Repository: anupamkliv/FedERA Branch: main Commit: 8db9bbcdc037 Files: 1415 Total size: 557.9 KB Directory structure: gitextract_o4_ojofq/ ├── .bandit ├── .coveragerc ├── .github/ │ └── workflows/ │ ├── publish.yml │ ├── pytest_coverage.yml │ ├── ubuntu.yml │ └── windows.yml ├── .gitignore ├── CONDUCT.md ├── Dockerfile ├── LICENSE ├── README.md ├── client_custom_dataset/ │ └── CUSTOM/ │ ├── test/ │ │ ├── Breast_1_131.npy │ │ ├── Breast_1_178.npy │ │ ├── Breast_1_38.npy │ │ ├── Breast_1_46.npy │ │ ├── Breast_1_56.npy │ │ ├── Breast_1_57.npy │ │ ├── Breast_1_58.npy │ │ ├── Chestxray_0_10192.npy │ │ ├── Chestxray_0_2599.npy │ │ ├── Chestxray_0_27.npy │ │ ├── Chestxray_10_2647.npy │ │ ├── Chestxray_10_7361.npy │ │ ├── Chestxray_11_6369.npy │ │ ├── Chestxray_12_3194.npy │ │ ├── Chestxray_12_6281.npy │ │ ├── Chestxray_1_1545.npy │ │ ├── Chestxray_2_10770.npy │ │ ├── Chestxray_2_1294.npy │ │ ├── Chestxray_2_2942.npy │ │ ├── Chestxray_3_4808.npy │ │ ├── Chestxray_3_563.npy │ │ ├── Chestxray_3_8533.npy │ │ ├── Chestxray_3_9185.npy │ │ ├── Chestxray_3_9271.npy │ │ ├── Chestxray_3_9492.npy │ │ ├── Chestxray_5_10699.npy │ │ ├── Chestxray_5_3477.npy │ │ ├── Chestxray_5_756.npy │ │ ├── Chestxray_5_7941.npy │ │ ├── Chestxray_5_8493.npy │ │ ├── Chestxray_6_7326.npy │ │ ├── Chestxray_7_10995.npy │ │ ├── Chestxray_7_8642.npy │ │ ├── Chestxray_7_9211.npy │ │ ├── Chestxray_8_2448.npy │ │ ├── Chestxray_9_1970.npy │ │ ├── Oct_0_10252.npy │ │ ├── Oct_0_10370.npy │ │ ├── Oct_0_1901.npy │ │ ├── Oct_0_2534.npy │ │ ├── Oct_0_4375.npy │ │ ├── Oct_0_521.npy │ │ ├── Oct_0_5368.npy │ │ ├── Oct_0_7082.npy │ │ ├── Oct_0_72818.npy │ │ ├── Oct_0_7570.npy │ │ ├── Oct_0_9852.npy │ │ ├── Oct_1_757.npy │ │ ├── Oct_2_26531.npy │ │ ├── Oct_2_3752.npy │ │ ├── Oct_2_8134.npy │ │ ├── Oct_2_8207.npy │ │ ├── Oct_2_9648.npy │ │ ├── Oct_3_1567.npy │ │ ├── Oct_3_1669.npy │ │ ├── Oct_3_2292.npy │ │ ├── Oct_3_2794.npy │ │ ├── Oct_3_4851.npy │ │ ├── Oct_3_4912.npy │ │ ├── Oct_3_5127.npy │ │ ├── Oct_3_5626.npy │ │ ├── Oct_3_5777.npy │ │ ├── Oct_3_6087.npy │ │ ├── Oct_3_6417.npy │ │ ├── Oct_3_9802.npy │ │ ├── Tissue_0_10673.npy │ │ ├── Tissue_0_12024.npy │ │ ├── Tissue_0_12038.npy │ │ ├── Tissue_0_14742.npy │ │ ├── Tissue_0_1487.npy │ │ ├── Tissue_0_17483.npy │ │ ├── Tissue_0_18694.npy │ │ ├── Tissue_0_20122.npy │ │ ├── Tissue_0_2373.npy │ │ ├── Tissue_0_3171.npy │ │ ├── Tissue_0_411.npy │ │ ├── Tissue_0_4904.npy │ │ ├── Tissue_0_6028.npy │ │ ├── Tissue_0_7434.npy │ │ ├── Tissue_0_8139.npy │ │ ├── Tissue_0_8962.npy │ │ ├── Tissue_0_9511.npy │ │ ├── Tissue_0_9949.npy │ │ ├── Tissue_3_8272.npy │ │ ├── Tissue_3_9593.npy │ │ ├── Tissue_4_10949.npy │ │ ├── Tissue_4_16558.npy │ │ ├── Tissue_4_17314.npy │ │ ├── Tissue_4_18139.npy │ │ ├── Tissue_4_2355.npy │ │ ├── Tissue_5_17494.npy │ │ ├── Tissue_5_22443.npy │ │ ├── Tissue_6_11839.npy │ │ ├── Tissue_6_12194.npy │ │ ├── Tissue_6_13317.npy │ │ ├── Tissue_6_14084.npy │ │ ├── Tissue_6_15019.npy │ │ ├── Tissue_6_15092.npy │ │ ├── Tissue_6_15204.npy │ │ ├── Tissue_6_17363.npy │ │ ├── Tissue_6_19355.npy │ │ ├── Tissue_6_19528.npy │ │ ├── Tissue_6_21927.npy │ │ ├── Tissue_6_37014.npy │ │ ├── Tissue_7_14747.npy │ │ ├── Tissue_7_14849.npy │ │ ├── Tissue_7_18085.npy │ │ ├── Tissue_7_18646.npy │ │ ├── Tissue_7_21396.npy │ │ ├── Tissue_7_21427.npy │ │ ├── Tissue_7_2508.npy │ │ ├── Tissue_7_2884.npy │ │ ├── Tissue_7_7116.npy │ │ └── Tissue_7_7248.npy │ └── train/ │ ├── Breast_0_0.npy │ ├── Breast_0_11.npy │ ├── Breast_0_115.npy │ ├── Breast_0_14.npy │ ├── Breast_0_20.npy │ ├── Breast_0_24.npy │ ├── Breast_0_28.npy │ ├── Breast_0_3.npy │ ├── Breast_0_30.npy │ ├── Breast_0_33.npy │ ├── Breast_0_45.npy │ ├── Breast_0_47.npy │ ├── Breast_0_64.npy │ ├── Breast_0_66.npy │ ├── Breast_0_71.npy │ ├── Breast_0_73.npy │ ├── Breast_0_74.npy │ ├── Breast_0_85.npy │ ├── Breast_1_1.npy │ ├── Breast_1_10.npy │ ├── Breast_1_12.npy │ ├── Breast_1_13.npy │ ├── Breast_1_15.npy │ ├── Breast_1_16.npy │ ├── Breast_1_17.npy │ ├── Breast_1_2.npy │ ├── Breast_1_21.npy │ ├── Breast_1_22.npy │ ├── Breast_1_227.npy │ ├── Breast_1_23.npy │ ├── Breast_1_249.npy │ ├── Breast_1_25.npy │ ├── Breast_1_27.npy │ ├── Breast_1_29.npy │ ├── Breast_1_32.npy │ ├── Breast_1_34.npy │ ├── Breast_1_35.npy │ ├── Breast_1_37.npy │ ├── Breast_1_40.npy │ ├── Breast_1_408.npy │ ├── Breast_1_41.npy │ ├── Breast_1_42.npy │ ├── Breast_1_43.npy │ ├── Breast_1_433.npy │ ├── Breast_1_48.npy │ ├── Breast_1_5.npy │ ├── Breast_1_50.npy │ ├── Breast_1_51.npy │ ├── Breast_1_52.npy │ ├── Breast_1_53.npy │ ├── Breast_1_54.npy │ ├── Breast_1_55.npy │ ├── Breast_1_6.npy │ ├── Breast_1_60.npy │ ├── Breast_1_61.npy │ ├── Breast_1_62.npy │ ├── Breast_1_64.npy │ ├── Breast_1_65.npy │ ├── Breast_1_68.npy │ ├── Breast_1_69.npy │ ├── Breast_1_7.npy │ ├── Breast_1_70.npy │ ├── Breast_1_72.npy │ ├── Breast_1_75.npy │ ├── Breast_1_76.npy │ ├── Breast_1_77.npy │ ├── Breast_1_8.npy │ ├── Breast_1_9.npy │ ├── Chestxray_0_10696.npy │ ├── Chestxray_0_10927.npy │ ├── Chestxray_0_11206.npy │ ├── Chestxray_0_2549.npy │ ├── Chestxray_0_44.npy │ ├── Chestxray_0_4584.npy │ ├── Chestxray_0_474.npy │ ├── Chestxray_0_4858.npy │ ├── Chestxray_0_5282.npy │ ├── Chestxray_0_6167.npy │ ├── Chestxray_0_6235.npy │ ├── Chestxray_0_6401.npy │ ├── Chestxray_0_6403.npy │ ├── Chestxray_0_6438.npy │ ├── Chestxray_0_6482.npy │ ├── Chestxray_0_7202.npy │ ├── Chestxray_0_752.npy │ ├── Chestxray_0_7551.npy │ ├── Chestxray_0_8453.npy │ ├── Chestxray_10_4463.npy │ ├── Chestxray_11_1275.npy │ ├── Chestxray_11_2068.npy │ ├── Chestxray_11_3915.npy │ ├── Chestxray_11_4063.npy │ ├── Chestxray_11_64880.npy │ ├── Chestxray_11_7570.npy │ ├── Chestxray_11_9678.npy │ ├── Chestxray_12_10521.npy │ ├── Chestxray_12_1472.npy │ ├── Chestxray_12_151.npy │ ├── Chestxray_12_194.npy │ ├── Chestxray_12_3225.npy │ ├── Chestxray_12_3458.npy │ ├── Chestxray_12_4112.npy │ ├── Chestxray_12_4694.npy │ ├── Chestxray_12_5795.npy │ ├── Chestxray_12_7320.npy │ ├── Chestxray_12_9084.npy │ ├── Chestxray_1_10242.npy │ ├── Chestxray_1_4797.npy │ ├── Chestxray_1_6404.npy │ ├── Chestxray_1_7034.npy │ ├── Chestxray_1_7388.npy │ ├── Chestxray_1_8324.npy │ ├── Chestxray_1_8326.npy │ ├── Chestxray_1_8907.npy │ ├── Chestxray_2_10082.npy │ ├── Chestxray_2_10125.npy │ ├── Chestxray_2_10149.npy │ ├── Chestxray_2_10349.npy │ ├── Chestxray_2_10524.npy │ ├── Chestxray_2_10638.npy │ ├── Chestxray_2_10750.npy │ ├── Chestxray_2_11137.npy │ ├── Chestxray_2_1160.npy │ ├── Chestxray_2_1177.npy │ ├── Chestxray_2_1408.npy │ ├── Chestxray_2_1727.npy │ ├── Chestxray_2_1832.npy │ ├── Chestxray_2_2029.npy │ ├── Chestxray_2_2134.npy │ ├── Chestxray_2_2535.npy │ ├── Chestxray_2_3045.npy │ ├── Chestxray_2_3462.npy │ ├── Chestxray_2_3820.npy │ ├── Chestxray_2_4261.npy │ ├── Chestxray_2_5098.npy │ ├── Chestxray_2_5125.npy │ ├── Chestxray_2_5669.npy │ ├── Chestxray_2_5987.npy │ ├── Chestxray_2_6509.npy │ ├── Chestxray_2_6807.npy │ ├── Chestxray_2_7015.npy │ ├── Chestxray_2_7113.npy │ ├── Chestxray_2_7131.npy │ ├── Chestxray_2_7152.npy │ ├── Chestxray_2_7430.npy │ ├── Chestxray_2_7706.npy │ ├── Chestxray_2_7865.npy │ ├── Chestxray_2_8054.npy │ ├── Chestxray_2_8099.npy │ ├── Chestxray_2_8955.npy │ ├── Chestxray_2_9745.npy │ ├── Chestxray_2_9882.npy │ ├── Chestxray_2_9918.npy │ ├── Chestxray_2_9951.npy │ ├── Chestxray_3_10171.npy │ ├── Chestxray_3_10717.npy │ ├── Chestxray_3_1072.npy │ ├── Chestxray_3_10733.npy │ ├── Chestxray_3_11004.npy │ ├── Chestxray_3_11050.npy │ ├── Chestxray_3_1627.npy │ ├── Chestxray_3_1767.npy │ ├── Chestxray_3_18163.npy │ ├── Chestxray_3_2116.npy │ ├── Chestxray_3_2130.npy │ ├── Chestxray_3_220.npy │ ├── Chestxray_3_2320.npy │ ├── Chestxray_3_2340.npy │ ├── Chestxray_3_2410.npy │ ├── Chestxray_3_2639.npy │ ├── Chestxray_3_2759.npy │ ├── Chestxray_3_3346.npy │ ├── Chestxray_3_3435.npy │ ├── Chestxray_3_3561.npy │ ├── Chestxray_3_3650.npy │ ├── Chestxray_3_3761.npy │ ├── Chestxray_3_3919.npy │ ├── Chestxray_3_3952.npy │ ├── Chestxray_3_421.npy │ ├── Chestxray_3_4216.npy │ ├── Chestxray_3_4402.npy │ ├── Chestxray_3_443.npy │ ├── Chestxray_3_4675.npy │ ├── Chestxray_3_4703.npy │ ├── Chestxray_3_4836.npy │ ├── Chestxray_3_4895.npy │ ├── Chestxray_3_4919.npy │ ├── Chestxray_3_4949.npy │ ├── Chestxray_3_512.npy │ ├── Chestxray_3_5301.npy │ ├── Chestxray_3_5605.npy │ ├── Chestxray_3_6751.npy │ ├── Chestxray_3_7412.npy │ ├── Chestxray_3_7657.npy │ ├── Chestxray_3_7667.npy │ ├── Chestxray_3_7801.npy │ ├── Chestxray_3_8160.npy │ ├── Chestxray_3_8298.npy │ ├── Chestxray_3_8713.npy │ ├── Chestxray_3_8954.npy │ ├── Chestxray_3_9047.npy │ ├── Chestxray_3_9081.npy │ ├── Chestxray_3_9523.npy │ ├── Chestxray_3_9811.npy │ ├── Chestxray_3_982.npy │ ├── Chestxray_4_1067.npy │ ├── Chestxray_4_1162.npy │ ├── Chestxray_4_1173.npy │ ├── Chestxray_4_2288.npy │ ├── Chestxray_4_3105.npy │ ├── Chestxray_4_4324.npy │ ├── Chestxray_4_6177.npy │ ├── Chestxray_4_7510.npy │ ├── Chestxray_4_7946.npy │ ├── Chestxray_4_8534.npy │ ├── Chestxray_4_8754.npy │ ├── Chestxray_4_9122.npy │ ├── Chestxray_4_9534.npy │ ├── Chestxray_4_9580.npy │ ├── Chestxray_5_10800.npy │ ├── Chestxray_5_1091.npy │ ├── Chestxray_5_1776.npy │ ├── Chestxray_5_3935.npy │ ├── Chestxray_5_4079.npy │ ├── Chestxray_5_554.npy │ ├── Chestxray_5_5938.npy │ ├── Chestxray_5_6184.npy │ ├── Chestxray_5_6391.npy │ ├── Chestxray_5_9471.npy │ ├── Chestxray_5_9729.npy │ ├── Chestxray_6_1513.npy │ ├── Chestxray_6_3188.npy │ ├── Chestxray_6_5225.npy │ ├── Chestxray_6_5805.npy │ ├── Chestxray_6_7399.npy │ ├── Chestxray_6_8382.npy │ ├── Chestxray_6_8988.npy │ ├── Chestxray_6_9056.npy │ ├── Chestxray_6_9185.npy │ ├── Chestxray_7_10100.npy │ ├── Chestxray_7_10571.npy │ ├── Chestxray_7_1145.npy │ ├── Chestxray_7_1717.npy │ ├── Chestxray_7_3195.npy │ ├── Chestxray_7_5008.npy │ ├── Chestxray_7_503.npy │ ├── Chestxray_7_61644.npy │ ├── Chestxray_7_69.npy │ ├── Chestxray_7_7171.npy │ ├── Chestxray_7_7260.npy │ ├── Chestxray_7_7293.npy │ ├── Chestxray_7_7542.npy │ ├── Chestxray_7_7828.npy │ ├── Chestxray_7_823.npy │ ├── Chestxray_7_9268.npy │ ├── Chestxray_8_17335.npy │ ├── Chestxray_8_187.npy │ ├── Chestxray_8_2918.npy │ ├── Chestxray_8_3663.npy │ ├── Chestxray_8_4409.npy │ ├── Chestxray_8_6667.npy │ ├── Chestxray_8_6771.npy │ ├── Chestxray_8_8104.npy │ ├── Chestxray_8_9794.npy │ ├── Chestxray_9_2208.npy │ ├── Chestxray_9_236.npy │ ├── Chestxray_9_2714.npy │ ├── Chestxray_9_6875.npy │ ├── Chestxray_9_7803.npy │ ├── Oct_0_10255.npy │ ├── Oct_0_1033.npy │ ├── Oct_0_10376.npy │ ├── Oct_0_10487.npy │ ├── Oct_0_1060.npy │ ├── Oct_0_1173.npy │ ├── Oct_0_1250.npy │ ├── Oct_0_1373.npy │ ├── Oct_0_1380.npy │ ├── Oct_0_1399.npy │ ├── Oct_0_1517.npy │ ├── Oct_0_1524.npy │ ├── Oct_0_1662.npy │ ├── Oct_0_1684.npy │ ├── Oct_0_1746.npy │ ├── Oct_0_1924.npy │ ├── Oct_0_2014.npy │ ├── Oct_0_2158.npy │ ├── Oct_0_2279.npy │ ├── Oct_0_2548.npy │ ├── Oct_0_2596.npy │ ├── Oct_0_2722.npy │ ├── Oct_0_2935.npy │ ├── Oct_0_2954.npy │ ├── Oct_0_2972.npy │ ├── Oct_0_3006.npy │ ├── Oct_0_3027.npy │ ├── Oct_0_3071.npy │ ├── Oct_0_3482.npy │ ├── Oct_0_3485.npy │ ├── Oct_0_3571.npy │ ├── Oct_0_3607.npy │ ├── Oct_0_3642.npy │ ├── Oct_0_3859.npy │ ├── Oct_0_3988.npy │ ├── Oct_0_4289.npy │ ├── Oct_0_4290.npy │ ├── Oct_0_4333.npy │ ├── Oct_0_449.npy │ ├── Oct_0_450.npy │ ├── Oct_0_4634.npy │ ├── Oct_0_5293.npy │ ├── Oct_0_5502.npy │ ├── Oct_0_5688.npy │ ├── Oct_0_5722.npy │ ├── Oct_0_5798.npy │ ├── Oct_0_5857.npy │ ├── Oct_0_596.npy │ ├── Oct_0_6403.npy │ ├── Oct_0_6663.npy │ ├── Oct_0_6760.npy │ ├── Oct_0_6922.npy │ ├── Oct_0_7047.npy │ ├── Oct_0_7081.npy │ ├── Oct_0_7107.npy │ ├── Oct_0_7151.npy │ ├── Oct_0_7218.npy │ ├── Oct_0_7233.npy │ ├── Oct_0_7331.npy │ ├── Oct_0_7354.npy │ ├── Oct_0_7581.npy │ ├── Oct_0_7616.npy │ ├── Oct_0_772.npy │ ├── Oct_0_7767.npy │ ├── Oct_0_7831.npy │ ├── Oct_0_7892.npy │ ├── Oct_0_7906.npy │ ├── Oct_0_7992.npy │ ├── Oct_0_8074.npy │ ├── Oct_0_8106.npy │ ├── Oct_0_8302.npy │ ├── Oct_0_8346.npy │ ├── Oct_0_8348.npy │ ├── Oct_0_8489.npy │ ├── Oct_0_851.npy │ ├── Oct_0_8843.npy │ ├── Oct_0_8955.npy │ ├── Oct_0_9002.npy │ ├── Oct_0_9224.npy │ ├── Oct_0_9362.npy │ ├── Oct_0_9377.npy │ ├── Oct_0_9453.npy │ ├── Oct_0_9587.npy │ ├── Oct_0_9755.npy │ ├── Oct_1_1207.npy │ ├── Oct_1_1753.npy │ ├── Oct_1_1796.npy │ ├── Oct_1_2078.npy │ ├── Oct_1_2182.npy │ ├── Oct_1_2987.npy │ ├── Oct_1_3566.npy │ ├── Oct_1_4093.npy │ ├── Oct_1_4149.npy │ ├── Oct_1_4366.npy │ ├── Oct_1_4436.npy │ ├── Oct_1_5388.npy │ ├── Oct_1_54.npy │ ├── Oct_1_5800.npy │ ├── Oct_1_6105.npy │ ├── Oct_1_6414.npy │ ├── Oct_1_65387.npy │ ├── Oct_1_7773.npy │ ├── Oct_1_8424.npy │ ├── Oct_1_8527.npy │ ├── Oct_1_8876.npy │ ├── Oct_1_8976.npy │ ├── Oct_1_9014.npy │ ├── Oct_1_902.npy │ ├── Oct_1_9097.npy │ ├── Oct_1_9702.npy │ ├── Oct_2_10143.npy │ ├── Oct_2_10483.npy │ ├── Oct_2_1052.npy │ ├── Oct_2_10661.npy │ ├── Oct_2_2610.npy │ ├── Oct_2_26446.npy │ ├── Oct_2_27682.npy │ ├── Oct_2_3036.npy │ ├── Oct_2_3249.npy │ ├── Oct_2_4057.npy │ ├── Oct_2_4370.npy │ ├── Oct_2_5788.npy │ ├── Oct_2_5962.npy │ ├── Oct_2_5985.npy │ ├── Oct_2_6369.npy │ ├── Oct_2_7924.npy │ ├── Oct_2_8401.npy │ ├── Oct_2_9099.npy │ ├── Oct_2_9163.npy │ ├── Oct_2_9652.npy │ ├── Oct_2_9760.npy │ ├── Oct_3_10010.npy │ ├── Oct_3_10059.npy │ ├── Oct_3_10063.npy │ ├── Oct_3_10164.npy │ ├── Oct_3_10261.npy │ ├── Oct_3_10341.npy │ ├── Oct_3_10348.npy │ ├── Oct_3_10360.npy │ ├── Oct_3_10441.npy │ ├── Oct_3_10444.npy │ ├── Oct_3_10811.npy │ ├── Oct_3_1092.npy │ ├── Oct_3_1269.npy │ ├── Oct_3_1370.npy │ ├── Oct_3_1588.npy │ ├── Oct_3_1671.npy │ ├── Oct_3_1799.npy │ ├── Oct_3_1820.npy │ ├── Oct_3_1869.npy │ ├── Oct_3_2031.npy │ ├── Oct_3_2211.npy │ ├── Oct_3_2217.npy │ ├── Oct_3_2259.npy │ ├── Oct_3_2277.npy │ ├── Oct_3_2300.npy │ ├── Oct_3_2320.npy │ ├── Oct_3_2345.npy │ ├── Oct_3_2388.npy │ ├── Oct_3_25.npy │ ├── Oct_3_2500.npy │ ├── Oct_3_2530.npy │ ├── Oct_3_2624.npy │ ├── Oct_3_271.npy │ ├── Oct_3_2777.npy │ ├── Oct_3_2915.npy │ ├── Oct_3_2921.npy │ ├── Oct_3_3087.npy │ ├── Oct_3_314.npy │ ├── Oct_3_3240.npy │ ├── Oct_3_3282.npy │ ├── Oct_3_3379.npy │ ├── Oct_3_3447.npy │ ├── Oct_3_3492.npy │ ├── Oct_3_36.npy │ ├── Oct_3_3660.npy │ ├── Oct_3_3685.npy │ ├── Oct_3_3771.npy │ ├── Oct_3_3895.npy │ ├── Oct_3_3983.npy │ ├── Oct_3_4006.npy │ ├── Oct_3_4101.npy │ ├── Oct_3_4393.npy │ ├── Oct_3_4416.npy │ ├── Oct_3_4499.npy │ ├── Oct_3_4657.npy │ ├── Oct_3_4711.npy │ ├── Oct_3_4786.npy │ ├── Oct_3_480.npy │ ├── Oct_3_4887.npy │ ├── Oct_3_4993.npy │ ├── Oct_3_4996.npy │ ├── Oct_3_4997.npy │ ├── Oct_3_5083.npy │ ├── Oct_3_5087.npy │ ├── Oct_3_5098.npy │ ├── Oct_3_5157.npy │ ├── Oct_3_5187.npy │ ├── Oct_3_5425.npy │ ├── Oct_3_5501.npy │ ├── Oct_3_5650.npy │ ├── Oct_3_5657.npy │ ├── Oct_3_5765.npy │ ├── Oct_3_5805.npy │ ├── Oct_3_5859.npy │ ├── Oct_3_5967.npy │ ├── Oct_3_5992.npy │ ├── Oct_3_6058.npy │ ├── Oct_3_6112.npy │ ├── Oct_3_619.npy │ ├── Oct_3_6209.npy │ ├── Oct_3_6240.npy │ ├── Oct_3_6277.npy │ ├── Oct_3_6411.npy │ ├── Oct_3_6561.npy │ ├── Oct_3_6614.npy │ ├── Oct_3_6674.npy │ ├── Oct_3_6794.npy │ ├── Oct_3_6903.npy │ ├── Oct_3_7007.npy │ ├── Oct_3_7102.npy │ ├── Oct_3_7352.npy │ ├── Oct_3_7453.npy │ ├── Oct_3_7833.npy │ ├── Oct_3_8099.npy │ ├── Oct_3_823.npy │ ├── Oct_3_8352.npy │ ├── Oct_3_8399.npy │ ├── Oct_3_8446.npy │ ├── Oct_3_852.npy │ ├── Oct_3_8542.npy │ ├── Oct_3_8627.npy │ ├── Oct_3_8752.npy │ ├── Oct_3_8781.npy │ ├── Oct_3_8817.npy │ ├── Oct_3_8890.npy │ ├── Oct_3_8911.npy │ ├── Oct_3_9019.npy │ ├── Oct_3_9356.npy │ ├── Oct_3_9482.npy │ ├── Oct_3_9592.npy │ ├── Oct_3_9744.npy │ ├── Oct_3_9875.npy │ ├── Oct_3_9938.npy │ ├── Tissue_0_10065.npy │ ├── Tissue_0_10235.npy │ ├── Tissue_0_10275.npy │ ├── Tissue_0_10363.npy │ ├── Tissue_0_10501.npy │ ├── Tissue_0_10715.npy │ ├── Tissue_0_10876.npy │ ├── Tissue_0_10970.npy │ ├── Tissue_0_1104.npy │ ├── Tissue_0_11215.npy │ ├── Tissue_0_11293.npy │ ├── Tissue_0_11304.npy │ ├── Tissue_0_11407.npy │ ├── Tissue_0_11543.npy │ ├── Tissue_0_11564.npy │ ├── Tissue_0_11609.npy │ ├── Tissue_0_11794.npy │ ├── Tissue_0_11911.npy │ ├── Tissue_0_12370.npy │ ├── Tissue_0_12488.npy │ ├── Tissue_0_12745.npy │ ├── Tissue_0_12909.npy │ ├── Tissue_0_13156.npy │ ├── Tissue_0_13261.npy │ ├── Tissue_0_13568.npy │ ├── Tissue_0_13699.npy │ ├── Tissue_0_13863.npy │ ├── Tissue_0_14042.npy │ ├── Tissue_0_14286.npy │ ├── Tissue_0_14369.npy │ ├── Tissue_0_14385.npy │ ├── Tissue_0_14542.npy │ ├── Tissue_0_14599.npy │ ├── Tissue_0_14672.npy │ ├── Tissue_0_14761.npy │ ├── Tissue_0_14805.npy │ ├── Tissue_0_1507.npy │ ├── Tissue_0_15264.npy │ ├── Tissue_0_154.npy │ ├── Tissue_0_15446.npy │ ├── Tissue_0_15489.npy │ ├── Tissue_0_15550.npy │ ├── Tissue_0_15846.npy │ ├── Tissue_0_15920.npy │ ├── Tissue_0_15943.npy │ ├── Tissue_0_16003.npy │ ├── Tissue_0_16179.npy │ ├── Tissue_0_16471.npy │ ├── Tissue_0_16578.npy │ ├── Tissue_0_16604.npy │ ├── Tissue_0_16671.npy │ ├── Tissue_0_16729.npy │ ├── Tissue_0_16738.npy │ ├── Tissue_0_16811.npy │ ├── Tissue_0_16840.npy │ ├── Tissue_0_17170.npy │ ├── Tissue_0_17171.npy │ ├── Tissue_0_17285.npy │ ├── Tissue_0_17376.npy │ ├── Tissue_0_17412.npy │ ├── Tissue_0_17486.npy │ ├── Tissue_0_17613.npy │ ├── Tissue_0_17929.npy │ ├── Tissue_0_18094.npy │ ├── Tissue_0_18387.npy │ ├── Tissue_0_18481.npy │ ├── Tissue_0_18485.npy │ ├── Tissue_0_18497.npy │ ├── Tissue_0_18545.npy │ ├── Tissue_0_18552.npy │ ├── Tissue_0_18581.npy │ ├── Tissue_0_18612.npy │ ├── Tissue_0_18634.npy │ ├── Tissue_0_18715.npy │ ├── Tissue_0_18737.npy │ ├── Tissue_0_18825.npy │ ├── Tissue_0_18933.npy │ ├── Tissue_0_19059.npy │ ├── Tissue_0_19110.npy │ ├── Tissue_0_19120.npy │ ├── Tissue_0_19162.npy │ ├── Tissue_0_19290.npy │ ├── Tissue_0_19312.npy │ ├── Tissue_0_19356.npy │ ├── Tissue_0_19436.npy │ ├── Tissue_0_19781.npy │ ├── Tissue_0_19861.npy │ ├── Tissue_0_19937.npy │ ├── Tissue_0_20028.npy │ ├── Tissue_0_20365.npy │ ├── Tissue_0_20416.npy │ ├── Tissue_0_20423.npy │ ├── Tissue_0_20465.npy │ ├── Tissue_0_20502.npy │ ├── Tissue_0_20740.npy │ ├── Tissue_0_20901.npy │ ├── Tissue_0_2091.npy │ ├── Tissue_0_21093.npy │ ├── Tissue_0_21164.npy │ ├── Tissue_0_21240.npy │ ├── Tissue_0_21398.npy │ ├── Tissue_0_21504.npy │ ├── Tissue_0_21546.npy │ ├── Tissue_0_21663.npy │ ├── Tissue_0_2183.npy │ ├── Tissue_0_22081.npy │ ├── Tissue_0_22085.npy │ ├── Tissue_0_22135.npy │ ├── Tissue_0_22355.npy │ ├── Tissue_0_22468.npy │ ├── Tissue_0_22513.npy │ ├── Tissue_0_22534.npy │ ├── Tissue_0_22550.npy │ ├── Tissue_0_22615.npy │ ├── Tissue_0_2285.npy │ ├── Tissue_0_22873.npy │ ├── Tissue_0_22928.npy │ ├── Tissue_0_23157.npy │ ├── Tissue_0_23203.npy │ ├── Tissue_0_23279.npy │ ├── Tissue_0_23295.npy │ ├── Tissue_0_23422.npy │ ├── Tissue_0_23562.npy │ ├── Tissue_0_2737.npy │ ├── Tissue_0_2896.npy │ ├── Tissue_0_2965.npy │ ├── Tissue_0_3072.npy │ ├── Tissue_0_3132.npy │ ├── Tissue_0_3156.npy │ ├── Tissue_0_3214.npy │ ├── Tissue_0_3264.npy │ ├── Tissue_0_3382.npy │ ├── Tissue_0_3487.npy │ ├── Tissue_0_3652.npy │ ├── Tissue_0_3802.npy │ ├── Tissue_0_3864.npy │ ├── Tissue_0_3966.npy │ ├── Tissue_0_4209.npy │ ├── Tissue_0_4414.npy │ ├── Tissue_0_4466.npy │ ├── Tissue_0_4795.npy │ ├── Tissue_0_488.npy │ ├── Tissue_0_4947.npy │ ├── Tissue_0_4996.npy │ ├── Tissue_0_5025.npy │ ├── Tissue_0_5105.npy │ ├── Tissue_0_5182.npy │ ├── Tissue_0_5184.npy │ ├── Tissue_0_5350.npy │ ├── Tissue_0_5512.npy │ ├── Tissue_0_5714.npy │ ├── Tissue_0_5863.npy │ ├── Tissue_0_5916.npy │ ├── Tissue_0_5970.npy │ ├── Tissue_0_6260.npy │ ├── Tissue_0_6261.npy │ ├── Tissue_0_6278.npy │ ├── Tissue_0_6368.npy │ ├── Tissue_0_65.npy │ ├── Tissue_0_653.npy │ ├── Tissue_0_6793.npy │ ├── Tissue_0_6952.npy │ ├── Tissue_0_7195.npy │ ├── Tissue_0_7443.npy │ ├── Tissue_0_7602.npy │ ├── Tissue_0_7610.npy │ ├── Tissue_0_7645.npy │ ├── Tissue_0_77.npy │ ├── Tissue_0_7807.npy │ ├── Tissue_0_8051.npy │ ├── Tissue_0_8072.npy │ ├── Tissue_0_8147.npy │ ├── Tissue_0_8340.npy │ ├── Tissue_0_836.npy │ ├── Tissue_0_8607.npy │ ├── Tissue_0_8697.npy │ ├── Tissue_0_8699.npy │ ├── Tissue_0_8796.npy │ ├── Tissue_0_8870.npy │ ├── Tissue_0_913.npy │ ├── Tissue_0_961.npy │ ├── Tissue_0_9629.npy │ ├── Tissue_0_9702.npy │ ├── Tissue_0_9822.npy │ ├── Tissue_1_1213.npy │ ├── Tissue_1_13054.npy │ ├── Tissue_1_13876.npy │ ├── Tissue_1_14595.npy │ ├── Tissue_1_14914.npy │ ├── Tissue_1_15318.npy │ ├── Tissue_1_15828.npy │ ├── Tissue_1_17215.npy │ ├── Tissue_1_17772.npy │ ├── Tissue_1_18026.npy │ ├── Tissue_1_19496.npy │ ├── Tissue_1_19798.npy │ ├── Tissue_1_1987.npy │ ├── Tissue_1_20308.npy │ ├── Tissue_1_20756.npy │ ├── Tissue_1_21050.npy │ ├── Tissue_1_2323.npy │ ├── Tissue_1_4574.npy │ ├── Tissue_1_483.npy │ ├── Tissue_1_6495.npy │ ├── Tissue_1_7192.npy │ ├── Tissue_1_7418.npy │ ├── Tissue_1_8376.npy │ ├── Tissue_1_8601.npy │ ├── Tissue_1_8790.npy │ ├── Tissue_1_9741.npy │ ├── Tissue_2_12291.npy │ ├── Tissue_2_12512.npy │ ├── Tissue_2_13211.npy │ ├── Tissue_2_14652.npy │ ├── Tissue_2_1538.npy │ ├── Tissue_2_15651.npy │ ├── Tissue_2_17690.npy │ ├── Tissue_2_20081.npy │ ├── Tissue_2_20348.npy │ ├── Tissue_2_20909.npy │ ├── Tissue_2_22554.npy │ ├── Tissue_2_32155.npy │ ├── Tissue_2_4327.npy │ ├── Tissue_2_5861.npy │ ├── Tissue_2_6910.npy │ ├── Tissue_2_9569.npy │ ├── Tissue_2_9599.npy │ ├── Tissue_2_9684.npy │ ├── Tissue_3_10163.npy │ ├── Tissue_3_11736.npy │ ├── Tissue_3_12666.npy │ ├── Tissue_3_12794.npy │ ├── Tissue_3_13262.npy │ ├── Tissue_3_13850.npy │ ├── Tissue_3_1443.npy │ ├── Tissue_3_15028.npy │ ├── Tissue_3_15037.npy │ ├── Tissue_3_1526.npy │ ├── Tissue_3_15283.npy │ ├── Tissue_3_15641.npy │ ├── Tissue_3_1584.npy │ ├── Tissue_3_16139.npy │ ├── Tissue_3_16275.npy │ ├── Tissue_3_16431.npy │ ├── Tissue_3_1673.npy │ ├── Tissue_3_17222.npy │ ├── Tissue_3_17288.npy │ ├── Tissue_3_17682.npy │ ├── Tissue_3_17823.npy │ ├── Tissue_3_18447.npy │ ├── Tissue_3_18679.npy │ ├── Tissue_3_18706.npy │ ├── Tissue_3_1891.npy │ ├── Tissue_3_19517.npy │ ├── Tissue_3_19530.npy │ ├── Tissue_3_19595.npy │ ├── Tissue_3_19725.npy │ ├── Tissue_3_21530.npy │ ├── Tissue_3_21678.npy │ ├── Tissue_3_21791.npy │ ├── Tissue_3_22927.npy │ ├── Tissue_3_23085.npy │ ├── Tissue_3_23161.npy │ ├── Tissue_3_3332.npy │ ├── Tissue_3_42135.npy │ ├── Tissue_3_480.npy │ ├── Tissue_3_492.npy │ ├── Tissue_3_5199.npy │ ├── Tissue_3_5292.npy │ ├── Tissue_3_5334.npy │ ├── Tissue_3_5376.npy │ ├── Tissue_3_5378.npy │ ├── Tissue_3_6005.npy │ ├── Tissue_3_6065.npy │ ├── Tissue_3_6238.npy │ ├── Tissue_3_6310.npy │ ├── Tissue_3_6344.npy │ ├── Tissue_3_6730.npy │ ├── Tissue_3_6861.npy │ ├── Tissue_3_7346.npy │ ├── Tissue_3_7502.npy │ ├── Tissue_3_7831.npy │ ├── Tissue_3_8562.npy │ ├── Tissue_3_9809.npy │ ├── Tissue_4_10991.npy │ ├── Tissue_4_11167.npy │ ├── Tissue_4_11184.npy │ ├── Tissue_4_11425.npy │ ├── Tissue_4_11664.npy │ ├── Tissue_4_11790.npy │ ├── Tissue_4_1208.npy │ ├── Tissue_4_12407.npy │ ├── Tissue_4_13305.npy │ ├── Tissue_4_14365.npy │ ├── Tissue_4_14431.npy │ ├── Tissue_4_14691.npy │ ├── Tissue_4_1501.npy │ ├── Tissue_4_15191.npy │ ├── Tissue_4_15266.npy │ ├── Tissue_4_15269.npy │ ├── Tissue_4_15273.npy │ ├── Tissue_4_15333.npy │ ├── Tissue_4_15662.npy │ ├── Tissue_4_1600.npy │ ├── Tissue_4_16887.npy │ ├── Tissue_4_17707.npy │ ├── Tissue_4_17947.npy │ ├── Tissue_4_18588.npy │ ├── Tissue_4_19067.npy │ ├── Tissue_4_19602.npy │ ├── Tissue_4_20012.npy │ ├── Tissue_4_21513.npy │ ├── Tissue_4_22008.npy │ ├── Tissue_4_22293.npy │ ├── Tissue_4_22427.npy │ ├── Tissue_4_3675.npy │ ├── Tissue_4_3731.npy │ ├── Tissue_4_4221.npy │ ├── Tissue_4_4419.npy │ ├── Tissue_4_44906.npy │ ├── Tissue_4_4744.npy │ ├── Tissue_4_6584.npy │ ├── Tissue_4_6589.npy │ ├── Tissue_4_6739.npy │ ├── Tissue_4_6827.npy │ ├── Tissue_4_6904.npy │ ├── Tissue_4_7830.npy │ ├── Tissue_4_7968.npy │ ├── Tissue_4_8010.npy │ ├── Tissue_4_8585.npy │ ├── Tissue_5_1049.npy │ ├── Tissue_5_10618.npy │ ├── Tissue_5_12264.npy │ ├── Tissue_5_12835.npy │ ├── Tissue_5_12894.npy │ ├── Tissue_5_14246.npy │ ├── Tissue_5_14371.npy │ ├── Tissue_5_14841.npy │ ├── Tissue_5_1639.npy │ ├── Tissue_5_17423.npy │ ├── Tissue_5_19155.npy │ ├── Tissue_5_19755.npy │ ├── Tissue_5_20836.npy │ ├── Tissue_5_21381.npy │ ├── Tissue_5_23079.npy │ ├── Tissue_5_23084.npy │ ├── Tissue_5_23166.npy │ ├── Tissue_5_4184.npy │ ├── Tissue_5_4787.npy │ ├── Tissue_5_5071.npy │ ├── Tissue_5_677.npy │ ├── Tissue_5_7029.npy │ ├── Tissue_5_7723.npy │ ├── Tissue_5_7938.npy │ ├── Tissue_5_8424.npy │ ├── Tissue_5_8628.npy │ ├── Tissue_6_10222.npy │ ├── Tissue_6_10237.npy │ ├── Tissue_6_10372.npy │ ├── Tissue_6_10478.npy │ ├── Tissue_6_10497.npy │ ├── Tissue_6_10944.npy │ ├── Tissue_6_10972.npy │ ├── Tissue_6_1101.npy │ ├── Tissue_6_11356.npy │ ├── Tissue_6_11461.npy │ ├── Tissue_6_11686.npy │ ├── Tissue_6_11775.npy │ ├── Tissue_6_119.npy │ ├── Tissue_6_12285.npy │ ├── Tissue_6_12712.npy │ ├── Tissue_6_12874.npy │ ├── Tissue_6_1291.npy │ ├── Tissue_6_1295.npy │ ├── Tissue_6_13046.npy │ ├── Tissue_6_13250.npy │ ├── Tissue_6_13321.npy │ ├── Tissue_6_1335.npy │ ├── Tissue_6_13369.npy │ ├── Tissue_6_13715.npy │ ├── Tissue_6_14052.npy │ ├── Tissue_6_14469.npy │ ├── Tissue_6_1468.npy │ ├── Tissue_6_15347.npy │ ├── Tissue_6_15468.npy │ ├── Tissue_6_15472.npy │ ├── Tissue_6_15603.npy │ ├── Tissue_6_15999.npy │ ├── Tissue_6_16265.npy │ ├── Tissue_6_16560.npy │ ├── Tissue_6_16636.npy │ ├── Tissue_6_16930.npy │ ├── Tissue_6_16985.npy │ ├── Tissue_6_17098.npy │ ├── Tissue_6_1725.npy │ ├── Tissue_6_17308.npy │ ├── Tissue_6_17357.npy │ ├── Tissue_6_17462.npy │ ├── Tissue_6_17968.npy │ ├── Tissue_6_17979.npy │ ├── Tissue_6_18189.npy │ ├── Tissue_6_1838.npy │ ├── Tissue_6_18661.npy │ ├── Tissue_6_18676.npy │ ├── Tissue_6_18883.npy │ ├── Tissue_6_18893.npy │ ├── Tissue_6_19159.npy │ ├── Tissue_6_19308.npy │ ├── Tissue_6_19401.npy │ ├── Tissue_6_19430.npy │ ├── Tissue_6_1946.npy │ ├── Tissue_6_19776.npy │ ├── Tissue_6_19930.npy │ ├── Tissue_6_19935.npy │ ├── Tissue_6_20138.npy │ ├── Tissue_6_20273.npy │ ├── Tissue_6_20359.npy │ ├── Tissue_6_20534.npy │ ├── Tissue_6_20857.npy │ ├── Tissue_6_20977.npy │ ├── Tissue_6_21009.npy │ ├── Tissue_6_21052.npy │ ├── Tissue_6_21207.npy │ ├── Tissue_6_21295.npy │ ├── Tissue_6_21514.npy │ ├── Tissue_6_21539.npy │ ├── Tissue_6_21851.npy │ ├── Tissue_6_21926.npy │ ├── Tissue_6_21973.npy │ ├── Tissue_6_2203.npy │ ├── Tissue_6_22133.npy │ ├── Tissue_6_22138.npy │ ├── Tissue_6_22348.npy │ ├── Tissue_6_22357.npy │ ├── Tissue_6_22475.npy │ ├── Tissue_6_22642.npy │ ├── Tissue_6_22907.npy │ ├── Tissue_6_2296.npy │ ├── Tissue_6_22988.npy │ ├── Tissue_6_23056.npy │ ├── Tissue_6_23287.npy │ ├── Tissue_6_23485.npy │ ├── Tissue_6_2518.npy │ ├── Tissue_6_2800.npy │ ├── Tissue_6_2841.npy │ ├── Tissue_6_2933.npy │ ├── Tissue_6_2978.npy │ ├── Tissue_6_3307.npy │ ├── Tissue_6_3309.npy │ ├── Tissue_6_3506.npy │ ├── Tissue_6_3536.npy │ ├── Tissue_6_4076.npy │ ├── Tissue_6_4133.npy │ ├── Tissue_6_4359.npy │ ├── Tissue_6_4373.npy │ ├── Tissue_6_4524.npy │ ├── Tissue_6_4591.npy │ ├── Tissue_6_4726.npy │ ├── Tissue_6_4749.npy │ ├── Tissue_6_4882.npy │ ├── Tissue_6_4905.npy │ ├── Tissue_6_5016.npy │ ├── Tissue_6_5115.npy │ ├── Tissue_6_5250.npy │ ├── Tissue_6_5648.npy │ ├── Tissue_6_5723.npy │ ├── Tissue_6_5851.npy │ ├── Tissue_6_5879.npy │ ├── Tissue_6_6110.npy │ ├── Tissue_6_6229.npy │ ├── Tissue_6_6292.npy │ ├── Tissue_6_6439.npy │ ├── Tissue_6_6542.npy │ ├── Tissue_6_6602.npy │ ├── Tissue_6_6749.npy │ ├── Tissue_6_6841.npy │ ├── Tissue_6_7207.npy │ ├── Tissue_6_7236.npy │ ├── Tissue_6_7756.npy │ ├── Tissue_6_8011.npy │ ├── Tissue_6_8196.npy │ ├── Tissue_6_8227.npy │ ├── Tissue_6_8382.npy │ ├── Tissue_6_8700.npy │ ├── Tissue_6_8818.npy │ ├── Tissue_6_8988.npy │ ├── Tissue_6_9130.npy │ ├── Tissue_6_9150.npy │ ├── Tissue_6_922.npy │ ├── Tissue_6_945.npy │ ├── Tissue_6_9465.npy │ ├── Tissue_6_9501.npy │ ├── Tissue_6_9528.npy │ ├── Tissue_6_9640.npy │ ├── Tissue_6_9849.npy │ ├── Tissue_7_1044.npy │ ├── Tissue_7_10916.npy │ ├── Tissue_7_10989.npy │ ├── Tissue_7_111.npy │ ├── Tissue_7_11487.npy │ ├── Tissue_7_11578.npy │ ├── Tissue_7_11828.npy │ ├── Tissue_7_12436.npy │ ├── Tissue_7_12736.npy │ ├── Tissue_7_12790.npy │ ├── Tissue_7_12842.npy │ ├── Tissue_7_13524.npy │ ├── Tissue_7_13934.npy │ ├── Tissue_7_1432.npy │ ├── Tissue_7_14811.npy │ ├── Tissue_7_14913.npy │ ├── Tissue_7_14926.npy │ ├── Tissue_7_14952.npy │ ├── Tissue_7_15198.npy │ ├── Tissue_7_15521.npy │ ├── Tissue_7_15876.npy │ ├── Tissue_7_1603.npy │ ├── Tissue_7_16273.npy │ ├── Tissue_7_16532.npy │ ├── Tissue_7_16716.npy │ ├── Tissue_7_17507.npy │ ├── Tissue_7_17704.npy │ ├── Tissue_7_17779.npy │ ├── Tissue_7_17938.npy │ ├── Tissue_7_18050.npy │ ├── Tissue_7_18300.npy │ ├── Tissue_7_18464.npy │ ├── Tissue_7_1861.npy │ ├── Tissue_7_20002.npy │ ├── Tissue_7_20065.npy │ ├── Tissue_7_2033.npy │ ├── Tissue_7_20552.npy │ ├── Tissue_7_2076.npy │ ├── Tissue_7_20864.npy │ ├── Tissue_7_21117.npy │ ├── Tissue_7_21211.npy │ ├── Tissue_7_21259.npy │ ├── Tissue_7_21986.npy │ ├── Tissue_7_22009.npy │ ├── Tissue_7_22260.npy │ ├── Tissue_7_22369.npy │ ├── Tissue_7_2242.npy │ ├── Tissue_7_2259.npy │ ├── Tissue_7_23231.npy │ ├── Tissue_7_23478.npy │ ├── Tissue_7_2718.npy │ ├── Tissue_7_3229.npy │ ├── Tissue_7_3920.npy │ ├── Tissue_7_4142.npy │ ├── Tissue_7_45543.npy │ ├── Tissue_7_4694.npy │ ├── Tissue_7_4763.npy │ ├── Tissue_7_4809.npy │ ├── Tissue_7_4851.npy │ ├── Tissue_7_5052.npy │ ├── Tissue_7_5664.npy │ ├── Tissue_7_6052.npy │ ├── Tissue_7_6219.npy │ ├── Tissue_7_640.npy │ ├── Tissue_7_6526.npy │ ├── Tissue_7_6702.npy │ ├── Tissue_7_7044.npy │ ├── Tissue_7_7065.npy │ ├── Tissue_7_7235.npy │ ├── Tissue_7_7722.npy │ ├── Tissue_7_7859.npy │ ├── Tissue_7_7914.npy │ ├── Tissue_7_7993.npy │ ├── Tissue_7_8054.npy │ ├── Tissue_7_8107.npy │ ├── Tissue_7_8200.npy │ ├── Tissue_7_904.npy │ ├── Tissue_7_9210.npy │ ├── Tissue_7_9691.npy │ └── Tissue_7_9759.npy ├── codecov.yml ├── configs/ │ ├── test_algorithms.json │ ├── test_datasets.json │ ├── test_models.json │ ├── test_modules.json │ ├── test_results.json │ └── test_scalability.json ├── docs/ │ ├── Makefile │ ├── make.bat │ ├── requirements.txt │ └── source/ │ ├── _autoapi_temp/ │ │ └── python/ │ │ └── module.rst │ ├── conf.py │ ├── contribution.rst │ ├── index.rst │ ├── installation.rst │ ├── overview.rst │ ├── refer.bib │ ├── reference.rst │ └── tutorials/ │ ├── algorithm.rst │ ├── code_carbon.rst │ ├── data_distribution.rst │ ├── dataset.rst │ ├── encryption.rst │ ├── models.rst │ ├── running.rst │ └── tutorial.rst ├── federa/ │ ├── __init__.py │ ├── client/ │ │ ├── __init__.py │ │ ├── src/ │ │ │ ├── ClientConnection_pb2.py │ │ │ ├── ClientConnection_pb2_grpc.py │ │ │ ├── __init__.py │ │ │ ├── client.py │ │ │ ├── client_lib.py │ │ │ ├── data_utils.py │ │ │ ├── get_data.py │ │ │ ├── net.py │ │ │ └── net_lib.py │ │ └── start_client.py │ ├── server/ │ │ ├── __init__.py │ │ ├── src/ │ │ │ ├── ClientConnection_pb2.py │ │ │ ├── ClientConnection_pb2_grpc.py │ │ │ ├── __init__.py │ │ │ ├── algorithms/ │ │ │ │ ├── fedadagrad.py │ │ │ │ ├── fedadam.py │ │ │ │ ├── fedavg.py │ │ │ │ ├── fedavgm.py │ │ │ │ ├── feddyn.py │ │ │ │ ├── fedyogi.py │ │ │ │ ├── mime.py │ │ │ │ ├── mimelite.py │ │ │ │ └── scaffold.py │ │ │ ├── client_connection_servicer.py │ │ │ ├── client_manager.py │ │ │ ├── client_wrapper.py │ │ │ ├── distribution.py │ │ │ ├── server.py │ │ │ ├── server_evaluate/ │ │ │ │ ├── __init__.py │ │ │ │ └── eval_lib.py │ │ │ ├── server_lib.py │ │ │ └── verification.py │ │ └── start_server.py │ └── tests/ │ ├── __init__.py │ ├── minitest.json │ ├── minitest.py │ └── misc.py ├── requirements.txt ├── server_custom_dataset/ │ └── CUSTOM/ │ └── test/ │ ├── Breast_1_131.npy │ ├── Breast_1_178.npy │ ├── Breast_1_38.npy │ ├── Breast_1_46.npy │ ├── Breast_1_56.npy │ ├── Breast_1_57.npy │ ├── Breast_1_58.npy │ ├── Chestxray_0_10192.npy │ ├── Chestxray_0_2599.npy │ ├── Chestxray_0_27.npy │ ├── Chestxray_10_2647.npy │ ├── Chestxray_10_7361.npy │ ├── Chestxray_11_6369.npy │ ├── Chestxray_12_3194.npy │ ├── Chestxray_12_6281.npy │ ├── Chestxray_1_1545.npy │ ├── Chestxray_2_10770.npy │ ├── Chestxray_2_1294.npy │ ├── Chestxray_2_2942.npy │ ├── Chestxray_3_4808.npy │ ├── Chestxray_3_563.npy │ ├── Chestxray_3_8533.npy │ ├── Chestxray_3_9185.npy │ ├── Chestxray_3_9271.npy │ ├── Chestxray_3_9492.npy │ ├── Chestxray_5_10699.npy │ ├── Chestxray_5_3477.npy │ ├── Chestxray_5_756.npy │ ├── Chestxray_5_7941.npy │ ├── Chestxray_5_8493.npy │ ├── Chestxray_6_7326.npy │ ├── Chestxray_7_10995.npy │ ├── Chestxray_7_8642.npy │ ├── Chestxray_7_9211.npy │ ├── Chestxray_8_2448.npy │ ├── Chestxray_9_1970.npy │ ├── Oct_0_10252.npy │ ├── Oct_0_10370.npy │ ├── Oct_0_1901.npy │ ├── Oct_0_2534.npy │ ├── Oct_0_4375.npy │ ├── Oct_0_521.npy │ ├── Oct_0_5368.npy │ ├── Oct_0_7082.npy │ ├── Oct_0_72818.npy │ ├── Oct_0_7570.npy │ ├── Oct_0_9852.npy │ ├── Oct_1_757.npy │ ├── Oct_2_26531.npy │ ├── Oct_2_3752.npy │ ├── Oct_2_8134.npy │ ├── Oct_2_8207.npy │ ├── Oct_2_9648.npy │ ├── Oct_3_1567.npy │ ├── Oct_3_1669.npy │ ├── Oct_3_2292.npy │ ├── Oct_3_2794.npy │ ├── Oct_3_4851.npy │ ├── Oct_3_4912.npy │ ├── Oct_3_5127.npy │ ├── Oct_3_5626.npy │ ├── Oct_3_5777.npy │ ├── Oct_3_6087.npy │ ├── Oct_3_6417.npy │ ├── Oct_3_9802.npy │ ├── Tissue_0_10673.npy │ ├── Tissue_0_12024.npy │ ├── Tissue_0_12038.npy │ ├── Tissue_0_14742.npy │ ├── Tissue_0_1487.npy │ ├── Tissue_0_17483.npy │ ├── Tissue_0_18694.npy │ ├── Tissue_0_20122.npy │ ├── Tissue_0_2373.npy │ ├── Tissue_0_3171.npy │ ├── Tissue_0_411.npy │ ├── Tissue_0_4904.npy │ ├── Tissue_0_6028.npy │ ├── Tissue_0_7434.npy │ ├── Tissue_0_8139.npy │ ├── Tissue_0_8962.npy │ ├── Tissue_0_9511.npy │ ├── Tissue_0_9949.npy │ ├── Tissue_3_8272.npy │ ├── Tissue_3_9593.npy │ ├── Tissue_4_10949.npy │ ├── Tissue_4_16558.npy │ ├── Tissue_4_17314.npy │ ├── Tissue_4_18139.npy │ ├── Tissue_4_2355.npy │ ├── Tissue_5_17494.npy │ ├── Tissue_5_22443.npy │ ├── Tissue_6_11839.npy │ ├── Tissue_6_12194.npy │ ├── Tissue_6_13317.npy │ ├── Tissue_6_14084.npy │ ├── Tissue_6_15019.npy │ ├── Tissue_6_15092.npy │ ├── Tissue_6_15204.npy │ ├── Tissue_6_17363.npy │ ├── Tissue_6_19355.npy │ ├── Tissue_6_19528.npy │ ├── Tissue_6_21927.npy │ ├── Tissue_6_37014.npy │ ├── Tissue_7_14747.npy │ ├── Tissue_7_14849.npy │ ├── Tissue_7_18085.npy │ ├── Tissue_7_18646.npy │ ├── Tissue_7_21396.npy │ ├── Tissue_7_21427.npy │ ├── Tissue_7_2508.npy │ ├── Tissue_7_2884.npy │ ├── Tissue_7_7116.npy │ └── Tissue_7_7248.npy ├── ssl/ │ ├── README.md │ ├── ca-config.json │ ├── ca-csr.json │ ├── client-csr.json │ └── server-csr.json ├── test/ │ ├── __init__.py │ ├── benchtest/ │ │ ├── __init__.py │ │ ├── test_results.py │ │ └── test_scalability.py │ ├── misc.py │ └── unittest/ │ ├── __init__.py │ ├── test_algorithms.py │ ├── test_datasets.py │ ├── test_models.py │ └── test_modules.py └── tutorials/ ├── Code_Carbon_Tutorial.ipynb ├── Federated_Algorithm_Tutorial.ipynb ├── Number_of_clients_Tutorial.ipynb ├── Verifcation_module_tutorial.ipynb ├── accuracy_plot.py ├── data_distribution.ipynb └── media_plot.ipynb ================================================ FILE CONTENTS ================================================ ================================================ FILE: .bandit ================================================ [bandit] targets: federa [bandit.test] severity: MEDIUM,HIGH ================================================ FILE: .coveragerc ================================================ [run] parallel = True concurrency = multiprocessing, thread [report] omit = federa/server/start_server.py federa/client/start_client.py federa/tests/* ================================================ FILE: .github/workflows/publish.yml ================================================ name: Build and Push Docker Image on: push: branches: - main env: DOCKER_HUB_USERNAME: anupamkliv DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} jobs: build-and-push: runs-on: ubuntu-latest steps: - name: Checkout repository uses: actions/checkout@v2 - name: Login to Docker Hub run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD - name: Build and tag Docker image run: | docker build -t anupamkliv/federa:latest . docker tag anupamkliv/federa:latest anupamkliv/federa:${{ github.sha }} - name: Copy README to Docker image run: | docker create --name temp_container anupamkliv/federa:latest docker cp README.md temp_container:/README.md docker commit temp_container anupamkliv/federa:latest docker rm temp_container - name: Push Docker image to Docker Hub run: | docker push anupamkliv/federa:latest docker push anupamkliv/federa:${{ github.sha }} ================================================ FILE: .github/workflows/pytest_coverage.yml ================================================ # This workflow will install Python dependencies, run tests and lint with a single version of Python # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions name: Pytest and code coverage on: [push] permissions: contents: read jobs: build: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - name: Set up Python 3.8 uses: actions/setup-python@v3 with: python-version: "3.8" - name: Install dependencies run: | python -m pip install --upgrade pip pip install pytest coverage pip install -r requirements.txt - name: Test with pytest and report code coverage run: | coverage run -m --source=federa -p test.unittest.test_algorithms coverage run -m --source=federa -p test.unittest.test_datasets coverage run -m --source=federa -p test.unittest.test_modules coverage run -m --source=federa -p test.unittest.test_models coverage combine continue-on-error: true - name: Upload coverage report uses: codecov/codecov-action@v3 with: token: ${{ secrets.CODECOV_TOKEN }} - name: Run Bandit run: | bandit -r --ini .bandit - name: Coveralls GitHub Action uses: coverallsapp/github-action@v2 ================================================ FILE: .github/workflows/ubuntu.yml ================================================ name: Ubuntu (latest) on: [push] permissions: contents: read jobs: interactive-kvasir: # from interactive-kvasir.yml runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - name: Set up Python 3.8 uses: actions/setup-python@v3 with: python-version: "3.8" - name: Install dependencies run: | python -m pip install --upgrade pip pip install -r requirements.txt - name: Interactive API - pytorch_kvasir_unet run: | pip install torch==1.13.1 pip install torchvision==0.14.1 python -m test.unittest.test_algorithms ================================================ FILE: .github/workflows/windows.yml ================================================ name: Windows (latest) on: [push] permissions: contents: read jobs: interactive-kvasir: # from interactive-kvasir.yml runs-on: windows-latest steps: - uses: actions/checkout@v3 - name: Set up Python 3.8 uses: actions/setup-python@v3 with: python-version: "3.8" - name: Install dependencies run: | python -m pip install --upgrade pip pip install -r requirements.txt - name: Interactive API - pytorch_kvasir_unet run: | pip install torch==1.13.1 pip install torchvision==0.14.1 python -m test.unittest.test_algorithms cli: # from taskrunner.yml needs: [interactive-kvasir] runs-on: windows-latest steps: - uses: actions/checkout@v3 - name: Set up Python 3.8 uses: actions/setup-python@v3 with: python-version: "3.8" - name: Install dependencies run: | python -m pip install --upgrade pip pip install -r requirements.txt - name: Test TaskRunner API run: | python -m test.unittest.test_algorithms ================================================ FILE: .gitignore ================================================ ================================================ FILE: CONDUCT.md ================================================ # Contributor Code of Conduct ## Introduction As contributors and maintainers of this Federated Learning framework, we are committed to creating a safe and welcoming environment for everyone. This code of conduct outlines our expectations for all those who participate in our community, as well as the consequences for unacceptable behavior. ## Code of Conduct We expect all contributors and users of the Federated Learning framework to abide by the following standards: - **Be respectful:** Treat others with respect and dignity, and avoid engaging in any behavior that could be considered abusive, threatening, or harassing. - **Be inclusive:** Ensure that everyone feels welcome, regardless of their background or identity. - **Be collaborative:** Encourage collaboration and teamwork, and work towards common goals. - **Be open-minded:** Be open to new ideas and perspectives, and consider feedback and criticism in a constructive manner. - **Be transparent:** Be transparent about the framework's development process, and communicate any changes or updates clearly and honestly. - **Be accountable:** Hold yourself and others accountable for their actions and behaviors, and enforce the code of conduct fairly and consistently. We believe in creating a safe and welcoming environment for all members of our community, and we will not tolerate any behavior that violates this code of conduct. If you witness or experience any violations of this code of conduct, please report them to [insert contact information here]. By participating in our community, you agree to abide by this code of conduct. ## Unacceptable Behavior Examples of unacceptable behavior include: - Harassment, intimidation, or discrimination in any form. - Physical, verbal, or written abuse, threats, or insults. - Offensive or derogatory comments. - Disrespectful or disruptive behavior. - Inappropriate use of nudity, sexual images, or offensive language. - Deliberate intimidation or stalking. - Unwelcome sexual attention or advances. - Advocating for or encouraging any of the above behavior. ## Consequences of Unacceptable Behavior If a contributor or user engages in behavior that violates our standards of conduct, we may take any action we deem appropriate, up to and including temporary or permanent expulsion from the community. Reporting Guidelines If you experience or witness behavior that violates our standards of conduct, please report it to us by contacting [insert contact information here]. All reports will be kept confidential, and we will work with you to determine an appropriate course of action. Attribution Thank you for your cooperation in maintaining a positive and inclusive community for everyone involved in the Federated Learning framework! ================================================ FILE: Dockerfile ================================================ # Dockerfile # Determine the system architecture ARG ARCHITECTURE # Use the official Python base image FROM python:3.10-slim # Install system dependencies RUN apt-get update && \ apt-get install -y \ git \ protobuf-compiler # Install PyTorch RUN pip install torch torchvision # Set the working directory WORKDIR /usr/src/federa # Copy the repository code into the container COPY . . # Install Python dependencies RUN pip install --upgrade pip && \ pip install -r requirements.txt # Expose the port for communication with the server EXPOSE 8214 # Set the entry point command ENTRYPOINT [] # Default argument values for the server #CMD ["python", "-m", "federa.server.start_server"] # Specify the command arguments as environment variables ENV SERVER_ARGS="" ENV CLIENT_ARGS="" # Start the server and client using the provided arguments CMD bash -c "python -m federa.server.start_server ${SERVER_ARGS} & \ python -m federa.client.start_client ${CLIENT_ARGS}" ================================================ FILE: LICENSE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [2023] [Indian Institute of Technology, Kharagpur, West Bengal, India] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: README.md ================================================ # Federated Learning Framework [![PyPI - Python Version](https://img.shields.io/badge/python-3.6%20%7C%203.7%20%7C%203.8%20%7C%203.9-blue)](https://pypi.org/project/federa/) [![Documentation Status](https://readthedocs.org/projects/federa/badge/?version=latest)](https://federa.readthedocs.io/en/latest/?badge=latest) [![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) [![Ubuntu (latest)](https://github.com/anupamkliv/FedERA/actions/workflows/ubuntu.yml/badge.svg)](https://github.com/anupamkliv/FedERA/actions/workflows/ubuntu.yml) [![Windows (latest)](https://github.com/anupamkliv/FedERA/actions/workflows/windows.yml/badge.svg)](https://github.com/anupamkliv/FedERA/actions/workflows/windows.yml) [![coveralls](https://coveralls.io/repos/github/anupamkliv/FedERA/badge.svg?branch=main)](https://coveralls.io/github/anupamkliv/FedERA?branch=main) [![codecov](https://codecov.io/github/Kasliwal17/FedERA/branch/main/graph/badge.svg?token=0U0pz1YIu7)](https://codecov.io/github/Kasliwal17/FedERA) [![Docker Pull](https://img.shields.io/docker/pulls/anupamkliv/federa)](https://hub.docker.com/r/anupamkliv/federa) [![security: bandit](https://img.shields.io/badge/security-bandit-yellow.svg)](https://github.com/PyCQA/bandit) [![linting: pylint](https://img.shields.io/badge/linting-pylint-yellowgreen)](https://github.com/pylint-dev/pylint) [![OpenSSF Best Practices](https://bestpractices.coreinfrastructure.org/projects/7364/badge)](https://bestpractices.coreinfrastructure.org/projects/7364) [![Downloads](https://static.pepy.tech/badge/federa)](https://pepy.tech/project/federa) `FedERA` is a highly dynamic and customizable framework that can accommodate many use cases with flexibility by implementing several functionalities over different federated learning algorithms, and essentially creating a plug-and-play architecture to accommodate different use cases. ## Supported Devices FedERA has been extensively tested on and works with the following devices: * Intel CPUs * Nvidia GPUs * Nvidia Jetson * Raspberry Pi * Intel NUC With `FedERA`, it is possible to operate the server and clients on **separate devices** or on a **single device** through various means, such as utilizing different terminals or implementing multiprocessing. ## Installation - Install the latest version from source code: ``` $ git clone https://github.com/anupamkliv/FedERA.git $ cd FedERA $ pip install -r requirements.txt ``` - Install the stable version (old version) via pip: ``` # assign the version federa==1.0.0 $ pip install federa ``` - Using Docker Create a docker image ``` docker build -t federa . ``` Run the docker image ``` docker run federa ``` ## Documentation Website documentation has been made availbale for `FedERA`. Please visit [FedERA Documentation](https://federa.readthedocs.io/en/latest/index.html) for more details. 1. [Overview](https://federa.readthedocs.io/en/latest/overview.html) 2. [Installation](https://federa.readthedocs.io/en/latest/installation.html) 3. [Tutorials](https://federa.readthedocs.io/en/latest/tutorials.html) 4. [Contribution](https://federa.readthedocs.io/en/latest/contribution.html) 5. [API Reference](https://federa.readthedocs.io/en/latest/api.html) ## Starting server ``` python -m federa.server.start_server \ --algorithm fedavg \ --clients 1 \ --rounds 10 \ --epochs 10 \ --batch_size 10 \ --lr 0.01 \ --dataset mnist \ ``` ## Starting client ``` python -m federa.client.start_client \ --ip localhost:8214 \ --device cpu \ ``` ## Arguments to the clients and server ### Server | Argument | Description | Default | | ---------- | ------------------------------------------------------------ | ------- | | algorithm | specifies the aggregation algorithm | fedavg | | clients | specifies number of clients selected per round | 1 | | fraction | specifies fraction of clients selected | 1 | | rounds | specifies total number of rounds | 1 | | model_path | specifies initial server model path | initial_model.pt | | epochs | specifies client epochs per round | 1 | | accept_conn| determines if connections accepted after FL begins | 1 | | verify | specifies if verification module runs before rounds | 0 | | threshold | specifies minimum verification score | 0 | | timeout | specifies client training time limit per round | None | | resize_size| specifies dataset resize dimension | 32 | | batch_size | specifies dataset batch size | 32 | | net | specifies network architecture | LeNet | | dataset | specifies dataset name | MNIST | | niid | specifies data distribution among clients | 1 | | carbon | specifies if carbon emissions tracked at client side | 0 | | encryption | specifies whether to use ssl encryption or not | 0 | | server_key | specifies path to server key certificate | server-key.pem| | server_cert | specifies path to server certificate | server.pem| ### Client | Argument | Description | Default | | ---------- | ------------------------------------------------------------ | ------- | | server_ip | specifies server IP address | localhost:8214 | | device | specifies device | cpu | | encryption | specifies whether to use ssl encryption or not | 0 | | ca | specifies path to CA certificate | ca.pem| | wait_time | specifies time to wait before reconnecting to server | 30 | ## Architecture Files architecture of `FedERA`. These contents may be helpful for users to understand our repo. ``` FedERA ├── federa │ ├── client │ │ ├── src │ | | ├── client_lib │ | | ├── client │ | | ├── ClientConnection_pb2_grpc │ | | ├── ClientConnection_pb2 │ | | ├── data_utils │ | | ├── distribution │ | | ├── get_data │ | | ├── net_lib │ | | ├── net │ │ └── start_client │ ├── server │ │ ├── src │ │ | ├── algorithms │ │ | ├── server_evaluate │ │ | ├── client_connection_servicer │ │ | ├── client_manager │ │ | ├── client_wrapper │ │ | ├── ClientConnection_pb2_grpc │ │ | ├── ClientConnection_pb2 │ │ | ├── server_lib │ │ | ├── server │ │ | ├── verification │ │ └── start_server | └── test | ├── minitest | └── misc │ └── test ├── misc ├── benchtest | ├── test_results | └── test_scalability └──unittest ├── test_algorithms ├── test_datasets ├── test_models └── test_modules ``` ## The framework is be composed of 4 modules, each module building upon the last: * **Module 1: Verification module** [docs](https://federa.readthedocs.io/en/latest/overview.html#verification-module) * **Module 2: Timeout module** [docs](https://federa.readthedocs.io/en/latest/overview.html#timeout-module) * **Module 3: Intermediate client connections module** [docs](https://federa.readthedocs.io/en/latest/overview.html#intermediate-client-connections-module) * **Module 4: Carbon emission tracking module** [docs](https://federa.readthedocs.io/en/latest/overview.html#carbon-emissions-tracking-module) ## Running tests Various unit tests and bench tests are available in the `test` directory. To run any tests, run the following command from the root directory: ``` python -m test.unittest.test_algorithms python -m test.unittest.test_datasets python -m test.unittest.test_models python -m test.unittest.test_modules ``` ## Federated Learning Algorithms Following federated learning algorithms are implemented in this framework: | Method | Paper | Publication | | ------------------- | ------------------------------------------------------------ | ---------------------------------------------------- | | FedAvg | [Communication-Efficient Learning of Deep Networks from Decentralized Data](http://proceedings.mlr.press/v54/mcmahan17a/mcmahan17a.pdf) | AISTATS'2017 | | FedDyn | [Federated Learning Based on Dynamic Regularization](https://openreview.net/forum?id=B7v4QMR6Z9w) | ICLR' 2021 | | Scaffold | [SCAFFOLD: Stochastic Controlled Averaging for Federated Learning]() | ICML'2020 | | Personalized FedAvg | [Improving Federated Learning Personalization via Model Agnostic Meta Learning](https://arxiv.org/pdf/1909.12488.pdf) | Pre-print | | FedAdagrad | [Adaptive Federated Optimization](https://arxiv.org/pdf/2003.00295.pdf) | ICML'2020 | | FedAdam | [Adaptive Federated Optimization](https://arxiv.org/pdf/2003.00295.pdf) | ICML'2020 | | FedYogi | [Adaptive Federated Optimization](https://arxiv.org/pdf/2003.00295.pdf) | ICML'2020 | | Mime | [Mime: Mimicking Centralized Stochastic Algorithms in Federated Learning](https://arxiv.org/pdf/2008.03606.pdf) | ICML'2020 | | Mimelite | [Mime: Mimicking Centralized Stochastic Algorithms in Federated Learning](https://arxiv.org/pdf/2008.03606.pdf) | ICML'2020 | ## Datasets & Data Partition Sophisticated in the real world, FL needs to handle various kind of data distribution scenarios, including iid and non-iid scenarios. Though there already exists some datasets and partition schemes for published data benchmark, it still can be very messy and hard for researchers to partition datasets according to their specific research problems, and maintain partition results during simulation. ### Data Partition We provide multiple Non-IID data partition schemes. Here we show the data partition visualization of several common used datasets as the examples. #### Balanced IID partition Each client has same number of samples, and same distribution for all class samples.
#### Non-IID partition 2
#### Non-IID partition 3
### Datasets Supported | Dataset | Training samples | Test samples | Classes | ---------------------- | ------------------------ | ------------------ | ------------------ | | MNIST | 60,000 | 10,000 | 10 | | FashionMnist | 60,000 | 10,000 | 10 | | CIFAR-10 | 50,000 | 10,000 | 10 | | CIFAR-100 | 50,000 | 10,000 | 100 | ### Custom Dataset Support We also provide a simple way to add your own dataset to the framework. The models employed in this framework were trained using a limited subset of the publicly accessible benchmark dataset MedMNIST v2 [(link)](https://medmnist.com/). We specifically selected four different medical image classes from this dataset, which include breast ultrasound (US), chest X-ray, retinal optical coherence tomography (OCT), and tissue microscopy. Each image within the dataset possesses dimensions of 28x28 pixels. For the framework's implementation, we utilized this custom dataset for both the side server and the client. Look into [docs](https://federa.readthedocs.io/en/latest/tutorials/dataset.html#adding-support-for-new-datasets) for more details. ## Models Supported `FedERA` has support for the following Deep Learning models, which are loaded from `torchvision.models`: * LeNet-5 * ResNet-18 * ResNet-50 * VGG-16 * AlexNet ### Custom Model Support We also provide a simple way to add your own models to the framework. Look into [docs](https://federa.readthedocs.io/en/latest/tutorials/dataset.html#adding-support-for-new-datasets) for more details. ## Carbon emission tracking In `FedERA` [CodeCarbon](https://github.com/mlco2/codecarbon) package is used to estimate the carbon emissions generated by clients during training. CodeCarbon is a Python package that provides an estimation of the carbon emissions associated with software code. ## Performance Evaluation under different Non-IID setting ### Plotting the accuracy of some algorithms against different Non-IID distributions
### Plotting accuracy on Non-IID distribution with different algorithms

### Comparing accuracy of different algorithm with different Non-IID distributions
## Contact For technical issues related to __**FedERA**__ development, please contact our development team through Github issues or email: **Principal Investigator** Dr Debdoot Sheet
Department of Electrical Engineering,
Indian Institute of Technology Kharagpur
email: debdoot@ee.iitkgp.ac.in **Contributor** Anupam Borthakur
Centre of Excellence in Artificial Intelligence,
Indian Institute of Technology Kharagpur
email: anupamborthakur@kgpian.iitkgp.ac.in
Github username: anupam-kliv Asim Manna
Centre of Excellence in Artificial Intelligence,
Indian Institute of Technology Kharagpur
email: asimmanna17@kgpian.iitkgp.ac.in
Github username: asimmanna17 Aditya Kasliwal
Manipal Institute of Technology
email: kasliwaladitya17@gmail.com
Github username: Kasliwal17 Dipayan Dewan
Centre of Excellence in Artificial Intelligence,
Indian Institute of Technology Kharagpur
email: diipayan93@kgpian.iitkgp.ac.in
Github username: dipayandewan94 ================================================ FILE: codecov.yml ================================================ codecov: require_ci_to_pass: yes ================================================ FILE: configs/test_algorithms.json ================================================ { "fedavg":{ "server": { "algorithm":"fedavg", "num_of_clients":1, "fraction_of_clients":1, "num_of_rounds":1, "initial_model_path":"initial_model.pt", "epochs":3, "accept_conn_after_FL_begin":0 , "verify": 0, "verification_threshold": 0, "timeout": 20, "resize_size": 32, "batch_size": 32, "net": "LeNet", "dataset": "MNIST", "device": "cpu", "niid": 2, "carbon": 0, "encryption": 0, "server_key": null, "server_cert": null }, "client": { "ip_address": "localhost:8214", "device":"cpu", "wait_time": 10, "encryption": 0, "ca": null } }, "fedadagrad":{ "server":{ "algorithm":"fedadagrad", "num_of_clients":1, "fraction_of_clients":1, "num_of_rounds":1, "initial_model_path":"initial_model.pt", "epochs":3, "accept_conn_after_FL_begin":0 , "verify": 0, "verification_threshold": 0, "timeout": 20, "resize_size": 32, "batch_size": 32, "net": "LeNet", "dataset": "CIFAR10", "device": "cpu", "niid": 2, "carbon": 0, "encryption": 0, "server_key": null, "server_cert": null }, "client":{ "ip_address": "localhost:8214", "device":"cpu", "wait_time": 10, "encryption": 0, "ca": null } }, "fedadam":{ "server":{ "algorithm":"fedadam", "num_of_clients":1, "fraction_of_clients":1, "num_of_rounds":1, "initial_model_path":"initial_model.pt", "epochs":3, "accept_conn_after_FL_begin":0 , "verify": 0, "verification_threshold": 0, "timeout": 20, "resize_size": 32, "batch_size": 32, "net": "LeNet", "dataset": "CIFAR100", "device": "cpu", "niid": 2, "carbon": 0, "encryption": 0, "server_key": null, "server_cert": null }, "client":{ "ip_address": "localhost:8214", "device":"cpu", "wait_time": 10, "encryption": 0, "ca": null } }, "fedavgm":{ "server":{ "algorithm":"fedavgm", "num_of_clients":1, "fraction_of_clients":1, "num_of_rounds":1, "initial_model_path":"initial_model.pt", "epochs":3, "accept_conn_after_FL_begin":0 , "verify": 0, "verification_threshold": 0, "timeout": 20, "resize_size": 224, "batch_size": 32, "net": "AlexNet", "dataset": "CIFAR100", "device": "cpu", "niid": 2, "carbon": 0, "encryption": 0, "server_key": null, "server_cert": null }, "client":{ "ip_address": "localhost:8214", "device":"cpu", "wait_time": 10, "encryption": 0, "ca": null } }, "feddyn":{ "server":{ "algorithm":"feddyn", "num_of_clients":1, "fraction_of_clients":1, "num_of_rounds":1, "initial_model_path":"initial_model.pt", "epochs":3, "accept_conn_after_FL_begin":0 , "verify": 0, "verification_threshold": 0, "timeout": 20, "resize_size": 32, "batch_size": 32, "net": "LeNet", "dataset": "MNIST", "device": "cpu", "niid": 2, "carbon": 0, "encryption": 0, "server_key": null, "server_cert": null }, "client":{ "ip_address": "localhost:8214", "device":"cpu", "wait_time": 10, "encryption": 0, "ca": null } }, "fedyogi":{ "server":{ "algorithm":"fedyogi", "num_of_clients":1, "fraction_of_clients":1, "num_of_rounds":1, "initial_model_path":"initial_model.pt", "epochs":3, "accept_conn_after_FL_begin":0 , "verify": 0, "verification_threshold": 0, "timeout": 20, "resize_size": 32, "batch_size": 32, "net": "LeNet", "dataset": "MNIST", "device": "cpu", "niid": 2, "carbon": 0, "encryption": 0, "server_key": null, "server_cert": null }, "client":{ "ip_address": "localhost:8214", "device":"cpu", "wait_time": 10, "encryption": 0, "ca": null } }, "mime":{ "server":{ "algorithm":"mime", "num_of_clients":1, "fraction_of_clients":1, "num_of_rounds":1, "initial_model_path":"initial_model.pt", "epochs":2, "accept_conn_after_FL_begin":0 , "verify": 0, "verification_threshold": 0, "timeout": 20, "resize_size": 32, "batch_size": 32, "net": "LeNet", "dataset": "MNIST", "device": "cpu", "niid": 2, "carbon": 0, "encryption": 0, "server_key": null, "server_cert": null }, "client":{ "ip_address": "localhost:8214", "device":"cpu", "wait_time": 10, "encryption": 0, "ca": null } }, "mimelite":{ "server":{ "algorithm":"mimelite", "num_of_clients":1, "fraction_of_clients":1, "num_of_rounds":1, "initial_model_path":"initial_model.pt", "epochs":3, "accept_conn_after_FL_begin":0 , "verify": 0, "verification_threshold": 0, "timeout": 20, "resize_size": 32, "batch_size": 32, "net": "LeNet", "dataset": "MNIST", "device": "cpu", "niid": 2, "carbon": 0, "encryption": 0, "server_key": null, "server_cert": null }, "client":{ "ip_address": "localhost:8214", "device":"cpu", "wait_time": 10, "encryption": 0, "ca": null } }, "scaffold":{ "server":{ "algorithm":"scaffold", "num_of_clients":1, "fraction_of_clients":1, "num_of_rounds":1, "initial_model_path":"initial_model.pt", "epochs":3, "accept_conn_after_FL_begin":0 , "verify": 0, "verification_threshold": 0, "timeout": 20, "resize_size": 32, "batch_size": 32, "net": "LeNet", "dataset": "MNIST", "device": "cpu", "niid": 2, "carbon": 0, "encryption": 0, "server_key": null, "server_cert": null }, "client":{ "ip_address": "localhost:8214", "device":"cpu", "wait_time": 10, "encryption": 0, "ca": null } } } ================================================ FILE: configs/test_datasets.json ================================================ { "MNIST":{ "server": { "algorithm":"fedavg", "num_of_clients":1, "fraction_of_clients":1, "num_of_rounds":1, "initial_model_path":"initial_model.pt", "epochs":1, "accept_conn_after_FL_begin":0 , "verify": 0, "verification_threshold": 0, "timeout": null, "resize_size": 32, "batch_size": 32, "net": "LeNet", "dataset": "MNIST", "device": "cpu", "niid": 2, "carbon": 0, "encryption": 0, "server_key": null, "server_cert": null }, "client": { "ip_address": "localhost:8214", "device":"cpu", "wait_time": 10, "encryption": 0, "ca": null } }, "FashionMNIST":{ "server": { "algorithm":"fedavg", "num_of_clients":1, "fraction_of_clients":1, "num_of_rounds":1, "initial_model_path":"initial_model.pt", "epochs":1, "accept_conn_after_FL_begin":0 , "verify": 0, "verification_threshold": 0, "timeout": null, "resize_size": 32, "batch_size": 32, "net": "LeNet", "dataset": "FashionMNIST", "device": "cpu", "niid": 2, "carbon": 0, "encryption": 0, "server_key": null, "server_cert": null }, "client": { "ip_address": "localhost:8214", "device":"cpu", "wait_time": 10, "encryption": 0, "ca": null } }, "CIFAR10":{ "server":{ "algorithm":"fedavg", "num_of_clients":1, "fraction_of_clients":1, "num_of_rounds":1, "initial_model_path":"initial_model.pt", "epochs":1, "accept_conn_after_FL_begin":0 , "verify": 0, "verification_threshold": 0, "timeout": null, "resize_size": 32, "batch_size": 32, "net": "resnet18", "dataset": "CIFAR10", "device": "cpu", "niid": 2, "carbon": 0, "encryption": 0, "server_key": null, "server_cert": null }, "client":{ "ip_address": "localhost:8214", "device":"cpu", "wait_time": 10, "encryption": 0, "ca": null } }, "CIFAR100":{ "server":{ "algorithm":"fedavg", "num_of_clients":1, "fraction_of_clients":1, "num_of_rounds":1, "initial_model_path":"initial_model.pt", "epochs":1, "accept_conn_after_FL_begin":0 , "verify": 0, "verification_threshold": 0, "timeout": null, "resize_size": 32, "batch_size": 32, "net": "resnet18", "dataset": "CIFAR100", "device": "cpu", "niid": 2, "carbon": 0, "encryption": 0, "server_key": null, "server_cert": null }, "client":{ "ip_address": "localhost:8214", "device":"cpu", "wait_time": 10, "encryption": 0, "ca": null } }, "CUSTOM":{ "server":{ "algorithm":"fedavg", "num_of_clients":1, "fraction_of_clients":1, "num_of_rounds":1, "initial_model_path":"initial_model.pt", "epochs":1, "accept_conn_after_FL_begin":0 , "verify": 0, "verification_threshold": 0, "timeout": null, "resize_size": 32, "batch_size": 32, "net": "LeNet", "dataset": "CUSTOM", "device": "cpu", "niid": 2, "carbon": 0, "encryption": 0, "server_key": null, "server_cert": null }, "client":{ "ip_address": "localhost:8214", "device":"cpu", "wait_time": 10, "encryption": 0, "ca": null } } } ================================================ FILE: configs/test_models.json ================================================ { "LeNet":{ "server":{ "algorithm":"fedavg", "num_of_clients":1, "fraction_of_clients":1, "num_of_rounds":1, "initial_model_path":"initial_model.pt", "epochs":1, "accept_conn_after_FL_begin":0 , "verify": 0, "verification_threshold": 0, "timeout": null, "resize_size": 32, "batch_size": 32, "net": "LeNet", "dataset": "MNIST", "device": "cpu", "niid": 2, "carbon": 0, "encryption": 0, "server_key": null, "server_cert": null }, "client":{ "ip_address": "localhost:8214", "device":"cpu", "wait_time": 10, "encryption": 0, "ca": null } }, "resnet18":{ "server":{ "algorithm":"fedavg", "num_of_clients":1, "fraction_of_clients":1, "num_of_rounds":1, "initial_model_path":"initial_model.pt", "epochs":1, "accept_conn_after_FL_begin":0 , "verify": 0, "verification_threshold": 0, "timeout": null, "resize_size": 32, "batch_size": 32, "net": "resnet18", "dataset": "CIFAR10", "device": "cpu", "niid": 2, "carbon": 0, "encryption": 0, "server_key": null, "server_cert": null }, "client":{ "ip_address": "localhost:8214", "device":"cpu", "wait_time": 10, "encryption": 0, "ca": null } }, "resnet50":{ "server":{ "algorithm":"fedavg", "num_of_clients":1, "fraction_of_clients":1, "num_of_rounds":1, "initial_model_path":"initial_model.pt", "epochs":1, "accept_conn_after_FL_begin":0 , "verify": 0, "verification_threshold": 0, "timeout": null, "resize_size": 32, "batch_size": 32, "net": "resnet50", "dataset": "CIFAR10", "device": "cpu", "niid": 2, "carbon": 0, "encryption": 0, "server_key": null, "server_cert": null }, "client":{ "ip_address": "localhost:8214", "device":"cpu", "wait_time": 10, "encryption": 0, "ca": null } }, "vgg16":{ "server":{ "algorithm":"fedavg", "num_of_clients":1, "fraction_of_clients":1, "num_of_rounds":1, "initial_model_path":"initial_model.pt", "epochs":1, "accept_conn_after_FL_begin":0 , "verify": 0, "verification_threshold": 0, "timeout": null, "resize_size": 32, "batch_size": 32, "net": "vgg16", "dataset": "CIFAR10", "device": "cpu", "niid": 2, "carbon": 0, "encryption": 0, "server_key": null, "server_cert": null }, "client":{ "ip_address": "localhost:8214", "device":"cpu", "wait_time": 10, "encryption": 0, "ca": null } }, "AlexNet":{ "server":{ "algorithm":"fedavg", "num_of_clients":1, "fraction_of_clients":1, "num_of_rounds":1, "initial_model_path":"initial_model.pt", "epochs":1, "accept_conn_after_FL_begin":0 , "verify": 0, "verification_threshold": 0, "timeout": 30, "resize_size": 224, "batch_size": 32, "net": "AlexNet", "dataset": "CIFAR10", "device": "cpu", "niid": 2, "carbon": 0, "encryption": 0, "server_key": null, "server_cert": null }, "client":{ "ip_address": "localhost:8214", "device":"cpu", "wait_time": 10, "encryption": 0, "ca": null } } } ================================================ FILE: configs/test_modules.json ================================================ { "verification":{ "server":{ "algorithm":"fedavg", "num_of_clients":2, "fraction_of_clients":1, "num_of_rounds":1, "initial_model_path":"initial_model.pt", "epochs":1, "accept_conn_after_FL_begin":0 , "verify": 1, "verification_threshold": 0.1, "timeout": null, "resize_size": 32, "batch_size": 32, "net": "LeNet", "dataset": "MNIST", "device": "cpu", "niid": 2, "carbon": 0, "encryption": 0, "server_key": null, "server_cert": null }, "client":{ "ip_address": "localhost:8214", "device":"cpu", "wait_time": 10, "encryption": 0, "ca": null } }, "timeout":{ "server":{ "algorithm":"fedavg", "num_of_clients":2, "fraction_of_clients":1, "num_of_rounds":1, "initial_model_path":"initial_model.pt", "epochs":1, "accept_conn_after_FL_begin":0 , "verify": 0, "verification_threshold": 0, "timeout": 60, "resize_size": 32, "batch_size": 32, "net": "LeNet", "dataset": "MNIST", "device": "cpu", "niid": 2, "carbon": 0, "encryption": 0, "server_key": null, "server_cert": null }, "client":{ "ip_address": "localhost:8214", "device":"cpu", "wait_time": 10, "encryption": 0, "ca": null } }, "intermediate":{ "server":{ "algorithm":"fedavg", "num_of_clients":1, "fraction_of_clients":1, "num_of_rounds":2, "initial_model_path":"initial_model.pt", "epochs":1, "accept_conn_after_FL_begin":1 , "verify": 0, "verification_threshold": 0, "timeout": null, "resize_size": 32, "batch_size": 32, "net": "LeNet", "dataset": "MNIST", "device": "cpu", "niid": 2, "carbon": 0, "encryption": 0, "server_key": null, "server_cert": null }, "client":{ "ip_address": "localhost:8214", "device":"cpu", "wait_time": 10, "encryption": 0, "ca": null } } } ================================================ FILE: configs/test_results.json ================================================ { "fedavg":{ "server": { "algorithm":"fedavg", "num_of_clients":2, "fraction_of_clients":1, "num_of_rounds":1, "initial_model_path":"initial_model.pt", "epochs":1, "accept_conn_after_FL_begin":0 , "verify": 0, "verification_threshold": 0, "timeout": null, "resize_size": 32, "batch_size": 32, "net": "LeNet", "dataset": "MNIST", "device": "cpu", "niid": 2, "carbon": 0, "encryption": 0, "server_key": null, "server_cert": null }, "client": { "ip_address": "localhost:8214", "device":"cpu", "wait_time": 10, "encryption": 0, "ca": null } }, "fedadagrad":{ "server":{ "algorithm":"fedadagrad", "num_of_clients":2, "fraction_of_clients":1, "num_of_rounds":2, "initial_model_path":"initial_model.pt", "epochs":1, "accept_conn_after_FL_begin":0 , "verify": 0, "verification_threshold": 0, "timeout": null, "resize_size": 32, "batch_size": 32, "net": "LeNet", "dataset": "MNIST", "device": "cpu", "niid": 2, "carbon": 0, "encryption": 0, "server_key": null, "server_cert": null }, "client":{ "ip_address": "localhost:8214", "device":"cpu", "wait_time": 10, "encryption": 0, "ca": null } }, "fedadam":{ "server":{ "algorithm":"fedadam", "num_of_clients":2, "fraction_of_clients":1, "num_of_rounds":1, "initial_model_path":"initial_model.pt", "epochs":1, "accept_conn_after_FL_begin":0 , "verify": 0, "verification_threshold": 0, "timeout": null, "resize_size": 32, "batch_size": 32, "net": "LeNet", "dataset": "MNIST", "device": "cpu", "niid": 2, "carbon": 0, "encryption": 0, "server_key": null, "server_cert": null }, "client":{ "ip_address": "localhost:8214", "device":"cpu", "wait_time": 10, "encryption": 0, "ca": null } }, "fedavgm":{ "server":{ "algorithm":"fedavgm", "num_of_clients":2, "fraction_of_clients":1, "num_of_rounds":1, "initial_model_path":"initial_model.pt", "epochs":1, "accept_conn_after_FL_begin":0 , "verify": 0, "verification_threshold": 0, "timeout": null, "resize_size": 32, "batch_size": 32, "net": "LeNet", "dataset": "MNIST", "device": "cpu", "niid": 2, "carbon": 0, "encryption": 0, "server_key": null, "server_cert": null }, "client":{ "ip_address": "localhost:8214", "device":"cpu", "wait_time": 10, "encryption": 0, "ca": null } }, "feddyn":{ "server":{ "algorithm":"feddyn", "num_of_clients":2, "fraction_of_clients":1, "num_of_rounds":1, "initial_model_path":"initial_model.pt", "epochs":1, "accept_conn_after_FL_begin":0 , "verify": 0, "verification_threshold": 0, "timeout": null, "resize_size": 32, "batch_size": 32, "net": "LeNet", "dataset": "MNIST", "device": "cpu", "niid": 2, "carbon": 0, "encryption": 0, "server_key": null, "server_cert": null }, "client":{ "ip_address": "localhost:8214", "device":"cpu", "wait_time": 10, "encryption": 0, "ca": null } }, "fedyogi":{ "server":{ "algorithm":"fedyogi", "num_of_clients":2, "fraction_of_clients":1, "num_of_rounds":1, "initial_model_path":"initial_model.pt", "epochs":1, "accept_conn_after_FL_begin":0 , "verify": 0, "verification_threshold": 0, "timeout": null, "resize_size": 32, "batch_size": 32, "net": "LeNet", "dataset": "MNIST", "device": "cpu", "niid": 2, "carbon": 0, "encryption": 0, "server_key": null, "server_cert": null }, "client":{ "ip_address": "localhost:8214", "device":"cpu", "wait_time": 10, "encryption": 0, "ca": null } }, "mime":{ "server":{ "algorithm":"mime", "num_of_clients":2, "fraction_of_clients":1, "num_of_rounds":1, "initial_model_path":"initial_model.pt", "epochs":1, "accept_conn_after_FL_begin":0 , "verify": 0, "verification_threshold": 0, "timeout": null, "resize_size": 32, "batch_size": 32, "net": "LeNet", "dataset": "MNIST", "device": "cpu", "niid": 2, "carbon": 0, "encryption": 0, "server_key": null, "server_cert": null }, "client":{ "ip_address": "localhost:8214", "device":"cpu", "wait_time": 10, "encryption": 0, "ca": null } }, "mimelite":{ "server":{ "algorithm":"mimelite", "num_of_clients":2, "fraction_of_clients":1, "num_of_rounds":1, "initial_model_path":"initial_model.pt", "epochs":1, "accept_conn_after_FL_begin":0 , "verify": 0, "verification_threshold": 0, "timeout": null, "resize_size": 32, "batch_size": 32, "net": "LeNet", "dataset": "MNIST", "device": "cpu", "niid": 2, "carbon": 0, "encryption": 0, "server_key": null, "server_cert": null }, "client":{ "ip_address": "localhost:8214", "device":"cpu", "wait_time": 10, "encryption": 0, "ca": null } }, "scaffold":{ "server":{ "algorithm":"scaffold", "num_of_clients":2, "fraction_of_clients":1, "num_of_rounds":1, "initial_model_path":"initial_model.pt", "epochs":1, "accept_conn_after_FL_begin":0 , "verify": 0, "verification_threshold": 0, "timeout": null, "resize_size": 32, "batch_size": 32, "net": "LeNet", "dataset": "MNIST", "device": "cpu", "niid": 2, "carbon": 0, "encryption": 0, "server_key": null, "server_cert": null }, "client":{ "ip_address": "localhost:8214", "device":"cpu", "wait_time": 10, "encryption": 0, "ca": null } } } ================================================ FILE: configs/test_scalability.json ================================================ { "2":{ "server":{ "algorithm":"fedavg", "num_of_clients":2, "fraction_of_clients":1, "num_of_rounds":1, "initial_model_path":"initial_model.pt", "epochs":1, "accept_conn_after_FL_begin":0 , "verify": 0, "verification_threshold": 0, "timeout": null, "resize_size": 32, "batch_size": 32, "net": "LeNet", "dataset": "MNIST", "device": "cpu", "niid": 2, "carbon": 0, "encryption": 0, "server_key": null, "server_cert": null }, "client":{ "ip_address": "localhost:8214", "device":"cpu", "wait_time": 10, "encryption": 0, "ca": null } }, "4":{ "server":{ "algorithm":"fedavg", "num_of_clients":4, "fraction_of_clients":1, "num_of_rounds":1, "initial_model_path":"initial_model.pt", "epochs":1, "accept_conn_after_FL_begin":0 , "verify": 0, "verification_threshold": 0, "timeout": null, "resize_size": 32, "batch_size": 32, "net": "LeNet", "dataset": "MNIST", "device": "cpu", "niid": 2, "carbon": 0, "encryption": 0, "server_key": null, "server_cert": null }, "client":{ "ip_address": "localhost:8214", "device":"cpu", "wait_time": 10, "encryption": 0, "ca": null } }, "6":{ "server":{ "algorithm":"fedavg", "num_of_clients":6, "fraction_of_clients":1, "num_of_rounds":1, "initial_model_path":"initial_model.pt", "epochs":1, "accept_conn_after_FL_begin":0 , "verify": 0, "verification_threshold": 0, "timeout": null, "resize_size": 32, "batch_size": 32, "net": "LeNet", "dataset": "MNIST", "device": "cpu", "niid": 2, "carbon": 0, "encryption": 0, "server_key": null, "server_cert": null }, "client":{ "ip_address": "localhost:8214", "device":"cpu", "wait_time": 10, "encryption": 0, "ca": null } }, "8":{ "server":{ "algorithm":"fedavg", "num_of_clients":8, "fraction_of_clients":1, "num_of_rounds":1, "initial_model_path":"initial_model.pt", "epochs":1, "accept_conn_after_FL_begin":0 , "verify": 0, "verification_threshold": 0, "timeout": null, "resize_size": 32, "batch_size": 32, "net": "LeNet", "dataset": "MNIST", "device": "cpu", "niid": 2, "carbon": 0, "encryption": 0, "server_key": null, "server_cert": null }, "client":{ "ip_address": "localhost:8214", "device":"cpu", "wait_time": 10, "encryption": 0, "ca": null } }, "10":{ "server":{ "algorithm":"fedavg", "num_of_clients":10, "fraction_of_clients":1, "num_of_rounds":1, "initial_model_path":"initial_model.pt", "epochs":1, "accept_conn_after_FL_begin":0 , "verify": 0, "verification_threshold": 0, "timeout": null, "resize_size": 32, "batch_size": 32, "net": "LeNet", "dataset": "MNIST", "device": "cpu", "niid": 2, "carbon": 0, "encryption": 0, "server_key": null, "server_cert": null }, "client":{ "ip_address": "localhost:8214", "device":"cpu", "wait_time": 10, "encryption": 0, "ca": null } }, "5_rounds":{ "server":{ "algorithm":"fedavg", "num_of_clients":2, "fraction_of_clients":1, "num_of_rounds":5, "initial_model_path":"initial_model.pt", "epochs":1, "accept_conn_after_FL_begin":0 , "verify": 0, "verification_threshold": 0, "timeout": null, "resize_size": 32, "batch_size": 32, "net": "LeNet", "dataset": "MNIST", "device": "cpu", "niid": 2, "carbon": 0, "encryption": 0, "server_key": null, "server_cert": null }, "client":{ "ip_address": "localhost:8214", "device":"cpu", "wait_time": 10, "encryption": 0, "ca": null } }, "10_rounds":{ "server":{ "algorithm":"fedavg", "num_of_clients":2, "fraction_of_clients":1, "num_of_rounds":10, "initial_model_path":"initial_model.pt", "epochs":1, "accept_conn_after_FL_begin":0 , "verify": 0, "verification_threshold": 0, "timeout": null, "resize_size": 32, "batch_size": 32, "net": "LeNet", "dataset": "MNIST", "device": "cpu", "niid": 2, "carbon": 0, "encryption": 0, "server_key": null, "server_cert": null }, "client":{ "ip_address": "localhost:8214", "device":"cpu", "wait_time": 10, "encryption": 0, "ca": null } }, "20_rounds":{ "server":{ "algorithm":"fedavg", "num_of_clients":2, "fraction_of_clients":1, "num_of_rounds":20, "initial_model_path":"initial_model.pt", "epochs":1, "accept_conn_after_FL_begin":0 , "verify": 0, "verification_threshold": 0, "timeout": null, "resize_size": 32, "batch_size": 32, "net": "LeNet", "dataset": "MNIST", "device": "cpu", "niid": 2, "carbon": 0, "encryption": 0, "server_key": null, "server_cert": null }, "client":{ "ip_address": "localhost:8214", "device":"cpu", "wait_time": 10, "encryption": 0, "ca": null } } } ================================================ FILE: docs/Makefile ================================================ # Minimal makefile for Sphinx documentation # # You can set these variables from the command line, and also # from the environment for the first two. SPHINXOPTS ?= SPHINXBUILD ?= sphinx-build SOURCEDIR = source BUILDDIR = build # Put it first so that "make" without argument is like "make help". help: @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) .PHONY: help Makefile # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) ================================================ FILE: docs/make.bat ================================================ @ECHO OFF pushd %~dp0 REM Command file for Sphinx documentation if "%SPHINXBUILD%" == "" ( set SPHINXBUILD=sphinx-build ) set SOURCEDIR=source set BUILDDIR=build if "%1" == "" goto help %SPHINXBUILD% >NUL 2>NUL if errorlevel 9009 ( echo. echo.The 'sphinx-build' command was not found. Make sure you have Sphinx echo.installed, then set the SPHINXBUILD environment variable to point echo.to the full path of the 'sphinx-build' executable. Alternatively you echo.may add the Sphinx directory to PATH. echo. echo.If you don't have Sphinx installed, grab it from echo.http://sphinx-doc.org/ exit /b 1 ) %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% goto end :help %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% :end popd ================================================ FILE: docs/requirements.txt ================================================ sphinx furo sphinxcontrib-napoleon sphinx-autoapi sphinx-design sphinxcontrib-bibtex ================================================ FILE: docs/source/_autoapi_temp/python/module.rst ================================================ {% if not obj.display %} :orphan: {% endif %} {{ obj.short_name }} {{ "=" * obj.short_name|length }} .. py:module:: {{ obj.name }} {% if obj.docstring %} .. autoapi-nested-parse:: {{ obj.docstring|indent(3) }} {% endif %} {% block subpackages %} {% set visible_subpackages = obj.subpackages|selectattr("display")|list %} {% if visible_subpackages %} .. toctree:: :titlesonly: :maxdepth: 3 {% for subpackage in visible_subpackages %} {{ subpackage.short_name }}/index.rst {% endfor %} {% endif %} {% endblock %} {% block submodules %} {% set visible_submodules = obj.submodules|selectattr("display")|list %} {% if visible_submodules %} .. toctree:: :titlesonly: :maxdepth: 1 {% for submodule in visible_submodules %} {{ submodule.short_name }}/index.rst {% endfor %} {% endif %} {% endblock %} {% block content %} {% if obj.all is not none %} {% set visible_children = obj.children|selectattr("short_name", "in", obj.all)|list %} {% elif obj.type is equalto("package") %} {% set visible_children = obj.children|selectattr("display")|list %} {% else %} {% set visible_children = obj.children|selectattr("display")|rejectattr("imported")|list %} {% endif %} {% if visible_children %} {{ obj.type|title }} Contents {{ "-" * obj.type|length }}--------- {% set visible_classes = visible_children|selectattr("type", "equalto", "class")|list %} {% set visible_functions = visible_children|selectattr("type", "equalto", "function")|list %} {% set visible_attributes = visible_children|selectattr("type", "equalto", "data")|list %} {% if "show-module-summary" in autoapi_options and (visible_classes or visible_functions) %} {% block classes scoped %} {% if visible_classes %} .. autoapisummary:: {% for klass in visible_classes %} {{ klass.id }} {% endfor %} {% endif %} {% endblock %} {% block functions scoped %} {% if visible_functions %} .. autoapisummary:: {% for function in visible_functions %} {{ function.id }} {% endfor %} {% endif %} {% endblock %} {% block attributes scoped %} {% if visible_attributes %} .. autoapisummary:: {% for attribute in visible_attributes %} {{ attribute.id }} {% endfor %} {% endif %} {% endblock %} {% endif %} {% for obj_item in visible_children %} {{ obj_item.render()|indent(0) }} {% endfor %} {% endif %} {% endblock %} ================================================ FILE: docs/source/conf.py ================================================ # Configuration file for the Sphinx documentation builder. # -- Project information project = 'FedERA' copyright = '2023, KLIV' author = 'KLIV' release = '0.1' version = '0.1.0' # -- General configuration extensions = [ 'autoapi.extension', # this one is really important 'sphinx.ext.viewcode', 'sphinx.ext.githubpages', 'sphinx.ext.mathjax', 'sphinx.ext.intersphinx', 'sphinx.ext.todo', 'sphinx.ext.autosectionlabel', # allows referring sections its title, affects `ref` 'sphinx_design', 'sphinxcontrib.bibtex', ] # for 'sphinxcontrib.bibtex' extension bibtex_bibfiles = ['refer.bib'] bibtex_default_style = 'unsrt' autodoc_mock_imports = ["numpy", "torch", "torchvision", "pandas"] autoclass_content = 'both' templates_path = ['_templates'] # configuration for 'autoapi.extension' autoapi_type = 'python' autoapi_dirs = ['../../federa'] autoapi_template_dir = '_autoapi_temp' add_module_names = False # makes Sphinx render package.module.Class as Class # Add more mapping for 'sphinx.ext.intersphinx' intersphinx_mapping = {'python': ('https://docs.python.org/3', None), 'PyTorch': ('http://pytorch.org/docs/master/', None), 'numpy': ('https://numpy.org/doc/stable/', None), 'pandas': ('https://pandas.pydata.org/pandas-docs/dev/', None)} # autosectionlabel throws warnings if section names are duplicated. # The following tells autosectionlabel to not throw a warning for # duplicated section names that are in different documents. autosectionlabel_prefix_document = True # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' # Config for 'sphinx.ext.todo' todo_include_todos = True # multi-language docs language = 'en' locale_dirs = ['../locales/'] # path is example but recommended. gettext_compact = False # optional. gettext_uuid = True # optional. # -- Options for HTML output html_theme = 'sphinx_rtd_theme' # -- Options for EPUB output epub_show_urls = 'footnote' ================================================ FILE: docs/source/contribution.rst ================================================ .. _contribution: ********************** Contribution to FedERA ********************** Reporting bugs -------------- To report bugs or request features, we utilize GitHub issues. If you come across a bug or have an idea for a feature, don't hesitate to open an issue. If you encounter any problems while using this software package, please submit a ticket to the Bug Tracker. Additionally, you can post pull requests or feature requests. Contributing to FedERA ---------------------- If you wish to contribute to the project by submitting code, you can do so by creating a Pull Request. By contributing code, you agree that your contributions will be licensed under `Apache License, Version 2.0 `_. We encourage you to contribute to the enhancement of **FedERA** or the implementation of existing FL methods within **FedERA**. The recommended method for contributing to **FedERA** is to fork the main repository on GitHub, clone it, and develop on a branch. Follow these steps: 1. Click on "Fork" to fork the project repository. 2. Clone your forked repository from your GitHub account to your local machine: .. code-block:: shell-session $ git clone https://github.com/anupamkliv/FedERA.git and then navigate to the FedLab directory using the command .. code-block:: shell-session $ cd FedERA 3. Create a new branch to save your changes using the command .. code-block:: shell-session $ git checkout -b my-feature 4. Develop the feature on your branch and use the command .. code-block:: shell-session $ git add modified_files followed by .. code-block:: shell-session $ git commit to save your changes. Pull Request Checklist ---------------------- - Please follow the file structure below for new features or create new file if there are something new. .. code-block:: shell-session FedERA ├── federa │ ├── client │ │ ├── src │ | | ├── client_lib │ | | ├── client │ | | ├── ClientConnection_pb2_grpc │ | | ├── ClientConnection_pb2 │ | | ├── data_utils │ | | ├── distribution │ | | ├── get_data │ | | ├── net_lib │ | | ├── net │ │ └── start_client │ ├── server │ │ ├── src │ │ | ├── algorithms │ │ | ├── server_evaluate │ │ | ├── client_connection_servicer │ │ | ├── client_manager │ │ | ├── client_wrapper │ │ | ├── ClientConnection_pb2_grpc │ │ | ├── ClientConnection_pb2 │ │ | ├── server_lib │ │ | ├── server │ │ | ├── verification │ │ └── start_server | └── test | ├── minitest | └── misc │ └── test ├── misc ├── benchtest | ├── test_results | └── test_scalability └──unittest ├── test_algorithms ├── test_datasets ├── test_models └── test_modules ================================================ FILE: docs/source/index.rst ================================================ FedERA =================================== **FedERA** is a highly dynamic and customizable framework that can accommodate many use cases with flexibility by implementing several functionalities over different federated learning algorithms, and essentially creating a plug-and-play architecture to accommodate different use cases. Check out the :doc:`overview` section for further information, including how to :ref:`installation` the project. .. note:: This project is under active development. Contents -------- .. toctree:: :maxdepth: 2 overview installation tutorials/tutorial contribution reference .. Citation .. ........ .. Please cite **FedERA** in your publications if it helps your research: .. .. code:: latex .. @article{ .. } Contacts ........ Contact the **FedERA** development team through Github issues or email: - Development Team: federa.team@gmail.com ================================================ FILE: docs/source/installation.rst ================================================ .. _installation: Installation ============ Install the package ------------------- Follow this procedure to prepare the environment and install **FedERA**: 1. Install a Python 3.8 (>=3.6, <=3.9) virtual environment using venv. See the `Venv installation guide `_ for details. 2. Create a new environment for the project. A. Using Virtual Environment .. code-block:: console $ python3 -m venv env B. Using conda .. code-block:: console $ conda create -n env python=3.9 3. Activate the virtual environment. A. Virtual Environment .. code-block:: console $ source env/bin/activate B. Conda Environment .. code-block:: console $ conda activate env 4. Install the **FedERA** package. A. Install the **stable version** with pip: .. code-block:: console $ pip install feder==$version$ B. Install the **latest version** from GitHub: 1. Clone the **FedERA** repository: .. code-block:: console $ git clone https://github.com/anupamkliv/FedERA.git $ cd FedERA 2. Install dependencies: .. code-block:: console $ pip install -r requirements.txt FedERA with Docker ------------------ Follow this procedure to build a Docker image of **FedERA**: .. note:: The purpose of the Docker edition of **FedERA** is to provide an isolated environment complete with the prerequisites to run. Once the execution is finished, the container can be eliminated, and the computation results will be accessible in a directory on the local host. 1. Install Docker on all nodes in the federation. See the `Docker installation guide `_ for details. 2. Check that Docker is running properly with the *Hello World* command: .. code-block:: console $ docker run hello-world Hello from Docker! This message shows that your installation appears to be working correctly. ... ... ... 3. Build the Docker image of **FedERA**: .. code-block:: console $ docker build -t federa . 4. Run the Docker image of **FedERA**: .. code-block:: console $ docker run federa ================================================ FILE: docs/source/overview.rst ================================================ .. _overview: ****************** Overview of FedERA ****************** Introduction ============ Federated Learning is a machine learning technique for training models on distributed data without sharing it. In traditional machine learning, large datasets must first be collected and then sent to one location where they can be combined before the model is trained on them. However, this process can cause privacy concerns as sensitive personal data may become publicly available. Federated learning attempts to address these concerns by keeping individual user's data local while still allowing for powerful powerful statistical analysis that can be used to create accurate models at scale. **FedAvg** is one of the foundational blocks of federated learning. A single communication round of FedAvg includes: * Waiting for a number of clients to connect to a server (Step 0) * Sending the clients a global model (Step 1) * Train the model with locally available data (Step 2) * Send the trained models back to the server (Step 3) The server then averages the weights of the models and calculates a new aggregated model. This process constitutes a single communication round and several such communication rounds occur to train a model. .. image:: ../imgs/fedavg_steps.png :align: center .. :class: only-light Overview ======== **FedERA** is a highly dynamic and customizable framework that can accommodate many use cases with flexibility by implementing several functionalities over vanilla FedAvg, and essentially creating a plug-and-play architecture to accommodate different use cases. Federated Learning ------------------ .. image:: ../imgs/phase1.png :align: center | | Establishing Connection between Server and Clients ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. image:: ../imgs/connection.png :align: center | | Communication with clients ~~~~~~~~~~~~~~~~~~~~~~~~~~ .. image:: ../imgs/communication.png :align: center | | Fractional and random subsampling ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ * The Client_Manager can be used to sample the already connected clients. * A minimum number of clients can be provided, upon which the Client_Manager will wait for that many clients to connect before returning a reference to them. * If a fraction is provided, the Client_Manager will return that fraction of available clients. * The Client_Manager can sample the clients based on their connection order or a random order. A function can also be provided to determine the selection of clients. Various modules in Feder ------------------------ Feder is composed of 4 modules, each module building upon the last. 1. **Verification module.** Before aggregating, the server will perform a special verification round to determine which models to accommodate during aggregation. 2. **Timeout module.** Instead of waiting indefinitely for a client to finish training, the server will be able to issue a timeout, upon the completion of which, even if it hasn’t completed all epochs, the client will stop training and return the results. 3. **Intermediate client connections module.** New clients will be able to join the server anytime and may even be included in a round that is already live. 4. **Carbon emissions tracking module.** The framework will be able to track the carbon emissions of the clients during the training process. Verification module ---------------------------- * After the server receives the trained weights, it aggregates all of them to form the new model. However, the selection of models for aggregation can be modified. * Before aggregation, the server passes the models to a Verification module, which then uses a predefined procedure to generate scores for models, and then returns only those models that have performed above a defined threshold. * The Verification module can be easily customized. Steps in the Verification module ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. image:: ../imgs/verification_steps.png :align: center | | Modified Federated Learning architecture ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. image:: ../imgs/verification_1.png :align: center .. image:: ../imgs/verification_2.png :align: center | | Timeout module -------------- * Often in real world scenarios, clients cannot keep training indefinitely. Therefore, a timeout functionality has been implemented. * The server can specify a timeout parameter as a Train order configuration. The client will then train till the timeout occurs, and then return the results. Steps in the Timeout module ~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. image:: ../imgs/timeout.png :align: center | Intermediate client connections module -------------------------------------- * Now, even during the middle of a communication round, the server can accept new client connections, incorporate them into the Client_Manager and even include them in the ongoing communication round as well. * The server can be easily configured to allow or reject new connections during different parts of Federated Learning. * Safeguards to notify when a client has disconnected anytime have been implemented. Carbon emissions tracking module -------------------------------- In **FedERA** CodeCarbon package is used to estimate the carbon emissions generated by clients during training. CodeCarbon is a Python package that provides an estimation of the carbon emissions associated with software code. Tested on ~~~~~~~~~ **FedERA** has been extensively tested on and works with the following devices: * Intel CPUs * Nvidia GPUs * Nvidia Jetson * Raspberry Pi * Intel NUC With **FedERA**, it is possible to operate the server and clients on separate devices or on a single device through various means, such as utilizing different terminals or implementing multiprocessing. .. image:: ../imgs/tested.png :align: center ================================================ FILE: docs/source/refer.bib ================================================ @article{codecarbon, author={Victor Schmidt and Kamal Goyal and Aditya Joshi and Boris Feld and Liam Conell and Nikolas Laskaris and Doug Blank and Jonathan Wilson and Sorelle Friedler and Sasha Luccioni}, title={{CodeCarbon: Estimate and Track Carbon Emissions from Machine Learning Computing}}, year={2021}, howpublished={\url{https://github.com/mlco2/codecarbon}}, DOI={10.5281/zenodo.4658424}, publisher={Zenodo}, } ================================================ FILE: docs/source/reference.rst ================================================ Reference ========= .. bibliography:: ================================================ FILE: docs/source/tutorials/algorithm.rst ================================================ .. _algorithm: ***************************** Federated Learning Algorithms ***************************** The implementation of federated learning algorithms in Feder consists of two components: the training part on the client side and the aggregation part on the server side. The training functions are coded in the net_lib.py file at client/src directory, while the aggregation functions are located in various files within the algorithms folder at server/src directory. The algorithms currently implemented in **FedERA** are: * FedAvg * FedDyn * FedAdam * FedAdagrad * Scaffold * FedAvgM * Mime * Mimelite * FedYogi Adding a new algorithm to **FedERA** ----------------------------------- To add a new algorithm to **FedERA**, you need to implement the training function on the client side and the aggregation function on the server side. The training function should be implemented in the net_lib.py file at client/src directory. The aggregation function should be implemented in a new file in the algorithms folder at server/src directory. Implementing the training function ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The training function should be implemented in the net_lib file, in a fashion similar to the following example of the mimelite algorithm: .. code-block:: python def train_mimelite(net, state, trainloader, epochs, deadline=None): #In the case of MimeLite, control_variate is nothing but a state like in case of momentum method x = deepcopy(net) criterion = torch.nn.CrossEntropyLoss() lr = 0.001 momentum = 0.9 net.train() for _ in tqdm(range(epochs)): for images, labels in trainloader: images, labels = images.to(DEVICE), labels.to(DEVICE) loss = criterion(net(images), labels) #Compute (full-batch) gradient of loss with respect to net's parameters grads = torch.autograd.grad(loss,net.parameters()) #Update net's parameters using gradients with torch.no_grad(): for param,grad,s in zip(net.parameters(), grads, state): param.data = param.data - lr * ((1-momentum) * grad.data + momentum * s.data) if deadline: current_time = time.time() if current_time >= deadline: print("deadline occurred.") break #Compute gradient wrt the received model (x) using the wholde dataset data = DataLoader(trainloader.dataset, batch_size = len(trainloader) * trainloader.batch_size, shuffle = True) for images, labels in data: images, labels = images.to(DEVICE), labels.to(DEVICE) output = x(images) loss = criterion(output, labels) #Calculate the loss with respect to y's output and labels gradient_x = torch.autograd.grad(loss,x.parameters()) return net, gradient_x After making the changes in the net_lib.py file, the client_lib.py file also needs to be updated so as to incorporate the newly defined algorithm. The client_lib.py file is located at client/src directory. The following code snippet shows the train function that needs to be updated in the client_lib.py file: .. code-block:: python def train(train_order_message): data_bytes = train_order_message.modelParameters data = torch.load( BytesIO(data_bytes), map_location="cpu" ) model_parameters, control_variate, control_variate2 = data['model_parameters'], data['control_variate'], data['control_variate2'] config_dict_bytes = train_order_message.configDict config_dict = json.loads( config_dict_bytes.decode("utf-8") ) carbon_tracker = config_dict["carbon_tracker"] model = get_net(config= config_dict) model.load_state_dict(model_parameters) model = model.to(device) epochs = config_dict["epochs"] if config_dict["timeout"]: deadline = time.time() + config_dict["timeout"] else: deadline = None #Run code carbon if the carbon-tracker flag is True if (carbon_tracker==1): tracker = OfflineEmissionsTracker(country_iso_code="IND", output_dir = save_dir_path) tracker.start() trainloader, testloader, _ = load_data(config_dict) print("training started") if (config_dict['algorithm'] == 'mimelite'): model, control_variate = train_mimelite(model, control_variate, trainloader, epochs, deadline) elif (config_dict['algorithm'] == 'scaffold'): model, control_variate = train_scaffold(model, control_variate, trainloader, epochs, deadline) elif (config_dict['algorithm'] == 'mime'): model, control_variate = train_mime(model, control_variate, control_variate2, trainloader, epochs, deadline) elif (config_dict['algorithm'] == 'fedavg'): model = train_fedavg(model, trainloader, epochs, deadline) elif (config_dict['algorithm'] == 'feddyn'): model = train_feddyn(model, trainloader, epochs, deadline) else: model = train_model(model, trainloader, epochs, deadline) print("training finished") if (carbon_tracker==1): emissions: float = tracker.stop() print(f"Emissions: {emissions} kg") myJSON = json.dumps(config_dict) json_path = save_dir_path + "/config.json" with open(json_path, "w") as jsonfile: jsonfile.write(myJSON) json_path = "config.json" with open(json_path, "w") as jsonfile: jsonfile.write(myJSON) trained_model_parameters = model.state_dict() #Create a dictionary where model_parameters and control_variate are stored which needs to be sent to the server data_to_send = {} data_to_send['model_parameters'] = trained_model_parameters data_to_send['control_variate'] = control_variate #If there is no control_variate, this will become None buffer = BytesIO() torch.save(data_to_send, buffer) buffer.seek(0) data_to_send_bytes = buffer.read() print("train eval") train_loss, train_accuracy = test_model(model, testloader) response_dict = {"train_loss": train_loss, "train_accuracy": train_accuracy} response_dict_bytes = json.dumps(response_dict).encode("utf-8") train_response_message = TrainResponse( modelParameters = data_to_send_bytes, responseDict = response_dict_bytes) save_model_state(model) return train_response_message Implementing the aggregation function ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The aggregation function should be implemented within a class in a new file in the algorithms folder at server/src directory. The following code snippet shows the aggregation function for the mimelite algorithm as deffined in the mimelite.py file: .. code-block:: python class mimelite(): def __init__(self, config): self.algorithm = "MimeLite" self.lr = 1.0 self.momentum = 0.9 def aggregate(self,server_model_state_dict, optimizer_state, state_dicts, gradients_x): keys = server_model_state_dict.keys() #List of keys in a state_dict avg_y = OrderedDict() #This will be our new server_model_state_dict for key in keys: current_key_tensors = [state_dict[key] for state_dict in state_dicts] current_key_sum = functools.reduce( lambda accumulator, tensor: accumulator + tensor, current_key_tensors ) current_key_average = current_key_sum / len(state_dicts) avg_y[key] = current_key_average #Average all the gradient_x in gradients_x avg_grads = [] for i in range(len(gradients_x[0])): #Average all the i'th element of gradient_x present in the gradients_x current_tensors = [gradient_x[i] for gradient_x in gradients_x] current_sum = functools.reduce(lambda accumulator, tensor: accumulator + tensor, current_tensors) current_average = current_sum / len(gradients_x) avg_grads.append(current_average) for state, grad in zip(optimizer_state, avg_grads): state.data = self.momentum * state.data + (1 - self.momentum) * grad.data return avg_y, optimizer_state ================================================ FILE: docs/source/tutorials/code_carbon.rst ================================================ .. _code_carbon: **************** Carbon Emissions **************** CodeCarbon is a Python package that provides an estimation of the carbon emissions associated with software code. It can be integrated into software development workflows and offers real-time feedback on the environmental impact of the code being developed. CodeCarbon helps developers and organizations become more environmentally conscious by optimizing their code and making better choices regarding the hardware and infrastructure used to run it. It enables companies to achieve their sustainability goals and demonstrate their commitment to reducing their environmental impact. To estimate the carbon emissions generated by clients during training, CodeCarbon has been utilized in the client_lib.py file, located in the client/src/ directory. By default, the client's location is set to India, but it can be modified to reflect the client's actual location. The following code snippet illustrates how CodeCarbon is used in the client_lib.py file: .. code-block:: python #Run code carbon if the carbon-tracker flag is True if (carbon_tracker==1): tracker = OfflineEmissionsTracker(country_iso_code="IND", output_dir = save_dir_path) tracker.start() trainloader, testloader, _ = load_data(config_dict) print("training started") if (config_dict['algorithm'] == 'mimelite'): model, control_variate = train_mimelite(model, control_variate, trainloader, epochs, deadline) elif (config_dict['algorithm'] == 'scaffold'): model, control_variate = train_scaffold(model, control_variate, trainloader, epochs, deadline) elif (config_dict['algorithm'] == 'mime'): model, control_variate = train_mime(model, control_variate, control_variate2, trainloader, epochs, deadline) elif (config_dict['algorithm'] == 'fedavg'): model = train_fedavg(model, trainloader, epochs, deadline) elif (config_dict['algorithm'] == 'feddyn'): model = train_feddyn(model, trainloader, epochs, deadline) else: model = train_model(model, trainloader, epochs, deadline) print("training finished") if (carbon_tracker==1): emissions: float = tracker.stop() print(f"Emissions: {emissions} kg") ================================================ FILE: docs/source/tutorials/data_distribution.rst ================================================ .. _data_distribution: ***************** Data Distribution ***************** **FedERA** allows the option to train with either IID or non-IID data distribution. To specify the data distribution, you can use the "--iid" flag. When the flag is set to "1", the data distribution is IID. However, if you set it to a value between "2-5", the data distribution will be non-IID. Each argument value corresponds to a different non-IID distribution. The non-IID distributions are defined as follows: .. code:: Python def data_distribution(config, trainset): labels = [] base_dir = os.getcwd() storepath = os.path.join(base_dir, 'Distribution/', config['dataset']+'/') seed = 10 random.seed(seed) num_users = 5 #Calculate the number of samples present per class for i in range(len(trainset)): labels.append(trainset[i][1]) unique_labels = np.unique(np.array(labels)) label_index_list = {} for key in unique_labels: label_index_list[key] = [] for index, label in enumerate(labels): label_index_list[label].append(index) num_classes = len(unique_labels) #Calculate the value of the probability distribution. For K=1, it will be iid distribution K = config['niid'] if (K==1): q_step = (1 - (1/num_classes)) else: q_step = (1 - (1/num_classes))/(K-1) #Shuffle the index position for all classes for i in range(len(label_index_list)): random.shuffle(label_index_list[i]) #Generate the different non-iid distribution. Data_presence_indicator will help to reduce the number of classes among the clients as the non-iid increases for j in range(K): dist = np.random.uniform(q_step, (1+j)*q_step, (num_classes, num_users)) if j != 0: data_presence_indicator = np.random.choice([0, 1], (num_classes, num_users), p=[j*q_step, 1-(j*q_step)]) if len(np.where(np.sum(data_presence_indicator, axis=0) == 0)[0])>0: for i in np.where(np.sum(data_presence_indicator, axis=0) == 0)[0]: zero_array = data_presence_indicator[:,i] zero_array[np.random.choice(len(zero_array),1)] =1 data_presence_indicator[:,i] = zero_array dist = np.multiply(dist,data_presence_indicator) psum = np.sum(dist, axis=1) for i in range(dist.shape[0]): dist[i] = dist[i]*len(label_index_list[i])/(psum[i]+0.00001) dist = np.floor(dist).astype(int) # If any client does not get any data then this logic helps to allocate the required samples among the clients gainers = list(np.where(np.sum(dist, axis=0) != 0))[0] if len(gainers) < num_users: losers = list(np.where(np.sum(dist, axis=0) == 0))[0] donors = np.random.choice(gainers, len(losers)) for index, donor in enumerate(donors): avail_digits = np.where(dist[:,donor] != 0)[0] for digit in avail_digits: transfer_frac = np.random.uniform(0.1,0.9) num_transfer = int(dist[digit, donor]*transfer_frac) dist[digit, donor] = dist[digit, donor] - num_transfer dist[digit, losers[index]] = num_transfer #Logic to check if the summation of all the samples among the clients is equal to the total number of samples present for that class. If not it will adjust. for num in range(num_classes): while dist[num].sum() != len(label_index_list[num]): index = random.randint(0,num_users-1) if dist[num].sum() < len(label_index_list[num]): dist[num][index]+=1 else: dist[num][index]-=1 #Division of samples number among the clients split = [[] for i in range(num_classes)] for num in range(num_classes): start = 0 for i in range(num_users): split[num].append(label_index_list[num][start:start+dist[num][i]]) start = start+dist[num][i] #Division of actual data points among the clients. datapoints = [[] for i in range(num_users)] class_histogram = [[] for i in range(num_users)] class_stats= [[] for i in range(num_users)] for i in range(num_users): for num in range(num_classes): datapoints[i] += split[num][i] class_histogram[i].append(len(split[num][i])) if(len(split[num][i])==0): class_stats[i].append(0) else: class_stats[i].append(1) #Store the dataset division in the folder if not os.path.exists(storepath): os.makedirs(storepath) file_name = 'data_split_niid_'+ str(K)+'.pt' torch.save({'datapoints': datapoints, 'histograms': class_histogram, 'class_statitics': class_stats}, storepath + file_name) Visualizing the non-IID data distribution for MNIST dataset ----------------------------------------------------------- Classwise distribution of samples among the clients for different non-IID distribution ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. figure:: ../../imgs/class_stats_0.png :align: center Classwise distribution of samples among the clients for non-IID distribution 1 | .. figure:: ../../imgs/class_stats_1.png :align: center Classwise distribution of samples among the clients for non-IID distribution 2 | .. figure:: ../../imgs/class_stats_2.png :align: center Classwise distribution of samples among the clients for non-IID distribution 3 | .. figure:: ../../imgs/class_stats_3.png :align: center C lasswise distribution of samples among the clients for non-IID distribution 4 | .. figure:: ../../imgs/class_stats_4.png :align: center Classwise distribution of samples among the clients for non-IID distribution 5 | Samplewise distribution of samples among the clients for different non-IID distribution ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. figure:: ../../imgs/sample_stats_0.png :align: center Samplewise distribution of samples among the clients for non-IID distribution 1 | .. figure:: ../../imgs/sample_stats_1.png :align: center Samplewise distribution of samples among the clients for non-IID distribution 2 | .. figure:: ../../imgs/sample_stats_2.png :align: center Samplewise distribution of samples among the clients for non-IID distribution 3 | .. figure:: ../../imgs/sample_stats_3.png :align: center Samplewise distribution of samples among the clients for non-IID distribution 4 | .. figure:: ../../imgs/sample_stats_4.png :align: center Samplewise distribution of samples among the clients for non-IID distribution 5 ================================================ FILE: docs/source/tutorials/dataset.rst ================================================ .. _dataset: ********* Datasets ********* The datasets used by **FedERA** are acquired by fetching them from torchvision.datasets. As of now, feder supports the following datasets: * MNIST * FashionMNIST * CIFAR10 * CIFAR100 Adding support for new datasets ------------------------------- There are two methods for incorporating support for new datasets in feder. One involves utilizing torchvision.datasets, while the other entails implementing support for a custom dataset. Adding support for a dataset available in torchvision.datasets ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The torchvision.datasets package consists of popular datasets used in computer vision. The datasets are downloaded and cached automatically. The datasets are subclasses of torch.utils.data.Dataset i.e. they have the same API. This makes it easy to incorporate support for new datasets in feder. All that is required is to add a a few lines of code in the get_data function in the get_data.py file. .. code-block:: python def get_data(config): # If the dataset is not custom, create a dataset folder if config['dataset'] != 'CUSTOM': dataset_path = "client_dataset" if not os.path.exists(dataset_path): os.makedirs(dataset_path) # Get the train and test datasets for each supported dataset if config['dataset'] == 'MNIST': # Apply transformations to the images apply_transform = transforms.Compose([transforms.Resize(config["resize_size"]), transforms.ToTensor()]) # Download and load the trainset trainset = datasets.MNIST(root='client_dataset/MNIST', train=True, download=True, transform=apply_transform) # Download and load the testset testset = datasets.MNIST(root='client_dataset/MNIST', train=False, download=True, transform=apply_transform) elif config['dataset'] == 'FashionMNIST': apply_transform = transforms.Compose([transforms.Resize(config['resize_size']), transforms.ToTensor()]) trainset = datasets.FashionMNIST(root='client_dataset/FashionMNIST', train=True, download=True, transform=apply_transform) testset = datasets.FashionMNIST(root='client_dataset/FashionMNIST', train=False, download=True, transform=apply_transform) elif config['dataset'] == 'CIFAR10': apply_transform = transforms.Compose([transforms.Resize(config['resize_size']), transforms.ToTensor()]) trainset = datasets.CIFAR10(root='client_dataset/CIFAR10', train=True, download=True, transform=apply_transform) testset = datasets.CIFAR10(root='client_dataset/CIFAR10', train=False, download=True, transform=apply_transform) elif config['dataset'] == 'CIFAR100': apply_transform = transforms.Compose([transforms.Resize(config['resize_size']), transforms.ToTensor()]) trainset = datasets.CIFAR100(root='client_dataset/CIFAR100', train=True, download=True, transform=apply_transform) testset = datasets.CIFAR100(root='client_dataset/CIFAR100', train=False, download=True, transform=apply_transform) elif config['dataset'] == 'CUSTOM': apply_transform = transforms.Compose([transforms.Resize(config['resize_size']), transforms.ToTensor()]) # Load the custom dataset trainset = customDataset(root='client_custom_dataset/CUSTOM/train', transform=apply_transform) testset = customDataset(root='client_custom_dataset/CUSTOM/test', transform=apply_transform) else: # Raise an error if an unsupported dataset is specified raise ValueError("Unsupported dataset type: {}".format(config['dataset'])) # Return the train and test datasets return trainset, testset For example, to add support for the STL10 dataset, the following lines of code can be added to the get_data function: .. code-block:: python elif config['dataset'] == 'STL10': apply_transform = transforms.Compose([transforms.Resize(config['resize_size']), transforms.ToTensor()]) trainset = datasets.STL10(root='client_dataset/STL10', split='train', download=True, transform=apply_transform) testset = datasets.STL10(root='client_dataset/STL10', split='test', download=True, transform=apply_transform) Adding support for a custom dataset ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ In order to incorporate support for a custom dataset, the train and test sets for the dataset must be included in the train and test folders, respectively, within the client/client_custom_dataset/CUSTOM/ directory. The train and test data must be stored in .npy files. The custom_dataset class which loads the custom data has been defined in the get_data.py file and can be changed as per the requirements of the custom dataset. The custom_dataset class is a subclass of torch.utils.data.Dataset and has the same API as the datasets in torchvision.datasets. The following code snippet shows the custom_dataset class: .. code-block:: python class customDataset(data.Dataset): def __init__(self, root, transform=None): """ Custom dataset class for loading image and label data from a folder of .npy files. Args: root (str): Path to the folder containing the .npy files. transform (callable, optional): A function/transform that takes in an PIL image and returns a transformed version. E.g, `transforms.RandomCrop` """ self.root = root samples = sample_return(root) self.samples = samples self.transform = transform def __getitem__(self, index): """ Retrieves a sample from the dataset at the given index. Args: index (int): Index of the sample to retrieve. Returns: img (PIL.Image): The image data. label (int): The label for the image data. """ img, label= self.samples[index] img = np.load(img) img = Image.fromarray(img) if self.transform is not None: img = self.transform(img) return img, label def __len__(self): return len(self.samples) ================================================ FILE: docs/source/tutorials/encryption.rst ================================================ .. _encryption: ********** Encryption ********** In the FedERA framework, encryption plays a crucial role in ensuring secure communication between the client and server during the Federated Learning process. This section provides guidance on generating and configuring the necessary certificates for TLS/SSL encryption. TLS Basics ========== To understand the encryption process, it's essential to grasp the fundamentals of TLS/SSL and chains of trust. TLS/SSL operates based on a transitive trust model, where trust in a certificate authority (CA) extends to the certificates it generates. Web browsers and operating systems have a "Trusted Roots" certificate store, automatically trusting certificates from public certificate authorities such as Let's Encrypt or GoDaddy. In the case of FedERA, we establish our own CA and need to inform the client about the CA certificate for trust verification. Additionally, the server certificate must contain the exact server name the client connects to for validation. Generate Certificates ===================== For the purpose of this example, we will set up a basic PKI Infrastructure using CloudFlare's CFSSL toolset, specifically the `cfssl` and `cfssljson` tools. You can download these tools from `here `_ . The `ssl` directory contains configuration files that can be modified, but for demonstration purposes, they can also be used as-is. Generate CA Certificate and Config ---------------------------------- To generate the CA certificate and configuration, navigate to the `ssl` directory and run the following command: .. code-block:: shell-session $ cd FedERA/ssl $ cfssl gencert -initca ca-csr.json | cfssljson -bare ca This command generates the `ca.pem` and `ca-key.pem` files. The `ca.pem` file is used by both the client and server for mutual verification. Generate Server and Client Certificates --------------------------------------- Server Certificate ~~~~~~~~~~~~~~~~~~ To generate the server certificate and key pair, run the following command in the `ssl` directory: .. code-block:: shell-session $ cd FedERA/ssl $ cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -hostname='127.0.0.1,localhost' server-csr.json | cfssljson -bare server This command creates the server certificate and key pair to be used by the server during TLS/SSL encryption. Note that you can modify the `hostname` parameter to match the name or IP address of the server on your network. Client Certificate ~~~~~~~~~~~~~~~~~~ To generate the client certificate and key pair, use the following command in the `ssl` directory: .. code-block:: shell-session $ cd FedERA/ssl $ cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json client-csr.json | cfssljson -bare client When generating the client certificate and key pair, a warning message may appear regarding the absence of a "hosts" field. This warning is expected and acceptable since the client certificate is only used for client identification, not server identification. TLS Server Identification and Encryption ======================================== In FedERA, the client trusts the certificate authority certificate, which subsequently enables trust in the server certificate. This is similar to how web browsers handle certificates, where pre-installed public certificate authority certificates establish trust. For one-way trust verification (client verifies server identity but not vice versa), the server does not necessarily need to present the CA certificate as part of its certificate chain. The server only needs to present enough of the certificate chain for the client to trace it back to a trusted CA certificate. In the FedERA framework, the gRPC server can be configured for SSL using the following code snippet: ---------------------------------------------------------------------------------------------------- On server side ~~~~~~~~~~~~~~ .. code-block:: python if configurations['encryption']==1: # Load the server's private key and certificate keyfile = configurations['server_key'] certfile = configurations['server_cert'] private_key = bytes(open(keyfile).read(), 'utf-8') certificate_chain = bytes(open(certfile).read(), 'utf-8') # Create SSL/TLS credentials object server_credentials = ssl_server_credentials([(private_key, certificate_chain)]) server.add_secure_port('localhost:8214', server_credentials) On client side ~~~~~~~~~~~~~~ .. code-block:: python if config["encryption"] == 1: ca_cert = 'ca.pem' root_certs = bytes(open(ca_cert).read(), 'utf-8') credentials = grpc.ssl_channel_credentials(root_certs) #create new gRPC channel to the server channel = grpc.secure_channel(ip_address, options=[ ('grpc.max_send_message_length', -1), ('grpc.max_receive_message_length', -1) ], credentials=credentials) Acknowledgments =============== This code and information were developed with the help of the repository `jottoekke/python-grpc-ssl `_, which provided valuable guidance in implementing the encryption functionality. ================================================ FILE: docs/source/tutorials/models.rst ================================================ .. _models: ******* Models ******* The models currently implemented in the framework are: * LeNet-5 * ResNet-18 * ResNet-50 * VGG-16 * AlexNet The `server_lib.py` file contains the implementation of Deep-Learning models for the server, while the `net.py` file contains the implementation of these models for the client. These models are either created by inheriting from torch.nn.module or are imported from torchvision.models. Adding support for a new model ------------------------------ There are two ways to incorporate support for new models in **FedERA**. One involves creating a new class that inherits from torch.nn.module and the other involves importing a model from torchvision.models. The first method is more flexible and allows for more customization, while the second method is easier to implement and is recommended for beginners. Adding support for a new model by inheriting from torch.nn.module ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ To add support for a new model by inheriting from torch.nn.module, the following steps need to be followed: 1. Create a new class that inherits from torch.nn.module and defines the model that needs to be implemented, and add it to `server_lib.py` file and the `net.py` file. The code for LeNet is given below as an example: .. code-block:: python class LeNet(nn.Module): def __init__(self, in_channels=1, num_classes=10): super(LeNet, self).__init__() self.conv1 = nn.Conv2d(in_channels, 6, kernel_size=5) self.pool1 = nn.MaxPool2d(kernel_size=2,stride=2) self.conv2 = nn.Conv2d(6, 16, kernel_size=5) self.pool2 = nn.MaxPool2d(kernel_size=2,stride=2) self.fc1 = nn.Linear(400, 120) self.fc2 = nn.Linear(120, 84) self.fc3 = nn.Linear(84, num_classes) self.relu = nn.ReLU() self.logSoftmax = nn.LogSoftmax(dim=1) def forward(self, x): x = self.conv1(x) x = self.relu(x) x = self.pool1(x) x = self.conv2(x) x = self.relu(x) x = self.pool2(x) x = x.view(-1, 400) x = self.fc1(x) x = self.relu(x) x = self.fc2(x) x = self.relu(x) x = self.fc3(x) x = self.logSoftmax(x) return x 2. The models implemented in `client_lib.py` and `net.py` files are imported via the use of `get_net.py` function defined in both the files. This function takes in the name of the model as a string and returns the corresponding model. To add support for a new model, the name of the model needs to be added to the `get_net.py` function in both the files and appropriate changes need to be made. The code for the `get_net.py` function in `client_lib.py` is given below as an example: .. code-block:: python def get_net(config): if config["net"] == 'LeNet': if config['dataset'] in ['MNIST', 'FashionMNIST', 'CUSTOM']: net = LeNet(in_channels=1, num_classes=10) elif config['dataset'] == 'CIFAR10': net = LeNet(in_channels=3, num_classes=10) else: net = LeNet(in_channels=3, num_classes=100) if config["net"] == 'resnet18': if config['dataset'] in ['MNIST', 'FashionMNIST']: net = models.resnet18(num_classes=10) elif config['dataset'] == 'CIFAR10': net = models.resnet18(num_classes=10) else: net = models.resnet18(num_classes=100) if config["net"] == 'resnet50': if config['dataset'] in ['MNIST', 'FashionMNIST']: net = models.resnet50(num_classes=10) elif config['dataset'] == 'CIFAR10': net = models.resnet50(num_classes=10) else: net = models.resnet50(num_classes=100) if config["net"] == 'vgg16': if config['dataset'] in ['MNIST', 'FashionMNIST']: net = models.vgg16(num_classes=10) elif config['dataset'] == 'CIFAR10': net = models.vgg16(num_classes=10) else: net = models.vgg16(num_classes=100) if config['net'] == 'AlexNet': if config['dataset'] in ['MNIST', 'FashionMNIST']: net = models.alexnet(num_classes=10) elif config['dataset'] == 'CIFAR10': net = models.alexnet(num_classes=10) else: net = models.alexnet(num_classes=100) return net Adding support for a new model by importing from torchvision.models ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ To add support for a new model by importing from torchvision.models, import the model from torchvision.models in `server_lib.py` and `net.py` files and make changes in the `get_net` function appropriately. The code that needs to be added in `get_net` function to import ResNet38 model is given below as an example: .. code-block:: python if config["net"] == 'resnet38': if config['dataset'] in ['MNIST', 'FashionMNIST']: net = models.resnet38(num_classes=10) elif config['dataset'] == 'CIFAR10': net = models.resnet38(num_classes=10) else: net = models.resnet38(num_classes=100) ================================================ FILE: docs/source/tutorials/running.rst ================================================ .. _running: ******************************* Running the Server and Clients ******************************* Starting the Server ------------------- The server is started by running the following command in the root directory of the framework: .. code-block:: console python -m federa.server.start_server Arguments that can be passed to the server are: ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. list-table:: Server Configuration Options :widths: 25 45 20 :header-rows: 1 * - Argument - Description - Default * - ``algorithm`` - specifies the aggregation algorithm - ``fedavg`` * - ``clients`` - specifies number of clients selected per round - ``1`` * - ``fraction`` - specifies fraction of clients selected - ``1`` * - ``rounds`` - specifies total number of rounds - ``1`` * - ``model_path`` - specifies initial server model path - ``initial_model.pt`` * - ``epochs`` - specifies client epochs per round - ``1`` * - ``accept_conn`` - determines if connections accepted after FL begins - ``1`` * - ``verify`` - specifies if verification module runs before rounds - ``0`` * - ``threshold`` - specifies minimum verification score - ``0`` * - ``timeout`` - specifies client training time limit per round - ``None`` * - ``resize_size`` - specifies dataset resize dimension - ``32`` * - ``batch_size`` - specifies dataset batch size - ``32`` * - ``net`` - specifies network architecture - ``LeNet`` * - ``dataset`` - specifies dataset name - ``MNIST`` * - ``niid`` - specifies data distribution among clients - ``1`` * - ``carbon`` - specifies if carbon emissions tracked at client side - ``0`` * - ``encryption`` - specifies whether to use SSL encryption or not - ``0`` * - ``server_key`` - specifies path to server key certificate - ``server-key.pem`` * - ``server_cert`` - specifies path to server certificate - ``server.pem`` Starting the Clients -------------------- The clients are started by running the following command in the root directory of the framework: .. code-block:: console python federa.client.start_client Arguments that can be passed to the clients are: ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. list-table:: Client Configuration Options :widths: 25 45 20 :header-rows: 1 * - Argument - Description - Default * - ``server_ip`` - specifies server IP address - ``localhost:8214`` * - ``device`` - specifies device - ``cpu`` * - ``encryption`` - specifies whether to use SSL encryption or not - ``0`` * - ``ca`` - specifies path to CA certificate - ``ca.pem`` * - ``wait_time`` - specifies time to wait before reconnecting to the server - ``30`` ================================================ FILE: docs/source/tutorials/tutorial.rst ================================================ .. _tutorial: ********* Tutorials ********* **FedERA** allows you to do federated learning in real time on various supported edge devices, including Intel CPUs, Nvidia GPUs, Nvidia Jetson, Raspberry Pi, Intel NUC. **FedERA** provides modular tools and standard algorithms to simplify federated learning implementation in real time using gRPC framework. .. card:: Running Server and Client :link: running :link-type: ref :class-card: sd-rounded-2 sd-border-1 Step-by-step guide on running server and clients on same and different devices. .. card:: How to Customize Federated Learning Algorithm? :link: algorithm :link-type: ref :class-card: sd-rounded-2 sd-border-1 Step-by-step guide on how to customize federated learning algorithm. .. card:: How to add Custom Dataset? :link: dataset :link-type: ref :class-card: sd-rounded-2 sd-border-1 Step-by-step guide on how to add a custom dataset. .. card:: How to add Custom Model? :link: models :link-type: ref :class-card: sd-rounded-2 sd-border-1 Step-by-step guide on how to add a custom model. .. card:: How to add use different Data Distribution? :link: data_distribution :link-type: ref :class-card: sd-rounded-2 sd-border-1 Step-by-step guide on how to use different data dstributions while training. .. card:: How to get Carbon Footprint? :link: code_carbon :link-type: ref :class-card: sd-rounded-2 sd-border-1 Step-by-step guide on how to get carbon footprint of the training process. .. card:: How to use Encryption? :link: encryption :link-type: ref :class-card: sd-rounded-2 sd-border-1 Step-by-step guide on how to use encryption in the training process. .. toctree:: :maxdepth: 2 running algorithm dataset models data_distribution code_carbon encryption .. Use :class:`NetworkManager` to customize communication strategies, including synchronous and asynchronous communication. ================================================ FILE: federa/__init__.py ================================================ __version__='0.0.0' ================================================ FILE: federa/client/__init__.py ================================================ ================================================ FILE: federa/client/src/ClientConnection_pb2.py ================================================ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: ClientConnection.proto """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() DESCRIPTOR = _descriptor.FileDescriptor( name='ClientConnection.proto', package='', syntax='proto3', serialized_options=None, #create_key=_descriptor._internal_create_key, serialized_pb=b'\n\x16\x43lientConnection.proto\"\xa3\x01\n\rServerMessage\x12\x1f\n\ntrainOrder\x18\x01 \x01(\x0b\x32\x0b.TrainOrder\x12\x1d\n\tevalOrder\x18\x02 \x01(\x0b\x32\n.EvalOrder\x12)\n\x0f\x64isconnectOrder\x18\x03 \x01(\x0b\x32\x10.DisconnectOrder\x12\'\n\x0esetParamsOrder\x18\x04 \x01(\x0b\x32\x0f.SetParamsOrder\"\x8a\x01\n\rClientMessage\x12%\n\rtrainResponse\x18\x01 \x01(\x0b\x32\x0e.TrainResponse\x12#\n\x0c\x65valResponse\x18\x02 \x01(\x0b\x32\r.EvalResponse\x12-\n\x11setParamsResponse\x18\x03 \x01(\x0b\x32\x12.SetParamsResponse\"9\n\nTrainOrder\x12\x17\n\x0fmodelParameters\x18\x01 \x01(\x0c\x12\x12\n\nconfigDict\x18\x02 \x01(\x0c\">\n\rTrainResponse\x12\x17\n\x0fmodelParameters\x18\x01 \x01(\x0c\x12\x14\n\x0cresponseDict\x18\x02 \x01(\x0c\"8\n\tEvalOrder\x12\x17\n\x0fmodelParameters\x18\x01 \x01(\x0c\x12\x12\n\nconfigDict\x18\x02 \x01(\x0c\"$\n\x0c\x45valResponse\x12\x14\n\x0cresponseDict\x18\x01 \x01(\x0c\")\n\x0eSetParamsOrder\x12\x17\n\x0fmodelParameters\x18\x01 \x01(\x0c\"\x13\n\x11SetParamsResponse\"9\n\x0f\x44isconnectOrder\x12\x0f\n\x07message\x18\x01 \x01(\t\x12\x15\n\rreconnectTime\x18\x02 \x01(\x05\x32\x41\n\x10\x43lientConnection\x12-\n\x07\x43onnect\x12\x0e.ClientMessage\x1a\x0e.ServerMessage(\x01\x30\x01\x62\x06proto3' ) _SERVERMESSAGE = _descriptor.Descriptor( name='ServerMessage', full_name='ServerMessage', filename=None, file=DESCRIPTOR, containing_type=None, #create_key=_descriptor._internal_create_key, fields=[ _descriptor.FieldDescriptor( name='trainOrder', full_name='ServerMessage.trainOrder', index=0, number=1, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='evalOrder', full_name='ServerMessage.evalOrder', index=1, number=2, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='disconnectOrder', full_name='ServerMessage.disconnectOrder', index=2, number=3, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='setParamsOrder', full_name='ServerMessage.setParamsOrder', index=3, number=4, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], oneofs=[ ], serialized_start=27, serialized_end=190, ) _CLIENTMESSAGE = _descriptor.Descriptor( name='ClientMessage', full_name='ClientMessage', filename=None, file=DESCRIPTOR, containing_type=None, #create_key=_descriptor._internal_create_key, fields=[ _descriptor.FieldDescriptor( name='trainResponse', full_name='ClientMessage.trainResponse', index=0, number=1, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='evalResponse', full_name='ClientMessage.evalResponse', index=1, number=2, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='setParamsResponse', full_name='ClientMessage.setParamsResponse', index=2, number=3, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], oneofs=[ ], serialized_start=193, serialized_end=331, ) _TRAINORDER = _descriptor.Descriptor( name='TrainOrder', full_name='TrainOrder', filename=None, file=DESCRIPTOR, containing_type=None, #create_key=_descriptor._internal_create_key, fields=[ _descriptor.FieldDescriptor( name='modelParameters', full_name='TrainOrder.modelParameters', index=0, number=1, type=12, cpp_type=9, label=1, has_default_value=False, default_value=b"", message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='configDict', full_name='TrainOrder.configDict', index=1, number=2, type=12, cpp_type=9, label=1, has_default_value=False, default_value=b"", message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], oneofs=[ ], serialized_start=333, serialized_end=390, ) _TRAINRESPONSE = _descriptor.Descriptor( name='TrainResponse', full_name='TrainResponse', filename=None, file=DESCRIPTOR, containing_type=None, #create_key=_descriptor._internal_create_key, fields=[ _descriptor.FieldDescriptor( name='modelParameters', full_name='TrainResponse.modelParameters', index=0, number=1, type=12, cpp_type=9, label=1, has_default_value=False, default_value=b"", message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='responseDict', full_name='TrainResponse.responseDict', index=1, number=2, type=12, cpp_type=9, label=1, has_default_value=False, default_value=b"", message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], oneofs=[ ], serialized_start=392, serialized_end=454, ) _EVALORDER = _descriptor.Descriptor( name='EvalOrder', full_name='EvalOrder', filename=None, file=DESCRIPTOR, containing_type=None, #create_key=_descriptor._internal_create_key, fields=[ _descriptor.FieldDescriptor( name='modelParameters', full_name='EvalOrder.modelParameters', index=0, number=1, type=12, cpp_type=9, label=1, has_default_value=False, default_value=b"", message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='configDict', full_name='EvalOrder.configDict', index=1, number=2, type=12, cpp_type=9, label=1, has_default_value=False, default_value=b"", message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], oneofs=[ ], serialized_start=456, serialized_end=512, ) _EVALRESPONSE = _descriptor.Descriptor( name='EvalResponse', full_name='EvalResponse', filename=None, file=DESCRIPTOR, containing_type=None, #create_key=_descriptor._internal_create_key, fields=[ _descriptor.FieldDescriptor( name='responseDict', full_name='EvalResponse.responseDict', index=0, number=1, type=12, cpp_type=9, label=1, has_default_value=False, default_value=b"", message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], oneofs=[ ], serialized_start=514, serialized_end=550, ) _SETPARAMSORDER = _descriptor.Descriptor( name='SetParamsOrder', full_name='SetParamsOrder', filename=None, file=DESCRIPTOR, containing_type=None, #create_key=_descriptor._internal_create_key, fields=[ _descriptor.FieldDescriptor( name='modelParameters', full_name='SetParamsOrder.modelParameters', index=0, number=1, type=12, cpp_type=9, label=1, has_default_value=False, default_value=b"", message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], oneofs=[ ], serialized_start=552, serialized_end=593, ) _SETPARAMSRESPONSE = _descriptor.Descriptor( name='SetParamsResponse', full_name='SetParamsResponse', filename=None, file=DESCRIPTOR, containing_type=None, #create_key=_descriptor._internal_create_key, fields=[ ], extensions=[ ], nested_types=[], enum_types=[ ], serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], oneofs=[ ], serialized_start=595, serialized_end=614, ) _DISCONNECTORDER = _descriptor.Descriptor( name='DisconnectOrder', full_name='DisconnectOrder', filename=None, file=DESCRIPTOR, containing_type=None, #create_key=_descriptor._internal_create_key, fields=[ _descriptor.FieldDescriptor( name='message', full_name='DisconnectOrder.message', index=0, number=1, type=9, cpp_type=9, label=1, has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='reconnectTime', full_name='DisconnectOrder.reconnectTime', index=1, number=2, type=5, cpp_type=1, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], oneofs=[ ], serialized_start=616, serialized_end=673, ) _SERVERMESSAGE.fields_by_name['trainOrder'].message_type = _TRAINORDER _SERVERMESSAGE.fields_by_name['evalOrder'].message_type = _EVALORDER _SERVERMESSAGE.fields_by_name['disconnectOrder'].message_type = _DISCONNECTORDER _SERVERMESSAGE.fields_by_name['setParamsOrder'].message_type = _SETPARAMSORDER _CLIENTMESSAGE.fields_by_name['trainResponse'].message_type = _TRAINRESPONSE _CLIENTMESSAGE.fields_by_name['evalResponse'].message_type = _EVALRESPONSE _CLIENTMESSAGE.fields_by_name['setParamsResponse'].message_type = _SETPARAMSRESPONSE DESCRIPTOR.message_types_by_name['ServerMessage'] = _SERVERMESSAGE DESCRIPTOR.message_types_by_name['ClientMessage'] = _CLIENTMESSAGE DESCRIPTOR.message_types_by_name['TrainOrder'] = _TRAINORDER DESCRIPTOR.message_types_by_name['TrainResponse'] = _TRAINRESPONSE DESCRIPTOR.message_types_by_name['EvalOrder'] = _EVALORDER DESCRIPTOR.message_types_by_name['EvalResponse'] = _EVALRESPONSE DESCRIPTOR.message_types_by_name['SetParamsOrder'] = _SETPARAMSORDER DESCRIPTOR.message_types_by_name['SetParamsResponse'] = _SETPARAMSRESPONSE DESCRIPTOR.message_types_by_name['DisconnectOrder'] = _DISCONNECTORDER _sym_db.RegisterFileDescriptor(DESCRIPTOR) ServerMessage = _reflection.GeneratedProtocolMessageType('ServerMessage', (_message.Message,), { 'DESCRIPTOR' : _SERVERMESSAGE, '__module__' : 'ClientConnection_pb2' # @@protoc_insertion_point(class_scope:ServerMessage) }) _sym_db.RegisterMessage(ServerMessage) ClientMessage = _reflection.GeneratedProtocolMessageType('ClientMessage', (_message.Message,), { 'DESCRIPTOR' : _CLIENTMESSAGE, '__module__' : 'ClientConnection_pb2' # @@protoc_insertion_point(class_scope:ClientMessage) }) _sym_db.RegisterMessage(ClientMessage) TrainOrder = _reflection.GeneratedProtocolMessageType('TrainOrder', (_message.Message,), { 'DESCRIPTOR' : _TRAINORDER, '__module__' : 'ClientConnection_pb2' # @@protoc_insertion_point(class_scope:TrainOrder) }) _sym_db.RegisterMessage(TrainOrder) TrainResponse = _reflection.GeneratedProtocolMessageType('TrainResponse', (_message.Message,), { 'DESCRIPTOR' : _TRAINRESPONSE, '__module__' : 'ClientConnection_pb2' # @@protoc_insertion_point(class_scope:TrainResponse) }) _sym_db.RegisterMessage(TrainResponse) EvalOrder = _reflection.GeneratedProtocolMessageType('EvalOrder', (_message.Message,), { 'DESCRIPTOR' : _EVALORDER, '__module__' : 'ClientConnection_pb2' # @@protoc_insertion_point(class_scope:EvalOrder) }) _sym_db.RegisterMessage(EvalOrder) EvalResponse = _reflection.GeneratedProtocolMessageType('EvalResponse', (_message.Message,), { 'DESCRIPTOR' : _EVALRESPONSE, '__module__' : 'ClientConnection_pb2' # @@protoc_insertion_point(class_scope:EvalResponse) }) _sym_db.RegisterMessage(EvalResponse) SetParamsOrder = _reflection.GeneratedProtocolMessageType('SetParamsOrder', (_message.Message,), { 'DESCRIPTOR' : _SETPARAMSORDER, '__module__' : 'ClientConnection_pb2' # @@protoc_insertion_point(class_scope:SetParamsOrder) }) _sym_db.RegisterMessage(SetParamsOrder) SetParamsResponse = _reflection.GeneratedProtocolMessageType('SetParamsResponse', ( _message.Message,),{ 'DESCRIPTOR' : _SETPARAMSRESPONSE, '__module__' : 'ClientConnection_pb2' # @@protoc_insertion_point(class_scope:SetParamsResponse) }) _sym_db.RegisterMessage(SetParamsResponse) DisconnectOrder = _reflection.GeneratedProtocolMessageType('DisconnectOrder', (_message.Message,), { 'DESCRIPTOR' : _DISCONNECTORDER, '__module__' : 'ClientConnection_pb2' # @@protoc_insertion_point(class_scope:DisconnectOrder) }) _sym_db.RegisterMessage(DisconnectOrder) _CLIENTCONNECTION = _descriptor.ServiceDescriptor( name='ClientConnection', full_name='ClientConnection', file=DESCRIPTOR, index=0, serialized_options=None, #create_key=_descriptor._internal_create_key, serialized_start=675, serialized_end=740, methods=[ _descriptor.MethodDescriptor( name='Connect', full_name='ClientConnection.Connect', index=0, containing_service=None, input_type=_CLIENTMESSAGE, output_type=_SERVERMESSAGE, serialized_options=None, #create_key=_descriptor._internal_create_key, ), ]) _sym_db.RegisterServiceDescriptor(_CLIENTCONNECTION) DESCRIPTOR.services_by_name['ClientConnection'] = _CLIENTCONNECTION # @@protoc_insertion_point(module_scope) ================================================ FILE: federa/client/src/ClientConnection_pb2_grpc.py ================================================ # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! """Client and server classes corresponding to protobuf-defined services.""" import grpc from . import ClientConnection_pb2 as ClientConnection__pb2 class ClientConnectionStub(): """Missing associated documentation comment in .proto file.""" def __init__(self, channel): """Constructor. Args: channel: A grpc.Channel. """ self.Connect = channel.stream_stream( '/ClientConnection/Connect', request_serializer=ClientConnection__pb2.ClientMessage.SerializeToString, response_deserializer=ClientConnection__pb2.ServerMessage.FromString, ) class ClientConnectionServicer(): """Missing associated documentation comment in .proto file.""" def Connect(self, request_iterator, context): """Missing associated documentation comment in .proto file.""" context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def add_ClientConnectionServicer_to_server(servicer, server): rpc_method_handlers = { 'Connect': grpc.stream_stream_rpc_method_handler( servicer.Connect, request_deserializer=ClientConnection__pb2.ClientMessage.FromString, response_serializer=ClientConnection__pb2.ServerMessage.SerializeToString, ), } generic_handler = grpc.method_handlers_generic_handler( 'ClientConnection', rpc_method_handlers) server.add_generic_rpc_handlers((generic_handler,)) # This class is part of an EXPERIMENTAL API. class ClientConnection(): """Missing associated documentation comment in .proto file.""" @staticmethod def Connect(request_iterator, target, options=(), channel_credentials=None, call_credentials=None, insecure=False, compression=None, wait_for_ready=None, timeout=None, metadata=None): return grpc.experimental.stream_stream(request_iterator, target, '/ClientConnection/Connect', ClientConnection__pb2.ClientMessage.SerializeToString, ClientConnection__pb2.ServerMessage.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata) ================================================ FILE: federa/client/src/__init__.py ================================================ ================================================ FILE: federa/client/src/client.py ================================================ from queue import Queue import torch import time import grpc from . import ClientConnection_pb2_grpc from .ClientConnection_pb2 import ClientMessage from .client_lib import train, evaluate, set_parameters def client_start(config): keep_going = True wait_time = config["wait_time"] ip_address = config["ip_address"] device = torch.device(config["device"]) #device = torch.device("cuda:2" if torch.cuda.is_available() else "cpu") while keep_going: #wait for specified time before reconnecting time.sleep(wait_time) if config["encryption"] == 1: ca_cert = config['ca'] root_certs = bytes(open(ca_cert).read(), 'utf-8') credentials = grpc.ssl_channel_credentials(root_certs) #create new gRPC channel to the server channel = grpc.secure_channel(ip_address, options=[ ('grpc.max_send_message_length', -1), ('grpc.max_receive_message_length', -1) ], credentials=credentials) else: channel = grpc.insecure_channel(ip_address, options=[ ('grpc.max_send_message_length', -1), ('grpc.max_receive_message_length', -1) ]) stub = ClientConnection_pb2_grpc.ClientConnectionStub(channel) client_buffer = Queue(maxsize = 10) print("Connected with server") #wait for incoming messages from the server in client_buffer #then according to fields present in them call the appropraite function for server_message in stub.Connect( iter(client_buffer.get, None) ): if server_message.HasField("evalOrder"): eval_order_message = server_message.evalOrder eval_response_message = evaluate(eval_order_message, device) message_to_server = ClientMessage(evalResponse = eval_response_message) client_buffer.put(message_to_server) if server_message.HasField("trainOrder"): train_order_message = server_message.trainOrder train_response_message = train(train_order_message, device) message_to_server = ClientMessage(trainResponse = train_response_message) client_buffer.put(message_to_server) if server_message.HasField("setParamsOrder"): set_parameters_order_message = server_message.setParamsOrder set_parameters(set_parameters_order_message, device) message_to_server = ClientMessage(setParamsResponse = None) client_buffer.put(message_to_server) if server_message.HasField("disconnectOrder"): print("Current FL process is done ") disconnect_order_message = server_message.disconnectOrder message = disconnect_order_message.message print(message) reconnect_time = disconnect_order_message.reconnectTime if reconnect_time == 0: keep_going = False break wait_time = reconnect_time ================================================ FILE: federa/client/src/client_lib.py ================================================ import torch from io import BytesIO import json import time import os from datetime import datetime from codecarbon import OfflineEmissionsTracker from .net import get_net from .net_lib import test_model, load_data from .net_lib import train_model, train_fedavg, train_scaffold, train_mimelite, train_mime, train_feddyn from torch.utils.data import DataLoader from .get_data import get_data import matplotlib.pyplot as plt import pandas as pd import numpy as np from .ClientConnection_pb2 import EvalResponse, TrainResponse #create a new directory inside FL_checkpoints and store the aggragted models in each round fl_timestamp = f"{datetime.now().strftime('%Y-%m-%d %H-%M-%S')}" save_dir_path = f"client_checkpoints/{fl_timestamp}" os.makedirs(save_dir_path) prev_grads = None def evaluate(eval_order_message, device): model_parameters_bytes = eval_order_message.modelParameters model_parameters = torch.load( BytesIO(model_parameters_bytes), map_location="cpu" ) config_dict_bytes = eval_order_message.configDict config_dict = json.loads( config_dict_bytes.decode("utf-8") ) client_id = config_dict["client_id"] state_dict = model_parameters print("Evaluation:",config_dict) with open("config.json", "r", encoding='utf-8') as jsonfile: config_dict = json.load(jsonfile) model = get_net(config= config_dict).to(device) model.load_state_dict(state_dict) _, testset = get_data(config= config_dict) testloader = DataLoader(testset, batch_size=config_dict['batch_size']) #_, testloader, _ = load_data(config_dict) eval_loss, eval_accuracy = test_model(model, testloader, device) response_dict = {"eval_loss": eval_loss, "eval_accuracy": eval_accuracy, "client_id": client_id} response_dict_bytes = json.dumps(response_dict).encode("utf-8") eval_response_message = EvalResponse(responseDict = response_dict_bytes) return eval_response_message def train(train_order_message, device): data_bytes = train_order_message.modelParameters data = torch.load( BytesIO(data_bytes), map_location="cpu" ) model_parameters, control_variate, = data['model_parameters'], data['control_variate'] control_variate2 = data['control_variate2'] config_dict_bytes = train_order_message.configDict config_dict = json.loads( config_dict_bytes.decode("utf-8") ) carbon_tracker = config_dict["carbon-tracker"] model = get_net(config= config_dict) model.load_state_dict(model_parameters) model = model.to(device) epochs = config_dict["epochs"] if config_dict["timeout"]: deadline = time.time() + config_dict["timeout"] else: deadline = None #Run code carbon if the carbon-tracker flag is True if carbon_tracker==1: tracker = OfflineEmissionsTracker(country_iso_code="IND", output_dir = save_dir_path) tracker.start() trainloader, testloader, _ = load_data(config_dict) print("Training started") if config_dict['algorithm'] == 'mimelite': model, control_variate = train_mimelite(model, control_variate, trainloader, epochs, device, deadline) elif config_dict['algorithm'] == 'scaffold': model, control_variate = train_scaffold(model, control_variate, trainloader, epochs, device, deadline) elif config_dict['algorithm'] == 'mime': model, control_variate = train_mime(model, control_variate, control_variate2, trainloader, epochs, device, deadline) elif config_dict['algorithm'] == 'fedavg': model = train_fedavg(model, trainloader, epochs, device, deadline) elif config_dict['algorithm'] == 'feddyn': global prev_grads model, prev_grads = train_feddyn(model, trainloader, epochs, device, deadline, prev_grads) else: model = train_model(model, trainloader, epochs, device, deadline) if carbon_tracker==1: emissions: float = tracker.stop() print(f"Emissions: {emissions} kg") myJSON = json.dumps(config_dict) json_path = save_dir_path + "/config.json" with open(json_path, "w", encoding='utf-8') as jsonfile: jsonfile.write(myJSON) json_path = "config.json" with open(json_path, "w", encoding='utf-8') as jsonfile: jsonfile.write(myJSON) trained_model_parameters = model.state_dict() #Create a dictionary where model_parameters and control_variate are stored which needs to be sent to the server data_to_send = {} data_to_send['model_parameters'] = trained_model_parameters data_to_send['control_variate'] = control_variate #If there is no control_variate, this will become None buffer = BytesIO() torch.save(data_to_send, buffer) buffer.seek(0) data_to_send_bytes = buffer.read() print("Evaluation") if config_dict['algorithm'] not in ('fedavg','feddyn','mime','mimelite'): for key in trained_model_parameters: trained_model_parameters[key] += model_parameters[key].to(device) train_loss, train_accuracy = test_model(model, testloader, device) response_dict = {"train_loss": train_loss, "train_accuracy": train_accuracy} response_dict_bytes = json.dumps(response_dict).encode("utf-8") train_response_message = TrainResponse( modelParameters = data_to_send_bytes, responseDict = response_dict_bytes) save_model_state(model) if carbon_tracker==1: plot_emission() return train_response_message #replace current model with the model provided def set_parameters(set_parameters_order_message, device): model_parameters_bytes = set_parameters_order_message.modelParameters model_parameters = torch.load( BytesIO(model_parameters_bytes), map_location="cpu" ) with open("config.json", "r", encoding='utf-8') as jsonfile: config_dict = json.load(jsonfile) model = get_net(config= config_dict).to(device) model.load_state_dict(model_parameters) save_model_state(model) #save the current model to model_checkpoints def save_model_state(model): file_num = len(os.listdir(f"{save_dir_path}")) filepath = f"{save_dir_path}/model_{file_num}.pt" state_dict = model.state_dict() torch.save(state_dict, filepath) #save plot for communication round-wise carbon emmision def plot_emission(): data = pd.read_csv(f"{save_dir_path}/emissions.csv") plt.plot(np.arange(len(data.index)),data['emissions']*1000) plt.xlabel('Communication Rounds') plt.ylabel('Carbon Emmision (gm)') plt.savefig(f"{save_dir_path}/emissions.png") ================================================ FILE: federa/client/src/data_utils.py ================================================ import torch #from PIL import Image from torch.utils import data #from torchvision import transforms #import pickle class distributionDataloader(data.Dataset): def __init__( self, config, trainset, data_idxs, clientID = 0, aug = False, ): self.aug = aug self.config = config self.img_size = config["resize_size"] self.trainset = trainset self.niid_degree = config["niid"] self.clientID = clientID self.mean = 33.3184 self.stdv = 78.5675 # self.data_idxs = torch.load(data_path)['datapoints'][clientID] self.data_idxs = data_idxs[clientID] def __len__(self): return len(self.data_idxs) def __getitem__(self, index): image = self.trainset[self.data_idxs[index]][0] label = self.trainset[self.data_idxs[index]][1] return image, label ================================================ FILE: federa/client/src/get_data.py ================================================ import os from torchvision import transforms,datasets from torch.utils import data import numpy as np from PIL import Image # Define a function to get the train and test datasets based on the given configuration def get_data(config): # If the dataset is not custom, create a dataset folder if config['dataset'] != 'CUSTOM': dataset_path = "client_dataset" if not os.path.exists(dataset_path): os.makedirs(dataset_path) # Get the train and test datasets for each supported dataset if config['dataset'] == 'MNIST': # Apply transformations to the images apply_transform = transforms.Compose([transforms.Resize(config["resize_size"]), transforms.ToTensor()]) # Download and load the trainset trainset = datasets.MNIST(root='client_dataset/MNIST', train=True, download=True, transform=apply_transform) # Download and load the testset testset = datasets.MNIST(root='client_dataset/MNIST', train=False, download=True, transform=apply_transform) elif config['dataset'] == 'FashionMNIST': apply_transform = transforms.Compose([transforms.Resize(config['resize_size']), transforms.ToTensor()]) trainset = datasets.FashionMNIST(root='client_dataset/FashionMNIST', train=True, download=True, transform=apply_transform) testset = datasets.FashionMNIST(root='client_dataset/FashionMNIST', train=False, download=True, transform=apply_transform) elif config['dataset'] == 'CIFAR10': apply_transform = transforms.Compose([transforms.Resize(config['resize_size']), transforms.ToTensor()]) trainset = datasets.CIFAR10(root='client_dataset/CIFAR10', train=True, download=True, transform=apply_transform) testset = datasets.CIFAR10(root='client_dataset/CIFAR10', train=False, download=True, transform=apply_transform) elif config['dataset'] == 'CIFAR100': apply_transform = transforms.Compose([transforms.Resize(config['resize_size']), transforms.ToTensor()]) trainset = datasets.CIFAR100(root='client_dataset/CIFAR100', train=True, download=True, transform=apply_transform) testset = datasets.CIFAR100(root='client_dataset/CIFAR100', train=False, download=True, transform=apply_transform) elif config['dataset'] == 'CUSTOM': apply_transform = transforms.Compose([transforms.Resize(config['resize_size']), transforms.ToTensor()]) # Load the custom dataset trainset = customDataset(root='client_custom_dataset/CUSTOM/train', transform=apply_transform) testset = customDataset(root='client_custom_dataset/CUSTOM/test', transform=apply_transform) else: # Raise an error if an unsupported dataset is specified raise ValueError(f"Unsupported dataset type: {config['dataset']}") # Return the train and test datasets return trainset, testset class customDataset(data.Dataset): def __init__(self, root, transform=None): """ Custom dataset class for loading image and label data from a folder of .npy files. Args: root (str): Path to the folder containing the .npy files. transform (callable, optional): A function/transform that takes an PIL image and returns a transformed version. E.g, `transforms.RandomCrop` """ self.root = root samples = sample_return(root) self.samples = samples self.transform = transform def __getitem__(self, index): """ Retrieves a sample from the dataset at the given index. Args: index (int): Index of the sample to retrieve. Returns: img (PIL.Image): The image data. label (int): The label for the image data. """ img, label= self.samples[index] img = np.load(img) img = Image.fromarray(img) if self.transform is not None: img = self.transform(img) return img, label def __len__(self): return len(self.samples) def sample_return(root): # Initialize an empty list to hold the samples newdataset = [] # Define a dictionary that maps label names to integer values labels = {'Breast': 0, 'Chestxray':1, 'Oct': 2, 'Tissue': 3} # Loop over each image in the root directory for image in os.listdir(root): # Initialize an empty list to hold the label label=[] # Get the full path of the image path = os.path.join(root, image) # Extract the label from the image filename labels_str = image.split('_')[0] label = labels[labels_str] # Create a tuple containing the image path and its label, and append it to the list of samples item = (path, label) newdataset.append(item) # Return the list of samples return newdataset ================================================ FILE: federa/client/src/net.py ================================================ from torch import nn from torchvision import models class LeNet(nn.Module): def __init__(self, in_channels=1, num_classes=10): super().__init__() self.conv1 = nn.Conv2d(in_channels, 6, kernel_size=5) self.pool1 = nn.MaxPool2d(kernel_size=2,stride=2) self.conv2 = nn.Conv2d(6, 16, kernel_size=5) self.pool2 = nn.MaxPool2d(kernel_size=2,stride=2) self.fc1 = nn.Linear(400, 120) self.fc2 = nn.Linear(120, 84) self.fc3 = nn.Linear(84, num_classes) self.relu = nn.ReLU() self.logSoftmax = nn.LogSoftmax(dim=1) def forward(self, x): x = self.conv1(x) x = self.relu(x) x = self.pool1(x) x = self.conv2(x) x = self.relu(x) x = self.pool2(x) x = x.view(-1, 400) x = self.fc1(x) x = self.relu(x) x = self.fc2(x) x = self.relu(x) x = self.fc3(x) x = self.logSoftmax(x) return x def get_net(config): if config["net"] == 'LeNet': if config['dataset'] in ['MNIST', 'FashionMNIST', 'CUSTOM']: net = LeNet(in_channels=1, num_classes=10) elif config['dataset'] == 'CIFAR10': net = LeNet(in_channels=3, num_classes=10) else: net = LeNet(in_channels=3, num_classes=100) if config["net"] == 'resnet18': if config['dataset'] == 'CIFAR10': net = models.resnet18(num_classes=10) else: net = models.resnet18(num_classes=100) if config["net"] == 'resnet50': if config['dataset'] == 'CIFAR10': net = models.resnet50(num_classes=10) else: net = models.resnet50(num_classes=100) if config["net"] == 'vgg16': if config['dataset'] == 'CIFAR10': net = models.vgg16(num_classes=10) else: net = models.vgg16(num_classes=100) if config['net'] == 'AlexNet': if config['dataset'] == 'CIFAR10': net = models.alexnet(num_classes=10) else: net = models.alexnet(num_classes=100) return net ================================================ FILE: federa/client/src/net_lib.py ================================================ import os import time from copy import deepcopy from math import ceil from tqdm import tqdm import torch from torch.utils.data import DataLoader from .data_utils import distributionDataloader from .get_data import get_data # DEVICE = torch.device("cuda:2" if torch.cuda.is_available() else "cpu") # #device id of this should be same in client_lib device def load_data(config): trainset, testset = get_data(config) # Data distribution for non-custom datasets if config['dataset'] != 'CUSTOM': datasets = distributionDataloader(config, trainset, config['datapoints'], config['client_idx']) trainloader = DataLoader(datasets, batch_size= config['batch_size'], shuffle=True) testloader = DataLoader(testset, batch_size=config['batch_size']) num_examples = {"trainset": len(datasets), "testset": len(testset)} else: trainloader = DataLoader(trainset, batch_size= config['batch_size'], shuffle=True) testloader = DataLoader(testset, batch_size=config['batch_size']) num_examples = {"trainset": len(trainset), "testset": len(testset)} # Return data loaders and number of examples in train and test datasets return trainloader, testloader, num_examples def flush_memory(): torch.cuda.empty_cache() def train_model(net, trainloader, epochs, device, deadline=None): """ Trains a neural network model on a given dataset using SGD optimizer with Cross Entropy Loss criterion. Args: net: neural network model trainloader: PyTorch DataLoader object for training dataset epochs: number of epochs to train the model deadline: optional deadline time for training Returns: trained model with the difference between trained model and the received model """ x = deepcopy(net) # Define the loss function and optimizer criterion = torch.nn.CrossEntropyLoss() optimizer = torch.optim.SGD(net.parameters(), lr=0.001, momentum=0.9) # Set the model to training mode net.train() # Train the model for the specified number of epochs for _ in tqdm(range(epochs)): for images, labels in trainloader: images, labels = images.to(device), labels.to(device) optimizer.zero_grad() loss = criterion(net(images), labels) loss.backward() optimizer.step() # Check if the deadline time has been reached if deadline: current_time = time.time() if current_time >= deadline: print("deadline occurred.") break # Calculate the difference between the trained model and the received model for param_net, param_x in zip(net.parameters(), x.parameters()): param_net.data = param_net.data - param_x.data return net def train_fedavg(net, trainloader, epochs, device, deadline=None): """ Trains a given neural network using the Federated Averaging (FedAvg) algorithm. Args: net: A PyTorch neural network model trainloader: A PyTorch DataLoader containing the training dataset epochs: An integer specifying the number of training epochs deadline: An optional deadline (in seconds) for the training process Returns: A trained PyTorch neural network model """ # Define loss function and optimizer criterion = torch.nn.CrossEntropyLoss() optimizer = torch.optim.SGD(net.parameters(), lr=0.001, momentum=0.9) # Set model to train mode net.train() # Train the model for the specified number of epochs for _ in tqdm(range(epochs)): for images, labels in trainloader: # Move data to device (GPU or CPU) images, labels = images.to(device), labels.to(device) # Zero the gradients optimizer.zero_grad() # Forward pass outputs = net(images) # Compute the loss loss = criterion(outputs, labels) # Backward pass loss.backward() # Update model parameters optimizer.step() # Check if deadline has been reached if deadline: current_time = time.time() if current_time >= deadline: print("Deadline occurred.") break # Return the trained model return net def train_feddyn(net, trainloader, epochs, device, deadline=None, prev_grads=None): """ Trains a given neural network using the FedDyn algorithm. Args: net: A PyTorch neural network model trainloader: A PyTorch DataLoader containing the training dataset epochs: An integer specifying the number of training epochs deadline: An optional deadline (in seconds) for the training process Returns: A trained PyTorch neural network model """ x = deepcopy(net) # prev_grads = None if prev_grads is not None: prev_grads = prev_grads.to(device) else: for param in net.parameters(): if not isinstance(prev_grads, torch.Tensor): prev_grads = torch.zeros_like(param.view(-1)) prev_grads.to(device) else: prev_grads = torch.cat((prev_grads, torch.zeros_like(param.view(-1))), dim=0) prev_grads.to(device) criterion = torch.nn.CrossEntropyLoss() lr = 0.1 alpha = 0.01 optimizer = torch.optim.SGD(net.parameters(), lr=lr) for _ in tqdm(range(epochs)): inputs,labels = next(iter(trainloader)) inputs, labels = inputs.float().to(device), labels.long().to(device) output = net(inputs) loss = criterion(output, labels) #Calculate the loss with respect to y's output and labels #Dynamic Regularisation lin_penalty = 0.0 curr_params = None for param in net.parameters(): if not isinstance(curr_params, torch.Tensor): curr_params = param.view(-1) else: curr_params = torch.cat((curr_params, param.view(-1)), dim=0) lin_penalty = torch.sum(curr_params * prev_grads) loss -= lin_penalty quad_penalty = 0.0 for y, z in zip(net.parameters(), x.parameters()): quad_penalty += torch.nn.functional.mse_loss(y.data, z.data, reduction='sum') loss += (alpha/2) * quad_penalty optimizer.zero_grad() loss.backward() torch.nn.utils.clip_grad_norm_(parameters=net.parameters(), max_norm=1) # Clip gradients optimizer.step() if deadline: current_time = time.time() if current_time >= deadline: print("deadline occurred.") break #Calculate the difference between updated model (y) and the received model (x) delta = None for y, z in zip(net.parameters(), x.parameters()): if not isinstance(delta, torch.Tensor): delta = torch.sub(y.data.view(-1), z.data.view(-1)) else: delta = torch.cat((delta, torch.sub(y.data.view(-1), z.data.view(-1))),dim=0) #Update prev_grads using delta which is scaled by alpha prev_grads = torch.sub(prev_grads, delta, alpha = alpha) return net, prev_grads def train_mimelite(net, state, trainloader, epochs, device, deadline=None): """ Trains a given neural network using the MimeLite algorithm. Args: net: A PyTorch neural network model trainloader: A PyTorch DataLoader containing the training dataset epochs: An integer specifying the number of training epochs deadline: An optional deadline (in seconds) for the training process Returns: A trained PyTorch neural network model In the case of MimeLite, control_variate is nothing but a state like in case of momentum method """ x = deepcopy(net) criterion = torch.nn.CrossEntropyLoss() lr = 0.001 momentum = 0.9 net.train() for _ in tqdm(range(epochs)): for images, labels in trainloader: images, labels = images.to(device), labels.to(device) loss = criterion(net(images), labels) #Compute (full-batch) gradient of loss with respect to net's parameters grads = torch.autograd.grad(loss,net.parameters()) #Update net's parameters using gradients with torch.no_grad(): for param,grad,s in zip(net.parameters(), grads, state): param.data = param.data - lr * ((1-momentum) * grad.data + momentum * s.to(device).data) if deadline: current_time = time.time() if current_time >= deadline: print("deadline occurred.") break #Compute gradient wrt the received model (x) using the wholde dataset data = DataLoader(trainloader.dataset, batch_size = len(trainloader) * trainloader.batch_size, shuffle = True) for images, labels in data: images, labels = images.to(device), labels.to(device) output = x(images) loss = criterion(output, labels) #Calculate the loss with respect to y's output and labels gradient_x = torch.autograd.grad(loss,x.parameters()) return net, gradient_x def train_mime(net, state, control_variate, trainloader, epochs, device, deadline=None): """ Trains a given neural network using the Mime algorithm. Args: net: A PyTorch neural network model trainloader: A PyTorch DataLoader containing the training dataset epochs: An integer specifying the number of training epochs deadline: An optional deadline (in seconds) for the training process Returns: A trained PyTorch neural network model """ x = deepcopy(net) criterion = torch.nn.CrossEntropyLoss() lr = 0.001 momentum = 0.9 net.train() x.train() #control_variate = control_variate.to(DEVICE) for epoch in tqdm(range(epochs)): for images, labels in trainloader: images, labels = images.to(device), labels.to(device) loss = criterion(net(images), labels) #Compute (full-batch) gradient of loss with respect to net's parameters grads_y = torch.autograd.grad(loss,net.parameters()) if epoch == 0: output = x(images) loss = criterion(output, labels) grads_x = torch.autograd.grad(loss,x.parameters()) #Update net's parameters using gradients with torch.no_grad(): for g_y, g_x, c in zip(grads_y, grads_x, control_variate): g_y.data -= g_x.data + c.to(device) for param,grad,s in zip(net.parameters(), grads_y, state): param.data = param.data - lr * ((1-momentum) * grad.data + momentum * s.to(device).data) if deadline: current_time = time.time() if current_time >= deadline: print("deadline occurred.") break #Compute gradient wrt the received model (x) using the wholde dataset data = DataLoader(trainloader.dataset, batch_size = len(trainloader) * trainloader.batch_size, shuffle = True) for images, labels in data: images, labels = images.to(device), labels.to(device) output = x(images) loss = criterion(output, labels) #Calculate the loss with respect to y's output and labels gradient_x = torch.autograd.grad(loss,x.parameters()) return net, gradient_x def train_scaffold(net, server_c, trainloader, epochs, device, deadline=None): """ Trains a given neural network using the Scaffold algorithm. Args: net: A PyTorch neural network model trainloader: A PyTorch DataLoader containing the training dataset epochs: An integer specifying the number of training epochs deadline: An optional deadline (in seconds) for the training process Returns: A trained PyTorch neural network model """ x = deepcopy(net) client_c = deepcopy(server_c) criterion = torch.nn.CrossEntropyLoss() lr = 0.001 for _ in tqdm(range(epochs)): for images, labels in trainloader: images, labels = images.to(device), labels.to(device) loss = criterion(net(images), labels) #Compute (full-batch) gradient of loss with respect to net's parameters grads = torch.autograd.grad(loss,net.parameters()) #Update y's parameters using gradients, client_c and server_c [Algorithm line no:10] for param,grad,s_c,c_c in zip(net.parameters(),grads,server_c,client_c): s_c, c_c = s_c.to(device), c_c.to(device) param.data = param.data - lr * (grad.data + (s_c.data - c_c.data)) if deadline: current_time = time.time() if current_time >= deadline: print("deadline occurred.") break delta_c = [torch.zeros_like(param) for param in net.parameters()] new_client_c = deepcopy(delta_c) for param_net, param_x in zip(net.parameters(), x.parameters()): param_net.data = param_net.data - param_x.data a = (ceil(len(trainloader.dataset) / trainloader.batch_size) * epochs * lr) for n_c, c_l, c_g, diff in zip(new_client_c, client_c, server_c, net.parameters()): c_l = c_l.to(device) c_g = c_g.to(device) n_c.data += c_l.data - c_g.data - diff.data / a #Calculate delta_c which equals to new_client_c-client_c for d_c, n_c_l, c_l in zip(delta_c, new_client_c, client_c): d_c = d_c.to(device) c_l = c_l.to(device) d_c.data.add_(n_c_l.data - c_l.data) return net, delta_c def test_model(net, testloader, device): """Evaluate the performance of a model on a test dataset. Args: net (torch.nn.Module): The neural network model to evaluate. testloader (torch.utils.data.DataLoader): The data loader for the test dataset. Returns: Tuple: The average loss and accuracy of the model on the test dataset. """ criterion = torch.nn.CrossEntropyLoss() net.eval() test_loss, correct, total = 0.0, 0, 0 with torch.no_grad(): for images, labels in tqdm(testloader): images, labels = images.to(device), labels.to(device) outputs = net(images) test_loss += criterion(outputs, labels).item() _, predicted = torch.max(outputs.data, 1) total += labels.size(0) correct += (predicted == labels).sum().item() test_loss /= len(testloader.dataset) accuracy = correct / total return test_loss, accuracy ================================================ FILE: federa/client/start_client.py ================================================ from .src.client import client_start import argparse parser = argparse.ArgumentParser() parser.add_argument("--ip", type=str, default = "localhost:8214", help="IP address of the server") parser.add_argument("--device", type=str, default = "cpu", help="Device to run the client on") parser.add_argument('--ca', type = str, default= 'ca.pem', help= 'path to CA certificate') parser.add_argument('--encryption', type = int, default= 0, help= '1 enables ssl encryption') parser.add_argument('--wait_time', type = int, default= 30, help= 'time to wait before sending the next request') args = parser.parse_args() configs = { "ip_address": args.ip, "wait_time": args.wait_time, "device": args.device, "encryption": args.encryption, "ca": args.ca, } if __name__ == '__main__': client_start(configs) ================================================ FILE: federa/server/__init__.py ================================================ ================================================ FILE: federa/server/src/ClientConnection_pb2.py ================================================ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: ClientConnection.proto """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() DESCRIPTOR = _descriptor.FileDescriptor( name='ClientConnection.proto', package='', syntax='proto3', serialized_options=None, #create_key=_descriptor._internal_create_key, serialized_pb=b'\n\x16\x43lientConnection.proto\"\xa3\x01\n\rServerMessage\x12\x1f\n\ntrainOrder\x18\x01 \x01(\x0b\x32\x0b.TrainOrder\x12\x1d\n\tevalOrder\x18\x02 \x01(\x0b\x32\n.EvalOrder\x12)\n\x0f\x64isconnectOrder\x18\x03 \x01(\x0b\x32\x10.DisconnectOrder\x12\'\n\x0esetParamsOrder\x18\x04 \x01(\x0b\x32\x0f.SetParamsOrder\"\x8a\x01\n\rClientMessage\x12%\n\rtrainResponse\x18\x01 \x01(\x0b\x32\x0e.TrainResponse\x12#\n\x0c\x65valResponse\x18\x02 \x01(\x0b\x32\r.EvalResponse\x12-\n\x11setParamsResponse\x18\x03 \x01(\x0b\x32\x12.SetParamsResponse\"9\n\nTrainOrder\x12\x17\n\x0fmodelParameters\x18\x01 \x01(\x0c\x12\x12\n\nconfigDict\x18\x02 \x01(\x0c\">\n\rTrainResponse\x12\x17\n\x0fmodelParameters\x18\x01 \x01(\x0c\x12\x14\n\x0cresponseDict\x18\x02 \x01(\x0c\"8\n\tEvalOrder\x12\x17\n\x0fmodelParameters\x18\x01 \x01(\x0c\x12\x12\n\nconfigDict\x18\x02 \x01(\x0c\"$\n\x0c\x45valResponse\x12\x14\n\x0cresponseDict\x18\x01 \x01(\x0c\")\n\x0eSetParamsOrder\x12\x17\n\x0fmodelParameters\x18\x01 \x01(\x0c\"\x13\n\x11SetParamsResponse\"9\n\x0f\x44isconnectOrder\x12\x0f\n\x07message\x18\x01 \x01(\t\x12\x15\n\rreconnectTime\x18\x02 \x01(\x05\x32\x41\n\x10\x43lientConnection\x12-\n\x07\x43onnect\x12\x0e.ClientMessage\x1a\x0e.ServerMessage(\x01\x30\x01\x62\x06proto3' ) _SERVERMESSAGE = _descriptor.Descriptor( name='ServerMessage', full_name='ServerMessage', filename=None, file=DESCRIPTOR, containing_type=None, #create_key=_descriptor._internal_create_key, fields=[ _descriptor.FieldDescriptor( name='trainOrder', full_name='ServerMessage.trainOrder', index=0, number=1, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='evalOrder', full_name='ServerMessage.evalOrder', index=1, number=2, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='disconnectOrder', full_name='ServerMessage.disconnectOrder', index=2, number=3, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='setParamsOrder', full_name='ServerMessage.setParamsOrder', index=3, number=4, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], oneofs=[ ], serialized_start=27, serialized_end=190, ) _CLIENTMESSAGE = _descriptor.Descriptor( name='ClientMessage', full_name='ClientMessage', filename=None, file=DESCRIPTOR, containing_type=None, #create_key=_descriptor._internal_create_key, fields=[ _descriptor.FieldDescriptor( name='trainResponse', full_name='ClientMessage.trainResponse', index=0, number=1, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='evalResponse', full_name='ClientMessage.evalResponse', index=1, number=2, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='setParamsResponse', full_name='ClientMessage.setParamsResponse', index=2, number=3, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], oneofs=[ ], serialized_start=193, serialized_end=331, ) _TRAINORDER = _descriptor.Descriptor( name='TrainOrder', full_name='TrainOrder', filename=None, file=DESCRIPTOR, containing_type=None, #create_key=_descriptor._internal_create_key, fields=[ _descriptor.FieldDescriptor( name='modelParameters', full_name='TrainOrder.modelParameters', index=0, number=1, type=12, cpp_type=9, label=1, has_default_value=False, default_value=b"", message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='configDict', full_name='TrainOrder.configDict', index=1, number=2, type=12, cpp_type=9, label=1, has_default_value=False, default_value=b"", message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], oneofs=[ ], serialized_start=333, serialized_end=390, ) _TRAINRESPONSE = _descriptor.Descriptor( name='TrainResponse', full_name='TrainResponse', filename=None, file=DESCRIPTOR, containing_type=None, #create_key=_descriptor._internal_create_key, fields=[ _descriptor.FieldDescriptor( name='modelParameters', full_name='TrainResponse.modelParameters', index=0, number=1, type=12, cpp_type=9, label=1, has_default_value=False, default_value=b"", message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='responseDict', full_name='TrainResponse.responseDict', index=1, number=2, type=12, cpp_type=9, label=1, has_default_value=False, default_value=b"", message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], oneofs=[ ], serialized_start=392, serialized_end=454, ) _EVALORDER = _descriptor.Descriptor( name='EvalOrder', full_name='EvalOrder', filename=None, file=DESCRIPTOR, containing_type=None, #create_key=_descriptor._internal_create_key, fields=[ _descriptor.FieldDescriptor( name='modelParameters', full_name='EvalOrder.modelParameters', index=0, number=1, type=12, cpp_type=9, label=1, has_default_value=False, default_value=b"", message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='configDict', full_name='EvalOrder.configDict', index=1, number=2, type=12, cpp_type=9, label=1, has_default_value=False, default_value=b"", message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], oneofs=[ ], serialized_start=456, serialized_end=512, ) _EVALRESPONSE = _descriptor.Descriptor( name='EvalResponse', full_name='EvalResponse', filename=None, file=DESCRIPTOR, containing_type=None, #create_key=_descriptor._internal_create_key, fields=[ _descriptor.FieldDescriptor( name='responseDict', full_name='EvalResponse.responseDict', index=0, number=1, type=12, cpp_type=9, label=1, has_default_value=False, default_value=b"", message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], oneofs=[ ], serialized_start=514, serialized_end=550, ) _SETPARAMSORDER = _descriptor.Descriptor( name='SetParamsOrder', full_name='SetParamsOrder', filename=None, file=DESCRIPTOR, containing_type=None, #create_key=_descriptor._internal_create_key, fields=[ _descriptor.FieldDescriptor( name='modelParameters', full_name='SetParamsOrder.modelParameters', index=0, number=1, type=12, cpp_type=9, label=1, has_default_value=False, default_value=b"", message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], oneofs=[ ], serialized_start=552, serialized_end=593, ) _SETPARAMSRESPONSE = _descriptor.Descriptor( name='SetParamsResponse', full_name='SetParamsResponse', filename=None, file=DESCRIPTOR, containing_type=None, #create_key=_descriptor._internal_create_key, fields=[ ], extensions=[ ], nested_types=[], enum_types=[ ], serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], oneofs=[ ], serialized_start=595, serialized_end=614, ) _DISCONNECTORDER = _descriptor.Descriptor( name='DisconnectOrder', full_name='DisconnectOrder', filename=None, file=DESCRIPTOR, containing_type=None, #create_key=_descriptor._internal_create_key, fields=[ _descriptor.FieldDescriptor( name='message', full_name='DisconnectOrder.message', index=0, number=1, type=9, cpp_type=9, label=1, has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='reconnectTime', full_name='DisconnectOrder.reconnectTime', index=1, number=2, type=5, cpp_type=1, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], oneofs=[ ], serialized_start=616, serialized_end=673, ) _SERVERMESSAGE.fields_by_name['trainOrder'].message_type = _TRAINORDER _SERVERMESSAGE.fields_by_name['evalOrder'].message_type = _EVALORDER _SERVERMESSAGE.fields_by_name['disconnectOrder'].message_type = _DISCONNECTORDER _SERVERMESSAGE.fields_by_name['setParamsOrder'].message_type = _SETPARAMSORDER _CLIENTMESSAGE.fields_by_name['trainResponse'].message_type = _TRAINRESPONSE _CLIENTMESSAGE.fields_by_name['evalResponse'].message_type = _EVALRESPONSE _CLIENTMESSAGE.fields_by_name['setParamsResponse'].message_type = _SETPARAMSRESPONSE DESCRIPTOR.message_types_by_name['ServerMessage'] = _SERVERMESSAGE DESCRIPTOR.message_types_by_name['ClientMessage'] = _CLIENTMESSAGE DESCRIPTOR.message_types_by_name['TrainOrder'] = _TRAINORDER DESCRIPTOR.message_types_by_name['TrainResponse'] = _TRAINRESPONSE DESCRIPTOR.message_types_by_name['EvalOrder'] = _EVALORDER DESCRIPTOR.message_types_by_name['EvalResponse'] = _EVALRESPONSE DESCRIPTOR.message_types_by_name['SetParamsOrder'] = _SETPARAMSORDER DESCRIPTOR.message_types_by_name['SetParamsResponse'] = _SETPARAMSRESPONSE DESCRIPTOR.message_types_by_name['DisconnectOrder'] = _DISCONNECTORDER _sym_db.RegisterFileDescriptor(DESCRIPTOR) ServerMessage = _reflection.GeneratedProtocolMessageType('ServerMessage', (_message.Message,), { 'DESCRIPTOR' : _SERVERMESSAGE, '__module__' : 'ClientConnection_pb2' # @@protoc_insertion_point(class_scope:ServerMessage) }) _sym_db.RegisterMessage(ServerMessage) ClientMessage = _reflection.GeneratedProtocolMessageType('ClientMessage', (_message.Message,), { 'DESCRIPTOR' : _CLIENTMESSAGE, '__module__' : 'ClientConnection_pb2' # @@protoc_insertion_point(class_scope:ClientMessage) }) _sym_db.RegisterMessage(ClientMessage) TrainOrder = _reflection.GeneratedProtocolMessageType('TrainOrder', (_message.Message,), { 'DESCRIPTOR' : _TRAINORDER, '__module__' : 'ClientConnection_pb2' # @@protoc_insertion_point(class_scope:TrainOrder) }) _sym_db.RegisterMessage(TrainOrder) TrainResponse = _reflection.GeneratedProtocolMessageType('TrainResponse', (_message.Message,), { 'DESCRIPTOR' : _TRAINRESPONSE, '__module__' : 'ClientConnection_pb2' # @@protoc_insertion_point(class_scope:TrainResponse) }) _sym_db.RegisterMessage(TrainResponse) EvalOrder = _reflection.GeneratedProtocolMessageType('EvalOrder', (_message.Message,), { 'DESCRIPTOR' : _EVALORDER, '__module__' : 'ClientConnection_pb2' # @@protoc_insertion_point(class_scope:EvalOrder) }) _sym_db.RegisterMessage(EvalOrder) EvalResponse = _reflection.GeneratedProtocolMessageType('EvalResponse', (_message.Message,), { 'DESCRIPTOR' : _EVALRESPONSE, '__module__' : 'ClientConnection_pb2' # @@protoc_insertion_point(class_scope:EvalResponse) }) _sym_db.RegisterMessage(EvalResponse) SetParamsOrder = _reflection.GeneratedProtocolMessageType('SetParamsOrder', (_message.Message,), { 'DESCRIPTOR' : _SETPARAMSORDER, '__module__' : 'ClientConnection_pb2' # @@protoc_insertion_point(class_scope:SetParamsOrder) }) _sym_db.RegisterMessage(SetParamsOrder) SetParamsResponse = _reflection.GeneratedProtocolMessageType('SetParamsResponse', (_message.Message,), { 'DESCRIPTOR' : _SETPARAMSRESPONSE, '__module__' : 'ClientConnection_pb2' # @@protoc_insertion_point(class_scope:SetParamsResponse) }) _sym_db.RegisterMessage(SetParamsResponse) DisconnectOrder = _reflection.GeneratedProtocolMessageType('DisconnectOrder', (_message.Message,), { 'DESCRIPTOR' : _DISCONNECTORDER, '__module__' : 'ClientConnection_pb2' # @@protoc_insertion_point(class_scope:DisconnectOrder) }) _sym_db.RegisterMessage(DisconnectOrder) _CLIENTCONNECTION = _descriptor.ServiceDescriptor( name='ClientConnection', full_name='ClientConnection', file=DESCRIPTOR, index=0, serialized_options=None, #create_key=_descriptor._internal_create_key, serialized_start=675, serialized_end=740, methods=[ _descriptor.MethodDescriptor( name='Connect', full_name='ClientConnection.Connect', index=0, containing_service=None, input_type=_CLIENTMESSAGE, output_type=_SERVERMESSAGE, serialized_options=None, #create_key=_descriptor._internal_create_key, ), ]) _sym_db.RegisterServiceDescriptor(_CLIENTCONNECTION) DESCRIPTOR.services_by_name['ClientConnection'] = _CLIENTCONNECTION ================================================ FILE: federa/server/src/ClientConnection_pb2_grpc.py ================================================ # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! """Client and server classes corresponding to protobuf-defined services.""" import grpc from . import ClientConnection_pb2 as ClientConnection__pb2 class ClientConnectionStub(): """Missing associated documentation comment in .proto file.""" def __init__(self, channel): """Constructor. Args: channel: A grpc.Channel. """ self.Connect = channel.stream_stream( '/ClientConnection/Connect', request_serializer=ClientConnection__pb2.ClientMessage.SerializeToString, response_deserializer=ClientConnection__pb2.ServerMessage.FromString, ) class ClientConnectionServicer(): """Missing associated documentation comment in .proto file.""" def Connect(self, request_iterator, context): """Missing associated documentation comment in .proto file.""" context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def add_ClientConnectionServicer_to_server(servicer, server): rpc_method_handlers = { 'Connect': grpc.stream_stream_rpc_method_handler( servicer.Connect, request_deserializer=ClientConnection__pb2.ClientMessage.FromString, response_serializer=ClientConnection__pb2.ServerMessage.SerializeToString, ), } generic_handler = grpc.method_handlers_generic_handler( 'ClientConnection', rpc_method_handlers) server.add_generic_rpc_handlers((generic_handler,)) # This class is part of an EXPERIMENTAL API. class ClientConnection(): """Missing associated documentation comment in .proto file.""" @staticmethod def Connect(request_iterator, target, options=(), channel_credentials=None, call_credentials=None, insecure=False, compression=None, wait_for_ready=None, timeout=None, metadata=None): return grpc.experimental.stream_stream(request_iterator, target, '/ClientConnection/Connect', ClientConnection__pb2.ClientMessage.SerializeToString, ClientConnection__pb2.ServerMessage.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata) ================================================ FILE: federa/server/src/__init__.py ================================================ ================================================ FILE: federa/server/src/algorithms/fedadagrad.py ================================================ import functools from collections import OrderedDict import torch #averages all of the given state dicts class fedadagrad(): def __init__(self, config): self.algorithm = "FedAdagrad" self.lr = 0.01 self.epsilon = 1e-6 self.state = None def aggregate(self,server_state_dict,state_dicts): keys = server_state_dict.keys() #List of keys in a state_dict #Averages the differences that we got by subtracting the server_model from client_model (delta_y) avg_delta_y = OrderedDict() for key in keys: current_key_tensors = [state_dict[key] for state_dict in state_dicts] current_key_sum = functools.reduce( lambda accumulator, tensor: accumulator + tensor, current_key_tensors ) current_key_average = current_key_sum / len(state_dicts) avg_delta_y[key] = current_key_average if not self.state: #If state = None, then the following line will execute. #So only at first round, it'll execute self.state = [torch.zeros_like(server_state_dict[key]) for key in server_state_dict.keys()] #Updates the server_state_dict for key, state in zip(keys, self.state): state.data += torch.square(avg_delta_y[key]) server_state_dict[key] += self.lr * avg_delta_y[key] / torch.sqrt(state.data + self.epsilon) return server_state_dict ================================================ FILE: federa/server/src/algorithms/fedadam.py ================================================ import functools from collections import OrderedDict import torch #averages all of the given state dicts class fedadam(): def __init__(self, config): self.algorithm = "FedAdam" self.lr = 0.01 self.beta1 = 0.9 self.beta2 = 0.999 self.epsilon = 1e-6 self.timestep = 1 self.m = None #1st moment vectpr self.v = None #2nd moment vector def aggregate(self,server_state_dict,state_dicts): keys = server_state_dict.keys() #List of keys in a state_dict #Averages the differences that we got by subtracting the server_model from client_model (delta_y) avg_delta_y = OrderedDict() for key in keys: current_key_tensors = [state_dict[key] for state_dict in state_dicts] current_key_sum = functools.reduce( lambda accumulator, tensor: accumulator + tensor, current_key_tensors ) current_key_average = current_key_sum / len(state_dicts) avg_delta_y[key] = current_key_average if not self.m: #If self.m = None, then the following line will execute. So only at first round, it'll execute self.m = [torch.zeros_like(server_state_dict[key]) for key in server_state_dict.keys()] self.v = [torch.zeros_like(server_state_dict[key]) for key in server_state_dict.keys()] #Updates the server_state_dict for key, m, v in zip(keys, self.m, self.v): m.data = self.beta1 * m.data + (1 - self.beta1) * avg_delta_y[key].data v.data = self.beta2 * v.data + (1 - self.beta2) * torch.square(avg_delta_y[key].data) m_bias_corr = m / (1 - self.beta1**self.timestep) v_bias_corr = v / (1 - self.beta2**self.timestep) server_state_dict[key].data += self.lr * m_bias_corr / (torch.sqrt(v_bias_corr) + self.epsilon) self.timestep += 1 #After each aggregation, timestep will increment by 1 return server_state_dict ================================================ FILE: federa/server/src/algorithms/fedavg.py ================================================ import functools from collections import OrderedDict #averages all of the given state dicts class fedavg(): def __init__(self, config): self.algorithm = "FedAvg" def aggregate(self,server_state_dict,state_dicts): #server_state_dict is of no use in FedAvg, # to maintain consistency with other algorithms; it is provided as an argument result_state_dict = OrderedDict() for key in state_dicts[0].keys(): current_key_tensors = [state_dict[key] for state_dict in state_dicts] current_key_sum = functools.reduce( lambda accumulator, tensor: accumulator + tensor, current_key_tensors ) current_key_average = current_key_sum / len(state_dicts) result_state_dict[key] = current_key_average return result_state_dict ================================================ FILE: federa/server/src/algorithms/fedavgm.py ================================================ import functools from collections import OrderedDict #averages all of the given state dicts class fedavgm(): def __init__(self, config): self.algorithm = "FedAvgM" self.momentum = 0.9 self.lr = 1 self.velocity = None def aggregate(self,server_state_dict,state_dicts): keys = server_state_dict.keys() #List of keys in a state_dict #Averages the differences that we got by subtracting the server_model from client_model (delta_y) avg_delta_y = OrderedDict() for key in keys: current_key_tensors = [state_dict[key] for state_dict in state_dicts] current_key_sum = functools.reduce( lambda accumulator, tensor: accumulator + tensor, current_key_tensors ) current_key_average = current_key_sum / len(state_dicts) avg_delta_y[key] = current_key_average #Updates the velocity if self.velocity: #This will be False at the first round for key in keys: self.velocity[key] = self.momentum * self.velocity[key] + avg_delta_y[key] else: self.velocity = avg_delta_y #Uses Nesterov gradient for key in keys: avg_delta_y[key] += self.momentum * self.velocity[key] #Updates server_state_dict for key in keys: server_state_dict[key] += self.lr * avg_delta_y[key] return server_state_dict ================================================ FILE: federa/server/src/algorithms/feddyn.py ================================================ import functools from collections import OrderedDict import torch #averages all of the given state dicts class feddyn(): def __init__(self, config): self.algorithm = "FedDyn" self.lr = 1.0 self.momentum = 0.9 self.h = None self.alpha = 0.01 def aggregate(self, server_model_state_dict, state_dicts): keys = server_model_state_dict.keys() #List of keys in a state_dict if not self.h: #If self.h = None, then the following line will execute. #So only at first round, it'll execute self.h = [torch.zeros_like(server_model_state_dict[key]) for key in server_model_state_dict.keys()] sum_y = OrderedDict() #This will be our new server_model_state_dict for key in keys: current_key_tensors = [state_dict[key] for state_dict in state_dicts] current_key_sum = functools.reduce( lambda accumulator, tensor: accumulator + tensor, current_key_tensors ) sum_y[key] = current_key_sum delta_x = [torch.zeros_like(server_model_state_dict[key]) for key in server_model_state_dict.keys()] for d_x, key in zip(delta_x, keys): d_x.data = sum_y[key]/len(state_dicts) - server_model_state_dict[key].to(sum_y[key].device) #Update h for h, d_x in zip(self.h, delta_x): h.data = h.data.to(d_x.data.device) h.data -= (self.alpha/len(state_dicts)) * d_x.data #Update x for key, h in zip(keys, self.h): server_model_state_dict[key] = (sum_y[key]/len(state_dicts)) - (h.data/self.alpha) return server_model_state_dict ================================================ FILE: federa/server/src/algorithms/fedyogi.py ================================================ import functools from collections import OrderedDict import torch #averages all of the given state dicts class fedyogi(): def __init__(self, config): self.algorithm = "FedYogi" self.lr = 0.01 self.beta1 = 0.9 self.beta2 = 0.999 self.epsilon = 1e-6 self.timestep = 1 self.m = None #1st moment vectpr self.v = None #2nd moment vector def aggregate(self,server_state_dict,state_dicts): keys = server_state_dict.keys() #List of keys in a state_dict #Averages the differences that we got by subtracting the server_model from client_model (delta_y) avg_delta_y = OrderedDict() for key in keys: current_key_tensors = [state_dict[key] for state_dict in state_dicts] current_key_sum = functools.reduce( lambda accumulator, tensor: accumulator + tensor, current_key_tensors ) current_key_average = current_key_sum / len(state_dicts) avg_delta_y[key] = current_key_average if not self.m: #If self.m = None, then the following line will execute. #So only at first round, it'll execute self.m = [torch.zeros_like(server_state_dict[key]) for key in server_state_dict.keys()] self.v = [torch.zeros_like(server_state_dict[key]) for key in server_state_dict.keys()] #Updates the server_state_dict for key, m, v in zip(keys, self.m, self.v): m.data = self.beta1 * m.data + (1 - self.beta1) * avg_delta_y[key].data v.data = v.data + (1 - self.beta2) * torch.sign( torch.square(avg_delta_y[key].data) - v.data ) * torch.square(avg_delta_y[key].data) m_bias_corr = m / (1 - self.beta1**self.timestep) v_bias_corr = v / (1 - self.beta2**self.timestep) server_state_dict[key].data += self.lr * m_bias_corr / (torch.sqrt(v_bias_corr) + self.epsilon) self.timestep += 1 #After each aggregation, timestep will increment by 1 return server_state_dict ================================================ FILE: federa/server/src/algorithms/mime.py ================================================ import functools from collections import OrderedDict #averages all of the given state dicts class mime(): def __init__(self, config): self.algorithm = "Mime" self.lr = 1.0 self.momentum = 0.9 def aggregate(self,server_model_state_dict, optimizer_state, state_dicts, gradients_x): keys = server_model_state_dict.keys() #List of keys in a state_dict avg_y = OrderedDict() #This will be our new server_model_state_dict for key in keys: current_key_tensors = [state_dict[key] for state_dict in state_dicts] current_key_sum = functools.reduce( lambda accumulator, tensor: accumulator + tensor, current_key_tensors ) current_key_average = current_key_sum / len(state_dicts) avg_y[key] = current_key_average #Average all the gradient_x in gradients_x avg_grads = [] for i in range(len(gradients_x[0])): #Average all the i'th element of gradient_x present in the gradients_x current_tensors = [gradient_x[i] for gradient_x in gradients_x] current_sum = functools.reduce(lambda accumulator, tensor: accumulator + tensor, current_tensors) current_average = current_sum / len(gradients_x) avg_grads.append(current_average) for state, grad in zip(optimizer_state, avg_grads): state.data = self.momentum * state.data + (1 - self.momentum) * grad.data control_variate = avg_grads return avg_y, optimizer_state, control_variate ================================================ FILE: federa/server/src/algorithms/mimelite.py ================================================ import functools from collections import OrderedDict #averages all of the given state dicts class mimelite(): def __init__(self, config): self.algorithm = "MimeLite" self.lr = 1.0 self.momentum = 0.9 def aggregate(self,server_model_state_dict, optimizer_state, state_dicts, gradients_x): keys = server_model_state_dict.keys() #List of keys in a state_dict avg_y = OrderedDict() #This will be our new server_model_state_dict for key in keys: current_key_tensors = [state_dict[key] for state_dict in state_dicts] current_key_sum = functools.reduce( lambda accumulator, tensor: accumulator + tensor, current_key_tensors ) current_key_average = current_key_sum / len(state_dicts) avg_y[key] = current_key_average #Average all the gradient_x in gradients_x avg_grads = [] for i in range(len(gradients_x[0])): #Average all the i'th element of gradient_x present in the gradients_x current_tensors = [gradient_x[i] for gradient_x in gradients_x] current_sum = functools.reduce(lambda accumulator, tensor: accumulator + tensor, current_tensors) current_average = current_sum / len(gradients_x) avg_grads.append(current_average) for state, grad in zip(optimizer_state, avg_grads): state.data = self.momentum * state.data + (1 - self.momentum) * grad.data return avg_y, optimizer_state ================================================ FILE: federa/server/src/algorithms/scaffold.py ================================================ import functools from collections import OrderedDict #averages all of the given state dicts class scaffold(): def __init__(self, config): self.algorithm = "SCAFFOLD" self.lr = 1.0 self.fraction = config["fraction_of_clients"] def aggregate(self,server_model_state_dict, control_variate, state_dicts, updated_control_variates): keys = server_model_state_dict.keys() #List of keys in a state_dict #Averages the differences that we got by subtracting the server_model from client_model (delta_y) delta_x = OrderedDict() for key in keys: current_key_tensors = [state_dict[key] for state_dict in state_dicts] current_key_sum = functools.reduce( lambda accumulator, tensor: accumulator + tensor, current_key_tensors ) current_key_average = current_key_sum / len(state_dicts) delta_x[key] = current_key_average #Average all the in gradients_x delta_c = [] for i in range(len(control_variate)): #Average all the i'th element of updated_control_variate present in the updated_control_variates current_tensors = [updated_control_variate[i] for updated_control_variate in updated_control_variates] current_sum = functools.reduce(lambda accumulator, tensor: accumulator + tensor, current_tensors) current_average = current_sum / len(updated_control_variates) delta_c.append(current_average) for key in keys: server_model_state_dict[key] += self.lr * delta_x[key] control_variate_list = list(range(len(control_variate))) for i in control_variate_list: control_variate[i] += self.fraction * delta_c[i] return server_model_state_dict, control_variate ================================================ FILE: federa/server/src/client_connection_servicer.py ================================================ from queue import Queue from . import ClientConnection_pb2_grpc from .client_wrapper import ClientWrapper #gRPC servicer that contains all functions that can be called by the client class ClientConnectionServicer( ClientConnection_pb2_grpc.ClientConnectionServicer ): def __init__(self, client_manager): self.client_manager = client_manager #called by every newly connected client. executes in a different thread for every client. #creates a client wrapper object for the client, registers with client manager, # and passes message from server to client #and vice-versa def Connect(self, request_iterator, context): client_id = context.peer() client_message_iterator = request_iterator send_buffer = Queue(maxsize = 1) recieve_buffer = Queue(maxsize = 1) client = ClientWrapper(send_buffer, recieve_buffer, client_id) register_result = self.client_manager.register(client) #if server is accepting connections, and registering was successful, True is returned if register_result: print(f"Client {client_id} connected.") client_index = self.client_manager.num_connected_clients() - 1 client.client_idx = client_index try: while True: server_message = send_buffer.get() yield server_message client_message = next(client_message_iterator) recieve_buffer.put(client_message) finally: client.is_connected = False self.client_manager.deregister(client_index) print(f"Client {client_id} has disconnected.") print(f"{self.client_manager.num_connected_clients()} clients remain active.") #server is not accepting connections or registering failed else: client.disconnect() server_message = send_buffer.get() yield server_message print(f"Client {client_id} attempted to connect. Connection refused.") ================================================ FILE: federa/server/src/client_manager.py ================================================ import random import threading from math import ceil #holds references to all live client_wrapper objects class ClientManager: def __init__(self): self.client_list = [] self.cv = threading.Condition() self.accepting_connections = True #set to false to stop accepting further connections #returns a list of references to client wrapper objects in the order they connected def select(self, num_of_clients = None, fraction = None, timeout = None): if num_of_clients: self.wait_for(num_of_clients, timeout = timeout) if num_of_clients and fraction: num_of_clients = ceil( fraction * num_of_clients ) if num_of_clients is None and fraction: num_of_clients = ceil( fraction * len(self.client_list) ) if num_of_clients is None and fraction is None: num_of_clients = len(self.client_list) selected_clients_list = self.client_list[:num_of_clients] return selected_clients_list #same as select but random order def random_select(self, num_of_clients = None, fraction = None, timeout = None): if num_of_clients: self.wait_for(num_of_clients, timeout = timeout) if num_of_clients and fraction: num_of_clients = ceil( fraction * num_of_clients ) if num_of_clients is None and fraction: num_of_clients = ceil( fraction * len(self.client_list) ) if num_of_clients is None and fraction is None: num_of_clients = len(self.client_list) client_list = self.client_list if len(client_list) < num_of_clients: return client_list selected_clients_list = random.sample(client_list, k=num_of_clients) return selected_clients_list #used to add a client wrapper object when accepting a new connection def register(self, client): if not self.accepting_connections: return False with self.cv: self.client_list.append(client) self.cv.notify_all() return True def num_connected_clients(self): return len(self.client_list) def deregister(self, client_index): self.client_list.pop(client_index) #wait for the number of clients to connect, indefinitely. # unless a timeout is specified, then just return after timeout def wait_for(self, minimum_clients, timeout): with self.cv: self.cv.wait_for( lambda: len(self.client_list) >= minimum_clients, timeout ) ================================================ FILE: federa/server/src/client_wrapper.py ================================================ from io import BytesIO import torch import json from .ClientConnection_pb2 import ServerMessage, TrainOrder, EvalOrder, SetParamsOrder, DisconnectOrder #serves as an abstraction of the actual connected client. #methods called here are called on the actual client with the same inputs and outputs class ClientWrapper: def __init__(self, send_buffer, recieve_buffer, client_id): #data is placed in this buffer to send to client self.send_buffer = send_buffer #data recieved from client is extracted from this buffer self.recieve_buffer = recieve_buffer self.client_id = client_id self.is_connected = True self.client_idx = None #orders the connected client to train using the given parameters def train(self, model_parameters, control_variate, control_variate2, config_dict): self.check_disconnection() #Create a dictionary where model_parameters and control_variate are stored data = {} data['model_parameters'] = model_parameters data['control_variate'] = control_variate data['control_variate2'] = control_variate2 #convert data to bytes buffer = BytesIO() torch.save(data, buffer) buffer.seek(0) data_bytes = buffer.read() ##add client index to config dict config_dict['client_idx'] = self.client_idx #convert config_dict to bytes config_dict_bytes = json.dumps(config_dict).encode("utf-8") #send bytes to client train_order_message = TrainOrder( modelParameters = data_bytes, configDict = config_dict_bytes) message_to_client = ServerMessage(trainOrder = train_order_message) self.send_buffer.put(message_to_client) #get trained model_parameters and response_dict from client client_message = self.recieve_buffer.get() train_response_message = client_message.trainResponse data_received_bytes = train_response_message.modelParameters data_received = torch.load( BytesIO(data_received_bytes), map_location="cpu" ) #updated_control_variate will become None when no control_variate is involved at all trained_model_parameters = data_received['model_parameters'] updated_control_variate = data_received['control_variate'] response_dict_bytes = train_response_message.responseDict response_dict = json.loads( response_dict_bytes.decode("utf-8") ) return trained_model_parameters, updated_control_variate, response_dict #orders the connected client to evaluate the given parameters def evaluate(self, model_parameters, config_dict): self.check_disconnection() #convert state_dict inside model_parameters to bytes buffer = BytesIO() torch.save(model_parameters, buffer) buffer.seek(0) model_parameters_bytes = buffer.read() #convert config_dict to bytes config_dict_bytes = json.dumps(config_dict).encode("utf-8") #send bytes to client eval_order_message = EvalOrder( modelParameters = model_parameters_bytes, configDict = config_dict_bytes) message_to_client = ServerMessage(evalOrder = eval_order_message) self.send_buffer.put(message_to_client) #get response dict as bytes from client client_message = self.recieve_buffer.get() eval_response_message = client_message.evalResponse response_dict_bytes = eval_response_message.responseDict response_dict = json.loads(response_dict_bytes.decode("utf-8")) return response_dict #orders the client to set its own parameters as the ones passed def set_parameters(self, model_parameters): self.check_disconnection() buffer = BytesIO() torch.save(model_parameters, buffer) buffer.seek(0) model_parameters_bytes = buffer.read() set_parameters_order_message = SetParamsOrder(modelParameters = model_parameters_bytes) message_to_client = ServerMessage(setParamsOrder = set_parameters_order_message) self.send_buffer.put(message_to_client) #client sends an empty set params message as response self.recieve_buffer.get() def check_disconnection(self): if not self.is_connected: raise Exception(f"Cannot execute command. {self.client_id} is disconnected.") def is_disconnected(self): return not self.is_connected #orders the client to disconnect. if a reconnect is specified (in seconds), #the client will attempt to reconnect after that time def disconnect(self, reconnect_time = 0, message = "Thank you for participating."): self.check_disconnection() disconnect_order_message = DisconnectOrder(reconnectTime = reconnect_time, message = message) message_to_client = ServerMessage(disconnectOrder = disconnect_order_message) self.send_buffer.put(message_to_client) ================================================ FILE: federa/server/src/distribution.py ================================================ import numpy as np import torch import random import os def data_distribution(config, trainset, num_users): labels = [] base_dir = os.getcwd() storepath = os.path.join(base_dir, 'Distribution/', config['dataset']+'/') seed = 10 random.seed(seed) #Calculate the number of samples present per class trainset_list = list(range(len(trainset))) for i in trainset_list: labels.append(trainset[i][1]) unique_labels = np.unique(np.array(labels)) label_index_list = {} for key in unique_labels: label_index_list[key] = [] for index, label in enumerate(labels): label_index_list[label].append(index) num_classes = len(unique_labels) #Calculate the value of the probability distribution. For K=1, it will be iid distribution K = config['niid'] if K==1: q_step = (1 - (1/num_classes)) else: q_step = (1 - (1/num_classes))/(K-1) #Shuffle the index position for all classes label_index_list_list = list(range(len(label_index_list))) for i in label_index_list_list: random.shuffle(label_index_list[i]) #Generate the different non-iid distribution. # Data_presence_indicator will help to reduce the number of classes -- # among the clients as the non-iid increases for j in range(K): dist = np.random.uniform(q_step, (1+j)*q_step, (num_classes, num_users)) if j != 0: data_presence_indicator = np.random.choice([0, 1], (num_classes, num_users), p=[j*q_step, 1-(j*q_step)]) if len(np.where(np.sum(data_presence_indicator, axis=0) == 0)[0])>0: for i in np.where(np.sum(data_presence_indicator, axis=0) == 0)[0]: zero_array = data_presence_indicator[:,i] zero_array[np.random.choice(len(zero_array),1)] =1 data_presence_indicator[:,i] = zero_array dist = np.multiply(dist,data_presence_indicator) psum = np.sum(dist, axis=1) for i in range(dist.shape[0]): dist[i] = dist[i]*len(label_index_list[i])/(psum[i]+0.00001) dist = np.floor(dist).astype(int) # If any client does not get any data then this logic helps to allocate the required samples among the clients gainers = list(np.where(np.sum(dist, axis=0) != 0))[0] if len(gainers) < num_users: losers = list(np.where(np.sum(dist, axis=0) == 0))[0] donors = np.random.choice(gainers, len(losers)) for index, donor in enumerate(donors): avail_digits = np.where(dist[:,donor] != 0)[0] for digit in avail_digits: transfer_frac = np.random.uniform(0.1,0.9) num_transfer = int(dist[digit, donor]*transfer_frac) dist[digit, donor] = dist[digit, donor] - num_transfer dist[digit, losers[index]] = num_transfer #Logic to check if the summation of all the samples among the clients is equal to # # the total number of samples present for that class. If not it will adjust. for num in range(num_classes): while dist[num].sum() != len(label_index_list[num]): index = random.randint(0,num_users-1) # nosec if dist[num].sum() < len(label_index_list[num]): dist[num][index]+=1 else: dist[num][index]-=1 #Division of samples number among the clients split = [[] for i in range(num_classes)] for num in range(num_classes): start = 0 for i in range(num_users): split[num].append(label_index_list[num][start:start+dist[num][i]]) start = start+dist[num][i] #Division of actual data points among the clients. datapoints = [[] for i in range(num_users)] class_histogram = [[] for i in range(num_users)] class_stats= [[] for i in range(num_users)] for i in range(num_users): for num in range(num_classes): datapoints[i] += split[num][i] class_histogram[i].append(len(split[num][i])) if len(split[num][i])==0: class_stats[i].append(0) else: class_stats[i].append(1) #Store the dataset division in the folder if not os.path.exists(storepath): os.makedirs(storepath) file_name = 'data_split_niid_'+ str(K)+'.pt' # torch.save({'datapoints': datapoints, 'histograms': class_histogram, # 'class_statitics': class_stats}, storepath + file_name) return datapoints ================================================ FILE: federa/server/src/server.py ================================================ from .client_manager import ClientManager from .client_connection_servicer import ClientConnectionServicer from .verification import verify from .server_evaluate import server_eval from .distribution import data_distribution from .server_lib import get_data import grpc from grpc import ssl_server_credentials from . import ClientConnection_pb2_grpc from concurrent import futures import os import json import threading import torch from datetime import datetime #the business logic of the server, i.e what interactions take place with the clients def server_runner(client_manager, configurations): print("\nServer Running") #get hyperparameters from the passed configurations dict config_dict = {"message": "eval"} algorithm = configurations["algorithm"] num_of_clients = configurations["num_of_clients"] fraction_of_clients = configurations["fraction_of_clients"] clients = client_manager.random_select(num_of_clients, fraction_of_clients) communRound = configurations["num_of_rounds"] initial_model_path = configurations["initial_model_path"] server_model_state_dict = torch.load(initial_model_path, map_location="cpu") epochs = configurations["epochs"] accept_conn_after_FL_begin = configurations["accept_conn_after_FL_begin"] verification = configurations["verify"] verification_threshold = configurations["verification_threshold"] timeout = configurations["timeout"] dataset = configurations["dataset"] net = configurations["net"] resize_size = configurations["resize_size"] batch_size = configurations["batch_size"] niid = configurations["niid"] carbon=configurations["carbon"] #create a new directory inside FL_checkpoints and store the aggragted models in each round fl_timestamp = f"{datetime.now().strftime('%Y-%m-%d %H-%M-%S')}" save_dir_path=f"server_results/{dataset}/{algorithm}/{niid}/{fl_timestamp}" if not os.path.exists(save_dir_path): os.makedirs(save_dir_path) torch.save(server_model_state_dict, f"{save_dir_path}/initial_model.pt") myJSON = json.dumps(configurations) json_path = save_dir_path + "/information.json" with open(json_path, "w", encoding='UTF-8') as jsonfile: jsonfile.write(myJSON) #create new file inside FL_results to store training results with open(f"{save_dir_path}/FL_results.txt", "w", encoding='UTF-8') as file: pass #Initialize the aggregation algorithm exec(f"from .algorithms.{algorithm} import {algorithm}") # nosec aggregator = eval(algorithm)(configurations) # nosec #If the algorithm is either scaffold or mimelite, then we need to make use of control variate if algorithm in ('scaffold', 'mimelite'): control_variate = [torch.zeros_like(server_model_state_dict[key]) for key in server_model_state_dict.keys() ] #At initialization, control variate should be zero as mentioned in the paper control_variate2 = None elif algorithm == 'mime': control_variate = [torch.zeros_like(server_model_state_dict[key]) for key in server_model_state_dict.keys()] control_variate2 = [torch.zeros_like(server_model_state_dict[key]) for key in server_model_state_dict.keys()] else: control_variate = None control_variate2 = None #run FL for given rounds _, trainset = get_data(configurations) datapoints = data_distribution(configurations, trainset, client_manager.num_connected_clients()) client_manager.accepting_connections = accept_conn_after_FL_begin config_dict = {"epochs": epochs, "timeout": timeout, "algorithm":algorithm, "message":"train", "dataset":dataset, "net":net, "resize_size":resize_size, "batch_size":batch_size, "niid": niid, "carbon-tracker":carbon, "datapoints":datapoints} for round in range(1, communRound + 1): clients = client_manager.random_select(client_manager.num_connected_clients(), fraction_of_clients) print(f"\nCR {round}/{communRound} with {len(clients)}/{client_manager.num_connected_clients()} client(s)") trained_model_state_dicts = [] updated_control_variates = [] with futures.ThreadPoolExecutor(max_workers=5) as executor: result_futures = {executor.submit( client.train, server_model_state_dict, control_variate, control_variate2, config_dict ) for client in clients} for client_index, result_future in zip(range(len(clients)), futures.as_completed(result_futures)): trained_model_state_dict, updated_control_variate, results = result_future.result() trained_model_state_dicts.append(trained_model_state_dict) updated_control_variates.append(updated_control_variate) print(f"Training results (client {clients[client_index].client_id}): ", results) if verification: print("Performing verification round...") if algorithm in ('fedavg','feddyn','mime','mimelite'): selected_state_dicts, selected_control_variates = verify(clients, trained_model_state_dicts, save_dir_path, verification_threshold, updated_control_variates) else: selected_state_dicts, selected_control_variates = verify(clients, trained_model_state_dicts, save_dir_path, verification_threshold, updated_control_variates, server_model_state_dict) print(f"\nAggregating {len(selected_state_dicts)}/{len(trained_model_state_dicts)} clients above threshold") else: selected_state_dicts = trained_model_state_dicts selected_control_variates = updated_control_variates #aggregate model, save it, then send to some client to evaluate#aggregate model, save it, # then send to some client to evaluate if control_variate2: server_model_state_dict, control_variate, control_variate2 = aggregator.aggregate(server_model_state_dict, control_variate, selected_state_dicts, selected_control_variates) elif control_variate: server_model_state_dict, control_variate = aggregator.aggregate(server_model_state_dict, control_variate, selected_state_dicts, selected_control_variates) else: server_model_state_dict = aggregator.aggregate(server_model_state_dict,selected_state_dicts) torch.save(server_model_state_dict, f"{save_dir_path}/round_{round}_aggregated_model.pt") #test on server test set print("Evaluating on server test set...") eval_result = server_eval(server_model_state_dict, configurations) eval_result["round"] = round print("Eval results: ", eval_result) #store the results with open(f"{save_dir_path}/FL_results.txt", "a", encoding='UTF-8') as file: file.write( str(eval_result) + "\n" ) #sync all connected clients with current global model and order them to disconnect for client in client_manager.random_select(): client.set_parameters(server_model_state_dict) client.disconnect() torch.save(server_model_state_dict, initial_model_path) print("Server runner stopped.") #starts the gRPC server and then runs server_runner concurrently def server_start(configurations): client_manager = ClientManager() client_connection_servicer = ClientConnectionServicer(client_manager) channel_opt = [('grpc.max_send_message_length', -1), ('grpc.max_receive_message_length', -1)] server = grpc.server(futures.ThreadPoolExecutor(max_workers=10), options=channel_opt) ClientConnection_pb2_grpc.add_ClientConnectionServicer_to_server( client_connection_servicer, server ) if configurations['encryption']==1: # Load the server's private key and certificate keyfile = configurations['server_key'] certfile = configurations['server_cert'] private_key = bytes(open(keyfile).read(), 'utf-8') certificate_chain = bytes(open(certfile).read(), 'utf-8') # Create SSL/TLS credentials object server_credentials = ssl_server_credentials([(private_key, certificate_chain)]) server.add_secure_port('localhost:8214', server_credentials) else: server.add_insecure_port('localhost:8214') server.start() server_runner_thread = threading.Thread(target = server_runner, args = (client_manager, configurations, )) server_runner_thread.start() server_runner_thread.join() server.stop(None) ================================================ FILE: federa/server/src/server_evaluate/__init__.py ================================================ from .eval_lib import server_eval ================================================ FILE: federa/server/src/server_evaluate/eval_lib.py ================================================ import torch from ..server_lib import load_data, get_net, test_model def server_eval(model_state_dict, config): device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") testloader, _ = load_data(config) model = get_net(config) model = model.to(device) model.load_state_dict(model_state_dict) eval_loss, eval_accuracy = test_model(model, testloader) eval_results = {"eval_loss": eval_loss, "eval_accuracy": eval_accuracy} return eval_results ================================================ FILE: federa/server/src/server_lib.py ================================================ import torch import os from tqdm import tqdm from torchvision import transforms,datasets from torch.utils.data import DataLoader from torch import nn from torchvision import models from torch.utils import data import numpy as np from PIL import Image device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") #serverlib and eval_lib should be on the same device def load_data(config): testset, _ = get_data(config) testloader = DataLoader(testset, batch_size=config['batch_size']) num_examples = {"testset": len(testset)} return testloader, num_examples ### Load different dataset def get_data(config): dataset_path="./server_dataset" if not os.path.exists(dataset_path): os.makedirs(dataset_path) if config['dataset'] == 'MNIST': apply_transform = transforms.Compose([transforms.Resize(config['resize_size']), transforms.ToTensor()]) testset = datasets.MNIST(root='./server_dataset/MNIST', train=False, download=True, transform=apply_transform) trainset = datasets.MNIST(root='./server_dataset/MNIST', train=True, download=True, transform=apply_transform) if config['dataset'] == 'FashionMNIST': apply_transform = transforms.Compose([transforms.Resize(config['resize_size']), transforms.ToTensor()]) testset = datasets.FashionMNIST(root='./server_dataset/FashionMNIST', train=False, download=True, transform=apply_transform) trainset = datasets.FashionMNIST(root='./server_dataset/FashionMNIST', train=True, download=True, transform=apply_transform) if config['dataset'] == 'CIFAR10': apply_transform = transforms.Compose([transforms.Resize(config['resize_size']), transforms.ToTensor()]) testset = datasets.CIFAR10(root='./server_dataset/CIFAR10', train=False, download=True, transform=apply_transform) trainset = datasets.CIFAR10(root='./server_dataset/CIFAR10', train=True, download=True, transform=apply_transform) if config['dataset'] == 'CIFAR100': apply_transform = transforms.Compose([transforms.Resize(config['resize_size']), transforms.ToTensor()]) testset = datasets.CIFAR100(root='./server_dataset/CIFAR100', train=False, download=True, transform=apply_transform) trainset = datasets.CIFAR100(root='./server_dataset/CIFAR100', train=True, download=True, transform=apply_transform) if config['dataset'] == 'CUSTOM': apply_transform = transforms.Compose([transforms.Resize(config['resize_size']), transforms.ToTensor()]) testset = customDataset(root='./server_custom_dataset/CUSTOM/test', transform=apply_transform) trainset = customDataset(root='./server_custom_dataset/CUSTOM/train', transform=apply_transform) return testset, trainset class customDataset(data.Dataset): def __init__(self, root, transform=None): self.root = root samples = sample_return(root) self.samples = samples self.transform = transform def __getitem__(self, index): img, label= self.samples[index] img = np.load(img) img = Image.fromarray(img) if self.transform is not None: img = self.transform(img) return img, label def __len__(self): return len(self.samples) def sample_return(root): newdataset = [] labels = {'Breast': 0, 'Chestxray':1, 'Oct': 2, 'Tissue': 3} for image in os.listdir(root): label=[] #print(image) path = os.path.join(root, image) #print(path) labels_str = image.split('_')[0] label = labels[labels_str] item = (path, label) newdataset.append(item) return newdataset class LeNet(nn.Module): def __init__(self, in_channels=1, num_classes=10): super().__init__() self.conv1 = nn.Conv2d(in_channels, 6, kernel_size=5) self.pool1 = nn.MaxPool2d(kernel_size=2,stride=2) self.conv2 = nn.Conv2d(6, 16, kernel_size=5) self.pool2 = nn.MaxPool2d(kernel_size=2,stride=2) self.fc1 = nn.Linear(400, 120) self.fc2 = nn.Linear(120, 84) self.fc3 = nn.Linear(84, num_classes) self.relu = nn.ReLU() self.logSoftmax = nn.LogSoftmax(dim=1) def forward(self, x): x = self.conv1(x) x = self.relu(x) x = self.pool1(x) x = self.conv2(x) x = self.relu(x) x = self.pool2(x) x = x.view(-1, 400) x = self.fc1(x) x = self.relu(x) x = self.fc2(x) x = self.relu(x) x = self.fc3(x) x = self.logSoftmax(x) return x def get_net(config): if config["net"] == 'LeNet': if config['dataset'] in ['MNIST', 'FashionMNIST', 'CUSTOM']: net = LeNet(in_channels=1, num_classes=10) elif config['dataset'] == 'CIFAR10': net = LeNet(in_channels=3, num_classes=10) else: net = LeNet(in_channels=3, num_classes=100) if config["net"] == 'resnet18': if config['dataset'] == 'CIFAR10': net = models.resnet18(num_classes=10) else: net = models.resnet18(num_classes=100) if config["net"] == 'resnet50': if config['dataset'] == 'CIFAR10': net = models.resnet50(num_classes=10) else: net = models.resnet50(num_classes=100) if config["net"] == 'vgg16': if config['dataset'] == 'CIFAR10': net = models.vgg16(num_classes=10) else: net = models.vgg16(num_classes=100) if config['net'] == 'AlexNet': if config['dataset'] == 'CIFAR10': net = models.alexnet(num_classes=10) else: net = models.alexnet(num_classes=100) return net def train_model(net, trainloader): criterion = torch.nn.CrossEntropyLoss() optimizer = torch.optim.SGD(net.parameters(), lr=0.001, momentum=0.9) net.train() dataiter = iter(trainloader) images, labels = next(dataiter) outputs = net(images) optimizer.zero_grad() loss = criterion(outputs, labels) loss.backward() optimizer.step() return net def test_model(net, testloader): criterion = torch.nn.CrossEntropyLoss() correct, total, loss = 0, 0, 0.0 net.eval() with torch.no_grad(): for images, labels in tqdm(testloader) : images, labels = images.to(device), labels.to(device) outputs = net(images) loss += criterion(outputs, labels).item() _, predicted = torch.max(outputs.data, 1) total += labels.size(0) correct += (predicted == labels).sum().item() loss /= len(testloader.dataset) accuracy = correct / total return loss, accuracy def save_intial_model(config): testloader, _ = load_data(config) net = get_net(config) net = train_model(net, testloader) torch.save(net.state_dict(), 'initial_model.pt') ================================================ FILE: federa/server/src/verification.py ================================================ from random import randint from collections import OrderedDict from concurrent import futures import copy ##modify the verify function to consider the updated control variates also def verify(clients, trained_model_state_dicts, save_dir_path, threshold = 0, updated_control_variates = None, server_model_state_dict = None): verification_dict = OrderedDict() config_dict = {"message": "verify"} ##if server_model_state_dict is not None then to each trained_model_state_dict, we need to add the server_model_state_dict if server_model_state_dict is not None: for i in range(len(trained_model_state_dicts)): for key in server_model_state_dict.keys(): trained_model_state_dicts[i][key] += server_model_state_dict[key] for i, client in zip( range(len(clients)), clients): verification_dict[client.client_id] = {"client_wrapper_object": client, "model": trained_model_state_dicts[i], "control_variates": updated_control_variates[i]} client_ids = list(verification_dict.keys()) client_ids_shuffled = random_derangement(client_ids) for i, client_id in zip( range(len(verification_dict)), verification_dict.keys() ): verification_dict[client_id]["assigned_client_id"] = client_ids_shuffled[i] with futures.ThreadPoolExecutor(max_workers = 20) as executor: result_futures = [] for client_id, client_info in verification_dict.items(): assigned_client_id = client_info["assigned_client_id"] assigned_client = verification_dict[assigned_client_id]["client_wrapper_object"] model_to_verify = client_info["model"] config_dict['client_id'] = client_id config_dict_s = copy.deepcopy(config_dict) result_futures.append(executor.submit(assigned_client.evaluate, model_to_verify, config_dict_s)) verification_results = [result_future.result() for result_future in futures.as_completed(result_futures)] for index in range(len(verification_results)): verification_dict[verification_results[index]["client_id"]]["score"] = verification_results[index]["eval_accuracy"] selected_client_models, ignored_client_models, selected_control_variates = [], [], [] for client_id, client_info in verification_dict.items(): if client_info["score"] >= threshold: selected_client_models.append(client_info["model"]) selected_control_variates.append(client_info["control_variates"]) client_info["selected"] = True else: ignored_client_models.append(verification_dict[client_id]["model"]) verification_dict[client_id]["selected"] = False #saves the client_id, its score, which client verified and fraction for the selected and ignored clients results_to_store = [] for client_id, client_info in verification_dict.items(): dict_to_store = { "client_id": client_id, "assigned_client_id": client_info["assigned_client_id"], "score": client_info["score"], "selected": client_info["selected"] } results_to_store.append(dict_to_store) selected_clients = [ client_dict for client_dict in results_to_store if client_dict["selected"] ] ignored_clients = [ client_dict for client_dict in results_to_store if not client_dict["selected"] ] num_of_selected_clients = len(selected_clients) num_of_ignored_clients = len(ignored_clients) num_of_total_clients = len(results_to_store) selected_info_dict = { "threshold": threshold, "selected": f"{num_of_selected_clients}/{num_of_total_clients}", "results": selected_clients } ignored_info_dict = { "threshold": threshold, "ignored": f"{num_of_ignored_clients}/{num_of_total_clients}", "results": ignored_clients } with open(f"{save_dir_path}/verification_selected_stats.txt", "a", encoding='UTF-8') as file: file.write( f"{selected_info_dict}\n" ) with open(f"{save_dir_path}/verification_ignored_stats.txt", "a", encoding='UTF-8') as file: file.write( f"{ignored_info_dict}\n" ) if server_model_state_dict is not None: for i in range(len(selected_client_models)): for key in server_model_state_dict.keys(): selected_client_models[i][key] -= server_model_state_dict[key] return selected_client_models, selected_control_variates def random_derangement(list_to_shuffle): for index1 in range(1, len(list_to_shuffle)): index2 = randint(0, index1 - 1) # nosec list_to_shuffle[index1], list_to_shuffle[index2] = list_to_shuffle[index2], list_to_shuffle[index1] return list_to_shuffle ================================================ FILE: federa/server/start_server.py ================================================ import argparse from .src.server import server_start from .src.server_lib import save_intial_model '# the parameters that can be passed while starting the server' parser = argparse.ArgumentParser() parser.add_argument('--algorithm', type= str, default = 'scaffold', help= 'Aggregation algorithm') parser.add_argument('--clients', type= int, default = 1, help= '#of clients to start') parser.add_argument('--fraction', type = float, default = 1, help = '''Fraction of clients to select out of the number provided or those available. Float between 0 to 1 inclusive''') parser.add_argument('--rounds', type= int, default = 1, help = 'Total number of CR') parser.add_argument('--model_path', default = 'initial_model.pt', help = "The path of the initial server model's state dict") parser.add_argument('--epochs', type = int, default = 1, help= '#of epochs for training') parser.add_argument('--accept_conn',type = int, default = 1, help = '''1, connections accpeted even after FL has begun, else 0.''') parser.add_argument('--verify', type = int, default = 0, help= '1 for True or 0') parser.add_argument('--threshold',type = float,default = 0, help = '''Minimum score clients must have in a verification round, .[0,1]''') parser.add_argument('--timeout', type = int, default=None, help= 'Time limit for training. Specified in seconds') parser.add_argument('--resize_size', type = int, default = 32, help= 'resize dimension') parser.add_argument('--batch_size', type = int, default = 32, help= 'batch size') parser.add_argument('--net', type = str, default = 'LeNet', help= 'client network') parser.add_argument('--dataset', type = str, default= 'MNIST', help= 'datsset.Use CUSTOME for local dataset') parser.add_argument('--niid', type = int, default= 1, help= 'value should be [1, 5]') parser.add_argument('--carbon', type = int, default= 0, help= '1 enable carbon emission at client') parser.add_argument('--encryption', type = int, default= 0, help= '1 enables ssl encryption') parser.add_argument('--server_key', type = str, default= 'server-key.pem', help= 'path to server key') parser.add_argument('--server_cert', type = str, default= 'server.pem', help= 'path to server certificate') args = parser.parse_args() configurations = { "algorithm": args.algorithm, "num_of_clients": args.clients, "fraction_of_clients": args.fraction, "num_of_rounds": args.rounds, "initial_model_path": args.model_path, "epochs": args.epochs, "accept_conn_after_FL_begin": args.accept_conn, "verify": args.verify, "verification_threshold": args.threshold, "timeout": args.timeout, "resize_size": args.resize_size, "batch_size": args.batch_size, "net": args.net, "dataset": args.dataset, "niid": args.niid, "carbon":args.carbon, "encryption": args.encryption, "server_key": args.server_key, "server_cert": args.server_cert } '# start the server with the given parameters' if __name__ == '__main__': save_intial_model(configurations) server_start(configurations) ================================================ FILE: federa/tests/__init__.py ================================================ ================================================ FILE: federa/tests/minitest.json ================================================ { "fedavg":{ "server": { "algorithm":"fedavg", "num_of_clients":1, "fraction_of_clients":1, "num_of_rounds":1, "initial_model_path":"initial_model.pt", "epochs":1, "accept_conn_after_FL_begin":0 , "verify": 0, "verification_threshold": 0, "timeout": null, "resize_size": 32, "batch_size": 32, "net": "LeNet", "dataset": "MNIST", "device": "cpu", "niid": 2, "carbon": 0, "encryption": 0, "server_key": null, "server_cert": null }, "client": { "ip_address": "localhost:8214", "device":"cpu", "wait_time": 10, "encryption": 0, "ca": null } }, "fedadam":{ "server":{ "algorithm":"fedadam", "num_of_clients":1, "fraction_of_clients":1, "num_of_rounds":1, "initial_model_path":"initial_model.pt", "epochs":1, "accept_conn_after_FL_begin":0 , "verify": 0, "verification_threshold": 0, "timeout": null, "resize_size": 32, "batch_size": 32, "net": "resnet18", "dataset": "CIFAR100", "device": "cpu", "niid": 2, "carbon": 0, "encryption": 0, "server_key": null, "server_cert": null }, "client":{ "ip_address": "localhost:8214", "device":"cpu", "wait_time": 10, "encryption": 0, "ca": null } }, "verification":{ "server":{ "algorithm":"fedavg", "num_of_clients":2, "fraction_of_clients":1, "num_of_rounds":1, "initial_model_path":"initial_model.pt", "epochs":1, "accept_conn_after_FL_begin":0 , "verify": 1, "verification_threshold": 0.1, "timeout": null, "resize_size": 32, "batch_size": 32, "net": "resnet50", "dataset": "CIFAR10", "device": "cpu", "niid": 2, "carbon": 0, "encryption": 0, "server_key": null, "server_cert": null }, "client":{ "ip_address": "localhost:8214", "device":"cpu", "wait_time": 10, "encryption": 0, "ca": null } }, "timeout":{ "server":{ "algorithm":"fedavg", "num_of_clients":2, "fraction_of_clients":1, "num_of_rounds":1, "initial_model_path":"initial_model.pt", "epochs":1, "accept_conn_after_FL_begin":0 , "verify": 0, "verification_threshold": 0, "timeout": 60, "resize_size": 32, "batch_size": 32, "net": "LeNet", "dataset": "FashionMNIST", "device": "cpu", "niid": 2, "carbon": 0, "encryption": 0, "server_key": null, "server_cert": null }, "client":{ "ip_address": "localhost:8214", "device":"cpu", "wait_time": 10, "encryption": 0, "ca": null } }, "intermediate":{ "server":{ "algorithm":"fedavg", "num_of_clients":1, "fraction_of_clients":1, "num_of_rounds":2, "initial_model_path":"initial_model.pt", "epochs":1, "accept_conn_after_FL_begin":1 , "verify": 0, "verification_threshold": 0, "timeout": null, "resize_size": 224, "batch_size": 32, "net": "AlexNet", "dataset": "CIFAR10", "device": "cpu", "niid": 2, "carbon": 0, "encryption": 0, "server_key": null, "server_cert": null }, "client":{ "ip_address": "localhost:8214", "device":"cpu", "wait_time": 10, "encryption": 0, "ca": null } } } ================================================ FILE: federa/tests/minitest.py ================================================ import unittest import os import sys from .misc import get_config, tester from ..server.src.server_lib import save_intial_model import logging logging.basicConfig(filename='test_results.log', level=logging.INFO) def create_train_test_for_fedavg(): """ Verify the FedAvg aggregation algorithm using two clients by implementing the following function. """ class TrainerTest(unittest.TestCase): @classmethod def setUpClass(cls): config = get_config('minitest', 'fedavg') save_intial_model(config['server']) def test_fedavg(self): print("\n==Fed Avg==") config = get_config('test_algorithms', 'fedavg') tester(config, 1) logging.info("Test 1 passed\n\tFedAvg algorithm ran successfully\n\tMNIST dataset ran successfully\n\tLeNet model ran succesfully\n") return TrainerTest def create_train_test_for_fedadam(): """ Verify the FedAdam aggregation algorithm using two clients by implementing the following function. """ class TrainerTest(unittest.TestCase): @classmethod def setUpClass(cls): config = get_config('minitest', 'fedadam') save_intial_model(config['server']) def test_fedadam(self): print("\n==Fed Adam==") config = get_config('test_algorithms', 'fedadam') tester(config, 1) logging.info("Test 2 passed\n\tFedAdam algorithm ran successfully\n\tCIFAR100 dataset ran successfully\n\tResnet-18 model ran succesfully\n") return TrainerTest def create_train_test_for_verification_module(): """ Verify the verification module using two clients by implementing the following function. """ class TrainerTest(unittest.TestCase): @classmethod def setUpClass(cls): config = get_config('minitest', 'verification') save_intial_model(config['server']) def test_verification_module(self): print('\n==Verfication Module Testing==') config = get_config('test_modules', 'verification') tester(config, 2) logging.info("Test 3 passed\n\tVerification module ran successfully\n\CIFAR10 dataset ran successfully\n\tResnet-50 model ran succesfully\n") return TrainerTest def create_train_test_for_timeout_module(): """ Verify the timeout module using two clients by implementing the following function. """ class TrainerTest(unittest.TestCase): @classmethod def setUpClass(cls): config = get_config('minitest', 'timeout') save_intial_model(config['server']) def test_timeout_module(self): print('\n==Timeout Module Testing==') config = get_config('test_modules', 'timeout') tester(config, 2) logging.info("Test 4 passed\n\tTimeout module ran successfully\n\tFashionMNIST dataset ran successfully\n\tLeNet model ran succesfully\n") return TrainerTest def create_train_test_for_intermediate_connection_module(): """ Verify the itermeidate connection module using two clients by implementing the following function. """ class TrainerTest(unittest.TestCase): @classmethod def setUpClass(cls): config = get_config('minitest', 'intermediate') save_intial_model(config['server']) def test_intermediate_module(self): print('\n==Intermediate Client Module Testing==') config = get_config('test_modules', 'intermediate') tester(config, 2, late=True) logging.info("Test 5 passed\n\tIntermediate client module ran successfully\n\tCIFAR10 dataset ran successfully\n\tAlexNet model ran succesfully\n") return TrainerTest class TestTrainer_verification(create_train_test_for_verification_module()): 'Test case for verification module' class TestTrainer_timeout(create_train_test_for_timeout_module()): 'Test case for timeout module' class TestTrainer_intermediate(create_train_test_for_intermediate_connection_module()): 'Test case for intermediate client connections module' class TestTrainer_fedavg(create_train_test_for_fedavg()): 'Test case for FedAvg' class TestTrainer_fedadam(create_train_test_for_fedadam()): 'Test case for FedAdam' if __name__ == '__main__': unittest.main() ================================================ FILE: federa/tests/misc.py ================================================ import os import sys import time import json from torch.multiprocessing import Process from torch import multiprocessing from ..server.src.server import server_start from ..client.src.client import client_start def get_config(action, action2, config_path=""): """ Get the configuration file as json from it """ root_path = os.path.dirname(os.path.realpath(__file__)) config_path = os.path.join(root_path, '') action = action + '.json' with open(os.path.join(config_path, action), encoding='UTF-8') as f1: config = json.load(f1) config = config[action2] return config def tester(configs , no_of_clients, late=None): """ Return the tester to each test algorithm. Late is introduced for intermediate connection """ multiprocessing.set_start_method('spawn', force=True) if late: no_of_clients -= 1 server = Process(target=server_start, args=(configs['server'],)) clients = [] server.start() time.sleep(5) for i in range(no_of_clients): client = Process(target=client_start, args=(configs['client'],)) clients.append(client) client.start() time.sleep(2) if late: time.sleep(3) client = Process(target=client_start, args=(configs['client'],)) clients.append(client) client.start() clients_list = list(range(len(clients))) for i in clients_list: clients[i].join() server.join() ================================================ FILE: requirements.txt ================================================ codecarbon==2.1.4 grpcio==1.53.0 numpy==1.23.4 Pillow==9.5.0 protobuf==3.20.2 torch>=2.0.0 torchvision>=0.15.1 tqdm==4.65.0 bandit matplotlib ================================================ FILE: ssl/README.md ================================================ # SSL Configuration ## CFSSL Integration This section provides instructions on using the CFSSL toolkit in conjunction with the files provided in this repository. To obtain the CFSSL toolkit, please visit the [CFSSL Website](https://cfssl.org/). ## File Customization Please note that the files in this directory should be customized with your own details, particularly the `ca-config.json` and `ca-csr.json` files. While minimal modifications are sufficient for basic testing purposes, it is recommended to update these files to align with your specific requirements. ## Certificate Authority Generation To generate the Certificate Authority (CA) files, execute the following command: ```sh cfssl gencert -initca ca-csr.json | cfssljson -bare ca ``` This command will generate the `ca.pem` and `ca-key.pem` files. These files are utilized for generating client and server certificates. The `ca.pem` file is used for mutual verification between clients and servers. ## Client Certificate Generation To generate a client certificate, use the following command: ```sh cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json client-csr.json | cfssljson -bare client ``` This command will generate the `client.pem` and `client-key.pem` files. **_Note:_** A warning message may appear during the execution of this command, indicating the lack of a "hosts" field in the certificate. However, for client certificates, the absence of this field is acceptable as they are not intended for use as servers. ## Server Certificate Generation To generate a server certificate, execute the following command: ```sh cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -hostname= server-csr.json | cfssljson -bare server ``` This command will generate the `server.pem` and `server-key.pem` files. ## Acknowledgements The code and information in this directory were developed with the help of the repository [joekottke/python-grpc-ssl](https://github.com/joekottke/python-grpc-ssl), which provided valuable guidance in implementing the encryption functionality. ================================================ FILE: ssl/ca-config.json ================================================ { "signing": { "profiles": { "default": { "usages": ["signing", "key encipherment", "server auth", "client auth"], "expiry": "8760h" } } } } ================================================ FILE: ssl/ca-csr.json ================================================ { "CN": "Example CA", "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "US", "L": "San Francisco", "O": "Example", "OU": "CertificateAuthority", "ST": "California" } ] } ================================================ FILE: ssl/client-csr.json ================================================ { "CN": "TestClient", "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "US", "L": "San Francisco", "O": "Example", "OU": "SRE-Operations", "ST": "California" } ] } ================================================ FILE: ssl/server-csr.json ================================================ { "CN": "server.example.com", "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "US", "L": "San Francisco", "O": "Example", "OU": "SRE-Operations", "ST": "California" } ] } ================================================ FILE: test/__init__.py ================================================ ================================================ FILE: test/benchtest/__init__.py ================================================ ================================================ FILE: test/benchtest/test_results.py ================================================ import unittest import os import sys from ..misc import get_config, tester sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from federa.server.src.server_lib import save_intial_model def create_train_test_for_fedavg(): """ Verify the FedAvg aggregation algorithm using two clients by implementing the following function. """ class TrainerTest(unittest.TestCase): @classmethod def setUpClass(cls): config = get_config('test_algorithms', 'fedavg') save_intial_model(config['server']) def test_fedavg(self): print("\n==Fed Avg==") config = get_config('test_algorithms', 'fedavg') tester(config, 1) return TrainerTest def create_train_test_for_fedadagrad(): """ Verify the FedAdagrad aggregation algorithm using two clients by implementing the following function. """ class TrainerTest(unittest.TestCase): @classmethod def setUpClass(cls): config = get_config('test_algorithms', 'fedadagrad') save_intial_model(config['server']) def test_fedadagrad(self): print("\n==Fed Adagrad==") config = get_config('test_algorithms', 'fedadagrad') tester(config, 1) return TrainerTest def create_train_test_for_fedadam(): """ Verify the FedAdam aggregation algorithm using two clients by implementing the following function. """ class TrainerTest(unittest.TestCase): @classmethod def setUpClass(cls): config = get_config('test_algorithms', 'fedadam') save_intial_model(config['server']) def test_fedadam(self): print("\n==Fed Adam==") config = get_config('test_algorithms', 'fedadam') tester(config, 1) return TrainerTest def create_train_test_for_fedavgm(): """ Verify the FedAvgM aggregation algorithm using two clients by implementing the following function. """ class TrainerTest(unittest.TestCase): @classmethod def setUpClass(cls): config = get_config('test_algorithms', 'fedavgm') save_intial_model(config['server']) def test_fedavgm(self): print("\n==Fed Avgm==") config = get_config('test_algorithms', 'fedavgm') tester(config, 1) return TrainerTest def create_train_test_for_feddyn(): """ Verify the FedDyn aggregation algorithm using two clients by implementing the following function. """ class TrainerTest(unittest.TestCase): @classmethod def setUpClass(cls): config = get_config('test_algorithms', 'feddyn') save_intial_model(config['server']) def test_feddyn(self): print("\n==Fed Dyn==") config = get_config('test_algorithms', 'feddyn') tester(config, 1) return TrainerTest def create_train_test_for_fedyogi(): """ Verify the FedYogi aggregation algorithm using two clients by implementing the following function. """ class TrainerTest(unittest.TestCase): @classmethod def setUpClass(cls): config = get_config('test_algorithms', 'fedyogi') save_intial_model(config['server']) def test_fedyogi(self): print("\n==Fed Yogi==") config = get_config('test_algorithms', 'fedyogi') tester(config, 1) return TrainerTest def create_train_test_for_mime(): """ Verify the Mime aggregation algorithm using two clients by implementing the following function. """ class TrainerTest(unittest.TestCase): @classmethod def setUpClass(cls): config = get_config('test_algorithms', 'mime') save_intial_model(config['server']) def test_mime(self): print("\n==Mime==") config = get_config('test_algorithms', 'mime') tester(config, 1) return TrainerTest def create_train_test_for_mimelite(): """ Verify the MimeLite aggregation algorithm using two clients by implementing the following function. """ class TrainerTest(unittest.TestCase): @classmethod def setUpClass(cls): config = get_config('test_algorithms', 'mimelite') save_intial_model(config['server']) def test_mimelite(self): print("\n===MimeLite==") config = get_config('test_algorithms', 'mimelite') tester(config, 1) return TrainerTest def create_train_test_for_scaffold(): """ Verify the Scaffold aggregation algorithm using two clients by implementing the following function. """ class TrainerTest(unittest.TestCase): @classmethod def setUpClass(cls): config = get_config('test_algorithms', 'scaffold') save_intial_model(config['server']) def test_scaffold(self): print("\n==Scaffold==") config = get_config('test_algorithms', 'scaffold') tester(config, 1) return TrainerTest class TestTrainer_fedavg(create_train_test_for_fedavg()): 'Test case for fedavg' class TestTrainer_fedadagrad(create_train_test_for_fedadagrad()): 'Test case for fedadagrad' class TestTrainer_fedadam(create_train_test_for_fedadam()): 'Test case for fedadam' class TestTrainer_fedavgm(create_train_test_for_fedavgm()): 'Test case for fedavgm' class TestTrainer_feddyn(create_train_test_for_feddyn()): 'Test case for feddyn' class TestTrainer_fedyogi(create_train_test_for_fedyogi()): 'Test case for fedyogi' class TestTrainer_mime(create_train_test_for_mime()): 'Test case for mime' class TestTrainer_mimelite(create_train_test_for_mimelite()): 'Test case for mimelite' class TestTrainer_scaffold(create_train_test_for_scaffold()): 'Test case for scaffold' if __name__ == '__main__': unittest.main() ================================================ FILE: test/benchtest/test_scalability.py ================================================ import os import sys import unittest sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from federa.server.src.server_lib import save_intial_model from ..misc import get_config, tester def create_train_test_for_four_clients(): """ Verify the scalability of clients using four clients by implementing the following function """ class TrainerTest(unittest.TestCase): @classmethod def setUpClass(cls): config = get_config('test_scalability', '4') save_intial_model(config['server']) def test_four_clients(self): print("\n== Testing for #client=4 ==") config = get_config('test_scalability', '4') tester(config, 4) return TrainerTest def create_train_test_for_six_clients(): """ Verify the scalability of clients using six clients by implementing the following function """ class TrainerTest(unittest.TestCase): @classmethod def setUpClass(cls): config = get_config('test_scalability', '6') save_intial_model(config['server']) def test_six_clients(self): print("\n== Testing for #client=6 ==") config = get_config('test_scalability', '6') tester(config, 6) return TrainerTest def create_train_test_for_five_rounds(): """ Verify the scalability of CR rounds using five round by implementing the following function """ class TrainerTest(unittest.TestCase): @classmethod def setUpClass(cls): config = get_config('test_scalability', '5_rounds') save_intial_model(config['server']) def test_five_communication_rounds(self): print("\n== Testing for Communication Rounds=5 ==") config = get_config('test_scalability', '5_rounds') tester(config, 2) return TrainerTest def create_train_test_for_ten_rounds(): """ Verify the scalability of CR rounds using ten round by implementing the following function """ class TrainerTest(unittest.TestCase): @classmethod def setUpClass(cls): config = get_config('test_scalability', '10_rounds') save_intial_model(config['server']) def test_ten_communication_rounds(self): print("\n== Testing for Communication Rounds=10 ==") config = get_config('test_scalability', '10_rounds') tester(config, 2) return TrainerTest class TestTrainer_4(create_train_test_for_four_clients()): 'Test case for four clients' class TestTrainer_6(create_train_test_for_six_clients()): 'Test case for six clients' class TestTrainer_5_rounds(create_train_test_for_five_rounds()): 'Test case for five communication rounds' class TestTrainer_10_rounds(create_train_test_for_ten_rounds()): 'Test case for ten communication rounds' if __name__ == '__main__': unittest.main() ================================================ FILE: test/misc.py ================================================ import os import sys import time import json from torch.multiprocessing import Process from torch import multiprocessing sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) from federa.server.src.server import server_start from federa.client.src.client import client_start def get_config(action, action2, config_path=""): """ Get the configuration file as json from it """ root_path = os.path.dirname( os.path.dirname(os.path.realpath(__file__))) config_path = os.path.join(root_path, 'configs') action = action + '.json' with open(os.path.join(config_path, action), encoding='UTF-8') as f1: config = json.load(f1) config = config[action2] return config def execute(process): os.system(f'{process}') def tester(configs , no_of_clients, late=None): """ Return the tester to each test algorithm. Late is introduced for intermediate connection """ multiprocessing.set_start_method('spawn', force=True) if late: no_of_clients -= 1 server = Process(target=server_start, args=(configs['server'],)) clients = [] server.start() time.sleep(5) for i in range(no_of_clients): client = Process(target=client_start, args=(configs['client'],)) clients.append(client) client.start() time.sleep(2) if late: time.sleep(3) client = Process(target=client_start, args=(configs['client'],)) clients.append(client) client.start() clients_list = list(range(len(clients))) for i in clients_list: clients[i].join() server.join() def get_result(dataset, algorithm): """ Return the result to each test algorithm. Dataset and algorithm defines as for which dataset the result is required """ dir_path = './server_results/'+dataset+'/'+algorithm lst = os.listdir(dir_path) lst.sort() lst = lst[-1] dir_path = dir_path+'/'+lst lst = os.listdir(dir_path) lst.sort() lst = lst[-1] print(lst) with open (f'{dir_path}/{lst}/FL_results.txt', 'r', encoding='UTF-8') as file: for line in file: pass result_dict = eval(line) return result_dict ================================================ FILE: test/unittest/__init__.py ================================================ ================================================ FILE: test/unittest/test_algorithms.py ================================================ import unittest import os import sys from ..misc import get_config, tester sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) from federa.server.src.server_lib import save_intial_model def create_train_test_for_fedavg(): """ Verify the FedAvg aggregation algorithm using two clients by implementing the following function. """ class TrainerTest(unittest.TestCase): @classmethod def setUpClass(cls): config = get_config('test_algorithms', 'fedavg') save_intial_model(config['server']) def test_fedavg(self): print("\n==Fed Avg==") config = get_config('test_algorithms', 'fedavg') tester(config, 1) return TrainerTest def create_train_test_for_fedadagrad(): """ Verify the FedAdagrad aggregation algorithm using two clients by implementing the following function. """ class TrainerTest(unittest.TestCase): @classmethod def setUpClass(cls): config = get_config('test_algorithms', 'fedadagrad') save_intial_model(config['server']) def test_fedadagrad(self): print("\n==Fed Adagrad==") config = get_config('test_algorithms', 'fedadagrad') tester(config, 1) return TrainerTest def create_train_test_for_fedadam(): """ Verify the FedAdam aggregation algorithm using two clients by implementing the following function. """ class TrainerTest(unittest.TestCase): @classmethod def setUpClass(cls): config = get_config('test_algorithms', 'fedadam') save_intial_model(config['server']) def test_fedadam(self): print("\n==Fed Adam==") config = get_config('test_algorithms', 'fedadam') tester(config, 1) return TrainerTest def create_train_test_for_fedavgm(): """ Verify the FedAvgM aggregation algorithm using two clients by implementing the following function. """ class TrainerTest(unittest.TestCase): @classmethod def setUpClass(cls): config = get_config('test_algorithms', 'fedavgm') save_intial_model(config['server']) def test_fedavgm(self): print("\n==Fed Avgm==") config = get_config('test_algorithms', 'fedavgm') tester(config, 1) return TrainerTest def create_train_test_for_feddyn(): """ Verify the FedDyn aggregation algorithm using two clients by implementing the following function. """ class TrainerTest(unittest.TestCase): @classmethod def setUpClass(cls): config = get_config('test_algorithms', 'feddyn') save_intial_model(config['server']) def test_feddyn(self): print("\n==Fed Dyn==") config = get_config('test_algorithms', 'feddyn') tester(config, 1) return TrainerTest def create_train_test_for_fedyogi(): """ Verify the FedYogi aggregation algorithm using two clients by implementing the following function. """ class TrainerTest(unittest.TestCase): @classmethod def setUpClass(cls): config = get_config('test_algorithms', 'fedyogi') save_intial_model(config['server']) def test_fedyogi(self): print("\n==Fed Yogi==") config = get_config('test_algorithms', 'fedyogi') tester(config, 1) return TrainerTest def create_train_test_for_mime(): """ Verify the Mime aggregation algorithm using two clients by implementing the following function. """ class TrainerTest(unittest.TestCase): @classmethod def setUpClass(cls): config = get_config('test_algorithms', 'mime') save_intial_model(config['server']) def test_mime(self): print("\n==Mime==") config = get_config('test_algorithms', 'mime') tester(config, 1) return TrainerTest def create_train_test_for_mimelite(): """ Verify the MimeLite aggregation algorithm using two clients by implementing the following function. """ class TrainerTest(unittest.TestCase): @classmethod def setUpClass(cls): config = get_config('test_algorithms', 'mimelite') save_intial_model(config['server']) def test_mimelite(self): print("\n===MimeLite==") config = get_config('test_algorithms', 'mimelite') tester(config, 1) return TrainerTest def create_train_test_for_scaffold(): """ Verify the Scaffold aggregation algorithm using two clients by implementing the following function. """ class TrainerTest(unittest.TestCase): @classmethod def setUpClass(cls): config = get_config('test_algorithms', 'scaffold') save_intial_model(config['server']) def test_scaffold(self): print("\n==Scaffold==") config = get_config('test_algorithms', 'scaffold') tester(config, 1) return TrainerTest class TestTrainer_fedavg(create_train_test_for_fedavg()): 'Test case for fedavg' class TestTrainer_fedadagrad(create_train_test_for_fedadagrad()): 'Test case for fedadagrad' class TestTrainer_fedadam(create_train_test_for_fedadam()): 'Test case for fedadam' class TestTrainer_fedavgm(create_train_test_for_fedavgm()): 'Test case for fedavgm' class TestTrainer_feddyn(create_train_test_for_feddyn()): 'Test case for feddyn' class TestTrainer_fedyogi(create_train_test_for_fedyogi()): 'Test case for fedyogi' class TestTrainer_mime(create_train_test_for_mime()): 'Test case for mime' class TestTrainer_mimelite(create_train_test_for_mimelite()): 'Test case for mimelite' class TestTrainer_scaffold(create_train_test_for_scaffold()): 'Test case for scaffold' if __name__ == '__main__': unittest.main() ================================================ FILE: test/unittest/test_datasets.py ================================================ import os import sys import unittest # add main directory to path sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from federa.server.src.server_lib import save_intial_model from ..misc import get_config, tester def create_train_test_for_MNIST(): """Verify the MNIST dataset using one client by implementing the following function""" class TrainerTest(unittest.TestCase): @classmethod def setUpClass(cls): config = get_config('test_datasets', 'MNIST') save_intial_model(config['server']) def test_MNIST(self): print("\n==MNIST Testing==") config = get_config('test_datasets', 'MNIST') tester(config, 1) return TrainerTest def create_train_test_for_FashionMnist(): """Verify the FashionMNIST dataset using one client by implementing the following function""" class TrainerTest(unittest.TestCase): @classmethod def setUpClass(cls): config = get_config('test_datasets', 'FashionMNIST') save_intial_model(config['server']) def test_FashionMnist(self): print("\n==FashionMNIST Testing==") config = get_config('test_datasets', 'FashionMNIST') tester(config, 1) return TrainerTest def create_train_test_for_CIFAR10(): """Verify the CIFAR100 dataset using one client by implementing the following function""" class TrainerTest(unittest.TestCase): @classmethod def setUpClass(cls): config = get_config('test_datasets', 'CIFAR10') save_intial_model(config['server']) def test_CIFAR10(self): print("\n==CIFAR10 Testing==") config = get_config('test_datasets', 'CIFAR10') tester(config, 1) return TrainerTest def create_train_test_for_CIFAR100(): """Verify the CIFAR100 dataset using one client by implementing the following function""" class TrainerTest(unittest.TestCase): @classmethod def setUpClass(cls): config = get_config('test_datasets', 'CIFAR100') save_intial_model(config['server']) def test_CIFAR100(self): print("\n==CIFAR100 Testing==") config = get_config('test_datasets', 'CIFAR100') tester(config, 1) return TrainerTest def create_train_test_for_CUSTOM(): """Verify the CUSTOM dataset using one client by implementing the following function""" class TrainerTest(unittest.TestCase): @classmethod def setUpClass(cls): config = get_config('test_datasets', 'CUSTOM') save_intial_model(config['server']) def test_CUSTOM(self): print("\n==CUSTOM Dataset Testing==") config = get_config('test_datasets', 'CUSTOM') tester(config, 1) return TrainerTest class TestTrainer_MNIST(create_train_test_for_MNIST()): 'Test case for MNIST dataset' class TestTrainer_FashionMNIST(create_train_test_for_FashionMnist()): 'Test case for FashionMNIST dataset' class TestTrainer_CIFAR10(create_train_test_for_CIFAR10()): 'Test case for CIFAR10 dataset' class TestTrainer_CIFAR100(create_train_test_for_CIFAR100()): 'Test case for CIFAR100 dataset' class TestTrainer_CUSTOM(create_train_test_for_CUSTOM()): 'Test case for CUSTOM dataset' if __name__ == '__main__': unittest.main() ================================================ FILE: test/unittest/test_models.py ================================================ import os import sys import unittest # add main directory to path sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from federa.server.src.server_lib import save_intial_model from ..misc import get_config, tester def create_train_test_for_LeNet(): """ Verify the LeNet-5 CNN model using one client by implementing the following function """ class TrainerTest(unittest.TestCase): @classmethod def setUpClass(cls): config = get_config('test_models', 'LeNet') save_intial_model(config['server']) def test_LeNet(self): print("\n==LeNet Testing==") config = get_config('test_models', 'LeNet') tester(config, 1) return TrainerTest def create_train_test_for_resnet18(): """ Verify the ResNet18 CNN model using one client by implementing the following function """ class TrainerTest(unittest.TestCase): @classmethod def setUpClass(cls): config = get_config('test_models', 'resnet18') save_intial_model(config['server']) def test_resnet18(self): print("\n===Resnet18 Testing==") config = get_config('test_models', 'resnet18') tester(config, 1) return TrainerTest def create_train_test_for_resnet50(): """ Verify the ResNet50 CNN model using one client by implementing the following function """ class TrainerTest(unittest.TestCase): @classmethod def setUpClass(cls): config = get_config('test_models', 'resnet50') save_intial_model(config['server']) def test_resnet18(self): print("\n==Resnet50 Testing==") config = get_config('test_models', 'resnet50') tester(config, 1) return TrainerTest def create_train_test_for_vgg16(): """ Verify the VGG16 CNN model using one client by implementing the following function """ class TrainerTest(unittest.TestCase): @classmethod def setUpClass(cls): config = get_config('test_models', 'vgg16') save_intial_model(config['server']) def test_vgg16(self): print("\n==VGG 16 Testing==") config = get_config('test_models', 'vgg16') tester(config, 1) return TrainerTest def create_train_test_for_AlexNet(): """ Verify the AlexNet CNN model using one client by implementing the following function """ class TrainerTest(unittest.TestCase): @classmethod def setUpClass(cls): config = get_config('test_models', 'AlexNet') save_intial_model(config['server']) def test_vgg16(self): print("\n==AlexNet Testing==") config = get_config('test_models', 'AlexNet') tester(config, 1) return TrainerTest class TestTrainer_LeNet(create_train_test_for_LeNet()): 'Test case for LeNet model' class TestTrainer_resnet18(create_train_test_for_resnet18()): 'Test case for resnet18 model' class TestTrainer_resnet50(create_train_test_for_resnet50()): 'Test case for resnet50 model' class TestTrainer_vgg16(create_train_test_for_vgg16()): 'Test case for vgg16 model' class TestTrainer_AlexNet(create_train_test_for_AlexNet()): 'Test case for AlexNet model' if __name__ == '__main__': unittest.main() ================================================ FILE: test/unittest/test_modules.py ================================================ import os import unittest import sys sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from federa.server.src.server_lib import save_intial_model from ..misc import get_config, tester def create_train_test_for_verification_module(): """ Verify the verification module using two clients by implementing the following function. """ class TrainerTest(unittest.TestCase): @classmethod def setUpClass(cls): config = get_config('test_modules', 'verification') save_intial_model(config['server']) def test_verification_module(self): print('\n==Verfication Module Testing==') config = get_config('test_modules', 'verification') tester(config, 2) return TrainerTest def create_train_test_for_timeout_module(): """ Verify the timeout module using two clients by implementing the following function. """ class TrainerTest(unittest.TestCase): @classmethod def setUpClass(cls): config = get_config('test_modules', 'timeout') save_intial_model(config['server']) def test_timeout_module(self): print('\n==Timeout Module Testing==') config = get_config('test_modules', 'timeout') tester(config, 2) return TrainerTest def create_train_test_for_intermediate_connection_module(): """ Verify the itermeidate connection module using two clients by implementing the following function. """ class TrainerTest(unittest.TestCase): @classmethod def setUpClass(cls): config = get_config('test_modules', 'intermediate') save_intial_model(config['server']) def test_intermediate_module(self): print('\n==Intermediate Client Module Testing==') config = get_config('test_modules', 'intermediate') tester(config, 2, late=True) return TrainerTest class TestTrainer_verification(create_train_test_for_verification_module()): 'Test case for verification module' class TestTrainer_timeout(create_train_test_for_timeout_module()): 'Test case for timeout module' class TestTrainer_intermediate(create_train_test_for_intermediate_connection_module()): 'Test case for intermediate client connections module' if __name__ == '__main__': unittest.main() ================================================ FILE: tutorials/Code_Carbon_Tutorial.ipynb ================================================ { "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Import Libraries" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "import sys\n", "import os\n", "from torch import multiprocessing\n", "import time\n", "sys.path.append(os.path.dirname(os.getcwd())) \n", "from server.src.server import server_start\n", "from server.src.server_lib import save_intial_model\n", "from client.src.client import client_start" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Use of Code Carbon Module" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Setup the configuration file" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "1. Different modules can easily be accessed with the help of the config file\n", "2. Carbon emission can easily be monitored with the help of this module\n", "3. Currently emission is tracked at the client side during the training\n", "4. The value of \"carbon\" need to be set as 1 in order to track the carbon emission at the clients\n", "4. Carbon emission result will be printed as well it be saved in a csv file\n", "5. Further there is another python file that take that csv file as input and plot the carbon emission graph over the different CR " ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Details of the config file\n", "1. Number of clients are taken as 2\n", "2. Dataset and model used for local training is MNIST and LeNet-5 respectively\n", "3. Carbon flag is set to 1\n", "4. Verification module and timeout module is off" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "configs ={\n", " \"algorithm\":\"fedavg\",\n", " \"num_of_clients\":2,\n", " \"fraction_of_clients\":1,\n", " \"num_of_rounds\":1,\n", " \"initial_model_path\":\"initial_model.pt\",\n", " \"epochs\":1,\n", " \"accept_conn_after_FL_begin\":1 ,\n", " \"verify\": 0,\n", " \"verification_threshold\": 0,\n", " \"timeout\": None,\n", " \"resize_size\": 32,\n", " \"batch_size\": 32,\n", " \"net\": \"LeNet\",\n", " \"dataset\": \"MNIST\",\n", " \"niid\": 2,\n", " \"carbon\": 1\n", "}" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Start the server\n", "1. Uncomment the multiprocessing line if you are using it on windows platform\n", "2. Output will be shown as the client part starts" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "Server Running\n", "Client ipv4:127.0.0.1:42074 connected.\n", "Client ipv4:127.0.0.1:42090 connected.\n", "\n", "Communication round 1/1 is starting with 2/2 client(s).\n", "Client ipv4:127.0.0.1:42090 has disconnected. Now 1 clients remain active.\n", "Client ipv4:127.0.0.1:42074 has disconnected. Now 0 clients remain active.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Process Process-1:\n", "KeyboardInterrupt\n", "Traceback (most recent call last):\n", " File \"/home/deepsip1/anaconda3/envs/anupam/lib/python3.6/multiprocessing/process.py\", line 258, in _bootstrap\n", " self.run()\n", " File \"/home/deepsip1/anaconda3/envs/anupam/lib/python3.6/multiprocessing/process.py\", line 93, in run\n", " self._target(*self._args, **self._kwargs)\n", " File \"/home/deepsip1/anupam/Framework/Github_25/fl_framework_initial-main/server/src/server.py\", line 138, in server_start\n", " server_runner_thread.join()\n", " File \"/home/deepsip1/anaconda3/envs/anupam/lib/python3.6/threading.py\", line 1056, in join\n", " self._wait_for_tstate_lock()\n", " File \"/home/deepsip1/anaconda3/envs/anupam/lib/python3.6/threading.py\", line 1072, in _wait_for_tstate_lock\n", " elif lock.acquire(block, timeout):\n" ] } ], "source": [ "save_intial_model(configs)\n", "#multiprocessing.set_start_method('spawn', force=False)\n", "server = multiprocessing.Process(target=server_start, args=(configs,))\n", "server.start()\n", "time.sleep(5)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Start the client" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Connected with server\n", "Connected with server\n" ] }, { "ename": "KeyboardInterrupt", "evalue": "", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 8\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mno_of_clients\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 9\u001b[0m \u001b[0mclients\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m \u001b[0mserver\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", "\u001b[0;32m~/anaconda3/envs/anupam/lib/python3.6/multiprocessing/process.py\u001b[0m in \u001b[0;36mjoin\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m 122\u001b[0m \u001b[0;32massert\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_parent_pid\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgetpid\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'can only join a child process'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 123\u001b[0m \u001b[0;32massert\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_popen\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'can only join a started process'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 124\u001b[0;31m \u001b[0mres\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_popen\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 125\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mres\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 126\u001b[0m \u001b[0m_children\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdiscard\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m~/anaconda3/envs/anupam/lib/python3.6/multiprocessing/popen_fork.py\u001b[0m in \u001b[0;36mwait\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m 48\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 49\u001b[0m \u001b[0;31m# This shouldn't block if wait() returned successfully.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 50\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpoll\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mWNOHANG\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mtimeout\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0.0\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 51\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreturncode\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 52\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m~/anaconda3/envs/anupam/lib/python3.6/multiprocessing/popen_fork.py\u001b[0m in \u001b[0;36mpoll\u001b[0;34m(self, flag)\u001b[0m\n\u001b[1;32m 26\u001b[0m \u001b[0;32mwhile\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 27\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 28\u001b[0;31m \u001b[0mpid\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwaitpid\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpid\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mflag\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 29\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mOSError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 30\u001b[0m \u001b[0;31m# Child process not yet created. See #1731717\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;31mKeyboardInterrupt\u001b[0m: " ] } ], "source": [ "no_of_clients = 2\n", "clients = []\n", "for i in range(no_of_clients):\n", " client = multiprocessing.Process(target=client_start)\n", " clients.append(client)\n", " client.start()\n", " time.sleep(2)\n", "for i in range(no_of_clients):\n", " clients[i].join()\n", "server.join()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Another one" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Details of the config file\n", "1. Number of clients are taken as 3\n", "2. Dataset and model used for local training is CIFAR-100 and ResNet-50 respectively\n", "3. Carbon flag is set to 1\n", "4. Verification module and timeout module is off" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "configs ={\n", " \"algorithm\":\"fedadagrad\",\n", " \"num_of_clients\":3,\n", " \"fraction_of_clients\":1,\n", " \"num_of_rounds\":2,\n", " \"initial_model_path\":\"initial_model.pt\",\n", " \"epochs\":1,\n", " \"accept_conn_after_FL_begin\":1 ,\n", " \"verify\": 1,\n", " \"verification_threshold\": 0.8,\n", " \"timeout\": None,\n", " \"resize_size\": 32,\n", " \"batch_size\": 32,\n", " \"net\": \"resnet50\",\n", " \"dataset\": \"CIFAR100\",\n", " \"niid\": 2,\n", " \"carbon\": 0\n", "}" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Start the server and client\n", "1. Uncomment the multiprocessing line if you are using it on windows platform\n", "2. Output will be shown as the client part starts\n", "3. Carbon emission result will be printed as well it be saved in a csv file\n", "4. Further there is another python file that take that csv file as input and plot the carbon emission graph over the different CR " ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "save_intial_model(configs)\n", "#multiprocessing.set_start_method('spawn', force=False)\n", "server = multiprocessing.Process(target=server_start, args=(configs,))\n", "server.start()\n", "time.sleep(5)\n", "no_of_clients = 3\n", "clients = []\n", "for i in range(no_of_clients):\n", " client = multiprocessing.Process(target=client_start)\n", " clients.append(client)\n", " client.start()\n", " time.sleep(2)\n", "for i in range(no_of_clients):\n", " clients[i].join()\n", "server.join()" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.9.16" } }, "nbformat": 4, "nbformat_minor": 2 } ================================================ FILE: tutorials/Federated_Algorithm_Tutorial.ipynb ================================================ { "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Import Libraries" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "import sys\n", "import os\n", "from torch import multiprocessing\n", "import time\n", "sys.path.append(os.path.dirname(os.getcwd())) \n", "from server.src.server import server_start\n", "from server.src.server_lib import save_intial_model\n", "from client.src.client import client_start" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Setup the configuration file" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "1. Choose the \"name\" of the algorithm in \"algorithms\" in configs.\n", "2. There are total 9 different federated learning algorithms pre-implemented\n", "3. Number of clients indicated the minimum amount of clients needed to start the process" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# 1. This is shown for Federated Averaging" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "configs ={\n", " \"algorithm\":\"fedavg\",\n", " \"num_of_clients\":2,\n", " \"fraction_of_clients\":1,\n", " \"num_of_rounds\":1,\n", " \"initial_model_path\":\"initial_model.pt\",\n", " \"epochs\":1,\n", " \"accept_conn_after_FL_begin\":1 ,\n", " \"verify\": 0,\n", " \"verification_threshold\": 0,\n", " \"timeout\": None,\n", " \"resize_size\": 32,\n", " \"batch_size\": 32,\n", " \"net\": \"LeNet\",\n", " \"dataset\": \"MNIST\",\n", " #\"device\": \"cpu\",\n", " \"niid\": 2,\n", " \"carbon\": 0\n", "}" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Start the server\n", "1. Uncomment the multiprocessing line if you are using it on windows platform\n", "2. Output will be shown as the client part starts" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "Server Running\n", "Client ipv4:127.0.0.1:42074 connected.\n", "Client ipv4:127.0.0.1:42090 connected.\n", "\n", "Communication round 1/1 is starting with 2/2 client(s).\n", "Client ipv4:127.0.0.1:42090 has disconnected. Now 1 clients remain active.\n", "Client ipv4:127.0.0.1:42074 has disconnected. Now 0 clients remain active.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Process Process-1:\n", "KeyboardInterrupt\n", "Traceback (most recent call last):\n", " File \"/home/deepsip1/anaconda3/envs/anupam/lib/python3.6/multiprocessing/process.py\", line 258, in _bootstrap\n", " self.run()\n", " File \"/home/deepsip1/anaconda3/envs/anupam/lib/python3.6/multiprocessing/process.py\", line 93, in run\n", " self._target(*self._args, **self._kwargs)\n", " File \"/home/deepsip1/anupam/Framework/Github_25/fl_framework_initial-main/server/src/server.py\", line 138, in server_start\n", " server_runner_thread.join()\n", " File \"/home/deepsip1/anaconda3/envs/anupam/lib/python3.6/threading.py\", line 1056, in join\n", " self._wait_for_tstate_lock()\n", " File \"/home/deepsip1/anaconda3/envs/anupam/lib/python3.6/threading.py\", line 1072, in _wait_for_tstate_lock\n", " elif lock.acquire(block, timeout):\n" ] } ], "source": [ "save_intial_model(configs)\n", "#multiprocessing.set_start_method('spawn', force=False)\n", "server = multiprocessing.Process(target=server_start, args=(configs,))\n", "server.start()\n", "time.sleep(5)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Start the client" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Connected with server\n", "Connected with server\n" ] }, { "ename": "KeyboardInterrupt", "evalue": "", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 8\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mno_of_clients\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 9\u001b[0m \u001b[0mclients\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m \u001b[0mserver\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", "\u001b[0;32m~/anaconda3/envs/anupam/lib/python3.6/multiprocessing/process.py\u001b[0m in \u001b[0;36mjoin\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m 122\u001b[0m \u001b[0;32massert\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_parent_pid\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgetpid\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'can only join a child process'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 123\u001b[0m \u001b[0;32massert\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_popen\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'can only join a started process'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 124\u001b[0;31m \u001b[0mres\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_popen\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 125\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mres\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 126\u001b[0m \u001b[0m_children\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdiscard\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m~/anaconda3/envs/anupam/lib/python3.6/multiprocessing/popen_fork.py\u001b[0m in \u001b[0;36mwait\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m 48\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 49\u001b[0m \u001b[0;31m# This shouldn't block if wait() returned successfully.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 50\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpoll\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mWNOHANG\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mtimeout\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0.0\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 51\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreturncode\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 52\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m~/anaconda3/envs/anupam/lib/python3.6/multiprocessing/popen_fork.py\u001b[0m in \u001b[0;36mpoll\u001b[0;34m(self, flag)\u001b[0m\n\u001b[1;32m 26\u001b[0m \u001b[0;32mwhile\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 27\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 28\u001b[0;31m \u001b[0mpid\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwaitpid\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpid\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mflag\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 29\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mOSError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 30\u001b[0m \u001b[0;31m# Child process not yet created. See #1731717\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;31mKeyboardInterrupt\u001b[0m: " ] } ], "source": [ "no_of_clients = 2\n", "clients = []\n", "for i in range(no_of_clients):\n", " client = multiprocessing.Process(target=client_start)\n", " clients.append(client)\n", " client.start()\n", " time.sleep(2)\n", "for i in range(no_of_clients):\n", " clients[i].join()\n", "server.join()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# 2. This is shown for Fed Adagrad" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "configs ={\n", " \"algorithm\":\"fedadagrad\",\n", " \"num_of_clients\":2,\n", " \"fraction_of_clients\":1,\n", " \"num_of_rounds\":1,\n", " \"initial_model_path\":\"initial_model.pt\",\n", " \"epochs\":1,\n", " \"accept_conn_after_FL_begin\":1 ,\n", " \"verify\": 0,\n", " \"verification_threshold\": 0,\n", " \"timeout\": None,\n", " \"resize_size\": 32,\n", " \"batch_size\": 32,\n", " \"net\": \"LeNet\",\n", " \"dataset\": \"MNIST\",\n", " #\"device\": \"cpu\",\n", " \"niid\": 2,\n", " \"carbon\": 0\n", "}" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Start the server and client\n", "1. Uncomment the multiprocessing line if you are using it on windows platform" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "save_intial_model(configs)\n", "#multiprocessing.set_start_method('spawn', force=False)\n", "server = multiprocessing.Process(target=server_start, args=(configs,))\n", "server.start()\n", "time.sleep(5)\n", "no_of_clients = 2\n", "clients = []\n", "for i in range(no_of_clients):\n", " client = multiprocessing.Process(target=client_start)\n", " clients.append(client)\n", " client.start()\n", " time.sleep(2)\n", "for i in range(no_of_clients):\n", " clients[i].join()\n", "server.join()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# 3. This is shown for Fed Adam\n", "1. Use algorithm =\"fedadam\"\n", "2. Number of epochs can also be changed through \"epochs\". Here Epochs= 2 is used" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "configs ={\n", " \"algorithm\":\"fedadam\",\n", " \"num_of_clients\":2,\n", " \"fraction_of_clients\":1,\n", " \"num_of_rounds\":1,\n", " \"initial_model_path\":\"initial_model.pt\",\n", " \"epochs\":2,\n", " \"accept_conn_after_FL_begin\":1 ,\n", " \"verify\": 0,\n", " \"verification_threshold\": 0,\n", " \"timeout\": None,\n", " \"resize_size\": 32,\n", " \"batch_size\": 32,\n", " \"net\": \"LeNet\",\n", " \"dataset\": \"MNIST\",\n", " #\"device\": \"cpu\",\n", " \"niid\": 2,\n", " \"carbon\": 0\n", "}" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Start the server and client\n", "1. Uncomment the multiprocessing line if you are using it on windows platform" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "save_intial_model(configs)\n", "#multiprocessing.set_start_method('spawn', force=False)\n", "server = multiprocessing.Process(target=server_start, args=(configs,))\n", "server.start()\n", "time.sleep(5)\n", "no_of_clients = 2\n", "clients = []\n", "for i in range(no_of_clients):\n", " client = multiprocessing.Process(target=client_start)\n", " clients.append(client)\n", " client.start()\n", " time.sleep(2)\n", "for i in range(no_of_clients):\n", " clients[i].join()\n", "server.join()" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.13" } }, "nbformat": 4, "nbformat_minor": 2 } ================================================ FILE: tutorials/Number_of_clients_Tutorial.ipynb ================================================ { "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Import Libraries" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "import sys\n", "import os\n", "from torch import multiprocessing\n", "import time\n", "sys.path.append(os.path.dirname(os.getcwd())) \n", "from server.src.server import server_start\n", "from server.src.server_lib import save_intial_model\n", "from client.src.client import client_start" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Setup the configuration file" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "1. Number of clients indicated the minimum amount of clients needed to start the process\n", "2. Update the number as how many clients initially need to be included\n", "3. Here number of clients 2" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "configs ={\n", " \"algorithm\":\"fedavg\",\n", " \"num_of_clients\":2,\n", " \"fraction_of_clients\":1,\n", " \"num_of_rounds\":1,\n", " \"initial_model_path\":\"initial_model.pt\",\n", " \"epochs\":1,\n", " \"accept_conn_after_FL_begin\":1 ,\n", " \"verify\": 0,\n", " \"verification_threshold\": 0,\n", " \"timeout\": None,\n", " \"resize_size\": 32,\n", " \"batch_size\": 32,\n", " \"net\": \"LeNet\",\n", " \"dataset\": \"MNIST\",\n", " #\"device\": \"cpu\",\n", " \"niid\": 2,\n", " \"carbon\": 0\n", "}" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Start the server\n", "1. Uncomment the multiprocessing line if you are using it on windows platform\n", "2. Output will be shown as the client part starts" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "Server Running\n", "Client ipv4:127.0.0.1:46254 connected.\n", "Client ipv4:127.0.0.1:46266 connected.\n", "\n", "Communication round 1/1 is starting with 2/2 client(s).\n" ] } ], "source": [ "save_intial_model(configs)\n", "#multiprocessing.set_start_method('spawn', force=False)\n", "server = multiprocessing.Process(target=server_start, args=(configs,))\n", "server.start()\n", "time.sleep(5)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Start the client" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "no_of_clients = 4\n", "clients = []\n", "for i in range(no_of_clients):\n", " client = multiprocessing.Process(target=client_start)\n", " clients.append(client)\n", " client.start()\n", " time.sleep(2)\n", "for i in range(no_of_clients):\n", " clients[i].join()\n", "server.join()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# 2. This is shown Number of clients =3" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "configs ={\n", " \"algorithm\":\"fedadagrad\",\n", " \"num_of_clients\":4,\n", " \"fraction_of_clients\":1,\n", " \"num_of_rounds\":1,\n", " \"initial_model_path\":\"initial_model.pt\",\n", " \"epochs\":1,\n", " \"accept_conn_after_FL_begin\":1 ,\n", " \"verify\": 0,\n", " \"verification_threshold\": 0,\n", " \"timeout\": None,\n", " \"resize_size\": 32,\n", " \"batch_size\": 32,\n", " \"net\": \"LeNet\",\n", " \"dataset\": \"MNIST\",\n", " #\"device\": \"cpu\",\n", " \"niid\": 2,\n", " \"carbon\": 0\n", "}" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Start the server and client\n", "1. Uncomment the multiprocessing line if you are using it on windows platform" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "save_intial_model(configs)\n", "#multiprocessing.set_start_method('spawn', force=False)\n", "server = multiprocessing.Process(target=server_start, args=(configs,))\n", "server.start()\n", "time.sleep(5)\n", "no_of_clients = 2\n", "clients = []\n", "for i in range(no_of_clients):\n", " client = multiprocessing.Process(target=client_start)\n", " clients.append(client)\n", " client.start()\n", " time.sleep(2)\n", "for i in range(no_of_clients):\n", " clients[i].join()\n", "server.join()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# 3. This is shown for number of clients =5\n" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "configs ={\n", " \"algorithm\":\"fedadam\",\n", " \"num_of_clients\":2,\n", " \"fraction_of_clients\":1,\n", " \"num_of_rounds\":1,\n", " \"initial_model_path\":\"initial_model.pt\",\n", " \"epochs\":2,\n", " \"accept_conn_after_FL_begin\":1 ,\n", " \"verify\": 0,\n", " \"verification_threshold\": 0,\n", " \"timeout\": None,\n", " \"resize_size\": 32,\n", " \"batch_size\": 32,\n", " \"net\": \"LeNet\",\n", " \"dataset\": \"MNIST\",\n", " #\"device\": \"cpu\",\n", " \"niid\": 2,\n", " \"carbon\": 0\n", "}" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Start the server and client\n", "1. Uncomment the multiprocessing line if you are using it on windows platform" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "save_intial_model(configs)\n", "#multiprocessing.set_start_method('spawn', force=False)\n", "server = multiprocessing.Process(target=server_start, args=(configs,))\n", "server.start()\n", "time.sleep(5)\n", "no_of_clients = 2\n", "clients = []\n", "for i in range(no_of_clients):\n", " client = multiprocessing.Process(target=client_start)\n", " clients.append(client)\n", " client.start()\n", " time.sleep(2)\n", "for i in range(no_of_clients):\n", " clients[i].join()\n", "server.join()" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.13" } }, "nbformat": 4, "nbformat_minor": 2 } ================================================ FILE: tutorials/Verifcation_module_tutorial.ipynb ================================================ { "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Import Libraries" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "import sys\n", "import os\n", "from torch import multiprocessing\n", "import time\n", "sys.path.append(os.path.dirname(os.getcwd())) \n", "from server.src.server import server_start\n", "from server.src.server_lib import save_intial_model\n", "from client.src.client import client_start" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Use of Verification Module" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Setup the configuration file" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "1. Different modules can easily be accessed with the help of the config file\n", "2. Thresshold is co-related with the verification module. If the verififcation mdoule is set to 1 then the thresshold value can be setup. \n", "3. The value of thresshold signifies the inference score of the model, so its value should be between 0-100\n", "4. The models received by the server will get shuffeld and send back to the clients participated in that communication for inference on their local dataset\n", "5. Clients once done the inference send back the scores only\n", "6. Server depending on the thresshold value choose the models whose scores are above that thresshold for aggregation on the communication round (CR)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Details of the config file\n", "1. Number of clients are taken as 2\n", "2. So the two models received by the server in each CR will be sent to the clients before aggreagtion for verification\n", "3. Verification is set to 1\n", "4. Thresshold value is set to 0.9\n", "5. Dataset and model used for local training is MNIST and LeNet-5 respectively" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "configs ={\n", " \"algorithm\":\"fedavg\",\n", " \"num_of_clients\":2,\n", " \"fraction_of_clients\":1,\n", " \"num_of_rounds\":1,\n", " \"initial_model_path\":\"initial_model.pt\",\n", " \"epochs\":1,\n", " \"accept_conn_after_FL_begin\":1 ,\n", " \"verify\": 1,\n", " \"verification_threshold\": 0.9,\n", " \"timeout\": None,\n", " \"resize_size\": 32,\n", " \"batch_size\": 32,\n", " \"net\": \"LeNet\",\n", " \"dataset\": \"MNIST\",\n", " #\"device\": \"cpu\",\n", " \"niid\": 2,\n", " \"carbon\": 0\n", "}" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Start the server\n", "1. Uncomment the multiprocessing line if you are using it on windows platform\n", "2. Output will be shown as the client part starts" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "Server Running\n", "Client ipv4:127.0.0.1:42074 connected.\n", "Client ipv4:127.0.0.1:42090 connected.\n", "\n", "Communication round 1/1 is starting with 2/2 client(s).\n", "Client ipv4:127.0.0.1:42090 has disconnected. Now 1 clients remain active.\n", "Client ipv4:127.0.0.1:42074 has disconnected. Now 0 clients remain active.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Process Process-1:\n", "KeyboardInterrupt\n", "Traceback (most recent call last):\n", " File \"/home/deepsip1/anaconda3/envs/anupam/lib/python3.6/multiprocessing/process.py\", line 258, in _bootstrap\n", " self.run()\n", " File \"/home/deepsip1/anaconda3/envs/anupam/lib/python3.6/multiprocessing/process.py\", line 93, in run\n", " self._target(*self._args, **self._kwargs)\n", " File \"/home/deepsip1/anupam/Framework/Github_25/fl_framework_initial-main/server/src/server.py\", line 138, in server_start\n", " server_runner_thread.join()\n", " File \"/home/deepsip1/anaconda3/envs/anupam/lib/python3.6/threading.py\", line 1056, in join\n", " self._wait_for_tstate_lock()\n", " File \"/home/deepsip1/anaconda3/envs/anupam/lib/python3.6/threading.py\", line 1072, in _wait_for_tstate_lock\n", " elif lock.acquire(block, timeout):\n" ] } ], "source": [ "save_intial_model(configs)\n", "#multiprocessing.set_start_method('spawn', force=False)\n", "server = multiprocessing.Process(target=server_start, args=(configs,))\n", "server.start()\n", "time.sleep(5)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Start the client" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Connected with server\n", "Connected with server\n" ] }, { "ename": "KeyboardInterrupt", "evalue": "", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 8\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mno_of_clients\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 9\u001b[0m \u001b[0mclients\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m \u001b[0mserver\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", "\u001b[0;32m~/anaconda3/envs/anupam/lib/python3.6/multiprocessing/process.py\u001b[0m in \u001b[0;36mjoin\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m 122\u001b[0m \u001b[0;32massert\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_parent_pid\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgetpid\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'can only join a child process'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 123\u001b[0m \u001b[0;32massert\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_popen\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'can only join a started process'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 124\u001b[0;31m \u001b[0mres\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_popen\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 125\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mres\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 126\u001b[0m \u001b[0m_children\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdiscard\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m~/anaconda3/envs/anupam/lib/python3.6/multiprocessing/popen_fork.py\u001b[0m in \u001b[0;36mwait\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m 48\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 49\u001b[0m \u001b[0;31m# This shouldn't block if wait() returned successfully.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 50\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpoll\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mWNOHANG\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mtimeout\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0.0\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 51\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreturncode\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 52\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m~/anaconda3/envs/anupam/lib/python3.6/multiprocessing/popen_fork.py\u001b[0m in \u001b[0;36mpoll\u001b[0;34m(self, flag)\u001b[0m\n\u001b[1;32m 26\u001b[0m \u001b[0;32mwhile\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 27\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 28\u001b[0;31m \u001b[0mpid\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwaitpid\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpid\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mflag\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 29\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mOSError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 30\u001b[0m \u001b[0;31m# Child process not yet created. See #1731717\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;31mKeyboardInterrupt\u001b[0m: " ] } ], "source": [ "no_of_clients = 2\n", "clients = []\n", "for i in range(no_of_clients):\n", " client = multiprocessing.Process(target=client_start)\n", " clients.append(client)\n", " client.start()\n", " time.sleep(2)\n", "for i in range(no_of_clients):\n", " clients[i].join()\n", "server.join()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Another one" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Details of the config file\n", "\n", "1. Number of clients are taken as 3\n", "2. So the two models received by the server in each CR will be sent to the clients before aggreagtion for verification\n", "3. Verification is set to 1\n", "4. Thresshold value is set to 0.8\n", "5. Dataset and model used for local training is CIFAR-10 and ResNet-18 respectively\n", "6. Number of communication round is 2" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "configs ={\n", " \"algorithm\":\"fedadagrad\",\n", " \"num_of_clients\":3,\n", " \"fraction_of_clients\":1,\n", " \"num_of_rounds\":2,\n", " \"initial_model_path\":\"initial_model.pt\",\n", " \"epochs\":1,\n", " \"accept_conn_after_FL_begin\":1 ,\n", " \"verify\": 1,\n", " \"verification_threshold\": 0.8,\n", " \"timeout\": None,\n", " \"resize_size\": 32,\n", " \"batch_size\": 32,\n", " \"net\": \"resnet18\",\n", " \"dataset\": \"CIFAR10\",\n", " #\"device\": \"cpu\",\n", " \"niid\": 2,\n", " \"carbon\": 0\n", "}" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Start the server and client\n", "1. Uncomment the multiprocessing line if you are using it on windows platform\n", "2. Output will be shown as the client part starts" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "save_intial_model(configs)\n", "#multiprocessing.set_start_method('spawn', force=False)\n", "server = multiprocessing.Process(target=server_start, args=(configs,))\n", "server.start()\n", "time.sleep(5)\n", "no_of_clients = 2\n", "clients = []\n", "for i in range(no_of_clients):\n", " client = multiprocessing.Process(target=client_start)\n", " clients.append(client)\n", " client.start()\n", " time.sleep(2)\n", "for i in range(no_of_clients):\n", " clients[i].join()\n", "server.join()" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.13" } }, "nbformat": 4, "nbformat_minor": 2 } ================================================ FILE: tutorials/accuracy_plot.py ================================================ import os import matplotlib.pyplot as plt def read_values(txt_path): """ Reads accuracy and round values from a text file. Args: txt_path (str): Path to the text file. Returns: tuple: Accuracy and round values as lists. """ accuracy = [] rounds = [] with open(txt_path, encoding='UTF-8') as f: for line in f.readlines(): line_data = line.split(',') acc = float(line_data[1].split(':')[1]) r = int(line_data[2].split(':')[1].split('}')[0]) accuracy.append(acc) rounds.append(r) return accuracy, rounds def plot_round_vs_accuracy_1(algorithm_values, niids): """ Plots round vs accuracy graphs for different NIID datasets and algorithms. Args: algorithm_values (list): List of tuples containing algorithm name and results for different NIIDs. niids (list): List of number of clients for which FL was performed. """ for niid in range(len(niids)): plt.figure() plt.title(f'NIID Dataset {niids[niid]} vs Accuracy') # Varying title based on the NIID dataset plt.xlabel('Rounds') plt.ylabel('Accuracy') for algorithm in range(len(algorithm_values)): accuracy, rounds = algorithm_values[algorithm][niid][1][0], algorithm_values[algorithm][niid][1][1] algorithm_name = algorithm_values[algorithm][0][0] plt.plot(rounds, accuracy, markersize=5, label=algorithm_name) plt.legend() plt.savefig(f'./media/NIID-{str(niids[niid])}_niid_vs_accuracy.png') plt.clf() del accuracy del rounds def plot_round_vs_accuracy_2(algorithm_values, niids): """ Plots round vs accuracy graphs for different algorithms and a single NIID dataset. Args: algorithm_values (list): List of tuples containing algorithm name and results for different NIIDs. niids (list): List of number of clients for which FL was performed. """ for i, algorithm_value in enumerate(algorithm_values): plt.figure() plt.xlabel('Rounds') plt.ylabel('Accuracy') for niid in range(len(niids)): plt.title(f'Round vs Accuracy for {str(algorithm_value[niid][0])}') #print(algorithm_value[niid][0]) accuracy, rounds = algorithm_value[niid][1][0], algorithm_value[niid][1][1] plt.plot(rounds, accuracy, markersize=5, label=f'NIID = {niid+1}') plt.legend() plt.savefig(f'./media/round_vs_accuracy_{str(algorithm_value[niid][0])}.png') def plot_niid_vs_accuracy(algorithm_values, niids): """ Plots NIID vs accuracy graph for different algorithms. Args: algorithm_values (list): List of tuples containing algorithm name and results for different NIIDs. niids (list): List of number of clients for which FL was performed. """ plt.figure() plt.title('NIID vs Accuracy') plt.xlabel('NIID') plt.ylabel('Accuracy') for algorithm_data in algorithm_values: algorithm_name = algorithm_data[0][0] accuracy = [] for niid in range(len(niids)): accuracy.append(algorithm_data[niid][1][0][-1]) plt.plot(niids, accuracy, 'o--', markersize=5, label=algorithm_name) plt.legend() plt.savefig('./media/Niid_vs_Accuracy.png') plt.clf() if __name__ == '__main__': # Check if the result directory exists results_path = '../server_results' if not os.path.exists(results_path): raise Exception("The result directory is not found") else: dataset_name = 'FashionMNIST' algorithm_names = ['fedavg', 'fedadam', 'fedavgm', 'fedadagrad', 'fedyogi'] # Get the list of number of clients niids = sorted([int(i) for i in os.listdir(os.path.join(results_path, dataset_name, algorithm_names[0]))]) # Get the FL results for each algorithm and each number of clients algorithm_values = [] for algorithm_name in algorithm_names: algorithm_path = os.path.join(results_path, dataset_name, algorithm_name) algorithm_niids = os.listdir(algorithm_path) algorithm_niids = sorted([int(i) for i in algorithm_niids]) algorithm_values.append([]) for niid in algorithm_niids: niid_path = os.path.join(algorithm_path, str(niid)) niid_runs = os.listdir(niid_path) latest_run = sorted(niid_runs, reverse=True)[0] results_file_path = os.path.join(niid_path, latest_run, 'FL_results.txt') values = read_values(results_file_path) # Assumes read_values function is defined elsewhere algorithm_values[-1].append((algorithm_name, values)) # Plot the results # Assumes plot_round_vs_accuracy_1 function is defined elsewhere plot_round_vs_accuracy_1(algorithm_values, niids) # Assumes plot_round_vs_accuracy_2 function is defined elsewhere plot_round_vs_accuracy_2(algorithm_values, niids) plot_niid_vs_accuracy(algorithm_values, niids) # Assumes plot_round_vs_accuracy_2 function is defined elsewhere ================================================ FILE: tutorials/data_distribution.ipynb ================================================ { "cells": [ { "attachments": {}, "cell_type": "markdown", "id": "39c0e501", "metadata": {}, "source": [ "### Import Libraries" ] }, { "cell_type": "code", "execution_count": 14, "id": "712f7d2b", "metadata": {}, "outputs": [], "source": [ "import os, torch, random\n", "import numpy as np\n", "import matplotlib.pyplot as plt\n", "from torchvision import transforms,datasets\n", "from torch.utils import data" ] }, { "attachments": {}, "cell_type": "markdown", "id": "2deef073", "metadata": {}, "source": [ "### Config file" ] }, { "cell_type": "code", "execution_count": 15, "id": "b21a520d", "metadata": {}, "outputs": [], "source": [ "config = {\"epochs\": 2, \"dataset\":\"MNIST\", \"net\": \"LeNet\", \"resize_size\":32, \"batch_size\":4,\"niid\": 5}" ] }, { "attachments": {}, "cell_type": "markdown", "id": "cb46fdd0", "metadata": {}, "source": [ "### Collecting data" ] }, { "cell_type": "code", "execution_count": 16, "id": "93d47069", "metadata": {}, "outputs": [], "source": [ "def get_data(config):\n", " # If the dataset is not custom, create a dataset folder\n", " if config['dataset'] != 'CUSTOM':\n", " dataset_path = \"client_dataset\"\n", " if not os.path.exists(dataset_path):\n", " os.makedirs(dataset_path)\n", "\n", " # Get the train and test datasets for each supported dataset\n", " if config['dataset'] == 'MNIST':\n", " # Apply transformations to the images\n", " apply_transform = transforms.Compose([transforms.Resize(config[\"resize_size\"]), transforms.ToTensor()])\n", " # Download and load the trainset\n", " trainset = datasets.MNIST(root='client_dataset/MNIST', train=True, download=True, transform=apply_transform)\n", " # Download and load the testset\n", " testset = datasets.MNIST(root='client_dataset/MNIST', train=False, download=True, transform=apply_transform)\n", " elif config['dataset'] == 'FashionMNIST':\n", " apply_transform = transforms.Compose([transforms.Resize(config['resize_size']), transforms.ToTensor()])\n", " trainset = datasets.FashionMNIST(root='client_dataset/FashionMNIST',\n", " train=True, download=True, transform=apply_transform)\n", " testset = datasets.FashionMNIST(root='client_dataset/FashionMNIST',\n", " train=False, download=True, transform=apply_transform)\n", " elif config['dataset'] == 'CIFAR10':\n", " apply_transform = transforms.Compose([transforms.Resize(config['resize_size']), transforms.ToTensor()])\n", " trainset = datasets.CIFAR10(root='client_dataset/CIFAR10',\n", " train=True, download=True, transform=apply_transform)\n", " testset = datasets.CIFAR10(root='client_dataset/CIFAR10',\n", " train=False, download=True, transform=apply_transform)\n", " elif config['dataset'] == 'CIFAR100':\n", " apply_transform = transforms.Compose([transforms.Resize(config['resize_size']), transforms.ToTensor()])\n", " trainset = datasets.CIFAR100(root='client_dataset/CIFAR100',\n", " train=True, download=True, transform=apply_transform)\n", " testset = datasets.CIFAR100(root='client_dataset/CIFAR100',\n", " train=False, download=True, transform=apply_transform)\n", " else:\n", " # Raise an error if an unsupported dataset is specified\n", " raise ValueError(f\"Unsupported dataset type: {config['dataset']}\")\n", " # Return the train and test datasets\n", " return trainset, testset\n", "\n" ] }, { "cell_type": "code", "execution_count": 17, "id": "7ecf15dc", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Downloading http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz\n", "Downloading http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz to client_dataset/MNIST/MNIST/raw/train-images-idx3-ubyte.gz\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "a3b70bf79cac4cd0b2c3d13d5de279d3", "version_major": 2, "version_minor": 0 }, "text/plain": [ " 0%| | 0/9912422 [00:00" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaEAAAGDCAYAAACCzK//AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAAu6UlEQVR4nO3deXxU5d028OtKWMImuxCgkKgQEoLGGml5CipgUV5xq1otca+laAVb0ILgY3mrb6EqlCJapRa3uoHIAy4P1IJQ+0jRUCMJgYBCZDMBBNkTJsnv/WPOPI4xy4RZ7gxc389nPsycc+ZerOXyPufM+dHMICIi4kKC6wGIiMipSyEkIiLOKIRERMQZhZCIiDijEBIREWcUQiIi4oxCSE4qJJ8i+Z8RaqsnycMkE73PK0neEYm2vfb+m+QtkWpPJB41cT0AkYYgWQygC4AKAJUACgG8AGCumVWZ2ZgGtHOHmf29tmPMbBuA1uGO2etvKoCzzOzGoPZHRKJtkXimlZDEo8vNrA2AXgCmA5gI4C+R7ICk/gNNJAYUQhK3zOyAmS0BcD2AW0hmknyO5MMAQLITybdIfkVyH8n3SSaQfBFATwBveqfbfk0yhaSR/CnJbQBWBG0LDqQzSX5I8gDJxSQ7eH1dRHJH8PhIFpO8mOSlACYDuN7r7xNv//+e3vPG9QDJz0nuJvkCybbevsA4biG5jeReklOi+09XJDYUQhL3zOxDADsADK62a4K3vTP8p/Am+w+3mwBsg39F1drMHgn6zoUA0gFcUkt3NwO4HUA3+E8Jzg5hfEsB/A7Aa15/59Rw2K3eawiAM+A/DTin2jGDAKQBGAbgQZLp9fUt0tgphORksQtAh2rbfACSAfQyM5+ZvW/1PyxxqpkdMbNjtex/0cwKzOwIgP8E8OPAjQthygEw08y2mNlhAPcDuKHaKuz/mtkxM/sEwCcAagozkbiiEJKTRXcA+6ptexTApwD+RnILyUkhtLO9Afs/B9AUQKeQR1m7bl57wW03gX8FF1AS9P4oInTThIhLCiGJeyTPhz+E/hm83cwOmdkEMzsDwOUAxpMcFthdS3P1rZS+E/S+J/yrrb0AjgBoGTSmRPhPA4ba7i74b7QIbrsCQGk93xOJawohiVskTyM5EsCrAP5qZvnV9o8keRZJAjgI/y3dld7uUvivvTTUjSQzSLYE8FsAr5tZJYBNAJJIXkayKYAHADQP+l4pgBSStf1/7hUAvyKZSrI1vr6GVHECYxSJGwohiUdvkjwE/6mxKQBmArithuN6A/g7gMMAVgN40sxWevumAXjAu3Pu3gb0/SKA5+A/NZYEYBzgv1MPwF0AngGwE/6VUfDdcgu8P78k+e8a2p3ntf0PAFsBlAEY24BxicQlqqidiIi4opWQiIg4oxASERFnFEIiIuKMQkhERJxRCImIiDN6UnCQTp06WUpKiuthiMhJZu3atXvNrHP9R556FEJBUlJSkJub63oYInKSIfl5/UedmnQ6TkREnFEIiYiIMwohERFxRiEkIiLOKIRERMQZhZCIiDijEBIREWcUQiIi4oxCSEREnFEIiYiIMwohERFxRiEkIiLO6AGmQfJ3HkDKpLej3k9x0qio9xHQP7VnTPqZP60iJv0AwIqLnohZX2X7Z8akn+tTJ8akHwB4Jml5TPoZfMGLMekHAHK4MCb9lAzJikk/pxKthERExBmFkIiIOKMQEhERZxRCIiLijEJIREScUQiJiIgzCiEREXFGISQiIs4ohERExBmFkIiIOKMQEhERZxRCIiLijEJIREScUQiJiIgzCiEREXFGISQiIs4ohERExBmFkIiIOKMQEhERZxRCIiLijEJIRESciVoIkexK8lWSn5EsJPkOyT4kU0gWeMdkk5wdRh+T69i3kmQRyTzvdfqJ9iMiItHRJBqNkiSARQCeN7MbvG1ZALoA2B44zsxyAeSG0dVkAL+rY3+O14eIiDRC0VoJDQHgM7OnAhvMLM/M3g8+iORFJN/y3rciOY/kRyQ/Jnmlt/1Wkm+QXEpyM8lHvO3TAbTwVjkvRWkeIiISRVFZCQHIBLC2gd+ZAmCFmd1Osh2AD0n+3duXBeBcAOUAikg+bmaTSN5tZll1tPksyUoACwE8bGZW/QCSowGMBoDE0zo3cMgiIhKOxnRjwnAAk0jmAVgJIAlAT2/fcjM7YGZlAAoB9AqhvRwz6w9gsPe6qaaDzGyumWWbWXZiy7ZhTkFERBoiWiG0HsB5DfwOAVxjZlneq6eZbfD2lQcdV4kQVnBmttP78xCAlwEMaOB4REQkyqIVQisANCf5s8AGkueTvLCO7ywDMNa7qQEkzw2hHx/JptU3kmxCspP3vimAkQAKGjIBERGJvqiEkHft5WoAP/Ru0V4PYCqAXXV87SEATQGs827hfiiEruZ6x1e/MaE5gGUk1wHIA7ATwJ8bNAkREYm6aN2YADPbBeDHtezO9I5ZCf/1H5jZMQA/r6Gd5wA8F/R5ZND7iQAm1vCdI2j46UAREYmxxnRjgoiInGIUQiIi4oxCSEREnFEIiYiIMwohERFxRiEkIiLOKIRERMQZhZCIiDijEBIREWcUQiIi4oxCSEREnFEIiYiIMwohERFxRiEkIiLOKIRERMQZhZCIiDijEBIREWcUQiIi4oxCSEREnKGZuR5Do5GdnW25ubmuhyEiJxmSa80s2/U4GiOthERExBmFkIiIOKMQEhERZxRCIiLijEJIREScUQiJiIgzCiEREXFGISQiIs4ohERExBmFkIiIOKMQEhERZ5q4HkBjkr/zAFImvR31foqTRkW9j4D+qT1j0s/8aRUx6QcAVlz0RMz6Kts/Myb9XJ86MSb9AMAzSctj0s/gC16MST8AkMOFMemnZEhWTPo5lWglJCIiziiERETEGYWQiIg4oxASERFnFEIiIuKMQkhERJxRCImIiDMKIRERcUYhJCIizuiJCSIiDeTz+bBjxw6UlZWFdPy7777b/5NPPimO7qgapSoABRUVFXecd955u2s6QCEkItJAO3bsQJs2bZCSkgKS9R5fWVlZkZmZuTcGQ2tUqqqquGfPnoySkpJnAFxR0zE6HSci0kBlZWXo2LFjSAF0KktISLDOnTsfAJBZ6zExHI+IyElDARSahIQEQx1ZoxASERFndE1IRCRMIZSAaQl8fl6o7RVPv2ztiYxj/Pjx3Vq3bl3529/+tvREvl+XsWPHdl+wYEHHgwcPJh49evTjSLWrlZCIiNTrqquu+mrNmjUbIt2uQkhEJA7NmTOnY58+fTLS0tIyrrrqqtTq+2fMmNEpMzMzPS0tLeOSSy4589ChQwkAMG/evPa9e/ful5aWlpGdnZ0GALm5uUn9+/dP79u3b0afPn0y8vPzm1dvb9iwYUd69erli/Q8dDpORCTO5ObmJj322GPJq1ev3picnFxRWlqaWP2YnJyc/RMmTNgLAOPGjes2e/bsTlOmTNk9ffr05L/97W+bUlNTfXv37k0EgMcff7zzXXfdVXrnnXfuKysrY0VF7ColayUkIhJnli1bdtrll1++Pzk5uQIAunTpUln9mLVr17Y477zz0vr06ZOxcOHCjuvXr08CgOzs7MM5OTkpM2bM6BQIm4EDBx6ZMWNG8pQpU7pu3ry5WevWrS1Wc1EIiYjEGTMDyTqDYvTo0alz5szZtmnTpsKJEyfuKi8vTwCAl19+edvDDz+8a/v27c2ysrL6lZSUJI4ZM2bf4sWLP23RokXViBEj+ixZsqRNbGaiEBIRiTuXXnrpwSVLlnQoKSlJBICaTscdPXo0oWfPnr7y8nK++uqrHQLb169f33zo0KFHZs2atat9+/YVW7ZsaVZYWNgsPT29/IEHHtg9fPjwr/Ly8lrEai5RuyZEsiuAWQDOB1AOoBjALwEcB/CWmWWSzAZws5mNO8E+JpvZ7+o5ZgmAM8ys1l/sioiEo3j6ZXXuLygoOJqZmRmxO8uys7PLJkyY8MXgwYP7JiQkWGZm5tGFCxcWBx8zadKkXQMGDEjv3r378fT09KOHDx9OBIBf/epXPYqLi5ubGQcNGnTw+9///rEpU6Z0XbBgQccmTZpY586dfdOmTdtVvc8xY8b0WLRoUYeysrKELl26nJ2Tk7N35syZ3zquoWgW+VN/9P+U+AMAz5vZU962LABtAGyHF0IR6OewmbWuY/+PAFwL4OxQ+mue3NuSb5kV7rDqVZw0Kup9BPRP7RmTfuZPi92FzBUXPRGzvsr2z4xJP9enToxJPwDwTNLymPQz+IIXY9IPAORwYUz6KRmSBQDYsGED0tPTQ/5epEMo3nzyySedzjnnnJSa9kXrdNwQAL5AAAGAmeWZ2fvBB5G8iORb3vtWJOeR/IjkxySv9LbfSvINkktJbib5iLd9OoAWJPNIvlR9ACRbAxgP4OEozVFERMIUrdNxmQAa+ovfKQBWmNntJNsB+JDk3719WQDOhf+0XhHJx81sEsm7zSyrlvYeAjADwNG6OiU5GsBoAEg8rXMDhywiIuFoTDcmDAcwiWQegJUAkgAEziUtN7MDZlYGoBBAr7oa8k79nWVmi+rr1Mzmmlm2mWUntmwbxvBFRKShorUSWg//tZiGIIBrzKzoGxvJ78G/AgqoRP3jHgjgPJLF3rGnk1xpZhc1cEwiIhJF0VoJrQDQnOTPAhtInk/ywjq+swzAWO+mBpA8N4R+fCSbVt9oZn8ys25mlgJgEIBNCiARkcYnKiFk/lvurgbwQ5KfkVwPYCqAum7newhAUwDrSBZ4n+sz1zv+WzcmiIhI4xe13wmZ2S4AP65ld6Z3zEr4r//AzI4B+HkN7TwH4LmgzyOD3k8EUOe9rWZWjDqq+omIhG1q3deTM4GWeB0hl3LA1AONqpTDoUOHEi6//PIzPv/88+aJiYkYPnz4V08++eTOSLTdmG5MEBGRRmrChAmlW7duXV9QUFC4Zs2a1vPnzz8tEu0qhERE4lAsSzm0adOm6vLLLz8EAElJSXb22Wcf3b59e7NIzEMhJCISZwKlHFatWrWpqKio8Omnn95W/ZicnJz9BQUFG4qKigrT0tKOzZ49uxMABEo5FBUVFS5duvRT4OtSDhs3bixct27dhtTU1OO19b13797Ed999t92IESMORmIuCiERkTjjqpSDz+fDj370ozNGjx5dmpGRUWtQNYRCSEQkzrgq5TBq1KiUM844o+zBBx/cHam5KIREROKMi1IO48aN63bw4MHEv/zlL9sjOReV9xYRCdfUA3XujvdSDp999lnTxx9/PDk1NbWsX79+GQAwevTo3ePHj98b7lyiUsohXqmUw4lTKYfwqJRDeFTKoXFzUcpBRESkXgohERFxRiEkIiLOKIRERMQZhZCIiDijEBIREWf0OyERkTD1f75/fYe0xNrQSznk35LfqEo5AMDgwYN77969u2llZSUHDBhw6IUXXtjWpEn4EaKVkIiI1Gvx4sWfFRUVFW7atGn9l19+2XTevHntI9GuQkhEJA7FspQDAHTo0KEKAHw+H30+H0lGZB4KIRGROOOqlMOgQYN6d+7c+ZxWrVpV3nbbbfsjMReFkIhInHFVyuGf//zn5pKSkk+OHz+e8Oabb0aksqpuTAjSv3tb5E6/LAY91f2ww0jKj1VHt8SqIyD0J3ZFwtCY9hYLUzE4Zj3FSknMemocQi3l8Prrr386cODAY7Nnz+64atWqNoC/lMOKFStaLVmypG1WVla/vLy89WPGjNk3ePDgI4sWLWo7YsSIPk8++WTxFVdccaimdlu2bGkjR478atGiRe2uvvrqsAvbaSUkIhJnYl3K4cCBAwmff/55U8Bf2G7p0qVt+/bteywSc9FKSEQkTPm31H3OId5LORw8eDDhsssuO+v48eOsqqriD37wg4P33XffnkjMRaUcgmRnZ1tubq7rYYhII6dSDg2jUg4iItIoKYRERMQZhZCIiDijEBIREWcUQiIi4oxCSEREnNHvhEREwrShb923aycCLTcg9FIO6Rs3NLpSDgFDhw49a/v27c03b968PhLtaSUkIiIhef7559u1atXqW8+pC4dWQkHydx5AyqS3o95PcdKoqPcR0D+1Z0z6mT+tIib9AMCKi56IWV9l+2fGpJ/rUyfGpB8AeCZpeUz6GXzBizHpBwByuDAm/ZQMyYpJP6GYM2dOx9mzZ3chifT09GP/9V//tTV4/4wZMzo9++yznX0+H1NSUspff/31rW3atKmaN29e+2nTpnVLSEiwNm3aVObm5hbl5uYm3Xbbbak+n49VVVVYuHDhZ/379y8Pbu/AgQMJs2fP7jJ37tzPb7jhhjMjNQ+FkIhInAmUcli9evXG5OTkipqeHZeTk7N/woQJewFg3Lhx3WbPnt1pypQpuwOlHFJTU3179+5NBL4u5XDnnXfuKysrY+Dp2sHGjx/f/Z577ilt3bp1VSTnotNxIiJxJtalHD744IMWW7dubX7zzTd/Fem5KIREROJMqKUc5syZs23Tpk2FEydO3FVeXp4A+Es5PPzww7u2b9/eLCsrq19JSUnimDFj9i1evPjTFi1aVI0YMaLPkiVL2gS39f7777cuKCho2b179/4XXHBB3+Li4uYDBgxIi8RcFEIiInEm1qUcJk6cuGf37t3rdu7cmf+Pf/xjY0pKSvmHH35YFIm56JqQiEiY0jfW/YDseC/lEE0q5RCkeXJvS75lVtT70d1x4dHdceHR3XEnLnB3nEo5NIxKOYiISKOkEBIREWcUQiIi4kxIIUTyOpJtvPcPkHyD5HejOzQRETnZhboS+k8zO0RyEIBLADwP4E/RG5aIiJwKQg2hwK9xLwPwJzNbDKBZdIYkIiKnilB/J7ST5NMALgbwe5LNoetJIiIAgCfGrKjvkJarsCLkUg6/eGpooyvlMGDAgLTdu3c3TUpKqgKA5cuXb+revXvYv80INYR+DOBSAI+Z2VckkwHcF27nIiISP1544YUtF1xwwdFIthnqauZpM3vDzDYDgJl9AeCmSA5ERERCN2fOnI59+vTJSEtLy7jqqqtSq++fMWNGp8zMzPS0tLSMSy655MxDhw4lAMC8efPa9+7du19aWlpGdnZ2GuB/Knf//v3T+/btm9GnT5+M/Pz85rGaR6gh1C/4A8lENKBKoIiIRE6glMOqVas2FRUVFT799NPbqh+Tk5Ozv6CgYENRUVFhWlrasdmzZ3cCgEAph6KiosKlS5d+CnxdymHjxo2F69at25Camnq8pn7vuOOOlL59+2bcd999yVVVkanoUGcIkbyf5CEAZ5M86L0OAdgNYHFERiAiIg0S61IOAPDaa69t2bRpU+Hq1as3fvDBB62ffPLJjpGYS50hZGbTzKwNgEfN7DTv1cbMOprZ/ZEYgIiINEysSzkAQGpqqg8A2rdvX3X99dfv+/DDD1tFYi4hnY4zs/tJdif5HyQvCLwiMQAREWmYWJdy8Pl8+OKLL5oAQHl5Od955522mZmZxyIxl5DujiM5HcANAArx9W+GDMA/IjEIEZF49ounhta5P95LORw7dizh4osv7u3z+VhVVcXBgwcfHD9+/J5IzCWkUg4kiwCcbWblITdMdgUwC8D5AMoBFAP4JYDjAN4ys0yS2QBuNrNxDR65v4/JZva7WvYtBZAMf9C+D+AXZvat86bBVMrhxKmUQ3hUyiE8KuXQuEWilMMWAE1D7ZAkASwCsNLMzjSzDACTAXQJPs7Mck80gDyT69j3YzM7B0AmgM4ArgujHxERiYJQf6x6FEAeyeXwr2oAAHUEyBAAPjN7KujYPAAgmRLYRvIiAPea2UiSrQA8DqC/N66pZraY5K0ArgDQEsCZABaZ2a+9U4QtSOYBWG9mOcEDMLODQXNsBv/pQxERaURCDaEl3itUmQAa+tiJKQBWmNntJNsB+JDk3719WQDOhT8Ai0g+bmaTSN5tZlm1NUhyGYABAP4bwOsNHI+IiERZSCFkZs+TbAGgp5kVRWkswwFcQfJe73MSgMAFjeVmdgAASBYC6AVge30NmtklJJMAvARgKIB3qx9DcjSA0QCQeFrncOcgIiINEGo9ocsB5AFY6n3OIlnXymg9Gv5EBQK4xsyyvFdPMwtcyAu+IaISoa/gYGZl8K/irqxl/1wzyzaz7MSWbRs4ZBERCUeoNyZMhf+01lfA/17f+dazioKsANCc5M8CG0ieT/LCOr6zDMBY76YGkDw3hHH5SH7rhgmSrb2HrIJkEwD/B8DGENoTEZEYCnVFUWFmB7x8CKj1Qr+ZGcmrAcwiOQlAGb6+Rbs2D8F/S/c6L4iKAYysZ1xzveP/Xe3GhFYAlnglJxLhD8WnampARCRcM66v768qtFzWgLNDE157q9GVcigrK+Ntt93Wc/Xq1W1I2m9+85udt95661fhthtqCBWQHAUgkWRvAOMAfFDXF8xsF/wlIGqS6R2zEsBK7/0xAD+voZ3nADwX9Hlk0PuJAL71AwszK4X/90kiIhIB999/f3Lnzp19xcXFBZWVldi9e3fIl0XqEurpuLHwP0m7HMArAA6i7lWNiIhEUaxLObzyyiudHn744RIASExMRODhqeEK9dlxR81sipmd713En+Jd8BcRkRiLdSmHvXv3JgL+030ZGRnpI0aMOGP79u3RXwmRnOX9+SbJJdVfkRiAiIg0TKxLOfh8PpaWljYdNGjQ4cLCwg3f+973jowdO/Y7kZhLfSuhwMOfHgMwo4aXiIjEWKxLOXTp0qUiKSmp6qabbvoKAG688cZ9BQUFLSMxl/rqCa31/lxV0ysSAxARkYaJdSmHhIQEDBs27MDbb7/dBgDeeeed03r37h39Ug4k81H3rdhnR2IQIiLxbMJrb9W5P95LOQDAzJkzd4waNSr13nvvTezYsWPFCy+8UFz9mBNRZykH73bsLvj2I3J6AdhlZp9GYhCNhUo5nDiVcgiPSjmER6UcGrdwSjn8AcBBM/s8+AX/U7X/EOFxiojIKaa+EEoxs3XVN5pZLoCUqIxIREROGfWFUFId+1rUsU9ERKRe9YXQR8EPIQ0g+VM0vF6QiIjIN9T3i9dfAlhEMgdfh042/JVKr47iuERE5BRQZwh5DwL9D5JD4D10FMDbZrYi6iMTEZGTXqiVVd8D8F6UxyIiEpd2THq/zv3tgJY78H7IpRx6TB/cqEo57N+/P2HgwIF9A59LS0ubXn311fvmzZtXb4Xr+kTkAXQiInLyat++fdXGjRsLA5/79euXft111+2PRNuhlnIQEZFGJNalHALy8/Obf/nll00vueSSw5GYh1ZCIiJxJlDKYfXq1RuTk5Mranp2XE5Ozv4JEybsBYBx48Z1mz17dqcpU6bsDpRySE1N9QVKNARKOdx55537ysrKGHi6dk2ef/75DldcccW+hITIrGG0EhIRiTOxLuUQbNGiRR1uuummfZGai0JIRCTOxLqUQ8Dq1atbVFZWcvDgwUcjNRedjgvSv3tb5E6/LAY9HYhBH375serollh1BIT+2MhIGBrT3mJhKgbHrKdYKYlZT43DpZdeevDaa689a/LkyaVdu3atLC0tTay+GqpeyiE5OdkHfF3KYejQoUeWLVvWbsuWLc327dtXmZ6eXt6vX7/dW7ZsaZ6Xl9fiiiuuOFS93xdffLHD1VdfHbFVEKAQEhEJW4/pdQf7yVDKAQCWLFnS4c0339wcqXkA9ZRyONVkZ2dbbm6u62GISCOnUg4NE04pBxERkahRCImIiDMKIRERcUYhJCIiziiERETEGYWQiIg4o98JiYiEaerUqfUd0vL1118PuZTD1KlTG1UpBwB4+umnO8yYMaMrAHTp0sU3f/78rYHHBoVDKyEREamTz+fD/fff/51Vq1Zt2rRpU2G/fv2OPfroo6dHom2FkIhIHIplKYeqqiqaGQ4dOpRQVVWFgwcPJnTr1u14JOah03EiInEm1qUcmjdvbjNnztz23e9+t1+LFi0qe/XqVf7CCy9si8RcFEJB8nceQMqkt6PeT3HSqKj3EdA/tWdM+pk/LexTwyFbcdETMeurbP/MmPRzferEmPQDAM8kLY9JP4MveDEm/QBADhfGpJ+SIVkx6ac+oZZyePDBB7sfOnQo8ciRI4kXXnjhAeDrUg7XXHPN/pycnP2Av5TDY489lrxjx45mN9xww/7+/fuXB7dVXl7OuXPndl6zZk1henp6+a233tpz8uTJyY888sgX4c5Fp+NEROJMrEs5/Otf/2oBAP369StPSEjAT37yk31r1qxpFYm5KIREROLMpZdeenDJkiUdSkpKEgGgptNx1Us5BLYHSjnMmjVrV/v27Su2bNnSrLCwsFl6enr5Aw88sHv48OFf5eXltQhuq1evXr5PP/00adeuXU0AYOnSpaf16dOnLBJz0ek4EZEw1XeLdryXckhJSfHdd999XwwaNCitSZMm1qNHj+Mvv/zy1kjMRaUcgjRP7m3Jt8yKej+6JhQeXRMKj64JnbjANSGVcmgYlXIQEZFGSSEkIiLOKIRERMQZhZCIiDijEBIREWcUQiIi4ox+JyQiEqblK86s75CWpSsQcimHYUM/a3SlHP785z+3f/TRR5Orqqp48cUXH3jqqad2RKJdrYRERKROJSUliQ8++GCPlStXbvr000/X7969u8nixYvb1P/N+imERETiUCxLORQVFTVPTU0t79atWwUADBs27OCCBQvaR2IeOh0nIhJnYl3KISMjo/yzzz5LKioqanbGGWccX7JkSXufz8dIzEUrIRGROBNqKYfzzjsvrU+fPhkLFy7suH79+iTg61IOM2bM6BQIm4EDBx6ZMWNG8pQpU7pu3ry5WevWrb/xPLfOnTtX/uEPf/j8uuuuO+P888/v27Nnz/LExMSIPPNNISQiEmdiXcoBAEaNGnVg3bp1G/Py8jampaWVnXnmmeXf7rXhFEIiInEm1qUcAGDnzp1NAGDPnj2JzzzzzOl33XXXnkjMRdeERETCNGzoZ3Xuj/dSDgAwZsyY7xQWFrYEgIkTJ+46++yzI7ISUimHICrlcOJUyiE8KuUQHpVyaNxUykFERBqlqIUQya4kXyX5GclCku+Q7EMyhWSBd0w2ydlh9DG5lu0tSb5NciPJ9SSnn2gfIiISPVEJIZIEsAjASjM708wyAEwG0CX4ODPLNbNxYXRVYwh5HjOzvgDOBfADkiPC6EdERKIgWiuhIQB8ZvZUYIOZ5ZnZ+8EHkbyI5Fve+1Yk55H8iOTHJK/0tt9K8g2SS0luJvmIt306gBYk80i+FNyumR01s/e898cB/BtAjyjNVURETlC0QigTQEMfwDcFwAozOx/+EHuUZCtvXxaA6wH0B3A9ye+Y2SQAx8wsy8xyamuUZDsAlwOo8WosydEkc0nmVh490MAhi4hIOBrTjQnDAUwimQdgJYAkAIFbu5ab2QEzKwNQCKBXKA2SbALgFQCzzWxLTceY2Vwzyzaz7MSWbcOcgoiINES0fie0HsC1DfwOAVxjZkXf2Eh+D0Dw/eiVCH3ccwFsNrNZDRyLiEjIur6XV98hLfFeXsilHEqGZDW6Ug5jx47tvmDBgo4HDx5MPHr06MeB7ceOHeO1116bmp+f37Jdu3YVCxYs2JKWlnY81HajtRJaAaA5yZ8FNpA8n+SFdXxnGYCx3k0NIHluCP34SDataQfJhwG0BfDLkEctIiI1uuqqq75as2bNt37r9Mc//rFT27ZtK7Zt21Zw9913l44fP75B19+jEkLm/wXs1QB+6N2ivR7AVADf+hVukIcANAWwzruF+6EQuprrHf+NGxNI9oD/GlMGgH97Ny/c0fCZiIg0TrEs5QAAw4YNO9KrVy9f9e1vvfVWu9tvv/1LALjtttv2f/DBB22qqqpCnkfUHttjZrsA/LiW3ZneMSvhv/4DMzsG4Oc1tPMcgOeCPo8Mej8RwLd+am5mO+A/vScictKJdSmHupSWljZLTU09DgBNmzZF69atK0tLS5sEnvBdn8Z0Y4KIiIQg1qUc6lLTo9/qe8J3MIWQiEiccVHKoTZdu3Y9vnXr1mYA4PP5cPjw4cTTTz/9W6FYG4WQiEiccVHKoTaXXXbZV/PmzesIAM8++2z7gQMHHkpICD1aVMpBRCRMgadr1+YkKeXQY9GiRR3KysoSunTpcnZOTs7emTNn7rrnnnv2XnPNNak9e/bMbNu2beVrr71Wd12LalTKIYhKOZw4lXIIj0o5hEelHBo3lXIQEZFGSSEkIiLOKIRERE6ALmWEpqqqigBq/fWqQkhEpIGSkpLw5ZdfKojqUVVVxT179rQFUFDbMbo7TkSkgXr06IEdO3Zgz549IR1fUlLSpLKyslOUh9UYVQEoqKioqPWxaQohEZEGatq0KVJTv/W4tlplZGTkm1l2FIcUt3Q6TkREnFEIiYiIMwohERFxRiEkIiLOKIRERMQZhZCIiDijEBIREWcUQiIi4oxKOQTJzs623Nxc18MQkZMMybX6sWrNtBISERFnFEIiIuKMQkhERJxRCImIiDMKIRERcUYhJCIiziiERETEGYWQiIg4oxASERFnFEIiIuKMQkhERJxRCImIiDNNXA+gMcnfeQApk96Oej/FSaOi3kdA/9SeMeln/rSKmPQDACsueiJmfZXtnxmTfq5PnRiTfgDgmaTlMeln8AUvxqQfAMjhwpj0UzIkKyb9nEq0EhIREWcUQiIi4oxCSEREnFEIiYiIMwohERFxRiEkIiLOKIRERMQZhZCIiDijEBIREWcUQiIi4oxCSEREnFEIiYiIMwohERFxRiEkIiLOKIRERMQZhZCIiDijEBIREWcUQiIi4oxCSEREnFEIiYiIM1ELIZJdSb5K8jOShSTfIdmHZArJAu+YbJKzw+hjch37/h/J7SQPn2j7IiISXVEJIZIEsAjASjM708wyAEwG0CX4ODPLNbNxYXRVawgBeBPAgDDaFhGRKIvWSmgIAJ+ZPRXYYGZ5ZvZ+8EEkLyL5lve+Fcl5JD8i+THJK73tt5J8g+RSkptJPuJtnw6gBck8ki9VH4CZ/cvMvojS/EREJAKaRKndTABrG/idKQBWmNntJNsB+JDk3719WQDOBVAOoIjk42Y2ieTdZpYVzkBJjgYwGgAST+scTlMiItJAjenGhOEAJpHMA7ASQBKAnt6+5WZ2wMzKABQC6BWpTs1srpllm1l2Ysu2kWpWRERCEK2V0HoA1zbwOwRwjZkVfWMj+T34V0ABlYjeuEVEJIaitRJaAaA5yZ8FNpA8n+SFdXxnGYCx3k0NIHluCP34SDYNb6giIuJKVELIzAzA1QB+6N2ivR7AVAC76vjaQwCaAljn3cL9UAhdzfWO/9aNCSQfIbkDQEuSO0hObeA0REQkyqJ2WsvMdgH4cS27M71jVsJ//QdmdgzAz2to5zkAzwV9Hhn0fiKAibX0/2sAvz6BoYuISIw0phsTRETkFKMQEhERZxRCIiLijEJIREScUQiJiIgzCiEREXFGISQiIs4ohERExBmFkIiIOKMQEhERZxRCIiLijEJIREScUQiJiIgzCiEREXFGISQiIs4ohERExBmFkIiIOKMQEhERZxRCIiLiDM3M9RgajezsbMvNzXU9DBE5yZBca2bZrsfRGGklJCIiziiERETEGYWQiIg4oxASERFnFEIiIuKMQkhERJxRCImIiDMKIRERcUYhJCIiziiERETEGYWQiIg4oxASERFnFEIiIuKMQkhERJxRKYcgJA8BKHI9jgjrBGCv60FEmOYUHzSnr/Uys86RHszJoInrATQyRSdbzQ+SuZpT46c5xYeTcU6u6XSciIg4oxASERFnFELfNNf1AKJAc4oPmlN8OBnn5JRuTBAREWe0EhIREWcUQgBIXkqyiOSnJCe5Hk+4SH6H5HskN5BcT/Ie12OKFJKJJD8m+ZbrsUQKyXYkXye50fvfbKDrMYWL5K+8f/cKSL5CMsn1mBqK5DySu0kWBG3rQPJdkpu9P9u7HOPJ4JQPIZKJAJ4AMAJABoCfkMxwO6qwVQCYYGbpAL4P4BcnwZwC7gGwwfUgIuyPAJaaWV8A5yDO50eyO4BxALLNLBNAIoAb3I7qhDwH4NJq2yYBWG5mvQEs9z5LGE75EAIwAMCnZrbFzI4DeBXAlY7HFBYz+8LM/u29PwT/X2rd3Y4qfCR7ALgMwDOuxxIpJE8DcAGAvwCAmR03s6+cDioymgBoQbIJgJYAdjkeT4OZ2T8A7Ku2+UoAz3vvnwdwVSzHdDJSCPn/ct4e9HkHToK/sANIpgA4F8Aax0OJhFkAfg2gyvE4IukMAHsAPOudZnyGZCvXgwqHme0E8BiAbQC+AHDAzP7mdlQR08XMvgD8/7EH4HTH44l7CiGANWw7KW4ZJNkawEIAvzSzg67HEw6SIwHsNrO1rscSYU0AfBfAn8zsXABHEOeneLzrJFcCSAXQDUArkje6HZU0Vgoh/8rnO0GfeyAOTx1UR7Ip/AH0kpm94Xo8EfADAFeQLIb/lOlQkn91O6SI2AFgh5kFVqqvwx9K8exiAFvNbI+Z+QC8AeA/HI8pUkpJJgOA9+dux+OJewoh4CMAvUmmkmwG/wXUJY7HFBaShP8awwYzm+l6PJFgZvebWQ8zS4H/f6MVZhb3/3VtZiUAtpNM8zYNA1DocEiRsA3A90m29P5dHIY4v9kiyBIAt3jvbwGw2OFYTgqn/ANMzayC5N0AlsF/F888M1vveFjh+gGAmwDkk8zztk02s3fcDUnqMBbAS95/BG0BcJvj8YTFzNaQfB3Av+G/U/NjxOGTBki+AuAiAJ1I7gDwGwDTAcwn+VP4w/Y6dyM8OeiJCSIi4oxOx4mIiDMKIRERcUYhJCIiziiERETEGYWQiIg4oxAS50gayRlBn+8lOTVCbT9H8tpItFVPP9d5T8B+rzGNS6SxUwhJY1AO4EckO7keSDDvCeuh+imAu8xsSLTGI3IyUghJY1AB/48Zf1V9R/UVA8nD3p8XkVxFcj7JTSSnk8wh+SHJfJJnBjVzMcn3veNGet9PJPkoyY9IriP586B23yP5MoD8GsbzE6/9ApK/97Y9CGAQgKdIPlrDd37tfecTktNr2P+gN44CknO9pwyA5DiShd74XvW2XUgyz3t9TLKNt/2+oLn8X29bK5Jve/0WkLw+tP85RGLnlH9igjQaTwBYR/KRBnznHADp8D9ufwuAZ8xsAP1F/MYC+KV3XAqACwGcCeA9kmcBuBn+pzufT7I5gP8hGXjS8wAAmWa2Nbgzkt0A/B7AeQD2A/gbyavM7LckhwK418xyq31nBPyP+/+emR0l2aGGecwxs996x78IYCSAN+F/kGmqmZWTbOcdey+AX5jZ/3gPqC0jORxAb2/cBLCE5AUAOgPYZWaXeW23De0fq0jsaCUkjYL3lO8X4C+GFqqPvNpJ5QA+AxAIkXz4gydgvplVmdlm+MOqL4DhAG72Hmu0BkBH+P8iB4APqweQ53wAK70Hc1YAeAn+WkB1uRjAs2Z21Jtn9fo0ADCE5BqS+QCGAujnbV8H/+N8boR/tQgA/wNgJslxANp54xjuvT6G/1E5fb255MO/Cvw9ycFmdqCesYrEnEJIGpNZ8F9bCa6nUwHv31PvNFWzoH3lQe+rgj5X4Zur/OrPpjL4VwxjzSzLe6UG1bw5Usv4air7UR/W0P/XO/1lr58EcK2Z9QfwZwCBUtiXwb9CPA/AWpJNzGw6gDsAtADwL5J9vT6mBc3lLDP7i5lt8r6bD2Cad9pQpFFRCEmj4a0S5sMfRAHF8P9FCvhr1DQ9gaavI5ngXSc6A0AR/A+svdMreQGSfVh/Mbk1AC4k2cm7aeEnAFbV852/AbidZEuvn+qn4wKBs9c7vXatd1wCgO+Y2XvwF/JrB6A1yTPNLN/Mfg8gF/5VzzKvj9bed7uTPN07fXjUzP4Kf5G5eC8RISchXROSxmYGgLuDPv8ZwGKSHwJYjtpXKXUpgj8sugAYY2ZlJJ+B/5Tdv70V1h7UU6rZzL4geT+A9+BffbxjZnU+yt/MlpLMApBL8jiAdwBMDtr/Fck/w79aKYa/tAjgf6L7X73rOATwB+/Yh0gOAVAJf8mH//auGaUDWO3d03AYwI0AzgLwKMkqAD4Ad9b7T0okxvQUbRERcUan40RExBmFkIiIOKMQEhERZxRCIiLijEJIREScUQiJiIgzCiEREXFGISQiIs78fy5NZxXnDeI/AAAAAElFTkSuQmCC", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaEAAAGDCAYAAACCzK//AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAAwcElEQVR4nO3deXxU9b0+8OdJWMImuxBASLAQEoJGDbTcggtYlwpW6lriXkvRCm1By+b1cqu3clWoRepFanG7dUNKweUHbUHUXikYNZIQCQhENsMiGCKQMCSf3x/nTB1DlpnMmTkJPO/Xa17MnOV7PiciT84y50Mzg4iIiB8S/C5AREROXQohERHxjUJIRER8oxASERHfKIRERMQ3CiEREfGNQkhOKiTnk/x3j8bqTfIrkonu59Uk7/BibHe8/0fyFq/GE2mKmvldgEgkSBYD6AbgOIBKAIUAngOwwMyqzGx8BOPcYWZ/r20ZM9sOoG20NbvbmwngW2Z2Y8j4l3sxtkhTpiMhaYpGm1k7AH0AzAIwBcAfvdwASf2CJhIHCiFpssys1MyWAbgewC0kM0k+Q/JBACDZheTrJL8keYDkuyQTSD4PoDeA19zTbb8imULSSP6Y5HYAq0KmhQbSmSTXkSwluZRkJ3dbF5LcGVofyWKSF5O8DMB0ANe72/vYnf+v03tuXfeR/IzkXpLPkWzvzgvWcQvJ7ST3k5wR25+uSHwohKTJM7N1AHYCGF5t1mR3elc4p/CmO4vbTQC2wzmiamtmD4escwGAdACX1rK5mwHcDqAHnFOCc8OobzmA3wB42d3e2TUsdqv7ughAXzinAedVW2YYgDQAIwHcTzK9vm2LNHYKITlZ7AbQqdq0AIBkAH3MLGBm71r9D0ucaWaHzexoLfOfN7MCMzsM4N8BXBe8cSFKOQDmmNlWM/sKwDQAN1Q7CvtPMztqZh8D+BhATWEm0qQohORk0RPAgWrTHgHwKYC/ktxKcmoY4+yIYP5nAJoD6BJ2lbXr4Y4XOnYzOEdwQSUh74/Ao5smRPykEJImj+RgOCH0j9DpZlZmZpPNrC+A0QAmkRwZnF3LcPUdKZ0R8r43nKOt/QAOA2gdUlMinNOA4Y67G86NFqFjHwewp571RJo0hZA0WSRPIzkKwEsA/tfM8qvNH0XyWyQJ4BCcW7or3dl74Fx7idSNJDNItgbwawCvmlklgE0AkkheQbI5gPsAtAxZbw+AFJK1/T/3IoBfkkwl2RZfX0M63oAaRZoMhZA0Ra+RLINzamwGgDkAbqthuX4A/g7gKwBrADxhZqvdeQ8BuM+9c+6eCLb9PIBn4JwaSwIwEXDu1ANwF4CnAOyCc2QUerfcIvfPL0h+WMO4C92x3wGwDUA5gAkR1CXSJFFN7URExC86EhIREd8ohERExDcKIRER8Y1CSEREfKMQEhER3+hJwSG6dOliKSkpfpchIieZDz74YL+Zda1/yVOPQihESkoKcnNz/S5DRE4yJD+rf6lTk07HiYiIbxRCIiLiG4WQiIj4RiEkIiK+UQiJiIhvFEIiIuIbhZCIiPhGISQiIr5RCImIiG8UQiIi4huFkIiI+EYhJCIivtEDTEPk7ypFytQ3Grx+cdJYD6sBBqX29nS8Vx467tlYqy78vWdjhaP84BzPxro+dYpnY9Wk6JJbPR8zh4s9Ha/koixPxxNpKB0JiYiIbxRCIiLiG4WQiIj4RiEkIiK+UQiJiIhvFEIiIuIbhZCIiPhGISQiIr5RCImIiG8UQiIi4huFkIiI+EYhJCIivlEIiYiIbxRCIiLiG4WQiIj4RiEkIiK+UQiJiIhvFEIiIuIbhZCIiPhGISQiIr5RCImIiG9iFkIku5N8ieQWkoUk3yTZn2QKyQJ3mWySc6PYxvQ65q0mWUQyz32d3tDtiIhIbDSLxaAkCWAJgGfN7AZ3WhaAbgB2BJczs1wAuVFsajqA39QxP8fdhoiINEKxOhK6CEDAzOYHJ5hZnpm9G7oQyQtJvu6+b0NyIcn3SX5E8gfu9FtJ/pnkcpKbST7sTp8FoJV7lPOnGO2HiIjEUEyOhABkAvggwnVmAFhlZreT7ABgHcm/u/OyAJwDoAJAEcnHzWwqybvNLKuOMZ8mWQlgMYAHzcyqL0ByHIBxAJB4WtcISxYRkWg0phsTLgEwlWQegNUAkgD0duetNLNSMysHUAigTxjj5ZjZIADD3ddNNS1kZgvMLNvMshNbt49yF0REJBKxCqENAM6LcB0CuNrMstxXbzP7xJ1XEbJcJcI4gjOzXe6fZQBeADAkwnpERCTGYhVCqwC0JPmT4ASSg0leUMc6KwBMcG9qAMlzwthOgGTz6hNJNiPZxX3fHMAoAAWR7ICIiMReTELIvfYyBsD33Fu0NwCYCWB3Has9AKA5gPXuLdwPhLGpBe7y1W9MaAlgBcn1APIA7ALwh4h2QkREYi5WNybAzHYDuK6W2ZnuMqvhXP+BmR0F8NMaxnkGwDMhn0eFvJ8CYEoN6xxG5KcDRUQkzhrTjQkiInKKUQiJiIhvFEIiIuIbhZCIiPhGISQiIr5RCImIiG8UQiIi4huFkIiI+EYhJCIivlEIiYiIbxRCIiLiG4WQiIj4RiEkIiK+UQiJiIhvFEIiIuIbhZCIiPhGISQiIr5RCImIiG8UQiIi4huamd81NBrZ2dmWm5vrdxkicpIh+YGZZftdR2OkIyEREfGNQkhERHyjEBIREd8ohERExDcKIRER8Y1CSEREfKMQEhER3yiERETENwohERHxjUJIRER8oxASERHfNPO7gMYkf1cpUqa+0eD1i5PGeliNY1Bq7wav+8pDxz2sJHyrLvx9zLdRfnBOg9YrS4/947uGn/+852PmcHFU65dclOVNISIe05GQiIj4RiEkIiK+UQiJiIhvFEIiIuIbhZCIiPhGISQiIr5RCImIiG8UQiIi4huFkIiI+EZPTBARiVAgEMDOnTtRXl4e1vJ/+9vfBn388cfFsa2qUaoCUHD8+PE7zjvvvL01LaAQEhGJ0M6dO9GuXTukpKSAZL3LV1ZWHs/MzNwfh9IalaqqKu7bty+jpKTkKQBX1rSMTseJiESovLwcnTt3DiuATmUJCQnWtWvXUgCZtS4Tx3pERE4aCqDwJCQkGOrIGoWQiIj4RteERESiFEYLmNbAZ+eFO17xrCs+aEgdkyZN6tG2bdvKX//613sasn5dJkyY0HPRokWdDx06lHjkyJGPvBpXR0IiIlKvq6666su1a9d+4vW4CiERkSZo3rx5nfv375+RlpaWcdVVV6VWnz979uwumZmZ6WlpaRmXXnrpmWVlZQkAsHDhwo79+vUbmJaWlpGdnZ0GALm5uUmDBg1KHzBgQEb//v0z8vPzW1Yfb+TIkYf79OkT8Ho/dDpORKSJyc3NTXr00UeT16xZszE5Ofn4nj17Eqsvk5OTc3Dy5Mn7AWDixIk95s6d22XGjBl7Z82alfzXv/51U2pqamD//v2JAPD44493veuuu/bceeedB8rLy3n8ePy6MutISESkiVmxYsVpo0ePPpicnHwcALp161ZZfZkPPvig1XnnnZfWv3//jMWLF3fesGFDEgBkZ2d/lZOTkzJ79uwuwbAZOnTo4dmzZyfPmDGj++bNm1u0bdvW4rUvCiERkSbGzECyzqAYN25c6rx587Zv2rSpcMqUKbsrKioSAOCFF17Y/uCDD+7esWNHi6ysrIElJSWJ48ePP7B06dJPW7VqVXX55Zf3X7ZsWbv47IlCSESkybnssssOLVu2rFNJSUkiANR0Ou7IkSMJvXv3DlRUVPCll17qFJy+YcOGliNGjDj82GOP7e7YsePxrVu3tigsLGyRnp5ecd999+295JJLvszLy2sVr32J2TUhkt0BPAZgMIAKAMUAfgHgGIDXzSyTZDaAm81sYgO3Md3MflPPMssA9DWzWr+xKyISjeJZV9Q5v6Cg4EhmZqZnd5ZlZ2eXT548+fPhw4cPSEhIsMzMzCOLFy8uDl1m6tSpu4cMGZLes2fPY+np6Ue++uqrRAD45S9/2au4uLilmXHYsGGHvvOd7xydMWNG90WLFnVu1qyZde3aNfDQQw/trr7N8ePH91qyZEmn8vLyhG7dup2Vk5Ozf86cOScsFymaeX/qj85Xid8D8KyZzXenZQFoB2AH3BDyYDtfmVnbOub/EMA1AM4KZ3stk/tZ8i2PNbie4qSxDV63NoNSezd43Vceit/FxVCrLvx9zLdRfnBOg9YrS8/2uJITDT//ec/HzOHiqNYvuSjLm0IEAPDJJ58gPT097OW9DqGm5uOPP+5y9tlnp9Q0L1an4y4CEAgGEACYWZ6ZvRu6EMkLSb7uvm9DciHJ90l+RPIH7vRbSf6Z5HKSm0k+7E6fBaAVyTySf6peAMm2ACYBeDBG+ygiIlGK1em4TACRfuN3BoBVZnY7yQ4A1pH8uzsvC8A5cE7rFZF83MymkrzbzLJqGe8BALMBHKlroyTHARgHAImndY2wZBERiUZjujHhEgBTSeYBWA0gCUDwXNRKMys1s3IAhQD61DWQe+rvW2a2pL6NmtkCM8s2s+zE1u2jKF9ERCIVqyOhDXCuxUSCAK42s6JvTCS/DecIKKgS9dc9FMB5JIvdZU8nudrMLoywJhERiaFYHQmtAtCS5E+CE0gOJnlBHeusADDBvakBJM8JYzsBks2rTzSz/zGzHmaWAmAYgE0KIBGRxicmIWTOLXdjAHyP5BaSGwDMBFDX7XwPAGgOYD3JAvdzfRa4y59wY4KIiDR+MfuekJntBnBdLbMz3WVWw7n+AzM7CuCnNYzzDIBnQj6PCnk/BcCUeuooRh1d/UREojaz7uvJmUBrvIqwWzlgZmmjauVQVlaWMHr06L6fffZZy8TERFxyySVfPvHEE7u8GLsx3ZggIiKN1OTJk/ds27ZtQ0FBQeHatWvbvvLKK6d5Ma5CSESkCYpnK4d27dpVjR49ugwAkpKS7KyzzjqyY8eOFl7sh0JIRKSJCbZyePvttzcVFRUVPvnkk9urL5OTk3OwoKDgk6KiosK0tLSjc+fO7QIAwVYORUVFhcuXL/8U+LqVw8aNGwvXr1//SWpq6rHatr1///7Ev/3tbx0uv/zyQ17si0JIRKSJ8auVQyAQwA9/+MO+48aN25ORkVFrUEVCISQi0sT41cph7NixKX379i2///7793q1LwohEZEmxo9WDhMnTuxx6NChxD/+8Y87vNwXtfcWEYnWzNI6Zzf1Vg5btmxp/vjjjyenpqaWDxw4MAMAxo0bt3fSpEn7o92XmLRyaKrUysEbauWgVg4nO7VyiIwfrRxERETqpRASERHfKIRERMQ3CiEREfGNQkhERHyjEBIREd/oe0IiIlEa9Oyg+hZpjQ/Cb+WQf0t+o2rlAADDhw/vt3fv3uaVlZUcMmRI2XPPPbe9WbPoI0RHQiIiUq+lS5duKSoqKty0adOGL774ovnChQs7ejGuQkhEpAmKZysHAOjUqVMVAAQCAQYCAZL0ZD8UQiIiTYxfrRyGDRvWr2vXrme3adOm8rbbbjvoxb4ohEREmhi/Wjn84x//2FxSUvLxsWPHEl577TVPOqvqxoQQg3q2R+6sK6IYoe6HGDZEfjQr3+JVFZEJ/4la0RgRl600zEzPRyzxfERpysJt5fDqq69+OnTo0KNz587t/Pbbb7cDnFYOq1atarNs2bL2WVlZA/Py8jaMHz/+wPDhww8vWbKk/eWXX97/iSeeKL7yyivLahq3devWNmrUqC+XLFnSYcyYMVE3ttORkIhIExPvVg6lpaUJn332WXPAaWy3fPny9gMGDDjqxb7oSEhEJEr5t9R9zqKpt3I4dOhQwhVXXPGtY8eOsaqqit/97ncP3Xvvvfu82Be1cgiRnZ1tubm5fpchIo2cWjlERq0cRESkUVIIiYiIbxRCIiLiG4WQiIj4RiEkIiK+UQiJiIhv9D0hEZEofTKg7tu1E4HWnyD8Vg7pGz9pdK0cgkaMGPGtHTt2tNy8efMGL8bTkZCIiITl2Wef7dCmTZsTnlMXDR0JhcjfVYqUqW94OmZx0lhPx/PCoNTeMRv7lYeOx2zs2rx59pmejHN96hRPxgl6Kmmlp+MFDT//+ZiMW10OF8dlO0ElF2XFdXtN3bx58zrPnTu3G0mkp6cf/ctf/rItdP7s2bO7PP30010DgQBTUlIqXn311W3t2rWrWrhwYceHHnqoR0JCgrVr164yNze3KDc3N+m2225LDQQCrKqqwuLFi7cMGjSoInS80tLShLlz53ZbsGDBZzfccIM3/9NBISQi0uQEWzmsWbNmY3Jy8vGanh2Xk5NzcPLkyfsBYOLEiT3mzp3bZcaMGXuDrRxSU1MD+/fvTwS+buVw5513HigvL2fw6dqhJk2a1PPnP//5nrZt21Z5uS86HSci0sTEu5XDe++912rbtm0tb7755i+93heFkIhIExNuK4d58+Zt37RpU+GUKVN2V1RUJABOK4cHH3xw944dO1pkZWUNLCkpSRw/fvyBpUuXftqqVauqyy+/vP+yZcvahY717rvvti0oKGjds2fPQeeff/6A4uLilkOGDEnzYl8UQiIiTUy8WzlMmTJl3969e9fv2rUr/5133tmYkpJSsW7duiIv9kXXhEREopS+se4HZDf1Vg6xpFYOIVom97PkWx7zdEzdHRd7ujsuNnR3XO3UyiEyauUgIiKNkkJIRER8oxASERHfKIRERMQ3CiEREfGNQkhERHwT1veESD5vZjfVN01E5FT0+/Gr6luk9dtYFXYrh5/NH9HoWjkMGTIkbe/evc2TkpKqAGDlypWbevbsGfV3MsL9surA0A8kExFBbwwREWn6nnvuua3nn3/+ES/HrPN0HMlpJMsAnEXykPsqA7AXwFIvCxERkfDNmzevc//+/TPS0tIyrrrqqtTq82fPnt0lMzMzPS0tLePSSy89s6ysLAEAFi5c2LFfv34D09LSMrKzs9MA56ncgwYNSh8wYEBG//79M/Lz81vGaz/qDCEze8jM2gF4xMxOc1/tzKyzmU2LU40iIhIi2Mrh7bff3lRUVFT45JNPbq++TE5OzsGCgoJPioqKCtPS0o7OnTu3CwAEWzkUFRUVLl++/FPg61YOGzduLFy/fv0nqampx2ra7h133JEyYMCAjHvvvTe5qsqbjg5h3ZhgZtNI9iT5byTPD748qUBERCIS71YOAPDyyy9v3bRpU+GaNWs2vvfee22feOKJzl7sS1ghRHIWgP8DcB+Ae93XPV4UICIikYl3KwcASE1NDQBAx44dq66//voD69ata+PFvoR7i/YYAGlm9n0zG+2+rvSiABERiUy8WzkEAgF8/vnnzQCgoqKCb775ZvvMzMyjXuxLuHfHbQXQHEBFfQuKiJxqfjZ/RJ3zm3orh6NHjyZcfPHF/QKBAKuqqjh8+PBDkyZN2ufFvoTVyoHkYgBnA1iJkCAys4l1rNMdwGMABrvrFAP4BYBjAF43s0yS2QBurmuceuqabma/qWXecgDJcIL2XQA/M7MTzpuGUiuH6KmVw9fUyiEyauVw8qqrlUO4R0LL3FdYSBLAEgDPmtkN7rQsAN0A7AguZ2a5AHLDHbcG0wHUGEIArjOzQ24trwK4FsBLUWxLREQ8FlYImdmzJFsB6G1m4bR0vQhAwMzmh4yRBwAkU4LTSF4I4B4zG0WyDYDHAQxy65ppZktJ3grgSgCtAZwJYImZ/cq9WaIVyTwAG8wsp1rNh0L2sQUAde8TEWlkwr07bjSAPADL3c9ZJOs6MsoEEOljJ2YAWGVmg+GE2CNuMAFAFoDr4QTU9STPMLOpAI6aWVb1AAqpewWcL9aWwTkaEhGRRiTcu+NmAhgC4EvgX0c1J3xDN0qXAJjqHtmsBpAEIHjxYqWZlZpZOYBCAH3CGdDMLoVzXaglgBqvHJIcRzKXZG7lkdLo9kBERCISbggdN7Pq/0LXdXprAyJ/thwBXO0e2WSZWW8zC17IC70rrxLhX8uCG1zLAPyglvkLzCzbzLITW7ePsGQREYlGuCFUQHIsgESS/Ug+DuC9OpZfBaAlyZ8EJ5AcTPKCOtZZAWCCeyMBSJ4TRl0Bks2rTyTZlmSy+74ZgO8D2BjGeCIiEkfhHlFMgHPNpgLAi3AC44HaFjYzIzkGwGMkpwIox9e3aNfmATi3dK93g6gYwKh66lrgLv9htetCbQAsI9kSQCKcUJxf0wAiItGafX19/1Sh9YoIzg5Nfvn1RtfKoby8nLfddlvvNWvWtCNp//Ef/7Hr1ltv/TLaccO9O+4InBCaEe7AZrYbwHW1zM50l1kN5/oPzOwogJ/WMM4zAJ4J+Twq5P0UACd8ucPM9sD5fpKIiHhg2rRpyV27dg0UFxcXVFZWYu/evWFfFqlLfa0cHnP/fI3ksuovLwoQEZHIxbuVw4svvtjlwQcfLAGAxMREBB+eGq36kiz41exHvdiYiIhEL9jKYc2aNRuTk5OP1/TsuJycnIOTJ0/eDwATJ07sMXfu3C4zZszYG2zlkJqaGti/f38i8HUrhzvvvPNAeXk5g0/XDgouN2nSpB7vvfdeuz59+lQsWLBg+xlnnBF1ENXXT+gD98+3a3pFu3EREYlcvFs5BAIB7tmzp/mwYcO+Kiws/OTb3/724QkTJpzhxb7Udzoun+T62l5eFCAiIpGJdyuHbt26HU9KSqq66aabvgSAG2+88UBBQUFrL/alvlu0fwjgLgCjq73udueJiEicxbuVQ0JCAkaOHFn6xhtvtAOAN99887R+/frFpZXDbwFMN7PPQieS7OrOG+1FESIiTdnkl1+vc35Tb+UAAHPmzNk5duzY1HvuuSexc+fOx5977rni6ss0RJ2tHEgWmFlmLfPyzWyQF0U0FmrlED21cviaWjlERq0cTl51tXKo73RcUh3zWtUxT0REpF71hdD7oY/eCSL5Y0T+lGwREZFvqO+a0C8ALCGZg69DJxtOf54xMaxLREROAXWGkPv4m38jeRHcR+0AeMPMVsW8MhEROemF++y4twC8FeNaRETkFBNuKwcRERHPefIUVBGRU9nOqe/WOb8D0Hon3g27lUOvWcMbVSuHgwcPJgwdOnRA8POePXuajxkz5sDChQt3RDu2QkhEROrUsWPHqo0bNxYGPw8cODD92muvPejF2DodJyLSBMW7lUNQfn5+yy+++KL5pZde+pUX+6EjIRGRJiberRxCPfvss52uvPLKAwkJ3hzD6EhIRKSJiXcrh1BLlizpdNNNNx3wal8UQiIiTUy8WzkErVmzplVlZSWHDx9+xKt90em4EIN6tkfurCs8HrXU4/Gilx/LwW+J5eA1C/8xkvE1E8NjNnI8lMRlK9IQl1122aFrrrnmW9OnT9/TvXv3yj179iRWPxqq3sohOTk5AHzdymHEiBGHV6xY0WHr1q0tDhw4UJmenl4xcODAvVu3bm2Zl5fX6sorryyrvt3nn3++05gxYzw7CgIUQiIiUes1q+5fOE6GVg4AsGzZsk6vvfbaZq/2A6inlcOpJjs723Jzc/0uQ0QaObVyiEw0rRxERERiRiEkIiK+UQiJiIhvFEIiIuIbhZCIiPhGISQiIr7R94RERKI0c+bM+hZp/eqrr4bdymHmzJmNqpUDADz55JOdZs+e3R0AunXrFnjllVe2BR8bFA0dCYmISJ0CgQCmTZt2xttvv71p06ZNhQMHDjz6yCOPnO7F2AohEZEmKJ6tHKqqqmhmKCsrS6iqqsKhQ4cSevToccyL/dDpOBGRJiberRxatmxpc+bM2X7uuecObNWqVWWfPn0qnnvuue1e7ItCKET+rlKkTH3D83GLk8Z6PmZDDErtHbOxX3ko6lPD9Vp14e9jOn75wTmej3l96hTPx6zPU0kr477N2gw//3nftp3DxZ6PWXJRludjNkS4rRzuv//+nmVlZYmHDx9OvOCCC0qBr1s5XH311QdzcnIOAk4rh0cffTR5586dLW644YaDgwYNqggdq6KiggsWLOi6du3awvT09Ipbb7219/Tp05Mffvjhz6PdF52OExFpYuLdyuGf//xnKwAYOHBgRUJCAn70ox8dWLt2bRsv9kUhJCLSxFx22WWHli1b1qmkpCQRAGo6HVe9lUNwerCVw2OPPba7Y8eOx7du3dqisLCwRXp6esV9992395JLLvkyLy+vVehYffr0CXz66adJu3fvbgYAy5cvP61///7lXuyLTseJiESpvlu0m3orh5SUlMC99977+bBhw9KaNWtmvXr1OvbCCy9s82Jf1MohRMvkfpZ8y2Oej6trQt7QNaHw6JqQI5bXhNTKITJq5SAiIo2SQkhERHyjEBIREd8ohERExDcKIRER8Y1CSEREfKPvCYmIRGnlqjPrW6T1nlUIu5XDyBFbGl0rhz/84Q8dH3nkkeSqqipefPHFpfPnz9/pxbg6EhIRkTqVlJQk3n///b1Wr1696dNPP92wd+/eZkuXLm1X/5r1UwiJiDRB8WzlUFRU1DI1NbWiR48exwFg5MiRhxYtWtTRi/3Q6TgRkSYm3q0cMjIyKrZs2ZJUVFTUom/fvseWLVvWMRAI0It90ZGQiEgTE24rh/POOy+tf//+GYsXL+68YcOGJODrVg6zZ8/uEgyboUOHHp49e3byjBkzum/evLlF27Ztv/E8t65du1b+9re//ezaa6/tO3jw4AG9e/euSExM9OSZbwohEZEmJt6tHABg7NixpevXr9+Yl5e3MS0trfzMM8+sOHGrkVMIiYg0MfFu5QAAu3btagYA+/btS3zqqadOv+uuu/Z5sS+6JiQiEqWRI7bUOb+pt3IAgPHjx59RWFjYGgCmTJmy+6yzzvLkSEitHEKolUPDqZVDzdTKQa0cALVyUCsHERFplGIWQiS7k3yJ5BaShSTfJNmfZArJAneZbJJzo9jG9Fqmtyb5BsmNJDeQnNXQbYiISOzEJIRIEsASAKvN7EwzywAwHUC30OXMLNfMJkaxqRpDyPWomQ0AcA6A75K8PIrtiIhIDMTqSOgiAAEzmx+cYGZ5ZvZu6EIkLyT5uvu+DcmFJN8n+RHJH7jTbyX5Z5LLSW4m+bA7fRaAViTzSP4pdFwzO2Jmb7nvjwH4EECvGO2riIg0UKxCKBNApA/gmwFglZkNhhNij5Bs487LAnA9gEEArid5hplNBXDUzLLMLKe2QUl2ADAaQI1Xa0mOI5lLMrfySGmEJYuISDQa040JlwCYSjIPwGoASQCCt3OtNLNSMysHUAigTzgDkmwG4EUAc81sa03LmNkCM8s2s+zE1u2j3AUREYlErL4ntAHANRGuQwBXm1nRNyaS3wYQej96JcKvewGAzWb2WIS1iIiErftbefUt0hpv5YXdyqHkoqxG18phwoQJPRctWtT50KFDiUeOHPkoOP3o0aO85pprUvPz81t36NDh+KJFi7ampaUdC3fcWB0JrQLQkuRPghNIDiZ5QR3rrAAwwb2pASTPCWM7AZLNa5pB8kEA7QH8IuyqRUSkRlddddWXa9euPeG7Tr/73e+6tG/f/vj27dsL7r777j2TJk2K6Pp7TELInG/AjgHwPfcW7Q0AZgI44Vu4IR4A0BzAevcW7gfC2NQCd/lv3JhAsheca0wZAD50b164I/I9ERFpnOLZygEARo4cebhPnz6B6tNff/31DrfffvsXAHDbbbcdfO+999pVVVWFvR8xe2yPme0GcF0tszPdZVbDuf4DMzsK4Kc1jPMMgGdCPo8KeT8FwAlfSTeznXBO74mInHTi3cqhLnv27GmRmpp6DACaN2+Otm3bVu7Zs6dZ8Anf9WlMNyaIiEgY4t3KoS41Pfqtvid8h1IIiYg0MX60cqhN9+7dj23btq0FAAQCAXz11VeJp59++gmhWBuFkIhIE+NHK4faXHHFFV8uXLiwMwA8/fTTHYcOHVqWkBB+tKiVg4hIlIJP167NSdLKodeSJUs6lZeXJ3Tr1u2snJyc/XPmzNn985//fP/VV1+d2rt378z27dtXvvzyy3X3tahGrRxCqJVDw6mVQ83UykGtHAC1clArBxERaZQUQiIi4huFkIhIA+hSRniqqqoIoNZvryqEREQilJSUhC+++EJBVI+qqiru27evPYCC2pbR3XEiIhHq1asXdu7ciX379oW1fElJSbPKysouMS6rMaoCUHD8+PFaH5umEBIRiVDz5s2RmnrC49pqlZGRkW9m2TEsqcnS6TgREfGNQkhERHyjEBIREd8ohERExDcKIRER8Y1CSEREfKMQEhER3yiERETEN2rlECI7O9tyc3P9LkNETjIkP9CXVWumIyEREfGNQkhERHyjEBIREd8ohERExDcKIRER8Y1CSEREfKMQEhER3yiERETENwohERHxjUJIRER8oxASERHfKIRERMQ3zfwuoDHJ31WKlKlvxGz8dulTPR1v/JrfeToeAJQfnOPpeNenTvF0vJo8lbQyZmPPv+CqBq1XclGWp3WInKx0JCQiIr5RCImIiG8UQiIi4huFkIiI+EYhJCIivlEIiYiIbxRCIiLiG4WQiIj4RiEkIiK+UQiJiIhvFEIiIuIbhZCIiPhGISQiIr5RCImIiG8UQiIi4huFkIiI+EYhJCIivlEIiYiIbxRCIiLiG4WQiIj4JmYhRLI7yZdIbiFZSPJNkv1JppAscJfJJjk3im1Mr2Pef5HcQfKrho4vIiKxFZMQIkkASwCsNrMzzSwDwHQA3UKXM7NcM5sYxaZqDSEArwEYEsXYIiISY7E6EroIQMDM5gcnmFmemb0buhDJC0m+7r5vQ3IhyfdJfkTyB+70W0n+meRykptJPuxOnwWgFck8kn+qXoCZ/dPMPo/R/omIiAeaxWjcTAAfRLjODACrzOx2kh0ArCP5d3deFoBzAFQAKCL5uJlNJXm3mWVFUyjJcQDGAUDiaV2jGUpERCLUmG5MuATAVJJ5AFYDSALQ25230sxKzawcQCGAPl5t1MwWmFm2mWUntm7v1bAiIhKGWB0JbQBwTYTrEMDVZlb0jYnkt+EcAQVVInZ1i4hIHMXqSGgVgJYkfxKcQHIwyQvqWGcFgAnuTQ0geU4Y2wmQbB5dqSIi4peYhJCZGYAxAL7n3qK9AcBMALvrWO0BAM0BrHdv4X4gjE0tcJc/4cYEkg+T3AmgNcmdJGdGuBsiIhJjMTutZWa7AVxXy+xMd5nVcK7/wMyOAvhpDeM8A+CZkM+jQt5PATCllu3/CsCvGlC6iIjESWO6MUFERE4xCiEREfGNQkhERHyjEBIREd8ohERExDcKIRER8Y1CSEREfKMQEhER3yiERETENwohERHxjUJIRER8oxASERHfKIRERMQ3CiEREfGNQkhERHyjEBIREd8ohERExDcKIRER8Y1CSEREfEMz87uGRiM7O9tyc3P9LkNETjIkPzCzbL/raIx0JCQiIr5RCImIiG8UQiIi4huFkIiI+EYhJCIivlEIiYiIbxRCIiLiG4WQiIj4RiEkIiK+UQiJiIhvFEIiIuIbhZCIiPhGISQiIr5RCImIiG/UyiEEyTIART6X0QXAfp9rAFRHY6sBUB2NrQYg/Dr6mFnXWBfTFDXzu4BGpsjvnh8kc/2uQXU0vhpUR+OroTHV0ZTpdJyIiPhGISQiIr5RCH3TAr8LQOOoAVAdoRpDDYDqCNUYagAaTx1Nlm5MEBER3+hISEREfKMQAkDyMpJFJD8lOdXjsc8g+RbJT0huIPlzd3onkn8judn9s2PIOtPcWopIXhoy/TyS+e68uSTZgHoSSX5E8nW/6iDZgeSrJDe6P5eh8a6D5C/d/x4FJF8kmRSPGkguJLmXZEHINM+2S7IlyZfd6WtJpkRQxyPuf5P1JJeQ7BDLOmqqIWTePSSNZBc/fhbu9AnutjaQfDjWdZyyzOyUfgFIBLAFQF8ALQB8DCDDw/GTAZzrvm8HYBOADAAPA5jqTp8K4L/d9xluDS0BpLq1Jbrz1gEYCoAA/h+AyxtQzyQALwB43f0c9zoAPAvgDvd9CwAd4lkHgJ4AtgFo5X5+BcCt8agBwPkAzgVQEDLNs+0CuAvAfPf9DQBejqCOSwA0c9//d6zrqKkGd/oZAFYA+AxAF59+FhcB+DuAlu7n02Ndx6n68r0Av1/uX5oVIZ+nAZgWw+0tBfA9OF+KTXanJcP5jtIJ23f/ZxzqLrMxZPqPADwZ4bZ7AVgJYAS+DqG41gHgNDgBwGrT41YHnBDaAaATnO/KvQ7nH+C41AAgpdo/eJ5tN7iM+74ZnC9SMpw6qs0bA+BPsa6jphoAvArgbADF+DqE4vqzgPOLycU1LBfTOk7Fl07Hff0PUtBOd5rn3MPwcwCsBdDNzD4HAPfP0+upp6f7Ppo6HwPwKwBVIdPiXUdfAPsAPE3ntOBTJNvEsw4z2wXgUQDbAXwOoNTM/hrPGqrxcrv/WsfMjgMoBdC5ATXdDue3+bjWQfJKALvM7ONqs+L9s+gPYLh7+uxtkoN9quOkpxByDp2r8/yWQZJtASwG8AszO9SAeqKqk+QoAHvN7INwV4lFHXB+EzwXwP+Y2TkADsM5BRW3OtxrLj+AczqlB4A2JG+MZw1hash2o66J5AwAxwH8KZ51kGwNYAaA+2uaHY8aQjQD0BHAdwDcC+AV9xqPL/9NTmYKIec3ljNCPvcCsNvLDZBsDieA/mRmf3Yn7yGZ7M5PBrC3nnp2uu8bWud3AVxJshjASwBGkPxfH+rYCWCnma11P78KJ5TiWcfFALaZ2T4zCwD4M4B/i3MNobzc7r/WIdkMQHsAB8IthOQtAEYByDH3/FEc6zgTzi8GH7t/T3sB+JBk9zjWELQTwJ/NsQ7O2YMuPtRx0lMIAe8D6EcylWQLOBcOl3k1uPvb0x8BfGJmc0JmLQNwi/v+FjjXioLTb3DvqEkF0A/AOvc0TRnJ77hj3hyyTr3MbJqZ9TKzFDj7uMrMbvShjhIAO0imuZNGAiiMcx3bAXyHZGt33ZEAPon3zyKEl9sNHesaOP+dwz1CvAzAFABXmtmRavXFvA4zyzez080sxf17uhPOTT0l8f5ZAPgLnGunINkfzg00+32o4+Tn90WpxvAC8H04d61tATDD47GHwTn0Xg8gz319H8454ZUANrt/dgpZZ4ZbSxFC7rYCkA2gwJ03Dw28uAngQnx9Y0Lc6wCQBSDX/Zn8Bc5pj7jWAeA/AWx0138ezt1OMa8BwItwrkMF4Pwj+2MvtwsgCcAiAJ/CuVurbwR1fArn2kXw7+n8WNZRUw3V5hfDvTHBh59FCwD/6477IYARsa7jVH3piQkiIuIbnY4TERHfKIRERMQ3CiEREfGNQkhERHyjEBIREd8ohMQ37lOSZ4d8vofkTI/GfobkNV6MVc92rqXzJPC3Yr2teuooZsgTp0WaCoWQ+KkCwA8b2z+eJBMjWPzHAO4ys4tiVY/IyUwhJH46Dqc98i+rz6h+JEPyK/fPC90HSr5CchPJWSRzSK5ze7mcGTLMxSTfdZcb5a6fSKdvzvt0+ub8NGTct0i+ACC/hnp+5I5fQPK/3Wn3w/ky8nySj1RbPpnkOyTz3HWGu9P/h2QunR41/xmyfDHJ35Bc484/l+QKkltIjg+p8R06vX4KSc4necL/wyRvdH8eeSSfdPc50f2ZFrj7ccLPXMQPzfwuQE55vwewniFNw8JwNoB0OM/f2grgKTMbQqdh4AQAv3CXSwFwAZxnkr1F8ltwHqdSamaDSbYE8H8k/+ouPwRAppltC90YyR5w+uucB+AggL+SvMrMfk1yBIB7zCy3Wo1j4bQI+S/3yKq1O32GmR1wp60keZaZrXfn7TCzoSR/C+AZOM/7SwKwAcD8kBoz4PTaWQ7gh3CevxesNR3A9QC+a2YBkk8AyHHH6Glmme5yHer/MYvEno6ExFfmPFH8OQATI1jtfTP73Mwq4DwiJRgi+XCCJ+gVM6sys81wwmoAnL5BN5PMg9NSozOc538BzjPAvhFArsEAVpvzwNPg06XPr69GALe517gGmVmZO/06kh8C+AjAQDiBEhR8ZmE+gLVmVmZm+wCUh4TGOjPbamaVcB43M6zadkfCCcv33X0cCad9xlYAfUk+7j4jrq4nuYvEjY6EpDF4DM7zuZ4OmXYc7i9J7gMhW4TMqwh5XxXyuQrf/Dtd/ZlUwUfuTzCzFaEzSF4Ip61ETSJuo25m75A8H8AVAJ53T9e9C+AeAIPN7CDJZ+Ac6QSF7kf1fQzuV037VL3WZ81s2gk7QZ4N4FIAPwNwHZyeQSK+0pGQ+M7MDsDpZPnjkMnFcH6jB5zeP80bMPS1JBPc60R94TxwcgWAO+m01wDJ/nSa6tVlLYALSHZxT6P9CMDbda1Asg+c/k1/gPMU9XPhdJU9DKCUZDcAlzdgn4bQeeJ7ApzTbv+oNn8lgGtInu7W0YlkH/fmjwQzWwzg3916RHynIyFpLGYDuDvk8x8ALCW5Ds4/rLUdpdSlCE5YdAMw3szKST4F55Tdh+4R1j4AV9U1iJl9TnIagLfgHGm8aWb1tWy4EMC9JAMAvgJws5ltI/kRnOszWwH8XwP2aQ2AWQAGAXgHwJJqtRaSvA/OdasEOE+G/hmAo3C62QZ/8TzhSEnED3qKtkgT4Z4yvMfMRvlciohndDpORER8oyMhERHxjY6ERETENwohERHxjUJIRER8oxASERHfKIRERMQ3CiEREfHN/we09iv2D4JY2QAAAABJRU5ErkJggg==", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaEAAAGDCAYAAACCzK//AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAAu3UlEQVR4nO3deXxU5d028OtKWMImuxCgkKgQEoLGGml5CipgUV5xq1otca+laAVb0ILgY3mrb6UqlCJapRa3uiLygMsDtSDUPlI01EhCIKCAbCaAIHvCJPm9f8yZxzFmmTDLPQPX9/OZDzPnnLkXa7m8zzlzfjQziIiIuJDkegAiInLyUgiJiIgzCiEREXFGISQiIs4ohERExBmFkIiIOKMQkhMKySdJ/meE2upJ8hDJZO/zcpK3RaJtr73/JnlTpNoTSURNXA9ApDFIbgHQBUAlgCoAxQCeBzDHzKrNbEwj2rnNzP5e1zFmthVA63DH7PU3FcAZZnZ9UPsjItG2SCLTSkgS0aVm1gZALwDTAEwE8JdIdkBS/4EmEgMKIUlYZrbfzBYBuBbATSSzST5L8kEAINmJ5FskvyK5l+T7JJNIvgCgJ4A3vdNtvyaZRtJI/pTkVgDLgrYFB9LpJD8kuZ/kQpIdvL4uILk9eHwkt5C8kOTFACYDuNbr7xNv//+e3vPGdR/Jz0nuIvk8ybbevsA4biK5leQeklOi+09XJDYUQpLwzOxDANsBDK6xa4K3vTP8p/Am+w+3GwBshX9F1drMHg76zvkAMgFcVEd3NwK4FUA3+E8JzgphfIsB/A7Aq15/Z9Vy2M3eawiA0+A/DTi7xjGDAGQAGAbgfpKZDfUtEu8UQnKi2AmgQ41tPgCpAHqZmc/M3reGH5Y41cwOm9nROva/YGZFZnYYwH8C+HHgxoUw5QGYYWabzOwQgHsBXFdjFfZ/zeyomX0C4BMAtYWZSEJRCMmJojuAvTW2PQLgUwB/I7mJ5KQQ2tnWiP2fA2gKoFPIo6xbN6+94LabwL+CCygNen8EEbppQsQlhZAkPJLnwh9C/wzebmYHzWyCmZ0G4FIA40kOC+yuo7mGVkrfCXrfE/7V1h4AhwG0DBpTMvynAUNtdyf8N1oEt10JoKyB74kkNIWQJCySp5AcCeAVAH81s8Ia+0eSPIMkARyA/5buKm93GfzXXhrrepJZJFsC+C2A182sCsAGACkkLyHZFMB9AJoHfa8MQBrJuv4/9zKAX5FMJ9kaX19DqjyOMYokDIWQJKI3SR6E/9TYFAAzANxSy3G9AfwdwCEAKwE8YWbLvX0PAbjPu3Pu7kb0/QKAZ+E/NZYCYBzgv1MPwB0AngawA/6VUfDdcvO8P78k+e9a2p3rtf0PAJsBlAMY24hxiSQkqqidiIi4opWQiIg4oxASERFnFEIiIuKMQkhERJxRCImIiDN6UnCQTp06WVpamuthiMgJZvXq1XvMrHPDR558FEJB0tLSkJ+f73oYInKCIfl5w0ednHQ6TkREnFEIiYiIMwohERFxRiEkIiLOKIRERMQZhZCIiDijEBIREWcUQiIi4oxCSEREnFEIiYiIMwohERFxRiEkIiLO6AGmQQp37EfapLej3s+WlFFR7yOgf3rPmPTz2kOVMekHAJZd8HjM+irfNyMm/VybPjEm/QDA0ylLY9LP4PNeiEk/AJDH+THpp3RITkz6OZloJSQiIs4ohERExBmFkIiIOKMQEhERZxRCIiLijEJIREScUQiJiIgzCiEREXFGISQiIs4ohERExBmFkIiIOKMQEhERZxRCIiLijEJIREScUQiJiIgzCiEREXFGISQiIs4ohERExBmFkIiIOKMQEhERZxRCIiLiTNRCiGRXkq+Q/IxkMcl3SPYhmUayyDsml+SsMPqYXM++5SRLSBZ4r1OPtx8REYmOJtFolCQBLADwnJld523LAdAFwLbAcWaWDyA/jK4mA/hdPfvzvD5ERCQORWslNASAz8yeDGwwswIzez/4IJIXkHzLe9+K5FySH5H8mOTl3vabSb5BcjHJjSQf9rZPA9DCW+W8GKV5iIhIFEVlJQQgG8DqRn5nCoBlZnYryXYAPiT5d29fDoCzAVQAKCH5mJlNInmnmeXU0+YzJKsAzAfwoJlZzQNIjgYwGgCST+ncyCGLiEg44unGhOEAJpEsALAcQAqAnt6+pWa238zKARQD6BVCe3lm1h/AYO91Q20HmdkcM8s1s9zklm3DnIKIiDRGtEJoLYBzGvkdArjKzHK8V08zW+ftqwg6rgohrODMbIf350EALwEY0MjxiIhIlEUrhJYBaE7yZ4ENJM8leX4931kCYKx3UwNInh1CPz6STWtuJNmEZCfvfVMAIwEUNWYCIiISfVEJIe/ay5UAfujdor0WwFQAO+v52gMAmgJY493C/UAIXc3xjq95Y0JzAEtIrgFQAGAHgD83ahIiIhJ10boxAWa2E8CP69id7R2zHP7rPzCzowB+Xks7zwJ4NujzyKD3EwFMrOU7h9H404EiIhJj8XRjgoiInGQUQiIi4oxCSEREnFEIiYiIMwohERFxRiEkIiLOKIRERMQZhZCIiDijEBIREWcUQiIi4oxCSEREnFEIiYiIMwohERFxRiEkIiLOKIRERMQZhZCIiDijEBIREWcUQiIi4oxCSEREnKGZuR5D3MjNzbX8/HzXwxCREwzJ1WaW63oc8UgrIRERcUYhJCIiziiERETEGYWQiIg4oxASERFnFEIiIuKMQkhERJxRCImIiDMKIRERcUYhJCIiziiERETEmSauBxBPCnfsR9qkt6Pez5aUUVHvI6B/es+Y9PPaQ5Ux6QcAll3weMz6Kt83Iyb9XJs+MSb9AMDTKUtj0s/g816IST8AkMf5MemndEhOTPo5mWglJCIiziiERETEGYWQiIg4oxASERFnFEIiIuKMQkhERJxRCImIiDMKIRERcUYhJCIizuiJCSIijeTz+bB9+3aUl5eHdPy7777b/5NPPtkS3VHFpWoARZWVlbedc845u2o7QCEkItJI27dvR5s2bZCWlgaSDR5fVVVVmZ2dvScGQ4sr1dXV3L17d1ZpaenTAC6r7RidjhMRaaTy8nJ07NgxpAA6mSUlJVnnzp33A8iu85gYjkdE5IShAApNUlKSoZ6sUQiJiIgzuiYkIhKmEErAtAQ+PyfU9rZMu2T18Yxj/Pjx3Vq3bl3129/+tux4vl+fsWPHdp83b17HAwcOJB85cuTjSLWrlZCIiDToiiuu+GrVqlXrIt2uQkhEJAHNnj27Y58+fbIyMjKyrrjiivSa+6dPn94pOzs7MyMjI+uiiy46/eDBg0kAMHfu3Pa9e/ful5GRkZWbm5sBAPn5+Sn9+/fP7Nu3b1afPn2yCgsLm9dsb9iwYYd79erli/Q8dDpORCTB5Ofnpzz66KOpK1euXJ+amlpZVlaWXPOYvLy8fRMmTNgDAOPGjes2a9asTlOmTNk1bdq01L/97W8b0tPTfXv27EkGgMcee6zzHXfcUXb77bfvLS8vZ2Vl7ColayUkIpJglixZcsqll166LzU1tRIAunTpUlXzmNWrV7c455xzMvr06ZM1f/78jmvXrk0BgNzc3EN5eXlp06dP7xQIm4EDBx6ePn166pQpU7pu3LixWevWrS1Wc1EIiYgkGDMDyXqDYvTo0emzZ8/eumHDhuKJEyfurKioSAKAl156aeuDDz64c9u2bc1ycnL6lZaWJo8ZM2bvwoULP23RokX1iBEj+ixatKhNbGaiEBIRSTgXX3zxgUWLFnUoLS1NBoDaTscdOXIkqWfPnr6Kigq+8sorHQLb165d23zo0KGHZ86cubN9+/aVmzZtalZcXNwsMzOz4r777ts1fPjwrwoKClrEai5RuyZEsiuAmQDOBVABYAuAXwI4BuAtM8smmQvgRjMbd5x9TDaz3zVwzCIAp5lZnb/YFREJx5Zpl9S7v6io6Eh2dnbE7izLzc0tnzBhwheDBw/um5SUZNnZ2Ufmz5+/JfiYSZMm7RwwYEBm9+7dj2VmZh45dOhQMgD86le/6rFly5bmZsZBgwYd+P73v390ypQpXefNm9exSZMm1rlzZ99DDz20s2afY8aM6bFgwYIO5eXlSV26dDkzLy9vz4wZM751XGPRLPKn/uj/KfEHAJ4zsye9bTkA2gDYBi+EItDPITNrXc/+HwG4GsCZofTXPLW3pd40M9xhNWhLyqio9xHQP71nTPp57aHYXchcdsHjMeurfN+MmPRzbfrEmPQDAE+nLI1JP4PPeyEm/QBAHufHpJ/SITkAgHXr1iEzMzPk70U6hBLNJ5980umss85Kq21ftE7HDQHgCwQQAJhZgZm9H3wQyQtIvuW9b0VyLsmPSH5M8nJv+80k3yC5mORGkg9726cBaEGygOSLNQdAsjWA8QAejNIcRUQkTNE6HZcNoLG/+J0CYJmZ3UqyHYAPSf7d25cD4Gz4T+uVkHzMzCaRvNPMcupo7wEA0wEcqa9TkqMBjAaA5FM6N3LIIiISjni6MWE4gEkkCwAsB5ACIHAuaamZ7TezcgDFAHrV15B36u8MM1vQUKdmNsfMcs0sN7ll2zCGLyIijRWtldBa+K/FNAYBXGVmJd/YSH4P/hVQQBUaHvdAAOeQ3OIdeyrJ5WZ2QSPHJCIiURStldAyAM1J/iywgeS5JM+v5ztLAIz1bmoAybND6MdHsmnNjWb2JzPrZmZpAAYB2KAAEhGJP1EJIfPfcnclgB+S/IzkWgBTAdR3O98DAJoCWEOyyPvckDne8d+6MUFEROJf1H4nZGY7Afy4jt3Z3jHL4b/+AzM7CuDntbTzLIBngz6PDHo/EUC997aa2RbUU9VPRCRsU+u/npwNtMTrCLmUA6buj6tSDgcPHky69NJLT/v888+bJycnY/jw4V898cQTOyLRdjzdmCAiInFqwoQJZZs3b15bVFRUvGrVqtavvfbaKZFoVyEkIpKAYlnKoU2bNtWXXnrpQQBISUmxM88888i2bduaRWIeCiERkQQTKOWwYsWKDSUlJcVPPfXU1prH5OXl7SsqKlpXUlJSnJGRcXTWrFmdACBQyqGkpKR48eLFnwJfl3JYv3598Zo1a9alp6cfq6vvPXv2JL/77rvtRowYcSASc1EIiYgkGFelHHw+H370ox+dNnr06LKsrKw6g6oxFEIiIgnGVSmHUaNGpZ122mnl999//65IzUUhJCKSYFyUchg3bly3AwcOJP/lL3/ZFsm5qLy3iEi4pu6vd3eil3L47LPPmj722GOp6enp5f369csCgNGjR+8aP378nnDnEpVSDolKpRyOn0o5hEelHMKjUg7xzUUpBxERkQYphERExBmFkIiIOKMQEhERZxRCIiLijEJIRESc0e+ERETC1P+5/g0d0hKrQy/lUHhTYVyVcgCAwYMH9961a1fTqqoqDhgw4ODzzz+/tUmT8CNEKyEREWnQwoULPyspKSnesGHD2i+//LLp3Llz20eiXYWQiEgCimUpBwDo0KFDNQD4fD76fD6SjMg8FEIiIgnGVSmHQYMG9e7cufNZrVq1qrrlllv2RWIuCiERkQTjqpTDP//5z42lpaWfHDt2LOnNN9+MSGVV3ZgQpH/3tsifdkkMeqr/YYeRVBirjm6KVUdA6E/sioShMe0tFqZicMx6ipXSmPUUH0It5fD6669/OnDgwKOzZs3quGLFijaAv5TDsmXLWi1atKhtTk5Ov4KCgrVjxozZO3jw4MMLFixoO2LEiD5PPPHElssuu+xgbe22bNnSRo4c+dWCBQvaXXnllWEXttNKSEQkwcS6lMP+/fuTPv/886aAv7Dd4sWL2/bt2/doJOailZCISJgKb6r/nEOil3I4cOBA0iWXXHLGsWPHWF1dzR/84AcH7rnnnt2RmItKOQTJzc21/Px818MQkTinUg6No1IOIiISlxRCIiLijEJIREScUQiJiIgzCiEREXFGISQiIs7od0IiImFa17f+27WTgZbrEHoph8z16+KulEPA0KFDz9i2bVvzjRs3ro1Ee1oJiYhISJ577rl2rVq1+tZz6sKhlVCQwh37kTbp7aj3syVlVNT7COif3jMm/bz2UGVM+gGAZRc8HrO+yvfNiEk/16ZPjEk/APB0ytKY9DP4vBdi0g8A5HF+TPopHZITk35CMXv27I6zZs3qQhKZmZlH/+u//mtz8P7p06d3euaZZzr7fD6mpaVVvP7665vbtGlTPXfu3PYPPfRQt6SkJGvTpk1Vfn5+SX5+fsott9yS7vP5WF1djfnz53/Wv3//iuD29u/fnzRr1qwuc+bM+fy66647PVLzUAiJiCSYQCmHlStXrk9NTa2s7dlxeXl5+yZMmLAHAMaNG9dt1qxZnaZMmbIrUMohPT3dt2fPnmTg61IOt99++97y8nIGnq4dbPz48d3vuuuustatW1dHci46HScikmBiXcrhgw8+aLF58+bmN95441eRnotCSEQkwYRaymH27NlbN2zYUDxx4sSdFRUVSYC/lMODDz64c9u2bc1ycnL6lZaWJo8ZM2bvwoULP23RokX1iBEj+ixatKhNcFvvv/9+66Kiopbdu3fvf9555/XdsmVL8wEDBmREYi4KIRGRBBPrUg4TJ07cvWvXrjU7duwo/Mc//rE+LS2t4sMPPyyJxFx0TUhEJEyZ6+t/QHail3KIJpVyCNI8tbel3jQz6v3o7rjw6O648OjuuOMXuDtOpRwaR6UcREQkLimERETEGYWQiIg4E1IIkbyGZBvv/X0k3yD53egOTURETnShroT+08wOkhwE4CIAzwH4U/SGJSIiJ4NQQyjwa9xLAPzJzBYCaBadIYmIyMki1N8J7SD5FIALAfyeZHPoepKICADg8THLGjqk5QosC7mUwy+eHBp3pRwGDBiQsWvXrqYpKSnVALB06dIN3bt3D/u3GaGG0I8BXAzgUTP7imQqgHvC7VxERBLH888/v+m88847Esk2Q13NPGVmb5jZRgAwsy8A3BDJgYiISOhmz57dsU+fPlkZGRlZV1xxRXrN/dOnT++UnZ2dmZGRkXXRRRedfvDgwSQAmDt3bvvevXv3y8jIyMrNzc0A/E/l7t+/f2bfvn2z+vTpk1VYWNg8VvMINYT6BX8gmYxGVAkUEZHICZRyWLFixYaSkpLip556amvNY/Ly8vYVFRWtKykpKc7IyDg6a9asTgAQKOVQUlJSvHjx4k+Br0s5rF+/vnjNmjXr0tPTj9XW72233ZbWt2/frHvuuSe1ujoyFR3qDSGS95I8COBMkge810EAuwAsjMgIRESkUWJdygEAXn311U0bNmwoXrly5foPPvig9RNPPNExEnOpN4TM7CEzawPgETM7xXu1MbOOZnZvJAYgIiKNE+tSDgCQnp7uA4D27dtXX3vttXs//PDDVpGYS0in48zsXpLdSf4HyfMCr0gMQEREGifWpRx8Ph+++OKLJgBQUVHBd955p212dvbRSMwlpLvjSE4DcB2AYnz9myED8I9IDEJEJJH94smh9e5P9FIOR48eTbrwwgt7+3w+VldXc/DgwQfGjx+/OxJzCamUA8kSAGeaWUXIDZNdAcwEcC6ACgBbAPwSwDEAb5lZNslcADea2bhGj9zfx2Qz+10d+xYDSIU/aN8H8Asz+9Z502Aq5XD8VMohPCrlEB6VcohvkSjlsAlA01A7JEkACwAsN7PTzSwLwGQAXYKPM7P84w0gz+R69v3YzM4CkA2gM4BrwuhHRESiINQfqx4BUEByKfyrGgBAPQEyBIDPzJ4MOrYAAEimBbaRvADA3WY2kmQrAI8B6O+Na6qZLSR5M4DLALQEcDqABWb2a+8UYQuSBQDWmlle8ADM7EDQHJvBf/pQRETiSKghtMh7hSobQGMfOzEFwDIzu5VkOwAfkvy7ty8HwNnwB2AJycfMbBLJO80sp64GSS4BMADAfwN4vZHjERGRKAsphMzsOZItAPQ0s5IojWU4gMtI3u19TgEQuKCx1Mz2AwDJYgC9AGxrqEEzu4hkCoAXAQwF8G7NY0iOBjAaAJJP6RzuHEREpBFCrSd0KYACAIu9zzkk61sZrUXjn6hAAFeZWY736mlmgQt5wTdEVCH0FRzMrBz+VdzldeyfY2a5Zpab3LJtI4csIiLhCPXGhKnwn9b6Cvjf6zvfelZRkGUAmpP8WWADyXNJnl/Pd5YAGOvd1ACSZ4cwLh/Jb90wQbK195BVkGwC4P8AWB9CeyIiEkOhrigqzWy/lw8BdV7oNzMjeSWAmSQnASjH17do1+UB+G/pXuMF0RYAIxsY1xzv+H/XuDGhFYBFXsmJZPhD8cnaGhARCdf0axv6qwotlzTi7NCEV9+Ku1IO5eXlvOWWW3quXLmyDUn7zW9+s+Pmm2/+Ktx2Qw2hIpKjACST7A1gHIAP6vuCme2EvwREbbK9Y5YDWO69Pwrg57W08yyAZ4M+jwx6PxHAt35gYWZl8P8+SUREIuDee+9N7dy5s2/Lli1FVVVV2LVrV8iXReoT6um4sfA/SbsCwMsADqD+VY2IiERRrEs5vPzyy50efPDBUgBITk5G4OGp4Qr12XFHzGyKmZ3rXcSf4l3wFxGRGIt1KYc9e/YkA/7TfVlZWZkjRow4bdu2bdFfCZGc6f35JslFNV+RGICIiDROrEs5+Hw+lpWVNR00aNCh4uLidd/73vcOjx079juRmEtDK6HAw58eBTC9lpeIiMRYrEs5dOnSpTIlJaX6hhtu+AoArr/++r1FRUUtIzGXhuoJrfb+XFHbKxIDEBGRxol1KYekpCQMGzZs/9tvv90GAN55551TevfuHf1SDiQLUf+t2GdGYhAiIolswqtv1bs/0Us5AMCMGTO2jxo1Kv3uu+9O7tixY+Xzzz+/peYxx6PeUg7e7dhd8O1H5PQCsNPMPo3EIOKFSjkcP5VyCI9KOYRHpRziWzilHP4A4ICZfR78gv+p2n+I8DhFROQk01AIpZnZmpobzSwfQFpURiQiIieNhkIopZ59LerZJyIi0qCGQuij4IeQBpD8KRpfL0hEROQbGvrF6y8BLCCZh69DJxf+SqVXRnFcIiJyEqg3hLwHgf4HySHwHjoK4G0zWxb1kYmIyAkv1Mqq7wF4L8pjERFJSNsnvV/v/nZAy+14P+RSDj2mDY6rUg779u1LGjhwYN/A57KysqZXXnnl3rlz5zZY4bohEXkAnYiInLjat29fvX79+uLA5379+mVec801+yLRdqilHEREJI7EupRDQGFhYfMvv/yy6UUXXXQoEvPQSkhEJMEESjmsXLlyfWpqamVtz47Ly8vbN2HChD0AMG7cuG6zZs3qNGXKlF2BUg7p6em+QImGQCmH22+/fW95eTkDT9euzXPPPdfhsssu25uUFJk1jFZCIiIJJtalHIItWLCgww033LA3UnNRCImIJJhYl3IIWLlyZYuqqioOHjz4SKTmotNxQfp3b4v8aZfEoKf9MejDrzBWHd0Uq46A0B8bGQlDY9pbLEzF4Jj1FCulMespPlx88cUHrr766jMmT55c1rVr16qysrLkmquhmqUcUlNTfcDXpRyGDh16eMmSJe02bdrUbO/evVWZmZkV/fr127Vp06bmBQUFLS677LKDNft94YUXOlx55ZURWwUBCiERkbD1mFZ/sJ8IpRwAYNGiRR3efPPNjZGaB9BAKYeTTW5uruXn57sehojEOZVyaJxwSjmIiIhEjUJIREScUQiJiIgzCiEREXFGISQiIs4ohERExBn9TkhEJExTp05t6JCWr7/+esilHKZOnRpXpRwA4Kmnnuowffr0rgDQpUsX32uvvbY58NigcGglJCIi9fL5fLj33nu/s2LFig0bNmwo7tev39FHHnnk1Ei0rRASEUlAsSzlUF1dTTPDwYMHk6qrq3HgwIGkbt26HYvEPHQ6TkQkwcS6lEPz5s1txowZW7/73e/2a9GiRVWvXr0qnn/++a2RmItCKEjhjv1Im/R21PvZkjIq6n0E9E/vGZN+Xnso7FPDIVt2weMx66t834yY9HNt+sSY9AMAT6csjUk/g897ISb9AEAe58ekn9IhOTHppyGhlnK4//77ux88eDD58OHDyeeff/5+4OtSDlddddW+vLy8fYC/lMOjjz6aun379mbXXXfdvv79+1cEt1VRUcE5c+Z0XrVqVXFmZmbFzTff3HPy5MmpDz/88BfhzkWn40REEkysSzn861//agEA/fr1q0hKSsJPfvKTvatWrWoVibkohEREEszFF198YNGiRR1KS0uTAaC203E1SzkEtgdKOcycOXNn+/btKzdt2tSsuLi4WWZmZsV99923a/jw4V8VFBS0CG6rV69evk8//TRl586dTQBg8eLFp/Tp06c8EnPR6TgRkTA1dIt2opdySEtL891zzz1fDBo0KKNJkybWo0ePYy+99NLmSMxFpRyCNE/tbak3zYx6P7omFB5dEwqPrgkdv8A1IZVyaByVchARkbikEBIREWcUQiIi4oxCSEREnFEIiYiIMwohERFxRr8TEhEJ09Jlpzd0SMuyZQi5lMOwoZ/FXSmHP//5z+0feeSR1Orqal544YX7n3zyye2RaFcrIRERqVdpaWny/fff32P58uUbPv3007W7du1qsnDhwjYNf7NhCiERkQQUy1IOJSUlzdPT0yu6detWCQDDhg07MG/evPaRmIdOx4mIJJhYl3LIysqq+Oyzz1JKSkqanXbaaccWLVrU3ufzMRJz0UpIRCTBhFrK4Zxzzsno06dP1vz58zuuXbs2Bfi6lMP06dM7BcJm4MCBh6dPn546ZcqUrhs3bmzWunXrbzzPrXPnzlV/+MMfPr/mmmtOO/fcc/v27NmzIjk5OSLPfFMIiYgkmFiXcgCAUaNG7V+zZs36goKC9RkZGeWnn356xbd7bTyFkIhIgol1KQcA2LFjRxMA2L17d/LTTz996h133LE7EnPRNSERkTANG/pZvfsTvZQDAIwZM+Y7xcXFLQFg4sSJO88888yIrIRUyiGISjkcP5VyCI9KOYRHpRzim0o5iIhIXIpaCJHsSvIVkp+RLCb5Dsk+JNNIFnnH5JKcFUYfk+vY3pLk2yTXk1xLctrx9iEiItETlRAiSQALACw3s9PNLAvAZABdgo8zs3wzGxdGV7WGkOdRM+sL4GwAPyA5Iox+REQkCqK1EhoCwGdmTwY2mFmBmb0ffBDJC0i+5b1vRXIuyY9Ifkzycm/7zSTfILmY5EaSD3vbpwFoQbKA5IvB7ZrZETN7z3t/DMC/AfSI0lxFROQ4RSuEsgE09gF8UwAsM7Nz4Q+xR0i28vblALgWQH8A15L8jplNAnDUzHLMLK+uRkm2A3ApgFqvxpIcTTKfZH7Vkf2NHLKIiIQjnm5MGA5gEskCAMsBpAAI3Nq11Mz2m1k5gGIAvUJpkGQTAC8DmGVmm2o7xszmmFmumeUmt2wb5hRERKQxovU7obUArm7kdwjgKjMr+cZG8nsAgu9Hr0Lo454DYKOZzWzkWEREQtb1vYKGDmmJ9wpCLuVQOiQn7ko5jB07tvu8efM6HjhwIPnIkSMfB7YfPXqUV199dXphYWHLdu3aVc6bN29TRkbGsVDbjdZKaBmA5iR/FthA8lyS59fznSUAxno3NYDk2SH04yPZtLYdJB8E0BbAL0MetYiI1OqKK674atWqVd/6rdMf//jHTm3btq3cunVr0Z133lk2fvz4Rl1/j0oImf8XsFcC+KF3i/ZaAFMBfOtXuEEeANAUwBrvFu4HQuhqjnf8N25MINkD/mtMWQD+7d28cFvjZyIiEp9iWcoBAIYNG3a4V69evprb33rrrXa33nrrlwBwyy237Pvggw/aVFdXhzyPqD22x8x2AvhxHbuzvWOWw3/9B2Z2FMDPa2nnWQDPBn0eGfR+IoBv/dTczLbDf3pPROSEE+tSDvUpKytrlp6efgwAmjZtitatW1eVlZU1CTzhuyHxdGOCiIiEINalHOpT26PfGnrCdzCFkIhIgnFRyqEuXbt2PbZ58+ZmAODz+XDo0KHkU0899VuhWBeFkIhIgnFRyqEul1xyyVdz587tCADPPPNM+4EDBx5MSgo9WlTKQUQkTIGna9flBCnl0GPBggUdysvLk7p06XJmXl7enhkzZuy866679lx11VXpPXv2zG7btm3Vq6++Wn9dixpUyiGISjkcP5VyCI9KOYRHpRzim0o5iIhIXFIIiYiIMwohEZHjoEsZoamuriaAOn+9qhASEWmklJQUfPnllwqiBlRXV3P37t1tARTVdYzujhMRaaQePXpg+/bt2L17d0jHl5aWNqmqquoU5WHFo2oARZWVlXU+Nk0hJCLSSE2bNkV6+rce11anrKysQjPLjeKQEpZOx4mIiDMKIRERcUYhJCIiziiERETEGYWQiIg4oxASERFnFEIiIuKMQkhERJxRKYcgubm5lp+f73oYInKCIblaP1atnVZCIiLijEJIREScUQiJiIgzCiEREXFGISQiIs4ohERExBmFkIiIOKMQEhERZxRCIiLijEJIREScUQiJiIgzCiEREXGmiesBxJPCHfuRNuntqPezJWVU1PsI6J/eMyb9jFn5x5j0AwDl+2bErK9r0yfGpJ+nU5bGpB8AePL8K2LST+mQnJj0I4lNKyEREXFGISQiIs4ohERExBmFkIiIOKMQEhERZxRCIiLijEJIREScUQiJiIgzCiEREXFGISQiIs4ohERExBmFkIiIOKMQEhERZxRCIiLijEJIREScUQiJiIgzCiEREXFGISQiIs4ohERExBmFkIiIOBO1ECLZleQrJD8jWUzyHZJ9SKaRLPKOySU5K4w+Jtez7/+R3Eby0PG2LyIi0RWVECJJAAsALDez080sC8BkAF2CjzOzfDMbF0ZXdYYQgDcBDAijbRERibJorYSGAPCZ2ZOBDWZWYGbvBx9E8gKSb3nvW5GcS/Ijkh+TvNzbfjPJN0guJrmR5MPe9mkAWpAsIPlizQGY2b/M7IsozU9ERCKgSZTazQawupHfmQJgmZndSrIdgA9J/t3blwPgbAAVAEpIPmZmk0jeaWY54QyU5GgAowEg+ZTO4TQlIiKNFE83JgwHMIlkAYDlAFIA9PT2LTWz/WZWDqAYQK9IdWpmc8ws18xyk1u2jVSzIiISgmithNYCuLqR3yGAq8ys5Bsbye/BvwIKqEL0xi0iIjEUrZXQMgDNSf4ssIHkuSTPr+c7SwCM9W5qAMmzQ+jHR7JpeEMVERFXohJCZmYArgTwQ+8W7bUApgLYWc/XHgDQFMAa7xbuB0Loao53/LduTCD5MMntAFqS3E5yaiOnISIiURa101pmthPAj+vYne0dsxz+6z8ws6MAfl5LO88CeDbo88ig9xMBTKyj/18D+PVxDF1ERGIknm5MEBGRk4xCSEREnFEIiYiIMwohERFxRiEkIiLOKIRERMQZhZCIiDijEBIREWcUQiIi4oxCSEREnFEIiYiIMwohERFxRiEkIiLOKIRERMQZhZCIiDijEBIREWcUQiIi4oxCSEREnFEIiYiIMzQz12OIG7m5uZafn+96GCJygiG52sxyXY8jHmklJCIiziiERETEGYWQiIg4oxASERFnFEIiIuKMQkhERJxRCImIiDMKIRERcUYhJCIiziiERETEGYWQiIg4oxASERFnFEIiIuKMQkhERJxRKYcgJA8CKHE9jgjrBGCP60FEmOaUGDSnr/Uys86RHsyJoInrAcSZkhOt5gfJfM0p/mlOieFEnJNrOh0nIiLOKIRERMQZhdA3zXE9gCjQnBKD5pQYTsQ5OaUbE0RExBmthERExBmFEACSF5MsIfkpyUmuxxMukt8h+R7JdSTXkrzL9ZgihWQyyY9JvuV6LJFCsh3J10mu9/43G+h6TOEi+Svv370iki+TTHE9psYiOZfkLpJFQds6kHyX5Ebvz/Yux3giOOlDiGQygMcBjACQBeAnJLPcjipslQAmmFkmgO8D+MUJMKeAuwCscz2ICPsjgMVm1hfAWUjw+ZHsDmAcgFwzywaQDOA6t6M6Ls8CuLjGtkkAlppZbwBLvc8ShpM+hAAMAPCpmW0ys2MAXgFwueMxhcXMvjCzf3vvD8L/l1p3t6MKH8keAC4B8LTrsUQKyVMAnAfgLwBgZsfM7Cung4qMJgBakGwCoCWAnY7H02hm9g8Ae2tsvhzAc9775wBcEcsxnYgUQv6/nLcFfd6OE+Av7ACSaQDOBrDK8VAiYSaAXwOodjyOSDoNwG4Az3inGZ8m2cr1oMJhZjsAPApgK4AvAOw3s7+5HVXEdDGzLwD/f+wBONXxeBKeQghgLdtOiFsGSbYGMB/AL83sgOvxhIPkSAC7zGy167FEWBMA3wXwJzM7G8BhJPgpHu86yeUA0gF0A9CK5PVuRyXxSiHkX/l8J+hzDyTgqYOaSDaFP4BeNLM3XI8nAn4A4DKSW+A/ZTqU5F/dDikitgPYbmaBlerr8IdSIrsQwGYz221mPgBvAPgPx2OKlDKSqQDg/bnL8XgSnkII+AhAb5LpJJvBfwF1keMxhYUk4b/GsM7MZrgeTySY2b1m1sPM0uD/32iZmSX8f12bWSmAbSQzvE3DABQ7HFIkbAXwfZItvX8XhyHBb7YIsgjATd77mwAsdDiWE8JJ/wBTM6skeSeAJfDfxTPXzNY6Hla4fgDgBgCFJAu8bZPN7B13Q5J6jAXwovcfQZsA3OJ4PGExs1UkXwfwb/jv1PwYCfikAZIvA7gAQCeS2wH8BsA0AK+R/Cn8YXuNuxGeGPTEBBERcUan40RExBmFkIiIOKMQEhERZxRCIiLijEJIREScUQiJcySN5PSgz3eTnBqhtp8leXUk2mqgn2u8J2C/F0/jEol3CiGJBxUAfkSyk+uBBPOesB6qnwK4w8yGRGs8IicihZDEg0r4f8z4q5o7aq4YSB7y/ryA5AqSr5HcQHIayTySH5IsJHl6UDMXknzfO26k9/1kko+Q/IjkGpI/D2r3PZIvASisZTw/8dovIvl7b9v9AAYBeJLkI7V859fedz4hOa2W/fd74ygiOcd7ygBIjiNZ7I3vFW/b+SQLvNfHJNt42+8Jmsv/9ba1Ivm2128RyWtD+59DJHZO+icmSNx4HMAakg834jtnAciE/3H7mwA8bWYD6C/iNxbAL73j0gCcD+B0AO+RPAPAjfA/3flcks0B/A/JwJOeBwDINrPNwZ2R7Abg9wDOAbAPwN9IXmFmvyU5FMDdZpZf4zsj4H/c//fM7AjJDrXMY7aZ/dY7/gUAIwG8Cf+DTNPNrIJkO+/YuwH8wsz+x3tAbTnJ4QB6e+MmgEUkzwPQGcBOM7vEa7ttaP9YRWJHKyGJC95Tvp+HvxhaqD7yaidVAPgMQCBECuEPnoDXzKzazDbCH1Z9AQwHcKP3WKNVADrC/xc5AHxYM4A85wJY7j2YsxLAi/DXAqrPhQCeMbMj3jxr1qcBgCEkV5EsBDAUQD9v+xr4H+dzPfyrRQD4HwAzSI4D0M4bx3Dv9TH8j8rp682lEP5V4O9JDjaz/Q2MVSTmFEIST2bCf20luJ5OJbx/T73TVM2C9lUEva8O+lyNb67yaz6byuBfMYw1sxzvlR5U8+ZwHeOrrexHQ1hL/1/v9Je9fgLA1WbWH8CfAQRKYV8C/wrxHACrSTYxs2kAbgPQAsC/SPb1+ngoaC5nmNlfzGyD991CAA95pw1F4opCSOKGt0p4Df4gCtgC/1+kgL9GTdPjaPoakknedaLTAJTA/8Da272SFyDZhw0Xk1sF4HySnbybFn4CYEUD3/kbgFtJtvT6qXk6LhA4e7zTa1d7xyUB+I6ZvQd/Ib92AFqTPN3MCs3s9wDy4V/1LPH6aO19tzvJU73Th0fM7K/wF5lL9BIRcgLSNSGJN9MB3Bn0+c8AFpL8EMBS1L1KqU8J/GHRBcAYMysn+TT8p+z+7a2wdqOBUs1m9gXJewG8B//q4x0zq/dR/ma2mGQOgHySxwC8A2By0P6vSP4Z/tXKFvhLiwD+J7r/1buOQwB/8I59gOQQAFXwl3z4b++aUSaAld49DYcAXA/gDACPkKwG4ANwe4P/pERiTE/RFhERZ3Q6TkREnFEIiYiIMwohERFxRiEkIiLOKIRERMQZhZCIiDijEBIREWcUQiIi4sz/B2HnZBWi1DS/AAAAAElFTkSuQmCC", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaEAAAGDCAYAAACCzK//AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAAv70lEQVR4nO3deXxU9b0+8OdJWMImuxBASLAQEoKiBlpuwQWsSgUrda1xQWsp2kJb0LJ5vdzqrVwVapF6kVrcbt2QUnD5QVsQtFcKRo0QAgFZZJNNEMKSMCSf3x/njI4hywyZmW8GnvfrNa/MnHPmez5nxDw5y5wPzQwiIiIuJLkuQEREzlwKIRERcUYhJCIiziiERETEGYWQiIg4oxASERFnFEJyWiE5k+S/R2msziQPk0z2Xy8leXc0xvbH+38k74jWeCKJqJ7rAkQiQXILgHYATgAoA1AI4AUAs8ys3MxGRjDO3Wb2j6qWMbOtAJrWtmZ/fZMBfMvMbg0Zf3A0xhZJZNoTkkQ01MyaAegCYAqAcQD+FM0VkNQfaCJxoBCShGVmB81sAYCbANxBMpvkcyQfBgCSbUi+SfJLkvtJvkcyieSLADoDeMM/3PZrkmkkjeSPSW4FsCRkWmggnUtyJcmDJOeTbOWv61KS20PrI7mF5OUkrwIwEcBN/vo+8ed/dXjPr+sBkp+R3EPyBZLN/XnBOu4guZXkPpKTYvvpisSHQkgSnpmtBLAdwIAKs8b609vCO4Q30VvcbgOwFd4eVVMzezTkPZcAyARwZRWrux3AXQA6wDskOD2M+hYC+C2AV/31nV/JYsP9x2UAusI7DDijwjL9AWQAGATgQZKZNa1bpK5TCMnpYieAVhWmBQCkAuhiZgEze89qvlniZDM7YmbHqpj/opkVmNkRAP8O4MbghQu1lAtgmpltMrPDACYAuLnCXth/mtkxM/sEwCcAKgszkYSiEJLTRUcA+ytMewzApwD+RnITyfFhjLMtgvmfAagPoE3YVVatgz9e6Nj14O3BBe0KeX4UUbpoQsQlhZAkPJJ94IXQP0Onm1mxmY01s64AhgIYQ3JQcHYVw9W0p3ROyPPO8Pa29gE4AqBxSE3J8A4DhjvuTngXWoSOfQLA7hreJ5LQFEKSsEieRXIIgFcA/K+Zra4wfwjJb5EkgEPwLuku82fvhnfuJVK3kswi2RjAbwC8bmZlANYDSCF5Ncn6AB4A0DDkfbsBpJGs6v+5lwH8imQ6yab4+hzSiVOoUSRhKIQkEb1BshjeobFJAKYBuLOS5boB+AeAwwCWA3jKzJb68x4B8IB/5dx9Eaz7RQDPwTs0lgJgNOBdqQfgXgDPANgBb88o9Gq5Of7PL0h+VMm4s/2x3wWwGUAJgFER1CWSkKimdiIi4or2hERExBmFkIiIOKMQEhERZxRCIiLijEJIRESc0Z2CQ7Rp08bS0tJclyEip5kPP/xwn5m1rXnJM49CKERaWhry8vJclyEipxmSn9W81JlJh+NERMQZhZCIiDijEBIREWcUQiIi4oxCSEREnFEIiYiIMwohERFxRiEkIiLOKIRERMQZhZCIiDijEBIREWcUQiIi4oxuYBpi9Y6DSBv/VszXsyXllpivozZ6pXeO6njD3+4S1fFuSh8X1fHqkpwrm7kuISy7LuvtugQ5TWhPSEREnFEIiYiIMwohERFxRiEkIiLOKIRERMQZhZCIiDijEBIREWcUQiIi4oxCSEREnFEIiYiIMwohERFxRiEkIiLOKIRERMQZhZCIiDijEBIREWcUQiIi4oxCSEREnFEIiYiIMwohERFxRiEkIiLOKIRERMSZmIUQyfYkXyG5kWQhybdJdieZRrLAXyaH5PRarGNiNfOWkiwime8/zj7V9YiISGzUi8WgJAlgHoDnzexmf1pvAO0AbAsuZ2Z5APJqsaqJAH5bzfxcfx0iIlIHxWpP6DIAATObGZxgZvlm9l7oQiQvJfmm/7wJydkkPyD5Mckf+NOHk/wLyYUkN5B81J8+BUAjfy/nzzHaDhERiaGY7AkByAbwYYTvmQRgiZndRbIFgJUk/+HP6w3gAgClAIpIPmlm40n+3Mx6VzPmsyTLAMwF8LCZWcUFSI4AMAIAks9qG2HJIiJSG3XpwoQrAIwnmQ9gKYAUAJ39eYvN7KCZlQAoBNAljPFyzawXgAH+47bKFjKzWWaWY2Y5yY2b13ITREQkErEKoTUALorwPQRwnZn19h+dzWytP680ZLkyhLEHZ2Y7/J/FAF4C0DfCekREJMZiFUJLADQk+ZPgBJJ9SF5SzXsWARjlX9QAkheEsZ4AyfoVJ5KsR7KN/7w+gCEACiLZABERib2YhJB/7mUYgO/5l2ivATAZwM5q3vYQgPoAVvmXcD8Uxqpm+ctXvDChIYBFJFcByAewA8AfI9oIERGJuVhdmAAz2wngxipmZ/vLLIV3/gdmdgzATysZ5zkAz4W8HhLyfByAcZW85wgiPxwoIiJxVpcuTBARkTOMQkhERJxRCImIiDMKIRERcUYhJCIiziiERETEGYWQiIg4oxASERFnFEIiIuKMQkhERJxRCImIiDMKIRERcUYhJCIiziiERETEGYWQiIg4oxASERFnFEIiIuKMQkhERJxRCImIiDM0M9c11Bk5OTmWl5fnugwROc2Q/NDMclzXURdpT0hERJxRCImIiDMKIRERcUYhJCIiziiERETEGYWQiIg4oxASERFnFEIiIuKMQkhERJxRCImIiDMKIRERcaae6wLqlJ0fA5Obu64CvdI7R3W84W93iep4VbkpfVxc1hMLz6Qsdl0CAGDAxS+6LuErgwZudF2CnAG0JyQiIs4ohERExBmFkIiIOKMQEhERZxRCIiLijEJIREScUQiJiIgzCiEREXFGISQiIs7ojgkiIhEKBALYvn07SkpKwlr+73//e69PPvlkS2yrqpPKARScOHHi7osuumhPZQsohEREIrR9+3Y0a9YMaWlpIFnj8mVlZSeys7P3xaG0OqW8vJx79+7N2rVr1zMArqlsGR2OExGJUElJCVq3bh1WAJ3JkpKSrG3btgcBZFe5TBzrERE5bSiAwpOUlGSoJmsUQiIi4ozOCYmI1FLa+LdqWqQx8NlF4Y63ZcrVH55KHWPGjOnQtGnTst/85je7T+X91Rk1alTHOXPmtD506FDy0aNHP47WuNoTEhGRGl177bVfrlixYm20x1UIiYgkoBkzZrTu3r17VkZGRta1116bXnH+1KlT22RnZ2dmZGRkXXnllecWFxcnAcDs2bNbduvWrWdGRkZWTk5OBgDk5eWl9OrVK7NHjx5Z3bt3z1q9enXDiuMNGjToSJcuXQLR3g4djhMRSTB5eXkpjz/+eOry5cvXpaamnti9e3dyxWVyc3MPjB07dh8AjB49usP06dPbTJo0ac+UKVNS//a3v61PT08P7Nu3LxkAnnzyybb33nvv7nvuuWd/SUkJT5w4Ebdt0Z6QiEiCWbRo0VlDhw49kJqaegIA2rVrV1ZxmQ8//LDRRRddlNG9e/esuXPntl6zZk0KAOTk5BzOzc1Nmzp1aptg2PTr1+/I1KlTUydNmtR+w4YNDZo2bWrx2haFkIhIgjEzkKw2KEaMGJE+Y8aMrevXry8cN27cztLS0iQAeOmll7Y+/PDDO7dt29agd+/ePXft2pU8cuTI/fPnz/+0UaNG5YMHD+6+YMGCZvHZEoWQiEjCueqqqw4tWLCg1a5du5IBoLLDcUePHk3q3LlzoLS0lK+88kqr4PQ1a9Y0HDhw4JEnnnhiZ8uWLU9s2rSpQWFhYYPMzMzSBx54YM8VV1zxZX5+fqN4bUvMzgmRbA/gCQB9AJQC2ALglwCOA3jTzLJJ5gC43cxGn+I6JprZb2tYZgGArmZW5Td2RURqY8uUq6udX1BQcDQ7OztqV5bl5OSUjB079vMBAwb0SEpKsuzs7KNz587dErrM+PHjd/bt2zezY8eOxzMzM48ePnw4GQB+9atfddqyZUtDM2P//v0Pfec73zk2adKk9nPmzGldr149a9u2beCRRx7ZWXGdI0eO7DRv3rxWJSUlSe3atTsvNzd337Rp005aLlI0i/6hP3pfJX4fwPNmNtOf1htAMwDb4IdQFNZz2MyaVjP/hwCuB3BeOOvL6ZBseSOqHC5ueqV3jup4w9/uEtXxqnJT+ri4rCcWnklZ7LoEAMCAi190XcJXBg3c6LqEOmvt2rXIzMwMe/loh1Ci+eSTT9qcf/75aZXNi9XhuMsABIIBBABmlm9m74UuRPJSkm/6z5uQnE3yA5Ifk/yBP304yb+QXEhyA8lH/elTADQimU/yzxULINkUwBgAD8doG0VEpJZidTguG0Ck3/idBGCJmd1FsgWAlST/4c/rDeACeIf1ikg+aWbjSf7czHpXMd5DAKYCOFrdSkmOADACADo3172gRETiqS5dmHAFgPEk8wEsBZACIHhcarGZHTSzEgCFAKo9vuQf+vuWmc2raaVmNsvMcswsp21jhZCISDzFak9oDbxzMZEggOvMrOgbE8lvw9sDCipDzXX3A3ARyS3+smeTXGpml0ZYk4iIxFCs9oSWAGhI8ifBCST7kLykmvcsAjDKv6gBJC8IYz0BkvUrTjSz/zGzDmaWBqA/gPUKIBGRuicmIWTeJXfDAHyP5EaSawBMBlDd5XwPAagPYBXJAv91TWb5y590YYKIiNR9MfuekJntBHBjFbOz/WWWwjv/AzM7BuCnlYzzHIDnQl4PCXk+DkC11wWb2RZU09VPRKTWJjevdnY20BivI+xWDph8sE61ciguLk4aOnRo188++6xhcnIyrrjiii+feuqpHdEYuy5dmCAiInXU2LFjd2/evHlNQUFB4YoVK5q+9tprZ0VjXIWQiEgCimcrh2bNmpUPHTq0GABSUlLsvPPOO7pt27YG0dgOhZCISIIJtnJYtmzZ+qKiosKnn356a8VlcnNzDxQUFKwtKioqzMjIODZ9+vQ2ABBs5VBUVFS4cOHCT4GvWzmsW7eucNWqVWvT09OPV7Xuffv2Jf/9739vMXjw4EPR2BaFkIhIgnHVyiEQCOCHP/xh1xEjRuzOysqqMqgioRASEUkwrlo53HLLLWldu3YtefDBB/dEa1sUQiIiCcZFK4fRo0d3OHToUPKf/vSnbdHcFrX3FhGprckHq52d6K0cNm7cWP/JJ59MTU9PL+nZs2cWAIwYMWLPmDFj9tV2W2LSyiFRqZVD7aiVQ+2plUNiUCuHyLho5SAiIlIjhZCIiDijEBIREWcUQiIi4oxCSEREnFEIiYiIM/qekIhILfV6vldNizTGh+G3clh9x+o61coBAAYMGNBtz5499cvKyti3b9/iF154YWu9erWPEO0JiYhIjebPn7+xqKiocP369Wu++OKL+rNnz24ZjXEVQiIiCSierRwAoFWrVuUAEAgEGAgESDIq26EQEhFJMK5aOfTv379b27Ztz2/SpEnZnXfeeSAa26IQEhFJMK5aOfzzn//csGvXrk+OHz+e9MYbb0Sls6ouTAjV4QJgcp7rKrA62gPeEe0BTz+TMcB1Cb7JrguQBBBuK4fXX3/90379+h2bPn1662XLljUDvFYOS5YsabJgwYLmvXv37pmfn79m5MiR+wcMGHBk3rx5zQcPHtz9qaee2nLNNdcUVzZu48aNbciQIV/OmzevxbBhw2rd2E57QiIiCSberRwOHjyY9Nlnn9UHvMZ2CxcubN6jR49j0dgW7QmJiNTS6juqP36R6K0cDh06lHT11Vd/6/jx4ywvL+d3v/vdQ/fff//eaGyLWjmEyMnJsbw894fjRKRuUyuHyKiVg4iI1EkKIRERcUYhJCIiziiERETEGYWQiIg4oxASERFn9D0hEZFaWtuj+su1k4HGaxF+K4fMdWvrXCuHoIEDB35r27ZtDTds2LAmGuNpT0hERMLy/PPPt2jSpMlJ96mrDe0Jhdr5MTC5ea2G6JXeOUrFeEYu/31Ux6tMyYFpURurODMnamNVZ8DFL8ZlPVUZNHCj0/WLzJgxo/X06dPbkURmZuaxv/71r5tD50+dOrXNs88+2zYQCDAtLa309ddf39ysWbPy2bNnt3zkkUc6JCUlWbNmzcry8vKK8vLyUu688870QCDA8vJyzJ07d2OvXr1KQ8c7ePBg0vTp09vNmjXrs5tvvvncaG2HQkhEJMEEWzksX758XWpq6onK7h2Xm5t7YOzYsfsAYPTo0R2mT5/eZtKkSXuCrRzS09MD+/btSwa+buVwzz337C8pKWHw7tqhxowZ0/EXv/jF7qZNm5ZHc1t0OE5EJMHEu5XD+++/32jz5s0Nb7/99i+jvS0KIRGRBBNuK4cZM2ZsXb9+feG4ceN2lpaWJgFeK4eHH35457Zt2xr07t27565du5JHjhy5f/78+Z82atSofPDgwd0XLFjQLHSs9957r2lBQUHjjh079rr44ot7bNmypWHfvn0zorEtCiERkQQT71YO48aN27tnz55VO3bsWP3uu++uS0tLK125cmVRNLZF54RERGopc131N8hO9FYOsaRWDiFyOiRb3oimtRpDV8fp6jg5/amVQ2TUykFEROokhZCIiDijEBIREWcUQiIi4oxCSEREnFEIiYiIM2F9T4jki2Z2W03TRETORH8YuaSmRRovw5KwWzn8bObAOtfKoW/fvhl79uypn5KSUg4AixcvXt+xY8eTbzIXoXC/rNoz9AXJZETQG0NERBLfCy+8sOniiy8+Gs0xqz0cR3ICyWIA55E85D+KAewBMD+ahYiISPhmzJjRunv37lkZGRlZ1157bXrF+VOnTm2TnZ2dmZGRkXXllVeeW1xcnAQAs2fPbtmtW7eeGRkZWTk5ORmAd1fuXr16Zfbo0SOre/fuWatXr24Yr+2oNoTM7BEzawbgMTM7y380M7PWZjYhTjWKiEiIYCuHZcuWrS8qKip8+umnt1ZcJjc390BBQcHaoqKiwoyMjGPTp09vAwDBVg5FRUWFCxcu/BT4upXDunXrCletWrU2PT39eGXrvfvuu9N69OiRdf/996eWl0eno0NYFyaY2QSSHUn+G8mLg4+oVCAiIhGJdysHAHj11Vc3rV+/vnD58uXr3n///aZPPfVU62hsS1ghRHIKgP8D8ACA+/3HfdEoQEREIhPvVg4AkJ6eHgCAli1blt900037V65c2SQa2xLuJdrDAGSY2ffNbKj/uCYaBYiISGTi3cohEAjg888/rwcApaWlfPvtt5tnZ2cfi8a2hHt13CYA9QGU1rSgiMiZ5mczB1Y7P9FbORw7dizp8ssv7xYIBFheXs4BAwYcGjNmzN5obEtYrRxIzgVwPoDFCAkiMxtdzXvaA3gCQB//PVsA/BLAcQBvmlk2yRwAt1c3Tg11TTSz31YxbyGAVHhB+x6An5nZScdNQ6mVQ+2plYOcCdTKITLVtXIId09ogf8IC0kCmAfgeTO72Z/WG0A7ANuCy5lZHoC8cMetxEQAlYYQgBvN7JBfy+sAbgDwSi3WJSIiURZWCJnZ8yQbAehsZuG0dL0MQMDMZoaMkQ8AJNOC00heCuA+MxtCsgmAJwH08uuabGbzSQ4HcA2AxgDOBTDPzH7tXyzRiGQ+gDVmlluh5kMh29gAgLr3iYjUMeFeHTcUQD6Ahf7r3iSr2zPKBhDpbScmAVhiZn3ghdhjfjABQG8AN8ELqJtInmNm4wEcM7PeFQMopO5F8L5YWwxvb0hEROqQcK+OmwygL4Avga/2ak76hm4tXQFgvL9nsxRACoDgCZbFZnbQzEoAFALoEs6AZnYlvPNCDQFUeuaQ5AiSeSTz9h7VzpKISDyFG0InzOxghWnV/cZeg8jvLUcA1/l7Nr3NrLOZBU/khV6VV4bwz2XBD64FAH5QxfxZZpZjZjltGzPCkkVEpDbCDaECkrcASCbZjeSTAN6vZvklABqS/ElwAsk+JC+p5j2LAIzyLyQAyQvCqCtAsn7FiSSbkkz1n9cD8H0A68IYT0RE4ijcPYpR8M7ZlAJ4GV5gPFTVwmZmJIcBeILkeAAl+PoS7ao8BO+S7lV+EG0BMKSGumb5y39U4bxQEwALSDYEkAwvFGdWNoCISG1NvammX1VovCiCo0NjX32zzrVyKCkp4Z133tl5+fLlzUjaf/zHf+wYPnz4l7UdN9yr447CC6FJ4Q5sZjsB3FjF7Gx/maXwzv/AzI4B+Gkl4zwH4LmQ10NCno8DMK6S9+yG9/0kERGJggkTJqS2bds2sGXLloKysjLs2bMn7NMi1amplcMT/s83SC6o+IhGASIiErl4t3J4+eWX2zz88MO7ACA5ORnBm6fWVk1JFvxa+uPRWJmIiNResJXD8uXL16Wmpp6o7N5xubm5B8aOHbsPAEaPHt1h+vTpbSZNmrQn2MohPT09sG/fvmTg61YO99xzz/6SkhIG764dFFxuzJgxHd5///1mXbp0KZ01a9bWc845p9ZBVFM/oQ/9n8sqe9R25SIiErl4t3IIBALcvXt3/f79+x8uLCxc++1vf/vIqFGjzonGttR0OG41yVVVPaJRgIiIRCberRzatWt3IiUlpfy22277EgBuvfXW/QUFBY2jsS01XaL9QwD3Ahha4fFzf56IiMRZvFs5JCUlYdCgQQffeuutZgDw9ttvn9WtW7e4tHL4HYCJZvZZ6ESSbf15Q6NRhIhIIhv76pvVzk/0Vg4AMG3atO233HJL+n333ZfcunXrEy+88MKWisucimpbOZAsMLPsKuatNrNe0SiirlArh9pTKwc5E6iVQ2Sqa+VQ0+G4lGrmNapmnoiISI1qCqEPQm+9E0Tyx4j8LtkiIiLfUNM5oV8CmEcyF1+HTg68/jzDYliXiIicAaoNIf/2N/9G8jL4t9oB8JaZLYl5ZSIictoL995x7wB4J8a1iIjIGSbcVg4iIiJRF5W7oIqInMm2j3+v2vktgMbb8V7YrRw6TRlQp1o5HDhwIKlfv349gq93795df9iwYftnz569rbZjK4RERKRaLVu2LF+3bl1h8HXPnj0zb7jhhgPRGFuH40REElC8WzkErV69uuEXX3xR/8orrzwcje3QnpCISIKJdyuHUM8//3yra665Zn9SUnT2YbQnJCKSYOLdyiHUvHnzWt122237o7UtCiERkQQT71YOQcuXL29UVlbGAQMGHI3WtuhwXKgOFwCT82o1xOoolfKVO6I9YGUGxmMlUTbZdQEizlx11VWHrr/++m9NnDhxd/v27ct2796dXHFvqGIrh9TU1ADwdSuHgQMHHlm0aFGLTZs2Ndi/f39ZZmZmac+ePfds2rSpYX5+fqNrrrmmuOJ6X3zxxVbDhg2L2l4QoBASEam1TlMGVDv/dGjlAAALFixo9cYbb2yI1nYANbRyONPk5ORYXl7t9oRE5PSnVg6RqU0rBxERkZhRCImIiDMKIRERcUYhJCIiziiERETEGYWQiIg4o+8JiYjU0uTJk2tapPHrr78ediuHyZMn16lWDgDw9NNPt5o6dWp7AGjXrl3gtdde2xy8bVBtaE9IRESqFQgEMGHChHOWLVu2fv369YU9e/Y89thjj50djbEVQiIiCSierRzKy8tpZiguLk4qLy/HoUOHkjp06HA8Gtuhw3EiIgkm3q0cGjZsaNOmTdt64YUX9mzUqFFZly5dSl944YWt0dgWhVCI1TsOIm38W67L+IYtKbfEfB290jtHfczXHqn1oeI6bcmlf3Bdwlf2tn+3Vu8fcPGLUaqkdgYN3Oi6hIQRbiuHBx98sGNxcXHykSNHki+55JKDwNetHK677roDubm5BwCvlcPjjz+eun379gY333zzgV69epWGjlVaWspZs2a1XbFiRWFmZmbp8OHDO0+cODH10Ucf/by226LDcSIiCSberRz+9a9/NQKAnj17liYlJeFHP/rR/hUrVjSJxrYohEREEsxVV111aMGCBa127dqVDACVHY6r2MohOD3YyuGJJ57Y2bJlyxObNm1qUFhY2CAzM7P0gQce2HPFFVd8mZ+f3yh0rC5dugQ+/fTTlJ07d9YDgIULF57VvXv3kmhsiw7HiYjUUk2XaCd6K4e0tLTA/fff/3n//v0z6tWrZ506dTr+0ksvbY7GtqiVQ4iGqd0s9Y4nXJfxDTonVDfpnFD0JdI5IbVyiIxaOYiISJ2kEBIREWcUQiIi4oxCSEREnFEIiYiIMwohERFxRt8TEhGppcVLzq1pkca7lyDsVg6DBm6sc60c/vjHP7Z87LHHUsvLy3n55ZcfnDlz5vZojKs9IRERqdauXbuSH3zwwU5Lly5d/+mnn67Zs2dPvfnz5zer+Z01UwiJiCSgeLZyKCoqapienl7aoUOHEwAwaNCgQ3PmzGkZje3Q4TgRkQQT71YOWVlZpRs3bkwpKipq0LVr1+MLFixoGQgEGI1t0Z6QiEiCCbeVw0UXXZTRvXv3rLlz57Zes2ZNCvB1K4epU6e2CYZNv379jkydOjV10qRJ7Tds2NCgadOm37ifW9u2bct+97vffXbDDTd07dOnT4/OnTuXJicnR+WebwohEZEEE+9WDgBwyy23HFy1atW6/Pz8dRkZGSXnnntu6clrjZxCSEQkwcS7lQMA7Nixox4A7N27N/mZZ545+957790bjW3ROSERkVqq6Q7gid7KAQBGjhx5TmFhYWMAGDdu3M7zzjsvKntCauUQQq0coketHOJHrRziT60cIqNWDiIiUifFLIRItif5CsmNJAtJvk2yO8k0kgX+Mjkkp9diHROrmN6Y5Fsk15FcQ3LKqa5DRERiJyYhRJIA5gFYambnmlkWgIkA2oUuZ2Z5Zja6FquqNIR8j5tZDwAXAPguycG1WI+IiMRArPaELgMQMLOZwQlmlm9m74UuRPJSkm/6z5uQnE3yA5Ifk/yBP304yb+QXEhyA8lH/elTADQimU/yz6HjmtlRM3vHf34cwEcAOsVoW0VE5BTFKoSyAUR6A75JAJaYWR94IfYYySb+vN4AbgLQC8BNJM8xs/EAjplZbzPLrWpQki0ADAWwuIr5I0jmkcwrO3owwpJFRKQ26tKFCVcAGE8yH8BSACkAgpdtLTazg2ZWAqAQQJdwBiRZD8DLAKab2abKljGzWWaWY2Y5yY2b13ITREQkErH6ntAaANdH+B4CuM7Mir4xkfw2gNDr0csQft2zAGwwsycirEVEJGzt38mvaZHGeCc/7FYOuy7rXedaOYwaNarjnDlzWh86dCj56NGjHwenHzt2jNdff3366tWrG7do0eLEnDlzNmVkZBwPd9xY7QktAdCQ5E+CE0j2IXlJNe9ZBGCUf1EDSF4QxnoCJOtXNoPkwwCaA/hl2FWLiEilrr322i9XrFhx0nedfv/737dp3rz5ia1btxb8/Oc/3z1mzJiIzr/HJITM+wbsMADf8y/RXgNgMoCTvoUb4iEA9QGs8i/hfiiMVc3yl//GhQkkO8E7x5QF4CP/4oW7I98SEZG6KZ6tHABg0KBBR7p06RKoOP3NN99scdddd30BAHfeeeeB999/v1l5eXnY2xGz2/aY2U4AN1YxO9tfZim88z8ws2MAflrJOM8BeC7k9ZCQ5+MAjKvkPdvhHd4TETntxLuVQ3V2797dID09/TgA1K9fH02bNi3bvXt3veAdvmtSly5MEBGRMMS7lUN1Krv1W013+A6lEBIRSTAuWjlUpX379sc3b97cAAACgQAOHz6cfPbZZ58UilVRCImIJBgXrRyqcvXVV385e/bs1gDw7LPPtuzXr19xUlL40aJWDiIitbTrst7Vzj9NWjl0mjdvXquSkpKkdu3anZebm7tv2rRpO3/xi1/su+6669I7d+6c3bx587JXX301otuhq5VDCLVyiB61cogftXKIP7VyiIxaOYiISJ2kEBIREWcUQiIip0CnMsJTXl5OAFV+e1UhJCISoZSUFHzxxRcKohqUl5dz7969zQEUVLWMro4TEYlQp06dsH37duzduzes5Xft2lWvrKysTYzLqovKARScOHGiytumKYRERCJUv359pKefdLu2KmVlZa02s5wYlpSwdDhOREScUQiJiIgzCiEREXFGISQiIs4ohERExBmFkIiIOKMQEhERZxRCIiLijFo5hMjJybG8vDzXZYjIaYbkh/qyauW0JyQiIs4ohERExBmFkIiIOKMQEhERZxRCIiLijEJIREScUQiJiIgzCiEREXFGISQiIs4ohERExBmFkIiIOKMQEhERZ+q5LqAuWb3jINLGv+W6jG9oljk+quO99siJqI7nwpJL/+C6hBqVHJgW1/UVZ7q7N+bMS651tu5423VZb9clnHa0JyQiIs4ohERExBmFkIiIOKMQEhERZxRCIiLijEJIREScUQiJiIgzCiEREXFGISQiIs4ohERExBmFkIiIOKMQEhERZxRCIiLijEJIREScUQiJiIgzCiEREXFGISQiIs4ohERExBmFkIiIOKMQEhERZ2IWQiTbk3yF5EaShSTfJtmdZBrJAn+ZHJLTa7GOidXM+y+S20gePtXxRUQktmISQiQJYB6ApWZ2rpllAZgIoF3ocmaWZ2aja7GqKkMIwBsA+tZibBERibFY7QldBiBgZjODE8ws38zeC12I5KUk3/SfNyE5m+QHJD8m+QN/+nCSfyG5kOQGko/606cAaEQyn+SfKxZgZv8ys89jtH0iIhIF9WI0bjaADyN8zyQAS8zsLpItAKwk+Q9/Xm8AFwAoBVBE8kkzG0/y52bWuzaFkhwBYAQAJJ/VtjZDiYhIhOrShQlXABhPMh/AUgApADr78xab2UEzKwFQCKBLtFZqZrPMLMfMcpIbN4/WsCIiEoZY7QmtAXB9hO8hgOvMrOgbE8lvw9sDCipD7OoWEZE4itWe0BIADUn+JDiBZB+Sl1TznkUARvkXNYDkBWGsJ0Cyfu1KFRERV2ISQmZmAIYB+J5/ifYaAJMB7KzmbQ8BqA9glX8J90NhrGqWv/xJFyaQfJTkdgCNSW4nOTnCzRARkRiL2WEtM9sJ4MYqZmf7yyyFd/4HZnYMwE8rGec5AM+FvB4S8nwcgHFVrP/XAH59CqWLiEic1KULE0RE5AyjEBIREWcUQiIi4oxCSEREnFEIiYiIMwohERFxRiEkIiLOKIRERMQZhZCIiDijEBIREWcUQiIi4oxCSEREnFEIiYiIMwohERFxRiEkIiLOKIRERMQZhZCIiDijEBIREWcUQiIi4gzNzHUNdUZOTo7l5eW5LkNETjMkPzSzHNd11EXaExIREWcUQiIi4oxCSEREnFEIiYiIMwohERFxRiEkIiLOKIRERMQZhZCIiDijEBIREWcUQiIi4oxCSEREnFEIiYiIMwohERFxRiEkIiLOqJVDCJLFAIpc1wGgDYB9rouA6qhMXalFdZysrtRSWR1dzKyti2LqunquC6hjiupCzw+Seaqj7tUB1J1aVMfJ6kotdaWORKHDcSIi4oxCSEREnFEIfdMs1wX4VMc31ZU6gLpTi+o4WV2ppa7UkRB0YYKIiDijPSEREXFGIQSA5FUki0h+SnJ8DMY/h+Q7JNeSXEPyF/70ViT/TnKD/7NlyHsm+PUUkbwyZPpFJFf786aT5CnUk0zyY5JvuqqDZAuSr5Nc538u/Rx+Hr/y/7sUkHyZZEo8aiE5m+QekgUh06K2XpINSb7qT19BMi2COh7z/9usIjmPZItY11FVLSHz7iNpJNu4+Ez86aP8da0h+Wg8PpPTnpmd0Q8AyQA2AugKoAGATwBkRXkdqQAu9J83A7AeQBaARwGM96ePB/Df/vMsv46GANL9+pL9eSsB9ANAAP8PwOBTqGcMgJcAvOm/jnsdAJ4HcLf/vAGAFo7q6AhgM4BG/uvXAAyPRy0ALgZwIYCCkGlRWy+AewHM9J/fDODVCOq4AkA9//l/x6OOqmrxp58DYBGAzwC0cfSZXAbgHwAa+q/Pjsdncro/nBfg+uH/A1kU8noCgAkxXud8AN+D98XYVH9aKrzvKZ1Ug/8/Xz9/mXUh038E4OkI190JwGIAA/F1CMW1DgBnwfvFzwrTXXweHQFsA9AK3vfm3oT3CzgutQBIq/CLLmrrDS7jP68H7wuUDKeOCvOGAfhzPOqoqhYArwM4H8AWfB1Ccf1M4P2Bcnkly8X8MzmdHzoc9/UvoaDt/rSY8He7LwCwAkA7M/scAPyfZ9dQU0f/eW1qfQLArwGUh0yLdx1dAewF8Cy9w4LPkGzioA6Y2Q4AjwPYCuBzAAfN7G8uavFFc71fvcfMTgA4CKD1KdR0F7y/4p3UQfIaADvM7JMKs+JdS3cAA/zDZ8tI9nFUx2lFIeTtJlcUk0sGSTYFMBfAL83s0CnUVKtaSQ4BsMfMPgz3LbGoA95ffhcC+B8zuwDAEXiHnuJdB/xzLj+AdxilA4AmJG91UUsNTmW90fh8JgE4AeDPLuog2RjAJAAPVjY7nrXA+3fbEsB3ANwP4DX/HI+T/zanC4WQ99fJOSGvOwHYGe2VkKwPL4D+bGZ/8SfvJpnqz08FsKeGmrb7z0+11u8CuIbkFgCvABhI8n8d1LEdwHYzW+G/fh1eKMW7DgC4HMBmM9trZgEAfwHwb45qQZTX+9V7SNYD0BzA/nALIXkHgCEAcs0/buSgjnPh/YHwif/vthOAj0i2d1DLdgB/Mc9KeEcT2jio47SiEAI+ANCNZDrJBvBOEi6I5gr8v5b+BGCtmU0LmbUAwB3+8zvgnSsKTr/Zv4ImHUA3ACv9wzPFJL/jj3l7yHtqZGYTzKyTmaXB284lZnargzp2AdhGMsOfNAhAYbzr8G0F8B2Sjf0xBgFY66iW4PjRWm/oWNfD++8d7h7IVQDGAbjGzI5WqC9udZjZajM728zS/H+32+Fd5LMr3rUA+Cu8c6kg2R3eBTX7HNRxenF9UqouPAB8H94VaxsBTIrB+P3h7WqvApDvP74P7xjwYgAb/J+tQt4zya+nCCFXWQHIAVDgz5uBUzyZCeBSfH1hQtzrANAbQJ7/mfwV3mEOJ58HgP8EsM4f50V4VznFvBYAL8M7DxWA98v1x9FcL4AUAHMAfArvKq2uEdTxKbxzFsF/rzNjXUdVtVSYvwX+hQkOPpMGAP7XH/cjAAPj8Zmc7g/dMUFERJzR4TgREXFGISQiIs4ohERExBmFkIiIOKMQEhERZxRC4ox/R+SpIa/vIzk5SmM/R/L6aIxVw3puoHcX8Hdiva4a6tjCkLtLiyQKhZC4VArgh3XtlyfJ5AgW/zGAe83ssljVI3I6UwiJSyfgtUL+VcUZFfdkSB72f17q3zzyNZLrSU4hmUtypd+35dyQYS4n+Z6/3BD//cn0euV8QK9Xzk9Dxn2H5EsAVldSz4/88QtI/rc/7UF4X0SeSfKxCsunknyXZL7/ngH+9P8hmUevH81/hiy/heRvSS73519IchHJjSRHhtT4Lr3+PoUkZ5I86f9hkrf6n0c+yaf9bU72P9MCfztO+sxFXKjnugA54/0BwCqGNAgLw/kAMuHda2sTgGfMrC+9ZoGjAPzSXy4NwCXw7j/2Dslvwbt1ykEz60OyIYD/I/k3f/m+ALLNbHPoykh2gNdT5yIABwD8jeS1ZvYbkgMB3GdmeRVqvAVei5D/8vesGvvTJ5nZfn/aYpLnmdkqf942M+tH8ncAnoN3r78UAGsAzAypMQteX52FAH4I7957wVozAdwE4LtmFiD5FIBcf4yOZpbtL9ei5o9ZJPa0JyROmXc38RcAjI7gbR+Y2edmVgrvdijBEFkNL3iCXjOzcjPbAC+sesDrFXQ7yXx47TRaw7vXF+Dd7+sbAeTrA2CpeTc5Dd5R+uKaagRwp3+Oq5eZFfvTbyT5EYCPAfSEFyhBwXsWrgawwsyKzWwvgJKQ0FhpZpvMrAzerWX6V1jvIHhh+YG/jYPgtc7YBKArySf9+8JVdxd3kbjRnpDUBU/AuxfXsyHTTsD/I8m/+WODkHmlIc/LQ16X45v/pivekyp4e/1RZrYodAbJS+G1lKhMxC3DzexdkhcDuBrAi/7huvcA3Aegj5kdIPkcvD2doNDtqLiNwe2qbJsq1vq8mU04aSPI8wFcCeBnAG6E1ydIxCntCYlzZrYfXtfKH4dM3gLvL3rA6/dT/xSGvoFkkn+eqCu8m0suAnAPvdYaINmdXkO96qwAcAnJNv5htB8BWFbdG0h2gde76Y/w7qB+IbyOskcAHCTZDsDgU9imvvTu+J4E77DbPyvMXwzgepJn+3W0ItnFv/gjyczmAvh3vx4R57QnJHXFVAA/D3n9RwDzSa6E94u1qr2U6hTBC4t2AEaaWQnJZ+AdsvvI38PaC+Da6gYxs89JTgDwDrw9jbfNrKY2DZcCuJ9kAMBhALeb2WaSH8M7P7MJwP+dwjYtBzAFQC8A7wKYV6HWQpIPwDtvlQTvLtA/A3AMXifb4B+eJ+0pibigu2iLJAj/kOF9ZjbEcSkiUaPDcSIi4oz2hERExBntCYmIiDMKIRERcUYhJCIiziiERETEGYWQiIg4oxASERFn/j8w/vNJKLgifQAAAABJRU5ErkJggg==", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaEAAAGDCAYAAACCzK//AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAAvgklEQVR4nO3deXxU9b0+8OdJWMImuxCwkIgQEoLFGujlCi6gKFe0ULVa434txQVsQQuC13Krv0pVqEVqlSoq1hWRgsuFWhFqrxQNGkkIBGSRTTZBEpaELJ/fH3PmOoYsM2bmfDPwvF+veTE558z5fA8iD99zzpwPzQwiIiIuJLgegIiInLwUQiIi4oxCSEREnFEIiYiIMwohERFxRiEkIiLOKITkhELySZL/FaV9dSN5iGSi9/MykrdGY9/e/v6H5I3R2p9IPGrkegAikSC5BUAnAOUAKgAUAJgLYLaZVZrZmAj2c6uZ/b2mbcxsK4CW9R2zV28qgDPM7LqQ/Q+Pxr5F4plmQhKPLjOzVgC6A5gGYCKAZ6JZgKT+gSbiA4WQxC0zO2hmiwBcDeBGkpkknyP5IACQ7EDyLZJfk9xP8gOSCSRfANANwJve6bZfkUwhaST/k+RWAEtDloUGUg+SH5E8SHIhyXZerfNJbg8dH8ktJC8keQmAyQCu9up95q3/v9N73rjuI/kFyT0k55Js7a0LjuNGkltJ7iM5Jba/uyL+UAhJ3DOzjwBsBzC4yqoJ3vKOCJzCmxzY3K4HsBWBGVVLM3s45DPnAUgHcHEN5W4AcAuALgicEpwZxvgWA/gtgFe9et+vZrObvNcFAE5H4DTgrCrbDAKQBmAogPtJptdVW6ShUwjJiWIngHZVlpUBSAbQ3czKzOwDq/thiVPN7LCZHa1h/Qtmlm9mhwH8F4CfBG9cqKdsADPMbJOZHQJwL4BrqszC/tvMjprZZwA+A1BdmInEFYWQnCi6AthfZdkjAD4H8DeSm0hOCmM/2yJY/wWAxgA6hD3KmnXx9he670YIzOCCdoW8P4Io3TQh4pJCSOIeyf4IhNA/Q5ebWbGZTTCz0wFcBmA8yaHB1TXsrq6Z0vdC3ndDYLa1D8BhAM1DxpSIwGnAcPe7E4EbLUL3XQ5gdx2fE4lrCiGJWyRPITkCwCsA/mJmeVXWjyB5BkkCKELglu4Kb/VuBK69ROo6khkkmwP4DYDXzawCwHoASSQvJdkYwH0AmoZ8bjeAFJI1/T/3MoBfkkwl2RLfXEMq/w5jFIkbCiGJR2+SLEbg1NgUADMA3FzNdj0B/B3AIQArADxhZsu8dQ8BuM+7c+7uCGq/AOA5BE6NJQEYBwTu1ANwO4CnAexAYGYUerfcPO/Xr0h+Us1+53j7/geAzQBKAIyNYFwicYlqaiciIq5oJiQiIs4ohERExBmFkIiIOKMQEhERZxRCIiLijJ4UHKJDhw6WkpLiehgicoJZtWrVPjPrWPeWJx+FUIiUlBTk5OS4HoaInGBIflH3VicnnY4TERFnFEIiIuKMQkhERJxRCImIiDMKIRERcUYhJCIiziiERETEGYWQiIg4oxASERFnFEIiIuKMQkhERJxRCImIiDN6gGmIvB0HkTLpbSe1tyRd66QuAPRN7eas9k3vdHdW++rUic5qZ13cylntXRf0c1ZbpCrNhERExBmFkIiIOKMQEhERZxRCIiLijEJIREScUQiJiIgzCiEREXFGISQiIs4ohERExBmFkIiIOKMQEhERZxRCIiLijEJIREScUQiJiIgzCiEREXFGISQiIs4ohERExBmFkIiIOKMQEhERZxRCIiLijEJIRESciVkIkexM8hWSG0kWkHyHZC+SKSTzvW2ySM6sR43JtaxbRrKQZK73OvW71hERkdhoFIudkiSABQCeN7NrvGX9AHQCsC24nZnlAMipR6nJAH5by/psr4aIiDRAsZoJXQCgzMyeDC4ws1wz+yB0I5Lnk3zLe9+C5BySH5P8lOSPvOU3kXyD5GKSG0g+7C2fBqCZN8t5MUbHISIiMRSTmRCATACrIvzMFABLzewWkm0AfETy7966fgDOAlAKoJDk42Y2ieSdZtavln0+S7ICwHwAD5qZVd2A5GgAowEg8ZSOEQ5ZRETqoyHdmDAMwCSSuQCWAUgC0M1b956ZHTSzEgAFALqHsb9sM+sLYLD3ur66jcxstpllmVlWYvPW9TwEERGJRKxCaA2AsyP8DAFcYWb9vFc3M1vrrSsN2a4CYczgzGyH92sxgJcADIhwPCIiEmOxCqGlAJqS/FlwAcn+JM+r5TNLAIz1bmoAybPCqFNGsnHVhSQbkezgvW8MYASA/EgOQEREYi8mIeRdexkF4CLvFu01AKYC2FnLxx4A0BjAau8W7gfCKDXb277qjQlNASwhuRpALoAdAP4c0UGIiEjMxerGBJjZTgA/qWF1prfNMgSu/8DMjgL4eTX7eQ7AcyE/jwh5PxHAxGo+cxiRnw4UERGfNaQbE0RE5CSjEBIREWcUQiIi4oxCSEREnFEIiYiIMwohERFxRiEkIiLOKIRERMQZhZCIiDijEBIREWcUQiIi4oxCSEREnFEIiYiIMwohERFxRiEkIiLOKIRERMQZhZCIiDijEBIREWcUQiIi4gzNzPUYGoysrCzLyclxPQwROcGQXGVmWa7H0RBpJiQiIs4ohERExBmFkIiIOKMQEhERZxRCIiLijEJIREScUQiJiIgzCiEREXFGISQiIs4ohERExBmFkIiIONPI9QAakrwdB5Ey6W0ntbckXeukLgD0Te3mrPZrD5U7q730/D86q11yYIaz2lenTnRW++mk95zVHnzuC85qS800ExIREWcUQiIi4oxCSEREnFEIiYiIMwohERFxRiEkIiLOKIRERMQZhZCIiDijEBIREWf0xAQRkQiRp6BVyzuQ2KgbGMa/5d99F30/++yzLbEfWYNTCSC/vLz81rPPPntPdRsohEREItSq5R3o1PkstD6lMUjWuX3jxizPzMzc58PQGpTKykru3bs3Y9euXU8DuLy6bXQ6TkQkQomNuoUdQCezhIQE69ix40EAmTVu4+N4REROCESCAihMCQkJhlqyRiEkIiLO6JqQiEg9nfnbrXVt0hz44uxw97dl2qWrvss4xo8f36Vly5YVv/nNb3Z/l8/XZuzYsV3nzZvXvqioKPHIkSOfRmu/mgmJiEidRo4c+fXKlSvXRnu/CiERkTg0a9as9r169cpIS0vLGDlyZGrV9dOnT++QmZmZnpaWlnHxxRf3KC4uTgCAOXPmtO3Zs2eftLS0jKysrDQAyMnJSerbt2967969M3r16pWRl5fXtOr+hg4derh79+5l0T4OnY4TEYkzOTk5SY8++mjyihUr1iUnJ5fv3r07seo22dnZByZMmLAPAMaNG9dl5syZHaZMmbJn2rRpyX/729/Wp6amlu3bty8RAB5//PGOt99+++7bbrttf0lJCcvL/et4rJmQiEicWbJkySmXXXbZgeTk5HIA6NSpU0XVbVatWtXs7LPPTuvVq1fG/Pnz269ZsyYJALKysg5lZ2enTJ8+vUMwbAYOHHh4+vTpyVOmTOm8YcOGJi1btjS/jkUhJCISZ8wMJGsNitGjR6fOmjVr6/r16wsmTpy4s7S0NAEAXnrppa0PPvjgzm3btjXp169fn127diWOGTNm/8KFCz9v1qxZ5fDhw3stWrSolT9HohASEYk7l1xySdGiRYva7dq1KxEAqjsdd+TIkYRu3bqVlZaW8pVXXmkXXL5mzZqmQ4YMOfzYY4/tbNu2bfmmTZuaFBQUNElPTy+977779gwbNuzr3NzcZn4dS8yuCZHsDOAxAP0BlALYAuAXAI4BeMvMMklmAbjBzMZ9xxqTzey3dWyzCMDpZlbjN3ZFROpj9eRuta7fupVHMjMzo3ZnWVZWVsmECRO+HDx4cO+EhATLzMw8Mn/+/C2h20yaNGnngAED0rt27XosPT39yKFDhxIB4Je//OVpW7ZsaWpmHDRoUNG//du/HZ0yZUrnefPmtW/UqJF17Nix7KGHHtpZteaYMWNOW7BgQbuSkpKETp06nZmdnb1vxowZx20XKZpF/9QfA18l/hDA82b2pLesH4BWALbBC6Eo1DlkZi1rWf9jAFcCODOcek2Te1ryjY/Vd1jfyZaka53UBYC+qbX/DxRLrz3k3wXQqpae/0dntUsOzHBW++rUic5qP530nrPag899IWr7atf2KfTo0Sns7aMdQvHms88+6/D9738/pbp1sToddwGAsmAAAYCZ5ZrZB6EbkTyf5Fve+xYk55D8mOSnJH/kLb+J5BskF5PcQPJhb/k0AM1I5pJ8seoASLYEMB7AgzE6RhERqadYnY7LBBDpN36nAFhqZreQbAPgI5J/99b1A3AWAqf1Ckk+bmaTSN5pZv1q2N8DAKYDOFJbUZKjAYwGgMRTOkY4ZBERqY+GdGPCMACTSOYCWAYgCUDwPNF7ZnbQzEoAFADoXtuOvFN/Z5jZgrqKmtlsM8sys6zE5q3rMXwREYlUrGZCaxC4FhMJArjCzAq/tZD8IQIzoKAK1D3ugQDOJrnF2/ZUksvM7PwIxyQiIjEUq5nQUgBNSf4suIBkf5Ln1fKZJQDGejc1gORZYdQpI9m46kIz+5OZdTGzFACDAKxXAImINDwxCSEL3HI3CsBFJDeSXANgKoDabud7AEBjAKtJ5ns/12W2t/1xNyaIiEjDF7PvCZnZTgA/qWF1prfNMgSu/8DMjgL4eTX7eQ7AcyE/jwh5PxFArfebmtkW1NLVT0Skvk6ZMajW9ZlAc7yOsFs5YOrBBtXKobi4OOGyyy47/YsvvmiamJiIYcOGff3EE0/siMa+G9KNCSIi0kBNmDBh9+bNm9fk5+cXrFy5suVrr712SjT2qxASEYlDfrZyaNWqVeVll11WDABJSUl25plnHtm2bVuTaByHQkhEJM4EWzksX758fWFhYcFTTz11XGvX7OzsA/n5+WsLCwsL0tLSjs6cObMDAARbORQWFhYsXrz4c+CbVg7r1q0rWL169drU1NRjNdXet29f4rvvvttm+PDhRdE4FoWQiEiccdXKoaysDD/+8Y9PHz169O6MjIwagyoSCiERkTjjqpXDtddem3L66aeX3H///XuidSwKIRGROOOilcO4ceO6FBUVJT7zzDPbonksau8tIlJPReP/Wev6eG/lsHHjxsaPP/54cmpqakmfPn0yAGD06NF7xo8fv6++xxKTVg7xSq0c/KdWDv5TK4f6UyuHyLho5SAiIlInhZCIiDijEBIREWcUQiIi4oxCSEREnFEIiYiIM/qekIhIPZ2zoM6vWDTHqvBbOeTdmNegWjkAwODBg3vu2bOncUVFBQcMGFA8d+7crY0a1T9CNBMSEZE6LVy4cGNhYWHB+vXr13z11VeN58yZ0zYa+1UIiYjEIT9bOQBAu3btKgGgrKyMZWVlJBmV41AIiYjEGVetHAYNGtSzY8eO32/RokXFzTfffCAax6IQEhGJM65aOfzzn//csGvXrs+OHTuW8Oabb0als6puTAjRt2tr5Ey71FH1g47qAnnOKgO40V3pdHelAQxxWt2VqRjstHq0rF27Fqec4u5PULitHF5//fXPBw4ceHTmzJntly9f3goItHJYunRpi0WLFrXu169fn9zc3DVjxozZP3jw4MMLFixoPXz48F5PPPHElssvv7y4uv02b97cRowY8fWCBQvajBo1qt6N7TQTEhGJM363cjh48GDCF1980RgINLZbvHhx6969ex+NxrFoJiQiUk95N9Z+PiE/Pz+uWzkUFRUlXHrppWccO3aMlZWVPOecc4ruueeevdE4FrVyCJGVlWU5OTmuhyEiDdzatWuRnh7+6bhoh1C8USsHERFpkBRCIiLijEJIREScUQiJiIgzCiEREXFGISQiIs7oe0IiIvW0tnftt2snAs3XIvxWDunr1ja4Vg5BQ4YMOWPbtm1NN2zYsCYa+9NMSEREwvL888+3adGixXHPqasPzYRC7fwUmNraSem+qd2c1AWAMSv+4Kx2yYEZzmoXp2c5qz343Bec1R46ZKOz2hI9s2bNaj9z5sxOJJGenn70r3/96+bQ9dOnT+/w7LPPdiwrK2NKSkrp66+/vrlVq1aVc+bMafvQQw91SUhIsFatWlXk5OQU5uTkJN18882pZWVlrKysxPz58zf27du3NHR/Bw8eTJg5c2an2bNnf3HNNdf0iNZxKIREROJMsJXDihUr1iUnJ5dX9+y47OzsAxMmTNgHAOPGjesyc+bMDlOmTNkTbOWQmppatm/fvkTgm1YOt9122/6SkhIGn64davz48V3vuuuu3S1btqyM5rHodJyISJzxu5XDhx9+2Gzz5s1Nb7jhhq+jfSwKIRGROBNuK4dZs2ZtXb9+fcHEiRN3lpaWJgCBVg4PPvjgzm3btjXp169fn127diWOGTNm/8KFCz9v1qxZ5fDhw3stWrSoVei+Pvjgg5b5+fnNu3bt2vfcc8/tvWXLlqYDBgxIi8axKIREROKM360cJk6cuHfPnj2rd+zYkfePf/xjXUpKSulHH31UGI1j0TUhEZF6Sl9X+wOy472VQyyplUOIrC6JljO6pZPaujvOf7o7Tr4rtXKIjFo5iIhIg6QQEhERZxRCIiLiTFghRPIqkq289/eRfIPkD2I7NBEROdGFOxP6LzMrJjkIwMUAngfwp9gNS0RETgbhhlDw27iXAviTmS0E0CQ2QxIRkZNFuN8T2kHyKQAXAvgdyabQ9SQREQDAH8csrWuT5suxNOxWDnc8OaTBtXIYMGBA2p49exonJSVVAsB77723vmvXrsc/ZC5C4YbQTwBcAuBRM/uaZDKAe+pbXERE4sfcuXM3nXvuuUeiuc9wZzNPmdkbZrYBAMzsSwDXR3MgIiISvlmzZrXv1atXRlpaWsbIkSNTq66fPn16h8zMzPS0tLSMiy++uEdxcXECAMyZM6dtz549+6SlpWVkZWWlAYGncvft2ze9d+/eGb169crIy8tr6tdxhBtCfUJ/IJmICLoEiohI9ARbOSxfvnx9YWFhwVNPPbW16jbZ2dkH8vPz1xYWFhakpaUdnTlzZgcACLZyKCwsLFi8ePHnwDetHNatW1ewevXqtampqceqq3vrrbem9O7dO+Oee+5JrqyMTkeHWkOI5L0kiwGcSbLIexUD2ANgYVRGICIiEfG7lQMAvPrqq5vWr19fsGLFinUffvhhyyeeeKJ9NI6l1hAys4fMrBWAR8zsFO/Vyszam9m90RiAiIhExu9WDgCQmppaBgBt27atvPrqq/d/9NFHLaJxLGGdjjOze0l2JfnvJM8NvqIxABERiYzfrRzKysrw5ZdfNgKA0tJSvvPOO60zMzOPRuNYwro7juQ0ANcAKMA33xkyAP+IxiBEROLZHU8OqXV9vLdyOHr0aMKFF17Ys6ysjJWVlRw8eHDR+PHj90bjWMJq5UCyEMCZZlYa9o7JzgAeA9AfQCmALQB+AeAYgLfMLJNkFoAbzGxcxCMP1JhsZr+tYd1iAMkIBO0HAO4ws+POm4ZSKwf/qZWD/9TKof7UyiEy0WjlsAlA43ALkiSABQCWmVkPM8sAMBlAp9DtzCznuwaQZ3It635iZt8HkAmgI4Cr6lFHRERiINwvqx4BkEvyPQRmNQCAWgLkAgBlZvZkyLa5AEAyJbiM5PkA7jazESRbAHgcQF9vXFPNbCHJmwBcDqA5gB4AFpjZr7xThM1I5gJYY2bZoQMws6KQY2yCwOlDERFpQMINoUXeK1yZACJ97MQUAEvN7BaSbQB8RPLv3rp+AM5CIAALST5uZpNI3mlm/WraIcklAAYA+B8Ar0c4HhERibGwQsjMnifZDEA3MyuM0ViGAbic5N3ez0kAghdK3jOzgwBAsgBAdwDb6tqhmV1MMgnAiwCGAHi36jYkRwMYDQDdWrO+xyAiIhEIt5/QZQByASz2fu5HsraZ0RpE/kQFArjCzPp5r25mFryQF3pDRAXCn8HBzEoQmMX9qIb1s80sy8yyOjZXCImI+CncGxOmInBa62vg/67vHPesohBLATQl+bPgApL9SZ5Xy2eWABjr3dQAkmeFMa4yksfdMEGypfeQVZBsBOA/AKwLY38iIuKjcGcU5WZ20MuHoBov9JuZkRwF4DGSkwCU4JtbtGvyAAK3dK/2gmgLgBF1jGu2t/0nVW5MaAFgkddyIhGBUHyyuh2IiNTX9Kvr+qsKzZdEcHZowqtvNbhWDiUlJbz55pu7rVixohVJ+/Wvf73jpptu+rq++w03hPJJXgsgkWRPAOMAfFjbB8xsJwItIKqT6W2zDMAy7/1RAD+vZj/PAXgu5OcRIe8nAphYzWd2I/D9JBERiYJ77703uWPHjmVbtmzJr6iowJ49e8K+LFKbcE/HjUXgSdqlAF4GUITaZzUiIhJDfrdyePnllzs8+OCDuwAgMTERwYen1le4z447YmZTzKy/dxF/infBX0REfOZ3K4d9+/YlAoHTfRkZGenDhw8/fdu2bbGfCZF8zPv1TZKLqr6iMQAREYmM360cysrKuHv37saDBg06VFBQsPaHP/zh4bFjx34vGsdS10wo+ICrRwFMr+YlIiI+87uVQ6dOncqTkpIqr7/++q8B4Lrrrtufn5/fPBrHUlc/oVXer8ure0VjACIiEhm/WzkkJCRg6NChB99+++1WAPDOO++c0rNnz9i3ciCZh9pvxT4zGoMQEYlnE159q9b18d7KAQBmzJix/dprr029++67E9u3b18+d+7cLVW3+S5qbeXg3Y7dCcc/Iqc7gJ1m9nk0BtFQqJWD/9TKwX9q5VB/auUQmfq0cvg9gCIz+yL0hcBTtX8f5XGKiMhJpq4QSjGz1VUXmlkOgJSYjEhERE4adYVQUi3rmtWyTkREpE51hdDHoQ8hDSL5n4i8X5CIiMi31PWN118AWEAyG9+EThYCnUpHxXBcIiJyEqg1hLwHgf47yQvgPXQUwNtmtjTmIxMRkRNeuJ1V3wfwfozHIiISl7ZP+qDW9W2A5tvxQditHE6bNrhBtXI4cOBAwsCBA3sHf969e3fjUaNG7Z8zZ06dHa7rEpUH0ImIyImrbdu2levWrSsI/tynT5/0q6666kA09h1uKwcREWlA/G7lEJSXl9f0q6++anzxxRcfisZxaCYkIhJngq0cVqxYsS45Obm8umfHZWdnH5gwYcI+ABg3blyXmTNndpgyZcqeYCuH1NTUsmCLhmArh9tuu21/SUkJg0/Xrs7zzz/f7vLLL9+fkBCdOYxmQiIiccbvVg6hFixY0O7666/fH61jUQiJiMQZv1s5BK1YsaJZRUUFBw8efCRax6LTcaG6nAVMzXFSOs9JVc+NLosPcVncoamuByBx7JJLLim68sorz5g8efLuzp07V+zevTux6myoaiuH5OTkMuCbVg5Dhgw5vGTJkjabNm1qsn///or09PTSPn367Nm0aVPT3NzcZpdffnlx1bovvPBCu1GjRkVtFgQohERE6u20aYNrXX8itHIAgEWLFrV78803N0TrOIA6WjmcbLKysiwnx81MSETih1o5RKY+rRxERERiRiEkIiLOKIRERMQZhZCIiDijEBIREWcUQiIi4oy+JyQiUk9Tp06ta5Pmr7/+etitHKZOndqgWjkAwFNPPdVu+vTpnQGgU6dOZa+99trm4GOD6kMzIRERqVVZWRnuvffe7y1fvnz9+vXrC/r06XP0kUceOTUa+1YIiYjEIT9bOVRWVtLMUFxcnFBZWYmioqKELl26HIvGceh0nIhInPG7lUPTpk1txowZW3/wgx/0adasWUX37t1L586duzUax6IQCpG34yBSJr3tpPaWpGud1AWAvqndnNV+7aF6n1L+zpae/0dntUsOzHBW++rUic5qP530nrPag899IWr7atf2KRQVufuzG24rh/vvv79rcXFx4uHDhxPPO++8g8A3rRyuuOKKA9nZ2QeAQCuHRx99NHn79u1NrrnmmgN9+/YtDd1XaWkpZ8+e3XHlypUF6enppTfddFO3yZMnJz/88MNf1vdYdDpORCTO+N3K4V//+lczAOjTp09pQkICfvrTn+5fuXJli2gci0JIRCTOXHLJJUWLFi1qt2vXrkQAqO50XNVWDsHlwVYOjz322M62bduWb9q0qUlBQUGT9PT00vvuu2/PsGHDvs7NzW0Wuq/u3buXff7550k7d+5sBACLFy8+pVevXiXROBadjhMRqafx46+odf3WrYzrVg4pKSll99xzz5eDBg1Ka9SokZ122mnHXnrppc3ROBa1cgjRNLmnJd/4mJPauibkP10T8t+JdE2oR49OYW8f7RCKN2rlICIiDZJCSEREnFEIiYiIMwohERFxRiEkIiLOKIRERMQZfU9IRKSePs4ZWdcmzXcvRditHIYO2djgWjn8+c9/bvvII48kV1ZW8sILLzz45JNPbo/GfjUTEhGRWu3atSvx/vvvP23ZsmXrP//88zV79uxptHDhwlZ1f7JuCiERkTjkZyuHwsLCpqmpqaVdunQpB4ChQ4cWzZs3r200jkOn40RE4ozfrRwyMjJKN27cmFRYWNjk9NNPP7Zo0aK2ZWVljMaxaCYkIhJnwm3lcPbZZ6f16tUrY/78+e3XrFmTBHzTymH69OkdgmEzcODAw9OnT0+eMmVK5w0bNjRp2bLlt57n1rFjx4rf//73X1x11VWn9+/fv3e3bt1KExMTo/LMN4WQiEic8buVAwBce+21B1evXr0uNzd3XVpaWkmPHj1Kj68aOYWQiEic8buVAwDs2LGjEQDs3bs38emnnz719ttv3xuNY9E1IRGReuqf9dda18d7KwcAGDNmzPcKCgqaA8DEiRN3nnnmmVGZCamVQwi1cvCfWjn4T60c6k+tHCKjVg4iItIgxSyESHYm+QrJjSQLSL5DshfJFJL53jZZJGfWo8bkGpY3J/k2yXUk15Cc9l1riIhI7MQkhEgSwAIAy8ysh5llAJgM4FvzVzPLMbNx9ShVbQh5HjWz3gDOAnAOyeH1qCMiIjEQq5nQBQDKzOzJ4AIzyzWzD0I3Ink+ybe89y1IziH5MclPSf7IW34TyTdILia5geTD3vJpAJqRzCX5Yuh+zeyImb3vvT8G4BMAp8XoWEVE5DuKVQhlAoj0AXxTACw1s/4IhNgjJFt46/oBuBpAXwBXk/yemU0CcNTM+plZdk07JdkGwGUAqr0iSnI0yRySORVHDkY4ZBERqY+GdGPCMACTSOYCWAYgCUDwtq33zOygmZUAKADQPZwdkmwE4GUAM81sU3XbmNlsM8sys6zE5q3reQgiIhKJWH1PaA2AKyP8DAFcYWaF31pI/hBA6P3oFQh/3LMBbDCzxyIci4hI2HqtOu6pOVU1x/u5Ybdy2HVBvwbXymHs2LFd582b176oqCjxyJEjnwaXHz16lFdeeWVqXl5e8zZt2pTPmzdvU1pa2rFw9xurmdBSAE1J/iy4gGR/kufV8pklAMZ6NzWA5Flh1Ckj2bi6FSQfBNAawC/CHrWIiFRr5MiRX69cufK47zr94Q9/6NC6devyrVu35t955527x48fH9H195iEkAW+ATsKwEXeLdprAEwFcNy3cEM8AKAxgNXeLdwPhFFqtrf9t25MIHkaAteYMgB84t28cGvkRyIi0jD52coBAIYOHXq4e/fuZVWXv/XWW21uueWWrwDg5ptvPvDhhx+2qqysDPs4YvbYHjPbCeAnNazO9LZZhsD1H5jZUQA/r2Y/zwF4LuTnESHvJwI47uvfZrYdgdN7IiInHL9bOdRm9+7dTVJTU48BQOPGjdGyZcuK3bt3Nwo+4bsuDenGBBERCYPfrRxqU92j3+p6wncohZCISJxx0cqhJp07dz62efPmJgBQVlaGQ4cOJZ566ql13qkRpBASEYkzLlo51OTSSy/9es6cOe0B4Nlnn207cODA4oSE8KNFrRxEROpp/dnHZcC3nCCtHE5bsGBBu5KSkoROnTqdmZ2dvW/GjBk777rrrn1XXHFFardu3TJbt25d8eqrr26M5FjUyiGEWjn4T60c/KdWDvWnVg6RUSsHERFpkBRCIiLijEJIRCRChspqb02W41VWVhJAjd9eVQiJiESoonwrDhaVKYjqUFlZyb1797YGkF/TNro7TkQkQsWH/gjsugP79nUDw/i3/FdfoVFFRUUHH4bW0FQCyC8vL6/xsWkKIRGRCJkVoaj4obC3v+iiTXlmlhXDIcUtnY4TERFnFEIiIuKMQkhERJxRCImIiDMKIRERcUYhJCIiziiERETEGYWQiIg4o1YOIbKysiwnJ8f1METkBENylb6sWj3NhERExBmFkIiIOKMQEhERZxRCIiLijEJIREScUQiJiIgzCiEREXFGISQiIs4ohERExBmFkIiIOKMQEhERZxRCIiLiTCPXA2hI8nYcRMqkt53UbpU+yUldAHjtoXJntZee/0dntUsOzHBWuzjd3bMsnzxvpLPauy7o56y2NEyaCYmIiDMKIRERcUYhJCIiziiERETEGYWQiIg4oxASERFnFEIiIuKMQkhERJxRCImIiDMKIRERcUYhJCIiziiERETEGYWQiIg4oxASERFnFEIiIuKMQkhERJxRCImIiDMKIRERcUYhJCIiziiERETEmZiFEMnOJF8huZFkAcl3SPYimUIy39smi+TMetSYXMu6/0dyG8lD33X/IiISWzEJIZIEsADAMjPrYWYZACYD6BS6nZnlmNm4epSqMYQAvAlgQD32LSIiMRarmdAFAMrM7MngAjPLNbMPQjcieT7Jt7z3LUjOIfkxyU9J/shbfhPJN0guJrmB5MPe8mkAmpHMJfli1QGY2b/M7MsYHZ+IiERBoxjtNxPAqgg/MwXAUjO7hWQbAB+R/Lu3rh+AswCUAigk+biZTSJ5p5n1q89ASY4GMBoAEk/pWJ9diYhIhBrSjQnDAEwimQtgGYAkAN28de+Z2UEzKwFQAKB7tIqa2WwzyzKzrMTmraO1WxERCUOsZkJrAFwZ4WcI4AozK/zWQvKHCMyAgioQu3GLiIiPYjUTWgqgKcmfBReQ7E/yvFo+swTAWO+mBpA8K4w6ZSQb12+oIiLiSkxCyMwMwCgAF3m3aK8BMBXAzlo+9gCAxgBWe7dwPxBGqdne9sfdmEDyYZLbATQnuZ3k1AgPQ0REYixmp7XMbCeAn9SwOtPbZhkC139gZkcB/Lya/TwH4LmQn0eEvJ8IYGIN9X8F4FffYegiIuKThnRjgoiInGQUQiIi4oxCSEREnFEIiYiIMwohERFxRiEkIiLOKIRERMQZhZCIiDijEBIREWcUQiIi4oxCSEREnFEIiYiIMwohERFxRiEkIiLOKIRERMQZhZCIiDijEBIREWcUQiIi4oxCSEREnKGZuR5Dg5GVlWU5OTmuhyEiJxiSq8wsy/U4GiLNhERExBmFkIiIOKMQEhERZxRCIiLijEJIREScUQiJiIgzCiEREXFGISQiIs4ohERExBmFkIiIOKMQEhERZxRCIiLijEJIREScUQiJiIgzauUQgmQxgEJH5TsA2Kfaqq3aJ2TtNDNr5ah2g9bI9QAamEJXPT9I5qi2aqv2iVvbRd14oNNxIiLijEJIREScUQh922zVVm3VVu0TrHaDphsTRETEGc2ERETEGYUQAJKXkCwk+TnJST7XnkNyD8l8n+t+j+T7JNeSXEPyLh9rJ5H8iORnXu3/9qt2yBgSSX5K8i2f624hmUcy1+87pki2Ifk6yXXef/eBPtVN8443+Coi+Qs/anv1f+n9Ocsn+TLJJB9r3+XVXePnMceTk/50HMlEAOsBXARgO4CPAfzUzAp8qn8ugEMA5ppZph81vbrJAJLN7BOSrQCsAjDSj+MmSQAtzOwQycYA/gngLjP7V6xrh4xhPIAsAKeY2Qgf624BkGVmvn9fheTzAD4ws6dJNgHQ3My+9nkMiQB2APihmX3hQ72uCPz5yjCzoyRfA/COmT3nQ+1MAK8AGADgGIDFAG4zsw2xrh1PNBMK/AH53Mw2mdkxBP7Q/Miv4mb2DwD7/aoXUvdLM/vEe18MYC2Arj7VNjM75P3Y2Hv59q8hkqcBuBTA037VdI3kKQDOBfAMAJjZMb8DyDMUwEY/AihEIwDNSDYC0BzATp/qpgP4l5kdMbNyAMsBjPKpdtxQCAX+4t0W8vN2+PSXcUNBMgXAWQBW+lgzkWQugD0A3jUz32oDeAzArwBU+lgzyAD8jeQqkqN9rHs6gL0AnvVOQz5NsoWP9YOuAfCyX8XMbAeARwFsBfAlgINm9jefyucDOJdke5LNAfwHgO/5VDtuKIQAVrPspDlHSbIlgPkAfmFmRX7VNbMKM+sH4DQAA7xTFzFHcgSAPWa2yo961TjHzH4AYDiAO7zTsX5oBOAHAP5kZmcBOAzA7+ufTQBcDmCejzXbInBmIxVAFwAtSF7nR20zWwvgdwDeReBU3GcAyv2oHU8UQoGZT+i/Tk6Df9N1p7zrMfMBvGhmb7gYg3dKaBmAS3wqeQ6Ay71rM68AGELyLz7Vhpnt9H7dA2ABAqeD/bAdwPaQGefrCISSn4YD+MTMdvtY80IAm81sr5mVAXgDwL/7VdzMnjGzH5jZuQicdtf1oCoUQoEbEXqSTPX+pXYNgEWOxxRz3s0BzwBYa2YzfK7dkWQb730zBP6iWOdHbTO718xOM7MUBP5bLzUzX/5lTLKFdxMIvFNhwxA4ZRNzZrYLwDaSad6ioQB8ufkmxE/h46k4z1YA/0ayufdnfigC1z99QfJU79duAH4M/4+/wTvpH2BqZuUk7wSwBEAigDlmtsav+iRfBnA+gA4ktwP4tZk940PpcwBcDyDPuzYDAJPN7B0faicDeN67UyoBwGtm5uut0o50ArAg8HchGgF4ycwW+1h/LIAXvX9sbQJws1+FvWsiFwH4uV81AcDMVpJ8HcAnCJwK+xT+Pr1gPsn2AMoA3GFmB3ysHRdO+lu0RUTEHZ2OExERZxRCIiLijEJIREScUQiJiIgzCiEREXFGISTOkTSS00N+vpvk1Cjt+zmSV0ZjX3XUucp7MvX7DWlcIg2dQkgaglIAPybZwfVAQnnfYwrXfwK43cwuiNV4RE5ECiFpCMoR+ALhL6uuqDpjIHnI+/V8kstJvkZyPclpJLO9PkV5JHuE7OZCkh94243wPp9I8hGSH5NcTfLnIft9n+RLAPKqGc9Pvf3nk/ydt+x+AIMAPEnykWo+8yvvM5+RnFbN+vu9ceSTnO19sx8kx5Es8Mb3irfsPH7Tl+fTkCcw3BNyLP/tLWtB8m2vbj7Jq8P7zyHin5P+iQnSYPwRwGqSD0fwme8j8Lj8/Qg8AeBpMxvAQIO+sQB+4W2XAuA8AD0AvE/yDAA3IPBE5f4kmwL4X5LBpysPAJBpZptDi5HsgsADKc8GcACBp2GPNLPfkBwC4G4zy6nymeEARiLQP+cIyXbVHMcsM/uNt/0LAEYAeBOBB4ymmllp8DFHAO5G4Jv3/+s9fLaE5DAAPb1xE8Ai78GoHQHsNLNLvX23Du+3VcQ/mglJg+A9wXsugHERfOxjry9SKYCNAIIhkodA8AS9ZmaVXjOxTQB6I/Dcthu8RxatBNAegb/IAeCjqgHk6Q9gmfcwzHIALyLQo6c2FwJ41syOeMdZXe+oC0iuJJkHYAiAPt7y1Qg8Zuc6fPP05f8FMIPkOABtvHEM816fIvB4mt7eseQhMAv8HcnBZnawjrGK+E4hJA3JYwhcWwntc1MO78+pd5qqSci60pD3lSE/V+Lbs/yqz6YyBGYMY82sn/dKDekzc7iG8VXX9qMurKb+NysDraafAHClmfUF8GcAwfbTlyIwQzwbwCqSjcxsGoBbATQD8C+Svb0aD4Ucyxne05vXe5/NA/CQd9pQpEFRCEmD4c0SXkMgiIK2IPAXKRDoC9P4O+z6KpIJ3nWi0wEUIvDA2tsYaGcBkr1Yd5O3lQDOI9nBu2nhpwh0y6zN3wDc4j3AE9WcjgsGzj7v9NqV3nYJAL5nZu8j0ICvDYCWJHuYWZ6Z/Q5ADgKzniVejZbeZ7uSPNU7fXjEzP6CQGM3v1s3iNRJ14SkoZkO4M6Qn/8MYCHJjwC8h5pnKbUpRCAsOgEYY2YlJJ9G4JTdJ94May8C125qZGZfkrwXwPsIzD7eMbOFdXxmMcl+AHJIHgPwDoDJIeu/JvlnBGYrWxBoLQIEnuj+F+86DgH83tv2AZIXAKhAoBXD/3jXjNIBrPDuaTgE4DoAZwB4hGQlAk9xvq3O3ykRn+kp2iIi4oxOx4mIiDMKIRERcUYhJCIiziiERETEGYWQiIg4oxASERFnFEIiIuKMQkhERJz5/zaAzk2SYEr7AAAAAElFTkSuQmCC", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaEAAAGDCAYAAACCzK//AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAAv2klEQVR4nO3deXxU9b0+8OdJWMImuxBASLAQEoJiDbTcigtYlAoqdS1xQWsp2kJb0LJ5vdzqrVwVy0XqRWpxu3VDSsHlB21B0F65YNQIIRAQiGyGRTCsCUPy+f1xztQxZJkhM/NN4Hm/XvNi5izfJSJPzjLnQzODiIiICwmuByAiImcvhZCIiDijEBIREWcUQiIi4oxCSEREnFEIiYiIMwohOaOQnEPyX6PUVleSR0gm+p9XkLwnGm377f0/kndGqz2R+qiB6wGIRIJkIYAOAE4CKAOQD+BFAHPNrNzMxkTQzj1m9veqtjGz7QCa13bMfn/TAHzLzG4LaX9oNNoWqc90JCT10XAzawGgG4DpACYC+GM0OyCpX9BE4kAhJPWWmRWb2WIAtwC4k2QmyedJPgIAJNuRfIvkVyQPkHyfZALJlwB0BfCmf7rt1yRTSBrJH5PcDmB5yLLQQDqf5BqSxSQXkWzj93U5yZ2h4yNZSPJKklcDmALgFr+/T/31/zy954/rQZKfk9xL8kWSLf11wXHcSXI7yf0kp8b2pysSHwohqffMbA2AnQAGVlg1wV/eHt4pvCne5nY7gO3wjqiam9ljIftcBiAdwFVVdHcHgLsBdIJ3SnBWGONbAuC3AF7z+7uwks1G+a8rAHSHdxpwdoVtLgGQBmAwgIdIptfUt0hdpxCSM8VuAG0qLAsASAbQzcwCZva+1fywxGlmdtTMjlex/iUzyzOzowD+FcDNwRsXaikbwJNmttXMjgCYDODWCkdh/25mx83sUwCfAqgszETqFYWQnCk6AzhQYdnjAD4D8FeSW0lOCqOdHRGs/xxAQwDtwh5l1Tr57YW23QDeEVxQUcj7Y4jSTRMiLimEpN4j2Q9eCP0jdLmZHTazCWbWHcBwAONJDg6urqK5mo6Uzgt53xXe0dZ+AEcBNA0ZUyK804Dhtrsb3o0WoW2fBLCnhv1E6jWFkNRbJM8hOQzAqwD+x8zWVVg/jOS3SBLAIXi3dJf5q/fAu/YSqdtIZpBsCuA3AN4wszIAmwAkkbyGZEMADwJoHLLfHgApJKv6f+4VAL8imUqyOb6+hnTyNMYoUm8ohKQ+epPkYXinxqYCeBLAXZVs1wPA3wEcAbAKwNNmtsJf9yiAB/075+6PoO+XADwP79RYEoBxgHenHoD7ADwLYBe8I6PQu+Xm+39+SfLjStqd57f9HoBtAEoAjI1gXCL1ElXUTkREXNGRkIiIOKMQEhERZxRCIiLijEJIREScUQiJiIgzelJwiHbt2llKSorrYYjIGeajjz7ab2bta97y7KMQCpGSkoKcnBzXwxCRMwzJz2ve6uyk03EiIuKMQkhERJxRCImIiDMKIRERcUYhJCIiziiERETEGYWQiIg4oxASERFnFEIiIuKMQkhERJxRCImIiDMKIRERcUYPMA2xblcxUia9Xet2CpNGRmE039QntWvU24yF1x896XoIzr1z4fmuhwAAuCV1YlTa6TJ9YFTaEamMjoRERMQZhZCIiDijEBIREWcUQiIi4oxCSEREnFEIiYiIMwohERFxRiEkIiLOKIRERMQZhZCIiDijEBIREWcUQiIi4oxCSEREnFEIiYiIMwohERFxRiEkIiLOKIRERMQZhZCIiDijEBIREWcUQiIi4oxCSEREnIlZCJHsSPJVkltI5pN8h2RPkikk8/xtskjOqkUfU6pZt4JkAclc/3Xu6fYjIiKx0SAWjZIkgIUAXjCzW/1lfQF0ALAjuJ2Z5QDIqUVXUwD8tpr12X4fIiJSB8XqSOgKAAEzmxNcYGa5ZvZ+6EYkLyf5lv++Gcl5JD8k+QnJ6/zlo0j+meQSkptJPuYvnw6giX+U86cYzUNERGIoJkdCADIBfBThPlMBLDezu0m2ArCG5N/9dX0BXASgFEAByafMbBLJn5tZ32rafI5kGYAFAB4xM6u4AcnRAEYDQOI57SMcsoiI1EZdujFhCIBJJHMBrACQBKCrv26ZmRWbWQmAfADdwmgv28z6ABjov26vbCMzm2tmWWaWldi0ZS2nICIikYhVCK0HcHGE+xDADWbW1391NbMN/rrSkO3KEMYRnJnt8v88DOBlAP0jHI+IiMRYrEJoOYDGJH8SXECyH8nLqtlnKYCx/k0NIHlRGP0ESDasuJBkA5Lt/PcNAQwDkBfJBEREJPZiEkL+tZcRAL7v36K9HsA0ALur2e1hAA0BrPVv4X44jK7m+ttXvDGhMYClJNcCyAWwC8AfIpqEiIjEXKxuTICZ7QZwcxWrM/1tVsC7/gMzOw7gp5W08zyA50M+Dwt5PxHAxEr2OYrITweKiEic1aUbE0RE5CyjEBIREWcUQiIi4oxCSEREnFEIiYiIMwohERFxRiEkIiLOKIRERMQZhZCIiDijEBIREWcUQiIi4oxCSEREnFEIiYiIMwohERFxRiEkIiLOKIRERMQZhZCIiDijEBIREWcUQiIi4gzNzPUY6oysrCzLyclxPQwROcOQ/MjMslyPoy7SkZCIiDijEBIREWcUQiIi4oxCSEREnFEIiYiIMwohERFxRiEkIiLOKIRERMQZhZCIiDijEBIREWcUQiIi4kwD1wOoS9btKkbKpLdr3U5h0sgojOabZmwYGPU2w3VL6sSotFMwZFRU2nElmwtcD8GZoiv6uh6CnKF0JCQiIs4ohERExBmFkIiIOKMQEhERZxRCIiLijEJIREScUQiJiIgzCiEREXFGISQiIs7oiQkiIhEKBALYuXMnSkpKwtr+b3/7W59PP/20MLajqpPKAeSdPHnynosvvnhvZRsohEREIrRz5060aNECKSkpIFnj9mVlZSczMzP3x2FodUp5eTn37duXUVRU9CyAayvbRqfjREQiVFJSgrZt24YVQGezhIQEa9++fTGAzCq3ieN4RETOGAqg8CQkJBiqyRqFkIiIOKNrQiIitRRGCZimwOcXh9te4fRrPjqdcYwfP75T8+bNy37zm9/sOZ39qzN27NjO8+fPb3vo0KHEY8eOfRKtdnUkJCIiNbr++uu/Wr169YZot6sQEhGph2bPnt22Z8+eGWlpaRnXX399asX1M2bMaJeZmZmelpaWcdVVV51/+PDhBACYN29e6x49evROS0vLyMrKSgOAnJycpD59+qT36tUro2fPnhnr1q1rXLG9wYMHH+3WrVsg2vPQ6TgRkXomJycn6YknnkhetWrVxuTk5JN79uxJrLhNdnb2wQkTJuwHgHHjxnWaNWtWu6lTp+6dPn168l//+tdNqampgf379ycCwFNPPdX+vvvu23PvvfceKCkp4cmTJ+M2Fx0JiYjUM0uXLj1n+PDhB5OTk08CQIcOHcoqbvPRRx81ufjii9N69uyZsWDBgrbr169PAoCsrKwj2dnZKTNmzGgXDJsBAwYcnTFjRvLUqVM7bt68uVHz5s0tXnNRCImI1DNmBpLVBsXo0aNTZ8+evX3Tpk35EydO3F1aWpoAAC+//PL2Rx55ZPeOHTsa9e3bt3dRUVHimDFjDixatOizJk2alA8dOrTn4sWLW8RnJgohEZF65+qrrz60ePHiNkVFRYkAUNnpuGPHjiV07do1UFpayldffbVNcPn69esbDxo06OjMmTN3t27d+uTWrVsb5efnN0pPTy998MEH9w4ZMuSr3NzcJvGaS8yuCZHsCGAmgH4ASgEUAvglgBMA3jKzTJJZAO4ws3Gn2ccUM/ttDdssBtDdzKr8xq6ISG0UTr+m2vV5eXnHMjMzo3ZnWVZWVsmECRO+GDhwYK+EhATLzMw8tmDBgsLQbSZNmrS7f//+6Z07dz6Rnp5+7MiRI4kA8Ktf/apLYWFhYzPjJZdccui73/3u8alTp3acP39+2wYNGlj79u0Djz766O6KfY4ZM6bLwoUL25SUlCR06NDhguzs7P1PPvnkKdtFimbRP/VH76vEHwB4wczm+Mv6AmgBYAf8EIpCP0fMrHk1638I4EYAF4TTX+PkHpZ858zaDguFSSNr3UZFMzYMjHqb4boldWJU2ikYMioq7biSzQWuh+BM0RV9XQ+hTtmwYQPS09PD3j7aIVTffPrpp+0uvPDClMrWxep03BUAAsEAAgAzyzWz90M3Ink5ybf8981IziP5IclPSF7nLx9F8s8kl5DcTPIxf/l0AE1I5pL8U8UBkGwOYDyAR2I0RxERqaVYnY7LBBDpN36nAlhuZneTbAVgDcm/++v6ArgI3mm9ApJPmdkkkj83s75VtPcwgBkAjlXXKcnRAEYDQOI57SMcsoiI1EZdujFhCIBJJHMBrACQBKCrv26ZmRWbWQmAfADdqmvIP/X3LTNbWFOnZjbXzLLMLCuxactaDF9ERCIVqyOh9fCuxUSCAG4ws4JvLCS/A+8IKKgMNY97AICLSRb6255LcoWZXR7hmEREJIZidSS0HEBjkj8JLiDZj+Rl1eyzFMBY/6YGkLwojH4CJBtWXGhm/21mncwsBcAlADYpgERE6p6YhJB5t9yNAPB9kltIrgcwDUB1t/M9DKAhgLUk8/zPNZnrb3/KjQkiIlL3xex7Qma2G8DNVazO9LdZAe/6D8zsOICfVtLO8wCeD/k8LOT9RADV3j9sZoWopqqfiEitTav+enIm0BRvIOxSDphWXKdKORw+fDhh+PDh3T///PPGiYmJGDJkyFdPP/30rmi0XZduTBARkTpqwoQJe7Zt27Y+Ly8vf/Xq1c1ff/31c6LRrkJIRKQeimcphxYtWpQPHz78MAAkJSXZBRdccGzHjh2NojEPhZCISD0TLOWwcuXKTQUFBfnPPPPM9orbZGdnH8zLy9tQUFCQn5aWdnzWrFntACBYyqGgoCB/yZIlnwFfl3LYuHFj/tq1azekpqaeqKrv/fv3J/7tb39rNXTo0EPRmItCSESknnFVyiEQCOCHP/xh99GjR+/JyMioMqgioRASEalnXJVyGDlyZEr37t1LHnroob3RmotCSESknnFRymHcuHGdDh06lPjHP/5xRzTnovLeIiK1Na242tX1vZTDli1bGj711FPJqampJb17984AgNGjR+8dP378/trOJSalHOorlXKonEo5eFTKQYJUyiEyLko5iIiI1EghJCIiziiERETEGYWQiIg4oxASERFnFEIiIuKMvickIlJLfV7oU9MmTfFR+KUc1t25rk6VcgCAgQMH9ti7d2/DsrIy9u/f//CLL764vUGD2keIjoRERKRGixYt2lJQUJC/adOm9V9++WXDefPmtY5GuwohEZF6KJ6lHACgTZs25QAQCAQYCARIMirzUAiJiNQzrko5XHLJJT3at29/YbNmzcruuuuug9GYi0JIRKSecVXK4R//+MfmoqKiT0+cOJHw5ptvRqWyqm5MCNGnc0vkTL8mCi1V/zDD0zEh6i3GXxdscT2EWilyPQARX7ilHN54443PBgwYcHzWrFltV65c2QLwSjksX7682eLFi1v27du3d25u7voxY8YcGDhw4NGFCxe2HDp0aM+nn3668Nprrz1cWbtNmza1YcOGfbVw4cJWI0aMqHVhOx0JiYjUM/Eu5VBcXJzw+eefNwS8wnZLlixp2atXr+PRmIuOhEREamndneuqXV/fSzkcOnQo4ZprrvnWiRMnWF5ezu9973uHHnjggX3RmItKOYTIysqynJwc18MQkTpOpRwio1IOIiJSJymERETEGYWQiIg4oxASERFnFEIiIuKMQkhERJzR94RERGppQ6/qb9dOBJpuQPilHNI3bqhzpRyCBg0a9K0dO3Y03rx58/potKcjIRERCcsLL7zQqlmzZqc8p642dCQUYt2uYqRMejsqbRUmjYxKO0EbXu0U1fYitfzy3zvt/3Rd16phVNp5NmlZVNqR2ptz2fXO+i66oq+zviuaPXt221mzZnUgifT09ON/+ctftoWunzFjRrvnnnuufSAQYEpKSukbb7yxrUWLFuXz5s1r/eijj3ZKSEiwFi1alOXk5BTk5OQk3XXXXamBQIDl5eVYsGDBlj59+pSGtldcXJwwa9asDnPnzv381ltvPT9a81AIiYjUM8FSDqtWrdqYnJx8srJnx2VnZx+cMGHCfgAYN25cp1mzZrWbOnXq3mAph9TU1MD+/fsTga9LOdx7770HSkpKGHy6dqjx48d3/sUvfrGnefPm5dGci07HiYjUM/Eu5fDBBx802bZtW+M77rjjq2jPRSEkIlLPhFvKYfbs2ds3bdqUP3HixN2lpaUJgFfK4ZFHHtm9Y8eORn379u1dVFSUOGbMmAOLFi36rEmTJuVDhw7tuXjx4hahbb3//vvN8/Lymnbu3LnPpZde2quwsLBx//7906IxF4WQiEg9E+9SDhMnTty3d+/etbt27Vr33nvvbUxJSSlds2ZNQTTmomtCIiK1lL6x+gdk1/dSDrGkUg4hGif3sOQ7Z0alLd0dVzfo7rgzT124O06lHCKjUg4iIlInKYRERMQZhZCIiDijEBIREWcUQiIi4oxCSEREnAnre0IkXzKz22taJiJyNvr9mOU1bdJ0JZaHXcrhZ3MG1blSDv3790/bu3dvw6SkpHIAWLZs2abOnTuf+pC5CIX7ZdXeoR9IJiKC2hgiIlL/vfjii1svvfTSY9Fss9rTcSQnkzwM4AKSh/zXYQB7ASyK5kBERCR8s2fPbtuzZ8+MtLS0jOuvvz614voZM2a0y8zMTE9LS8u46qqrzj98+HACAMybN691jx49eqelpWVkZWWlAd5Tufv06ZPeq1evjJ49e2asW7eucbzmUW0ImdmjZtYCwONmdo7/amFmbc1scpzGKCIiIYKlHFauXLmpoKAg/5lnntlecZvs7OyDeXl5GwoKCvLT0tKOz5o1qx0ABEs5FBQU5C9ZsuQz4OtSDhs3bsxfu3bthtTU1BOV9XvPPfek9OrVK+OBBx5ILi+PTkWHsG5MMLPJJDuT/BeSlwZfURmBiIhEJN6lHADgtdde27pp06b8VatWbfzggw+aP/30022jMZewQojkdAD/C+BBAA/4r/ujMQAREYlMvEs5AEBqamoAAFq3bl1+yy23HFizZk2zaMwl3Fu0RwBIM7MfmNlw/3VtNAYgIiKRiXcph0AggC+++KIBAJSWlvKdd95pmZmZeTwacwn37ritABoCKK1pQxGRs83P5gyqdn19L+Vw/PjxhCuvvLJHIBBgeXk5Bw4ceGj8+PH7ojGXsEo5kFwA4EIAyxASRGY2rpp9OgKYCaCfv08hgF8COAHgLTPLJJkF4I7q2qlhXFPM7LdVrFsCIBle0L4P4Gdmdsp501Aq5VA1lXJQKYe6QqUc6p/qSjmEeyS02H+FhSQBLATwgpnd6i/rC6ADgB3B7cwsB0BOuO1WYgqASkMIwM1mdsgfyxsAbgLwai36EhGRKAsrhMzsBZJNAHQ1s3BKul4BIGBmc0LayAUAkinBZSQvB3C/mQ0j2QzAUwD6+OOaZmaLSI4CcC2ApgDOB7DQzH7t3yzRhGQugPVmll1hzIdC5tgIgKr3iYjUMeHeHTccQC6AJf7nviSrOzLKBBDpYyemAlhuZv3ghdjjfjABQF8At8ALqFtInmdmkwAcN7O+FQMoZNxL4X2x9jC8oyEREalDwr07bhqA/gC+Av55VHPKN3RraQiASf6RzQoASQC6+uuWmVmxmZUAyAfQLZwGzewqeNeFGgOo9MohydEkc0jmlB0rrt0MREQkIuGG0Ekzq/gvdHWnt9Yj8mfLEcAN/pFNXzPrambBC3mhd+WVIfxrWfCDazGA66pYP9fMsswsK7FpywiHLCIitRFuCOWRHAkgkWQPkk8B+KCa7ZcDaEzyJ8EFJPuRvKyafZYCGOvfSACSF4UxrgDJU25/ItmcZLL/vgGAHwDYGEZ7IiISR+EeUYyFd82mFMAr8ALj4ao2NjMjOQLATJKTAJTg61u0q/IwvFu61/pBVAhgWA3jmutv/3GF60LNACwm2RhAIrxQnFNZAyIitTXjlpr+qULTpRGcHZrw2lt1rpRDSUkJ77rrrq6rVq1qQdL+7d/+bdeoUaO+qm274d4ddwxeCE0Nt2Ez2w3g5ipWZ/rbrIB3/QdmdhzATytp53kAz4d8HhbyfiKAiZXsswfe95NERCQKJk+enNy+fftAYWFhXllZGfbu3Rv2ZZHq1FTKYab/55skF1d8RWMAIiISuXiXcnjllVfaPfLII0UAkJiYiODDU2urpiR7yf/ziWh0JiIitRcs5bBq1aqNycnJJyt7dlx2dvbBCRMm7AeAcePGdZo1a1a7qVOn7g2WckhNTQ3s378/Efi6lMO99957oKSkhMGnawcFtxs/fnynDz74oEW3bt1K586du/28886rdRDVVE/oI//PlZW9atu5iIhELt6lHAKBAPfs2dPwkksuOZKfn7/hO9/5ztGxY8eeF4251HQ6bh3JtVW9ojEAERGJTLxLOXTo0OFkUlJS+e233/4VANx2220H8vLymkZjLjXdov1DAPcBGF7h9XN/nYiIxFm8SzkkJCRg8ODBxW+//XYLAHjnnXfO6dGjR1xKOfwOwBQz+zx0Icn2/rrh0RiEiEh9NuG1t6pdX99LOQDAk08+uXPkyJGp999/f2Lbtm1Pvvjii4UVtzkd1ZZyIJlnZplVrFtnZn2iMYi6QqUcqqZSDirlUFeolEP9U10ph5pOxyVVs65JNetERERqVFMIfRj66J0gkj9G5E/JFhER+Yaargn9EsBCktn4OnSy4NXnGRHDcYmIyFmg2hDyH3/zLySvgP+oHQBvm9nymI9MRETOeOE+O+5dAO/GeCwiInKWCbeUg4iISNRF5SmoIiJns52T3q92fSug6U68H3Yphy7TB9apUg4HDx5MGDBgQK/g5z179jQcMWLEgXnz5u2obdsKIRERqVbr1q3LN27cmB/83Lt37/SbbrrpYDTa1uk4EZF6KN6lHILWrVvX+Msvv2x41VVXHYnGPHQkJCJSz8S7lEOoF154oc211157ICEhOscwOhISEaln4l3KIdTChQvb3H777QeiNReFkIhIPRPvUg5Bq1atalJWVsaBAwcei9ZcdDouRJ/OLZEz/ZootVYcpXY86dOi2lzk/bvt3rlpGOh6COKb5noAdcDVV1996MYbb/zWlClT9nTs2LFsz549iRWPhiqWckhOTg4AX5dyGDRo0NGlS5e22rp1a6MDBw6Upaenl/bu3Xvv1q1bG+fm5ja59tprD1fs96WXXmozYsSIqB0FAQohEZFa6zK9+l9SzoRSDgCwePHiNm+++ebmaM0DqKGUw9kmKyvLcnJyXA9DROo4lXKITG1KOYiIiMSMQkhERJxRCImIiDMKIRERcUYhJCIiziiERETEGX1PSESklqZNm1bTJk3feOONsEs5TJs2rU6VcgCAZ555ps2MGTM6AkCHDh0Cr7/++rbgY4NqQ0dCIiJSrUAggMmTJ5+3cuXKTZs2bcrv3bv38ccff/zcaLStEBIRqYfiWcqhvLycZobDhw8nlJeX49ChQwmdOnU6EY156HSciEg9E+9SDo0bN7Ynn3xy+7e//e3eTZo0KevWrVvpiy++uD0ac1EIhVi3qxgpk96OWnuFSSOj1lbQ74sWRr3NeLiuVcOot1nT87pEzlThlnJ46KGHOh8+fDjx6NGjiZdddlkx8HUphxtuuOFgdnb2QcAr5fDEE08k79y5s9Gtt956sE+fPqWhbZWWlnLu3LntV69enZ+enl46atSorlOmTEl+7LHHvqjtXHQ6TkSknol3KYf/+7//awIAvXv3Lk1ISMCPfvSjA6tXr24WjbkohERE6pmrr7760OLFi9sUFRUlAkBlp+MqlnIILg+Wcpg5c+bu1q1bn9y6dWuj/Pz8Runp6aUPPvjg3iFDhnyVm5vbJLStbt26BT777LOk3bt3NwCAJUuWnNOzZ8+SaMxFp+NERGqpplu063sph5SUlMADDzzwxSWXXJLWoEED69Kly4mXX355WzTmolIOIRon97DkO2dGrT1dE/qargnJmUSlHCKjUg4iIlInKYRERMQZhZCIiDijEBIREWcUQiIi4oxCSEREnNH3hEREamnZ8vNr2qTpnuUIu5TD4EFb6lwphz/84Q+tH3/88eTy8nJeeeWVxXPmzNkZjXZ1JCQiItUqKipKfOihh7qsWLFi02effbZ+7969DRYtWtSi5j1rphASEamH4lnKoaCgoHFqamppp06dTgLA4MGDD82fP791NOah03EiIvVMvEs5ZGRklG7ZsiWpoKCgUffu3U8sXry4dSAQYDTmoiMhEZF6JtxSDhdffHFaz549MxYsWNB2/fr1ScDXpRxmzJjRLhg2AwYMODpjxozkqVOndty8eXOj5s2bf+N5bu3bty/73e9+9/lNN93UvV+/fr26du1ampiYGJVnvimERETqmXiXcgCAkSNHFq9du3Zjbm7uxrS0tJLzzz+/9NReI6cQEhGpZ+JdygEAdu3a1QAA9u3bl/jss8+ee9999+2Lxlx0TUhEpJYGD9pS7fr6XsoBAMaMGXNefn5+UwCYOHHi7gsuuCAqR0Iq5RBCpRxiR6Uc5EyiUg6RUSkHERGpk2IWQiQ7knyV5BaS+STfIdmTZArJPH+bLJKzatHHlCqWNyX5NsmNJNeTnH66fYiISOzEJIRIEsBCACvM7HwzywAwBUCH0O3MLMfMxtWiq0pDyPeEmfUCcBGA75EcWot+REQkBmJ1JHQFgICZzQkuMLNcM3s/dCOSl5N8y3/fjOQ8kh+S/ITkdf7yUST/THIJyc0kH/OXTwfQhGQuyT+Ftmtmx8zsXf/9CQAfA+gSo7mKiMhpilUIZQKI9AF8UwEsN7N+8ELscZLN/HV9AdwCoA+AW0ieZ2aTABw3s75mll1VoyRbARgOYFkV60eTzCGZU3asOMIhi4hIbdSlGxOGAJhEMhfACgBJALr665aZWbGZlQDIB9AtnAZJNgDwCoBZZra1sm3MbK6ZZZlZVmLTlrWcgoiIRCJW3xNaD+DGCPchgBvMrOAbC8nvAAi9H70M4Y97LoDNZjYzwrGIiISt47u5NW3SFO/mhl3KoeiKvnWulMPYsWM7z58/v+2hQ4cSjx079klw+fHjx3njjTemrlu3rmmrVq1Ozp8/f2taWtqJcNuN1ZHQcgCNSf4kuIBkP5KXVbPPUgBj/ZsaQPKiMPoJkKz0CygkHwHQEsAvwx61iIhU6vrrr/9q9erVp3zX6b/+67/atWzZ8uT27dvzfv7zn+8ZP358RNffYxJC5n0DdgSA7/u3aK8HMA3AKd/CDfEwgIYA1vq3cD8cRldz/e2/cWMCyS7wrjFlAPjYv3nhnshnIiJSN8WzlAMADB48+Gi3bt0CFZe/9dZbre6+++4vAeCuu+46+MEHH7QoLy8Pex4xe2yPme0GcHMVqzP9bVbAu/4DMzsO4KeVtPM8gOdDPg8LeT8RwMRK9tkJ7/SeiMgZJ96lHKqzZ8+eRqmpqScAoGHDhmjevHnZnj17GgSf8F2TunRjgoiIhCHepRyqU9mj32p6wncohZCISD3jopRDVTp27Hhi27ZtjQAgEAjgyJEjieeee+4poVgVhZCISD3jopRDVa655pqv5s2b1xYAnnvuudYDBgw4nJAQfrSolIOISC0VXdG32vVnSCmHLgsXLmxTUlKS0KFDhwuys7P3P/nkk7t/8Ytf7L/hhhtSu3btmtmyZcuy1157rfq6FhWolEMIlXKIHZVykDOJSjlERqUcRESkTlIIiYiIMwohEZHToEsZ4SkvLyeAKr+9qhASEYlQUlISvvzySwVRDcrLy7lv376WAPKq2kZ3x4mIRKhLly7YuXMn9u3bF9b2RUVFDcrKytrFeFh1UTmAvJMnT1b52DSFkIhIhBo2bIjU1FMe11aljIyMdWaWFcMh1Vs6HSciIs4ohERExBmFkIiIOKMQEhERZxRCIiLijEJIREScUQiJiIgzCiEREXFGpRxCZGVlWU5OjuthiMgZhuRH+rJq5XQkJCIiziiERETEGYWQiIg4oxASERFnFEIiIuKMQkhERJxRCImIiDMKIRERcUYhJCIiziiERETEGYWQiIg4oxASERFnGrgeQF2yblcxUia9Xas2CpNGRmk0p+qT2jVmbbsw6p1uMWn3ltSJUWmnYMioqLQTL4MHbXE9BJGI6UhIREScUQiJiIgzCiEREXFGISQiIs4ohERExBmFkIiIOKMQEhERZxRCIiLijEJIREScUQiJiIgzCiEREXFGISQiIs4ohERExBmFkIiIOKMQEhERZxRCIiLijEJIREScUQiJiIgzCiEREXFGISQiIs7ELIRIdiT5KsktJPNJvkOyJ8kUknn+NlkkZ9WijynVrPsPkjtIHjnd9kVEJLZiEkIkCWAhgBVmdr6ZZQCYAqBD6HZmlmNm42rRVZUhBOBNAP1r0baIiMRYrI6ErgAQMLM5wQVmlmtm74duRPJykm/575uRnEfyQ5KfkLzOXz6K5J9JLiG5meRj/vLpAJqQzCX5p4oDMLP/M7MvYjQ/ERGJggYxajcTwEcR7jMVwHIzu5tkKwBrSP7dX9cXwEUASgEUkHzKzCaR/LmZ9a3NQEmOBjAaABLPaV+bpkREJEJ16caEIQAmkcwFsAJAEoCu/rplZlZsZiUA8gF0i1anZjbXzLLMLCuxactoNSsiImGI1ZHQegA3RrgPAdxgZgXfWEh+B94RUFAZYjduERGJo1gdCS0H0JjkT4ILSPYjeVk1+ywFMNa/qQEkLwqjnwDJhrUbqoiIuBKTEDIzAzACwPf9W7TXA5gGYHc1uz0MoCGAtf4t3A+H0dVcf/tTbkwg+RjJnQCaktxJclqE0xARkRiL2WktM9sN4OYqVmf626yAd/0HZnYcwE8raed5AM+HfB4W8n4igIlV9P9rAL8+jaGLiEic1KUbE0RE5CyjEBIREWcUQiIi4oxCSEREnFEIiYiIMwohERFxRiEkIiLOKIRERMQZhZCIiDijEBIREWcUQiIi4oxCSEREnFEIiYiIMwohERFxRiEkIiLOKIRERMQZhZCIiDijEBIREWcUQiIi4gzNzPUY6oysrCzLyclxPQwROcOQ/MjMslyPoy7SkZCIiDijEBIREWcUQiIi4oxCSEREnFEIiYiIMwohERFxRiEkIiLOKIRERMQZhZCIiDijEBIREWcUQiIi4oxCSEREnFEIiYiIMwohERFxRqUcQpA8DKDAUfftAOx31Lfr/s/mubvuX3OPj25m1j5OfdUrDVwPoI4pcFXzg2SOy3ojLvs/m+fuun/NXTV+XNPpOBERcUYhJCIiziiEvmnuWdq36/7P5rm77l9zF6d0Y4KIiDijIyEREXFGIQSA5NUkC0h+RnJSlNo8j+S7JDeQXE/yF/7yNiT/RnKz/2frkH0m+2MoIHlVyPKLSa7z180iyQjGkUjyE5JvxbN/kq1IvkFyo/8zGBDPuZP8lf9zzyP5CsmkWPZPch7JvSTzQpZFrT+SjUm+5i9fTTKlhr4f93/2a0kuJNkqFn1X1X/IuvtJGsl28e6f5Fi/j/UkH4tV/1JLZnZWvwAkAtgCoDuARgA+BZARhXaTAXzbf98CwCYAGQAeAzDJXz4JwH/67zP8vhsDSPXHlOivWwNgAAAC+H8AhkYwjvEAXgbwlv85Lv0DeAHAPf77RgBaxbHvzgC2AWjif34dwKhY9g/gUgDfBpAXsixq/QG4D8Ac//2tAF6roe8hABr47/8zVn1X1b+//DwASwF8DqBdPPsHcAWAvwNo7H8+N1b961XLfytdD8D1y/9LtzTk82QAk2PQzyIA34f3Zdhkf1kyvO8mndKv/z/vAH+bjSHLfwTgmTD77AJgGYBB+DqEYt4/gHPghQArLI/L3OGF0A4AbeB9F+4teP8ox7R/ACkV/iGMWn/Bbfz3DeB9yZJV9V1hXCMA/ClWfVfVP4A3AFwIoBBfh1Bc+of3i8eVlfwsYtK/Xqf/0um4r//BCtrpL4sa//D9IgCrAXQwsy8AwP/z3BrG0dl/fzrjmwng1wDKQ5bFo//uAPYBeI7eqcBnSTaLU98ws10AngCwHcAXAIrN7K/x6j9ENPv75z5mdhJAMYC2YY7jbni/2cetb5LXAthlZp9WWBWvufcEMNA/fbaSZL849y9hUgh5h94VRe2WQZLNASwA8EszO3Qa4zit8ZEcBmCvmX0U1kCj238DeKdH/tvMLgJwFN7pqHj0Df/ay3XwTrd0AtCM5G3x6j8Mp9Pf6f4spgI4CeBP8eqbZFMAUwE8VNnqWPfvawCgNYDvAngAwOv+NZ64/ewlPAoh7zee80I+dwGwOxoNk2wIL4D+ZGZ/9hfvIZnsr08GsLeGcez030c6vu8BuJZkIYBXAQwi+T9x6n8ngJ1mttr//Aa8UIrX3K8EsM3M9plZAMCfAfxLHPsPimZ//9yHZAMALQEcqK5zkncCGAYg2/xzSXHq+3x4vwB86v/96wLgY5Id49R/cJ8/m2cNvLMB7eLYv4RJIQR8CKAHyVSSjeBdeFxc20b937r+CGCDmT0ZsmoxgDv993fCu1YUXH6rfydOKoAeANb4p3EOk/yu3+YdIftUycwmm1kXM0vx57TczG6LR/9mVgRgB8k0f9FgAPnxmju803DfJdnU328wgA1x7D8omv2FtnUjvP+e1R2NXA1gIoBrzexYhTHFtG8zW2dm55pZiv/3bye8m3SK4tG/7y/wroWCZE94N8fsj2P/Ei7XF6XqwgvAD+DdvbYFwNQotXkJvEP2tQBy/dcP4J1LXgZgs/9nm5B9pvpjKEDIXVgAsgDk+etmI8KLogAux9c3JsSlfwB9AeT48/8LvFMjcZs7gH8HsNHf9yV4d0PFrH8Ar8C7/hSA94/uj6PZH4AkAPMBfAbvLq7uNfT9GbzrGMG/e3Ni0XdV/VdYXwj/xoR49Q8vdP7Hb+9jAINi1b9etXvpiQkiIuKMTseJiIgzCiEREXFGISQiIs4ohERExBmFkIiIOKMQEmf8pyvPCPl8P8lpUWr7eZI3RqOtGvq5id5Twt+NdV81jKOQIU+qFqkvFELiUimAH9a1fzxJJkaw+Y8B3GdmV8RqPCJnMoWQuHQSXonlX1VcUfFIhuQR/8/L/QdSvk5yE8npJLNJrvFrwZwf0syVJN/3txvm759Ir9bOh/Rq7fw0pN13Sb4MYF0l4/mR334eyf/0lz0E70vJc0g+XmH7ZJLvkcz19xnoL/9vkjn0atz8e8j2hSR/S3KVv/7bJJeS3EJyTMgY36NXHyif5BySp/w/TPI2/+eRS/IZf86J/s80z5/HKT9zERcauB6AnPV+D2AtQ4qOheFCAOnwnt+1FcCzZtafXuHAsQB+6W+XAuAyeM8ye5fkt+A9jqXYzPqRbAzgf0n+1d++P4BMM9sW2hnJTvBq8lwM4CCAv5K83sx+Q3IQgPvNLKfCGEfCKxHyH/6RVVN/+VQzO+AvW0byAjNb66/bYWYDSP4OwPPwnv+XBGA9gDkhY8yAV6NnCYAfwns2X3Cs6QBuAfA9MwuQfBpAtt9GZzPL9LdrVfOPWST2dCQkTpn3ZPEXAYyLYLcPzewLMyuF94iVYIisgxc8Qa+bWbmZbYYXVr3g1RW6g2QuvNIabeE9PwzwniH2jQDy9QOwwrwHogafSH1pTWMEcJd/jauPmR32l99M8mMAnwDoDS9QgoLPLFwHYLWZHTazfQBKQkJjjZltNbMyeI+ruaRCv4PhheWH/hwHwyutsRVAd5JP+c+Vq+6J7iJxoyMhqQtmwnu+13Mhy07C/yXJf6Bko5B1pSHvy0M+l+Obf6crPpMq+Mj+sWa2NHQFycvhlZyoTNjl1P/Zkdl7JC8FcA2Al/zTde8DuB9APzM7SPJ5eEc6QaHzqDjH4Lwqm1PFsb5gZpNPmQR5IYCrAPwMwM3w6gyJOKUjIXHOzA7Aq4T545DFhfB+owe82kANT6Ppm0gm+NeJusN7YOVSAPfSK7MBkj3pFdyrzmoAl5Fs559G+xGAldXtQLIbvHpOf4D3NPVvw6s4exRAMckOAIaexpz603viewK8027/qLB+GYAbSZ7rj6MNyW7+zR8JZrYAwL/64xFxTkdCUlfMAPDzkM9/ALCI5Bp4/7BWdZRSnQJ4YdEBwBgzKyH5LLxTdh/7R1j7AFxfXSNm9gXJyQDehXek8Y6Z1VTS4XIAD5AMADgC4A4z20byE3jXZ7YC+N/TmNMqANMB9AHwHoCFFcaaT/JBeNetEuA9WfpnAI7Dq3Qb/MXzlCMlERf0FG2ResI/ZXi/mQ1zPBSRqNHpOBERcUZHQiIi4oyOhERExBmFkIiIOKMQEhERZxRCIiLijEJIREScUQiJiIgz/x89kuXFwhQj8wAAAABJRU5ErkJggg==", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaEAAAGDCAYAAACCzK//AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAAvD0lEQVR4nO3de3hU5bk+/vtOOISTnIUAQiJCSAgaa6RlF1TAovxEi1WrJR7QbSlawRa0ILgtu/qrVA1lI7VKFRXrEZENHjbUilC7pWjQSEIgIBA5mQCCnBMmyfP9Y1Y2Y8xhxszMO4H7c125mFlrzXqfhcjNu9aa9dDMICIi4kKc6wJEROT0pRASERFnFEIiIuKMQkhERJxRCImIiDMKIRERcUYhJKcUkk+S/I8w7asnySMk4733K0neHo59e/v7H5K3hGt/Io1RE9cFiISCZBGALgDKAVQAKACwAMA8M6s0s/Eh7Od2M/t7bduY2XYArRtaszfeDADnmNmNAfsfGY59izRmmglJY3SlmbUB0AvATABTADwTzgFI6h9oIlGgEJJGy8wOmtlSANcDuIVkOsnnSD4EACQ7kXyL5Nck95P8gGQcyRcA9ATwpne67Tckk0gayX8nuR3AioBlgYHUm+RHJA+SXEKygzfWJSR3BtZHsojkpSQvBzANwPXeeJ956//v9J5X1/0kvyC5h+QCkm29dVV13EJyO8l9JKdH9ndXJDoUQtLomdlHAHYCGFJt1WRveWf4T+FN829uNwHYDv+MqrWZPRLwmYsBpAK4rJbhbgZwG4Bu8J8SnBNEfcsA/B7Aq95459Ww2VjvZyiAs+E/DTi32jaDAaQAGA7gAZKp9Y0tEusUQnKq2A2gQ7VlPgCJAHqZmc/MPrD6H5Y4w8yOmtnxWta/YGb5ZnYUwH8A+GnVjQsNlAVglpltNbMjAO4DcEO1Wdh/mtlxM/sMwGcAagozkUZFISSniu4A9ldb9iiAzwH8jeRWklOD2M+OENZ/AaApgE5BV1m7bt7+AvfdBP4ZXJXigNfHEKabJkRcUghJo0fyQvhD6J+By83ssJlNNrOzAVwJYBLJ4VWra9ldfTOlswJe94R/trUPwFEALQNqiof/NGCw+90N/40WgfsuB1BSz+dEGjWFkDRaJM8gOQrAKwD+amZ51daPInkOSQI4BP8t3RXe6hL4r72E6kaSaSRbAvgdgNfNrALAJgAJJK8g2RTA/QCaB3yuBEASydr+n3sZwK9JJpNsjZPXkMq/Q40ijYZCSBqjN0kehv/U2HQAswDcWsN2fQD8HcARAKsBPGFmK711DwO437tz7p4Qxn4BwHPwnxpLADAR8N+pB+BOAE8D2AX/zCjwbrmF3q9fkfykhv3O9/b9DwDbAJQCmBBCXSKNEtXUTkREXNFMSEREnFEIiYiIMwohERFxRiEkIiLOKIRERMQZPSk4QKdOnSwpKcl1GSJyilm7du0+M+tc/5anH4VQgKSkJOTk5LguQ0ROMSS/qH+r05NOx4mIiDMKIRERcUYhJCIiziiERETEGYWQiIg4oxASERFnFEIiIuKMQkhERJxRCImIiDMKIRERcUYhJCIiziiERETEGT3ANEDeroNImvq20xqKEsY4HR8ABiT3dF0CXnu43HUJAIB3zuvtugRcnzzFdQnoMXOI6xLkFKWZkIiIOKMQEhERZxRCIiLijEJIREScUQiJiIgzCiEREXFGISQiIs4ohERExBmFkIiIOKMQEhERZxRCIiLijEJIREScUQiJiIgzCiEREXFGISQiIs4ohERExBmFkIiIOKMQEhERZxRCIiLijEJIREScUQiJiIgzEQshkl1JvkJyC8kCku+Q7EsyiWS+t00myTkNGGNaHetWkiwkmev9nPldxxERkchoEomdkiSAxQCeN7MbvGUZALoA2FG1nZnlAMhpwFDTAPy+jvVZ3hgiIhKDIjUTGgrAZ2ZPVi0ws1wz+yBwI5KXkHzLe92K5HySH5P8lOSPveVjSb5BchnJzSQf8ZbPBNDCm+W8GKHjEBGRCIrITAhAOoC1IX5mOoAVZnYbyXYAPiL5d29dBoDzAZQBKCT5uJlNJXmXmWXUsc9nSVYAWATgITOz6huQHAdgHADEn9E5xJJFRKQhYunGhBEAppLMBbASQAKAnt6698zsoJmVAigA0CuI/WWZ2QAAQ7yfm2rayMzmmVmmmWXGt2zbwEMQEZFQRCqE1gO4IMTPEMA1Zpbh/fQ0sw3eurKA7SoQxAzOzHZ5vx4G8BKAgSHWIyIiERapEFoBoDnJn1ctIHkhyYvr+MxyABO8mxpA8vwgxvGRbFp9IckmJDt5r5sCGAUgP5QDEBGRyItICHnXXq4G8CPvFu31AGYA2F3Hxx4E0BTAOu8W7geDGGqet331GxOaA1hOch2AXAC7APwlpIMQEZGIi9SNCTCz3QB+WsvqdG+blfBf/4GZHQfwixr28xyA5wLejwp4PQXAlBo+cxShnw4UEZEoi6UbE0RE5DSjEBIREWcUQiIi4oxCSEREnFEIiYiIMwohERFxRiEkIiLOKIRERMQZhZCIiDijEBIREWcUQiIi4oxCSEREnFEIiYiIMwohERFxRiEkIiLOKIRERMQZhZCIiDijEBIREWcUQiIi4gzNzHUNMSMzM9NycnJclyEipxiSa80s03UdsUgzIRERcUYhJCIiziiERETEGYWQiIg4oxASERFnFEIiIuKMQkhERJxRCImIiDMKIRERcUYhJCIiziiERETEmSauC4glebsOImnq205rKEoY43R8AMjeMMR1Cbg+eYrrEgAAhSPGui4BWVzkugQUD81wXYKcojQTEhERZxRCIiLijEJIREScUQiJiIgzCiEREXFGISQiIs4ohERExBmFkIiIOKMQEhERZ/TEBBGREPl8PuzcuROlpaVBbf/uu+8O+Oyzz4oiW1VMqgSQX15efvsFF1ywp6YNFEIiIiHauXMn2rRpg6SkJJCsd/uKiory9PT0fVEoLaZUVlZy7969acXFxU8DuKqmbXQ6TkQkRKWlpejYsWNQAXQ6i4uLs86dOx8EkF7rNlGsR0TklKEACk5cXJyhjqxRCImIiDO6JiQi0kBBtIBpCXxxQbD7K5p5xdrvUsekSZO6tW7duuJ3v/tdyXf5fF0mTJjQfeHChR0PHToUf+zYsU/DtV/NhEREpF6jR4/+es2aNRvCvV+FkIhIIzR37tyOffv2TUtJSUkbPXp0cvX12dnZndLT01NTUlLSLrvsst6HDx+OA4D58+e379OnT/+UlJS0zMzMFADIyclJGDBgQGq/fv3S+vbtm5aXl9e8+v6GDx9+tFevXr5wH4dOx4mINDI5OTkJjz32WOLq1as3JiYmlpeUlMRX3yYrK+vA5MmT9wHAxIkTu82ZM6fT9OnT98ycOTPxb3/726bk5GTfvn374gHg8ccf73znnXeW3HHHHftLS0tZXl4etWPRTEhEpJFZvnz5GVdeeeWBxMTEcgDo0qVLRfVt1q5d2+KCCy5I6du3b9qiRYs6rl+/PgEAMjMzj2RlZSVlZ2d3qgqbQYMGHc3Ozk6cPn16182bNzdr3bq1RetYFEIiIo2MmYFknUExbty45Llz527ftGlTwZQpU3aXlZXFAcBLL720/aGHHtq9Y8eOZhkZGf2Li4vjx48fv3/JkiWft2jRonLkyJF9ly5d2iY6R6IQEhFpdC6//PJDS5cu7VBcXBwPADWdjjt27Fhcz549fWVlZXzllVc6VC1fv35982HDhh2dPXv27vbt25dv3bq1WUFBQbPU1NSy+++/f8+IESO+zs3NbRGtY4nYNSGSXQHMBnAhgDIARQB+BeAEgLfMLJ1kJoCbzWzidxxjmpn9vp5tlgI428xq/cauiEhDFM28os71+fn5x9LT08N2Z1lmZmbp5MmTvxwyZEi/uLg4S09PP7Zo0aKiwG2mTp26e+DAgandu3c/kZqaeuzIkSPxAPDrX/+6R1FRUXMz4+DBgw/94Ac/OD59+vSuCxcu7NikSRPr3Lmz7+GHH95dfczx48f3WLx4cYfS0tK4Ll26nJuVlbVv1qxZ39ouVDQL/6k/+r9K/CGA583sSW9ZBoA2AHbAC6EwjHPEzFrXsf4nAK4FcG4w4zVP7GOJt8xuaFkNUpQwxun4AJC9YYjrEnB98hTXJQAACkeMdV0CsrjIdQkoHprhuoSYsmHDBqSmpga9fbhDqLH57LPPOp133nlJNa2L1Om4oQB8VQEEAGaWa2YfBG5E8hKSb3mvW5GcT/Jjkp+S/LG3fCzJN0guI7mZ5CPe8pkAWpDMJfli9QJItgYwCcBDETpGERFpoEidjksHEOo3fqcDWGFmt5FsB+Ajkn/31mUAOB/+03qFJB83s6kk7zKzjFr29yCAbADH6hqU5DgA4wAg/ozOIZYsIiINEUs3JowAMJVkLoCVABIA9PTWvWdmB82sFEABgF517cg79XeOmS2ub1Azm2dmmWaWGd+ybQPKFxGRUEVqJrQe/msxoSCAa8ys8BsLye/DPwOqUoH66x4E4AKSRd62Z5JcaWaXhFiTiIhEUKRmQisANCf586oFJC8keXEdn1kOYIJ3UwNInh/EOD6STasvNLM/m1k3M0sCMBjAJgWQiEjsiUgImf+Wu6sB/IjkFpLrAcwAUNftfA8CaApgHcl873195nnbf+vGBBERiX0R+56Qme0G8NNaVqd726yE//oPzOw4gF/UsJ/nADwX8H5UwOspAOq8l9fMilBHVz8RkQabUff15HSgJV5H0K0cMONgTLVyOHz4cNyVV1559hdffNE8Pj4eI0aM+PqJJ57YFY59x9KNCSIiEqMmT55csm3btvX5+fkFa9asaf3aa6+dEY79KoRERBqhaLZyaNOmTeWVV155GAASEhLs3HPPPbZjx45m4TgOhZCISCNT1cph1apVmwoLCwueeuqp7dW3ycrKOpCfn7+hsLCwICUl5ficOXM6AUBVK4fCwsKCZcuWfQ6cbOWwcePGgnXr1m1ITk4+UdvY+/bti3/33XfbjRw58lA4jkUhJCLSyLhq5eDz+fCTn/zk7HHjxpWkpaXVGlShUAiJiDQyrlo5jBkzJunss88ufeCBB/aE61gUQiIijYyLVg4TJ07sdujQofhnnnlmRziPRe29RUQaasbBOlc39lYOW7Zsafr4448nJicnl/bv3z8NAMaNG7dn0qRJ+xp6LBFp5dBYqZWDn1o5nKRWDn5q5fBNauUQGhetHEREROqlEBIREWcUQiIi4oxCSEREnFEIiYiIMwohERFxRt8TEhFpoAHPD6hvk5ZYG3wrh7xb8mKqlQMADBkypM+ePXuaVlRUcODAgYcXLFiwvUmThkeIZkIiIlKvJUuWbCksLCzYtGnT+q+++qrp/Pnz24djvwohEZFGKJqtHACgQ4cOlQDg8/no8/lIMizHoRASEWlkXLVyGDx4cJ/OnTuf16pVq4pbb731QDiORSEkItLIuGrl8M9//nNzcXHxZydOnIh78803w9JZVTcmBBjQvS1yZl7huIq6H4QYDZNdFxBDemCL6xJQ7LoAiTnBtnJ4/fXXPx80aNDxOXPmdFy1alUbwN/KYcWKFa2WLl3aNiMjo39ubu768ePH7x8yZMjRxYsXtx05cmTfJ554ouiqq646XNN+W7ZsaaNGjfp68eLF7a6++uoGN7bTTEhEpJGJdiuHgwcPxn3xxRdNAX9ju2XLlrXt16/f8XAci2ZCIiINlHdLXp3rG3srh0OHDsVdccUV55w4cYKVlZX84Q9/eOjee+/dG45jUSuHAJmZmZaTk+O6DBGJcWrlEBq1chARkZikEBIREWcUQiIi4oxCSEREnFEIiYiIMwohERFxRt8TEhFpoA396r5dOx5ouQHBt3JI3bgh5lo5VBk2bNg5O3bsaL558+b14difZkIiIhKU559/vl2rVq2+9Zy6htBMKEDeroNImvq20xqKEsY4HR8ANrzSzXUJWHHJn1yXAAAoPTDLdQm4PnmK6xLwdMJ7rkvAkxePdl0CiodmuC7h/8ydO7fjnDlzupBEamrq8f/+7//eFrg+Ozu707PPPtvZ5/MxKSmp7PXXX9/Wpk2byvnz57d/+OGHu8XFxVmbNm0qcnJyCnNychJuvfXWZJ/Px8rKSixatGjLgAEDygL3d/Dgwbg5c+Z0mTdv3hc33HBD73Adh0JIRKSRqWrlsHr16o2JiYnlNT07Lisr68DkyZP3AcDEiRO7zZkzp9P06dP3VLVySE5O9u3bty8eONnK4Y477thfWlrKqqdrB5o0aVL3u+++u6R169aV4TwWnY4TEWlkot3K4cMPP2yxbdu25jfffPPX4T4WhZCISCMTbCuHuXPnbt+0aVPBlClTdpeVlcUB/lYODz300O4dO3Y0y8jI6F9cXBw/fvz4/UuWLPm8RYsWlSNHjuy7dOnSNoH7+uCDD1rn5+e37N69+4CLLrqoX1FRUfOBAwemhONYFEIiIo1MtFs5TJkyZe+ePXvW7dq1K+8f//jHxqSkpLKPPvqoMBzHomtCIiINlLqx7gdkN/ZWDpGkVg4Bmif2scRbZjutQXfH+enuuJN0d5xfLN0dp1YOoVErBxERiUkKIRERcUYhJCIizgQVQiSvI9nGe30/yTdIfi+ypYmIyKku2JnQf5jZYZKDAVwG4HkAf45cWSIicjoINoSqvo17BYA/m9kSAM0iU5KIiJwugv2e0C6STwG4FMAfSDaHrieJiAAA/jR+RX2btFyFFUG3cvjlk8NirpXDwIEDU/bs2dM0ISGhEgDee++9Td27d//2Q+ZCFGwI/RTA5QAeM7OvSSYCuLehg4uISOOxYMGCrRdddNGxcO4z2NnMU2b2hpltBgAz+xLATeEsREREgjd37tyOffv2TUtJSUkbPXp0cvX12dnZndLT01NTUlLSLrvsst6HDx+OA4D58+e379OnT/+UlJS0zMzMFMD/VO4BAwak9uvXL61v375peXl5zaN1HMGGUP/ANyTjEUKXQBERCZ+qVg6rVq3aVFhYWPDUU09tr75NVlbWgfz8/A2FhYUFKSkpx+fMmdMJAKpaORQWFhYsW7bsc+BkK4eNGzcWrFu3bkNycvKJmsa9/fbbk/r165d27733JlZWhqejQ50hRPI+kocBnEvykPdzGMAeAEvCUoGIiIQk2q0cAODVV1/dumnTpoLVq1dv/PDDD1s/8cQTHcNxLHWGkJk9bGZtADxqZmd4P23MrKOZ3ReOAkREJDTRbuUAAMnJyT4AaN++feX111+//6OPPmoVjmMJ6nScmd1HsjvJfyN5UdVPOAoQEZHQRLuVg8/nw5dfftkEAMrKyvjOO++0TU9PPx6OYwnq7jiSMwHcAKAAJ78zZAD+EY4iREQas18+OazO9Y29lcPx48fjLr300j4+n4+VlZUcMmTIoUmTJu0Nx7EE1cqBZCGAc82sLOgdk10BzAZwIYAyAEUAfgXgBIC3zCydZCaAm81sYsiV+8eYZma/r2XdMgCJ8AftBwB+aWbfOm8aSK0c/NTK4SS1cvBTKwc/tXL4bsLRymErgKbBDkiSABYDWGlmvc0sDcA0AF0CtzOznO8aQJ5pdaz7qZmdByAdQGcA1zVgHBERiYBgv6x6DEAuyffgn9UAAOoIkKEAfGb2ZMC2uQBAMqlqGclLANxjZqNItgLwOIABXl0zzGwJybEArgLQEkBvAIvN7DfeKcIWJHMBrDezrMACzOxQwDE2g//0oYiIxJBgQ2ip9xOsdAChPnZiOoAVZnYbyXYAPiL5d29dBoDz4Q/AQpKPm9lUkneZWUZtOyS5HMBAAP8D4PUQ6xERkQgLKoTM7HmSLQD0NLPCCNUyAsBVJO/x3icA6Om9fs/MDgIAyQIAvQDsqG+HZnYZyQQALwIYBuDd6tuQHAdgHADEn9G5occgIiIhCLaf0JUAcgEs895nkKxrZrQeoT9RgQCuMbMM76enmVVdyAu8IaICwc/gYGal8M/iflzL+nlmlmlmmfEt24ZYsoiINESwNybMgP+01tfA/13f+dazigKsANCc5M+rFpC8kOTFdXxmOYAJ3k0NIHl+EHX5SH7rhgmSrb2HrIJkEwD/H4CNQexPRESiKNgZRbmZHfTyoUqtF/rNzEheDWA2yakASnHyFu3aPAj/Ld3rvCAqAjCqnrrmedt/Uu3GhFYAlnotJ+LhD8Una9qBiEhDZV9f319VaLk8hLNDk199K+ZaOZSWlvLWW2/tuXr16jYk7be//e2usWPHft3Q/QYbQvkkxwCIJ9kHwEQAH9b1ATPbDX8LiJqke9usBLDSe30cwC9q2M9zAJ4LeD8q4PUUAN/6EoWZlcD//SQREQmD++67L7Fz586+oqKi/IqKCuzZsyfoyyJ1CfZ03AT4n6RdBuBlAIdQ96xGREQiKNqtHF5++eVODz30UDEAxMfHo+rhqQ0V7LPjjpnZdDO70LuIP9274C8iIlEW7VYO+/btiwf8p/vS0tJSR44cefaOHTsiPxMiOdv79U2SS6v/hKMAEREJTbRbOfh8PpaUlDQdPHjwkYKCgg3f//73j06YMOGscBxLfTOhF7xfHwOQXcOPiIhEWbRbOXTp0qU8ISGh8qabbvoaAG688cb9+fn5LcNxLPX1E1rr/bqqpp9wFCAiIqGJdiuHuLg4DB8+/ODbb7/dBgDeeeedM/r06RP5Vg4k81D3rdjnhqMIEZHGbPKrb9W5vrG3cgCAWbNm7RwzZkzyPffcE9+xY8fyBQsWFFXf5ruos5WDdzt2F3z7ETm9AOw2s8/DUUSsUCsHP7VyOEmtHPzUysFPrRy+m4a0cvgjgENm9kXgD/xP1f5jmOsUEZHTTH0hlGRm66ovNLMcAEkRqUhERE4b9YVQQh3rWtSxTkREpF71hdDHgQ8hrULy3xF6vyAREZFvqO8br78CsJhkFk6GTib8nUqvjmBdIiJyGqgzhLwHgf4byaHwHjoK4G0zWxHxykRE5JQXbGfV9wG8H+FaREQapZ1TP6hzfTug5U58EHQrhx4zh8RUK4cDBw7EDRo0qF/V+5KSkqZXX331/vnz59fb4bo+YXkAnYiInLrat29fuXHjxoKq9/3790+97rrrDoRj38G2chARkRgS7VYOVfLy8pp/9dVXTS+77LIj4TgOzYRERBqZqlYOq1ev3piYmFhe07PjsrKyDkyePHkfAEycOLHbnDlzOk2fPn1PVSuH5ORkX1WLhqpWDnfcccf+0tJSVj1duybPP/98h6uuump/XFx45jCaCYmINDLRbuUQaPHixR1uuumm/eE6FoWQiEgjE+1WDlVWr17doqKigkOGDDkWrmPR6bgAA7q3Rc7MKxxXcdDx+EDqDNcVAME/GjLShrkuICbMwBDXJWCG6wJiyOWXX37o2muvPWfatGklXbt2rSgpKYmvPhuq3sohMTHRB5xs5TBs2LCjy5cvb7d169Zm+/fvr0hNTS3r37//nq1btzbPzc1tcdVVVx2uPu4LL7zQ4eqrrw7bLAhQCImINFiPmXWH9KnQygEAli5d2uHNN9/cHK7jAOpp5XC6yczMtJycHNdliEiMUyuH0DSklYOIiEjEKIRERMQZhZCIiDijEBIREWcUQiIi4oxCSEREnNH3hEREGmjGjBn1bdLy9ddfD7qVw4wZM2KqlQMAPPXUUx2ys7O7AkCXLl18r7322raqxwY1hGZCIiJSJ5/Ph/vuu++sVatWbdq0aVNB//79jz/66KNnhmPfCiERkUYomq0cKisraWY4fPhwXGVlJQ4dOhTXrVu3E+E4Dp2OExFpZKLdyqF58+Y2a9as7d/73vf6t2jRoqJXr15lCxYs2B6OY1EIBcjbdRBJU992WkNRwhin4wPAhle6uS4BKy75k+sSAAA/btfUdQkoHDHWdQkYPmyL6xIkQLCtHB544IHuhw8fjj969Gj8xRdffBA42crhmmuuOZCVlXUA8LdyeOyxxxJ37tzZ7IYbbjgwYMCAssB9lZWVcd68eZ3XrFlTkJqaWjZ27Nie06ZNS3zkkUe+bOix6HSciEgjE+1WDv/6179aAED//v3L4uLi8LOf/Wz/mjVrWoXjWBRCIiKNzOWXX35o6dKlHYqLi+MBoKbTcdVbOVQtr2rlMHv27N3t27cv37p1a7OCgoJmqampZffff/+eESNGfJ2bm9sicF+9evXyff755wm7d+9uAgDLli07o2/fvqXhOBadjhMRaaD6btFu7K0ckpKSfPfee++XgwcPTmnSpIn16NHjxEsvvbQtHMeiVg4Bmif2scRbZjutQdeE/HRN6CRdE4o9auUQGrVyEBGRmKQQEhERZxRCIiLijEJIREScUQiJiIgzCiEREXFG3xMSEWmg91b0rm+TliUrEHQrh+HDtsRcK4e//OUv7R999NHEyspKXnrppQeffPLJneHYr2ZCIiJSp+Li4vgHHnigx8qVKzd9/vnn6/fs2dNkyZIlber/ZP0UQiIijVA0WzkUFhY2T05OLuvWrVs5AAwfPvzQwoUL24fjOHQ6TkSkkYl2K4e0tLSyLVu2JBQWFjY7++yzTyxdurS9z+djOI5FMyERkUYm2FYOF1xwQUrfvn3TFi1a1HH9+vUJwMlWDtnZ2Z2qwmbQoEFHs7OzE6dPn9518+bNzVq3bv2N57l17ty54o9//OMX11133dkXXnhhv549e5bFx8eH5ZlvCiERkUYm2q0cAGDMmDEH161btzE3N3djSkpKae/evcu+PWroFEIiIo1MtFs5AMCuXbuaAMDevXvjn3766TPvvPPOveE4Fl0TEhFpoPqeMt7YWzkAwPjx488qKChoCQBTpkzZfe6554ZlJqRWDgHUysFPrRxOUisHP7Vy+Ca1cgiNWjmIiEhMilgIkexK8hWSW0gWkHyHZF+SSSTzvW0ySc5pwBjTalnekuTbJDeSXE9y5ncdQ0REIiciIUSSABYDWGlmvc0sDcA0AF0CtzOzHDOb2IChagwhz2Nm1g/A+QB+SHJkA8YREZEIiNRMaCgAn5k9WbXAzHLN7IPAjUheQvIt73UrkvNJfkzyU5I/9paPJfkGyWUkN5N8xFs+E0ALkrkkXwzcr5kdM7P3vdcnAHwCoEeEjlVERL6jSIVQOoBQH8A3HcAKM7sQ/hB7lGQrb10GgOsBDABwPcmzzGwqgONmlmFmWbXtlGQ7AFcCeK+W9eNI5pDMqTh2MMSSRUSkIWLpxoQRAKaSzAWwEkACgJ7euvfM7KCZlQIoANArmB2SbALgZQBzzGxrTduY2TwzyzSzzPiWbRt4CCIiEopIfU9oPYBrQ/wMAVxjZoXfWEh+H0Dg/egVCL7ueQA2m9nsEGsREQla1/dz69ukJd7PDbqVQ/HQjJhr5TBhwoTuCxcu7Hjo0KH4Y8eOfVq1/Pjx47z22muT8/LyWrZr16584cKFW1NSUk4Eu99IzYRWAGhO8udVC0heSPLiOj6zHMAE76YGkDw/iHF8JGv8IgfJhwC0BfCroKsWEZEajR49+us1a9Z867tO//Vf/9Wpbdu25du3b8+/6667SiZNmhTS9feIhJD5vwF7NYAfebdorwcwA8C3voUb4EEATQGs827hfjCIoeZ523/jxgSSPeC/xpQG4BPv5oXbQz8SEZHYFM1WDgAwfPjwo7169fJVX/7WW2+1u+22274CgFtvvfXAhx9+2KaysjLo44jYY3vMbDeAn9ayOt3bZiX8139gZscB/KKG/TwH4LmA96MCXk8BMKWGz+yE//SeiMgpJ9qtHOpSUlLSLDk5+QQANG3aFK1bt64oKSlpUvWE7/rE0o0JIiIShGi3cqhLTY9+q+8J34EUQiIijYyLVg616dq164lt27Y1AwCfz4cjR47En3nmmd8KxdoohEREGhkXrRxqc8UVV3w9f/78jgDw7LPPth80aNDhuLjgo0WtHEREGqh4aEad60+RVg49Fi9e3KG0tDSuS5cu52ZlZe2bNWvW7rvvvnvfNddck9yzZ8/0tm3bVrz66qshPXJdrRwCqJWDn1o5nKRWDn5q5fBNauUQGrVyEBGRmKQQEhERZxRCIiLfgS5lBKeyspIAav32qkJIRCRECQkJ+OqrrxRE9aisrOTevXvbAsivbRvdHSciEqIePXpg586d2Lt3b1DbFxcXN6moqOgU4bJiUSWA/PLy8lofm6YQEhEJUdOmTZGc/K3HtdUqLS0tz8wyI1hSo6XTcSIi4oxCSEREnFEIiYiIMwohERFxRiEkIiLOKIRERMQZhZCIiDijEBIREWfUyiFAZmam5eTkuC5DRE4xJNfqy6o100xIREScUQiJiIgzCiEREXFGISQiIs4ohERExBmFkIiIOKMQEhERZxRCIiLijEJIREScUQiJiIgzCiEREXFGISQiIs40cV1ALMnbdRBJU992WkNRwhin4wPAgOSerkvA2Hd6uS4BAHB98hTXJeDphPdcl4AhF73gugRkcZHrElA8NMN1CacczYRERMQZhZCIiDijEBIREWcUQiIi4oxCSEREnFEIiYiIMwohERFxRiEkIiLOKIRERMQZhZCIiDijEBIREWcUQiIi4oxCSEREnFEIiYiIMwohERFxRiEkIiLOKIRERMQZhZCIiDijEBIREWcUQiIi4kzEQohkV5KvkNxCsoDkOyT7kkwime9tk0lyTgPGmFbHuv+f5A6SR77r/kVEJLIiEkIkCWAxgJVm1tvM0gBMA9AlcDszyzGziQ0YqtYQAvAmgIEN2LeIiERYpGZCQwH4zOzJqgVmlmtmHwRuRPISkm95r1uRnE/yY5Kfkvyxt3wsyTdILiO5meQj3vKZAFqQzCX5YvUCzOxfZvZlhI5PRETCoEmE9psOYG2In5kOYIWZ3UayHYCPSP7dW5cB4HwAZQAKST5uZlNJ3mVmGQ0plOQ4AOMAIP6Mzg3ZlYiIhCiWbkwYAWAqyVwAKwEkAOjprXvPzA6aWSmAAgC9wjWomc0zs0wzy4xv2TZcuxURkSBEaia0HsC1IX6GAK4xs8JvLCS/D/8MqEoFIle3iIhEUaRmQisANCf586oFJC8keXEdn1kOYIJ3UwNInh/EOD6STRtWqoiIuBKREDIzA3A1gB95t2ivBzADwO46PvYggKYA1nm3cD8YxFDzvO2/dWMCyUdI7gTQkuROkjNCPAwREYmwiJ3WMrPdAH5ay+p0b5uV8F//gZkdB/CLGvbzHIDnAt6PCng9BcCUWsb/DYDffIfSRUQkSmLpxgQRETnNKIRERMQZhZCIiDijEBIREWcUQiIi4oxCSEREnFEIiYiIMwohERFxRiEkIiLOKIRERMQZhZCIiDijEBIREWcUQiIi4oxCSEREnFEIiYiIMwohERFxRiEkIiLOKIRERMQZhZCIiDhDM3NdQ8zIzMy0nJwc12WIyCmG5Fozy3RdRyzSTEhERJxRCImIiDMKIRERcUYhJCIiziiERETEGYWQiIg4oxASERFnFEIiIuKMQkhERJxRCImIiDMKIRERcUYhJCIiziiERETEGYWQiIg4o1YOAUgeBlDouIxOAPaphpioAYiNOlRD46+hl5l1Dncxp4ImrguIMYWue36QzFENsVFDrNShGlTDqUyn40RExBmFkIiIOKMQ+qZ5rguAaqgSCzUAsVGHavBTDacg3ZggIiLOaCYkIiLOKIQAkLycZCHJz0lOdVTDfJJ7SOa7GN+r4SyS75PcQHI9ybsd1JBA8iOSn3k1/Ge0awioJZ7kpyTfcjR+Eck8krkkc1zU4NXRjuTrJDd6fzYGRXn8FO/3oOrnEMlfRbMGr45fe38m80m+TDIh2jWcik7703Ek4wFsAvAjADsBfAzgZ2ZWEOU6LgJwBMACM0uP5tgBNSQCSDSzT0i2AbAWwOho/l6QJIBWZnaEZFMA/wRwt5n9K1o1BNQyCUAmgDPMbJSD8YsAZJqZ0+/GkHwewAdm9jTJZgBamtnXjmqJB7ALwPfN7Isojtsd/j+LaWZ2nORrAN4xs+eiVcOpSjMhYCCAz81sq5mdAPAKgB9Huwgz+weA/dEet1oNX5rZJ97rwwA2AOge5RrMzI54b5t6P1H/lxLJHgCuAPB0tMeOJSTPAHARgGcAwMxOuAogz3AAW6IZQAGaAGhBsgmAlgB2O6jhlKMQ8v8luyPg/U5E+S/eWEQyCcD5ANY4GDueZC6APQDeNbOo1wBgNoDfAKh0MHYVA/A3kmtJjnNUw9kA9gJ41js1+TTJVo5qAYAbALwc7UHNbBeAxwBsB/AlgINm9rdo13EqUggBrGHZaX2OkmRrAIsA/MrMDkV7fDOrMLMMAD0ADCQZ1dOTJEcB2GNma6M5bg1+aGbfAzASwC+9U7bR1gTA9wD82czOB3AUgKvrps0AXAVgoYOx28N/hiQZQDcArUjeGO06TkUKIf/M56yA9z1wGk+zveswiwC8aGZvuKzFO+2zEsDlUR76hwCu8q7JvAJgGMm/RrkGmNlu79c9ABbDf+o42nYC2BkwG30d/lByYSSAT8ysxMHYlwLYZmZ7zcwH4A0A/+agjlOOQsh/I0Ifksnev7RuALDUcU1OeDcFPANgg5nNclRDZ5LtvNct4P+ff2M0azCz+8ysh5klwf/nYYWZRfVfvSRbeTeHwDv9NQJA1O+cNLNiADtIpniLhgOI6k07AX4GB6fiPNsB/IBkS+//k+HwXzOVBjrtH2BqZuUk7wKwHEA8gPlmtj7adZB8GcAlADqR3Angt2b2TJTL+CGAmwDkeddkAGCamb0TxRoSATzv3QUVB+A1M3Nyi7RjXQAs9v99hyYAXjKzZY5qmQDgRe8faVsB3BrtAki2hP8O1l9Ee2wAMLM1JF8H8AmAcgCfQk9PCIvT/hZtERFxR6fjRETEGYWQiIg4oxASERFnFEIiIuKMQkhERJxRCIlzJI1kdsD7e0jOCNO+nyN5bTj2Vc8413lPmH4/luoSiXUKIYkFZQB+QrKT60ICed9VCta/A7jTzIZGqh6RU5FCSGJBOfxf/Pt19RXVZwwkj3i/XkJyFcnXSG4iOZNklteLKI9k74DdXEryA2+7Ud7n40k+SvJjkutI/iJgv++TfAlAXg31/Mzbfz7JP3jLHgAwGMCTJB+t4TO/8T7zGcmZNax/wKsjn+Q87xv5IDmRZIFX3yvesot5sq/OpwFPVbg34Fj+01vWiuTb3rj5JK8P7j+HSPSc9k9MkJjxJwDrSD4SwmfOA5AKfwuMrQCeNrOB9DfjmwDgV952SQAuBtAbwPskzwFwM/xPQr6QZHMA/0uy6qnIAwGkm9m2wMFIdgPwBwAXADgA/xOuR5vZ70gOA3CPmeVU+8xIAKPh739zjGSHGo5jrpn9ztv+BQCjALwJ/4NCk82srOpRRgDuAfBLM/tf70GzpSRHAOjj1U0AS72HnXYGsNvMrvD23Ta431aR6NFMSGKC97TuBQAmhvCxj70eSGUAtgCoCpE8+IOnymtmVmlmm+EPq37wP4vtZu/xRGsAdIT/L3IA+Kh6AHkuBLDSe4hlOYAX4e+1U5dLATxrZse846ypZ9RQkmtI5gEYBqC/t3wd/I/LuRH+2SIA/C+AWSQnAmjn1THC+/kU/sfK9POOJQ/+WeAfSA4xs4P11CoSdQohiSWz4b+2Etivphzen1PvNFWzgHVlAa8rA95X4puz/OrPpjL4ZwwTzCzD+0kO6A9ztJb6amr7UR/WMP7Jlf4W0U8AuNbMBgD4C4CqttFXwD9DvADAWpJNzGwmgNsBtADwL5L9vDEeDjiWc8zsGTPb5H02D8DD3mlDkZiiEJKY4c0SXoM/iKoUwf8XKeDv59L0O+z6OpJx3nWiswEUwv/A2jvob10Bkn1Zf7O2NQAuJtnJu2nhZwBW1fOZvwG4zXsAJ2o4HVcVOPu802vXetvFATjLzN6Hv7leOwCtSfY2szwz+wOAHPhnPcu9MVp7n+1O8kzv9OExM/sr/A3ZXLVgEKmVrglJrMkGcFfA+78AWELyIwDvofZZSl0K4Q+LLgDGm1kpyafhP2X3iTfD2gv/tZtamdmXJO8D8D78s493zGxJPZ9ZRjIDQA7JEwDeATAtYP3XJP8C/2ylCP7WIoD/ie5/9a7jEMAfvW0fJDkUQAX8LRX+x7tmlApgtXdPwxEANwI4B8CjJCsB+ADcUe/vlEiU6SnaIiLijE7HiYiIMwohERFxRiEkIiLOKIRERMQZhZCIiDijEBIREWcUQiIi4oxCSEREnPl/22ehs2VWzrwAAAAASUVORK5CYII=", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaEAAAGDCAYAAACCzK//AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAAvsElEQVR4nO3de3xU9Z0+8OdJuISb3IUAQoKFkBAUNdCyBS9gUSqo1GtNvbalaIW2oAXBddnqVlaFukhdpBYRfrUqUha0LrQFQbtSMGiEEAj3m5gAgtwThuTz++OcqWNIJjPM5czA83695sXkXL7ncyYJT86ZM+dDM4OIiIgXUrwuQEREzl8KIRER8YxCSEREPKMQEhERzyiERETEMwohERHxjEJIzikkZ5D81yiN1ZnkMZKp7tfLSf4oGmO74/0vyXujNZ5IMqrndQEi4SC5A0A7AKcBVAIoBjAHwEwzqzKzkWGM8yMz+1tty5jZLgBNI63Z3d4kAN8wsx8EjD8kGmOLJDMdCUkyGmZmzQB0ATAZwDgAv4/mBkjqDzSROFAISdIys8NmtgjAHQDuJZlLcjbJpwCAZBuS75D8kuRBkh+QTCE5F0BnAG+7p9t+STKDpJH8IcldAJYFTAsMpItJriZ5mORCkq3cbV1Nck9gfSR3kLyW5PUAJgC4w93ep+78f57ec+t6nOROkvtIziHZ3J3nr+NekrtIHiA5Mbavrkh8KIQk6ZnZagB7AAyoNmusO70tnFN4E5zF7W4Au+AcUTU1s2cC1rkKQDaA62rZ3D0AHgDQAc4pwWkh1LcYwK8BvOFu79IaFrvPfVwDoCuc04DTqy3TH0AWgEEAniCZXde2RRKdQkjOFXsBtKo2zQcgHUAXM/OZ2QdW980SJ5nZcTM7Wcv8uWZWZGbHAfwrgNv9Fy5EKB/AVDPbZmbHADwG4M5qR2H/bmYnzexTAJ8CqCnMRJKKQkjOFR0BHKw27VkAWwD8heQ2kuNDGGd3GPN3AqgPoE3IVdaugzte4Nj14BzB+ZUGPD+BKF00IeIlhZAkPZJ94ITQ3wOnm9lRMxtrZl0BDAMwhuQg/+xahqvrSOmigOed4RxtHQBwHEDjgJpS4ZwGDHXcvXAutAgc+zSAsjrWE0lqCiFJWiQvIDkUwOsA/p+Zras2fyjJb5AkgCNwLumudGeXwXnvJVw/IJlDsjGAXwF4y8wqAWwCkEbyBpL1ATwOoGHAemUAMkjW9jv3RwC/IJlJsim+eg/p9FnUKJI0FEKSjN4meRTOqbGJAKYCuL+G5boB+BuAYwBWAnjRzJa7854G8Lh75dwjYWx7LoDZcE6NpQEYDThX6gF4CMDLAD6Dc2QUeLXcPPffL0h+XMO4s9yx3wewHUA5gFFh1CWSlKimdiIi4hUdCYmIiGcUQiIi4hmFkIiIeEYhJCIinlEIiYiIZ3Sn4ABt2rSxjIwMr8sQkXPMmjVrDphZ27qXPP8ohAJkZGSgoKDA6zJE5BxDcmfdS52fdDpOREQ8oxASERHPKIRERMQzCiEREfGMQkhERDyjEBIREc8ohERExDMKIRER8YxCSEREPKMQEhERzyiERETEMwohERHxjG5gGmjvJ8Ck5hEPM2XDgCgUc267I3NcROu/nLY04hoGXDn3rNcdNHBrxNsXER0JiYiIhxRCIiLiGYWQiIh4RiEkIiKeUQiJiIhnFEIiIuIZhZCIiHhGISQiIp5RCImIiGcUQiIi4hmFkIiIeEYhJCIinlEIiYiIZxRCIiLiGYWQiIh4RiEkIiKeUQiJiIhnFEIiIuIZhZCIiHhGISQiIp5RCImIiGdiFkIk25N8neRWksUk3yXZnWQGySJ3mTyS0yLYxoQg85aTLCFZ6D4uPNvtiIhIbNSLxaAkCWABgFfN7E53Wm8A7QDs9i9nZgUACiLY1AQAvw4yP9/dhoiIJKBYHQldA8BnZjP8E8ys0Mw+CFyI5NUk33GfNyE5i+RHJD8heZM7/T6SfyK5mORmks+40ycDaOQe5fwhRvshIiIxFJMjIQC5ANaEuc5EAMvM7AGSLQCsJvk3d15vAJcBqABQQvIFMxtP8mEz6x1kzFdIVgKYD+ApM7PqC5AcAWAEAHRuzjBLFhGRSCTShQmDAYwnWQhgOYA0AJ3deUvN7LCZlQMoBtAlhPHyzawXgAHu4+6aFjKzmWaWZ2Z5bRsrhERE4ilWIbQewBVhrkMAt5hZb/fR2cw2uPMqAparRAhHcGb2mfvvUQCvAegbZj0iIhJjsQqhZQAakvyxfwLJPiSvCrLOEgCj3IsaQPKyELbjI1m/+kSS9Ui2cZ/XBzAUQFE4OyAiIrEXkxBy33sZDuA77iXa6wFMArA3yGpPAqgPYK17CfeTIWxqprt89QsTGgJYQnItgEIAnwH4XVg7ISIiMRerCxNgZnsB3F7L7Fx3meVw3v+BmZ0E8JMaxpkNYHbA10MDno8DMK6GdY4j/NOBIiISZ4l0YYKIiJxnFEIiIuIZhZCIiHhGISQiIp5RCImIiGcUQiIi4hmFkIiIeEYhJCIinlEIiYiIZxRCIiLiGYWQiIh4RiEkIiKeUQiJiIhnFEIiIuIZhZCIiHhGISQiIp5RCImIiGcUQiIi4hmFkIiIeIZm5nUNCSMvL88KCgq8LkNEzjEk15hZntd1JCIdCYmIiGcUQiIi4hmFkIiIeEYhJCIinlEIiYiIZxRCIiLiGYWQiIh4RiEkIiKeUQiJiIhnFEIiIuIZhZCIiHimntcFJJS9nwCTmkc8zJ7ydyIe4+W0pRGtP+DKuRGtn8/5Ea0vQOk1vb0uQSTh6UhIREQ8oxASERHPKIRERMQzCiEREfGMQkhERDyjEBIREc8ohERExDMKIRER8YxCSEREPKM7JoiIhMnn82HPnj0oLy8Pafm//vWvvT799NMdsa0qIVUBKDp9+vSPrrjiin01LaAQEhEJ0549e9CsWTNkZGSAZJ3LV1ZWns7NzT0Qh9ISSlVVFffv359TWlr6MoAba1pGp+NERMJUXl6O1q1bhxRA57OUlBRr27btYQC5tS4Tx3pERM4ZCqDQpKSkGIJkjUJIREQ8o/eEREQilDH+z3Ut0hjYeUWo4+2YfMOas6ljzJgxHZo2bVr5q1/9quxs1g9m1KhRHefNm9f6yJEjqSdOnPgkWuPqSEhEROp08803f7lq1aoN0R5XISQikoSmT5/eunv37jlZWVk5N998c2b1+VOmTGmTm5ubnZWVlXPdddddfPTo0RQAmDVrVstu3br1zMrKysnLy8sCgIKCgrRevXpl9+jRI6d79+4569ata1h9vEGDBh3v0qWLL9r7odNxIiJJpqCgIO25555LX7ly5cb09PTTZWVlqdWXyc/PPzR27NgDADB69OgO06ZNazNx4sR9kydPTv/LX/6yKTMz03fgwIFUAHjhhRfaPvTQQ2UPPvjgwfLycp4+fTpu+6IjIRGRJLNkyZILhg0bdig9Pf00ALRr166y+jJr1qxpdMUVV2R17949Z/78+a3Xr1+fBgB5eXnH8vPzM6ZMmdLGHzb9+vU7PmXKlPSJEye237x5c4OmTZtavPZFISQikmTMDCSDBsWIESMyp0+fvmvTpk3F48aN21tRUZECAK+99tqup556au/u3bsb9O7du2dpaWnqyJEjDy5cuHBLo0aNqoYMGdJ90aJFzeKzJwohEZGkc/311x9ZtGhRq9LS0lQAqOl03IkTJ1I6d+7sq6io4Ouvv97KP339+vUNBw4cePz555/f27Jly9Pbtm1rUFxc3CA7O7vi8ccf3zd48OAvCwsLG8VrX2L2nhDJ9gCeB9AHQAWAHQB+DuAUgHfMLJdkHoB7zGz0WW5jgpn9uo5lFgHoama1fmJXRCQSOybfEHR+UVHRidzc3KhdWZaXl1c+duzYzwcMGNAjJSXFcnNzT8yfP39H4DLjx4/f27dv3+yOHTueys7OPnHs2LFUAPjFL37RaceOHQ3NjP379z/yrW996+TEiRPbz5s3r3W9evWsbdu2vqeffnpv9W2OHDmy04IFC1qVl5entGvX7pL8/PwDU6dOPWO5cNEs+qf+6HyU+EMAr5rZDHdabwDNAOyGG0JR2M4xM2saZP73ANwK4JJQtpfXIdUKRtQ6XMj2lL8T8Rgvpy2NaP0BV86NaP18zo9ofQFKr+ntdQkSIxs2bEB2dnbIy0c7hJLNp59+2ubSSy/NqGlerE7HXQPA5w8gADCzQjP7IHAhkleTfMd93oTkLJIfkfyE5E3u9PtI/onkYpKbST7jTp8MoBHJQpJ/qF4AyaYAxgB4Kkb7KCIiEYrV6bhcAOF+4ncigGVm9gDJFgBWk/ybO683gMvgnNYrIfmCmY0n+bCZ9a5lvCcBTAFwIthGSY4AMAIAOjfXvaBEROIpkS5MGAxgPMlCAMsBpAHo7M5bamaHzawcQDGALsEGck/9fcPMFtS1UTObaWZ5ZpbXtrFCSEQknmJ1JLQeznsx4SCAW8ys5GsTyW/COQLyq0TddfcDcAXJHe6yF5JcbmZXh1mTiIjEUKyOhJYBaEjyx/4JJPuQvCrIOksAjHIvagDJy0LYjo9k/eoTzey/zayDmWUA6A9gkwJIRCTxxCSEzLnkbjiA75DcSnI9gEkAgl3O9ySA+gDWkixyv67LTHf5My5MEBGRxBezzwmZ2V4At9cyO9ddZjmc939gZicB/KSGcWYDmB3w9dCA5+MAjKujjh0I0tVPRCRik5oHnZ0LNMZbCLmVAyYdTqhWDkePHk0ZNmxY1507dzZMTU3F4MGDv3zxxRc/i8bYiXRhgoiIJKixY8eWbd++fX1RUVHxqlWrmr755psXRGNchZCISBKKZyuHZs2aVQ0bNuwoAKSlpdkll1xyYvfu3Q2isR8KIRGRJONv5bBixYpNJSUlxS+99NKu6svk5+cfKioq2lBSUlKclZV1ctq0aW0AwN/KoaSkpHjx4sVbgK9aOWzcuLF47dq1GzIzM0/Vtu0DBw6k/vWvf20xZMiQI9HYF4WQiEiS8aqVg8/nw/e+972uI0aMKMvJyak1qMKhEBIRSTJetXK46667Mrp27Vr+xBNP7IvWviiERESSjBetHEaPHt3hyJEjqb///e93R3Nf1N5bRCRSkw4HnZ3srRy2bt1a/4UXXkjPzMws79mzZw4AjBgxYt+YMWMORLovMWnlkKzUyuErauUQObVyOHeplUN4vGjlICIiUieFkIiIeEYhJCIinlEIiYiIZxRCIiLiGYWQiIh4Rp8TEhGJUK9Xe9W1SGOsCb2Vw7p71yVUKwcAGDBgQLd9+/bVr6ysZN++fY/OmTNnV716kUeIjoRERKROCxcu3FpSUlK8adOm9V988UX9WbNmtYzGuAohEZEkFM9WDgDQqlWrKgDw+Xz0+XwkGZX9UAiJiCQZr1o59O/fv1vbtm0vbdKkSeX9999/KBr7ohASEUkyXrVy+Pvf/765tLT001OnTqW8/fbbUemsqgsTAnW4DJhUEPEwnaJQyiQMiHiESJRGuHURiZ1QWzm89dZbW/r163dy2rRprVesWNEMcFo5LFu2rMmiRYua9+7du2dhYeH6kSNHHhwwYMDxBQsWNB8yZEj3F198cceNN954tKZxGzdubEOHDv1ywYIFLYYPHx5xYzsdCYmIJJl4t3I4fPhwys6dO+sDTmO7xYsXN+/Ro8fJaOyLjoRERCK07t51QecneyuHI0eOpNxwww3fOHXqFKuqqvjtb3/7yKOPPro/GvuiVg4B8vLyrKAg8tNxInJuUyuH8KiVg4iIJCSFkIiIeEYhJCIinlEIiYiIZxRCIiLiGYWQiIh4Rp8TEhGJ0IYewS/XTgUab0DorRyyN25IuFYOfgMHDvzG7t27G27evHl9NMbTkZCIiITk1VdfbdGkSZMz7lMXCR0JBVj32WFkjP+z12UkrR1pd0VlnA2vd4jKOAIsu/q3XpfwNTe1qB/R+i+nLY24hgFXzj3rdQcN3Brx9qNl+vTpradNm9aOJLKzs0/+z//8z/bA+VOmTGnzyiuvtPX5fMzIyKh46623tjdr1qxq1qxZLZ9++ukOKSkp1qxZs8qCgoKSgoKCtPvvvz/T5/OxqqoK8+fP39qrV6+KwPEOHz6cMm3atHYzZ87ceeedd14crf1QCImIJBl/K4eVK1duTE9PP13TvePy8/MPjR079gAAjB49usO0adPaTJw4cZ+/lUNmZqbvwIEDqcBXrRwefPDBg+Xl5fTfXTvQmDFjOv7sZz8ra9q0aVU090Wn40REkky8Wzl8+OGHjbZv397wnnvu+TLa+6IQEhFJMqG2cpg+ffquTZs2FY8bN25vRUVFCuC0cnjqqaf27t69u0Hv3r17lpaWpo4cOfLgwoULtzRq1KhqyJAh3RctWtQscKwPPvigaVFRUeOOHTv2uvLKK3vs2LGjYd++fbOisS8KIRGRJBPvVg7jxo3bv2/fvrWfffbZuvfff39jRkZGxerVq0uisS96T0hEJELZG4PfIDvZWznEklo5BGiY3s3S733e6zKSlq6OSzy6Ou5M0bg6Tq0cwqNWDiIikpAUQiIi4hmFkIiIeEYhJCIinlEIiYiIZxRCIiLimZA+J0RyrpndXdc0EZHz0W9HLqtrkcYrsCzkVg4/nTEw4Vo59O3bN2vfvn3109LSqgBg6dKlmzp27HjmTebCFOqHVXsGfkEyFWH0xhARkeQ3Z86cbVdeeeWJaI4Z9HQcycdIHgVwCckj7uMogH0AFkazEBERCd306dNbd+/ePScrKyvn5ptvzqw+f8qUKW1yc3Ozs7Kycq677rqLjx49mgIAs2bNatmtW7eeWVlZOXl5eVmAc1fuXr16Zffo0SOne/fuOevWrWsYr/0IGkJm9rSZNQPwrJld4D6amVlrM3ssTjWKiEgAfyuHFStWbCopKSl+6aWXdlVfJj8//1BRUdGGkpKS4qysrJPTpk1rAwD+Vg4lJSXFixcv3gJ81cph48aNxWvXrt2QmZl5qqbt/uhHP8ro0aNHzqOPPppeVRWdjg4hXZhgZo+R7EjyX0he6X9EpQIREQlLvFs5AMAbb7yxbdOmTcUrV67c+OGHHzZ98cUXW0djX0IKIZKTAfwfgMcBPOo+HolGASIiEp54t3IAgMzMTB8AtGzZsuqOO+44uHr16ibR2JdQL9EeDiDLzL5rZsPcx43RKEBERMIT71YOPp8Pn3/+eT0AqKio4Lvvvts8Nzf3ZDT2JdSr47YBqA+goq4FRUTONz+dMTDo/GRv5XDy5MmUa6+9tpvP52NVVRUHDBhwZMyYMfujsS8htXIgOR/ApQCWIiCIzGx0kHXaA3geQB93nR0Afg7gFIB3zCyXZB6Ae4KNU0ddE8zs17XMWwwgHU7QfgDgp2Z2xnnTQGrlEBm1ckg8auVwJrVyiL9grRxCPRJa5D5CQpIAFgB41czudKf1BtAOwG7/cmZWAKAg1HFrMAFAjSEE4HYzO+LW8haA2wC8HsG2REQkykIKITN7lWQjAJ3NLJSWrtcA8JnZjIAxCgGAZIZ/GsmrATxiZkNJNgHwAoBebl2TzGwhyfsA3AigMYCLASwws1+6F0s0IlkIYL2Z5Ver+UjAPjYAoO59IiIJJtSr44YBKASw2P26N8lgR0a5AMK97cREAMvMrA+cEHvWDSYA6A3gDjgBdQfJi8xsPICTZta7egAF1L0Ezgdrj8I5GhIRkQQS6tVxkwD0BfAl8M+jmjM+oRuhwQDGu0c2ywGkAejszltqZofNrBxAMYAuoQxoZtfBeV+oIYAa3zkkOYJkAcmCyhOHI9sDEREJS6ghdNrMqv8PHez01nqEf285ArjFPbLpbWadzcz/Rl7gVXmVCP29LLjBtQjATbXMn2lmeWaWl9q4eZgli4hIJEINoSKSdwFIJdmN5AsAPgyy/DIADUn+2D+BZB+SVwVZZwmAUe6FBCB5WQh1+UiecbkNyaYk093n9QB8F8DGEMYTEZE4CvWIYhSc92wqAPwRTmA8WdvCZmYkhwN4nuR4AOX46hLt2jwJ55LutW4Q7QAwtI66ZrrLf1ztfaEmABaRbAggFU4ozqhpABGRSE25o67/qtB4SRhnh8a+8U7CtXIoLy/n/fff33nlypXNSNq//du/fXbfffd9Gem4oV4ddwJOCE0MdWAz2wvg9lpm57rLLIfz/g/M7CSAn9QwzmwAswO+HhrwfByAcTWsUwbn80kiIhIFjz32WHrbtm19O3bsKKqsrMS+fftCflskmLpaOTzv/vs2yUXVH9EoQEREwhfvVg5//OMf2zz11FOlAJCamgr/zVMjVVeS+T9a/Fw0NiYiIpHzt3JYuXLlxvT09NM13TsuPz//0NixYw8AwOjRoztMmzatzcSJE/f5WzlkZmb6Dhw4kAp81crhwQcfPFheXk7/3bX9/MuNGTOmw4cfftisS5cuFTNnztx10UUXRRxEdfUTWuP+u6KmR6QbFxGR8MW7lYPP52NZWVn9/v37HysuLt7wzW9+8/ioUaMuisa+1HU6bh3JtbU9olGAiIiEJ96tHNq1a3c6LS2t6u677/4SAH7wgx8cLCoqahyNfanrEu3vAXgIwLBqj4fdeSIiEmfxbuWQkpKCQYMGHf7zn//cDADefffdC7p16xaXVg6/ATDBzHYGTiTZ1p03LBpFiIgks7FvvBN0frK3cgCAqVOn7rnrrrsyH3nkkdTWrVufnjNnzo7qy5yNoK0cSBaZWW4t89aZWa9oFJEo1MohMmrlkHjUyuFMauUQf8FaOdR1Oi4tyLxGQeaJiIjUqa4Q+ijw1jt+JH+I8O+SLSIi8jV1vSf0cwALSObjq9DJg9OfZ3gM6xIRkfNA0BByb3/zLySvgXurHQB/NrNlMa9MRETOeaHeO+49AO/FuBYRETnPhNrKQUREJOqichdUEZHz2Z7xHwSd3wJovAcfhNzKodPkAQnVyuHQoUMp/fr16+H/uqysrP7w4cMPzpo1a3ekYyuEREQkqJYtW1Zt3Lix2P91z549s2+77bZD0Rhbp+NERJJQvFs5+K1bt67hF198Uf+66647Fo390JGQiEiSiXcrh0CvvvpqqxtvvPFgSkp0jmF0JCQikmTi3coh0IIFC1rdfffdB6O1LwohEZEkE+9WDn4rV65sVFlZyQEDBpyI1r7odFyAXh2bo2DyDV6XkcQOR2WU7ElRGUYAhH6LzeQwCQOiMkqyu/7664/ceuut35gwYUJZ+/btK8vKylKrHw1Vb+WQnp7uA75q5TBw4MDjS5YsabFt27YGBw8erMzOzq7o2bPnvm3btjUsLCxsdOONNx6tvt25c+e2Gj58eNSOggCFkIhIxDpNDh6O50IrBwBYtGhRq7fffntztPYDqKOVw/kmLy/PCgoKvC5DRBKcWjmEJ5JWDiIiIjGjEBIREc8ohERExDMKIRER8YxCSEREPKMQEhERz+hzQiIiEZo0aVJdizR+6623Qm7lMGnSpIRq5QAAL730UqspU6a0B4B27dr53nzzze3+2wZFQkdCIiISlM/nw2OPPXbRihUrNm3atKm4Z8+eJ5999tkLozG2QkhEJAnFs5VDVVUVzQxHjx5NqaqqwpEjR1I6dOhwKhr7odNxIiJJJt6tHBo2bGhTp07ddfnll/ds1KhRZZcuXSrmzJmzKxr7ohAKtPcTYFLziIfpldk5CsWc3/53w4sRrf9y2tKIaxhw5dyI1h80cGvENYjUJNRWDk888UTHo0ePph4/fjz1qquuOgx81crhlltuOZSfn38IcFo5PPfcc+l79uxpcOeddx7q1atXReBYFRUVnDlzZttVq1YVZ2dnV9x3332dJ0yYkP7MM898Hum+6HSciEiSiXcrh3/84x+NAKBnz54VKSkp+P73v39w1apVTaKxLwohEZEkc/311x9ZtGhRq9LS0lQAqOl0XPVWDv7p/lYOzz///N6WLVue3rZtW4Pi4uIG2dnZFY8//vi+wYMHf1lYWNgocKwuXbr4tmzZkrZ37956ALB48eILunfvXh6NfdHpOBGRCNV1iXayt3LIyMjwPfroo5/3798/q169etapU6dTr7322vZo7ItaOQTI65BqBSOaRjyO3hOKnN4TkkSmVg7hUSsHERFJSAohERHxjEJIREQ8oxASERHPKIRERMQzCiEREfGMPickIhKhpcsurmuRxmXLEHIrh0EDtyZcK4ff/e53LZ999tn0qqoqXnvttYdnzJixJxrj6khIRESCKi0tTX3iiSc6LV++fNOWLVvW79u3r97ChQub1b1m3RRCIiJJKJ6tHEpKShpmZmZWdOjQ4TQADBo06Mi8efNaRmM/dDpORCTJxLuVQ05OTsXWrVvTSkpKGnTt2vXUokWLWvp8PkZjX3QkJCKSZEJt5XDFFVdkde/ePWf+/Pmt169fnwZ81cphypQpbfxh069fv+NTpkxJnzhxYvvNmzc3aNq06dfu59a2bdvK3/zmNztvu+22rn369OnRuXPnitTU1Kjc800hJCKSZOLdygEA7rrrrsNr167dWFhYuDErK6v84osvrjhzq+FTCImIJJl4t3IAgM8++6weAOzfvz/15ZdfvvChhx7aH4190XtCIiIRquuO6cneygEARo4ceVFxcXFjABg3btzeSy65JCpHQmrlEECtHBKHWjlIIlMrh/ColYOIiCSkmIUQyfYkXye5lWQxyXdJdieZQbLIXSaP5LQItjGhlumNSf6Z5EaS60lOPtttiIhI7MQkhEgSwAIAy83sYjPLATABQLvA5cyswMxGR7CpGkPI9ZyZ9QBwGYBvkxwSwXZERCQGYnUkdA0An5nN8E8ws0Iz+yBwIZJXk3zHfd6E5CySH5H8hORN7vT7SP6J5GKSm0k+406fDKARyUKSfwgc18xOmNl77vNTAD4G0ClG+yoiImcpViGUCyDcG/BNBLDMzPrACbFnSTZx5/UGcAeAXgDuIHmRmY0HcNLMeptZfm2DkmwBYBiAGt+pJjmCZAHJgv0ndJGGiEg8JdKFCYMBjCdZCGA5gDQA/svMlprZYTMrB1AMoEsoA5KsB+CPAKaZ2baaljGzmWaWZ2Z5bRtH5S4UIiISolh9Tmg9gFvDXIcAbjGzkq9NJL8JIPB69EqEXvdMAJvN7PkwaxERCVn79wrrWqQx3isMuZVD6TW9E66Vw6hRozrOmzev9ZEjR1JPnDjxiX/6yZMneeutt2auW7eucYsWLU7PmzdvW1ZW1qlQx43VkdAyAA1J/tg/gWQfklcFWWcJgFHuRQ0geVkI2/GRrF/TDJJPAWgO4OchVy0iIjW6+eabv1y1atUZn3X6r//6rzbNmzc/vWvXrqKHH364bMyYMWG9/x6TEDLnE7DDAXzHvUR7PYBJAM74FG6AJwHUB7DWvYT7yRA2NdNd/msXJpDsBOc9phwAH7sXL/wo/D0REUlM8WzlAACDBg063qVLF1/16e+8806LBx544AsAuP/++w99+OGHzaqqqkLej5jdtsfM9gK4vZbZue4yy+G8/wMzOwngJzWMMxvA7ICvhwY8HwdgXA3r7IFzek9E5JwT71YOwZSVlTXIzMw8BQD169dH06ZNK8vKyur57/Bdl0S6MEFEREIQ71YOwdR067e67vAdSCEkIpJkvGjlUJv27duf2r59ewMA8Pl8OHbsWOqFF154RijWRiEkIpJkvGjlUJsbbrjhy1mzZrUGgFdeeaVlv379jqakhB4tauUgIhKh0mt6B51/jrRy6LRgwYJW5eXlKe3atbskPz//wNSpU/f+7Gc/O3DLLbdkdu7cObd58+aVb7zxRli3j1crhwBq5ZA41MpBEplaOYRHrRxERCQhKYRERMQzCiERkbOgtzJCU1VVRQC1fnpVISQiEqa0tDR88cUXCqI6VFVVcf/+/c0BFNW2jK6OExEJU6dOnbBnzx7s378/pOVLS0vrVVZWtolxWYmoCkDR6dOna71tmkJIRCRM9evXR2bmGbdrq1VOTs46M8uLYUlJS6fjRETEMwohERHxjEJIREQ8oxASERHPKIRERMQzCiEREfGMQkhERDyjEBIREc+olUOAvLw8Kygo8LoMETnHkFyjD6vWTEdCIiLiGYWQiIh4RiEkIiKeUQiJiIhnFEIiIuIZhZCIiHhGISQiIp5RCImIiGcUQiIi4hmFkIiIeEYhJCIinlEIiYiIZ+p5XUBC2fsJMKl5xMP8tnRBFIqJvZta1I94jJfTlkY8xoAr5571uoMGbo14+yLiHR0JiYiIZxRCIiLiGYWQiIh4RiEkIiKeUQiJiIhnFEIiIuIZhZCIiHhGISQiIp5RCImIiGcUQiIi4hmFkIiIeEYhJCIinlEIiYiIZxRCIiLiGYWQiIh4RiEkIiKeUQiJiIhnFEIiIuIZhZCIiHhGISQiIp6JWQiRbE/ydZJbSRaTfJdkd5IZJIvcZfJITotgGxOCzPsPkrtJHjvb8UVEJLZiEkIkCWABgOVmdrGZ5QCYAKBd4HJmVmBmoyPYVK0hBOBtAH0jGFtERGIsVkdC1wDwmdkM/wQzKzSzDwIXInk1yXfc501IziL5EclPSN7kTr+P5J9ILia5meQz7vTJABqRLCT5h+oFmNk/zOzzGO2fiIhEQb0YjZsLYE2Y60wEsMzMHiDZAsBqkn9z5/UGcBmACgAlJF8ws/EkHzaz3pEUSnIEgBEA0Lk5IxlKRETClEgXJgwGMJ5kIYDlANIAdHbnLTWzw2ZWDqAYQJdobdTMZppZnpnltW2sEBIRiadYHQmtB3BrmOsQwC1mVvK1ieQ34RwB+VUidnWLiEgcxepIaBmAhiR/7J9Asg/Jq4KsswTAKPeiBpC8LITt+EjWj6xUERHxSkxCyMwMwHAA33Ev0V4PYBKAvUFWexJAfQBr3Uu4nwxhUzPd5c+4MIHkMyT3AGhMcg/JSWHuhoiIxFjMTmuZ2V4At9cyO9ddZjmc939gZicB/KSGcWYDmB3w9dCA5+MAjKtl+78E8MuzKF1EROIkkS5MEBGR84xCSEREPKMQEhERzyiERETEMwohERHxjEJIREQ8oxASERHPKIRERMQzCiEREfGMQkhERDyjEBIREc8ohERExDMKIRER8YxCSEREPKMQEhERzyiERETEMwohERHxjEJIREQ8oxASERHP0My8riFh5OXlWUFBgddliMg5huQaM8vzuo5EpCMhERHxjEJIREQ8oxASERHPKIRERMQzCiEREfGMQkhERDyjEBIREc8ohERExDMKIRER8YxCSEREPKMQEhERzyiERETEMwohERHxjEJIREQ8o1YOAUgeBVDidR0A2gA4oBoAqI7qVEdi1QCEVkcXM2sbj2KSTT2vC0gwJYnQ84Nkgdd1JEINqkN1JHoNiVRHstLpOBER8YxCSEREPKMQ+rqZXhfgSoQ6EqEGQHVUpzq+kgg1AIlTR1LShQkiIuIZHQmJiIhnFEIASF5PsoTkFpLjYzD+RSTfI7mB5HqSP3OnTyL5GclC9/HdgHUec+spIXldwPQrSK5z500jyTDq2OGuW0iywJ3WiuRfSW52/20Z4xqyAva3kOQRkj+Px2tBchbJfSSLAqZFbf9JNiT5hjt9FcmMMOp4luRGkmtJLiDZwp2eQfJkwOsyI8Z1RO37EGEdbwTUsINkYSxfD9b+Oxr3n4/zjpmd1w8AqQC2AugKoAGATwHkRHkb6QAud583A7AJQA6ASQAeqWH5HLeOhgAy3fpS3XmrAfQDQAD/C2BIGHXsANCm2rRnAIx3n48H8J+xrKGG174UQJd4vBYArgRwOYCiWOw/gIcAzHCf3wngjTDqGAygnvv8PwPqyAhcrto4sagjat+HSOqoNn8KgCdi+Xqg9t/RuP98nG8PHQkBfQFsMbNtZnYKwOsAbormBszsczP72H1+FMAGAB2DrHITgNfNrMLMtgPYAqAvyXQAF5jZSnN+kucAuDnC8m4C8Kr7/NWA8eJRwyAAW81sZx31RaUOM3sfwMEaxo/W/geO9RaAQTUdndVUh5n9xcxOu1/+A0CnYPsSqzqCiOvrEbCfBHA7gD8GKy7SOoL8jsb95+N8oxByftB2B3y9B8EDIiLuIfhlAFa5kx52T8HMCjjUr62mju7zs63VAPyF5BqSI9xp7czsc8D5RQRwYYxrCHQnvv6fSzxfC79o7v8/13ED5TCA1mdR0wNw/oL2yyT5CckVJAcEbCtWdUTr+xCN12MAgDIz2xwwLaavR7Xf0UT8+TinKIScQ+bqYnLJIMmmAOYD+LmZHQHw3wAuBtAbwOdwTjsEqynSWr9tZpcDGALgpySvDFZujGpwBicbALgRwDx3UrxfizpLPIvtRlwTyYkATgP4gzvpcwCdzewyAGMAvEbyghjWEc3vQzS+R9/H1/9QienrUcPvaG28ej3OOQoh5y+ViwK+7gRgb7Q3QrI+nB/uP5jZnwDAzMrMrNLMqgD8Ds6pwWA17cHXT9OEVauZ7XX/3Qdggbu9MvcUgv+Uxr5Y1hBgCICPzazMrSmur0WAaO7/P9chWQ9Ac4R+ugsk7wUwFEC+eyoH7umeL9zna+C899A9VnVE+fsQ6etRD8D3ALwRUF/MXo+afkeRQD8f5yqFEPARgG4kM92/zu8EsCiaG3DP+/4ewAYzmxowPT1gseEA/FcHLQJwp3s1TSaAbgBWu6cDjpL8ljvmPQAWhlhDE5LN/M/hvBFe5G7rXnexewPGi3oN1XztL9x4vhbVRHP/A8e6FcAyf5jUheT1AMYBuNHMTgRMb0sy1X3e1a1jWwzriOb34azrcF0LYKOZ/fP0Vqxej9p+R5EgPx/ntEivbDgXHgC+C+dqmK0AJsZg/P5wDrvXAih0H98FMBfAOnf6IgDpAetMdOspQcBVXwDy4PzHsBXAdLgfOA6hhq5wrub5FMB6/37COSe9FMBm999WsaohYP3GAL4A0DxgWsxfCzih9zkAH5y/Sn8Yzf0HkAbn9OIWOFdIdQ2jji1w3i/w/3z4r6K6xf1+fQrgYwDDYlxH1L4PkdThTp8NYGS1ZWPyeqD239G4/3ycbw/dMUFERDyj03EiIuIZhZCIiHhGISQiIp5RCImIiGcUQiIi4hmFkHiGpJGcEvD1IyQnRWns2SRvjcZYdWznNjp3Xn4v1tuqo44dJNt4WYPI2VAIiZcqAHwv0f7z9H8YMkQ/BPCQmV0Tq3pEzmUKIfHSaTitkX9RfUb1IxmSx9x/r3ZvXPkmyU0kJ5PMJ7maTg+XiwOGuZbkB+5yQ931U+n07vnIvUnnTwLGfY/ka3A+rFm9nu+74xeR/E932hNwPuQ4g+Sz1ZZPJ/k+nZ43Rf4bbZL8b5IFdHrW/HvA8jtI/prkSnf+5SSXkNxKcmRAje/T6TdUTHIGyTN+h0n+wH09Ckm+5O5zqvuaFrn7ccZrLuKFel4XIOe93wJYS/KZMNa5FEA2nPtubQPwspn1pdOIbBSAn7vLZQC4Cs4NOd8j+Q04t1E5bGZ9SDYE8H8k/+Iu3xdArjm35v8nkh3g9Pi5AsAhOHciv9nMfkVyIJz+OwXVarwLwBIz+w/3yKqxO32imR10py0leYmZrXXn7TazfiR/A+duAd+G8yn79QD8zdv6wullsxPAYjj3VnsroNZsAHfAuVmtj+SLAPLdMTqaWa67XIu6X2aR2NORkHjKnDsVzwEwOozVPjKn/0sFnFuj+ENkHZzg8XvTzKrMaQOwDUAPOPfMu4dOp85VcG7L0s1dfnX1AHL1AbDczPabcwv+P8BpxBa0RgD3u+9x9TKnRw0A3E7yYwCfAOgJJ1D8/PcsXAdglZkdNbP9AMoDQmO1Ob2vKuHc7qZ/te0OghOWH7n7OAjOLZu2AehK8gX3PnXB7hAtEjc6EpJE8Dyc+4C9EjDtNNw/ktwbQTYImFcR8Lwq4OsqfP1nuvo9qfy32h9lZksCZ5C8GsDxWuoLu/GYmb1Pp1XGDQDmuqfrPgDwCIA+ZnaI5Gw4Rzp+gftRfR/9+1XTPlWv9VUze+yMnSAvBXAdgJ/CaRT3QLj7JRJtOhISz5nZQQBvwnmT328HnL/oAacjZf2zGPo2kinu+0Rd4dxocgmAB+ncth8ku9O5q3gwqwBcRbKNexrt+wBWBFuBZBcA+8zsd3Duznw5gAvgBN1hku3gtLMIV186d3xPgXPa7e/V5i8FcCvJC906WpHs4l78kWJm8wH8q1uPiOd0JCSJYgqAhwO+/h2AhSRXw/mPtbajlGBK4IRFOzh3Yy4n+TKcU3Yfu0dY+1F3W/DPST4G4D04RxrvmlldbSOuBvAoSR+AYwDuMbPtJD+B8/7MNgD/dxb7tBLAZAC9ALwPpy9UYK3FJB+H875VCpw7U/8UwEkArwRcyHDGkZKIF3QXbZEk4Z4yfMTMhnpcikjU6HSciIh4RkdCIiLiGR0JiYiIZxRCIiLiGYWQiIh4RiEkIiKeUQiJiIhnFEIiIuKZ/w8Qy2mZDmODWQAAAABJRU5ErkJggg==", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaEAAAGDCAYAAACCzK//AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAAu/ElEQVR4nO3de3hU5bk+/vtOOIQAchYCCIkIISForIFudkEFLMpPVKharVGrbkvRCragBcFt2dVfpWqoRWqVIirWIyAbPGyoBaF2S8GgkYRAOMvJBBDknDBJnu8fs7IZYw4TMzPvBO7PdeViZq01630WIjfvWmvWQzODiIiICzGuCxARkbOXQkhERJxRCImIiDMKIRERcUYhJCIiziiERETEGYWQnFFIPkfyP0O0r24kj5GM9d6vIHl3KPbt7e9/SP40VPsTaYgauS5ApC5I7gDQEUApgDIA+QDmAphlZuVmNqYO+7nbzP5e3TZmthNAi/rW7I03FcAFZnZrwP6Hh2LfIg2ZZkLSEF1jZi0BdAcwDcBEAC+EcgCS+geaSAQohKTBMrPDZrYYwE0AfkoyjeRLJB8DAJLtSb5L8muSB0l+RDKG5CsAugF4xzvd9muSiSSN5H+Q3AlgecCywEDqQXINycMkF5Fs6411OcndgfWR3EHyCpJXAZgM4CZvvM+99f93es+r62GSX5DcR3IuyVbeuoo6fkpyJ8kDJKeE93dXJDIUQtLgmdkaALsBDKq0aoK3vAP8p/Am+ze32wDshH9G1cLMngj4zGUAUgBcWc1wtwO4C0Bn+E8JzgiiviUAfgfgTW+8i6rY7A7vZzCA8+E/DTiz0jYDASQDGArgEZIptY0tEu0UQnKm2AugbaVlPgAJALqbmc/MPrLaH5Y41cyOm9nJata/YmZ5ZnYcwH8C+HHFjQv1lAlgupltM7NjAB4CcHOlWdh/mdlJM/scwOcAqgozkQZFISRnii4ADlZa9iSALQD+RnIbyUlB7GdXHdZ/AaAxgPZBV1m9zt7+AvfdCP4ZXIXCgNcnEKKbJkRcUghJg0eyH/wh9M/A5WZ21MwmmNn5AK4BMJ7k0IrV1eyutpnSeQGvu8E/2zoA4DiA+ICaYuE/DRjsfvfCf6NF4L5LARTV8jmRBk0hJA0WyXNIjgDwBoC/mllupfUjSF5AkgCOwH9Ld5m3ugj+ay91dSvJVJLxAH4LYL6ZlQHYBCCO5NUkGwN4GEDTgM8VAUgkWd3/c68D+BXJJJItcPoaUul3qFGkwVAISUP0Dsmj8J8amwJgOoA7q9iuJ4C/AzgGYBWAZ81shbfucQAPe3fOPVCHsV8B8BL8p8biAIwD/HfqAbgXwGwAe+CfGQXeLTfP+/Urkp9Wsd853r7/AWA7gGIAY+tQl0iDRDW1ExERVzQTEhERZxRCIiLijEJIREScUQiJiIgzCiEREXFGTwoO0L59e0tMTHRdhoicYdauXXvAzDrUvuXZRyEUIDExEdnZ2a7LEJEzDMkvat/q7KTTcSIi4oxCSEREnFEIiYiIMwohERFxRiEkIiLOKIRERMQZhZCIiDijEBIREWcUQiIi4oxCSEREnFEIiYiIMwohERFxRg8wDbT3M2BqK6cl9E3q5nR8AHjr8VLXJeD9i3q4LgEAcFPSRNclYHbcMtclYNClr7guAUOHbHVdgoSBZkIiIuKMQkhERJxRCImIiDMKIRERcUYhJCIiziiERETEGYWQiIg4oxASERFnFEIiIuKMQkhERJxRCImIiDMKIRERcUYhJCIiziiERETEGYWQiIg4oxASERFnFEIiIuKMQkhERJxRCImIiDMKIRERcUYhJCIizoQthEh2IvkGya0k80m+T7IXyUSSed42GSRn1GOMyTWsW0GygGSO93Pudx1HRETCo1E4dkqSABYCeNnMbvaWpQPoCGBXxXZmlg0gux5DTQbwuxrWZ3pjiIhIFArXTGgwAJ+ZPVexwMxyzOyjwI1IXk7yXe91c5JzSH5C8jOS13nL7yD5NsklJDeTfMJbPg1AM2+W82qYjkNERMIoLDMhAGkA1tbxM1MALDezu0i2BrCG5N+9dekALgZQAqCA5DNmNonkfWaWXsM+XyRZBmABgMfMzCpvQHI0gNEA0K0V61iyiIjURzTdmDAMwCSSOQBWAIgD0M1bt8zMDptZMYB8AN2D2F+mmfUFMMj7ua2qjcxslpllmFlGh3iFkIhIJIUrhNYDuKSOnyGA680s3fvpZmYbvHUlAduVIYgZnJnt8X49CuA1AP3rWI+IiIRZuEJoOYCmJH9WsYBkP5KX1fCZpQDGejc1gOTFQYzjI9m48kKSjUi29143BjACQF5dDkBERMIvLCHkXXsZBeCH3i3a6wFMBbC3ho89CqAxgHXeLdyPBjHULG/7yjcmNAWwlOQ6ADkA9gD4S50OQkREwi5cNybAzPYC+HE1q9O8bVbAf/0HZnYSwM+r2M9LAF4KeD8i4PVEABOr+Mxx1P10oIiIRFg03ZggIiJnGYWQiIg4oxASERFnFEIiIuKMQkhERJxRCImIiDMKIRERcUYhJCIiziiERETEGYWQiIg4oxASERFnFEIiIuKMQkhERJxRCImIiDMKIRERcUYhJCIiziiERETEGYWQiIg4oxASERFnaGaua4gaGRkZlp2d7boMETnDkFxrZhmu64hGmgmJiIgzCiEREXFGISQiIs4ohERExBmFkIiIOKMQEhERZxRCIiLijEJIREScUQiJiIgzCiEREXFGISQiIs40cl1AVNn7GTC1ldMSsjYMcjo+ANyUNNF1CZgdt8x1CQCAQZe+4roEZHKB6xJQODjddQlyhtJMSEREnFEIiYiIMwohERFxRiEkIiLOKIRERMQZhZCIiDijEBIREWcUQiIi4oxCSEREnNETE0RE6sjn82H37t0oLi4OavsPPvig7+eff74jvFVFpXIAeaWlpXdfcskl+6raQCEkIlJHu3fvRsuWLZGYmAiStW5fVlZWmpaWdiACpUWV8vJy7t+/P7WwsHA2gGur2kan40RE6qi4uBjt2rULKoDOZjExMdahQ4fDANKq3SaC9YiInDEUQMGJiYkx1JA1CiEREXFG14REROopcdJ7tW0SD3xxSbD72zHt6rXfpY7x48d3btGiRdlvf/vbou/y+ZqMHTu2y7x589odOXIk9sSJE5+Far+aCYmISK1Gjhz59erVqzeEer8KIRGRBmjmzJntevXqlZqcnJw6cuTIpMrrs7Ky2qelpaUkJyenXnnllT2OHj0aAwBz5sxp07Nnzz7JycmpGRkZyQCQnZ0d17dv35TevXun9urVKzU3N7dp5f0NHTr0ePfu3X2hPg6djhMRaWCys7PjnnrqqYRVq1ZtTEhIKC0qKoqtvE1mZuahCRMmHACAcePGdZ4xY0b7KVOm7Js2bVrC3/72t01JSUm+AwcOxALAM8880+Hee+8tuueeew4WFxeztLQ0YseimZCISAOzdOnSc6655ppDCQkJpQDQsWPHssrbrF27ttkll1yS3KtXr9QFCxa0W79+fRwAZGRkHMvMzEzMyspqXxE2AwYMOJ6VlZUwZcqUTps3b27SokULi9SxKIRERBoYMwPJGoNi9OjRSTNnzty5adOm/IkTJ+4tKSmJAYDXXntt52OPPbZ3165dTdLT0/sUFhbGjhkz5uCiRYu2NGvWrHz48OG9Fi9e3DIyR6IQEhFpcK666qojixcvbltYWBgLAFWdjjtx4kRMt27dfCUlJXzjjTfaVixfv3590yFDhhx/+umn97Zp06Z027ZtTfLz85ukpKSUPPzww/uGDRv2dU5OTrNIHUvYrgmR7ATgaQD9AJQA2AHglwBOAXjXzNJIZgC43czGfccxJpvZ72rZZjGA882s2m/siojUx45pV9e4Pi8v70RaWlrI7izLyMgonjBhwpeDBg3qHRMTY2lpaScWLFiwI3CbSZMm7e3fv39Kly5dTqWkpJw4duxYLAD86le/6rpjx46mZsaBAwce+bd/+7eTU6ZM6TRv3rx2jRo1sg4dOvgef/zxvZXHHDNmTNeFCxe2LS4ujunYseOFmZmZB6ZPn/6t7eqKZqE/9Uf/V4k/BvCymT3nLUsH0BLALnghFIJxjplZixrW/wjADQAuDGa8jM6xlj262t1FRNaGQU7HB4Cbkia6LgGz45a5LgEAMOjSV1yXgEwucF0CCgenuy4hqmzYsAEpKSlBbx/qEGpoPv/88/YXXXRRYlXrwnU6bjAAX0UAAYCZ5ZjZR4Ebkbyc5Lve6+Yk55D8hORnJK/zlt9B8m2SS0huJvmEt3wagGYkc0i+WrkAki0AjAfwWJiOUURE6ilcp+PSANT1G79TACw3s7tItgawhuTfvXXpAC6G/7ReAclnzGwSyfvMLL2a/T0KIAvAiZoGJTkawGgA6NZKz4ISEYmkaLoxYRiASSRzAKwAEAegm7dumZkdNrNiAPkAute0I+/U3wVmtrC2Qc1slpllmFlGh3iFkIhIJIVrJrQe/msxdUEA15tZwTcWkt+HfwZUoQy11z0AwCUkd3jbnktyhZldXseaREQkjMI1E1oOoCnJn1UsINmP5GU1fGYpgLHeTQ0geXEQ4/hINq680Mz+bGadzSwRwEAAmxRAIiLRJywhZP5b7kYB+CHJrSTXA5gKoKbb+R4F0BjAOpJ53vvazPK2/9aNCSIiEv3C9j0hM9sL4MfVrE7ztlkB//UfmNlJAD+vYj8vAXgp4P2IgNcTAdR4P7GZ7UANXf1EROptaqsaV6cB8ZiPoFs5YOrhqGrlcPTo0Zhrrrnm/C+++KJpbGwshg0b9vWzzz67JxT7jqYbE0REJEpNmDChaPv27evz8vLyV69e3eKtt946JxT7VQiJiDRAkWzl0LJly/JrrrnmKADExcXZhRdeeGLXrl1NQnEcCiERkQamopXDypUrNxUUFOQ///zzOytvk5mZeSgvL29DQUFBfnJy8skZM2a0B4CKVg4FBQX5S5Ys2QKcbuWwcePG/HXr1m1ISko6Vd3YBw4ciP3ggw9aDx8+/EgojkUhJCLSwLhq5eDz+fCjH/3o/NGjRxelpqZWG1R1oRASEWlgXLVyuOWWWxLPP//84kceeWRfqI5FISQi0sC4aOUwbty4zkeOHIl94YUXdoXyWNTeW0SkvqYernF1Q2/lsHXr1sbPPPNMQlJSUnGfPn1SAWD06NH7xo8ff6C+xxKWVg4NlVo5+KmVw2lq5eCnVg7fpFYOdeOilYOIiEitFEIiIuKMQkhERJxRCImIiDMKIRERcUYhJCIizuh7QiIi9dT35b61bRKPtcG3csj9aW5UtXIAgEGDBvXct29f47KyMvbv3//o3LlzdzZqVP8I0UxIRERqtWjRoq0FBQX5mzZtWv/VV181njNnTptQ7FchJCLSAEWylQMAtG3bthwAfD4ffT4fSYbkOBRCIiINjKtWDgMHDuzZoUOHi5o3b1525513HgrFsSiEREQaGFetHP75z39uLiws/PzUqVMx77zzTkg6q+rGhECdLwamZjstYYLT0aPHVLh/hp7fVNcFoNB1ARJ1gm3lMH/+/C0DBgw4OWPGjHYrV65sCfhbOSxfvrz54sWLW6Wnp/fJyclZP2bMmIODBg06vnDhwlbDhw/v9eyzz+649tprj1a13/j4eBsxYsTXCxcubD1q1Kh6N7bTTEhEpIGJdCuHw4cPx3zxxReNAX9juyVLlrTq3bv3yVAci2ZCIiL1lPvT3BrXN/RWDkeOHIm5+uqrLzh16hTLy8v5gx/84MiDDz64PxTHolYOATIyMiw72+3pOBGJfmrlUDdq5SAiIlFJISQiIs4ohERExBmFkIiIOKMQEhERZxRCIiLijL4nJCJSTxt613y7diwQvwHBt3JI2bgh6lo5VBgyZMgFu3btarp58+b1odifZkIiIhKUl19+uXXz5s2/9Zy6+tBMKEDunsNInPSe0xp2xN3idHwA2PBGZ9clYPnlf3JdAgDgutaNXZeA2XHLXJeAQZe+4roEZHKB6xJQODjddQn/Z+bMme1mzJjRkSRSUlJO/vd///f2wPVZWVntX3zxxQ4+n4+JiYkl8+fP396yZcvyOXPmtHn88cc7x8TEWMuWLcuys7MLsrOz4+68884kn8/H8vJyLFiwYGvfvn1LAvd3+PDhmBkzZnScNWvWFzfffHOPUB2HQkhEpIGpaOWwatWqjQkJCaVVPTsuMzPz0IQJEw4AwLhx4zrPmDGj/ZQpU/ZVtHJISkryHThwIBY43crhnnvuOVhcXMyKp2sHGj9+fJf777+/qEWLFuWhPBadjhMRaWAi3crh448/brZ9+/amt99++9ehPhaFkIhIAxNsK4eZM2fu3LRpU/7EiRP3lpSUxAD+Vg6PPfbY3l27djVJT0/vU1hYGDtmzJiDixYt2tKsWbPy4cOH91q8eHHLwH199NFHLfLy8uK7dOnS99JLL+29Y8eOpv37908OxbEohEREGphIt3KYOHHi/n379q3bs2dP7j/+8Y+NiYmJJWvWrCkIxbHompCISD2lbKz5AdkNvZVDOKmVQ4CmCT0t4adPO61Bd8f56e6403R3nF803R2nVg51o1YOIiISlRRCIiLijEJIREScCSqESN5IsqX3+mGSb5P8XnhLExGRM12wM6H/NLOjJAcCuBLAywD+HL6yRETkbBBsCFV8G/dqAH82s0UAmoSnJBEROVsE+z2hPSSfB3AFgN+TbApdTxIRAQD8aczy2jaJX4nlQbdy+MVzQ6KulUP//v2T9+3b1zguLq4cAJYtW7apS5cu337IXB0FG0I/BnAVgKfM7GuSCQAerO/gIiLScMydO3fbpZdeeiKU+wx2NvO8mb1tZpsBwMy+BHBbKAsREZHgzZw5s12vXr1Sk5OTU0eOHJlUeX1WVlb7tLS0lOTk5NQrr7yyx9GjR2MAYM6cOW169uzZJzk5OTUjIyMZ8D+Vu2/fvim9e/dO7dWrV2pubm7TSB1HsCHUJ/ANyVjUoUugiIiETkUrh5UrV24qKCjIf/7553dW3iYzM/NQXl7ehoKCgvzk5OSTM2bMaA8AFa0cCgoK8pcsWbIFON3KYePGjfnr1q3bkJSUdKqqce++++7E3r17pz744IMJ5eWh6ehQYwiRfIjkUQAXkjzi/RwFsA/AopBUICIidRLpVg4A8Oabb27btGlT/qpVqzZ+/PHHLZ599tl2oTiWGkPIzB43s5YAnjSzc7yflmbWzsweCkUBIiJSN5Fu5QAASUlJPgBo06ZN+U033XRwzZo1zUNxLEGdjjOzh0h2IfnvJC+t+AlFASIiUjeRbuXg8/nw5ZdfNgKAkpISvv/++63S0tJOhuJYgro7juQ0ADcDyMfp7wwZgH+EoggRkYbsF88NqXF9Q2/lcPLkyZgrrriip8/nY3l5OQcNGnRk/Pjx+0NxLEG1ciBZAOBCMysJesdkJwBPA+gHoATADgC/BHAKwLtmlkYyA8DtZjauzpX7x5hsZr+rZt0SAAnwB+1HAH5hZt86bxpIrRz81MrhNLVy8FMrBz+1cvhuQtHKYRuAoP9vJEkACwGsMLMeZpYKYDKAjoHbmVn2dw0gz+Qa1v3YzC4CkAagA4Ab6zGOiIiEQbBfVj0BIIfkMvhnNQCAGgJkMACfmT0XsG0OAJBMrFhG8nIAD5jZCJLNATwDoK9X11QzW0TyDgDXAogH0APAQjP7tXeKsBnJHADrzSwzsAAzOxJwjE3gP30oIiJRJNgQWuz9BCsNQF0fOzEFwHIzu4tkawBrSP7dW5cO4GL4A7CA5DNmNonkfWaWXt0OSS4F0B/A/wCYX8d6REQkzIIKITN7mWQzAN3MrCBMtQwDcC3JB7z3cQC6ea+XmdlhACCZD6A7gF217dDMriQZB+BVAEMAfFB5G5KjAYwGgNhzOtT3GEREpA6C7Sd0DYAcAEu89+kka5oZrUfdn6hAANebWbr3083MKi7kBd4QUYbgZ3Aws2L4Z3HXVbN+lpllmFlGbHyrOpYsIiL1EeyNCVPhP631NfB/13e+9ayiAMsBNCX5s4oFJPuRvKyGzywFMNa7qQEkLw6iLh/Jb90wQbKF95BVkGwE4P8DsDGI/YmISAQFO6MoNbPDXj5UqPZCv5kZyVEAniY5CUAxTt+iXZ1H4b+le50XRDsAjKilrlne9p9WujGhOYDFXsuJWPhD8bmqdiAiUl9ZN9X2VxXil9bh7NCEN9+NulYOxcXFvPPOO7utWrWqJUn7zW9+s+eOO+74ur77DTaE8kjeAiCWZE8A4wB8XNMHzGwv/C0gqpLmbbMCwArv9UkAP69iPy8BeCng/YiA1xMBTKziM0Xwfz9JRERC4KGHHkro0KGDb8eOHXllZWXYt29f0JdFahLs6bix8D9JuwTA6wCOoOZZjYiIhFGkWzm8/vrr7R977LFCAIiNjUXFw1PrK9hnx50wsylm1s+7iD/Fu+AvIiIRFulWDgcOHIgF/Kf7UlNTU4YPH37+rl27wj8TIvm09+s7JBdX/glFASIiUjeRbuXg8/lYVFTUeODAgcfy8/M3fP/73z8+duzY80JxLLXNhCoeGPUUgKwqfkREJMIi3cqhY8eOpXFxceW33Xbb1wBw6623HszLy4sPxbHU1k9orffryqp+QlGAiIjUTaRbOcTExGDo0KGH33vvvZYA8P7775/Ts2fP8LdyIJmLmm/FvjAURYiINGQT3ny3xvUNvZUDAEyfPn33LbfckvTAAw/EtmvXrnTu3Lk7Km/zXdTYysG7Hbsjvv2InO4A9prZllAUES3UysFPrRxOUysHP7Vy8FMrh++mPq0c/gDgiJl9EfgD/1O1/xDiOkVE5CxTWwglmtm6ygvNLBtAYlgqEhGRs0ZtIRRXw7pmNawTERGpVW0h9EngQ0grkPwP1L1fkIiIyDfU9o3XXwJYSDITp0MnA/5OpaPCWJeIiJwFagwh70Gg/05yMLyHjgJ4z8yWh70yERE54wXbWfVDAB+GuRYRkQZp96SPalzfGojfjY+CbuXQddqgqGrlcOjQoZgBAwb0rnhfVFTUeNSoUQfnzJlTa4fr2oTkAXQiInLmatOmTfnGjRvzK9736dMn5cYbbzwUin0H28pBRESiSKRbOVTIzc1t+tVXXzW+8sorj4XiODQTEhFpYCpaOaxatWpjQkJCaVXPjsvMzDw0YcKEAwAwbty4zjNmzGg/ZcqUfRWtHJKSknwVLRoqWjncc889B4uLi1nxdO2qvPzyy22vvfbagzExoZnDaCYkItLARLqVQ6CFCxe2ve222w6G6lgUQiIiDUykWzlUWLVqVbOysjIOGjToRKiORafjAvTt0grZ0652XMVhx+MDKVNdVwAE/2jIM99UDHJdAoCprgtAoesCoshVV1115IYbbrhg8uTJRZ06dSorKiqKrTwbqtzKISEhwQecbuUwZMiQ40uXLm29bdu2JgcPHixLSUkp6dOnz75t27Y1zcnJaXbttdcerTzuK6+80nbUqFEhmwUBCiERkXrrOq3mfyicCa0cAGDx4sVt33nnnc2hOg6gllYOZ5uMjAzLzs52XYaIRDm1cqib+rRyEBERCRuFkIiIOKMQEhERZxRCIiLijEJIREScUQiJiIgz+p6QiEg9TZ06tbZN4ufPnx90K4epU6dGVSsHAHj++efbZmVldQKAjh07+t56663tFY8Nqg/NhEREpEY+nw8PPfTQeStXrty0adOm/D59+px88sknzw3FvhVCIiINUCRbOZSXl9PMcPTo0Zjy8nIcOXIkpnPnzqdCcRw6HSci0sBEupVD06ZNbfr06Tu/973v9WnWrFlZ9+7dS+bOnbszFMeiEAq09zNgaiunJfRN6uZ0fAAYs+qPrkvAda0buy4BADA7bpnrEjDo0ldcl4ChQ7a6LkECBNvK4ZFHHuly9OjR2OPHj8dedtllh4HTrRyuv/76Q5mZmYcAfyuHp556KmH37t1Nbr755kN9+/YtCdxXSUkJZ82a1WH16tX5KSkpJXfccUe3yZMnJzzxxBNf1vdYdDpORKSBiXQrh3/961/NAKBPnz4lMTEx+MlPfnJw9erVzUNxLAohEZEG5qqrrjqyePHitoWFhbEAUNXpuMqtHCqWV7RyePrpp/e2adOmdNu2bU3y8/ObpKSklDz88MP7hg0b9nVOTk6zwH11797dt2XLlri9e/c2AoAlS5ac06tXr+JQHItOx4mI1FNtt2g39FYOiYmJvgcffPDLgQMHJjdq1Mi6du166rXXXtseimNRK4cAGZ1jLXt0C6c16JqQn64JnaZrQtFHrRzqRq0cREQkKimERETEGYWQiIg4oxASERFnFEIiIuKMQkhERJzR94REROpp2fIetW0SX7QcQbdyGDpka9S1cvjLX/7S5sknn0woLy/nFVdccfi5557bHYr9aiYkIiI1KiwsjH3kkUe6rlixYtOWLVvW79u3r9GiRYta1v7J2imEREQaoEi2cigoKGialJRU0rlz51IAGDp06JF58+a1CcVx6HSciEgDE+lWDqmpqSVbt26NKygoaHL++eefWrx4cRufz8dQHItmQiIiDUywrRwuueSS5F69eqUuWLCg3fr16+OA060csrKy2leEzYABA45nZWUlTJkypdPmzZubtGjR4hvPc+vQoUPZH/7why9uvPHG8/v169e7W7duJbGxsSF55ptCSESkgYl0KwcAuOWWWw6vW7duY05Ozsbk5OTiHj16lHx71LpTCImINDCRbuUAAHv27GkEAPv374+dPXv2uffee+/+UByLrgmJiNRTbU8Zb+itHABgzJgx5+Xn58cDwMSJE/deeOGFIZkJqZVDALVy8FMrh9PUysFPrRy+Sa0c6katHEREJCqFLYRIdiL5BsmtJPNJvk+yF8lEknneNhkkZ9RjjMnVLI8n+R7JjSTXk5z2XccQEZHwCUsIkSSAhQBWmFkPM0sFMBlAx8DtzCzbzMbVY6gqQ8jzlJn1BnAxgB+QHF6PcUREJAzCNRMaDMBnZs9VLDCzHDP7KHAjkpeTfNd73ZzkHJKfkPyM5HXe8jtIvk1yCcnNJJ/wlk8D0IxkDslXA/drZifM7EPv9SkAnwLoGqZjFRGR7yhcIZQGoK4P4JsCYLmZ9YM/xJ4k2dxblw7gJgB9AdxE8jwzmwTgpJmlm1lmdTsl2RrANQCqvMJMcjTJbJLZ+0/oJg0RkUiKphsThgGYRDIHwAoAcQAqbhVbZmaHzawYQD6A7sHskGQjAK8DmGFm26raxsxmmVmGmWV0iA/JUyhERCRI4fqe0HoAN9TxMwRwvZkVfGMh+X0AgfejlyH4umcB2GxmT9exFhGRoHX6MKe2TeLxYU7QrRwKB6dHXSuHsWPHdpk3b167I0eOxJ44ceKziuUnT57kDTfckJSbmxvfunXr0nnz5m1LTk4+Fex+wzUTWg6gKcmfVSwg2Y/kZTV8ZimAsd5NDSB5cRDj+EhW+YUSko8BaAXgl0FXLSIiVRo5cuTXq1ev/tZ3nf74xz+2b9WqVenOnTvz7rvvvqLx48fX6fp7WELI/N+AHQXgh94t2usBTAXwrW/hBngUQGMA67xbuB8NYqhZ3vbfuDGBZFf4rzGlAvjUu3nh7rofiYhIdIpkKwcAGDp06PHu3bv7Ki9/9913W991111fAcCdd9556OOPP25ZXl4e9HGE7bE9ZrYXwI+rWZ3mbbMC/us/MLOTAH5exX5eAvBSwPsRAa8nAphYxWd2w396T0TkjBPpVg41KSoqapKUlHQKABo3bowWLVqUFRUVNap4wndtounGBBERCUKkWznUpKpHv9X2hO9ACiERkQbGRSuH6nTq1OnU9u3bmwCAz+fDsWPHYs8999xvhWJ1FEIiIg2Mi1YO1bn66qu/njNnTjsAePHFF9sMGDDgaExM8NGiVg4iIvVUODi9xvVnSCuHrgsXLmxbXFwc07FjxwszMzMPTJ8+fe/9999/4Prrr0/q1q1bWqtWrcrefPPNOj1yXa0cAqiVg59aOZymVg5+auXwTWrlUDdq5SAiIlFJISQiIs4ohEREvgNdyghOeXk5AVT77VWFkIhIHcXFxeGrr75SENWivLyc+/fvbwUgr7ptdHeciEgdde3aFbt378b+/fuD2r6wsLBRWVlZ+zCXFY3KAeSVlpZW+9g0hZCISB01btwYSUnfelxbtVJTU3PNLCOMJTVYOh0nIiLOKIRERMQZhZCIiDijEBIREWcUQiIi4oxCSEREnFEIiYiIMwohERFxRq0cAmRkZFh2drbrMkTkDENyrb6sWjXNhERExBmFkIiIOKMQEhERZxRCIiLijEJIREScUQiJiIgzCiEREXFGISQiIs4ohERExBmFkIiIOKMQEhERZxRCIiLiTCPXBUST3D2HkTjpPac17Ii7xen4APCnwoWuS8B1rRu7LgEAMDtumesSMOjSV1yXgKFDtrouQc5QmgmJiIgzCiEREXFGISQiIs4ohERExBmFkIiIOKMQEhERZxRCIiLijEJIREScUQiJiIgzCiEREXFGISQiIs4ohERExBmFkIiIOKMQEhERZxRCIiLijEJIREScUQiJiIgzCiEREXFGISQiIs4ohERExJmwhRDJTiTfILmVZD7J90n2IplIMs/bJoPkjHqMMbmGdf8/yV0kj33X/YuISHiFJYRIEsBCACvMrIeZpQKYDKBj4HZmlm1m4+oxVLUhBOAdAP3rsW8REQmzcM2EBgPwmdlzFQvMLMfMPgrciOTlJN/1XjcnOYfkJyQ/I3mdt/wOkm+TXEJyM8knvOXTADQjmUPy1coFmNm/zOzLMB2fiIiEQKMw7TcNwNo6fmYKgOVmdhfJ1gDWkPy7ty4dwMUASgAUkHzGzCaRvM/M0utTKMnRAEYDQOw5HeqzKxERqaNoujFhGIBJJHMArAAQB6Cbt26ZmR02s2IA+QC6h2pQM5tlZhlmlhEb3ypUuxURkSCEaya0HsANdfwMAVxvZgXfWEh+H/4ZUIUyhK9uERGJoHDNhJYDaEryZxULSPYjeVkNn1kKYKx3UwNIXhzEOD6SjetXqoiIuBKWEDIzAzAKwA+9W7TXA5gKYG8NH3sUQGMA67xbuB8NYqhZ3vbfujGB5BMkdwOIJ7mb5NQ6HoaIiIRZ2E5rmdleAD+uZnWat80K+K//wMxOAvh5Fft5CcBLAe9HBLyeCGBiNeP/GsCvv0PpIiISIdF0Y4KIiJxlFEIiIuKMQkhERJxRCImIiDMKIRERcUYhJCIiziiERETEGYWQiIg4oxASERFnFEIiIuKMQkhERJxRCImIiDMKIRERcUYhJCIiziiERETEGYWQiIg4oxASERFnFEIiIuKMQkhERJyhmbmuIWpkZGRYdna26zJE5AxDcq2ZZbiuIxppJiQiIs4ohERExBmFkIiIOKMQEhERZxRCIiLijEJIREScUQiJiIgzCiEREXFGISQiIs4ohERExBmFkIiIOKMQEhERZxRCIiLijEJIREScUSuHACSPAihwXEZ7AAdUQ1TUAERHHaqh4dfQ3cw6hLqYM0Ej1wVEmQLXPT9IZquG6KghWupQDarhTKbTcSIi4oxCSEREnFEIfdMs1wVANVSIhhqA6KhDNfiphjOQbkwQERFnNBMSERFnFEIASF5FsoDkFpKTHNUwh+Q+knkuxvdqOI/khyQ3kFxP8n4HNcSRXEPyc6+G/4p0DQG1xJL8jOS7jsbfQTKXZA7JbBc1eHW0Jjmf5Ebvz8aACI+f7P0eVPwcIfnLSNbg1fEr789kHsnXScZFuoYz0Vl/Oo5kLIBNAH4IYDeATwD8xMzyI1zHpQCOAZhrZmmRHDughgQACWb2KcmWANYCGBnJ3wuSBNDczI6RbAzgnwDuN7N/RaqGgFrGA8gAcI6ZjXAw/g4AGWbm9LsxJF8G8JGZzSbZBEC8mX3tqJZYAHsAfN/MvojguF3g/7OYamYnSb4F4H0zeylSNZypNBMC+gPYYmbbzOwUgDcAXBfpIszsHwAORnrcSjV8aWafeq+PAtgAoEuEazAzO+a9bez9RPxfSiS7ArgawOxIjx1NSJ4D4FIALwCAmZ1yFUCeoQC2RjKAAjQC0IxkIwDxAPY6qOGMoxDy/yW7K+D9bkT4L95oRDIRwMUAVjsYO5ZkDoB9AD4ws4jXAOBpAL8GUO5g7AoG4G8k15Ic7aiG8wHsB/Cid2pyNsnmjmoBgJsBvB7pQc1sD4CnAOwE8CWAw2b2t0jXcSZSCAGsYtlZfY6SZAsACwD80syORHp8Myszs3QAXQH0JxnR05MkRwDYZ2ZrIzluFX5gZt8DMBzAL7xTtpHWCMD3APzZzC4GcByAq+umTQBcC2Ceg7HbwH+GJAlAZwDNSd4a6TrORAoh/8znvID3XXEWT7O96zALALxqZm+7rMU77bMCwFURHvoHAK71rsm8AWAIyb9GuAaY2V7v130AFsJ/6jjSdgPYHTAbnQ9/KLkwHMCnZlbkYOwrAGw3s/1m5gPwNoB/d1DHGUch5L8RoSfJJO9fWjcDWOy4Jie8mwJeALDBzKY7qqEDydbe62bw/8+/MZI1mNlDZtbVzBLh//Ow3Mwi+q9eks29m0Pgnf4aBiDid06aWSGAXSSTvUVDAUT0pp0AP4GDU3GenQD+jWS89//JUPivmUo9nfUPMDWzUpL3AVgKIBbAHDNbH+k6SL4O4HIA7UnuBvAbM3shwmX8AMBtAHK9azIAMNnM3o9gDQkAXvbugooB8JaZOblF2rGOABb6/75DIwCvmdkSR7WMBfCq94+0bQDujHQBJOPhv4P155EeGwDMbDXJ+QA+BVAK4DPo6Qkhcdbfoi0iIu7odJyIiDijEBIREWcUQiIi4oxCSEREnFEIiYiIMwohcY6kkcwKeP8Ayakh2vdLJG8Ixb5qGedG7wnTH0ZTXSLRTiEk0aAEwI9ItnddSCDvu0rB+g8A95rZ4HDVI3ImUghJNCiF/4t/v6q8ovKMgeQx79fLSa4k+RbJTSSnkcz0ehHlkuwRsJsrSH7kbTfC+3wsySdJfkJyHcmfB+z3Q5KvAcitop6fePvPI/l7b9kjAAYCeI7kk1V85tfeZz4nOa2K9Y94deSRnOV9Ix8kx5HM9+p7w1t2GU/31fks4KkKDwYcy395y5qTfM8bN4/kTcH95xCJnLP+iQkSNf4EYB3JJ+rwmYsApMDfAmMbgNlm1p/+ZnxjAfzS2y4RwGUAegD4kOQFAG6H/0nI/Ug2BfC/JCueitwfQJqZbQ8cjGRnAL8HcAmAQ/A/4Xqkmf2W5BAAD5hZdqXPDAcwEv7+NydItq3iOGaa2W+97V8BMALAO/A/KDTJzEoqHmUE4AEAvzCz//UeNFtMchiAnl7dBLDYe9hpBwB7zexqb9+tgvttFYkczYQkKnhP654LYFwdPvaJ1wOpBMBWABUhkgt/8FR4y8zKzWwz/GHVG/5nsd3uPZ5oNYB28P9FDgBrKgeQpx+AFd5DLEsBvAp/r52aXAHgRTM74R1nVT2jBpNcTTIXwBAAfbzl6+B/XM6t8M8WAeB/AUwnOQ5Aa6+OYd7PZ/A/Vqa3dyy58M8Cf09ykJkdrqVWkYhTCEk0eRr+ayuB/WpK4f059U5TNQlYVxLwujzgfTm+Ocuv/Gwqg3/GMNbM0r2fpID+MMerqa+qth+1YRXjn17pbxH9LIAbzKwvgL8AqGgbfTX8M8RLAKwl2cjMpgG4G0AzAP8i2dsb4/GAY7nAzF4ws03eZ3MBPO6dNhSJKgohiRreLOEt+IOowg74/yIF/P1cGn+HXd9IMsa7TnQ+gAL4H1h7D/2tK0CyF2tv1rYawGUk23s3LfwEwMpaPvM3AHd5D+BEFafjKgLngHd67QZvuxgA55nZh/A312sNoAXJHmaWa2a/B5AN/6xnqTdGC++zXUie650+PGFmf4W/IZurFgwi1dI1IYk2WQDuC3j/FwCLSK4BsAzVz1JqUgB/WHQEMMbMiknOhv+U3afeDGs//NduqmVmX5J8CMCH8M8+3jezRbV8ZgnJdADZJE8BeB/A5ID1X5P8C/yzlR3wtxYB/E90/6t3HYcA/uBt+yjJwQDK4G+p8D/eNaMUAKu8exqOAbgVwAUAniRZDsAH4J5af6dEIkxP0RYREWd0Ok5ERJxRCImIiDMKIRERcUYhJCIiziiERETEGYWQiIg4oxASERFnFEIiIuLM/wP43JCJeL3JcgAAAABJRU5ErkJggg==", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" } ], "source": [ "\n", "#Generate the different non-iid distribution.\n", "# Data_presence_indicator will help to reduce the number of classes --\n", "# among the clients as the non-iid increases\n", "for j in range(K):\n", " dist = np.random.uniform(q_step, (1+j)*q_step, (num_classes, num_users))\n", " if j != 0:\n", " data_presence_indicator = np.random.choice([0, 1], (num_classes, num_users), p=[j*q_step, 1-(j*q_step)])\n", " if len(np.where(np.sum(data_presence_indicator, axis=0) == 0)[0])>0:\n", " for i in np.where(np.sum(data_presence_indicator, axis=0) == 0)[0]:\n", " zero_array = data_presence_indicator[:,i]\n", " zero_array[np.random.choice(len(zero_array),1)] =1\n", " data_presence_indicator[:,i] = zero_array\n", " dist = np.multiply(dist,data_presence_indicator)\n", " psum = np.sum(dist, axis=1)\n", " for i in range(dist.shape[0]):\n", " dist[i] = dist[i]*len(label_index_list[i])/(psum[i]+0.00001)\n", " dist = np.floor(dist).astype(int)\n", "\n", " # If any client does not get any data then this logic helps to allocate the required samples among the clients\n", " gainers = list(np.where(np.sum(dist, axis=0) != 0))[0]\n", " if len(gainers) < num_users:\n", " losers = list(np.where(np.sum(dist, axis=0) == 0))[0]\n", " donors = np.random.choice(gainers, len(losers))\n", " for index, donor in enumerate(donors):\n", " avail_digits = np.where(dist[:,donor] != 0)[0]\n", " for digit in avail_digits:\n", " transfer_frac = np.random.uniform(0.1,0.9)\n", " num_transfer = int(dist[digit, donor]*transfer_frac)\n", " dist[digit, donor] = dist[digit, donor] - num_transfer\n", " dist[digit, losers[index]] = num_transfer\n", "\n", " #Logic to check if the summation of all the samples among the clients is equal to\n", " # # the total number of samples present for that class. If not it will adjust.\n", " for num in range(num_classes):\n", " while dist[num].sum() != len(label_index_list[num]):\n", " index = random.randint(0,num_users-1)\n", " if dist[num].sum() < len(label_index_list[num]):\n", " dist[num][index]+=1\n", " else:\n", " dist[num][index]-=1\n", " \n", "\n", " #Division of samples number among the clients\n", " split = [[] for i in range(num_classes)]\n", " for num in range(num_classes):\n", " start = 0\n", " for i in range(num_users):\n", " split[num].append(label_index_list[num][start:start+dist[num][i]])\n", " start = start+dist[num][i]\n", "\n", " #Division of actual data points among the clients.\n", " datapoints = [[] for i in range(num_users)]\n", " class_histogram = [[] for i in range(num_users)]\n", " class_stats= [[] for i in range(num_users)]\n", " for i in range(num_users):\n", " for num in range(num_classes):\n", " datapoints[i] += split[num][i]\n", " class_histogram[i].append(len(split[num][i]))\n", " if len(split[num][i])==0:\n", " class_stats[i].append(0)\n", " else:\n", " class_stats[i].append(1)\n", "\n", " #Store the dataset division in the folder\n", " if not os.path.exists(storepath):\n", " os.makedirs(storepath)\n", " file_name = 'data_split_niid_'+ str(K)+'.pt'\n", "\n", " torch.save({'datapoints': datapoints, 'histograms': class_histogram,\n", " 'class_statitics': class_stats}, storepath + file_name)\n", " class_stats=np.array(class_stats) \n", " class_stats=class_stats.transpose()\n", " filename_class=f'class_stats_{j}.png'\n", " filename_sample=f'sample_stats_{j}.png'\n", " #print(dist)\n", " #dist[dist==1]==0\n", " \n", " plot_sample_stats(dist, num_users=num_users,filename=filename_sample)\n", " #print(dist)\n", " plot_class_stats(class_stats, num_users=num_users,filename=filename_class)\n", " #print(class_stats)" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.9.6" } }, "nbformat": 4, "nbformat_minor": 5 } ================================================ FILE: tutorials/media_plot.ipynb ================================================ { "cells": [ { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "FashionMNIST\n" ] }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAABZ4UlEQVR4nO3dd3hUVd7A8e+ZXlMmhYQESKgBQugg0oJYQEDABnbgVVZ37bvuru6L4rruuq6va1ldFgvq7torShEVwYZ0pPcaSCCkTzJ9zvvHnUwSEiBIEBLP53l4yNxz5t4zc5PfPffcc39XSClRFEVRmj/d2W6AoiiK0jRUQFcURWkhVEBXFEVpIVRAVxRFaSFUQFcURWkhDGdrw4mJiTIjI+NsbV5RFKVZWr169VEpZVJDZWctoGdkZLBq1aqztXlFUZRmSQix73hlashFURSlhVABXVEUpYVoVEAXQowSQmwTQuwUQvy+gfJ4IcQHQoj1QogVQojspm+qoiiKciInDehCCD3wHDAa6AZcI4Todky1B4B1Usoc4Ebg6aZuqKIoinJijemhDwB2Sil3Syn9wJvA+GPqdAO+AJBSbgUyhBCtmrSliqIoygk1ZpZLGnCg1us8YOAxdX4ALge+EUIMANoB6cDh2pWEENOB6QBt27Y95cZu+fpLvn7zNSqKjuJMSGTo5BvpOnTEKa9HURSlJWpMD100sOzYFI2PAfFCiHXAHcBaIFjvTVLOllL2k1L2S0pqcBrlcW35+ksWzXqKiqOFICUVRwtZNOsptnz95SmtR1EUpaVqTEDPA9rUep0OHKpdQUpZLqWcKqXshTaGngTsaapGAnz92iyCwVCdZcFgiCVzniMYCDTlphRFUZqlxgy5rAQ6CSEygYPAZODa2hWEEHFAVWSM/WbgKylleVM2tKLcTUMnC1WVXtbM/4g1n7xHZXlFnbKcYcNIaJ9Fj5GXYDSZ8VVVotPrMZotTdk0RVGUc8JJA7qUMiiEuB34FNADL0spNwkhbo2UzwK6Aq8JIULAZuB/mrqh1kAQj9FYb7kpGCSlQ2fSw4fZhq1O2fqvvoKvvsLv8bD6/X/jPaYjn+oyENd9KJfceie6FbM5vHMrfmnEHhuDPTYec1JbRPZErfKhdRAOgckGRhuYHGCyg1EdHBRFOTc06tZ/KeV8YP4xy2bV+nkZ0Klpm1ZX50PFbGiTRFhXM0qkC4fperCIttk5+HcF6FCwS2sP4DPoMQVDpD56O+UdOtHOEGSHT1fn/d78CootW1ny2otsWjSPgKw7AqUXkuSO33Dl//4J3Tu3s3FHIT4M2A1+HAY/9jbdcEx7C6szBl4eDe4CMNq1QG+yQdtBMPy32sqW/g3CQW25ya7VS+gIbfpr5Yc3gcESKYvU0enP5FeqKEoLc9ZyuZyqVAQcKGRbqguv0YAlEKRLfrG2HEh79nmqVqzEt2MH3k2bEDt2ABA3eioum43wUT0ddu4GtIAf0OswhcJ0+vv/sbfIh2/Flxw8VEy5tabH3fZwGUkXpDL/2SfYsyqOMHF1G3UAUg4/xMTfz8R3KInFmyVBATZDCLu+DEfiGtJabSYtqxvBr18m7M7HaJKI6uNGzuSagD57BIR8ddc/YDpc+jftzOCf59cK9g7twNB1HHSfCAEvLPtHzfLqA0ZyFsRnQCgAFQWRA40d9CYQDV3rVhSlOWs2Af2lHpdz6Y4tmGPHgtmF2VcMJZ/wUqeuPAksCMSzr+0QrB2HY71Mj9WoJ96io6tNG4YJ3PQLwmvXoC84iO5gHuLQQYxt2mHI7EXHTDD9K5YuvlKMTht6hwlpllg7uXDddCvb16zDWbKX0r27KAvqKRFWAM6v2E/yZffx3p8fpHBPMbL66wxG/q9002neBzgTEinOG8rHR/YDYAlLLBJs247QPeUrss4fRpH+Bg4dPITNasBhM2Ayg7EkFQdAKEDAmIEIetBJHzrvQfBXQus+2na8pbD4kfpf2kV/hMF3Qel+eLZPzXKh14L/qD9D7+vh6E6Ye3v9A0av6yCtD5Tnw7b5Ncur6yR1AWscBP0Q8mvLdSqbhKKcLc0moG9x9CKjS1/0QhuG8FkS2NzlejbbtJkv8zcU8PmWOtPeaZ9k56IeaQDMKE5iRbg/JPeHZND1CtMvQc/bkbofidbE4SZlz1ESKwqxBHzs7JRDrtlBl0FDOPznZ3DJZKyJKSQlJaCPt2HqEE+H84bjD0q2fTUXWbiHqkoflZ4wyUY3g+PzMd/0Mv+5/x485WVg1L5uP1AOWKWfhF07SGzTjtIdB1lU5YYSrT2GYAjrxqMMSMqi18WXsneBn+2FR7AEQ1gQWM0OXPv20G6QRDhacdhwH+HKCnQ2M3qbCZ3FgNnbETuALQFfv5noRBCdMYBOF0AEqrQhHwAZAp0BqoohkAf+KvC7oX2uFtALt8K8e+vvlGvfgc4Xw87P4M3IdXKjrWbI6KpXtPfvXgrLZ9U/YPS/BZyttANKwfr6BwxXJuiNIKU6o1CURmg2AX2E34T+mL9pvdBzoU/rEY5rl4hjTyWdOsbTq2MCrngLBrOeUCiMXq/j/tFZFFb48ARCeAMhvIEwsdZaF1knXc/64ipWBMN4fEH0ZSVkxRnIjRTvMziJPXKQ5P27ifVXArCp53D6TrqZrMHD2fCHv1NgiuGQI4N8ewL5Bhcre3TlTwlJ5N54My+98R7JgSIsIS8iEKBVjJ++XR20m3Q9z9xwRb3PazaHiHdITBYL+7Zu5XAnG1tNiXUr5W1jxIK55FwyjsMbN7GpsgiT14fZ68ccDOLqvYeu/Ydgi4lj78NvEK6IzALS6dA5ncRdkUir356HTOzMwbUd0dls6GOc6Jwx6F0OLN50bIBsMwjf2E/QmXXozVrsF2EPpPbS1pfYRTsbqD4QBKq0MwhLrFbud0PpAQhUasur6/W4SgvoOz6FTx+ov9Pv3ghxbeCrJ2DpXyPDSY5IwLfBTR9r29jwLuxaXP+AMfBW7TrE4c3gPlz/gOE4tXshFOVc12wCuj3U8HJrWIvyBfMP0Kk8BAVHWffN0Wj5xdO706lPK8T2Cna8u5P4FBuxyTYSbQbMdiNl6VXEJtmY0rsNRQfdWOxGzDYDZpsRs73m67nhg5eiPwfKyqncs48UsxmAcGUl2a3sdD+0D92Bmhzv/jZTEWIEnXv0Zsw/X6c8oSOlsckUxSWxJiYZ0bsjGQguvv23zPl4KfaqI1i8pZj8VXjjnCT27Upq5yxevmt6nc+sI0xWbCH6+GSSMtqzbuVajrTOo/iIA7/DQTjynVB0EMemDRg69uK9fiMxlW5HLwwYhA6zFCSVlXDBkcNUCCsHNm1D563C5nZj8FYBYLlpKpm9e1Ne4ubQ9ce0wW4n8Y7bSZgyhaCIJ//fW9E5neidTnQxSeidTuxHg1gSINzuAnxD2qJzONDHxKB3OhG1Zyz1uhbaj4gcCNxawA9UgT1yAGs7EM6/PXIgqKw5MOi175+SPdpZQPXykB+EDs77pVb+/fOw9t91f3FMTnggT/v5o1/B9kV1Dxix6XDVHK18xQtQsrfuBWtninYNA7QL2qHAMQcMu3Z2cZaou6p/nppNQHe4zLiLfQ0uBxj7P9nsWX+UvL1lFOW5wRcGoFXbGACWzdUuiJYUVFFSUBV9f6d+rYhNgm/e3sHudYX11j95xgAS0hzsWHWYjUsP1gr4Fsw2Iz0ygpgdDtL/+RKeigAmXRBd6RE4chBLRjsAQsUltC3cR+CHZRCqOTKl/ukRDP2vpEtGJr9LXI6p7WBMbdtibNsWY2oqQq/H76li3K//wMG8Q5QdOUxl4SGqjh7G3HcU3fv2wheWLP37Q0AsQkjsxiA2nZ9+rgME2w/DlZbOzs0/cLn1HXYEE6kIWakImvAEDezeuZ6BpcXslfG8l9WJ9MLdhInBo7fg1VnJLD/K1UcOs/yQjzf6XU1csBRjMIwpGMIWCHCJM4UEYNOCOVh/+BKdP0QwoCcU0KGTEpPOTIcuXVj7zVpsd9SdyRo2mkh89M+0umwM+zbso/zpp9A5HBhiYjDExmCMcZLsPIIlox3B2Gz8Lid6pwNd9QHBakVUD8MMu0/7Vy0U0A4I1eVDfw09r6k5c/BXggzX1G9znnbaUfuAEa51o/Oer2DnF9ryaq2yawL63Dvh4DEPa0nvDzd/rv3874lQdrDuASO9PwyPtPnbp7WDUPTsw64NN6X11cqP7tAuZFcfMAyWEw5Bbfn6SxbN/gdBv/b3UnG0kEWz/wGggnoL12wC+qDxHfjyv1sJ+mv+EA0mHYPGdwAgrUs8aV3i67ynwhvAaTESCkveifWjrwzRJcZG1zgbqUYjiQk2WmVqAT9/V2mD27XHaQeM9YvzKNhdVq+8+9DWAGz+5hCrF9R+kIjAZDnIlMczMbfPxDtjDgc2H8Uo/RgDleirSvHrM4mVEv/efeS//QnhYAhDsApD0IswGmj70ovYBwygbWwCiVt3YMzqhuniURjT09GZTAB43W4uv/9hyguPUH70SOT/QlzXPUOrtulsWrGG757/O5AFgMOqw2kNc1WnEmIGTqLM5iB171p+afqE0mQL/pABd9CEWzpwu8PoDHp6pgj8HfazbWuhdieCHjDDd/NeoWvrI2TveYI951s55InBbvBj1IdZaB3NxX16I6Vkuy6WeUNuxuipwh7wYA94sQc8TElNB2Dphjycuw/hiCx3BDwYwyFKs3uQktGOt2d/QO9XnqjzvYeEjnb//Q/OPr1Z8MK7GD94i5DVRtjuQNodCIeTi+7/JQaXi7U7PZTuqcAUG4MprjXmuBhs8bFEriDgy7kWQ6/r0euOEyQnRXr34TAEPVrgrx3wR/0F3EfqHjDs2nBOOBQilNCNkLAT8noIVVZhoQBTbD5+TxWF+/cRWvQKocoSQlIQkjpaW8tx9r6MsmF/ZufK7wktmkkoGCQkBUGpIyf+MK7B15Hf+RZWz/+I0PYvCKMniJ6Q1FFaKev8nQAE/T6+fm0WXRMqtANGYieIa6vNoKosVFNlW4hmE9A7D0wBYNlHu3AX+3C4zAwa3yG6vCFOi3bKq9cJXr1nCAs3FjBvQz6fHDgCwH3dujAMkFIy6X8HUFHsxV3sw13ixV3iIybRisWuraP4kLuBNrXC4jAipeTIvgqS2zkxWvQYTXqMZj1GiwGDURvj91YGKCnw4KsK4q2ShIMxGPNL6TlR4Bg6hCO3PseOVUcia5YYdSHiPg1x9QCoWrGCla8tp8q2B2OgCkOwCrPdSNt7bqXD8M6kmqzYPXoyu/bDNrYdBqcj2sb2ffpz1Yw/1wr2R6g4egTzzfdhiItnx/tv8e1b/0ZLmAkWq4kYu5ErcxOw9r2aguJiKnauJMfzJT3aSkJhgTtoojJoorLtxViWPc62YgefF3TEH9ZTczfvDj577H6y//sh1+QYabtmN3n55djsNiwOK2a7i4qy/UBPLrwkmyOd7sAnzHikibywHq9PMmaANjMnZdggluj/F9xuqKxAV+lG76mkc5p2MC0s8xDrrsJcXIQ14MHm9+AM+AjfNYWiKjcrX3qF7JWLCAuBRwhkIIjeF8S/9Eu2b17PZ6/PI2b/DvwGI0GDgZDOgS22FTc98wBfv/0fNmzai8/tRuh1oNMRysihVf/hTOpp4oPH/0hphYdQMIAIB5GhIGkjL6fbiItJ3r+X1+67/ZjfGj2X3PYg2bkXUrhtC28+eB9a/ru0aI2x026ky8DzKNl3kCWvvRAtE0Kg1wsyz7sQV/vheCvdHNm9A53PjF6E0YsgBsJUeg00dFd1Rbkb3r5Be3HBDBj2GyjLg6dzaioZLFpwv+hh6HOjNtQ0947I/RW1psTmXK1d8HYfge0LI1Nia51hJHTQrm+EgtpFdzVV9ifRbAI6aEH9RAH8RNq4bNwyrD23DGvPwVIPCzcWMDDTBcCa/SU88P5GLu2RyqU9UujVN7ne+6f931Cqyvy4i7VgX1HixZViRwiB3xukKM9NVbm/znv6jGqHEAKfJ0j+zjJSO8XhjDfjiLdgcRhxumrmvOdc0IY23RLwVQXwVQXxVQbQRQ4G8ddfh6jqTvGWUny+MFJqfxiHlxbTYTiUzf2YResTKY8xIOQhDCEvJhGkzfmdGDk1m2SDiUN7HGBLpVVHB+16Gjm8N0hMYgU9LxpNm+45lBYU4C4ppOJoIeVHCzFfMQN0eja88A/Wf74Q0J5ZYjSbiIuP4Yb/GYnIHMpXD3zBmpLOhGrdlKUjTF9XHq1G3UHe1k241y9kx/rNFPntRKfxAM6Ne/H5w5RtWsqaVdvrfHednEdZnX8NJeV+ynevpypfe59ZH6RDvJuLc0rAeg8v3D4Nd1EhhakCsAE2stqY6NM/Dn1CDP+ZPpWg38eyTunRdbe1m8m2WdA57Xw6S0vdXxon0PLJBWlXXEi3bSuRuhnsWLEMXVkFsT4vOinRSUm7dctJee1ZdN99jSPeReGeLdgqywkKPSFhYN/rH3F08Tpu+vtvOf/q63hv4Vq8Hj9evQmPwcL89/fQ88By/nx5d664/2H+tHAHPikwGU0YTUaeyY9n0FbJNf168KuX3mTWN/swGI3YzEasJj1rjXoqYhxkp8WS8dRsNhwsw2LUpupajHre/92tuIvqDx86XS649RvtDCNGOxhijYMxT0aGmmqdYcRnauWhgPbPc7BWnSrt/om0PlC4TQv4x5r8BmRdCru+gNevrpkqW32N4fLZkN4P9n2nzYA69oDR9ybtOkXxbu0axbEHjNj0s3qN4lwlpDw2ceJPo1+/fvJceUj08t1FPLFoG6v2lSAldEp2MLpHKrcMzYz28hsjFAxTWar18CuKfbhS7SS1dVJe5GHBrA24S3x43TX5B4Zf05ns4ekU51fy2cubcMRbcMSbI/8spHWOxxFvrrMNKSUBXwhfVZBwKExsko2Q2832L7ZRur8Iz9EKvGWV+LyS1FFDOG9CB/LuvocvjvbGbW9NzV1NkNkzkUtvy8G3ezf/fe4APl8Ys82IxWbAbDOQ2TOJboMTKDtSwJpPNxLwlhLwliBlgPOvnk5MgoUXb7u8TjCv5jQFmf7vhbz98P0c2LyhTllSm3Qm3X0HPr2Tec8/y9H9e/B7vXXquBJiiGmdgTXWReXhfezfUTfXm8OqJ7P/UJyt0ggeXMeGVZsw6iQxlhBtY310TyzHfvtidm/chFj/Fvptn6AngF5IHAYfsSYfPFRKWeER9F8+jH7jW5FerkSHBKMd8b9aDrrAi9cTWPMpoYAgHNATCpmQpjgSXvoBgMP3jMezcTchH4T9EPaFETYznb9dA8DmMRcgduXX/V1JdpH91bcALJkwHktxMT6TGa/RTJXRgrlTJy79v8eQUnLbtJmEwpIyo4PSyL/xw7rzhyv74vGH6Prgwjrrnlz5MamF+wjKmuETgwjhbduJZe0mYTXquf68dozJSeWo28ffP9sePRhYTdr/Qzom0iXFSbk3wOp9JVgjBwyrSfs/0WHGatJD0KfNIPJX1Z3FlNYHHMlQtAs2fVD/gJF7PyR1hm0L4POZdWdIBb1w23fQqjssnw0L7qOeO9eCqz1885Q2A6o60FdfuL7+XbDGw+aPtBlQxx4w+k0DvQEKt0PlkZrl1XWs8fW32RTWvw1f/FE7M4pNh5EPamc7p0AIsVpK2a+hsmbVQz9TBrZP4J1bz+dwuZeFGwuYvyGf15bt5fYR2ijrNzuOEm830i01puZCXAP0Bh0xiVZiEq11lsckWJn0hwEABP0h3CU+3KU+YpO0ejIssceZqSjykr+zFF+VNj475pc5OOLN7NtUxBevbK4J+C4LjjgznQdEzlbMNjqP7Y1e3/BNPcn33sPlO3fh27cPz75DVB4sJGiy0XqCNlUw//4HSCtKIGCNJxyXSMjhIuRMRMpELA4HosLJ7vUxIGMALY/9B0+soefINg0Gc4AKv4Gdq49w8a134av0smdDMRa7FavTgi3GRpXBgT3OzDWP/K0RewgKdu2g/OgRKkuKqSwtwV1STGxKawZOvBq4hvU3X0t5RTnlXgN5pQa+22cn68XZjLnzPhhwPh89kYTZZsMeE4M9xonDaScx/xCu1mlwye9h8DQtkAR92r9aF02Nw27E2LW/dgNV0Kvd0Wuo2cetJg6BAbbI+yN1YlKj5R3HxBHcuYewN0jIEyAcEOiSYqPl3ez78ZaVEvboCJULwn4dltBe4DGEEPxm09sEjkl1Z90dA1cux6AXfLHsd8hACGHWgUmHWeemIN7KGtGaiqAZh/DT35RHatUuisoGU6azICsTACj3BPh0UwEef4iqQIjq/t1jl/egS4qT3YWVTJ2zst7+eOaa3lzWszXf7atgysubMBt10YBvMdh4ZIKBAQ5YV5XA7AMj6pxBWGP0TNKn0wbYlziMNee/h8WgxxI5WFj10DkuBivgzZpIqFVfzNKLIeSpOWA4Is/Pad0b+v9P3emw/kptiAe0C8rbFtSUVWf+7h+5SP/987B6Tt0PZ7DA/0buafnkXm1IqfYBw5kKV0Zmva1+VTuLqH2Xtj0JssZo5YXbIRzQlu/4Ahb9QbsOA1B2AD6+U/v5FIP68aiAXkurGAs3nZ/BTednUOkLYjJowerhjzex44ibjAQbo3ukMqZHKt1bnzi4H4/BpCeulY24VjWJxBLSHIz9Vc/o64AvhLvEG70ga3UYycxJxF3qo6zQw8FtJfi9Idp0c+GIN7NlWT5L39iGPcaEPd4SHdbpe2k7rA4TwbgURO9kXLm56Bq48Jf063uJ2bmTwP4D+A/sJLB/H+ZOnUgbdSkA+ydPYkRRMbTJRKS1R6a0RZ/Vg8TBqWxa4MBdWf/6gk7vxFcVIK5VGqWHq1i9YDdQWqfO0EmdyBnRhuJDlXz87Lqa6aKRKaXdh7YmJTOWqnI/5cVOzA4XscnV5VrdarfOeo2q8lIqS0qoLC2msqSEmETtwmQoGMRdXETB7p1UlZYQjsw06jfucoZfPw2fKZEXZtyHPS4ee7wr+n9HNpLeNZtQ+xGU2rviiHdhstrq7/dhDfQgazH98n1M1S+k1C6ohmrO1JKfm6sFm+qDSdALZme0vO1ffkOo6DBht5tQRQVhdxWGdG0ygFGvw5aZRrC0grAnQMgTxOM2kG6soF+flUgJW99OBSkoRc9UZgIQH7weBv2BjFgTb3z2aPT+A+x2cDhxFhuBtnSIMfBRuyP4LXZ8FhsekxWPyUYvl/bdp8ZauXloZvT+Do8/hCcQwmbSzg4qvAF2HHbXK7+gazJtXDaW7ynmt++ur/edLbx7KFkpRt7aVMlDcwsin1VgMViwmOx8lKGjtQneK+nAO3tN0QOG1ajH4tTzB2nGDnyfPpWN4nLtzMKgw6YP4hA+BqFDDxT1/AXBNqMxSw9m6cUU8qCn1gXl1r20A3j0gFEZOTBE7PxcC/ihWsOtiV1qAvrcO+DA98f/5Qh4tB67Cuhnlt1c89W8Of08Pt10mAUb85n91W7+uWQX1w1sy6MTewDaMMiPCe7HYzTriU+xR18nt4sh+YaYOnX8niAGky5S7qTfpRm4S3xUlngpzq9k3+Zi+l2aAcCGJXmsmr8XoRPYY03RIZ2RU7piMOrxpXcnkJSFY4wZm9OEOCboJ915J/59+/Af2E9g3378674h1nQZCddfSOdDRax1mqndURdh6FmUT/ehachAAMPeDVz3q3SCZidBvRWfJ4SvKkBSWy1o6Y060rPitWsHVUHKCj349lWQmaPNQz+aV8FnL22u9z2N+VUOGT0SObC1mO/e21lruCgGsy2BjN7axcSq8iDDrp+B2W7AZNERDnsIeisw27XvWIbDdBt2AZUlxbhLisnfsTV6QEjvmk1pQQGv/Po2AAwmM/b4eBzxLs6bOImMXn2pKitl99pVOKoPCPEurM7jHPCF0MZ+a4//xrWpX68W08gTJy9NfeXTugv+ng1lR6IvO4w5QjggCJlSCI96hlB5BabIlFqCQexDBhMuryDkriBcXExo3z5Eb62DYXGXYXr6cUyAo9Ym7L//HUyZQuvyI0x45ObI/Qcx6B3a1FJX1rWQ1p9BcfBWqzz0MTGR+xRc6BxOjJEDwpgeqQzMdOGpFey9gRDp8VqHp2+7eP5wadeaA0Lk/9p/n2EJxZX+Wu8P84dLuwLw+ebDvPhN/Ucz7O7RBYD/Wx3k9eVBwBj558RpMbAhkmLpT4f68fXuDCwmPRaDDqtJT7LJzOOR9bzd/s/si/sDdr3EofPj0PtwWWtuSNzT+7fQqQBL2EvKknsbflpQWV6Dy38MFdAbIcFh5tqBbbl2YFtKKv0s2lxAuwQtGOwrquS6F5czOjuFS3uk0qtNXJMG9+MxWWt2XXK7GJLb1Q34ta+NdOiThD3OHJ294y7xUVJQiT5yBrJm0T62fa/1gnQ6gT3OTFyKjcvu7AVARbdcAh1COF0WXHFmLA4DIhiZtleViqFVXwL+7yFcATonBst5iMPa9ZHgkSMcuGlKTcMMBgzx8STfey9xAyYQLCzE+/IceiUmoE9JwJCYgN7lwtSuHXqHFkJSO8ZxzUMDIwE/EP0/IU0r1xt0OOLM+KqCFBdUaXUqg3Tq3woSYN/GIpa+vq3uFyjg2oe0Jyke2FJJRdkAzA4jqa20nr/Jqqfb0NTId2llyDW/IuAtx1dVhqeilKrSkmjemiN7d/PpP5+qs3qd3sDE384go1dfDu/eyfovFuKIT8AeH489zoUj3oUrvQ1GU91rJE1i5IPaqXzAgxBgcoTAaIVxD0HOyLrttNlo/eijx12VISWFjkuXEC4vJ1ThJlxRTqi8Aks3LWDqLGacI0cSqignXOEmVFFOID+fULk2RuTbtZvDj/yp3nrTn/sHzpEjkSu+x/eb32CIiSHO6cTldKKPcWK8807o1IlOnkJa7/lWu3s5xokuTis3iRBg5Iq+6VzRN73e+qv9bnQWd4zsFD078Aa1gF99pnp1vzb0aRsfuXtcq1P7qmLrOCuZiXbtgBMIUVLpJxSuqfHZlsMs3nqkzrJOyQ5ytRFW7ltuZdW+OAC+MSWSrjvKsQ6TyI+b6lGfCuinKN5uYlL/muehegIhOrdy8sp3e3nh6z20jrUwukcqvxjenmTn2cuVXvugkpjuJDHdedy6/UZn0KFPcnQGj7uk7gXK1Qv3cXBbzewUvUFH606xXHZXb3Z3moDeGIfe2qPOe3Z1assIQO9y0XbOywSLigkVHSVYVEyw6CjGVO1XOFBwmJI33kAec1G09RNPEDt2DFVr15L/wB8wJCSgT0jAkuDCnpBA7GWXYXJZCJWXk2goZdRNHdDZ7XU+d/VBrUPvJFyt7fgqA9GzAG9lAFts5E7fsIxe26g+YIRDkuxhWg9/+8pS1i4yoz2IKwmhE5htBtp003qx5cUuOg/+DUJ4QFYSDrkJBdzEpWgzSQ7vPci2Zd/iq6z7AJbr//IUrdp3ZMu3S1k5972aHn6cC3t8PN2GjsBssxPwehE6HQaTiUapPn0/zYtvAEKvx9iqFbRq+JnvxtatSf3jw8d9v61fXzp9+w2h8nJtyKi8nHBFBZZs7ffFkJxEzJgx2lBSRQWhigr8e/chIx0Gz7p1HP7LY/XWm/nRR1i6dKb49dcp/L8nozecVd+tnPqnRzAkJhJYt5bguh8wxTixOmPQObW7lWWqA2Ew0DMthl5t4o7b/mlDMpk2JPO45S/cqF2bDITC2tmBP0SoVmfqwXHdKHL78QRCPP7G1TxmfBGbqBmeqZIm/hK4iqePu4VTowL6acpKieHlKf0p8wT4Ysth5m/I540V+7njAu2C6vLdRQgh6NcuvsHx63PBsWP6x7rk5u41c/RLtf+rzxC8xtgG3+MzamcMOquVxatshIIWrM52WFubsHYxEnDEYAesPbJptfAbTCKA3lNGuKSYUHExlmxtmqTObMbcpQuhoiJ8O3dS9f1RQmVl2AcOxNSmDe6lX3HoPm0MW5jN6BNcGFwJtH78r5jbt8ezcRNVq1ZqB4HIQcHQ0YU+IQ0R6WF3GZhCl1rTYatnEhnN2jhw1/NTScmMxVtrSqnfG4qe4XjdYcoKzfiq9Pg8FmTYhdlmIC5F6+Ef2ulCmG7BbAwhRBVGkxerM0B8qhbw87a68XssHCk/TGD7TvyecpCSLucNAeC7995m1dy3sdgddcb4L55+BwaTiaP79+J1u7Xef7wLk8WqBe8mGpc9HcJgwJCQgCEhocFyS5cupDw447jvj504EeeFF9YE/PIKQhXlGNO0g62lUyfirryCUHkFYbdWHjhyGPTavnN//TVFs/5Vb71dVq9CGAwcefxvlLzxBroYJ/rqgO+Moc3sfyF0Oio+/xzfrt01OY6cDvSxsVh79QJA+v1gNGLU6zDqdcQcMysuJz0u+vOj8y7ixY0HuHjzKmQVCBss6taPVdkXncpXekKNCuhCiFHA02j3CL4opXzsmPJY4D9oUyAMwBNSyjn1VtSCxVqNXN4nncv7pOMNhLAYtV+oZxbv4NudRSQ7zYzKTmF0dioDMl3HvyvxHGR1mrA6TSS3q1/mcFkaTMlgddb0Js02AxVFXgr3V+B1a0Exa1AK7bITkFLy5p9WEA5KhACz3YjVYadryEfvi8Gc1ZWDI+/A6jBhdRqxOYxYrDp08drZj7V3b1r/9TGCR4sIFhcROlpEsLgYXSRtctXy7znytyfqta/jl4sxpqZS8s47lM+bj8Hl0oJ9QgL6BBdx48cjhIFQRQWxsQbiU46fyGvAuPYMGNceiBwMvCH83poUDzkj0mnT1RU9O/BVBTCY9JisWhtDoXZgGEsoEERnBrMpTEJrvfbgFGD/JjsGy/kEQ1WUF1dRXnQUne4g+kg+nE9n/5eCHcui2zOYLDgTk5n29+cB2PTVl1SWFOGo1ft3xCdgcdQeFT83Cb0efVwc+ri4Bstt/ftj69//uO9PuusuEm6+RRsqih4UyhGR3w/beQNBr6u5hlBegfT5ogf78kWLKJ/7cZ116l0uOn+nTTnNu/de3F8uqXMNwZSZSdr/ab9zJW+9TfBoIXpnDH87sB77mnXISK4lWQVD166n+8C80/qOajtpQBdC6IHngIvQHhi9UggxV0pZ+yrVr4DNUspxQogkYJsQ4r+RZ4z+7FQHc4B/3dCPxVuPMH99Pm+tPMBry/ZxYddkXrxJ+yVs6guqP7XjpWQYcmXNA6wu/p/udd4TCoUJBSL1JVx4Uzc8bj+eigCeCj8edwCzTfvV9HmCrJq/lzoDm0D/sZkMGJtJwJnIJ2taYXWkY3EYtTMAhxGLz0Yy4Lz+JoKDLsUYqMDoKUWWlhAsOhrtMQohkD4fnk0bCR0tIlxZCUIQN1F79OCRJ5+k9I03tUyUkYBvaNWK9KefAqDy++8JlZTUHAxcLoyxsXWucbTuFE/rTsef1zxqunY2Eg5L/B4t6MuwjAaV8yYOobyoL77KmmsIca1qZtsEQ30xOlpDuBIpK5HhSoSuZmz+85c+JOjdVWeb1pgkfvmC1ud6c+bfCHjLsMXEa8E+IYFWmW3o1F+7U7c5/44KIdA77OgddoypqfXKnbm5OHNzj/v+tMcfJ/WRRyLXELQDgvTXhLWY0aMxt++gXUOIHBSEoebvv3zePKpWrACgod8ASyhA2w9ehTtu/NGfsbbG9NAHADullLsBhBBvAuPRnh1aTQJOoe11B1CMdtvdz57DbOCynq25rGdrKn1Bvtx2BLtJ+9rLqgJc/NRSLshKZnR2KoM6JGA8zlzyc9WPScmg1+uic+aFTmgXL4/DYjdy23Mj8LoDeNx+vBUBPO4A8SmRISIJiekOPG4/ZYUeCvaU43UHiEm0ktwuhpL8Kt59Zkt0fUaLBasjk6HbysnokQhDLyVf3wtL5AzAYgKz9BAIgskAMZdcgjG1NaGiIoJFRYSKiwiV1eT0KX71NdxfflmnzcY2bej42SIAjjz1FIFDhzAkJGJIcKFPSMTUJh1bP23sVYZCiMjwgE4nsNiN0XQT1U70/QDc9JdRhENhfJ4gvsog3qpAdLgIYMg19+AuqcBdUkJVWTHeijKS2mjXVEKBMAW7iggH85FyC0jtbCsmqSOd+vfB5wny/M3/A9KH3ujEaHFissbSoW9Pcm+YgN8bZNW8ldjjY4lNSsDqtGKxG7DFmjGaWkZeGJ3ZjC4pCUNS/bO02DFjYMyY47633WuvIoNBQhUV7Dh/MDRwI2cwP7+Bd/44jQnoacCBWq/zgIHH1PkHMBc4BDiBSVLWTmenEUJMB6YDtG3b9tjiFs9uNjA2p3X0dYUvwIDMBOauO8QbKw4QZzNycbdW/DK3IxmJ9hOs6dxyOikZGkOnE9hiTNhi6l8UtMeZueSW7DrLZFhGL4jGJlkZ9YtsPBUBvNVnAe5AdEio7KiHdZ8fIByq+4d22d1xtMlycdjSgW/2hbE6jFgzTFizjVicJhJKvDjiLcT8fiZy8m2Y/BUYqkoQpUV1emjB/AI8q9cQLCqKXvi19upFxptvALDn8isI5OdrQz6JCRhcCdj69sF1000AuL/5Fp3NGh0S0jkcDfaWdXqdNizliHyujz9mx/VPEczPJyY1lY733E3sNePqvU9v1HHHnD9Hh4KqyqooPXIUW0ytM4wuA6kqK8RXVUbAW47XfZCCXQZgAu4SH8vefhyIzKsXFoSw0+m8XMbdNZWjeRW899jLmG2xWBxx0bOAHrkZJLeLobLMR/7OMsx2A5Za9yCYLPpme1ZwLBGZ2WVITSV46FC9ckMDZw4/VmMCeoNTJ495fQmwDrgA6AB8JoT4WkpZ5/42KeVsYDZot/6fcmtbmPR4G89e0xtvIMTS7YUs2JDP/A0F3JarXVD94UApxZV+BndMjN7kpJyc0AlE5NfWYjfSoXf93DzV2nVP4NZ/5OL3hvBU+LUzgQo/SZFZQVaHkdYd4/BU+Kkq91N00I3HHaDbYO2PcM8OL9++W503xYre0Bar08gVJT4c8Wb81/2Ggr4lWBwGLGYwSQ86syAcCqPT64i7fCL+/Qe03n9REb5du9DF1MxIOvTrX9c5IxAmE3FXXhm9kJg/cyZ6p7NmOCghAe/OnRQ++ffoASR46BD5Mx4EIHZcA0HdoIseMONT7KR1qemJmq0GJj34y3rvqTlgWhj9q99QVniUiiJtHn9VWQmJadoAQyjgwV34GcfeeuZ3T2T0r/6Hg9sKmPfMMwidA6GzI4QddHZG3Tqczv0z2b+5iOUf7cZc+zkFNgM9hqfhiLdQUeylvNATvdHMbDNgNJ+bB4Pke+4mf8aDdWZ0CYuF5HvubrJtNCag5wG173xIR+uJ1zYVeExqe3mnEGIPWr7WFU3SyhbOYtRzSfcULumegj8YjgbvV5ft5f01B3FaDFzUtRWje6QytFNinTF65fQJITBbDZitBjgm9qd2jCO1Y1ydZXXn+CcT18oW6flHhoQq/NFrAEcPutn87SECvrpPaLn1vC6ghy3289nuPow1wYQ1w6idCcSYSI2MW9sen423sBSTrwxDRRG68kKsXbU54NLvx71kKcGiIgjU3Hmqs9vJj8lmV6/L8EWev9th91z0f3+qwYD+Y78zAL1BT7dhg49bLzkjidte+C+VpSV10jakddE+Q3yKHntMKVXlewj6asamSw/FA5mUHz1I/pZnEAYHQtiR2AiHbLRqdzmO+M7sXH2Ib9/ZDKImP75OJ7j24fOITbKyfUUB25YfxmwzaDedRQ4M3YelYTTpqSj24vcEow+0MRh1Z+xgEDtuHHsO6lm90ovXEIslWEbf/hZix13aZNtoTEBfCXQSQmQCB4HJwLXH1NkPjAS+FkK0AroAu5uslT8jtXvij12ew7ic1szfkM+izYd5f+1BstNi+OSOoYB2Ee1cnQrZktX+g3e6LHWyZh6r3+gM+o3OIOgP4XEH8Eb+6Y3Vd/nG4PMEo0NCR/Z5kVIy9OrOAGzYKti9NgjYATtCtMXlszN5nNZbr7j/FSqKvJhNYcz4MYWr2P3SO+xvexHhyBOdfJYEtna5Fra9QafjtPNMEUJgi4nFFhNLUtuMeuVJbdOZ/vxLSCnxezyRtA3F0Tn8rTu66NAvp9YB4QBBTyUyfAEAZks+vrJZCJ0ek9WJyRqDwRxDVVk7YpPaU150hOK8DYRCFoIBK36fCYGe7kO1aY8/fHGAH76oGVHWG3SY7QZu+stgdDrBxqV5FOwuj5wdaAcEq9NI5/7aEGNlmXbNwWIzRvfp8WxfXsDyzTaCRu33xWuMY/lmHbblBU02ZHnSgC6lDAohbgc+RZu2+LKUcpMQ4tZI+SzgEeAVIcQGtCGa30kp698SpZwSk0HHiKxkRmQl8+dQmO92FeHxa9eaA6EwuX9bQp928VyanUJul2Qt+51yTjKY9Dhd+nrBv1P/Vie86DloQgeyh6bVzAJy+6Pz3wEKD1SQt6UEb2VND120G4XU1f3TDuvN7O40gXP1eUVCCMw2G2abDVfrmjs/Xa3TtQRrtQT8PnSRGUBpXTowYsovovl7qg8I1ReFjYY8ju59s/aGsDqcVBR3wZWaRlxyEW26bENvdKI32BE6Bzq9GQgDeiqKfRzaWRq99wCoE9CXvr6NPT9ooc5g0mG2GXG1tkfvsl73+X7cJT7MNgM/fHGggQePhFn20a6fLqADSCnnA/OPWTar1s+HgIubpEVKg4x6HcM714xtVvqCDOucxKebCvj4h0NYjXouyErmttwOZKc1fLOP0vyc/KavyJTHUBhvZRBPhZ83H1neYN3j3QTW3NROlxCbnEKf0ccfRupy/jBaZXbEHe3ha4HfHhsHQGXJAXauWIAM1w20Qye/icXuwGTaiNWynMQUF7bYeCz2WEz2WGQ4jNDpyB6WQpuu8fiqQtEppSZLTVg9uL2Ug9tKokNuQd8Wgt5vaqXJGIK7uGuTfTfqTtFmKs5m4i+X9+CR8d1ZsbeY+RvyWbjxMDednwHAtoIKthaUM7JrKxxmtZtbOp2+5sLm8W72cpxgaKilstgdpHTsfNzygROuYsBlV+CpKMcdGeOvLC3BbNNmmQmdjoDfz8FtW6gsLSYUCGA0W+gzSru7c+Pi19ixchmOyA1b9jgXxpRUiAxuDRgbA2NjsDrj+M//voW36jOiM7rDFQSrPsPiMKLNJzl96i+9mTPodZzfIZHzOyTy8GXZ0SlJH647yD+X7MJk0Hr2Y3qkckHX5Hq3Jistz8mev6vUJXQ6bLFx2CK99tr6jB4XPQOQUuKrqsRTUTN5r0O/gdhiY3FHhnsK9+2h7Mhhhl6jTTtdPGcWeZs3nmDrQUKeb4jM5j79z6KeWNQyhcOS1ftLmLc+n4UbCygo95LkNLP8/pHodIJgKIyhmd3EpDTe9uUFp3Szl3JmHN69k9LD+VSWFPPlqy80XEkIfv3mxw2XNVhdPbHoZ0enE/TPcNE/w8WDY7ux9kAJeSUedDqBlJLRT39NWryVS3ukcnG3VsTZGpnJT2kWzvTNXkrjtGrfkVbttftKVs37kIqjDTzrNSGxybanumg/AzqdoG87F+N7aVO1fMEwI7KS2XnEzW/fXU+/P33ODS8t57udamJSS/Hh2oMMfmwxmb+fx+DHFvPh2oNnu0k/e0Mn34jhmPz3BpOZoZObJo8LqB76z5LFqOeBS7ty/+gsNhwsY/4G7TmqJVXa1LcDxVV8veMoF3dvRaLjDDyAQTmjPlx7kPvf34AnoM2sOFjq4f73tQd1T4g8xUn56XUdqk0a/frN16goOoozIZGhk2+MLm8KagxdAbQLPmEJep3g1e/28tDcTegEnNc+gdE9Urmke6uz+sAOpYaUEn8ojC8YxhfJWpnk1A682woquO7F7znqrp/oNC3Oyre/b5rZFMrZo8bQlZMSQqCPTJG5cVA7BmS6WLAhn3kb8pnx4UYe+WQza2dchN1sqJOeALQe4d8+3cahUg+t46zcd0mXFtsTDIclvmAYfzCMLxjSgmowRIckLWnWrkI3eSUefIFQtF4oLLm6v5Y9Y+HGAjYcLI28XwvIJoOORyZo88mfXLSN73cXR9ftD4ZJdJp5+xeDALjx5RV8tb3uOGy31Bjm36XdPXzfuz80GMwBDpV6ztTXopwjVEBX6hFC0DU1hq6pMdxzUWd2HHGz8WBZ9MG817+0XLuwmp2KTgd/XbDtJzu9D4TCVHiDWsALhLWeaiBMZpIdh9nAoVIP6/PKagVbLShO6NWaBIeZVXuLmb+hAF8wVBNUgyEendiDRIeZd1fn8ep3e2veH9nG4l8PJ85m4m+LtvHPJbvqtWvbn0ZhNuh57bu9vLpsX50yg05EA/rnWw7z/po8LEY9JoMOs0FHgr1mWCskJTqddp+B2aDDbNTTyllTPjby3Fpz5L1mg67OsNhD47ox/bXVFFXWD+qt46yn/f0r5zYV0JUTEkLQuZWTzq20DIBSSoZ1SuST9fn88ZPNDb7HEwgxc+4myr0BfAEtYOZ2SSY7LZYDxVU8v2RXnR6oLxjmtuEdGNQhgbX7S/jNOz9EA3V1wJ11fV9yuySzeOsRfvHv1fW2+db08xjYPoHle4q4560f6pUPzHSR4DCz44ibd1YdwGzUYdJrAdOk1+GNHJBsJj1JTnM0WGpBVx99wtTwzknEWo2R92plJoMOfSS/y9TBmVzWK61WwNXXOZt5/Iocnriq53G/7/suyTrh/qg+MBxP33YuZoztVmcMHcBq1HPfJV1O+F6l+VNj6MqPtqvQzcj/W9qouo9MyOaG89qxtaCcG15aUStg6jEbdNxzUWeGd05ix+EKnvpiB+ZjAuZVfdPp1MrJgeIqFm89UifYmgw6+rWLJ95uorTKT36ZN9r7ra7jMBua1WP/TtfPaRjs5+ZEY+gqoCunZfBjiznYwNhsSoyZeXcOjfaAjXpxTuaoVpTm5kQBXc1DV07LfZd0wXpMfnarUc/vR3clwWHGYTZgMpy5HNOKotRQY+jKaak+jVen94py9qmArpy2Cb3TVABXlHOAGnJRFEVpIRoV0IUQo4QQ24QQO4UQv2+g/D4hxLrIv41CiJAQwtX0zVUURVGO56QBXQihB54DRgPdgGuEEN1q15FS/k1K2UtK2Qu4H1gqpSw+A+1VFEVRjqMxPfQBwE4p5W4ppR94Exh/gvrXAG80ReMURVGUxmtMQE8DDtR6nRdZVo8QwgaMAt47Tvl0IcQqIcSqwsL6eYEVRVGUH68xAb2hCcTHuxtpHPDt8YZbpJSzpZT9pJT9kpKSGqqiKIqi/EiNCeh5QO0EEunAoePUnYwablEURTkrGhPQVwKdhBCZQggTWtCee2wlIUQsMBz4qGmbqCiKojTGSW8sklIGhRC3A58CeuBlKeUmIcStkfJZkaoTgUVSysoz1lpFURTluFRyLkVRlGZEPbFIUZTTEggEyMvLw+v1nu2m/GxYLBbS09MxGo2Nfo8K6IqinFReXh5Op5OMjAyVOfMnIKWkqKiIvLw8MjMzG/0+lctFUZST8nq9JCQkqGD+ExFCkJCQcMpnRCqgK4rSKCqY/7R+zPetArqiKM3GM888Q9euXbnuuusaVT83N5ef0+QLNYauKEqTO1PPNH3++edZsGDBKY0r/5yoHrqiKE3qw7UHuf/9DRws9SCBg6Ue7n9/Ax+uPXha67311lvZvXs3l112GY8++ijTpk2jf//+9O7dm48+0u5n9Hg8TJ48mZycHCZNmoTHU/O829tuu41+/frRvXt3HnroIQAWLFjA1VdfHa2zZMkSxo0bB8BLL71E586dyc3N5ZZbbuH2228/rfb/FFQPXVGUUzbpX8vqLRubk8oNgzJ4fOFWPIFQnTJPIMTMjzcxoXcaxZV+bvvP6jrlb/1i0Em3OWvWLBYuXMiXX37Jk08+yQUXXMDLL79MaWkpAwYM4MILL+Rf//oXNpuN9evXs379evr06RN9/6OPPorL5SIUCjFy5EjWr1/PRRddxC9+8QsqKyux2+289dZbTJo0iUOHDvHII4+wZs0anE4nF1xwAT179vyR39ZPR/XQFUVpUvllDc/MKK0KNNk2Fi1axGOPPUavXr3Izc3F6/Wyf/9+vvrqK66//noAcnJyyMnJib7n7bffpk+fPvTu3ZtNmzaxefNmDAYDo0aN4uOPPyYYDDJv3jzGjx/PihUrGD58OC6XC6PRyFVXXdVkbT+TVA9dUZRTdqIedes4KwdLPfWWp8VZAXDZTY3qkZ+IlJL33nuPLl261CtraHbInj17eOKJJ1i5ciXx8fFMmTIlOiVw0qRJPPfcc7hcLvr374/T6eRs3UF/ulQPXVGUJnXfJV2wGvV1llmNeu67pH7w/bEuueQSnn322WjgXbt2LQDDhg3jv//9LwAbN25k/fr1AJSXl2O324mNjeXw4cMsWLAguq7c3FzWrFnDCy+8wKRJkwAYMGAAS5cupaSkhGAwyHvvNfiIh3OO6qEritKkqmeznIlZLtVmzJjB3XffTU5ODlJKMjIy+OSTT7jtttuYOnUqOTk59OrViwEDBgDQs2dPevfuTffu3Wnfvj2DBw+Orkuv1zN27FheeeUVXn31VQDS0tJ44IEHGDhwIK1bt6Zbt27ExsY2WfvPFJWcS1GUk9qyZQtdu3Y92834SbndbhwOB8FgkIkTJzJt2jQmTpz4k7ahoe/9RMm51JCLoihKA2bOnEmvXr3Izs4mMzOTCRMmnO0mnZQaclEURWnAE088cbabcMpUD11RFKWFaFRAF0KMEkJsE0LsFEL8/jh1coUQ64QQm4QQS5u2mYqiKMrJnHTIRQihB54DLkJ7YPRKIcRcKeXmWnXigOeBUVLK/UKI5DPUXkVRFOU4GtNDHwDslFLullL6gTeB8cfUuRZ4X0q5H0BKeaRpm6koiqKcTGMCehpwoNbrvMiy2joD8UKIJUKI1UKIGxtakRBiuhBilRBiVWFh4Y9rsaIoP1tnOn3ukiVLGDt27I9t3lnXmFkuDWVZP3byugHoC4wErMAyIcT3Usrtdd4k5WxgNmjz0E+9uYqiNAvr34Yv/ghleRCbDiMfhJyrT/6+k1Dpc0+sMQE9D2hT63U6cKiBOkellJVApRDiK6AnsB1FUX5e1r8NH98JgUg+l7ID2ms4raBeO33u5MmT2bVrFxs2bCAYDDJz5kzGjx+Px+Nh6tSpbN68ma5du9ZLn7ty5Uo8Hg9XXnklDz/8MAALFy7k7rvvJjExsU52xhUrVnD33Xfj8XiwWq3MmTOHLl268Morr/Dhhx8SCoXYuHEjv/71r/H7/fz73//GbDYzf/58XC7Xj/6cp6MxAX0l0EkIkQkcBCajjZnX9hHwDyGEATABA4G/N2VDFUU5h8wZU39Z9wkw4Bb4/OGaYF4t4IEFv9MCemURvH3MqOzUeSfd5JlIn9u5c2duueUWFi9eTMeOHaO5XACysrL46quvMBgMfP755zzwwAPRnC4bN25k7dq1eL1eOnbsyF//+lfWrl3LPffcw2uvvcbdd9/d2G+ySZ00oEspg0KI24FPAT3wspRykxDi1kj5LCnlFiHEQmA9EAZelFJuPJMNVxTlHFV+nAdZeIqbbBOLFi1i7ty50Zt/aqfPvfNO7WygofS5s2fPJhgMkp+fz+bNmwmHw2RmZtKpUycArr/+embPng1AWVkZN910Ezt27EAIQSBQk/53xIgROJ1OnE4nsbGx0Ydi9OjRI5oQ7Gxo1J2iUsr5wPxjls065vXfgL81XdMURTlnnahHHZuuDbPUWx4ZubUnNKpHfiJNmT73eA9jnjFjBiNGjOCDDz5g79695ObmRsvMZnP0Z51OF32t0+kIBoOn89FOi7pTVFGUpjXyQTBa6y4zWrXlTaSp0udmZWWxZ88edu3aBcAbb7wR3UZZWRlpadqEvldeeaXJ2n4mqYCuKErTyrkaxj0T6ZEL7f9xzzTJLJdqM2bMIBAIkJOTQ3Z2NjNmzAC0C59ut5ucnBwef/zxBtPnTps2LZo+12KxMHv2bMaMGcOQIUNo165ddBu//e1vuf/++xk8eDChUKh+I85BKn2uoign9XNMn3suUOlzFUVRfqZUQFcURWkhVEBXFEVpIVRAVxRFaSFUQFcURWkhVEBXFEVpIVRAVxSl2Wiq9LmvvPIKt99+e1M376xTD4lWFKXJzds9j6fXPE1BZQEp9hTu6nMXY9o3kNDrFKn0uSemeuiKojSpebvnMfO7meRX5iOR5FfmM/O7mczbfXr5W2qnz3300UeZNm0a/fv3p3fv3nz00UcAeDweJk+eTE5ODpMmTaqTPnfOnDl07tyZ4cOH8+233wJQUVFBZmZmNPFWeXk5GRkZBAIBcnNz+d3vfseAAQPo3LkzX3/99Wm1/6egeuiKopyyqQun1lt2ScYlTM6azFOrn8Ib8tYp84a8/GXFXxjTfgwl3hLuXXJvnfI5o+acdJunkz43Pz+fhx56iNWrVxMbG8uIESPo3bs3TqeT3Nxc5s2bx4QJE3jzzTe54oorMBqNAASDQVasWMH8+fN5+OGH+fzzz3/sV/aTUD10RVGa1OGqww0uL/OVNdk2Fi1axGOPPUavXr3Izc2tkz73+uuvB+qmz12+fDm5ubkkJSVhMpnq5D2/+eabmTNHO6DMmTOHqVNrDlaXX345AH379mXv3r1N1v4zRfXQFUU5ZSfqUafYU8ivzK+3PNWeCkC8Jb5RPfITOdX0uSdaPnjwYPbu3cvSpUsJhUJkZ2dHy6rT4ur1+rOaFrexVA9dUZQmdVefu7DoLXWWWfQW7upzV5Nt41TT5w4cOJAlS5ZQVFREIBDgnXfeqbO+G2+8kWuuuaZO77w5alRAF0KMEkJsE0LsFEL8voHyXCFEmRBiXeRf0yU+VhSlWRnTfgwzz59Jqj0VgSDVnsrM82c2ySyXaqeaPjc1NZWZM2cyaNAgLrzwwjqPpgO47rrrKCkp4ZprrmmyNp4NJ02fK4TQoz3s+SK0h0GvBK6RUm6uVScX+I2UcmxjN6zS5ypK89HS0+e+++67fPTRR/z73/8+202p41TT5zZmDH0AsFNKuTuysjeB8cDmE75LURSlGbjjjjtYsGAB8+fPP3nlc1xjAnoaUPsBgXnAwAbqDRJC/AAcQuutbzq2ghBiOjAdoG3btqfeWkVRlCb27LPPnu0mNJnGjKE3dGn42HGaNUA7KWVP4Fngw4ZWJKWcLaXsJ6Xsl5SUdEoNVRRFUU6sMQE9D2hT63U6Wi88SkpZLqV0R36eDxiFEIlN1kpFURTlpBoT0FcCnYQQmUIIEzAZmFu7ghAiRUQmeQohBkTWW9TUjVUURVGO76Rj6FLKoBDiduBTQA+8LKXcJIS4NVI+C7gSuE0IEQQ8wGR5tp4+rSiK8jPVqHnoUsr5UsrOUsoOUspHI8tmRYI5Usp/SCm7Syl7SinPk1J+dyYbrSjKz1NTpc9tqdSt/4qiNLmyjz/myN+fIpifjyE1leR77iZ23LjTXq9Kn3ti6tZ/RVGaVNnHH5M/40GChw6BlAQPHSJ/xoOUffzxaa33dNPn3nbbbfTr14/u3bvz0EMPAbBgwQKuvvrqaJ0lS5YwLnLgeemll+jcuTO5ubnccsst0QdiTJkyhdtuu40RI0bQvn17li5dyrRp0+jatStTpkw5rc94ulQPXVGUU7bvhhvrLXOOHoXr2ms58uTfkd666XOl10vBo38mdtw4giUlHLyzbl6Xdv9+7aTbPJ30uQCPPvooLpeLUCjEyJEjWb9+PRdddBG/+MUvqKysxG6389ZbbzFp0iQOHTrEI488wpo1a3A6nVxwwQX07Nkzuq6SkhIWL17M3LlzGTduHN9++y0vvvgi/fv3Z926dfTq1esUv9GmoXroiqI0qWBBQYPLw6WlTbaNU02fC/D222/Tp08fevfuzaZNm9i8eTMGg4FRo0bx8ccfEwwGmTdvHuPHj2fFihUMHz4cl8uF0WjkqquuqrP9cePGIYSgR48etGrVih49eqDT6ejevftZTbOreuiKopyyE/WoDamp2nDLsctbt9b+j49vVI/8RE41fe6ePXt44oknWLlyJfHx8UyZMgVv5Cxi0qRJPPfcc7hcLvr374/T6eRkk/Sq0+rqdLroz9Wvz2aaXdVDVxSlSSXfczfCUjd9rrBYSL7n7ibbxqmmzy0vL8dutxMbG8vhw4dZsGBBdF25ubmsWbOGF154IfrgiwEDBrB06VJKSkoIBoO89957Tdb2M0n10BVFaVLVs1nOxCyXajNmzODuu+8mJycHKSUZGRl88skn3HbbbUydOpWcnBx69eoVTZ/bs2dPevfuTffu3Wnfvj2DBw+Orkuv1zN27FheeeUVXn31VQDS0tJ44IEHGDhwIK1bt6Zbt27ExsY2WfvPlJOmzz1TVPpcRWk+Wnr63Ia43W4cDgfBYJCJEycybdo0Jk6c+JO24VTT56ohF0VRlAbMnDmTXr16kZ2dTWZmJhMmTDjbTTopNeSiKIrSgCeeeOJsN+GUqR66oihKC6ECuqIoSguhArqiKEoLoQK6oihKC6ECuqIozcaZTp+7ZMkSxo4d+2Ob96Ps3buX7OzsJlmXmuWiKEqT2768gGUf7cJd7MPhMjNofAc6D0w57fU2p/S5oVAIvV7/k26zUT10IcQoIcQ2IcROIcTvT1CvvxAiJIS4sumaqChKc7J9eQFf/ncr7mIfAO5iH1/+dyvblzectKuxzkT6XICFCxeSlZXFkCFDeP/996PLV6xYwfnnn0/v3r05//zz2bZtGwBVVVVcffXV0W0MHDgwehbgcDh48MEHGThwIMuWLeOPf/wj/fv3Jzs7m+nTp0dTFaxevZqePXsyaNAgnnvuudP6Xmo7aQ9dCKEHngMuQntg9EohxFwp5eYG6v0V7VF1iqK0YB/835p6yzr2TaZHbjrLPtxF0B+uUxb0h/nq7e10HpiCx+1n4b821imf+Os+nMyZSJ/buXNnbrnlFhYvXkzHjh2juVwAsrKy+OqrrzAYDHz++ec88MADvPfeezz//PPEx8ezfv16Nm7cWCdVbmVlJdnZ2fzxj38EoFu3bjz44IMA3HDDDXzyySeMGzeOqVOn8uyzzzJ8+HDuu+++k3/hjdSYHvoAYKeUcreU0g+8CYxvoN4dwHvAkSZrnaIozY67xNfgcl9l02UhbKr0uVu3biUzM5NOnTohhIi+F6CsrIyrrrqK7Oxs7rnnHjZt2gTAN998w+TJkwHIzs6usw29Xs8VV1wRff3ll18ycOBAevToweLFi9m0aRNlZWWUlpYyfPhwQAv0TaUxY+hpwIFar/OAgbUrCCHSgInABUD/JmudoijnpBP1qB0uc3S45djlAFaHqVE98hNpyvS5DdUHLQHYiBEj+OCDD9i7dy+5ubnRbR+PxWKJjpt7vV5++ctfsmrVKtq0acPMmTPxer1IKY+7zdPVmB56Q1s+9hM9BfxOShk64YqEmC6EWCWEWFVYWNjIJiqK0pwMGt8Bg6luaDGYdAwa36HJttFU6XOzsrLYs2cPu3btAuCNN96IbqOsrIy0tDQAXnnllejyIUOG8PbbbwOwefNmNmzY0GAbqw8YiYmJuN1u3n33XQDi4uKIjY3lm2++AYi2tyk0JqDnAW1qvU4Hjs1e3w94UwixF7gSeF4IMeHYFUkpZ0sp+0kp+yUlJf24FiuKck7rPDCFEddlRXvkDpeZEddlNcksl2ozZswgEAiQk5NDdnY2M2bMALQLn263m5ycHB5//PEG0+dOmzYtmj7XYrEwe/ZsxowZw5AhQ2jXrl10G7/97W+5//77GTx4MKFQTV/1l7/8JYWFheTk5PDXv/6VnJycBlPrxsXFccstt9CjRw8mTJhA//41gxdz5szhV7/6FYMGDcJqtTbZ93LS9LlCCAOwHRgJHARWAtdKKTcdp/4rwCdSyndPtF6VPldRmo+fY/rc4wmFQgQCASwWC7t27WLkyJFs374dk8nU5Ns61fS5Jx1Dl1IGhRC3o81e0QMvSyk3CSFujZTPOv1mK4qiNA9VVVWMGDGCQCCAlJJ//vOfZySY/xiNurFISjkfmH/MsgYDuZRyyuk3S1EU5dzkdDpP6e7Tn5K69V9RFKWFUAFdURSlhVABXVEUpYVQAV1RFKWFUAFdUZRm40ynz22MBx98kM8//7xJ19lUVPpcRVGa3Javv+TrN1+jougozoREhk6+ka5DR5z2es+F9LnVibfORaqHrihKk9ry9Zcsmv0PKo4WgpRUHC1k0ex/sOXrL09rvaeTPvell17innvuia7rhRde4N577wXgySefJDs7m+zsbJ566qlonUceeYSsrCwuuugirrnmGp544gkApkyZEr2N/1yjeuiKopyytx6u/1iELucNpdclY/j6jVcJ+usm5wr6fSx+ZTZdh46gqryMj//+lzrlkx567KTbPJ30udVB/vHHH8doNDJnzhz+9a9/sXr1aubMmcPy5cuRUjJw4ECGDx9OKBTivffeY+3atQSDQfr06UPfvn1P4xv7aaiArihKk6ooKmpwuddd0WTbWLRoEXPnzo32mmunz73zzjuBuulz7XY7F1xwAZ988gldu3YlEAjQo0cPnn76aSZOnIjdbgfg8ssv5+uvvyYcDjN+/PhonpVx48Y1WdvPJBXQFUU5ZSfqUTsTE7XhlnrLtYR8tpjYRvXIT+RU0+cC3Hzzzfz5z38mKyuLqVOnRtdzvPU3R2oMXVGUJjV08o0YTOY6ywwmM0Mn39hk2zjV9LkAAwcO5MCBA7z++utcc8010foffvghVVVVVFZW8sEHHzB06FCGDBnCxx9/jNfrxe12M2/evCZr+5mkeuiKojSp6tksZ2KWS7UZM2Zw9913k5OTg5SSjIwMPvnkE2677TamTp1KTk4OvXr1iqbPrXb11Vezbt064uPjAejTpw9TpkyJ1rv55pvp3bs3AJdddhk9e/akXbt29OvXr8EUueeak6bPPVNU+lxFaT5aSvrcsWPHcs899zBy5MiT1nW73TgcDqqqqhg2bBizZ8+u84zSn8Kpps9VQy6KorR4paWldO7cGavV2qhgDjB9+nR69epFnz59uOKKK37yYP5jqCEXRVFavLi4OLZv335K73n99dfPUGvOHNVDVxRFaSEaFdCFEKOEENuEEDuFEPXuKBBCjBdCrBdCrIs8BHpI0zdVUZSzqblO5Wuufsz3fdKALoTQA88Bo4FuwDVCiG7HVPsC6Cml7AVMA1485ZYoinLOslgsFBUVqaD+E5FSUlRUhMViOaX3NWYMfQCwU0q5G0AI8SYwHthca+PuWvXtgNrritKCpKenk5eXR2Fh/RuGlDPDYrGQnp5+Su9pTEBPAw7Uep0HDDy2khBiIvAXIBkYc0qtUBTlnGY0Gs9qhkOlcRozht7QfbT1euBSyg+klFnABOCRBlckxPTIGPsqdaRXFEVpWo0J6HlAm1qv04FDx6sspfwK6CCESGygbLaUsp+Usl9SUtIpN1ZRFEU5vsYE9JVAJyFEphDCBEwG5tauIIToKCIZcYQQfQAT0HDKNUVRFOWMOOkYupQyKIS4HfgU0AMvSyk3CSFujZTPAq4AbhRCBAAPMEmqy+GKoig/KZXLRVEUpRlRuVwURVF+BlRAVxRFaSFUQFcURWkhVEBXFEVpIVRAVxRFaSFUQFcURWkhVEBXFEVpIVRAVxRFaSFUQFcURWkhVEBXFEVpIVRAVxRFaSFUQFcURWkhVEBXFEVpIVRAVxRFaSFUQFcURWkhVEBXFEVpIVRAVxRFaSEaFdCFEKOEENuEEDuFEL9voPw6IcT6yL/vhBA9m76piqIoyomcNKALIfTAc8BooBtwjRCi2zHV9gDDpZQ5wCPA7KZuqKIoinJijemhDwB2Sil3Syn9wJvA+NoVpJTfSSlLIi+/B9KbtpmKoijKyTQmoKcBB2q9zossO57/ARY0VCCEmC6EWCWEWFVYWNj4ViqKoign1ZiALhpYJhusKMQItID+u4bKpZSzpZT9pJT9kpKSGt9KRVEU5aQMjaiTB7Sp9TodOHRsJSFEDvAiMFpKWdQ0zVMURVEaqzE99JVAJyFEphDCBEwG5tauIIRoC7wP3CCl3N70zVQURVFO5qQ9dCllUAhxO/ApoAdellJuEkLcGimfBTwIJADPCyEAglLKfmeu2YqiKMqxhJQNDoefcf369ZOrVq06K9tWFEVproQQq4/XYVZ3iiqKorQQKqAriqK0ECqgK4qitBAqoCuKorQQKqAriqK0ECqgK4qitBAqoCuKorQQKqAriqK0ECqgK4qitBAqoCuKorQQKqAriqK0ECqgK4qitBAqoCuKorQQKqAriqK0ECqgK4qitBAqoCuKorQQjQroQohRQohtQoidQojfN1CeJYRYJoTwCSF+0/TN1MzbPY+L372YnFdzuPjdi5m3e96Z2pRyCtR+OfeofXJuOtP75aSPoBNC6IHngIvQHhi9UggxV0q5uVa1YuBOYEKTtq6WebvnMfO7mXhDXgDyK/OZ+d1MAMa0H3OmNquchNov5x61T85NP8V+Oekj6IQQg4CZUspLIq/vB5BS/qWBujMBt5TyiZNt+FQfQXfxuxeTX5lfb3mqPZVFVy6ix6s9Gnzfhps2AJyw3BP0MOC/A+qVOY1Ovrv2O0q8JQx7a1i98nRHOguuWEBBZQEXvXtRvfIeiT14fczr7Cnbw2UfXlavfHDaYGZdOIutxVu56uOr6pWPaT+Gx4Y+xrJDy5j+2fR65dd3vZ7fDfgdC/cs5L6v7qtX/qtev+LWnrfy1ta3+NPyP9Ur/9+B/8ukrEnM+mEWz617rl7534b9jVGZo/jrir/yny3/qVc++6LZPPTdQw3uF4B3xr1DliuLWz+/lW8PfluvfO6EuWTGZnLtvGvZcHRDvfLPrvyMFHsKo98bTZ47r175V5O+It4Sz/mvn09FoKJe+YrrVmA1WE/rd+NMl5+p372GmHQmcpJyABiePpwp2VMAmLpwar26l2RcwuSsyXiCHn75+S/rlY/vOJ4JHSdQ4i3h3iX31iuf1GUSozJHUVBZwP1f31+v/KbuN5HbJpc9ZXv447I/1iufnjOdQa0HsbV4K39d8dd65Xf1uYteyb1Yd2QdT695ul757wb8jixXFssOLWP2+tn1yh8c9CCZsZksObCEVze9Wq/8L0P/Qoo9hYV7FvLWtrfqlT+Z+yTxlng+3PkhH+38qF758xc+j9Vg5c2tb/Lp3k8BWF+4Hn/YX69udQxrrNN9BF0acKDW67zIslMmhJguhFglhFhVWFh4Su8tqCw4peXKT0N9/81HQ8FE+ekc7/tvyr+hxvTQrwIukVLeHHl9AzBASnlHA3VncpZ66MrZofbLuUftk3NTU+2X0+2h5wFtar1OBw41eutN5K4+d2HRW+oss+gt3NXnrp+6KUotar+ce9Q+OTf9FPvlpBdFgZVAJyFEJnAQmAxc22QtaKTqiwZPr3magsoCUuwp3NXnLnWR5yxT++Xco/bJuemn2C8nHXIBEEJcCjwF6IGXpZSPCiFuBZBSzhJCpACrgBggDLiBblLK8uOt81SHXBRFUZQTD7k0poeOlHI+MP+YZbNq/VyANhSjKIqinCXqTlFFUZQWQgV0RVGUFkIFdEVRlBZCBXRFUZQWolGzXM7IhoUoBPb9yLcnAkebsDlnk/os56aW8llayucA9VmqtZNSJjVUcNYC+ukQQqw63rSd5kZ9lnNTS/ksLeVzgPosjaGGXBRFUVoIFdAVRVFaiOYa0Ovnw2y+1Gc5N7WUz9JSPgeoz3JSzXIMXVEURamvufbQFUVRlGOogK4oitJCnNMBXQjxshDiiBBi43HKhRDimcjDq9cLIfr81G1sjEZ8jlwhRJkQYl3k34M/dRsbSwjRRgjxpRBiixBikxCiXjLn5rBfGvk5msV+EUJYhBArhBA/RD7Lww3UOef3CTT6szSL/QLaM5mFEGuFEJ80UNb0+0RKec7+A4YBfYCNxym/FFgACOA8YPnZbvOP/By5wCdnu52N/CypQJ/Iz05gO1qq5Ga1Xxr5OZrFfol8z47Iz0ZgOXBec9snp/BZmsV+ibT1XuD1htp7JvbJOd1Dl1J+BRSfoMp44DWp+R6IE0Kk/jSta7xGfI5mQ0qZL6VcE/m5AthC/WfMnvP7pZGfo1mIfM/uyEtj5N+xsx3O+X0Cjf4szYIQIh0YA7x4nCpNvk/O6YDeCE32AOtzwKDIaeYCIUT3s92YxhBCZAC90XpRtTWr/XKCzwHNZL9ETu3XAUeAz6SUzXafNOKzQPPYL08Bv0V76E9DmnyfNPeALhpY1hyP5mvQ8jP0BJ4FPjy7zTk5IYQDeA+4W9Z/MlWz2S8n+RzNZr9IKUNSyl5oD5oZIITIPqZKs9knjfgs5/x+EUKMBY5IKVefqFoDy05rnzT3gH5OPMD6dEkpy6tPM6X2dCijECLxLDfruIQQRrQg+F8p5fsNVGkW++Vkn6O57RcAKWUpsAQYdUxRs9gntR3vszST/TIYuEwIsRd4E7hACPGfY+o0+T5p7gF9LnBj5GrxeUCZlDL/bDfqVAkhUoQQIvLzALT9UnR2W9WwSDtfArZIKZ88TrVzfr805nM0l/0ihEgSQsRFfrYCFwJbj6l2zu8TaNxnaQ77RUp5v5QyXUqZAUwGFksprz+mWpPvk0Y9U/RsEUK8gXZFO1EIkQc8hHaRBKk903Q+2pXinUAVMPXstPTEGvE5rgRuE0IEAQ8wWUYug5+DBgM3ABsi45wADwBtoVntl8Z8juayX1KBV4UQerTg9raU8hNR60HuNI99Ao37LM1lv9RzpveJuvVfURSlhWjuQy6KoihKhAroiqIoLYQK6IqiKC2ECuiKoigthAroiqIoLYQK6IqiKC2ECuiKoigtxP8DyR1GEmoDAXUAAAAASUVORK5CYII=", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" } ], "source": [ "import os\n", "import matplotlib.pyplot as plt\n", "\n", "path = './results'\n", "\n", "# Get a list of datasets\n", "datasets = os.listdir(path)\n", "print(datasets[1])\n", "# Get a list of algorithms for the first dataset\n", "algorithms = os.listdir(os.path.join(path, datasets[1]))\n", "\n", "# Get a list of niids for the first algorithm\n", "niids = [int(i) for i in os.listdir(os.path.join(path, datasets[1], algorithms[0]))]\n", "\n", "# Get the latest results for each algorithm and niid\n", "results = []\n", "for algorithm in algorithms:\n", " algorithm_results = []\n", " for niid in niids:\n", " niid_path = os.path.join(path, datasets[1], algorithm, str(niid))\n", " latest_result = sorted(os.listdir(niid_path), reverse=True)[0]\n", " result_file = os.path.join(niid_path, latest_result, 'FL_results.txt')\n", " with open(result_file) as f:\n", " data = f.readlines()\n", " accuracy = float(data[-1].split(',')[1].split(':')[1])\n", " algorithm_results.append(accuracy)\n", " results.append(algorithm_results)\n", "\n", "# Plot the results\n", "for i, algorithm in enumerate(algorithms):\n", " plt.plot(niids, results[i], 'o--',label=algorithm)\n", "plt.legend()\n", "plt.show()\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "asimenv", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.9.6" }, "orig_nbformat": 4 }, "nbformat": 4, "nbformat_minor": 2 }