Repository: RollingPlain/IVIF_ZOO Branch: main Commit: 53b4c12ccfc9 Files: 6 Total size: 102.6 KB Directory structure: gitextract_r6h374q9/ ├── Metric/ │ ├── Metric_torch.py │ ├── Nabf.py │ ├── Qabf.py │ ├── eval_torch.py │ └── ssim.py └── README.md ================================================ FILE CONTENTS ================================================ ================================================ FILE: Metric/Metric_torch.py ================================================ import numpy as np from scipy.signal import convolve2d from Qabf import get_Qabf from Nabf import get_Nabf import math import torch import torch.nn.functional as F import torch.fft from ssim import ssim, ms_ssim from sklearn.metrics import normalized_mutual_info_score def EN_function(image_tensor): histogram = torch.histc(image_tensor, bins=256, min=0, max=255) histogram = histogram / histogram.sum() entropy = -torch.sum(histogram * torch.log2(histogram + 1e-7)) return entropy def CE_function(ir_img_tensor, vi_img_tensor, f_img_tensor): ir_img_tensor = torch.sigmoid(ir_img_tensor) vi_img_tensor = torch.sigmoid(vi_img_tensor) f_img_tensor = torch.sigmoid(f_img_tensor) epsilon = 1e-7 f_img_tensor = torch.clamp(f_img_tensor, epsilon, 1.0 - epsilon) true_tensor = (ir_img_tensor + vi_img_tensor) / 2 true_tensor = torch.clamp(true_tensor, epsilon, 1.0 - epsilon) CE = F.binary_cross_entropy(f_img_tensor, true_tensor) return CE def QNCIE_function(ir_img_tensor, vi_img_tensor, f_img_tensor): def normalize1(img_tensor): img_min = img_tensor.min() img_max = img_tensor.max() return (img_tensor - img_min) / (img_max - img_min) def NCC(img1, img2): mean1 = torch.mean(img1) mean2 = torch.mean(img2) numerator = torch.sum((img1 - mean1) * (img2 - mean2)) denominator = torch.sqrt(torch.sum((img1 - mean1) ** 2) * torch.sum((img2 - mean2) ** 2)) return numerator / (denominator + 1e-10) ir_img_tensor = normalize1(ir_img_tensor) vi_img_tensor = normalize1(vi_img_tensor) f_img_tensor = normalize1(f_img_tensor) NCCxy = NCC(ir_img_tensor, vi_img_tensor) NCCxf = NCC(ir_img_tensor, f_img_tensor) NCCyf = NCC(vi_img_tensor, f_img_tensor) R = torch.tensor([[1, NCCxy, NCCxf], [NCCxy, 1, NCCyf], [NCCxf, NCCyf, 1]], dtype=torch.float32) r = torch.linalg.eigvals(R).real K = 3 b = 256 HR = torch.sum(r * torch.log2(r / K) / K) HR = -HR / np.log2(b) QNCIE = 1 - HR.item() return QNCIE def TE_function(ir_img_tensor, vi_img_tensor, f_img_tensor, q=1, ksize=256): def compute_entropy(img_tensor, q, ksize): img_tensor = img_tensor.view(-1).float() histogram = torch.histc(img_tensor, bins=ksize, min=0, max=ksize - 1) probabilities = histogram / torch.sum(histogram) if q == 1: entropy = -torch.sum(probabilities * torch.log2(probabilities + 1e-10)) else: entropy = (1 / (q - 1)) * (1 - torch.sum(probabilities ** q)) return entropy.item() TE_ir = compute_entropy(ir_img_tensor, q, ksize) TE_vi = compute_entropy(vi_img_tensor, q, ksize) TE_f = compute_entropy(f_img_tensor, q, ksize) TE = TE_ir + TE_vi - TE_f return TE def EI_function(f_img_tensor): sobel_kernel_x = torch.tensor([[-1., 0., 1.], [-2., 0., 2.], [-1., 0., 1.]]).to(f_img_tensor.device) sobel_kernel_y = torch.tensor([[-1., -2., -1.], [0., 0., 0.], [1., 2., 1.]]).to(f_img_tensor.device) sobel_kernel_x = sobel_kernel_x.view(1, 1, 3, 3) sobel_kernel_y = sobel_kernel_y.view(1, 1, 3, 3) gx = F.conv2d(f_img_tensor.unsqueeze(0).unsqueeze(0), sobel_kernel_x, padding=1) gy = F.conv2d(f_img_tensor.unsqueeze(0).unsqueeze(0), sobel_kernel_y, padding=1) g = torch.sqrt(gx ** 2 + gy ** 2) EI = torch.mean(g).item() return EI def SF_function(image_tensor): RF = image_tensor[1:, :] - image_tensor[:-1, :] CF = image_tensor[:, 1:] - image_tensor[:, :-1] RF1 = torch.sqrt(torch.mean(RF ** 2)) CF1 = torch.sqrt(torch.mean(CF ** 2)) SF = torch.sqrt(RF1 ** 2 + CF1 ** 2) return SF def SD_function(image_tensor): m, n = image_tensor.shape u = torch.mean(image_tensor) SD = torch.sqrt(torch.sum((image_tensor - u) ** 2) / (m * n)) return SD def PSNR_function(A, B, F): A = A.float() / 255.0 B = B.float() / 255.0 F = F.float() / 255.0 m, n = F.shape MSE_AF = torch.mean((F - A) ** 2) MSE_BF = torch.mean((F - B) ** 2) MSE = 0.5 * MSE_AF + 0.5 * MSE_BF PSNR = 20 * torch.log10(1 / torch.sqrt(MSE)) return PSNR def MSE_function(A, B, F): A = A.float() / 255.0 B = B.float() / 255.0 F = F.float() / 255.0 m, n = F.shape MSE_AF = torch.mean((F - A) ** 2) MSE_BF = torch.mean((F - B) ** 2) MSE = 0.5 * MSE_AF + 0.5 * MSE_BF return MSE def fspecial_gaussian(shape, sigma): m, n = [(ss-1.)/2. for ss in shape] y, x = np.ogrid[-m:m+1, -n:n+1] h = np.exp(-(x*x + y*y) / (2.*sigma*sigma)) h[h < np.finfo(h.dtype).eps*h.max()] = 0 sumh = h.sum() if sumh != 0: h /= sumh return h def fspecial_gaussian(size, sigma): x = torch.linspace(-size[0]//2, size[0]//2, size[0]) y = torch.linspace(-size[1]//2, size[1]//2, size[1]) x, y = torch.meshgrid(x, y) g = torch.exp(-(x**2 + y**2) / (2 * sigma**2)) return g / g.sum() def convolve2d(input, kernel): kernel = kernel.unsqueeze(0).unsqueeze(0).to(input.device) # Add batch and channel dimensions return F.conv2d(input.unsqueeze(0).unsqueeze(0), kernel, padding=kernel.shape[2] // 2)[0][0] def vifp_mscale(ref, dist): sigma_nsq = 2 num = 0 den = 0 for scale in range(1, 5): N = 2 ** (4 - scale + 1) + 1 win = fspecial_gaussian((N, N), N / 5) if scale > 1: ref = convolve2d(ref, win) dist = convolve2d(dist, win) ref = ref[::2, ::2] dist = dist[::2, ::2] mu1 = convolve2d(ref, win) mu2 = convolve2d(dist, win) mu1_sq = mu1 * mu1 mu2_sq = mu2 * mu2 mu1_mu2 = mu1 * mu2 sigma1_sq = convolve2d(ref * ref, win) - mu1_sq sigma2_sq = convolve2d(dist * dist, win) - mu2_sq sigma12 = convolve2d(ref * dist, win) - mu1_mu2 sigma1_sq[sigma1_sq < 0] = 0 sigma2_sq[sigma2_sq < 0] = 0 g = sigma12 / (sigma1_sq + 1e-10) sv_sq = sigma2_sq - g * sigma12 g[sigma1_sq < 1e-10] = 0 sv_sq[sigma1_sq < 1e-10] = sigma2_sq[sigma1_sq < 1e-10] sigma1_sq[sigma1_sq < 1e-10] = 0 g[sigma2_sq < 1e-10] = 0 sv_sq[sigma2_sq < 1e-10] = 0 sv_sq[g < 0] = sigma2_sq[g < 0] g[g < 0] = 0 sv_sq[sv_sq <= 1e-10] = 1e-10 num += torch.sum(torch.log10(1 + g**2 * sigma1_sq / (sv_sq + sigma_nsq))) den += torch.sum(torch.log10(1 + sigma1_sq / sigma_nsq)) vifp = num / den return vifp def VIF_function(A, B, F): VIF = vifp_mscale(A, F) + vifp_mscale(B, F) return VIF def CC_function(A, B, F): rAF = torch.sum((A - torch.mean(A)) * (F - torch.mean(F))) / torch.sqrt(torch.sum((A - torch.mean(A)) ** 2) * torch.sum((F - torch.mean(F)) ** 2)) rBF = torch.sum((B - torch.mean(B)) * (F - torch.mean(F))) / torch.sqrt(torch.sum((B - torch.mean(B)) ** 2) * torch.sum((F - torch.mean(F)) ** 2)) CC = torch.mean(torch.tensor([rAF, rBF])) return CC def corr2(a, b): a = a - torch.mean(a) b = b - torch.mean(b) r = torch.sum(a * b) / torch.sqrt(torch.sum(a * a) * torch.sum(b * b)) return r def SCD_function(A, B, F): r = corr2(F - B, A) + corr2(F - A, B) return r def Qabf_function(A, B, F): return get_Qabf(A, B, F) def Nabf_function(A, B, F): return Nabf_function(A, B, F) def Hab(im1, im2, gray_level): hang, lie = im1.shape count = hang * lie N = gray_level h = np.zeros((N, N)) for i in range(hang): for j in range(lie): h[im1[i, j], im2[i, j]] = h[im1[i, j], im2[i, j]] + 1 h = h / np.sum(h) im1_marg = np.sum(h, axis=0) im2_marg = np.sum(h, axis=1) H_x = 0 H_y = 0 for i in range(N): if (im1_marg[i] != 0): H_x = H_x + im1_marg[i] * math.log2(im1_marg[i]) for i in range(N): if (im2_marg[i] != 0): H_x = H_x + im2_marg[i] * math.log2(im2_marg[i]) H_xy = 0 for i in range(N): for j in range(N): if (h[i, j] != 0): H_xy = H_xy + h[i, j] * math.log2(h[i, j]) MI = H_xy - H_x - H_y return MI def MI_function(A, B, F, gray_level=256): MIA = Hab(A, F, gray_level) MIB = Hab(B, F, gray_level) MI_results = MIA + MIB return MI_results def entropy(im, gray_level=256): hist, _ = np.histogram(im, bins=gray_level, range=(0, gray_level), density=True) H = -np.sum(hist * np.log2(hist + 1e-10)) return H def NMI_function(A, B, F, gray_level=256): MIA = Hab(A, F, gray_level) MIB = Hab(B, F, gray_level) MI_results = MIA + MIB H_A = entropy(A, gray_level) H_B = entropy(B, gray_level) NMI = 2 * MI_results / (H_A + H_B + 1e-10) return NMI def AG_function(image_tensor): grady, gradx = torch.gradient(image_tensor) s = torch.sqrt((gradx ** 2 + grady ** 2) / 2) AG = torch.sum(s) / (image_tensor.shape[0] * image_tensor.shape[1]) return AG def SSIM_function(A, B, F): ssim_A = ssim(A, F) ssim_B = ssim(B, F) SSIM = (ssim_A + 1 * ssim_B) / 2 return SSIM.item() def MS_SSIM_function(A, B, F): ssim_A = ms_ssim(A, F) ssim_B = ms_ssim(B, F) MS_SSIM = (ssim_A + 1 * ssim_B) / 2 return MS_SSIM.item() def Nabf_function(A, B, F): Nabf = get_Nabf(A, B, F) return Nabf def Qy_function(ir_img_tensor, vi_img_tensor, f_img_tensor): def gaussian_filter(window_size, sigma): gauss = torch.tensor([np.exp(-(x - window_size // 2) ** 2 / float(2 * sigma ** 2)) for x in range(window_size)], device=ir_img_tensor.device) gauss = gauss / gauss.sum() gauss = gauss.view(1, 1, -1).repeat(1, 1, 1, 1) return gauss def ssim_yang(img1, img2): window_size = 7 sigma = 1.5 window = gaussian_filter(window_size, sigma) window = window.expand(1, 1, window_size, window_size) mu1 = F.conv2d(img1, window, stride=1, padding=window_size // 2) mu2 = F.conv2d(img2, window, stride=1, padding=window_size // 2) mu1_sq = mu1.pow(2) mu2_sq = mu2.pow(2) mu1_mu2 = mu1 * mu2 sigma1_sq = F.conv2d(img1.pow(2), window, padding=window_size // 2) - mu1_sq sigma2_sq = F.conv2d(img2.pow(2), window, padding=window_size // 2) - mu2_sq sigma12 = F.conv2d(img1 * img2, window, padding=window_size // 2) - mu1_mu2 C1 = 0.01**2 C2 = 0.03**2 ssim_map = ((2 * mu1_mu2 + C1) * (2 * sigma12 + C2)) / ((mu1_sq + mu2_sq + C1) * (sigma1_sq + sigma2_sq + C2)) mssim = ssim_map.mean().item() return mssim, ssim_map, sigma1_sq, sigma2_sq ir_img_tensor = ir_img_tensor.unsqueeze(0).unsqueeze(0).double() vi_img_tensor = vi_img_tensor.unsqueeze(0).unsqueeze(0).double() f_img_tensor = f_img_tensor.unsqueeze(0).unsqueeze(0).double() _, ssim_map1, sigma1_sq1, sigma2_sq1 = ssim_yang(ir_img_tensor, vi_img_tensor) _, ssim_map2, _, _ = ssim_yang(ir_img_tensor, f_img_tensor) _, ssim_map3, _, _ = ssim_yang(vi_img_tensor, f_img_tensor) bin_map = (ssim_map1 >= 0.75).double() ramda = sigma1_sq1 / (sigma1_sq1 + sigma2_sq1 + 1e-10) Q1 = (ramda * ssim_map2 + (1 - ramda) * ssim_map3) * bin_map Q2 = torch.max(ssim_map2, ssim_map3) * (1 - bin_map) Qy = (Q1 + Q2).mean().item() return Qy def gaussian2d(n1, n2, sigma, device): x = torch.arange(-15, 16, device=device, dtype=torch.double) y = torch.arange(-15, 16, device=device, dtype=torch.double) x, y = torch.meshgrid(x, y) G = torch.exp(-(x**2 + y**2) / (2 * sigma**2)) / (2 * torch.pi * sigma**2) return G def contrast(G1, G2, img): buff = F.conv2d(img.unsqueeze(0).unsqueeze(0), G1.unsqueeze(0).unsqueeze(0), padding=G1.shape[-1] // 2) buff1 = F.conv2d(img.unsqueeze(0).unsqueeze(0), G2.unsqueeze(0).unsqueeze(0), padding=G2.shape[-1] // 2) return buff / (buff1 + 1e-10) - 1 def Qcb_function(ir_img_tensor, vi_img_tensor, f_img_tensor): device = ir_img_tensor.device ir_img_tensor = ir_img_tensor.double().to(device) vi_img_tensor = vi_img_tensor.double().to(device) f_img_tensor = f_img_tensor.double().to(device) ir_img_tensor = (ir_img_tensor - ir_img_tensor.min()) / (ir_img_tensor.max() - ir_img_tensor.min()) vi_img_tensor = (vi_img_tensor - vi_img_tensor.min()) / (vi_img_tensor.max() - vi_img_tensor.min()) f_img_tensor = (f_img_tensor - f_img_tensor.min()) / (f_img_tensor.max() - f_img_tensor.min()) f0 = 15.3870 f1 = 1.3456 a = 0.7622 k = 1 h = 1 p = 3 q = 2 Z = 0.0001 hang, lie = ir_img_tensor.shape[-2:] u, v = torch.meshgrid(torch.fft.fftfreq(hang, device=device), torch.fft.fftfreq(lie, device=device), indexing='ij') u = u * (hang / 30) v = v * (lie / 30) r = torch.sqrt(u**2 + v**2) Sd = (torch.exp(-(r / f0)**2) - a * torch.exp(-(r / f1)**2)).to(device) fim1 = torch.fft.ifft2(torch.fft.fft2(ir_img_tensor) * Sd).real fim2 = torch.fft.ifft2(torch.fft.fft2(vi_img_tensor) * Sd).real ffim = torch.fft.ifft2(torch.fft.fft2(f_img_tensor) * Sd).real G1 = gaussian2d(hang, lie, 2, device).to(device) G2 = gaussian2d(hang, lie, 4, device).to(device) C1 = contrast(G1, G2, fim1) C2 = contrast(G1, G2, fim2) Cf = contrast(G1, G2, ffim) C1P = (k * (torch.abs(C1)**p)) / (h * (torch.abs(C1)**q) + Z) C2P = (k * (torch.abs(C2)**p)) / (h * (torch.abs(C2)**q) + Z) CfP = (k * (torch.abs(Cf)**p)) / (h * (torch.abs(Cf)**q) + Z) mask1 = (C1P < CfP).double() Q1F = (C1P / CfP) * mask1 + (CfP / C1P) * (1 - mask1) mask2 = (C2P < CfP).double() Q2F = (C2P / CfP) * mask2 + (CfP / C2P) * (1 - mask2) ramda1 = (C1P**2) / (C1P**2 + C2P**2 + 1e-10) ramda2 = (C2P**2) / (C1P**2 + C2P**2 + 1e-10) Q = ramda1 * Q1F + ramda2 * Q2F Qcb = Q.mean().item() return Qcb ================================================ FILE: Metric/Nabf.py ================================================ import numpy as np from scipy.signal import convolve2d import math import torch def sobel_fn(x): vtemp = np.array([[-1, 0, 1], [-2, 0, 2], [-1, 0, 1]]) / 8 htemp = np.array([[-1, -2, -1], [0, 0, 0], [1, 2, 1]]) / 8 a, b = htemp.shape x_ext = per_extn_im_fn(x, a) p, q = x_ext.shape gv = np.zeros((p - 2, q - 2)) gh = np.zeros((p - 2, q - 2)) gv = convolve2d(x_ext, vtemp, mode='valid') gh = convolve2d(x_ext, htemp, mode='valid') return gv, gh def per_extn_im_fn(x, wsize): hwsize = (wsize - 1) // 2 # Half window size excluding centre pixel. p, q = x.shape xout_ext = np.zeros((p + wsize - 1, q + wsize - 1)) xout_ext[hwsize: p + hwsize, hwsize: q + hwsize] = x if wsize - 1 == hwsize + 1: xout_ext[0: hwsize, :] = xout_ext[2, :].reshape(1, -1) xout_ext[p + hwsize: p + wsize - 1, :] = xout_ext[-3, :].reshape(1, -1) xout_ext[:, 0: hwsize] = xout_ext[:, 2].reshape(-1, 1) xout_ext[:, q + hwsize: q + wsize - 1] = xout_ext[:, -3].reshape(-1, 1) return xout_ext def get_Nabf(I1, I2, f): Td=2 wt_min=0.001 P=1 Lg=1.5 Nrg=0.9999 kg=19 sigmag=0.5 Nra=0.9995 ka=22 sigmaa=0.5 I1 = I1.cpu().numpy() if isinstance(I1, torch.Tensor) else I1 I2 = I2.cpu().numpy() if isinstance(I2, torch.Tensor) else I2 f = f.cpu().numpy() if isinstance(f, torch.Tensor) else f xrcw = f.astype(np.float64) x1 = I1.astype(np.float64) x2 = I2.astype(np.float64) gvA,ghA=sobel_fn(x1) gA=np.sqrt(ghA**2+gvA**2) gvB,ghB=sobel_fn(x2) gB=np.sqrt(ghB**2+gvB**2) gvF,ghF=sobel_fn(xrcw) gF=np.sqrt(ghF**2+gvF**2) gAF=np.zeros(gA.shape) gBF=np.zeros(gB.shape) aA=np.zeros(ghA.shape) aB=np.zeros(ghB.shape) aF=np.zeros(ghF.shape) p,q=xrcw.shape maskAF1 = (gA == 0) | (gF == 0) maskAF2 = (gA > gF) gAF[~maskAF1] = np.where(maskAF2, gF / gA, gA / gF)[~maskAF1] maskBF1 = (gB == 0) | (gF == 0) maskBF2 = (gB > gF) gBF[~maskBF1] = np.where(maskBF2, gF / gB, gB / gF)[~maskBF1] aA = np.where((gvA == 0) & (ghA == 0), 0, np.arctan(gvA / ghA)) aB = np.where((gvB == 0) & (ghB == 0), 0, np.arctan(gvB / ghB)) aF = np.where((gvF == 0) & (ghF == 0), 0, np.arctan(gvF / ghF)) aAF=np.abs(np.abs(aA-aF)-np.pi/2)*2/np.pi aBF=np.abs(np.abs(aB-aF)-np.pi/2)*2/np.pi QgAF = Nrg / (1 + np.exp(-kg * (gAF - sigmag))) QaAF = Nra / (1 + np.exp(-ka * (aAF - sigmaa))) QAF = np.sqrt(QgAF * QaAF) QgBF = Nrg / (1 + np.exp(-kg * (gBF - sigmag))) QaBF = Nra / (1 + np.exp(-ka * (aBF - sigmaa))) QBF = np.sqrt(QgBF * QaBF) wtA = wt_min * np.ones((p, q)) wtB = wt_min * np.ones((p, q)) cA = np.ones((p, q)) cB = np.ones((p, q)) wtA = np.where(gA >= Td, cA * gA ** Lg, 0) wtB = np.where(gB >= Td, cB * gB ** Lg, 0) wt_sum = np.sum(wtA + wtB) QAF_wtsum = np.sum(QAF * wtA) / wt_sum QBF_wtsum = np.sum(QBF * wtB) / wt_sum QABF = QAF_wtsum + QBF_wtsum Qdelta = np.abs(QAF - QBF) QCinfo = (QAF + QBF - Qdelta) / 2 QdeltaAF = QAF - QCinfo QdeltaBF = QBF - QCinfo QdeltaAF_wtsum = np.sum(QdeltaAF * wtA) / wt_sum QdeltaBF_wtsum = np.sum(QdeltaBF * wtB) / wt_sum QdeltaABF = QdeltaAF_wtsum + QdeltaBF_wtsum QCinfo_wtsum = np.sum(QCinfo * (wtA + wtB)) / wt_sum QABF11 = QdeltaABF + QCinfo_wtsum rr = np.zeros((p, q)) rr = np.where(gF <= np.minimum(gA, gB), 1, 0) LABF = np.sum(rr * ((1 - QAF) * wtA + (1 - QBF) * wtB)) / wt_sum na1 = np.where((gF > gA) & (gF > gB), 2 - QAF - QBF, 0) NABF1 = np.sum(na1 * (wtA + wtB)) / wt_sum na = np.where((gF > gA) & (gF > gB), 1, 0) NABF = np.sum(na * ((1 - QAF) * wtA + (1 - QBF) * wtB)) / wt_sum return NABF ================================================ FILE: Metric/Qabf.py ================================================ import numpy as np import math from scipy.signal import convolve2d def sobel_fn(x): vtemp = np.array([[-1, 0, 1], [-2, 0, 2], [-1, 0, 1]]) / 8 htemp = np.array([[-1, -2, -1], [0, 0, 0], [1, 2, 1]]) / 8 a, b = htemp.shape x_ext = per_extn_im_fn(x, a) p, q = x_ext.shape gv = np.zeros((p - 2, q - 2)) gh = np.zeros((p - 2, q - 2)) gv = convolve2d(x_ext, vtemp, mode='valid') gh = convolve2d(x_ext, htemp, mode='valid') return gv, gh def per_extn_im_fn(x, wsize): hwsize = (wsize - 1) // 2 p, q = x.shape xout_ext = np.zeros((p + wsize - 1, q + wsize - 1)) xout_ext[hwsize: p + hwsize, hwsize: q + hwsize] = x if wsize - 1 == hwsize + 1: xout_ext[0: hwsize, :] = xout_ext[2, :].reshape(1, -1) xout_ext[p + hwsize: p + wsize - 1, :] = xout_ext[-3, :].reshape(1, -1) xout_ext[:, 0: hwsize] = xout_ext[:, 2].reshape(-1, 1) xout_ext[:, q + hwsize: q + wsize - 1] = xout_ext[:, -3].reshape(-1, 1) return xout_ext def get_Qabf(pA, pB, pF): L = 1 Tg = 0.9994 kg = -15 Dg = 0.5; Ta = 0.9879 ka = -22 Da = 0.8 h1 = np.array([[1, 2, 1], [0, 0, 0], [-1, -2, -1]]).astype(np.float32) h2 = np.array([[0, 1, 2], [-1, 0, 1], [-2, -1, 0]]).astype(np.float32) h3 = np.array([[-1, 0, 1], [-2, 0, 2], [-1, 0, 1]]).astype(np.float32) strA = pA strB = pB strF = pF def flip180(arr): return np.flip(arr) def convolution(k, data): k = flip180(k) data = np.pad(data, ((1, 1), (1, 1)), 'constant', constant_values=(0, 0)) img_new = convolve2d(data, k, mode='valid') return img_new def getArray(img): SAx = convolution(h3, img) SAy = convolution(h1, img) gA = np.sqrt(np.multiply(SAx, SAx) + np.multiply(SAy, SAy)) n, m = img.shape aA = np.zeros((n, m)) zero_mask = SAx == 0 aA[~zero_mask] = np.arctan(SAy[~zero_mask] / SAx[~zero_mask]) aA[zero_mask] = np.pi / 2 return gA, aA gA, aA = getArray(strA) gB, aB = getArray(strB) gF, aF = getArray(strF) def getQabf(aA, gA, aF, gF): mask = (gA > gF) GAF = np.where(mask, gF / gA, np.where(gA == gF, gF, gA / gF)) AAF = 1 - np.abs(aA - aF) / (math.pi / 2) QgAF = Tg / (1 + np.exp(kg * (GAF - Dg))) QaAF = Ta / (1 + np.exp(ka * (AAF - Da))) QAF = QgAF * QaAF return QAF QAF = getQabf(aA, gA, aF, gF) QBF = getQabf(aB, gB, aF, gF) deno = np.sum(gA + gB) nume = np.sum(np.multiply(QAF, gA) + np.multiply(QBF, gB)) output = nume / deno return output ================================================ FILE: Metric/eval_torch.py ================================================ import numpy as np from PIL import Image from Metric_torch import * from natsort import natsorted from tqdm import tqdm import os import torch import warnings from openpyxl import Workbook, load_workbook from openpyxl.utils import get_column_letter warnings.filterwarnings("ignore") device = torch.device("cuda" if torch.cuda.is_available() else "cpu") def write_excel(excel_name='metric.xlsx', worksheet_name='VIF', column_index=0, data=None): try: workbook = load_workbook(excel_name) except FileNotFoundError: workbook = Workbook() worksheet = workbook.create_sheet(title=worksheet_name) if worksheet_name not in workbook.sheetnames else workbook[ worksheet_name] column = get_column_letter(column_index + 1) for i, value in enumerate(data): cell = worksheet[column + str(i + 1)] cell.value = value workbook.save(excel_name) def evaluation_one(ir_name, vi_name, f_name): f_img = Image.open(f_name).convert('L') ir_img = Image.open(ir_name).convert('L') vi_img = Image.open(vi_name).convert('L') f_img_tensor = torch.tensor(np.array(f_img)).float().to(device) ir_img_tensor = torch.tensor(np.array(ir_img)).float().to(device) vi_img_tensor = torch.tensor(np.array(vi_img)).float().to(device) f_img_int = np.array(f_img).astype(np.int32) f_img_double = np.array(f_img).astype(np.float32) ir_img_int = np.array(ir_img).astype(np.int32) ir_img_double = np.array(ir_img).astype(np.float32) vi_img_int = np.array(vi_img).astype(np.int32) vi_img_double = np.array(vi_img).astype(np.float32) CE = CE_function(ir_img_tensor, vi_img_tensor, f_img_tensor) NMI = NMI_function(ir_img_int, vi_img_int, f_img_int, gray_level=256) QNCIE = QNCIE_function(ir_img_tensor, vi_img_tensor, f_img_tensor) TE = TE_function(ir_img_tensor, vi_img_tensor, f_img_tensor) EI = EI_function(f_img_tensor) Qy = Qy_function(ir_img_tensor, vi_img_tensor, f_img_tensor) Qcb = Qcb_function(ir_img_tensor, vi_img_tensor, f_img_tensor) EN = EN_function(f_img_tensor) MI = MI_function(ir_img_int, vi_img_int, f_img_int, gray_level=256) SF = SF_function(f_img_tensor) SD = SD_function(f_img_tensor) AG = AG_function(f_img_tensor) PSNR = PSNR_function(ir_img_tensor, vi_img_tensor, f_img_tensor) MSE = MSE_function(ir_img_tensor, vi_img_tensor, f_img_tensor) VIF = VIF_function(ir_img_tensor, vi_img_tensor, f_img_tensor) CC = CC_function(ir_img_tensor, vi_img_tensor, f_img_tensor) SCD = SCD_function(ir_img_tensor, vi_img_tensor, f_img_tensor) Qabf = Qabf_function(ir_img_double, vi_img_double, f_img_double) Nabf = Nabf_function(ir_img_tensor, vi_img_tensor, f_img_tensor) SSIM = SSIM_function(ir_img_double, vi_img_double, f_img_double) MS_SSIM = MS_SSIM_function(ir_img_double, vi_img_double, f_img_double) return CE, NMI, QNCIE, TE, EI, Qy, Qcb, EN, MI, SF, AG, SD, CC, SCD, VIF, MSE, PSNR, Qabf, Nabf, SSIM, MS_SSIM if __name__ == '__main__': if __name__ == '__main__': with_mean = True config = { 'dataroot': '/mnt/disk1/IVIF/', # Change to your local infrared and visible images path 'results_root': '/mnt/disk1/IVIF/', # Change to your local fusion images path 'dataset': 'M3FD_4200', # Specify the dataset name 'save_dir': '/mnt/disk4/test' # Directory for saving metrics } ir_dir = os.path.join(config['dataroot'], config['dataset'], 'Ir') # Infrared images directory vi_dir = os.path.join(config['dataroot'], config['dataset'], 'Vis') # Visible images directory f_dir = os.path.join(config['results_root'], config['dataset']) # Fusion images directory os.makedirs(config['save_dir'], exist_ok=True) filelist = natsorted(os.listdir(ir_dir))[:300] metric_save_name = os.path.join(config['save_dir'], f'metric_{config["dataset"]}.xlsx') # Metrics file name # Change to the directory name of the fusion images you want to evaluate Method_list = [ 'BDLFusion', 'CAF', 'CDDFuse', 'CoCoNet', 'DATFuse', 'DDcGAN', 'DDFM', 'DeFusion', 'Densefuse', 'DIDFuse', 'EMMA', 'FusinDN', 'GANMcC', 'IF-FILM', 'IGNet', 'IRFS', 'LRRNet', 'MetaFusion', 'MFEIF', 'MRFS', 'PAIF', 'PMGI', 'PSFusion', 'ReCoNet', 'RFN-Nest', 'SDCFusion', 'SDNet', 'SeAFusion', 'SegMif', 'SHIP', 'SuperFusion', 'SwinFusion', 'TarDAL', 'Text-IF', 'TGFuse', 'TIMFusion', 'U2Fusion', 'UMFusion', 'YDTR', 'FusionGAN', 'DetFusion', 'MoE-Fusion', 'PromptF' ] # Starting index for the method 'BDLFusion' start_index = Method_list.index('BDLFusion') for i, Method in enumerate(Method_list[start_index:], start=start_index): CE_list = [] NMI_list = [] QNCIE_list = [] TE_list = [] EI_list = [] Qy_list = [] Qcb_list = [] EN_list = [] MI_list = [] SF_list = [] AG_list = [] SD_list = [] CC_list = [] SCD_list = [] VIF_list = [] MSE_list = [] PSNR_list = [] Qabf_list = [] Nabf_list = [] SSIM_list = [] MS_SSIM_list = [] filename_list = [''] sub_f_dir = os.path.join(f_dir, Method) eval_bar = tqdm(filelist) for _, item in enumerate(eval_bar): ir_name = os.path.join(ir_dir, item) vi_name = os.path.join(vi_dir, item) f_name = os.path.join(sub_f_dir, item) if os.path.exists(f_name): print(ir_name, vi_name, f_name) CE, NMI, QNCIE, TE, EI, Qy, Qcb, EN, MI, SF, AG, SD, CC, SCD, VIF, MSE, PSNR, Qabf, Nabf, SSIM, MS_SSIM = evaluation_one(ir_name, vi_name, f_name) CE_list.append(CE) NMI_list.append(NMI) QNCIE_list.append(QNCIE) TE_list.append(TE) EI_list.append(EI) Qy_list.append(Qy) Qcb_list.append(Qcb) EN_list.append(EN) MI_list.append(MI) SF_list.append(SF) AG_list.append(AG) SD_list.append(SD) CC_list.append(CC) SCD_list.append(SCD) VIF_list.append(VIF) MSE_list.append(MSE) PSNR_list.append(PSNR) Qabf_list.append(Qabf) Nabf_list.append(Nabf) SSIM_list.append(SSIM) MS_SSIM_list.append(MS_SSIM) filename_list.append(item) eval_bar.set_description("{} | {}".format(Method, item)) if with_mean: CE_tensor = torch.tensor(CE_list).mean().item() CE_list.append(CE_tensor) NMI_tensor = torch.tensor(NMI_list).mean().item() NMI_list.append(NMI_tensor) QNCIE_tensor = torch.tensor(QNCIE_list).mean().item() QNCIE_list.append(QNCIE_tensor) TE_tensor = torch.tensor(TE_list).mean().item() TE_list.append(TE_tensor) EI_tensor = torch.tensor(EI_list).mean().item() EI_list.append(EI_tensor) Qy_tensor = torch.tensor(Qy_list).mean().item() Qy_list.append(Qy_tensor) Qcb_tensor = torch.tensor(Qcb_list).mean().item() Qcb_list.append(Qcb_tensor) EN_tensor = torch.tensor(EN_list).mean().item() EN_list.append(EN_tensor) MI_tensor = torch.tensor(MI_list).mean().item() MI_list.append(MI_tensor) SF_tensor = torch.tensor(SF_list).mean().item() SF_list.append(SF_tensor) AG_tensor = torch.tensor(AG_list).mean().item() AG_list.append(AG_tensor) SD_tensor = torch.tensor(SD_list).mean().item() SD_list.append(SD_tensor) CC_tensor = torch.tensor(CC_list).mean().item() CC_list.append(CC_tensor) SCD_tensor = torch.tensor(SCD_list).mean().item() SCD_list.append(SCD_tensor) VIF_tensor = torch.tensor(VIF_list).mean().item() VIF_list.append(VIF_tensor) MSE_tensor = torch.tensor(MSE_list).mean().item() MSE_list.append(MSE_tensor) PSNR_tensor = torch.tensor(PSNR_list).mean().item() PSNR_list.append(PSNR_tensor) Qabf_list.append(np.mean(Qabf_list)) Nabf_tensor = torch.tensor(Nabf_list).mean().item() Nabf_list.append(Nabf_tensor) SSIM_tensor = torch.tensor(SSIM_list).mean().item() SSIM_list.append(SSIM_tensor) MS_SSIM_tensor = torch.tensor(MS_SSIM_list).mean().item() MS_SSIM_list.append(MS_SSIM_tensor) filename_list.append('mean') CE_list.insert(0, '{}'.format(Method)) NMI_list.insert(0, '{}'.format(Method)) QNCIE_list.insert(0, '{}'.format(Method)) TE_list.insert(0, '{}'.format(Method)) EI_list.insert(0, '{}'.format(Method)) Qy_list.insert(0, '{}'.format(Method)) Qcb_list.insert(0, '{}'.format(Method)) EN_list.insert(0, '{}'.format(Method)) MI_list.insert(0, '{}'.format(Method)) SF_list.insert(0, '{}'.format(Method)) AG_list.insert(0, '{}'.format(Method)) SD_list.insert(0, '{}'.format(Method)) CC_list.insert(0, '{}'.format(Method)) SCD_list.insert(0, '{}'.format(Method)) VIF_list.insert(0, '{}'.format(Method)) MSE_list.insert(0, '{}'.format(Method)) PSNR_list.insert(0, '{}'.format(Method)) Qabf_list.insert(0, '{}'.format(Method)) Nabf_list.insert(0, '{}'.format(Method)) SSIM_list.insert(0, '{}'.format(Method)) MS_SSIM_list.insert(0, '{}'.format(Method)) if i == start_index: write_excel(metric_save_name, 'CE', 0, filename_list) write_excel(metric_save_name, 'NMI', 0, filename_list) write_excel(metric_save_name, 'QNCIE', 0, filename_list) write_excel(metric_save_name, 'TE', 0, filename_list) write_excel(metric_save_name, 'EI', 0, filename_list) write_excel(metric_save_name, 'Qy', 0, filename_list) write_excel(metric_save_name, 'Qcb', 0, filename_list) write_excel(metric_save_name, 'EN', 0, filename_list) write_excel(metric_save_name, "MI", 0, filename_list) write_excel(metric_save_name, "SF", 0, filename_list) write_excel(metric_save_name, "AG", 0, filename_list) write_excel(metric_save_name, "SD", 0, filename_list) write_excel(metric_save_name, "CC", 0, filename_list) write_excel(metric_save_name, "SCD", 0, filename_list) write_excel(metric_save_name, "VIF", 0, filename_list) write_excel(metric_save_name, "MSE", 0, filename_list) write_excel(metric_save_name, "PSNR", 0, filename_list) write_excel(metric_save_name, "Qabf", 0, filename_list) write_excel(metric_save_name, "Nabf", 0, filename_list) write_excel(metric_save_name, "SSIM", 0, filename_list) write_excel(metric_save_name, "MS_SSIM", 0, filename_list) write_excel(metric_save_name, 'CE', i + 1, [x.item() if isinstance(x, torch.Tensor) else float(x) if isinstance(x, (int, float)) else x for x in CE_list]) write_excel(metric_save_name, 'NMI', i + 1, [x.item() if isinstance(x, torch.Tensor) else float(x) if isinstance(x, (int, float)) else x for x in NMI_list]) write_excel(metric_save_name, 'QNCIE', i + 1, [x.item() if isinstance(x, torch.Tensor) else float(x) if isinstance(x, (int, float)) else x for x in QNCIE_list]) write_excel(metric_save_name, 'TE', i + 1, [x.item() if isinstance(x, torch.Tensor) else float(x) if isinstance(x, (int, float)) else x for x in TE_list]) write_excel(metric_save_name, 'EI', i + 1, [x.item() if isinstance(x, torch.Tensor) else float(x) if isinstance(x, (int, float)) else x for x in EI_list]) write_excel(metric_save_name, 'Qy', i + 1, [x.item() if isinstance(x, torch.Tensor) else float(x) if isinstance(x, (int, float)) else x for x in Qy_list]) write_excel(metric_save_name, 'Qcb', i + 1, [x.item() if isinstance(x, torch.Tensor) else float(x) if isinstance(x, (int, float)) else x for x in Qcb_list]) write_excel(metric_save_name, 'EN', i + 1, [x.item() if isinstance(x, torch.Tensor) else float(x) if isinstance(x, (int, float)) else x for x in EN_list]) write_excel(metric_save_name, 'MI', i + 1, [x.item() if isinstance(x, torch.Tensor) else float(x) if isinstance(x, (int, float)) else x for x in MI_list]) write_excel(metric_save_name, 'SF', i + 1, [x.item() if isinstance(x, torch.Tensor) else float(x) if isinstance(x, (int, float)) else x for x in SF_list]) write_excel(metric_save_name, 'AG', i + 1, [x.item() if isinstance(x, torch.Tensor) else float(x) if isinstance(x, (int, float)) else x for x in AG_list]) write_excel(metric_save_name, 'SD', i + 1, [x.item() if isinstance(x, torch.Tensor) else float(x) if isinstance(x, (int, float)) else x for x in SD_list]) write_excel(metric_save_name, 'CC', i + 1, [x.item() if isinstance(x, torch.Tensor) else float(x) if isinstance(x, (int, float)) else x for x in CC_list]) write_excel(metric_save_name, 'SCD', i + 1, [x.item() if isinstance(x, torch.Tensor) else float(x) if isinstance(x, (int, float)) else x for x in SCD_list]) write_excel(metric_save_name, 'VIF', i + 1, [x.item() if isinstance(x, torch.Tensor) else float(x) if isinstance(x, (int, float)) else x for x in VIF_list]) write_excel(metric_save_name, 'MSE', i + 1, [x.item() if isinstance(x, torch.Tensor) else float(x) if isinstance(x, (int, float)) else x for x in MSE_list]) write_excel(metric_save_name, 'PSNR', i + 1, [x.item() if isinstance(x, torch.Tensor) else float(x) if isinstance(x, (int, float)) else x for x in PSNR_list]) write_excel(metric_save_name, 'Qabf', i + 1, Qabf_list) write_excel(metric_save_name, 'Nabf', i + 1, [x.item() if isinstance(x, torch.Tensor) else float(x) if isinstance(x, (int, float)) else x for x in Nabf_list]) write_excel(metric_save_name, 'SSIM', i + 1, [x.item() if isinstance(x, torch.Tensor) else float(x) if isinstance(x, (int, float)) else x for x in SSIM_list]) write_excel(metric_save_name, 'MS_SSIM', i + 1, [x.item() if isinstance(x, torch.Tensor) else float(x) if isinstance(x, (int, float)) else x for x in MS_SSIM_list]) ================================================ FILE: Metric/ssim.py ================================================ import warnings import torch import torch.nn as nn import torch.nn.functional as F import torchvision.transforms.functional as TF import numpy as np def _fspecial_gauss_1d(size, sigma): coords = torch.arange(size, dtype=torch.float32) coords -= size // 2 g = torch.exp(-(coords ** 2) / (2 * sigma ** 2)) g /= g.sum() return g.unsqueeze(0).unsqueeze(0) def gaussian_filter(input, win): assert all([ws == 1 for ws in win.shape[1:-1]]), win.shape if len(input.shape) == 4: conv = F.conv2d elif len(input.shape) == 5: conv = F.conv3d else: raise NotImplementedError(input.shape) C = input.shape[1] out = input for i, s in enumerate(input.shape[2:]): if s >= win.shape[-1]: perms = list(range(win.ndim)) perms[2 + i] = perms[-1] perms[-1] = 2 + i out = conv(out, weight=win.permute(perms), stride=1, padding=0, groups=C) else: warnings.warn( f"Skipping Gaussian Smoothing at dimension 2+{i} for input: {input.shape} and win size: {win.shape[-1]}" ) return out def _ssim(X, Y, data_range, win, K=(0.01, 0.03)): K1, K2 = K compensation = 1.0 C1 = (K1 * data_range) ** 2 C2 = (K2 * data_range) ** 2 win = win.type_as(X) mu1 = gaussian_filter(X, win) mu2 = gaussian_filter(Y, win) mu1_sq = mu1.pow(2) mu2_sq = mu2.pow(2) mu1_mu2 = mu1 * mu2 sigma1_sq = compensation * (gaussian_filter(X * X, win) - mu1_sq) sigma2_sq = compensation * (gaussian_filter(Y * Y, win) - mu2_sq) sigma12 = compensation * (gaussian_filter(X * Y, win) - mu1_mu2) cs_map = (2 * sigma12 + C2) / (sigma1_sq + sigma2_sq + C2) ssim_map = ((2 * mu1_mu2 + C1) / (mu1_sq + mu2_sq + C1)) * cs_map ssim_per_channel = torch.flatten(ssim_map, 2).mean(-1) cs = torch.flatten(cs_map, 2).mean(-1) return ssim_per_channel, cs def ssim(X, Y, data_range=255, size_average=True, win_size=11, win_sigma=1.5, win=None, K=(0.01, 0.03), nonnegative_ssim=False): X = TF.to_tensor(X).unsqueeze(0).unsqueeze(0) Y = TF.to_tensor(Y).unsqueeze(0).unsqueeze(0) if not X.shape == Y.shape: raise ValueError("Input images should have the same dimensions.") for d in range(len(X.shape) - 1, 1, -1): X = torch.squeeze(X, dim=d) Y = torch.squeeze(Y, dim=d) if len(X.shape) not in (4, 5): raise ValueError(f"Input images should be 4-d or 5-d tensors, but got {X.shape}") if not X.dtype == Y.dtype: raise ValueError("Input images should have the same dtype.") if win is not None: # set win_size win_size = win.shape[-1] if not (win_size % 2 == 1): raise ValueError("Window size should be odd.") if win is None: win = _fspecial_gauss_1d(win_size, win_sigma) win = win.repeat([X.shape[1]] + [1] * (len(X.shape) - 1)) ssim_per_channel, _ = _ssim(X, Y, data_range=data_range, win=win, K=K) if nonnegative_ssim: ssim_per_channel = F.relu(ssim_per_channel) if size_average: return ssim_per_channel.mean() else: return ssim_per_channel.mean(dim=1) def ms_ssim( X, Y, data_range=255, size_average=True, win_size=11, win_sigma=1.5, win=None, weights=None, K=(0.01, 0.03) ): X = TF.to_tensor(X).unsqueeze(0).unsqueeze(0) Y = TF.to_tensor(Y).unsqueeze(0).unsqueeze(0) if not X.shape == Y.shape: raise ValueError("Input images should have the same dimensions.") for d in range(len(X.shape) - 1, 1, -1): X = X.squeeze(dim=d) Y = Y.squeeze(dim=d) if not X.dtype == Y.dtype: raise ValueError("Input images should have the same dtype.") if len(X.shape) == 4: avg_pool = F.avg_pool2d elif len(X.shape) == 5: avg_pool = F.avg_pool3d else: raise ValueError(f"Input images should be 4-d or 5-d tensors, but got {X.shape}") if win is not None: win_size = win.shape[-1] if not (win_size % 2 == 1): raise ValueError("Window size should be odd.") smaller_side = min(X.shape[-2:]) assert smaller_side > (win_size - 1) * ( 2 ** 4 ), "Image size should be larger than %d due to the 4 downsamplings in ms-ssim" % ((win_size - 1) * (2 ** 4)) if weights is None: weights = [0.0448, 0.2856, 0.3001, 0.2363, 0.1333] weights = torch.tensor(weights, dtype=X.dtype) if win is None: win = _fspecial_gauss_1d(win_size, win_sigma) win = win.repeat([X.shape[1]] + [1] * (len(X.shape) - 1)) levels = weights.shape[0] mcs = [] for i in range(levels): ssim_per_channel, cs = _ssim(X, Y, win=win, data_range=data_range, K=K) if i < levels - 1: mcs.append(F.relu(cs)) padding = [s % 2 for s in X.shape[2:]] X = avg_pool(X, kernel_size=2, padding=padding) Y = avg_pool(Y, kernel_size=2, padding=padding) ssim_per_channel = F.relu(ssim_per_channel) mcs_and_ssim = torch.stack(mcs + [ssim_per_channel], dim=0) ms_ssim_val = torch.prod(mcs_and_ssim ** weights.reshape((-1, 1, 1)), dim=0) if size_average: return ms_ssim_val.mean() else: return ms_ssim_val.mean(dim=1) class SSIM(nn.Module): def __init__( self, data_range=255, size_average=True, win_size=11, win_sigma=1.5, channel=3, spatial_dims=2, K=(0.01, 0.03), nonnegative_ssim=False, ): super(SSIM, self).__init__() self.win_size = win_size self.win = _fspecial_gauss_1d(win_size, win_sigma).tile([channel, 1] + [1] * spatial_dims) self.size_average = size_average self.data_range = data_range self.K = K self.nonnegative_ssim = nonnegative_ssim def forward(self, X, Y): return ssim( X, Y, data_range=self.data_range, size_average=self.size_average, win=self.win, K=self.K, nonnegative_ssim=self.nonnegative_ssim, ).item() class MS_SSIM(nn.Module): def __init__( self, data_range=255, size_average=True, win_size=11, win_sigma=1.5, channel=3, spatial_dims=2, weights=None, K=(0.01, 0.03), ): super(MS_SSIM, self).__init__() self.win_size = win_size self.win = _fspecial_gauss_1d(win_size, win_sigma).tile([channel, 1] + [1] * spatial_dims) self.size_average = size_average self.data_range = data_range self.weights = weights self.K = K def forward(self, X, Y): return ms_ssim( X, Y, data_range=self.data_range, size_average=self.size_average, win=self.win, weights=self.weights, K=self.K, ).item() ================================================ FILE: README.md ================================================ ## Latest News 🔥🔥 [2024-12-12] Our survey paper [__Infrared and Visible Image Fusion: From Data Compatibility to Task Adaption.__] has been accepted by IEEE Transactions on Pattern Analysis and Machine Intelligence! ([Paper](https://ieeexplore.ieee.org/abstract/document/10812907))([中文版](https://pan.baidu.com/s/1EIRYSULa-pd2FRmIdG693g?pwd=aiey)) [2026-04-15] We have updated the repository with state-of-the-art methods for both Image Fusion and Video Fusion. # IVIF Zoo Welcome to IVIF Zoo, a comprehensive repository dedicated to Infrared and Visible Image Fusion (IVIF). Based on our survey paper [__Infrared and Visible Image Fusion: From Data Compatibility to Task Adaption.__ *Jinyuan Liu, Guanyao Wu, Zhu Liu, Di Wang, Zhiying Jiang, Long Ma, Wei Zhong, Xin Fan, Risheng Liu**], this repository aims to serve as a central hub for researchers, engineers, and enthusiasts in the field of IVIF. Here, you'll find a wide array of resources, tools, and datasets, curated to accelerate advancements and foster collaboration in infrared-visible image fusion technologies. *** ![preview](assets/light2.png) A detailed spectrogram depicting almost all wavelength and frequency ranges, particularly expanding the range of the human visual system and annotating corresponding computer vision and image fusion datasets. ![preview](assets/pipeline1.png) The diagram of infrared and visible image fusion for practical applications. Existing image fusion methods majorly focus on the design of architectures and training strategies for visual enhancement, few considering the adaptation for downstream visual perception tasks. Additionally, from the data compatibility perspective, pixel misalignment and adversarial attacks of image fusion are two major challenges. Additionally, integrating comprehensive semantic information for tasks like semantic segmentation, object detection, and salient object detection remains underexplored, posing a critical obstacle in image fusion. ![preview](assets/sankey1.png) A classification sankey diagram containing typical fusion methods. *** ## 导航(Navigation) - [数据集 (Datasets)](#数据集datasets) - [方法集 (Method Set)](#方法集method-set) - [纯融合方法 (Fusion for Visual Enhancement)](#纯融合方法fusion-for-visual-enhancement) - [数据兼容方法 (Data Compatible)](#数据兼容方法data-compatible) - [面向应用方法 (Application-oriented)](#面向应用方法application-oriented) - [评价指标 (Evaluation Metric)](#评价指标evaluation-metric) ### [🔥🚀资源库 (Resource Library)](#资源库resource-library) `It covers all results of our survey paper, available for download from Baidu Cloud.` - 💥[融合 (Fusion)](#融合fusion) - ✂️[分割 (Segmentation)](#分割segmentation) `Based on SegFormer` - 🔍[检测 (Detection)](#检测detection) `Based on YOLO-v5` - [计算效率 (Computational Efficiency)](#计算效率computational-efficiency) # 数据集(Datasets) ## 图像数据集（Image Datasets）
Dataset Img pairs Resolution Color Obj/Cats Cha-Sc Anno DownLoad
TNO 261 768×576 ❌ few ✔ ❌ Link
RoadScene 🔥 221 Various ✔ medium ❌ ❌ Link
VIFB 21 Various Various few ❌ ❌ Link
MS 2999 768×576 ✔ 14146 / 6 ❌ ✔ Link
LLVIP 16836 1280×720 ✔ pedestrian / 1 ❌ ✔ Link
M3FD 🔥 4200 1024×768 ✔ 33603 / 6 ✔ ✔ Link
MFNet 1569 640×480 ✔ abundant / 8 ❌ ✔ Link
FMB 🔥 1500 800×600 ✔ abundant / 14 ❌ ✔ Link
## 视频数据集（Video Datasets）
Dataset Video Count Total Frames Resolution DownLoad
VF-Bench 797 Over 200,000 2K/540p/480p Link
HDO 24 7,500 640×480 Link
M3SVD 220 153,797 640×480 Link
If the M3FD and FMB datasets are helpful to you, please cite the following paper: ``` @inproceedings{liu2022target, title={Target-aware dual adversarial learning and a multi-scenario multi-modality benchmark to fuse infrared and visible for object detection}, author={Liu, Jinyuan and Fan, Xin and Huang, Zhanbo and Wu, Guanyao and Liu, Risheng and Zhong, Wei and Luo, Zhongxuan}, booktitle={Proceedings of the IEEE/CVF conference on computer vision and pattern recognition}, pages={5802--5811}, year={2022} } ``` ``` @inproceedings{liu2023multi, title={Multi-interactive feature learning and a full-time multi-modality benchmark for image fusion and segmentation}, author={Liu, Jinyuan and Liu, Zhu and Wu, Guanyao and Ma, Long and Liu, Risheng and Zhong, Wei and Luo, Zhongxuan and Fan, Xin}, booktitle={Proceedings of the IEEE/CVF international conference on computer vision}, pages={8115--8124}, year={2023} } ``` # 方法集(Method Set) ## 纯融合方法(Fusion for Visual Enhancement)
Aspects
(分类)		 Methods
(方法)		 Title
(标题)		 Venue
(发表场所)		 Source
(资源)
Auto-Encoder DenseFuse Densefuse: A fusion approach to infrared and visible images TIP '18 Paper/Code
Auto-Encoder SEDRFuse Sedrfuse: A symmetric encoder–decoder with residual block network for infrared and visible image fusion TIM '20 Paper/Code
Auto-Encoder DIDFuse Didfuse: Deep image decomposition for infrared and visible image fusion IJCAI '20 Paper/Code
Auto-Encoder MFEIF Learning a deep multi-scale feature ensemble and an edge-attention guidance for image fusion TCSVT '21 Paper/Code
Auto-Encoder RFN-Nest Rfn-nest: An end-to-end residual fusion network for infrared and visible images TIM '21 Paper/Code
Auto-Encoder SFAFuse Self-supervised feature adaption for infrared and visible image fusion InfFus '21 Paper/Code
Auto-Encoder SMoA Smoa: Searching a modality-oriented architecture for infrared and visible image fusion SPL '21 Paper/Code
Auto-Encoder Re2Fusion Res2fusion: Infrared and visible image fusion based on dense res2net and double nonlocal attention models TIM '22 Paper/Code
Auto-Encoder RPFNet Residual Prior-driven Frequency-aware Network for Image Fusion ACM MM '25 Paper/Code
Auto-Encoder TTD Test-Time Dynamic Image Fusion NeurIPS '24 Paper/Code
GAN FusionGAN Fusiongan: A generative adversarial network for infrared and visible image fusion InfFus '19 Paper/Code
GAN DDcGAN Learning a generative model for fusing infrared and visible images via conditional generative adversarial network with dual discriminators TIP '19 Paper/Code
GAN AtFGAN Attentionfgan: Infrared and visible image fusion using attention-based generative adversarial networks TMM '20 Paper
GAN DPAL Infrared and visible image fusion via detail preserving adversarial learning InfFus '20 Paper/Code
GAN D2WGAN Infrared and visible image fusion using dual discriminators generative adversarial networks with wasserstein distance InfSci '20 Paper
GAN GANMcC Ganmcc: A generative adversarial network with multiclassification constraints for infrared and visible image fusion TIM '20 Paper/Code
GAN ICAFusion Infrared and visible image fusion via interactive compensatory attention adversarial learning TMM '22 Paper/Code
GAN TCGAN Transformer based conditional gan for multimodal image fusion TMM '23 Paper/Code
GAN DCFusion DCFusion: A Dual-Frequency Cross-Enhanced Fusion Network for Infrared and Visible Image Fusion TIM '23 Paper
GAN FreqGAN Freqgan: Infrared and visible image fusion via unified frequency adversarial learning TCSVT '24 Paper/Code
GAN DDBF Dispel Darkness for Better Fusion: A Controllable Visual Enhancer based on Cross-modal Conditional Adversarial Learning CVPR '24 Paper/Code
GAN CCF Conditional Controllable Image Fusion NeurIPS '24 Paper/Code
CNN BIMDL A bilevel integrated model with data-driven layer ensemble for multi-modality image fusion TIP '20 Paper
CNN MgAN-Fuse Multigrained attention network for infrared and visible image fusion TIM '20 Paper
CNN AUIF Efficient and model-based infrared and visible image fusion via algorithm unrolling TCSVT '21 Paper/Code
CNN RXDNFuse Rxdnfuse: A aggregated residual dense network for infrared and visible image fusion InfFus '21 Paper
CNN STDFusionNet Stdfusionnet: An infrared and visible image fusion network based on salient target detection TIM '21 Paper/Code
CNN CUFD Cufd: An encoder–decoder network for visible and infrared image fusion based on common and unique feature decomposition CVIU '22 Paper/Code
CNN Dif-Fusion Dif-fusion: Towards high color fidelity in infrared and visible image fusion with diffusion models TIP '23 Paper/Code
CNN L2Net L2Net: Infrared and Visible Image Fusion Using Lightweight Large Kernel Convolution Network TIP '23 Paper/Code
CNN IGNet Learning a graph neural network with cross modality interaction for image fusion ACMMM '23 Paper/Code
CNN LRRNet Lrrnet: A novel representation learning guided fusion network for infrared and visible images TPAMI '23 Paper/Code
CNN MetaFusion Metafusion: Infrared and visible image fusion via meta-feature embedding from object detection CVPR '23 Paper/Code
CNN PSFusion Rethinking the necessity of image fusion in high-level vision tasks: A practical infrared and visible image fusion network based on progressive semantic injection and scene fidelity InfFus '23 Paper/Code
CNN LUT-Fuse LUT-Fuse: Towards Extremely Fast Infrared and Visible Image Fusion via Distillation to Learnable Look-Up Tables ICCV '25 Paper/Code
CNN PMAINet Progressive Modality-Adaptive Interactive Network for Multi-Modality Image Fusion IJCAI '25 Paper
Transformer SwinFusion Swinfusion: Cross-domain long-range learning for general image fusion via swin transformer JAS '22 Paper/Code
Transformer YDTR Ydtr: Infrared and visible image fusion via y-shape dynamic transformer TMM '22 Paper/Code
Transformer IFT Image fusion transformer ICIP '22 Paper/Code
Transformer CDDFuse Cddfuse: Correlation-driven dual-branch feature decomposition for multi-modality image fusion CVPR '23 Paper/Code
Transformer TGFuse Tgfuse: An infrared and visible image fusion approach based on transformer and generative adversarial network TIP '23 Paper/Code
Transformer CMTFusion Cross-modal transformers for infrared and visible image fusion TCSVT '23 Paper/Code
Transformer Text-IF Text-if: Leveraging semantic text guidance for degradation-aware and interactive image fusion CVPR '24 Paper/Code
Transformer PromptF Promptfusion: Harmonized semantic prompt learning for infrared and visible image fusion JAS '24
Transformer MaeFuse MaeFuse: Transferring Omni Features With Pretrained Masked Autoencoders for Infrared and Visible Image Fusion via Guided Training TIP '25 Paper/Code
Transformer Fusion with Language-driven Infrared and Visible Image Fusion with Language-Driven Loss in CLIP Embedding Space ACM MM '24 Paper/Code
## 数据兼容方法(Data Compatible)
Aspects
(分类)		 Methods
(方法)		 Title
(标题)		 Venue
(发表场所)		 Source
(资源)
Registration UMIR Unsupervised multi-modal image registration via geometry preserving image-to-image translation CVPR ‘20 Paper/Code
Registration ReCoNet Reconet: Recurrent correction network for fast and efficient multi-modality image fusion ECCV ‘22 Paper/Code
Registration SuperFusion Superfusion: A versatile image registration and fusion network with semantic awareness JAS ‘22 Paper/Code
Registration UMFusion Unsupervised misaligned infrared and visible image fusion via cross-modality image generation and registration IJCAI ‘22 Paper/Code
Registration GCRF General cross-modality registration framework for visible and infrared UAV target image registration SR ‘23 Paper
Registration MURF MURF: mutually reinforcing multi-modal image registration and fusion TPAMI ‘23 Paper/Code
Registration SemLA Semantics lead all: Towards unified image registration and fusion from a semantic perspective InfFus ‘23 Paper/Code
Registration - A Deep Learning Framework for Infrared and Visible Image Fusion Without Strict Registration IJCV ‘23 Paper
Attack PAIFusion PAIF: Perception-aware infrared-visible image fusion for attack-tolerant semantic segmentation ACMMM ‘23 Paper/Code
General FusionDN FusionDN: A unified densely connected network for image fusion AAAI ‘20 Paper/Code
General IFCNN IFCNN: A general image fusion framework based on convolutional neural network InfFus ‘20 Paper/Code
General PMGI Rethinking the image fusion: A fast unified image fusion network based on proportional maintenance of gradient and intensity AAAI ‘20 Paper/Code
General U2Fusion U2Fusion: A unified unsupervised image fusion network TPAMI ‘20 Paper/Code
General SDNet SDNet: A versatile squeeze-and-decomposition network for real-time image fusion IJCV ‘21 Paper/Code
General CoCoNet CoCoNet: Coupled contrastive learning network with multi-level feature ensemble for multi-modality image fusion IJCV ‘23 Paper/Code
General DDFM DDFM: Denoising diffusion model for multi-modality image fusion ICCV ‘23 Paper/Code
General EMMA Equivariant multi-modality image fusion CVPR ‘24 Paper/Code
General FILM Image fusion via vision-language model ICML ‘24 Paper/Code
General VDMUFusion VDMUFusion: A Versatile Diffusion Model-Based Unsupervised Framework for Image Fusion TIP ‘24 Paper/Code
General TC-MoA Task-Customized Mixture of Adapters for General Image Fusion CVPR '24 Paper/Code
General SHIP Probing Synergistic High-Order Interaction in Infrared and Visible Image Fusion CVPR '24 Paper/Code
Transformer GIFNet One Model for ALL: Low-Level Task Interaction Is a Key to Task-Agnostic Image Fusion CVPR '25 Paper/Code
## 面向应用方法(Application-oriented)
Aspects
(分类)		 Methods
(方法)		 Title
(标题)		 Venue
(发表场所)		 Source
(资源)
Perception DetFusion A detection-driven infrared and visible image fusion network ACMMM ‘22 Paper/Code
Perception SeAFusion Image fusion in the loop of high-level vision tasks: A semantic-aware real-time infrared and visible image fusion network InfFus ‘22 Paper/Code
Perception TarDAL Target-aware dual adversarial learning and a multi-scenario multimodality benchmark to fuse infrared and visible for object detection CVPR ‘22 Paper/Code
Perception BDLFusion Bi-level dynamic learning for jointly multi-modality image fusion and beyond IJCAI ‘23 Paper/Code
Perception IRFS An interactively reinforced paradigm for joint infrared-visible image fusion and saliency object detection InfFus ‘23 Paper/Code
Perception MetaFusion Metafusion: Infrared and visible image fusion via meta-feature embedding from object detection CVPR ‘23 Paper/Code
Perception MoE-Fusion Multi-modal gated mixture of local-to-global experts for dynamic image fusion ICCV ‘23 Paper/Code
Perception SegMiF Multi-interactive feature learning and a full-time multimodality benchmark for image fusion and segmentation ICCV ‘23 Paper/Code
Perception CAF Where elegance meets precision: Towards a compact, automatic, and flexible framework for multi-modality image fusion and applications IJCAI ‘24 Paper/Code
Perception MRFS Mrfs: Mutually reinforcing image fusion and segmentation CVPR ‘24 Paper/Code
Perception TIMFusion A task-guided, implicitly searched and meta-initialized deep model for image fusion TPAMI ‘24 Paper/Code
Perception SAGE Every SAM Drop Counts: Embracing Semantic Priors for Multi-Modality Image Fusion and Beyond CVPR ‘25 Paper/Code
Perception DCEvo DCEvo: Discriminative Cross-Dimensional Evolutionary Learning for Infrared and Visible Image Fusion CVPR ‘25 Paper/Code
Perception TDFusion Task-driven Image Fusion with Learnable Fusion Loss CVPR ‘25 Paper/Code
Perception EVIF Event-based Visible and Infrared Fusion via Multi-task Collaboration  CVPR ‘24 Paper/Code
Perception MMAIF MMAIF: Multi-task and Multi-degradation All-in-One for Image Fusion with Language Guidance ICCV ‘25 Paper/Code
Perception CMFS CMFS: CLIP-Guided Modality Interaction for Mitigating Noise in Multi-Modal Image Fusion and Segmentation IJCAI ‘25 Paper/Code
Perception A²RNet A²RNet: Adversarial Attack Resilient Network for Robust Infrared and Visible Image Fusion AAAI ‘25 Paper/Code
Perception SDSFusion SDSFusion: A Semantic-Aware Infrared and Visible Image Fusion Network for Degraded Scenes TIP ‘25 Paper/Code
Perception S4Fusion S4Fusion: Saliency-Aware Selective State Space Model for Infrared and Visible Image Fusion TIP ‘25 Paper/Code
Perception FreeFusion FreeFusion: Infrared and Visible Image Fusion via Cross Reconstruction Learning TPAMI ‘25 Paper
Perception MulFS-CAP MulFS-CAP: Multimodal Fusion-Supervised Cross-Modality Alignment Perception for Unregistered Infrared-Visible Image Fusion TPAMI ‘25 Paper/Code
# 最新研究进展(Latest Research Progress)
Methods
(方法)		 Title
(标题)		 Venue
(发表场所)		 Source
(资源)
SAGE Every SAM Drop Counts: Embracing Semantic Priors for Multi-Modality Image Fusion and Beyond CVPR ‘25 Paper/Code
DCEvo DCEvo: Discriminative Cross-Dimensional Evolutionary Learning for Infrared and Visible Image Fusion CVPR ‘25 Paper/Code
TDFusion Task-driven Image Fusion with Learnable Fusion Loss CVPR ‘25 Paper/Code
GIFNet One Model for ALL: Low-Level Task Interaction Is a Key to Task-Agnostic Image Fusion CVPR '25 Paper/Code
RIS-Fuse Highlight What You Want: Weakly-Supervised Instance-Level Controllable Infrared-Visible Image Fusion ICCV '25 Paper/Code
SCA The Source Image is the Best Attention for Infrared and Visible Image Fusion ICCV '25 Paper
TITA Balancing Task-invariant Interaction and Task-specific Adaptation for Unified Image Fusion ICCV '25 Paper/Code
DreamFuse DreamFuse: Adaptive Image Fusion with Diffusion Transformer ICCV '25 Paper/Code
TRACE Toward a Training-Free Plug-and-Play Refinement Framework for Infrared and Visible Image Registration and Fusion ACM MM '25 Paper/Code
HRFusion Prior-Constrained Relevant Feature driven Image Fusion with Hybrid Feature via Mode Decomposition ACM MM '25 Paper/Code
TG-ECNet Task-Gated Multi-Expert Collaboration Network for Degraded Multi-Modal Image Fusion ICML '25 Paper/Code
Deno-IF Deno-IF: Unsupervised Noisy Visible and Infrared Image Fusion via Mimicking Textures from Clean Images NeurIPS '25 Paper/Code
HCLFuse Revisiting Generative Infrared and Visible Image Fusion Based on Human Cognitive Laws NeurIPS '25 Paper/Code
RFfusion Efficient Rectified Flow for Image Fusion NeurIPS '25 Paper/Code
ControlFusion ControlFusion: A Controllable Image Fusion Framework with Language-Vision Degradation Prompts NeurIPS '25 Paper/Code
PDFuse Projection-Manifold Regularized Latent Diffusion for Robust General Image Fusion NeurIPS '25 Paper/Code
DAFusion Domain Adaptation Guided Infrared and Visible Image Fusion AAAI '26 Paper/Code
HMMF A Hybrid Space Model for Misaligned Multi-modality Image Fusion AAAI '26 Paper/Code
CtrlFuse CtrlFuse: Mask-Prompt Guided Controllable Infrared and Visible Image Fusion AAAI '26 Paper/Code
MdaIF MdaIF: Robust One-Stop Multi-Degradation-Aware Image Fusion with Language-Driven Semantics AAAI '26 Paper/Code
SMC Self-supervised Multiplex Consensus Mamba for General Image Fusion AAAI '26 Paper
OCCO OCCO: LVM-Guided Infrared and Visible Image Fusion Framework Based on Object-Aware and Contextual Contrastive Learning IJCV '25 Paper/Code
MagicFuse One Model for ALL: Low-Level Task Interaction Is a Key to Task-Agnostic Image Fusion CVPR '26 Paper
UniFusion One Model for ALL: Low-Level Task Interaction Is a Key to Task-Agnostic Image Fusion CVPR '26 Paper/Code
OmniFuse OmniFuse: Composite Degradation-Robust Image Fusion With Language-Driven Semantics TPAMI '25 Paper/Code
DiTFuse Towards Uniffed Semantic and Controllable Image Fusion: A Diffusion Transformer Approach TPAMI '25 Paper/Code
Mask-DiFuser Mask-DiFuser: A Masked Diffusion Model for Unified Unsupervised Image Fusion TPAMI '25 Paper/Code
FuseAgent FUSEAGENT: A VLM-DRIVEN AGENT FOR UNIFIED IN-THE-WILD IMAGE FUSION ICLR '26 Paper/Code
URFusion URFusion: Unsupervised Unified Degradation-Robust Image Fusion Network TIP '25 Paper/Code
# 视频融合(Video Fusion)
Methods
(方法)		 Title
(标题)		 Venue
(发表场所)		 Source
(资源)
UniVF A Unified Solution to Video Fusion: FromMulti-Frame Learning to Benchmarking NeurIPS '25 Paper/Code
RCVS RCVS: A Unified Registration and Fusion Framework for Video Streams TMM '24 Paper
VideoFusion VideoFusion: A Spatio-Temporal Collaborative Network for Multi-modal Video Fusion CVPR '26 Paper/Code
CMVF CMVF: Cross-modal unregistered video fusion via spatio-temporal consistency InfFus '26 Paper/Code
Seq-IF Seq-IF: Sequentially Consistent Infrared-Visible Video Fusion under Time-Varying Illumination for Perception Enhancement TCSVT '26 Paper
TemCoCo TemCoCo: Temporally Consistent Multi-modal Video Fusion with Visual-Semantic Collaboration ICCV '25 Paper/Code
# 评价指标(Evaluation Metric) We integrated the code for calculating metrics and used GPU acceleration with PyTorch, significantly improving the speed of computing metrics across multiple methods and images. You can find it at [Metric](https://github.com/RollingPlain/IVIF_ZOO/tree/main/Metric) If you want to calculate metrics using our code, you can run: ```python # Please modify the data path in 'eval_torch.py'. python eval_torch.py ``` # 资源库(Resource Library) ## 融合(Fusion) Fusion images from multiple datasets in the IVIF domain are organized in the following form: each subfolder contains fusion images generated by different methods, facilitating research and comparison for users. ``` Fusion ROOT ├── IVIF | ├── FMB | | ├── ... | | ├── CAF # All the file names are named after the methods | | └── ... | ├── # The other files follow the same structure shown above. | ├── M3FD_300 # Mini version of M3FD dataset with 300 images | ├── RoadScene | ├── TNO | └── M3FD_4200.zip # Full version of the M3FD dataset with 4200 images ``` You can directly download from here. Download：[Baidu Yun](https://pan.baidu.com/s/1S6l-CUqE2nRPXeX2P_VScg?pwd=wgtn) ## 分割(Segmentation) Segmentation data is organized in the following form: it contains multiple directories to facilitate the management of segmentation-related data and results. ``` Segmentation ROOT ├── Segformer | ├── datasets | | ├── ... | | ├── CAF # All the file names are named after the methods | | | └──VOC2007 | | | ├── JPEGImages # Fusion result images in JPG format | | | └── SegmentationClass # Ground truth for segmentation | | └── ... # The other files follow the same structure shown above. | ├── model_data | | ├── backbone # Backbone used for segmentation | | └── model # Saved model files | | ├── ... | | ├── CAF.pth # All the model names are named after the methods | | └── ... | ├── results # Saved model files and training results | | ├── iou # IoU results for segmentation validation | | ├── ... | | ├── CAF.txt # All the file names are named after the methods | | └── ... | | └── predict #Visualization of segmentation | | ├── ... | | ├── CAF # All the file names are named after the methods | | └── ... | └── hyperparameters.md # Hyperparameter settings ``` You can directly download from here. Download：[Baidu Yun](https://pan.baidu.com/s/1IZOZU17CA6-zeR8zb1LW3Q?pwd=5rcp) ## 检测(Detection) Detection data is organized in the following form: it contains multiple directories to facilitate the management of detection-related data and results. ``` Detection ROOT ├── M3FD | ├── Fused Results | | ├── ... | | ├── CAF # All the file names are named after the methods | | | ├── Images # Fusion result images in PNG format | | | └── Labels # Ground truth for detection | | └── ... # The other files follow the same structure shown above. | ├── model_data | | └── model # Saved model files | | ├── ... | | ├── CAF.pth # All the model names are named after the methods | | └── ... | ├── results # Saved model files and training results | | └── predict #Visualization of detection | | ├── ... | | ├── CAF # All the file names are named after the methods | | └── ... | └── hyperparameters.md # Hyperparameter settings ``` You can directly download from here. Download：[Baidu Yun](https://pan.baidu.com/s/1mC3wTM1DjbBz5mIaDYJLDQ?pwd=a36k) ## 计算效率(Computational Efficiency) - **FLOPS and Params**: - We utilize the `profile` function from the `thop` package to compute the FLOPs (G) and Params (M) counts of the model. ```python from thop import profile # Create ir, vi input tensor ir = torch.randn(1, 1, 1024, 768).to(device) vi = torch.randn(1, 3, 1024, 768).to(device) # Assume 'model' is your network model flops, params = profile(model, inputs=(ir, vi)) ``` - **Time**: - To measure the Time (ms) of the model, we exclude the initial image to compute the average while testing a random selection of 10 image sets from the M3FD dataset, each with a resolution of 1024×768, on the Nvidia GeForce 4090. To eliminate CPU influence, we employ CUDA official event functions to measure running time on the GPU. ```python import torch # Create CUDA events start = torch.cuda.Event(enable_timing=True) end = torch.cuda.Event(enable_timing=True) # Record the start time start.record() # Execute the model # Assume 'model' is your network model fus = model(ir, vi) # Record the end time end.record() ```