Showing preview only (3,037K chars total). Download the full file or copy to clipboard to get everything.
Repository: dvlab-research/BlockNeRFPytorch
Branch: main
Commit: 3d7008d0a012
Files: 273
Total size: 2.3 MB
Directory structure:
gitextract_b74j6bbn/
├── .all-contributorsrc
├── .gitignore
├── FourierGrid/
│ ├── FourierGrid_ckpt_manager.py
│ ├── FourierGrid_grid.py
│ ├── FourierGrid_model.py
│ ├── __init__.py
│ ├── arf.py
│ ├── bbox_compute.py
│ ├── camera_utils.py
│ ├── common_data_loaders/
│ │ ├── __init__.py
│ │ ├── load_blendedmvs.py
│ │ ├── load_blender.py
│ │ ├── load_co3d.py
│ │ ├── load_common_data.py
│ │ ├── load_deepvoxels.py
│ │ ├── load_free.py
│ │ ├── load_llff.py
│ │ ├── load_nerfpp.py
│ │ ├── load_nsvf.py
│ │ └── load_tankstemple.py
│ ├── configs/
│ │ ├── blendedmvs/
│ │ │ ├── Character.py
│ │ │ ├── Fountain.py
│ │ │ ├── Jade.py
│ │ │ └── Statues.py
│ │ ├── co3d/
│ │ │ └── donut_369_40208_78816.py
│ │ ├── custom/
│ │ │ ├── Madoka.py
│ │ │ ├── Madoka_long.py
│ │ │ ├── Otobai.py
│ │ │ ├── default_forward_facing.py
│ │ │ ├── default_ubd_inward_facing.py
│ │ │ ├── sm01_desktop.py
│ │ │ ├── sm02_multiple_desktop.py
│ │ │ └── sm03_meeting.py
│ │ ├── deepvoxels/
│ │ │ ├── armchair.py
│ │ │ ├── cube.py
│ │ │ ├── greek.py
│ │ │ └── vase.py
│ │ ├── default.py
│ │ ├── free_dataset/
│ │ │ ├── grass.py
│ │ │ └── lab.py
│ │ ├── lf/
│ │ │ ├── africa.py
│ │ │ ├── basket.py
│ │ │ ├── lf_default.py
│ │ │ ├── ship.py
│ │ │ ├── statue.py
│ │ │ └── torch.py
│ │ ├── linemod/
│ │ │ ├── ape.py
│ │ │ ├── benchvise.py
│ │ │ ├── camera.py
│ │ │ ├── can.py
│ │ │ ├── cat.py
│ │ │ ├── driller.py
│ │ │ ├── duck.py
│ │ │ ├── eggbox.py
│ │ │ ├── glue.py
│ │ │ ├── holepuncher.py
│ │ │ ├── iron.py
│ │ │ ├── lamp.py
│ │ │ └── phone.py
│ │ ├── llff/
│ │ │ ├── fern.py
│ │ │ ├── fern_lg.py
│ │ │ ├── flower.py
│ │ │ ├── flower_lg.py
│ │ │ ├── fortress.py
│ │ │ ├── fortress_lg.py
│ │ │ ├── horns.py
│ │ │ ├── horns_lg.py
│ │ │ ├── leaves.py
│ │ │ ├── leaves_lg.py
│ │ │ ├── llff_default.py
│ │ │ ├── llff_default_lg.py
│ │ │ ├── orchids.py
│ │ │ ├── orchids_lg.py
│ │ │ ├── room.py
│ │ │ ├── room_lg.py
│ │ │ ├── trex.py
│ │ │ └── trex_lg.py
│ │ ├── mega/
│ │ │ ├── building.py
│ │ │ ├── building_no_block.py
│ │ │ ├── quad.py
│ │ │ └── rubble.py
│ │ ├── nerf/
│ │ │ ├── chair.py
│ │ │ ├── drums.py
│ │ │ ├── ficus.py
│ │ │ ├── hotdog.py
│ │ │ ├── lego.py
│ │ │ ├── materials.py
│ │ │ ├── mic.py
│ │ │ ├── ship.py
│ │ │ └── ship.tensorf.py
│ │ ├── nerf_studio/
│ │ │ ├── Giannini_Hall.py
│ │ │ ├── nerf_studio_default.py
│ │ │ └── stump.py
│ │ ├── nerf_unbounded/
│ │ │ ├── bicycle.py
│ │ │ ├── bicycle_single.py
│ │ │ ├── bonsai.py
│ │ │ ├── bonsai_single.py
│ │ │ ├── counter.py
│ │ │ ├── counter_single.py
│ │ │ ├── garden.py
│ │ │ ├── garden_single.py
│ │ │ ├── kitchen.py
│ │ │ ├── kitchen_single.py
│ │ │ ├── nerf_unbounded_default.py
│ │ │ ├── room.py
│ │ │ ├── room_single.py
│ │ │ ├── stump.py
│ │ │ └── stump_single.py
│ │ ├── nsvf/
│ │ │ ├── Bike.py
│ │ │ ├── Lifestyle.py
│ │ │ ├── Palace.py
│ │ │ ├── Robot.py
│ │ │ ├── Spaceship.py
│ │ │ ├── Steamtrain.py
│ │ │ ├── Toad.py
│ │ │ └── Wineholder.py
│ │ ├── tankstemple/
│ │ │ ├── Barn.py
│ │ │ ├── Barn_lg.py
│ │ │ ├── Caterpillar.py
│ │ │ ├── Caterpillar_lg.py
│ │ │ ├── Family.py
│ │ │ ├── Family_lg.py
│ │ │ ├── Ignatius.py
│ │ │ ├── Ignatius_lg.py
│ │ │ ├── Truck.py
│ │ │ ├── Truck_lg.py
│ │ │ ├── barn_single.py
│ │ │ ├── caterpillar_single.py
│ │ │ └── family_single.py
│ │ ├── tankstemple_unbounded/
│ │ │ ├── M60.py
│ │ │ ├── Playground.py
│ │ │ ├── Train.py
│ │ │ ├── Truck.py
│ │ │ ├── m60_single.py
│ │ │ ├── playground_single.py
│ │ │ ├── train_single.py
│ │ │ ├── truck_single.py
│ │ │ └── tt_default.py
│ │ └── waymo/
│ │ ├── block_0_llff.py
│ │ ├── block_0_tt.py
│ │ ├── waymo_base.py
│ │ ├── waymo_block.py
│ │ └── waymo_no_block.py
│ ├── cuda/
│ │ ├── adam_upd.cpp
│ │ ├── adam_upd_kernel.cu
│ │ ├── render_utils.cpp
│ │ ├── render_utils_kernel.cu
│ │ ├── setup.py
│ │ ├── total_variation.cpp
│ │ ├── total_variation_kernel.cu
│ │ ├── ub360_utils.cpp
│ │ └── ub360_utils_kernel.cu
│ ├── dcvgo.py
│ ├── dmpigo.py
│ ├── dvgo.py
│ ├── grid.py
│ ├── load_everything.py
│ ├── load_mega.py
│ ├── load_waymo.py
│ ├── masked_adam.py
│ ├── pose_utils/
│ │ ├── __init__.py
│ │ ├── image_operators.py
│ │ ├── linemod_constants.py
│ │ ├── linemod_evaluator.py
│ │ ├── model_operations.py
│ │ ├── pose_operators.py
│ │ ├── projection.py
│ │ └── visualization.py
│ ├── pycolmap/
│ │ ├── LICENSE.txt
│ │ ├── README.md
│ │ ├── __init__.py
│ │ ├── pycolmap/
│ │ │ ├── __init__.py
│ │ │ ├── camera.py
│ │ │ ├── database.py
│ │ │ ├── image.py
│ │ │ ├── rotation.py
│ │ │ └── scene_manager.py
│ │ └── tools/
│ │ ├── colmap_to_nvm.py
│ │ ├── delete_images.py
│ │ ├── impute_missing_cameras.py
│ │ ├── save_cameras_as_ply.py
│ │ ├── transform_model.py
│ │ ├── write_camera_track_to_bundler.py
│ │ └── write_depthmap_to_ply.py
│ ├── run_colmap2standard.py
│ ├── run_export_bbox.py
│ ├── run_export_coarse.py
│ ├── run_gen_cam_paths.py
│ ├── run_gtk_analysis.py
│ ├── run_render.py
│ ├── run_train.py
│ ├── tools/
│ │ ├── __init__.py
│ │ ├── colmap_utils/
│ │ │ ├── __init__.py
│ │ │ ├── colmap_read_model.py
│ │ │ ├── colmap_wrapper.py
│ │ │ └── pose_utils.py
│ │ ├── imgs2poses.py
│ │ ├── vis_train.py
│ │ └── vis_volume.py
│ ├── trajectory_generators/
│ │ ├── __init__.py
│ │ ├── interp_traj.py
│ │ ├── mega_traj.py
│ │ └── waymo_traj.py
│ └── utils.py
├── LICENSE
├── README.md
├── block_nerf/
│ ├── __init__.py
│ ├── block_nerf_lightning.py
│ ├── block_nerf_model.py
│ ├── block_visualize.py
│ ├── learning_utils.py
│ ├── metrics.py
│ ├── rendering.py
│ └── waymo_dataset.py
├── data_preprocess/
│ ├── __init__.py
│ ├── comvog_organize_by_cam_pos.py
│ ├── download_waymo.sh
│ ├── fetch_data_from_tf_record.py
│ ├── gdown_download_folder.py
│ ├── get_one_block_meta_and_visualize.py
│ ├── process_mega.sh
│ ├── split_block.py
│ ├── vis.py
│ └── visualize_cameras.py
├── docs/
│ ├── classified_weekly_nerf/
│ │ ├── dynamic.md
│ │ ├── editing.md
│ │ ├── fast.md
│ │ ├── generalization.md
│ │ ├── human.md
│ │ ├── lighting.md
│ │ ├── others.md
│ │ ├── pose-slam.md
│ │ ├── reconstruction.md
│ │ ├── semantic.md
│ │ ├── texture.md
│ │ └── video.md
│ ├── classified_weekly_nerf_cn/
│ │ ├── dynamic.md
│ │ ├── editing.md
│ │ ├── fast.md
│ │ ├── generalization.md
│ │ ├── human.md
│ │ ├── lighting.md
│ │ ├── others.md
│ │ ├── pose-slam.md
│ │ ├── reconstruction.md
│ │ ├── semantic.md
│ │ ├── texture.md
│ │ └── video.md
│ ├── contribute_weekly_nerf.md
│ ├── get_pytorch_waymo_dataset.md
│ ├── parse_markdown.py
│ ├── render_docs_from_csv.py
│ ├── sample_logs/
│ │ ├── create_cluster_mask.txt
│ │ └── merge_sub_modules.txt
│ ├── weekly_nerf.md
│ ├── weekly_nerf_cn.md
│ └── weekly_nerf_meta_data.xlsx
├── eval_block_nerf.py
├── requirements.txt
├── run_FourierGrid.py
├── scripts/
│ ├── block_nerf_eval.sh
│ ├── block_nerf_train.sh
│ ├── create_cluster_mask.sh
│ ├── download_nerfstudio.sh
│ ├── gen_path_FourierGrid.sh
│ ├── merge_sub_modules.sh
│ ├── one_block_train.sh
│ ├── render_FourierGrid.sh
│ ├── sfm_FourierGrid.sh
│ ├── train_FourierGrid.sh
│ └── visualize_FourierGrid.sh
├── setup.py
└── train_block_nerf.py
================================================
FILE CONTENTS
================================================
================================================
FILE: .all-contributorsrc
================================================
{
"files": [
"README.md"
],
"imageSize": 100,
"commit": false,
"contributors": [
{
"login": "sjtuytc",
"name": "Zelin Zhao",
"avatar_url": "https://avatars.githubusercontent.com/u/31123348?v=4",
"profile": "https://sjtuytc.github.io/",
"contributions": [
"code",
"maintenance"
]
},
{
"login": "SEUleaderYang",
"name": "EZ-Yang",
"avatar_url": "https://avatars.githubusercontent.com/u/55042050?v=4",
"profile": "https://github.com/SEUleaderYang",
"contributions": [
"code"
]
},
{
"login": "Alex-Alison-Zhang",
"name": "Alex-Zhang",
"avatar_url": "https://avatars.githubusercontent.com/u/71915735?v=4",
"profile": "https://github.com/Alex-Alison-Zhang",
"contributions": [
"bug"
]
},
{
"login": "FanLu97",
"name": "Fan Lu",
"avatar_url": "https://avatars.githubusercontent.com/u/45007531?v=4",
"profile": "https://fanlu97.github.io/",
"contributions": [
"bug"
]
},
{
"login": "MaybeShewill-CV",
"name": "MaybeShewill-CV",
"avatar_url": "https://avatars.githubusercontent.com/u/15725187?v=4",
"profile": "https://maybeshewill-cv.github.io",
"contributions": [
"bug"
]
},
{
"login": "buer1121",
"name": "buer1121",
"avatar_url": "https://avatars.githubusercontent.com/u/48516434?v=4",
"profile": "https://github.com/buer1121",
"contributions": [
"bug"
]
}
],
"contributorsPerLine": 7,
"projectName": "LargeScaleNeRFPytorch",
"projectOwner": "sjtuytc",
"repoType": "github",
"repoHost": "https://github.com",
"skipCi": true,
"commitConvention": "angular"
}
================================================
FILE: .gitignore
================================================
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib64/
parts/
sdist/
var/
wheels/
pip-wheel-metadata/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
target/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# pyenv
.python-version
# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
#Pipfile.lock
# PEP 582; used by e.g. github.com/David-OConnor/pyflow
__pypackages__/
# Celery stuff
celerybeat-schedule
celerybeat.pid
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
# Pyre type checker
.pyre/
data/
data/*
logs/
logs
data
logs/*
ckpts/
ckpts/*
data
*.png
.vscode
.vscode/*
================================================
FILE: FourierGrid/FourierGrid_ckpt_manager.py
================================================
from FourierGrid.FourierGrid_model import FourierGridModel
from FourierGrid import utils, dvgo, dcvgo, dmpigo
import torch
import pdb
import os
from tqdm import tqdm
from FourierGrid.run_train import create_new_model
import torch.nn.functional as F
class FourierGridCheckpointManager:
def __init__(self, args, cfg) -> None:
super(FourierGridCheckpointManager, self).__init__()
self.args = args
self.cfg = cfg
def load_all_info(self, model, optimizer, ckpt_path, no_reload_optimizer):
ckpt = torch.load(ckpt_path)
start = ckpt['global_step']
model.load_state_dict(ckpt['model_state_dict'])
if not no_reload_optimizer:
optimizer.load_state_dict(ckpt['optimizer_state_dict'])
return model, optimizer, start
def load_existing_model(self, args, cfg, cfg_train, reload_ckpt_path, device):
# not used in training
FourierGrid_datasets = ["waymo", "mega", "nerfpp"]
if cfg.data.dataset_type in FourierGrid_datasets or cfg.model == 'FourierGrid':
model_class = FourierGridModel
elif cfg.data.ndc:
model_class = dmpigo.DirectMPIGO
elif cfg.data.unbounded_inward:
model_class = dcvgo.DirectContractedVoxGO
else:
model_class = dvgo.DirectVoxGO
model, _ = self.load_model(model_class, reload_ckpt_path)
model = model.to(device)
optimizer = utils.create_optimizer_or_freeze_model(model, cfg_train, global_step=0,
verbose=False)
model, optimizer, start = self.load_all_info(
model, optimizer, reload_ckpt_path, args.no_reload_optimizer)
return model, optimizer, start
def save_model(self, global_step, model, optimizer, save_path):
torch.save({
'global_step': global_step,
'model_kwargs': model.get_kwargs(),
'model_state_dict': model.state_dict(),
'optimizer_state_dict': optimizer.state_dict(),
}, save_path)
print(f'Saved checkpoints at', save_path)
def load_model(self, model_class, ckpt_path):
ckpt = torch.load(ckpt_path)
model_args = ckpt['model_kwargs']
model = model_class(**model_args)
model.load_state_dict(ckpt['model_state_dict'])
return model, model_args
@torch.no_grad()
def merge_blocks(self, args, cfg, device):
stage = 'fine'
exp_folder = os.path.join(cfg.basedir, cfg.expname)
paths = [os.path.join(exp_folder, f'{stage}_last_{block_id}.tar') for block_id in range(args.block_num)]
model_class = FourierGridModel
cp1 = paths[0]
m1, m1_args = self.load_model(model_class, cp1)
m1 = m1.to(device)
merged_model, _ = create_new_model(args, cfg, cfg.fine_model_and_render, cfg.fine_train,
m1_args['xyz_min'], m1_args['xyz_max'], stage, None, device)
merged_model = merged_model.to(device)
merged_state_dict = m1.state_dict()
# merge the grids consequently
for idx, cur_cp in enumerate(paths[1:]):
print(f"Meging grid {idx} / {len(paths[1:])}: {cur_cp} ...")
cur_m, _ = self.load_model(model_class, cur_cp)
cur_m = cur_m.to(device)
for key in merged_state_dict:
print(f"Merging model key: {key} ...")
if key in ['density.grid', 'k0.grid'] or 'rgb' in key:
g1, g2 = merged_state_dict[key], cur_m.state_dict()[key]
merged_g = torch.min(g1, g2)
# merged_g = torch.max(g1, g2)
# del g1
# del g2
merged_state_dict[key] = merged_g
# else:
# merged_state_dict[key] = merged_model.state_dict()[key]
# del cur_m
torch.cuda.empty_cache()
if "mask_cache.mask" in merged_state_dict:
merged_state_dict.pop("mask_cache.mask")
merged_model.load_state_dict(merged_state_dict, strict=False)
merged_model.update_occupancy_cache()
# merged_model.export_geometry_for_visualize(os.path.join(exp_folder, "debug.npz"))
return merged_model
================================================
FILE: FourierGrid/FourierGrid_grid.py
================================================
import os
import time
import functools
import numpy as np
from einops import rearrange
import torch
import torch.nn as nn
import torch.nn.functional as F
import pdb
import render_utils_cuda
import total_variation_cuda
def create_grid(type, **kwargs):
if type == 'DenseGrid':
return FourierGrid(**kwargs)
else:
raise NotImplementedError
class NeRFPosEmbedding(nn.Module):
def __init__(self, num_freqs: int, logscale=True):
"""
Defines a function that embeds x to (x, sin(2^k x), cos(2^k x), ...)
"""
super(NeRFPosEmbedding, self).__init__()
if logscale:
self.freq_bands = 2 ** torch.linspace(0, num_freqs - 1, num_freqs)
else:
self.freq_bands = torch.linspace(1, 2 ** (num_freqs - 1), num_freqs)
def forward(self, x: torch.Tensor) -> torch.Tensor:
out = [x]
for freq in self.freq_bands:
out += [torch.sin(freq * x), torch.cos(freq * x)]
return torch.cat(out, -1)
'''
Dense 3D grid
'''
class FourierGrid(nn.Module):
def __init__(self, channels, world_size, xyz_min, xyz_max, use_nerf_pos, fourier_freq_num, config):
super(FourierGrid, self).__init__()
self.channels = channels
self.world_size = world_size
self.register_buffer('xyz_min', torch.Tensor(xyz_min))
self.register_buffer('xyz_max', torch.Tensor(xyz_max))
if use_nerf_pos:
self.nerf_pos_num_freq = fourier_freq_num
self.nerf_pos = NeRFPosEmbedding(num_freqs=self.nerf_pos_num_freq)
self.pos_embed_output_dim = 1 + self.nerf_pos_num_freq * 2
self.grid = nn.Parameter(torch.zeros([self.pos_embed_output_dim, channels, *world_size]))
else:
self.nerf_pos_num_freq = -1
self.pos_embed_output_dim = -1
self.nerf_pos = None
self.grid = nn.Parameter(torch.zeros([1, channels, *world_size]))
def forward(self, xyz):
'''
xyz: global coordinates to query
'''
shape = xyz.shape[:-1]
xyz = xyz.reshape(1,1,1,-1,3)
ind_norm = ((xyz - self.xyz_min) / (self.xyz_max - self.xyz_min)).flip((-1,)) * 2 - 1
if self.nerf_pos is not None:
pos_embed = self.nerf_pos(ind_norm)
out = 0
batch_pos_emb = rearrange(pos_embed, '1 1 1 b (n d) -> n 1 1 b d', d=3)
batch_out = F.grid_sample(self.grid, batch_pos_emb, mode='bilinear', align_corners=True)
out = batch_out.mean(0).reshape(self.channels, -1).T.reshape(*shape, self.channels)
else:
out = F.grid_sample(self.grid, ind_norm, mode='bilinear', align_corners=True)
out = out.reshape(self.channels,-1).T.reshape(*shape, self.channels)
if self.channels == 1:
out = out.squeeze(-1)
return out
def scale_volume_grid(self, new_world_size):
if self.channels == 0:
self.grid = nn.Parameter(torch.zeros([1, self.channels, *new_world_size]))
else:
self.grid = nn.Parameter(
F.interpolate(self.grid.data, size=tuple(new_world_size), mode='trilinear', align_corners=True))
def total_variation_add_grad(self, wx, wy, wz, dense_mode):
'''Add gradients by total variation loss in-place'''
total_variation_cuda.total_variation_add_grad(
self.grid, self.grid.grad, wx, wy, wz, dense_mode)
def get_dense_grid(self):
return self.grid
@torch.no_grad()
def __isub__(self, val):
self.grid.data -= val
return self
def extra_repr(self):
return f'channels={self.channels}, world_size={self.world_size.tolist()}'
def compute_tensorf_feat(xy_plane, xz_plane, yz_plane, x_vec, y_vec, z_vec, f_vec, ind_norm):
# Interp feature (feat shape: [n_pts, n_comp])
xy_feat = F.grid_sample(xy_plane, ind_norm[:,:,:,[1,0]], mode='bilinear', align_corners=True).flatten(0,2).T
xz_feat = F.grid_sample(xz_plane, ind_norm[:,:,:,[2,0]], mode='bilinear', align_corners=True).flatten(0,2).T
yz_feat = F.grid_sample(yz_plane, ind_norm[:,:,:,[2,1]], mode='bilinear', align_corners=True).flatten(0,2).T
x_feat = F.grid_sample(x_vec, ind_norm[:,:,:,[3,0]], mode='bilinear', align_corners=True).flatten(0,2).T
y_feat = F.grid_sample(y_vec, ind_norm[:,:,:,[3,1]], mode='bilinear', align_corners=True).flatten(0,2).T
z_feat = F.grid_sample(z_vec, ind_norm[:,:,:,[3,2]], mode='bilinear', align_corners=True).flatten(0,2).T
# Aggregate components
feat = torch.cat([
xy_feat * z_feat,
xz_feat * y_feat,
yz_feat * x_feat,
], dim=-1)
feat = torch.mm(feat, f_vec)
return feat
def compute_tensorf_val(xy_plane, xz_plane, yz_plane, x_vec, y_vec, z_vec, ind_norm):
# Interp feature (feat shape: [n_pts, n_comp])
xy_feat = F.grid_sample(xy_plane, ind_norm[:,:,:,[1,0]], mode='bilinear', align_corners=True).flatten(0,2).T
xz_feat = F.grid_sample(xz_plane, ind_norm[:,:,:,[2,0]], mode='bilinear', align_corners=True).flatten(0,2).T
yz_feat = F.grid_sample(yz_plane, ind_norm[:,:,:,[2,1]], mode='bilinear', align_corners=True).flatten(0,2).T
x_feat = F.grid_sample(x_vec, ind_norm[:,:,:,[3,0]], mode='bilinear', align_corners=True).flatten(0,2).T
y_feat = F.grid_sample(y_vec, ind_norm[:,:,:,[3,1]], mode='bilinear', align_corners=True).flatten(0,2).T
z_feat = F.grid_sample(z_vec, ind_norm[:,:,:,[3,2]], mode='bilinear', align_corners=True).flatten(0,2).T
# Aggregate components
feat = (xy_feat * z_feat).sum(-1) + (xz_feat * y_feat).sum(-1) + (yz_feat * x_feat).sum(-1)
return feat
''' Mask grid
It supports query for the known free space and unknown space.
'''
class MaskGrid(nn.Module):
def __init__(self, path=None, mask_cache_thres=None, mask=None, xyz_min=None, xyz_max=None):
super(MaskGrid, self).__init__()
if path is not None:
st = torch.load(path)
self.mask_cache_thres = mask_cache_thres
density = F.max_pool3d(st['model_state_dict']['density.grid'], kernel_size=3, padding=1, stride=1)
alpha = 1 - torch.exp(-F.softplus(density + st['model_state_dict']['act_shift']) * st['model_kwargs']['voxel_size_ratio'])
mask = (alpha >= self.mask_cache_thres).squeeze(0).squeeze(0)
xyz_min = torch.Tensor(st['model_kwargs']['xyz_min'])
xyz_max = torch.Tensor(st['model_kwargs']['xyz_max'])
else:
mask = mask.bool()
xyz_min = torch.Tensor(xyz_min)
xyz_max = torch.Tensor(xyz_max)
self.register_buffer('mask', mask)
xyz_len = xyz_max - xyz_min
self.register_buffer('xyz2ijk_scale', (torch.Tensor(list(mask.shape)) - 1) / xyz_len)
self.register_buffer('xyz2ijk_shift', -xyz_min * self.xyz2ijk_scale)
@torch.no_grad()
def forward(self, xyz):
'''Skip know freespace
@xyz: [..., 3] the xyz in global coordinate.
'''
shape = xyz.shape[:-1]
xyz = xyz.reshape(-1, 3)
mask = render_utils_cuda.maskcache_lookup(self.mask, xyz, self.xyz2ijk_scale, self.xyz2ijk_shift)
mask = mask.reshape(shape)
return mask
def extra_repr(self):
return f'mask.shape=list(self.mask.shape)'
================================================
FILE: FourierGrid/FourierGrid_model.py
================================================
import os
import time
import functools
import numpy as np
import pdb
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch_scatter import segment_coo
from FourierGrid.grid import DenseGrid
from . import FourierGrid_grid
from .dvgo import Raw2Alpha, Alphas2Weights
from .dmpigo import create_full_step_id
from FourierGrid import utils, dvgo, dcvgo, dmpigo
import render_utils_cuda
import ub360_utils_cuda
def get_rays(H, W, K, c2w, inverse_y, flip_x, flip_y, mode='center'):
i, j = torch.meshgrid(
torch.linspace(0, W-1, W, device=c2w.device),
torch.linspace(0, H-1, H, device=c2w.device)) # pytorch's meshgrid has indexing='ij'
i = i.t().float()
j = j.t().float()
if mode == 'lefttop':
pass
elif mode == 'center':
i, j = i+0.5, j+0.5
elif mode == 'random':
i = i+torch.rand_like(i)
j = j+torch.rand_like(j)
else:
raise NotImplementedError
if flip_x:
i = i.flip((1,))
if flip_y:
j = j.flip((0,))
if inverse_y:
dirs = torch.stack([(i-K[0][2])/K[0][0], (j-K[1][2])/K[1][1], torch.ones_like(i)], -1)
else:
dirs = torch.stack([(i-K[0][2])/K[0][0], -(j-K[1][2])/K[1][1], -torch.ones_like(i)], -1)
# Rotate ray directions from camera frame to the world frame
rays_d = torch.sum(dirs[..., np.newaxis, :] * c2w[:3,:3], -1) # dot product, equals to: [c2w.dot(dir) for dir in dirs]
# Translate camera frame's origin to the world frame. It is the origin of all rays.
rays_o = c2w[:3,3].expand(rays_d.shape)
return rays_o, rays_d
def ndc_rays(H, W, focal, near, rays_o, rays_d):
# Shift ray origins to near plane
t = -(near + rays_o[...,2]) / rays_d[...,2]
rays_o = rays_o + t[...,None] * rays_d
# Projection
o0 = -1./(W/(2.*focal)) * rays_o[...,0] / rays_o[...,2]
o1 = -1./(H/(2.*focal)) * rays_o[...,1] / rays_o[...,2]
o2 = 1. + 2. * near / rays_o[...,2]
d0 = -1./(W/(2.*focal)) * (rays_d[...,0]/rays_d[...,2] - rays_o[...,0]/rays_o[...,2])
d1 = -1./(H/(2.*focal)) * (rays_d[...,1]/rays_d[...,2] - rays_o[...,1]/rays_o[...,2])
d2 = -2. * near / rays_o[...,2]
rays_o = torch.stack([o0,o1,o2], -1)
rays_d = torch.stack([d0,d1,d2], -1)
return rays_o, rays_d
def get_rays_of_a_view(H, W, K, c2w, ndc, inverse_y, flip_x, flip_y, mode='center'):
rays_o, rays_d = get_rays(H, W, K, c2w, inverse_y=inverse_y, flip_x=flip_x, flip_y=flip_y, mode=mode)
viewdirs = rays_d / rays_d.norm(dim=-1, keepdim=True)
if ndc:
rays_o, rays_d = ndc_rays(H, W, K[0][0], 1., rays_o, rays_d)
return rays_o, rays_d, viewdirs
class NeRFPosEmbedding(nn.Module):
def __init__(self, num_freqs: int, logscale=True):
"""
Defines a function that embeds x to (x, sin(2^k x), cos(2^k x), ...)
"""
super(NeRFPosEmbedding, self).__init__()
if logscale:
self.freq_bands = 2 ** torch.linspace(0, num_freqs - 1, num_freqs)
else:
self.freq_bands = torch.linspace(1, 2 ** (num_freqs - 1), num_freqs)
def forward(self, x: torch.Tensor) -> torch.Tensor:
out = [x]
for freq in self.freq_bands:
out += [torch.sin(freq * x), torch.cos(freq * x)]
return torch.cat(out, -1)
class FourierMSELoss(nn.Module):
def __init__(self,):
super(FourierMSELoss, self).__init__()
def forward(self, pred, gt):
fft_dim = -1
pred_fft = torch.fft.fft(pred, dim=fft_dim)
gt_fft = torch.fft.fft(gt, dim=fft_dim)
pred_real, pred_imag = pred_fft.real, pred_fft.imag
gt_real, gt_imag = gt_fft.real, gt_fft.imag
real_loss = F.mse_loss(pred_real, gt_real)
return real_loss
class FourierMSELoss(nn.Module):
def __init__(self, num_freqs=7, logscale=True):
super(FourierMSELoss, self).__init__()
# self.nerf_pos = NeRFPosEmbedding(num_freqs=num_freqs, logscale=logscale)
def forward(self, pred, gt):
# pred_embed = self.nerf_pos(pred)
# gt_embed = self.nerf_pos(gt)
# return F.mse_loss(pred_embed, gt_embed)
fft_dim = -1
pred_fft = torch.fft.fft(pred, dim=fft_dim)
gt_fft = torch.fft.fft(gt, dim=fft_dim)
pred_real, pred_imag = pred_fft.real, pred_fft.imag
gt_real, gt_imag = gt_fft.real, gt_fft.imag
real_loss = F.mse_loss(pred_real, gt_real)
# imag_loss = F.mse_loss(pred_imag, gt_imag)
return real_loss
'''Model'''
class FourierGridModel(nn.Module):
def __init__(self, xyz_min, xyz_max, num_voxels_density=0, num_voxels_base_density=0, num_voxels_rgb=0,
num_voxels_base_rgb=0, num_voxels_viewdir=0, alpha_init=None, mask_cache_world_size=None, fast_color_thres=0,
bg_len=0.2, contracted_norm='inf', density_type='DenseGrid', k0_type='DenseGrid', density_config={}, k0_config={},
rgbnet_dim=0, rgbnet_depth=3, rgbnet_width=128, fourier_freq_num=5, viewbase_pe=4, img_emb_dim=-1, verbose=False,
**kwargs):
super(FourierGridModel, self).__init__()
xyz_min = torch.Tensor(xyz_min)
xyz_max = torch.Tensor(xyz_max)
assert len(((xyz_max - xyz_min) * 100000).long().unique()), 'scene bbox must be a cube in DirectContractedVoxGO'
self.register_buffer('scene_center', (xyz_min + xyz_max) * 0.5)
self.register_buffer('scene_radius', (xyz_max - xyz_min) * 0.5)
# xyz_min/max are the boundary that separates fg and bg scene in NDC.
self.register_buffer('xyz_min', torch.Tensor([-1,-1,-1]) - bg_len)
self.register_buffer('xyz_max', torch.Tensor([1,1,1]) + bg_len)
if isinstance(fast_color_thres, dict):
self._fast_color_thres = fast_color_thres
self.fast_color_thres = fast_color_thres[0]
else:
self._fast_color_thres = None
self.fast_color_thres = fast_color_thres
self.bg_len = bg_len
self.contracted_norm = contracted_norm
self.verbose = verbose
# determine based grid resolution
self.fourier_freq_num = fourier_freq_num
self.num_voxels_base_density = num_voxels_base_density
self.voxel_size_base_density = ((self.xyz_max - self.xyz_min).prod() / self.num_voxels_base_density).pow(1/3)
self.num_voxels_base_rgb = num_voxels_base_rgb
self.voxel_size_base_rgb = ((self.xyz_max - self.xyz_min).prod() / self.num_voxels_base_rgb).pow(1/3)
self.num_voxels_viewdir = num_voxels_viewdir
self.voxel_size_viewdir = ((torch.Tensor([1,1,1]) - torch.Tensor([-1,-1,-1])).prod() / self.num_voxels_viewdir).pow(1/3)
# determine init grid resolution
self._set_grid_resolution(num_voxels_density, num_voxels_rgb)
# determine the density bias shift
self.alpha_init = alpha_init
self.register_buffer('act_shift', torch.FloatTensor([np.log(1/(1-alpha_init) - 1)]))
if self.verbose:
print('FourierGrid: set density bias shift to', self.act_shift)
# init density voxel grid
self.density_type = density_type
self.density_config = density_config
self.world_size = self.world_size_density
self.density = FourierGrid_grid.create_grid(
density_type, channels=1, world_size=self.world_size_density,
xyz_min=self.xyz_min, xyz_max=self.xyz_max, use_nerf_pos=True,
fourier_freq_num=self.fourier_freq_num, config=self.density_config)
# init color representation
self.rgbnet_kwargs = {
'rgbnet_dim': rgbnet_dim,
'rgbnet_depth': rgbnet_depth,
'rgbnet_width': rgbnet_width,
'viewbase_pe': viewbase_pe,
}
self.k0_type = k0_type
self.k0_config = k0_config
self.img_embed_dim = img_emb_dim
if 'sample_num' not in kwargs:
self.sample_num = -1
else:
self.sample_num = kwargs['sample_num']
if img_emb_dim > 0 and self.sample_num > 0: # use apperance embeddings
self.img_embeddings = nn.Embedding(num_embeddings=self.sample_num,
embedding_dim=self.img_embed_dim)
else:
self.img_embeddings = None
self.img_embed_dim = 0
pos_emb = False
if pos_emb and self.sample_num > 0: # use apperance embeddings
self.pos_emb = torch.zeros((self.sample_num, 3), requires_grad=True)
else:
self.pos_emb = None
# rgbnet configurations
if rgbnet_dim <= 0:
# color voxel grid (coarse stage)
self.k0_dim = 3
self.k0 = FourierGrid_grid.create_grid(
k0_type, channels=self.k0_dim, world_size=self.world_size_rgb,
xyz_min=self.xyz_min, xyz_max=self.xyz_max, use_nerf_pos=False,
fourier_freq_num=self.fourier_freq_num, config=self.k0_config)
self.rgbnet = None
else:
# feature voxel grid + shallow MLP (fine stage)
self.k0_dim = rgbnet_dim
self.k0 = FourierGrid_grid.create_grid(
k0_type, channels=self.k0_dim, world_size=self.world_size_rgb,
xyz_min=self.xyz_min, xyz_max=self.xyz_max, use_nerf_pos=True,
fourier_freq_num=self.fourier_freq_num, config=self.k0_config)
self.register_buffer('viewfreq', torch.FloatTensor([(2**i) for i in range(viewbase_pe)]))
dim0 = (3+3*viewbase_pe*2) # view freq dim
dim0 += self.k0_dim
self.rgbnet = nn.Sequential(
nn.Linear(dim0, rgbnet_width), nn.ReLU(inplace=True),
*[
nn.Sequential(nn.Linear(rgbnet_width, rgbnet_width), nn.ReLU(inplace=True))
for _ in range(rgbnet_depth-2)
],
nn.Linear(rgbnet_width, 3),
)
nn.init.constant_(self.rgbnet[-1].bias, 0)
if self.verbose:
print('FourierGrid: feature voxel grid', self.k0)
print('FourierGrid: mlp', self.rgbnet)
use_view_grid = num_voxels_viewdir > 0
if use_view_grid:
self.vd = FourierGrid_grid.create_grid(k0_type, channels=3, world_size=self.world_size_viewdir,
xyz_min=torch.Tensor([-1, -1, -1]), xyz_max=torch.Tensor([1, 1, 1]),
fourier_freq_num=self.fourier_freq_num, use_nerf_pos=False,)
else:
self.vd = None
# Using the coarse geometry if provided (used to determine known free space and unknown space)
# Re-implement as occupancy grid
if mask_cache_world_size is None:
mask_cache_world_size = self.world_size_density
mask = torch.ones(list(mask_cache_world_size), dtype=torch.bool)
self.mask_cache = FourierGrid_grid.MaskGrid(
path=None, mask=mask,
xyz_min=self.xyz_min, xyz_max=self.xyz_max)
@torch.no_grad()
def FourierGrid_get_training_rays(self, rgb_tr_ori, train_poses, HW, Ks, ndc, inverse_y, flip_x, flip_y):
if self.pos_emb is not None:
train_poses[:, :3, 3] = train_poses[:, :3, 3] + self.pos_emb
assert len(rgb_tr_ori) == len(train_poses) and len(rgb_tr_ori) == len(Ks) and len(rgb_tr_ori) == len(HW)
eps_time = time.time()
DEVICE = rgb_tr_ori[0].device
N = sum(im.shape[0] * im.shape[1] for im in rgb_tr_ori)
rgb_tr = torch.zeros([N,3], device=DEVICE)
rays_o_tr = torch.zeros_like(rgb_tr)
rays_d_tr = torch.zeros_like(rgb_tr)
viewdirs_tr = torch.zeros_like(rgb_tr)
indexs_tr = torch.zeros_like(rgb_tr) # image indexs
imsz = []
top = 0
cur_idx = 0
for c2w, img, (H, W), K in zip(train_poses, rgb_tr_ori, HW, Ks):
assert img.shape[:2] == (H, W)
rays_o, rays_d, viewdirs = get_rays_of_a_view(
H=H, W=W, K=K, c2w=c2w, ndc=ndc,
inverse_y=inverse_y, flip_x=flip_x, flip_y=flip_y)
n = H * W
rgb_tr[top:top+n].copy_(img.flatten(0,1))
rays_o_tr[top:top+n].copy_(rays_o.flatten(0,1).to(DEVICE))
rays_d_tr[top:top+n].copy_(rays_d.flatten(0,1).to(DEVICE))
viewdirs_tr[top:top+n].copy_(viewdirs.flatten(0,1).to(DEVICE))
indexs_tr[top:top+n].copy_(torch.tensor(cur_idx).long().to(DEVICE))
cur_idx += 1
imsz.append(n)
top += n
assert top == N
eps_time = time.time() - eps_time
return rgb_tr, rays_o_tr, rays_d_tr, viewdirs_tr, indexs_tr, imsz
def gather_training_rays(self, data_dict, images, cfg, i_train, cfg_train, poses, HW, Ks, render_kwargs):
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
if data_dict['irregular_shape']:
rgb_tr_ori = [images[i].to('cpu' if cfg.data.load2gpu_on_the_fly else device) for i in i_train]
else:
rgb_tr_ori = images[i_train].to('cpu' if cfg.data.load2gpu_on_the_fly else device)
indexs_train = None
FourierGrid_datasets = ["waymo", "mega", "nerfpp"]
if cfg.data.dataset_type in FourierGrid_datasets or cfg.model == 'FourierGrid':
rgb_tr, rays_o_tr, rays_d_tr, viewdirs_tr, indexs_train, imsz = self.FourierGrid_get_training_rays(
rgb_tr_ori=rgb_tr_ori, train_poses=poses[i_train], HW=HW[i_train], Ks=Ks[i_train],
ndc=cfg.data.ndc, inverse_y=cfg.data.inverse_y,
flip_x=cfg.data.flip_x, flip_y=cfg.data.flip_y, )
elif cfg_train.ray_sampler == 'in_maskcache':
rgb_tr, rays_o_tr, rays_d_tr, viewdirs_tr, imsz = dvgo.get_training_rays_in_maskcache_sampling(
rgb_tr_ori=rgb_tr_ori,
train_poses=poses[i_train],
HW=HW[i_train], Ks=Ks[i_train],
ndc=cfg.data.ndc, inverse_y=cfg.data.inverse_y,
flip_x=cfg.data.flip_x, flip_y=cfg.data.flip_y,
model=self, render_kwargs=render_kwargs)
elif cfg_train.ray_sampler == 'flatten':
rgb_tr, rays_o_tr, rays_d_tr, viewdirs_tr, imsz = dvgo.get_training_rays_flatten(
rgb_tr_ori=rgb_tr_ori,
train_poses=poses[i_train],
HW=HW[i_train], Ks=Ks[i_train], ndc=cfg.data.ndc, inverse_y=cfg.data.inverse_y,
flip_x=cfg.data.flip_x, flip_y=cfg.data.flip_y)
else:
rgb_tr, rays_o_tr, rays_d_tr, viewdirs_tr, imsz = dvgo.get_training_rays(
rgb_tr=rgb_tr_ori,
train_poses=poses[i_train],
HW=HW[i_train], Ks=Ks[i_train], ndc=cfg.data.ndc, inverse_y=cfg.data.inverse_y,
flip_x=cfg.data.flip_x, flip_y=cfg.data.flip_y)
index_generator = dvgo.batch_indices_generator(len(rgb_tr), cfg_train.N_rand)
batch_index_sampler = lambda: next(index_generator)
return rgb_tr, rays_o_tr, rays_d_tr, viewdirs_tr, indexs_train, imsz, batch_index_sampler
def _set_grid_resolution(self, num_voxels_density, num_voxels_rgb):
# Determine grid resolution
self.num_voxels_density = num_voxels_density
self.num_voxels_rgb = num_voxels_rgb
self.voxel_size_density = ((self.xyz_max - self.xyz_min).prod() / self.num_voxels_density).pow(1/3)
self.voxel_size_rgb = ((self.xyz_max - self.xyz_min).prod() / self.num_voxels_rgb).pow(1/3)
self.voxel_size_viewdir = ((torch.Tensor([1,1,1]) - torch.Tensor([-1,-1,-1])).prod() / self.num_voxels_viewdir).pow(1/3)
self.world_size_density = ((self.xyz_max - self.xyz_min) / self.voxel_size_density).long()
self.world_size_rgb = ((self.xyz_max - self.xyz_min) / self.voxel_size_rgb).long()
self.world_size_viewdir = (torch.Tensor([1,1,1]) - torch.Tensor([-1,-1,-1]) / self.voxel_size_viewdir).long()
self.world_len_density = self.world_size_density[0].item()
self.world_len_rgb = self.world_size_rgb[0].item()
self.world_len_viewdir = self.world_size_viewdir[0].item()
self.voxel_size_ratio_density = self.voxel_size_density / self.voxel_size_base_density
self.voxel_size_ratio_rgb = self.voxel_size_rgb / self.voxel_size_base_rgb
def get_kwargs(self):
return {
'xyz_min': self.xyz_min.cpu().numpy(),
'xyz_max': self.xyz_max.cpu().numpy(),
'num_voxels_density': self.num_voxels_density,
'num_voxels_rgb': self.num_voxels_rgb,
'num_voxels_viewdir': self.num_voxels_viewdir,
'fourier_freq_num': self.fourier_freq_num,
'num_voxels_base_density': self.num_voxels_base_density,
'num_voxels_base_rgb': self.num_voxels_base_rgb,
'alpha_init': self.alpha_init,
'voxel_size_ratio_density': self.voxel_size_ratio_density,
'voxel_size_ratio_rgb': self.voxel_size_ratio_rgb,
'mask_cache_world_size': list(self.mask_cache.mask.shape),
'fast_color_thres': self.fast_color_thres,
'contracted_norm': self.contracted_norm,
'density_type': self.density_type,
'k0_type': self.k0_type,
'density_config': self.density_config,
'k0_config': self.k0_config,
'sample_num': self.sample_num,
**self.rgbnet_kwargs,
}
@torch.no_grad()
def maskout_near_cam_vox(self, cam_o, near_clip):
ind_norm = ((cam_o - self.xyz_min) / (self.xyz_max - self.xyz_min)).flip((-1,)) * 2 - 1
pos_embed = self.density.nerf_pos(ind_norm).squeeze()
# maskout grid points that between cameras and their near planes
self_grid_xyz = torch.stack(torch.meshgrid(
torch.linspace(-1, 1, self.world_size_density[0]),
torch.linspace(-1, 1, self.world_size_density[1]),
torch.linspace(-1, 1, self.world_size_density[2]),
), -1)
for i in range(self.density.pos_embed_output_dim):
cur_pos_embed = pos_embed[:, 3*i:3*(i+1)].unsqueeze(0).unsqueeze(0).unsqueeze(0)
nearest_dist = torch.stack([(self_grid_xyz.unsqueeze(-2) - co).pow(2).sum(-1).sqrt().amin(-1) for co in cur_pos_embed.split(10)]).amin(0)
self.density.grid[0][i][nearest_dist <= near_clip] = -100
def voxel_count_views(self, rays_o_tr, rays_d_tr, imsz, near, far, stepsize, downrate=1, irregular_shape=False):
print('FourierGrid: voxel_count_views start')
far = 1e9 # the given far can be too small while rays stop when hitting scene bbox
eps_time = time.time()
N_samples = int(np.linalg.norm(np.array(self.world_size_density.cpu())+1) / stepsize) + 1
rng = torch.arange(N_samples)[None].float()
count = torch.zeros_like(self.density.get_dense_grid())
device = rng.device
for rays_o_, rays_d_ in zip(rays_o_tr.split(imsz), rays_d_tr.split(imsz)):
ones = DenseGrid(1, self.world_size_density, self.xyz_min, self.xyz_max)
if irregular_shape:
rays_o_ = rays_o_.split(10000)
rays_d_ = rays_d_.split(10000)
else:
rays_o_ = rays_o_[::downrate, ::downrate].to(device).flatten(0,-2).split(10000)
rays_d_ = rays_d_[::downrate, ::downrate].to(device).flatten(0,-2).split(10000)
for rays_o, rays_d in zip(rays_o_, rays_d_):
vec = torch.where(rays_d==0, torch.full_like(rays_d, 1e-6), rays_d)
rate_a = (self.xyz_max - rays_o) / vec
rate_b = (self.xyz_min - rays_o) / vec
t_min = torch.minimum(rate_a, rate_b).amax(-1).clamp(min=near, max=far)
t_max = torch.maximum(rate_a, rate_b).amin(-1).clamp(min=near, max=far)
step = stepsize * self.voxel_size_density * rng
interpx = (t_min[...,None] + step/rays_d.norm(dim=-1,keepdim=True))
rays_pts = rays_o[...,None,:] + rays_d[...,None,:] * interpx[...,None]
ones(rays_pts).sum().backward()
with torch.no_grad():
count += (ones.grid.grad > 1)
eps_time = time.time() - eps_time
print('FourierGrid: voxel_count_views finish (eps time:', eps_time, 'sec)')
return count
@torch.no_grad()
def scale_volume_grid(self, num_voxels_density, num_voxels_rgb):
if self.verbose:
print('FourierGrid: scale_volume_grid start')
self._set_grid_resolution(num_voxels_density, num_voxels_rgb)
self.density.scale_volume_grid(self.world_size_density)
self.k0.scale_volume_grid(self.world_size_rgb)
if np.prod(self.world_size_density.tolist()) <= 256**3:
self_grid_xyz = torch.stack(torch.meshgrid(
torch.linspace(self.xyz_min[0], self.xyz_max[0], self.world_size_density[0]),
torch.linspace(self.xyz_min[1], self.xyz_max[1], self.world_size_density[1]),
torch.linspace(self.xyz_min[2], self.xyz_max[2], self.world_size_density[2]),
), -1)
self_alpha = F.max_pool3d(self.activate_density(self.density.get_dense_grid()), kernel_size=3, padding=1, stride=1)[0,0]
self.mask_cache = FourierGrid_grid.MaskGrid(
path=None, mask=self.mask_cache(self_grid_xyz) & (self_alpha>self.fast_color_thres),
xyz_min=self.xyz_min, xyz_max=self.xyz_max)
print('FourierGrid: scale_volume_grid finish')
@torch.no_grad()
def update_occupancy_cache(self):
ori_p = self.mask_cache.mask.float().mean().item()
cache_grid_xyz = torch.stack(torch.meshgrid(
torch.linspace(self.xyz_min[0], self.xyz_max[0], self.mask_cache.mask.shape[0]),
torch.linspace(self.xyz_min[1], self.xyz_max[1], self.mask_cache.mask.shape[1]),
torch.linspace(self.xyz_min[2], self.xyz_max[2], self.mask_cache.mask.shape[2]),
), -1)
cache_grid_density = self.density(cache_grid_xyz)[None,None]
cache_grid_alpha = self.activate_density(cache_grid_density)
cache_grid_alpha = F.max_pool3d(cache_grid_alpha, kernel_size=3, padding=1, stride=1)[0,0]
self.mask_cache.mask &= (cache_grid_alpha > self.fast_color_thres)
new_p = self.mask_cache.mask.float().mean().item()
if self.verbose:
print(f'FourierGrid: update mask_cache {ori_p:.4f} => {new_p:.4f}')
def update_occupancy_cache_lt_nviews(self, rays_o_tr, rays_d_tr, imsz, render_kwargs, maskout_lt_nviews):
# TODO: Check or remove this function. This is untested and unused for now.
if self.verbose:
print('FourierGrid: update mask_cache lt_nviews start')
eps_time = time.time()
count = torch.zeros_like(self.density.get_dense_grid()).long()
device = count.device
for rays_o_, rays_d_ in zip(rays_o_tr.split(imsz), rays_d_tr.split(imsz)):
ones = FourierGrid_grid.FourierGrid(1, self.world_size_density, self.xyz_min, self.xyz_max)
for rays_o, rays_d in zip(rays_o_.split(8192), rays_d_.split(8192)):
ray_pts, indexs, inner_mask, t, rays_d_e = self.sample_ray(
ori_rays_o=rays_o.to(device), ori_rays_d=rays_d.to(device),
**render_kwargs)
ones(ray_pts).sum().backward()
count.data += (ones.grid.grad > 1)
ori_p = self.mask_cache.mask.float().mean().item()
self.mask_cache.mask &= (count >= maskout_lt_nviews)[0,0]
new_p = self.mask_cache.mask.float().mean().item()
if self.verbose:
print(f'FourierGrid: update mask_cache {ori_p:.4f} => {new_p:.4f}')
eps_time = time.time() - eps_time
if self.verbose:
print(f'FourierGrid: update mask_cache lt_nviews finish (eps time:', eps_time, 'sec)')
def density_total_variation_add_grad(self, weight, dense_mode):
w = weight * self.world_size_density.max() / 128
self.density.total_variation_add_grad(w, w, w, dense_mode)
def k0_total_variation_add_grad(self, weight, dense_mode):
w = weight * self.world_size_rgb.max() / 128
self.k0.total_variation_add_grad(w, w, w, dense_mode)
def activate_density(self, density, interval=None):
interval = interval if interval is not None else self.voxel_size_ratio_density
shape = density.shape
return Raw2Alpha.apply(density.flatten(), self.act_shift, interval).reshape(shape)
def hit_coarse_geo(self, rays_o, rays_d, near, far, stepsize, **render_kwargs):
'''Check whether the rays hit the solved coarse geometry or not'''
far = 1e9 # the given far can be too small while rays stop when hitting scene bbox
shape = rays_o.shape[:-1]
rays_o = rays_o.reshape(-1, 3).contiguous()
rays_d = rays_d.reshape(-1, 3).contiguous()
stepdist = stepsize * self.voxel_size_density
ray_pts, mask_outbbox, ray_id = render_utils_cuda.sample_pts_on_rays(
rays_o, rays_d, self.xyz_min, self.xyz_max, near, far, stepdist)[:3]
mask_inbbox = ~mask_outbbox
hit = torch.zeros([len(rays_o)], dtype=torch.bool)
hit[ray_id[mask_inbbox][self.mask_cache(ray_pts[mask_inbbox])]] = 1
return hit.reshape(shape)
def sample_ray(self, ori_rays_o, ori_rays_d, stepsize, is_train=False, **render_kwargs):
'''Sample query points on rays: central sampling.
Ori_rays_o needs to be properly scaled!
All the output points are sorted from near to far.
Input:
rays_o, rayd_d: both in [N, 3] indicating ray configurations.
stepsize: the number of voxels of each sample step.
Output:
ray_pts: [M, 3] storing all the sampled points.
ray_id: [M] the index of the ray of each point.
step_id: [M] the i'th step on a ray of each point.
'''
# NDC coordinates
rays_o = (ori_rays_o - self.scene_center) / self.scene_radius
rays_d = ori_rays_d / ori_rays_d.norm(dim=-1, keepdim=True)
N_inner = int(2 / (2+2*self.bg_len) * self.world_len_density / stepsize) + 1
N_outer = N_inner
t_boundary = 1.5 # default t_boundary=2, waymo=1.5
b_inner = torch.linspace(0, t_boundary, N_inner+1)
b_outer = t_boundary / torch.linspace(1, 1/128, N_outer+1)
t = torch.cat([
(b_inner[1:] + b_inner[:-1]) * 0.5,
(b_outer[1:] + b_outer[:-1]) * 0.5,
])
ray_pts = rays_o[:,None,:] + rays_d[:,None,:] * t[None,:,None]
if self.contracted_norm == 'inf':
norm = ray_pts.abs().amax(dim=-1, keepdim=True)
elif self.contracted_norm == 'l2':
norm = ray_pts.norm(dim=-1, keepdim=True)
else:
raise NotImplementedError
seperate_boundary = 1.0
B = 1 + self.bg_len
order = 1 # default order = 1
A = B * (seperate_boundary**order) - seperate_boundary ** (order + 1)
ray_pts = torch.where(
norm<=seperate_boundary,
ray_pts,
ray_pts / norm * (B - A/ (norm ** order))
)
indexs = None
rays_d_extend = None
inner_mask = norm<=seperate_boundary # this variable is not important
return ray_pts, indexs, inner_mask.squeeze(-1), t, rays_d_extend
def forward(self, rays_o, rays_d, viewdirs, global_step=None, is_train=False, **render_kwargs):
'''Volume rendering
@rays_o: [N, 3] the starting point of the N shooting rays.
@rays_d: [N, 3] the shooting direction of the N rays.
@viewdirs: [N, 3] viewing direction to compute positional embedding for MLP.
'''
assert len(rays_o.shape)==2 and rays_o.shape[-1]==3, 'Only support point queries in [N, 3] format'
if isinstance(self._fast_color_thres, dict) and global_step in self._fast_color_thres:
if self.verbose:
print(f'FourierGrid: update fast_color_thres {self.fast_color_thres} => {self._fast_color_thres[global_step]}')
self.fast_color_thres = self._fast_color_thres[global_step]
ret_dict = {}
num_rays = len(rays_o)
# sample points on rays
ray_pts, ray_indexs, inner_mask, t, rays_d_e = self.sample_ray(
ori_rays_o=rays_o, ori_rays_d=rays_d, is_train=global_step is not None, **render_kwargs)
n_max = len(t)
interval = render_kwargs['stepsize'] * self.voxel_size_ratio_density
ray_id, step_id = create_full_step_id(ray_pts.shape[:2])
# skip oversampled points outside scene bbox
mask = inner_mask.clone() # default
# changing shapes, only needed when the above procedures are commented out
t = t[None].repeat(num_rays, 1)
# query for alpha w/ post-activation
density = self.density(ray_pts)
alpha = self.activate_density(density, interval)
# apply fast color thresh
if self.fast_color_thres > 0:
# masked inner points, change this for other scenes!!!
mask = (alpha > self.fast_color_thres)
ray_pts = ray_pts[mask]
if rays_d_e is not None:
rays_d_e = rays_d_e[mask]
inner_mask = inner_mask[mask]
t = t[mask]
# changed because the above masking functions are removed
ray_id = ray_id[mask.flatten()]
step_id = step_id[mask.flatten()]
density = density[mask]
alpha = alpha[mask]
# compute accumulated transmittance
weights, alphainv_last = Alphas2Weights.apply(alpha, ray_id, num_rays)
if self.fast_color_thres > 0:
mask = (weights > self.fast_color_thres)
# print(f"Masked ratio: {1 - mask.sum() / mask.numel()}.")
ray_pts = ray_pts[mask]
if rays_d_e is not None:
rays_d_e = rays_d_e[mask]
inner_mask = inner_mask[mask]
t = t[mask]
ray_id = ray_id[mask]
step_id = step_id[mask]
density = density[mask]
alpha = alpha[mask]
weights = weights[mask]
else:
ray_pts = ray_pts.reshape(-1, ray_pts.shape[-1])
weights = weights.reshape(-1)
inner_mask = inner_mask.reshape(-1)
# query for color
k0 = self.k0(ray_pts)
if self.rgbnet is None:
# no view-depend effect
rgb = torch.sigmoid(k0)
elif self.vd is not None:
viewdirs_color = self.vd(viewdirs)[ray_id]
rgb_logit = k0 + viewdirs_color
rgb = torch.sigmoid(rgb_logit)
else:
# view-dependent color emission
viewdirs_emb = (viewdirs.unsqueeze(-1) * self.viewfreq).flatten(-2)
viewdirs_emb = torch.cat([viewdirs, viewdirs_emb.sin(), viewdirs_emb.cos()], -1)
viewdirs_emb = viewdirs_emb.flatten(0,-2)[ray_id]
rgb_feat = torch.cat([k0, viewdirs_emb], -1)
rgb_logit = self.rgbnet(rgb_feat)
rgb = torch.sigmoid(rgb_logit)
# Ray marching, rendering equations here.
rgb_marched = segment_coo(
src=weights.unsqueeze(-1) * rgb,
index=ray_id,
out=torch.zeros([num_rays, 3]),
reduce='sum')
if render_kwargs.get('rand_bkgd', False):
rgb_marched += (alphainv_last.unsqueeze(-1) * torch.rand_like(rgb_marched))
s = 1 - 1/(1+t) # [0, inf] => [0, 1]
ret_dict.update({
'alphainv_last': alphainv_last,
'weights': weights,
'rgb_marched': rgb_marched,
'raw_density': density,
'raw_alpha': alpha,
'raw_rgb': rgb,
'ray_id': ray_id,
'step_id': step_id,
'n_max': n_max,
't': t,
's': s,
})
if render_kwargs.get('render_depth', False):
with torch.no_grad():
depth = segment_coo(
src=(weights * s),
index=ray_id,
out=torch.zeros([num_rays]),
reduce='sum')
ret_dict.update({'depth': depth})
return ret_dict
def export_geometry_for_visualize(self, save_path):
with torch.no_grad():
dense_grid = self.density.get_dense_grid()
alpha = self.activate_density(dense_grid).squeeze().cpu().numpy()
color_grid = self.k0.get_dense_grid()
rgb = torch.sigmoid(color_grid).squeeze().permute(1,2,3,0).cpu().numpy()
np.savez_compressed(save_path, alpha=alpha, rgb=rgb)
print(f"Geometry is saved at {save_path}.")
class DistortionLoss(torch.autograd.Function):
@staticmethod
def forward(ctx, w, s, n_max, ray_id):
n_rays = ray_id.max()+1
interval = 1/n_max
w_prefix, w_total, ws_prefix, ws_total = ub360_utils_cuda.segment_cumsum(w, s, ray_id)
loss_uni = (1/3) * interval * w.pow(2)
loss_bi = 2 * w * (s * w_prefix - ws_prefix)
ctx.save_for_backward(w, s, w_prefix, w_total, ws_prefix, ws_total, ray_id)
ctx.interval = interval
return (loss_bi.sum() + loss_uni.sum()) / n_rays
@staticmethod
@torch.autograd.function.once_differentiable
def backward(ctx, grad_back):
w, s, w_prefix, w_total, ws_prefix, ws_total, ray_id = ctx.saved_tensors
interval = ctx.interval
grad_uni = (1/3) * interval * 2 * w
w_suffix = w_total[ray_id] - (w_prefix + w)
ws_suffix = ws_total[ray_id] - (ws_prefix + w*s)
grad_bi = 2 * (s * (w_prefix - w_suffix) + (ws_suffix - ws_prefix))
grad = grad_back * (grad_bi + grad_uni)
return grad, None, None, None
distortion_loss = DistortionLoss.apply
================================================
FILE: FourierGrid/__init__.py
================================================
================================================
FILE: FourierGrid/arf.py
================================================
import numpy as np
import torch
import torch.nn as nn
import imageio
import cv2
import pdb
import os
class ARF:
def __init__(self, cfg, data_dict, device):
super(ARF, self).__init__()
assert 'arf' in cfg, "ARF should not be initialized according to cfg!"
self.style_root = cfg.arf.style_root
self.style_id = cfg.arf.style_id
style_img_p = os.path.join(self.style_root, str(self.style_id) + ".jpg")
self.device = device
# assume the images are of the same height / width.
self.w, self.h = data_dict['HW'][0][1], data_dict['HW'][0][0]
# initialize style image.
self.np_style_img = None
self.style_img = self.load_style_img(style_img_p)
def load_style_img(self, style_img_p):
# resize style image such that its long side matches the long side of content images
style_img = imageio.imread(style_img_p).astype(np.float32) / 255.0
style_h, style_w = style_img.shape[:2]
content_long_side = max([self.w, self.h])
if style_h > style_w:
style_img = cv2.resize(
style_img,
(int(content_long_side / style_h * style_w), content_long_side),
interpolation=cv2.INTER_AREA,
)
else:
style_img = cv2.resize(
style_img,
(content_long_side, int(content_long_side / style_w * style_h)),
interpolation=cv2.INTER_AREA,
)
style_img = cv2.resize(
style_img,
(style_img.shape[1] // 2, style_img.shape[0] // 2),
interpolation=cv2.INTER_AREA,
)
self.np_style_img = style_img
style_img = torch.from_numpy(style_img).to(device=self.device)
return style_img
def match_colors_for_image_set(self, image_set, train_save_dir): # code from ARF
"""
image_set: [N, H, W, 3]
style_img: [H, W, 3]
"""
imageio.imwrite(
os.path.join(train_save_dir, "style_image.png"),
np.clip(self.np_style_img * 255.0, 0.0, 255.0).astype(np.uint8),
)
sh = image_set.shape
image_set = image_set.reshape(-1, 3)
image_set = torch.tensor(image_set).to(self.device)
style_img = self.style_img.reshape(-1, 3).to(image_set.device)
mu_c = image_set.mean(0, keepdim=True)
mu_s = style_img.mean(0, keepdim=True)
cov_c = torch.matmul((image_set - mu_c).transpose(1, 0), image_set - mu_c) / float(image_set.size(0))
cov_s = torch.matmul((style_img - mu_s).transpose(1, 0), style_img - mu_s) / float(style_img.size(0))
u_c, sig_c, _ = torch.svd(cov_c)
u_s, sig_s, _ = torch.svd(cov_s)
u_c_i = u_c.transpose(1, 0)
u_s_i = u_s.transpose(1, 0)
scl_c = torch.diag(1.0 / torch.sqrt(torch.clamp(sig_c, 1e-8, 1e8)))
scl_s = torch.diag(torch.sqrt(torch.clamp(sig_s, 1e-8, 1e8)))
tmp_mat = u_s @ scl_s @ u_s_i @ u_c @ scl_c @ u_c_i
tmp_vec = mu_s.view(1, 3) - mu_c.view(1, 3) @ tmp_mat.T
image_set = image_set @ tmp_mat.T + tmp_vec.view(1, 3)
image_set = image_set.contiguous().clamp_(0.0, 1.0).view(sh)
color_tf = torch.eye(4).float().to(tmp_mat.device)
color_tf[:3, :3] = tmp_mat
color_tf[:3, 3:4] = tmp_vec.T
return image_set, color_tf
================================================
FILE: FourierGrid/bbox_compute.py
================================================
import torch
import numpy as np
from FourierGrid import utils, dvgo
import time
from FourierGrid.load_everything import load_existing_model
from tqdm import tqdm
import pdb
def _compute_bbox_by_cam_frustrm_unbounded(cfg, HW, Ks, poses, i_train, near_clip):
# Find a tightest cube that cover all camera centers
xyz_min = torch.Tensor([np.inf, np.inf, np.inf])
xyz_max = -xyz_min
for (H, W), K, c2w in tqdm(zip(HW[i_train], Ks[i_train], poses[i_train]), total=len(HW[i_train])):
rays_o, rays_d, viewdirs = dvgo.get_rays_of_a_view(
H=H, W=W, K=K, c2w=c2w,
ndc=cfg.data.ndc, inverse_y=cfg.data.inverse_y,
flip_x=cfg.data.flip_x, flip_y=cfg.data.flip_y)
pts = rays_o + rays_d * near_clip
xyz_min = torch.minimum(xyz_min, pts.amin((0,1)))
xyz_max = torch.maximum(xyz_max, pts.amax((0,1)))
center = (xyz_min + xyz_max) * 0.5
radius = (center - xyz_min).max() * cfg.data.unbounded_inner_r
xyz_min = center - radius
xyz_max = center + radius
return xyz_min, xyz_max
def FourierGrid_compute_bbox_by_cam_frustrm_nerfpp(cfg, HW, Ks, poses, i_train, near_clip):
# Find a tightest cube that cover all camera centers
xyz_min = torch.Tensor([np.inf, np.inf, np.inf])
xyz_max = -xyz_min
for (H, W), K, c2w in tqdm(zip(HW[i_train], Ks[i_train], poses[i_train]), total=len(HW[i_train])):
rays_o, rays_d, viewdirs = dvgo.get_rays_of_a_view(
H=H, W=W, K=K, c2w=c2w,
ndc=cfg.data.ndc, inverse_y=cfg.data.inverse_y,
flip_x=cfg.data.flip_x, flip_y=cfg.data.flip_y)
pts = rays_o + rays_d * near_clip
xyz_min = torch.minimum(xyz_min, pts.amin((0,1)))
xyz_max = torch.maximum(xyz_max, pts.amax((0,1)))
center = (xyz_min + xyz_max) * 0.5
radius = (center - xyz_min).max() * cfg.data.unbounded_inner_r
xyz_min = center - radius
xyz_max = center + radius
return xyz_min, xyz_max
def FourierGrid_compute_bbox_by_cam_frustrm_waymo(cfg, HW, Ks, poses, i_train, near_clip):
xs, ys, zs = [], [], []
for (H, W), K, c2w in tqdm(zip(HW[i_train], Ks[i_train], poses[i_train]), total=len(HW[i_train])):
xs.append(c2w[:, 3][0].item())
ys.append(c2w[:, 3][1].item())
zs.append(c2w[:, 3][2].item())
zmin, zmax = min(zs), max(zs)
xmin, xmax = min(xs), max(xs)
ymin, ymax = min(ys), max(ys)
x_extend = 0.05 # 0.05
y_extend = 0.01
z_extend = 0.01 # 0.01
xyz_min = [xmin - x_extend, ymin - y_extend, zmin - z_extend]
xyz_max = [xmax + x_extend, ymax + y_extend, zmax + z_extend]
xyz_min, xyz_max = torch.tensor(xyz_min), torch.tensor(xyz_max)
center = (xyz_min + xyz_max) * 0.5
radius = (center - xyz_min).max() * cfg.data.unbounded_inner_r
xyz_min = center - radius
xyz_max = center + radius
# # manually set xyz_min and xyz_max
# xyz_min = torch.tensor([-1.0, -1.0, -1.0])
# xyz_max = torch.tensor([1.0, 1.0, 1.0])
return xyz_min, xyz_max
def FourierGrid_compute_bbox_by_cam_frustrm_mega(cfg, HW, Ks, poses, i_train, near_clip):
xs, ys, zs = [], [], []
for (H, W), K, c2w in tqdm(zip(HW[i_train], Ks[i_train], poses[i_train]), total=len(HW[i_train])):
xs.append(c2w[:, 3][0].item())
ys.append(c2w[:, 3][1].item())
zs.append(c2w[:, 3][2].item())
zmin, zmax = min(zs), max(zs)
xmin, xmax = min(xs), max(xs)
ymin, ymax = min(ys), max(ys)
x_distance, y_distance, z_distance = abs(xmax - xmin), abs(ymax - ymin), abs(zmax - zmin)
boundary_ratio = cfg.data.boundary_ratio
xyz_min = [xmin - boundary_ratio*x_distance, ymin - boundary_ratio * y_distance, zmin - boundary_ratio * z_distance]
xyz_max = [xmax + boundary_ratio*x_distance, ymax + boundary_ratio * y_distance, zmax + boundary_ratio * z_distance]
xyz_min, xyz_max = torch.tensor(xyz_min), torch.tensor(xyz_max)
center = (xyz_min + xyz_max) * 0.5
radius = (center - xyz_min).max() * cfg.data.unbounded_inner_r
xyz_min = center - radius
xyz_max = center + radius
return xyz_min, xyz_max
def _compute_bbox_by_cam_frustrm_bounded(cfg, HW, Ks, poses, i_train, near, far):
xyz_min = torch.Tensor([np.inf, np.inf, np.inf])
xyz_max = -xyz_min
for (H, W), K, c2w in zip(HW[i_train], Ks[i_train], poses[i_train]):
rays_o, rays_d, viewdirs = dvgo.get_rays_of_a_view(
H=H, W=W, K=K, c2w=c2w,
ndc=cfg.data.ndc, inverse_y=cfg.data.inverse_y,
flip_x=cfg.data.flip_x, flip_y=cfg.data.flip_y)
if cfg.data.ndc:
pts_nf = torch.stack([rays_o+rays_d*near, rays_o+rays_d*far])
else:
pts_nf = torch.stack([rays_o+viewdirs*near, rays_o+viewdirs*far])
xyz_min = torch.minimum(xyz_min, pts_nf.amin((0,1,2)))
xyz_max = torch.maximum(xyz_max, pts_nf.amax((0,1,2)))
return xyz_min, xyz_max
def compute_bbox_by_cam_frustrm(args, cfg, HW, Ks, poses, i_train, near, far, **kwargs):
verbose = args.block_num <= 1
if verbose:
print('compute_bbox_by_cam_frustrm: start')
if cfg.data.dataset_type == "waymo":
xyz_min, xyz_max = FourierGrid_compute_bbox_by_cam_frustrm_waymo(
cfg, HW, Ks, poses, i_train, kwargs.get('near_clip', None))
elif cfg.data.dataset_type == "nerfpp" or cfg.model == 'FourierGrid':
xyz_min, xyz_max = FourierGrid_compute_bbox_by_cam_frustrm_nerfpp(
cfg, HW, Ks, poses, i_train, kwargs.get('near_clip', None))
elif cfg.data.unbounded_inward:
xyz_min, xyz_max = _compute_bbox_by_cam_frustrm_unbounded(
cfg, HW, Ks, poses, i_train, kwargs.get('near_clip', None))
else:
xyz_min, xyz_max = _compute_bbox_by_cam_frustrm_bounded(
cfg, HW, Ks, poses, i_train, near, far)
if verbose:
print('compute_bbox_by_cam_frustrm: xyz_min', xyz_min)
print('compute_bbox_by_cam_frustrm: xyz_max', xyz_max)
print('compute_bbox_by_cam_frustrm: finish')
return xyz_min, xyz_max
@torch.no_grad()
def compute_bbox_by_coarse_geo(model_class, model_path, thres, device, args, cfg):
print('compute_bbox_by_coarse_geo: start')
eps_time = time.time()
# model = utils.load_model(model_class, model_path)
# TODO: validate this, this should be fine
model, _, _ = load_existing_model(args, cfg, cfg.fine_train, model_path, device=device)
model.to(device)
interp = torch.stack(torch.meshgrid(
torch.linspace(0, 1, model.world_size[0]),
torch.linspace(0, 1, model.world_size[1]),
torch.linspace(0, 1, model.world_size[2]),
), -1)
dense_xyz = model.xyz_min * (1-interp) + model.xyz_max * interp
density = model.density(dense_xyz)
alpha = model.activate_density(density)
mask = (alpha > thres)
if mask.max() > 0:
active_xyz = dense_xyz[mask]
else:
print("Warning! No activated voxels found.")
mask = (alpha > -1)
active_xyz = dense_xyz[mask]
xyz_min = active_xyz.amin(0)
xyz_max = active_xyz.amax(0)
print('compute_bbox_by_coarse_geo: xyz_min', xyz_min)
print('compute_bbox_by_coarse_geo: xyz_max', xyz_max)
eps_time = time.time() - eps_time
print('compute_bbox_by_coarse_geo: finish (eps time:', eps_time, 'secs)')
return xyz_min, xyz_max
================================================
FILE: FourierGrid/camera_utils.py
================================================
import numpy as np
import enum
from dataclasses import dataclass
from typing import List, Mapping, Optional, Text, Tuple, Union
from torch import Tensor
import torch
@dataclass
class Rays:
origins: Tensor
directions: Tensor
viewdirs: Tensor
radii: Tensor
near: Tensor
far: Tensor
def split_rays(rays, batch_size):
ret = []
origins_all = rays.origins.split(batch_size)
directions_all = rays.directions.split(batch_size)
viewdirs_all = rays.viewdirs.split(batch_size)
radii_all = rays.radii.split(batch_size)
near_all = rays.near.split(batch_size)
far_all = rays.far.split(batch_size)
for o, d, v, r, n, f in zip(origins_all, directions_all, viewdirs_all, radii_all, near_all, far_all):
ret.append(Rays(o, d, v, r, n, f))
return ret
def intrinsic_matrix(fx: float,
fy: float,
cx: float,
cy: float,
):
"""Intrinsic matrix for a pinhole camera in OpenCV coordinate system."""
return np.array([
[fx, 0, cx],
[0, fy, cy],
[0, 0, 1.]], dtype=np.float32)
class ProjectionType(enum.Enum):
"""Camera projection type (standard perspective pinhole or fisheye model)."""
PERSPECTIVE = 'perspective'
FISHEYE = 'fisheye'
def convert_to_ndc(origins: Tensor,
directions: Tensor,
pixtocam: Tensor,
near: float = 1.) -> Tuple[Tensor, Tensor]:
"""Converts a set of rays to normalized device coordinates (NDC).
Args:
origins: ndarray(float32), [..., 3], world space ray origins.
directions: ndarray(float32), [..., 3], world space ray directions.
pixtocam: ndarray(float32), [3, 3], inverse intrinsic matrix.
near: float, near plane along the negative z axis.
xnp: either numpy or jax.numpy.
Returns:
origins_ndc: ndarray(float32), [..., 3].
directions_ndc: ndarray(float32), [..., 3].
This function assumes input rays should be mapped into the NDC space for a
perspective projection pinhole camera, with identity extrinsic matrix (pose)
and intrinsic parameters defined by inputs focal, width, and height.
The near value specifies the near plane of the frustum, and the far plane is
assumed to be infinity.
The ray bundle for the identity pose camera will be remapped to parallel rays
within the (-1, -1, -1) to (1, 1, 1) cube. Any other ray in the original
world space can be remapped as long as it has dz < 0 (ray direction has a
negative z-coord); this allows us to share a common NDC space for "forward
facing" scenes.
Note that
projection(origins + t * directions)
will NOT be equal to
origins_ndc + t * directions_ndc
and that the directions_ndc are not unit length. Rather, directions_ndc is
defined such that the valid near and far planes in NDC will be 0 and 1.
See Appendix C in https://arxiv.org/abs/2003.08934 for additional details.
"""
# Shift ray origins to near plane, such that oz = -near.
# This makes the new near bound equal to 0.
t = -(near + origins[..., 2]) / directions[..., 2]
origins = origins + t[..., None] * directions
dx, dy, dz = torch.moveaxis(directions, -1, 0)
ox, oy, oz = torch.moveaxis(origins, -1, 0)
xmult = 1. / pixtocam[0, 2] # Equal to -2. * focal / cx
ymult = 1. / pixtocam[1, 2] # Equal to -2. * focal / cy
# Perspective projection into NDC for the t = 0 near points
# origins + 0 * directions
origins_ndc = torch.stack([xmult * ox / oz, ymult * oy / oz,
-torch.ones_like(oz)], dim=-1)
# Perspective projection into NDC for the t = infinity far points
# origins + infinity * directions
infinity_ndc = torch.stack([xmult * dx / dz, ymult * dy / dz,
torch.ones_like(oz)],
dim=-1)
# directions_ndc points from origins_ndc to infinity_ndc
directions_ndc = infinity_ndc - origins_ndc
return origins_ndc, directions_ndc
def pixels_to_rays(
pix_x_int: Tensor,
pix_y_int: Tensor,
pixtocams: Tensor,
camtoworlds: Tensor,
distortion_params: Optional[Mapping[str, float]] = None,
pixtocam_ndc: Optional[Tensor] = None,
camtype: ProjectionType = ProjectionType.PERSPECTIVE,
) -> Tuple[Tensor, Tensor, Tensor, Tensor, Tensor]:
"""Calculates rays given pixel coordinates, intrinisics, and extrinsics.
Given 2D pixel coordinates pix_x_int, pix_y_int for cameras with
inverse intrinsics pixtocams and extrinsics camtoworlds (and optional
distortion coefficients distortion_params and NDC space projection matrix
pixtocam_ndc), computes the corresponding 3D camera rays.
Vectorized over the leading dimensions of the first four arguments.
Args:
pix_x_int: int array, shape SH, x coordinates of image pixels.
pix_y_int: int array, shape SH, y coordinates of image pixels.
pixtocams: float array, broadcastable to SH + [3, 3], inverse intrinsics.
camtoworlds: float array, broadcastable to SH + [3, 4], camera extrinsics.
distortion_params: dict of floats, optional camera distortion parameters.
pixtocam_ndc: float array, [3, 3], optional inverse intrinsics for NDC.
camtype: camera_utils.ProjectionType, fisheye or perspective camera.
xnp: either numpy or jax.numpy.
Returns:
origins: float array, shape SH + [3], ray origin points.
directions: float array, shape SH + [3], ray direction vectors.
viewdirs: float array, shape SH + [3], normalized ray direction vectors.
radii: float array, shape SH + [1], ray differential radii.
imageplane: float array, shape SH + [2], xy coordinates on the image plane.
If the image plane is at world space distance 1 from the pinhole, then
imageplane will be the xy coordinates of a pixel in that space (so the
camera ray direction at the origin would be (x, y, -1) in OpenGL coords).
"""
# Must add half pixel offset to shoot rays through pixel centers.
def pix_to_dir(x, y):
return torch.stack([x + .5, y + .5, torch.ones_like(x)], dim=-1)
# We need the dx and dy rays to calculate ray radii for mip-NeRF cones.
pixel_dirs_stacked = torch.stack([
pix_to_dir(pix_x_int, pix_y_int),
pix_to_dir(pix_x_int + 1, pix_y_int),
pix_to_dir(pix_x_int, pix_y_int + 1)
], dim=0)
# For jax, need to specify high-precision matmul.
mat_vec_mul = lambda A, b: torch.matmul(A, b[..., None])[..., 0]
# Apply inverse intrinsic matrices.
camera_dirs_stacked = mat_vec_mul(pixtocams, pixel_dirs_stacked)
if distortion_params is not None:
# Correct for distortion.
x, y = _radial_and_tangential_undistort(
camera_dirs_stacked[..., 0],
camera_dirs_stacked[..., 1],
**distortion_params)
camera_dirs_stacked = torch.stack([x, y, torch.ones_like(x)], -1)
if camtype == ProjectionType.FISHEYE:
theta = torch.sqrt(torch.sum(torch.square(camera_dirs_stacked[..., :2]), dim=-1))
theta = theta.clip(None, np.pi)
sin_theta_over_theta = torch.sin(theta) / theta
camera_dirs_stacked = torch.stack([
camera_dirs_stacked[..., 0] * sin_theta_over_theta,
camera_dirs_stacked[..., 1] * sin_theta_over_theta,
torch.cos(theta),
], dim=-1)
# Flip from OpenCV to OpenGL coordinate system.
camera_dirs_stacked = torch.matmul(camera_dirs_stacked,
torch.diag(torch.tensor([1., -1., -1.],
dtype=torch.float32,
device=camera_dirs_stacked.device)))
# Extract 2D image plane (x, y) coordinates.
imageplane = camera_dirs_stacked[0, ..., :2]
# Apply camera rotation matrices.
directions_stacked = mat_vec_mul(camtoworlds[..., :3, :3],
camera_dirs_stacked)
# Extract the offset rays.
directions, dx, dy = directions_stacked
origins = torch.broadcast_to(camtoworlds[..., :3, -1], directions.shape)
viewdirs = directions / torch.linalg.norm(directions, ord=2, dim=-1, keepdim=True)
if pixtocam_ndc is None:
# Distance from each unit-norm direction vector to its neighbors.
dx_norm = torch.linalg.norm(dx - directions, ord=2, dim=-1)
dy_norm = torch.linalg.norm(dy - directions, ord=2, dim=-1)
else:
# Convert ray origins and directions into projective NDC space.
origins_dx, _ = convert_to_ndc(origins, dx, pixtocam_ndc)
origins_dy, _ = convert_to_ndc(origins, dy, pixtocam_ndc)
origins, directions = convert_to_ndc(origins, directions, pixtocam_ndc)
# In NDC space, we use the offset between origins instead of directions.
dx_norm = torch.linalg.norm(origins_dx - origins, ord=2, dim=-1)
dy_norm = torch.linalg.norm(origins_dy - origins, ord=2, dim=-1)
# Cut the distance in half, multiply it to match the variance of a uniform
# distribution the size of a pixel (1/12, see the original mipnerf paper).
radii = (0.5 * (dx_norm + dy_norm))[..., None] * 2 / np.sqrt(12.)
return origins, directions, viewdirs, radii, imageplane
def _compute_residual_and_jacobian(
x: Tensor,
y: Tensor,
xd: Tensor,
yd: Tensor,
k1: float = 0.0,
k2: float = 0.0,
k3: float = 0.0,
k4: float = 0.0,
p1: float = 0.0,
p2: float = 0.0,
) -> Tuple[Tensor, Tensor, Tensor, Tensor, Tensor, Tensor]:
"""Auxiliary function of radial_and_tangential_undistort()."""
# Adapted from https://github.com/google/nerfies/blob/main/nerfies/camera.py
# let r(x, y) = x^2 + y^2;
# d(x, y) = 1 + k1 * r(x, y) + k2 * r(x, y) ^2 + k3 * r(x, y)^3 +
# k4 * r(x, y)^4;
r = x * x + y * y
d = 1.0 + r * (k1 + r * (k2 + r * (k3 + r * k4)))
# The perfect projection is:
# xd = x * d(x, y) + 2 * p1 * x * y + p2 * (r(x, y) + 2 * x^2);
# yd = y * d(x, y) + 2 * p2 * x * y + p1 * (r(x, y) + 2 * y^2);
#
# Let's define
#
# fx(x, y) = x * d(x, y) + 2 * p1 * x * y + p2 * (r(x, y) + 2 * x^2) - xd;
# fy(x, y) = y * d(x, y) + 2 * p2 * x * y + p1 * (r(x, y) + 2 * y^2) - yd;
#
# We are looking for a solution that satisfies
# fx(x, y) = fy(x, y) = 0;
fx = d * x + 2 * p1 * x * y + p2 * (r + 2 * x * x) - xd
fy = d * y + 2 * p2 * x * y + p1 * (r + 2 * y * y) - yd
# Compute derivative of d over [x, y]
d_r = (k1 + r * (2.0 * k2 + r * (3.0 * k3 + r * 4.0 * k4)))
d_x = 2.0 * x * d_r
d_y = 2.0 * y * d_r
# Compute derivative of fx over x and y.
fx_x = d + d_x * x + 2.0 * p1 * y + 6.0 * p2 * x
fx_y = d_y * x + 2.0 * p1 * x + 2.0 * p2 * y
# Compute derivative of fy over x and y.
fy_x = d_x * y + 2.0 * p2 * y + 2.0 * p1 * x
fy_y = d + d_y * y + 2.0 * p2 * x + 6.0 * p1 * y
return fx, fy, fx_x, fx_y, fy_x, fy_y
def _radial_and_tangential_undistort(
xd: Tensor,
yd: Tensor,
k1: float = 0,
k2: float = 0,
k3: float = 0,
k4: float = 0,
p1: float = 0,
p2: float = 0,
eps: float = 1e-9,
max_iterations=10) -> Tuple[Tensor, Tensor]:
"""Computes undistorted (x, y) from (xd, yd)."""
# From https://github.com/google/nerfies/blob/main/nerfies/camera.py
# Initialize from the distorted point.
x = xd.clone()
y = yd.clone()
for _ in range(max_iterations):
fx, fy, fx_x, fx_y, fy_x, fy_y = _compute_residual_and_jacobian(
x=x, y=y, xd=xd, yd=yd, k1=k1, k2=k2, k3=k3, k4=k4, p1=p1, p2=p2)
denominator = fy_x * fx_y - fx_x * fy_y
x_numerator = fx * fy_y - fy * fx_y
y_numerator = fy * fx_x - fx * fy_x
step_x = torch.where(
torch.abs(denominator) > eps, x_numerator / denominator,
torch.zeros_like(denominator))
step_y = torch.where(
torch.abs(denominator) > eps, y_numerator / denominator,
torch.zeros_like(denominator))
x = x + step_x
y = y + step_y
return x, y
================================================
FILE: FourierGrid/common_data_loaders/__init__.py
================================================
================================================
FILE: FourierGrid/common_data_loaders/load_blendedmvs.py
================================================
import os
import glob
import torch
import numpy as np
import imageio
import json
import torch.nn.functional as F
import cv2
def load_blendedmvs_data(basedir):
pose_paths = sorted(glob.glob(os.path.join(basedir, 'pose', '*txt')))
rgb_paths = sorted(glob.glob(os.path.join(basedir, 'rgb', '*png')))
all_poses = []
all_imgs = []
i_split = [[], []]
for i, (pose_path, rgb_path) in enumerate(zip(pose_paths, rgb_paths)):
i_set = int(os.path.split(rgb_path)[-1][0])
all_imgs.append((imageio.imread(rgb_path) / 255.).astype(np.float32))
all_poses.append(np.loadtxt(pose_path).astype(np.float32))
i_split[i_set].append(i)
imgs = np.stack(all_imgs, 0)
poses = np.stack(all_poses, 0)
i_split.append(i_split[-1])
path_intrinsics = os.path.join(basedir, 'intrinsics.txt')
H, W = imgs[0].shape[:2]
K = np.loadtxt(path_intrinsics)
focal = float(K[0,0])
render_poses = torch.Tensor(np.loadtxt(os.path.join(basedir, 'test_traj.txt')).reshape(-1,4,4).astype(np.float32))
return imgs, poses, render_poses, [H, W, focal], K, i_split
================================================
FILE: FourierGrid/common_data_loaders/load_blender.py
================================================
import os
import torch
import numpy as np
import imageio
import pdb
import json
import torch.nn.functional as F
import cv2
trans_t = lambda t : torch.Tensor([
[1,0,0,0],
[0,1,0,0],
[0,0,1,t],
[0,0,0,1]]).float()
rot_phi = lambda phi : torch.Tensor([
[1,0,0,0],
[0,np.cos(phi),-np.sin(phi),0],
[0,np.sin(phi), np.cos(phi),0],
[0,0,0,1]]).float()
rot_theta = lambda th : torch.Tensor([
[np.cos(th),0,-np.sin(th),0],
[0,1,0,0],
[np.sin(th),0, np.cos(th),0],
[0,0,0,1]]).float()
def pose_spherical(theta, phi, radius):
c2w = trans_t(radius)
c2w = rot_phi(phi/180.*np.pi) @ c2w
c2w = rot_theta(theta/180.*np.pi) @ c2w
c2w = torch.Tensor(np.array([[-1,0,0,0],[0,0,1,0],[0,1,0,0],[0,0,0,1]])) @ c2w
return c2w
def load_blender_data(basedir, half_res=False, testskip=1):
splits = ['train', 'val', 'test']
metas = {}
for s in splits:
with open(os.path.join(basedir, 'transforms_{}.json'.format(s)), 'r') as fp:
metas[s] = json.load(fp)
all_imgs = []
all_poses = []
counts = [0]
for s in splits:
meta = metas[s]
imgs = []
poses = []
if s=='train' or testskip==0:
skip = 1
else:
skip = testskip
for frame in meta['frames'][::skip]:
fname = os.path.join(basedir, frame['file_path'] + '.png')
imgs.append(imageio.imread(fname))
poses.append(np.array(frame['transform_matrix']))
imgs = (np.array(imgs) / 255.).astype(np.float32) # keep all 4 channels (RGBA)
poses = np.array(poses).astype(np.float32)
counts.append(counts[-1] + imgs.shape[0])
all_imgs.append(imgs)
all_poses.append(poses)
i_split = [np.arange(counts[i], counts[i+1]) for i in range(3)]
imgs = np.concatenate(all_imgs, 0)
poses = np.concatenate(all_poses, 0)
pdb.set_trace()
H, W = imgs[0].shape[:2]
camera_angle_x = float(meta['camera_angle_x'])
focal = .5 * W / np.tan(.5 * camera_angle_x)
render_poses = torch.stack([pose_spherical(angle, -30.0, 4.0) for angle in np.linspace(-180,180,160+1)[:-1]], 0)
if half_res:
H = H//2
W = W//2
focal = focal/2.
imgs_half_res = np.zeros((imgs.shape[0], H, W, 4))
for i, img in enumerate(imgs):
imgs_half_res[i] = cv2.resize(img, (W, H), interpolation=cv2.INTER_AREA)
imgs = imgs_half_res
# imgs = tf.image.resize_area(imgs, [400, 400]).numpy()
return imgs, poses, render_poses, [H, W, focal], i_split
================================================
FILE: FourierGrid/common_data_loaders/load_co3d.py
================================================
import os
import json
import gzip
import glob
import torch
import numpy as np
import imageio
import torch.nn.functional as F
import cv2
def load_co3d_data(cfg):
# load meta
with gzip.open(cfg.annot_path, 'rt', encoding='utf8') as zipfile:
annot = [v for v in json.load(zipfile) if v['sequence_name'] == cfg.sequence_name]
with open(cfg.split_path) as f:
split = json.load(f)
train_im_path = set()
test_im_path = set()
for k, lst in split.items():
for v in lst:
if v[0] == cfg.sequence_name:
if 'known' in k:
train_im_path.add(v[-1])
else:
test_im_path.add(v[-1])
assert len(annot) == len(train_im_path) + len(test_im_path), 'Mismatch: '\
f'{len(annot)} == {len(train_im_path) + len(test_im_path)}'
# load datas
imgs = []
masks = []
poses = []
Ks = []
i_split = [[], []]
remove_empty_masks_cnt = [0, 0]
for i, meta in enumerate(annot):
im_fname = meta['image']['path']
assert im_fname in train_im_path or im_fname in test_im_path
sid = 0 if im_fname in train_im_path else 1
if meta['mask']['mass'] == 0:
remove_empty_masks_cnt[sid] += 1
continue
im_path = os.path.join(cfg.datadir, im_fname)
mask_path = os.path.join(cfg.datadir, meta['mask']['path'])
mask = imageio.imread(mask_path) / 255.
if mask.max() < 0.5:
remove_empty_masks_cnt[sid] += 1
continue
Rt = np.concatenate([meta['viewpoint']['R'], np.array(meta['viewpoint']['T'])[:,None]], 1)
pose = np.linalg.inv(np.concatenate([Rt, [[0,0,0,1]]]))
imgs.append(imageio.imread(im_path) / 255.)
masks.append(mask)
poses.append(pose)
assert imgs[-1].shape[:2] == tuple(meta['image']['size'])
half_image_size_wh = np.float32(meta['image']['size'][::-1]) * 0.5
principal_point = np.float32(meta['viewpoint']['principal_point'])
focal_length = np.float32(meta['viewpoint']['focal_length'])
principal_point_px = -1.0 * (principal_point - 1.0) * half_image_size_wh
focal_length_px = focal_length * half_image_size_wh
Ks.append(np.array([
[focal_length_px[0], 0, principal_point_px[0]],
[0, focal_length_px[1], principal_point_px[1]],
[0, 0, 1],
]))
i_split[sid].append(len(imgs)-1)
if sum(remove_empty_masks_cnt) > 0:
print('load_co3d_data: removed %d train / %d test due to empty mask' % tuple(remove_empty_masks_cnt))
print(f'load_co3d_data: num images {len(i_split[0])} train / {len(i_split[1])} test')
imgs = np.array(imgs)
masks = np.array(masks)
poses = np.stack(poses, 0)
Ks = np.stack(Ks, 0)
render_poses = poses[i_split[-1]]
i_split.append(i_split[-1])
# visyalization hwf
H, W = np.array([im.shape[:2] for im in imgs]).mean(0).astype(int)
focal = Ks[:,[0,1],[0,1]].mean()
return imgs, masks, poses, render_poses, [H, W, focal], Ks, i_split
================================================
FILE: FourierGrid/common_data_loaders/load_common_data.py
================================================
import numpy as np
import pdb
from .load_llff import load_llff_data
from .load_free import load_free_data
from .load_blender import load_blender_data
from .load_nsvf import load_nsvf_data
from .load_blendedmvs import load_blendedmvs_data
from .load_tankstemple import load_tankstemple_data
from .load_deepvoxels import load_dv_data
from .load_co3d import load_co3d_data
from .load_nerfpp import load_nerfpp_data
def load_common_data(args):
K, depths = None, None
near_clip = None
if not 'training_ids' in args:
training_ids = None
else:
training_ids = args['training_ids']
if args.dataset_type == 'llff':
images, depths, poses, bds, render_poses, i_test = load_llff_data(
args.datadir, args.factor, args.width, args.height,
recenter=True, bd_factor=args.bd_factor,
spherify=args.spherify,
load_depths=args.load_depths,
movie_render_kwargs=args.movie_render_kwargs)
hwf = poses[0,:3,-1]
poses = poses[:,:3,:4]
print('Loaded llff', images.shape, render_poses.shape, hwf, args.datadir)
if not isinstance(i_test, list):
i_test = [i_test]
if args.llffhold > 0:
print('Auto LLFF holdout,', args.llffhold)
i_test = np.arange(images.shape[0])[::args.llffhold]
i_val = i_test
i_train = np.array([i for i in np.arange(int(images.shape[0])) if
(i not in i_test and i not in i_val)])
print('DEFINING BOUNDS')
if args.ndc:
near = 0.
far = 1.
else:
near_clip = max(np.ndarray.min(bds) * .9, 0)
_far = max(np.ndarray.max(bds) * 1., 0)
near = 0
far = inward_nearfar_heuristic(poses[i_train, :3, 3])[1]
print('near_clip', near_clip)
print('original far', _far)
print('NEAR FAR', near, far)
elif args.dataset_type == 'free':
images, depths, intri, poses, bds, render_poses, i_test = load_free_data(
args, args.datadir, args.factor, args.width, args.height,
recenter=True, bd_factor=args.bd_factor,
spherify=args.spherify,
load_depths=args.load_depths,
movie_render_kwargs=args.movie_render_kwargs, training_ids=training_ids, sc=args.pose_scale)
i_val = i_test
i_train = np.array([i for i in np.arange(int(images.shape[0])) if (i not in i_test and i not in i_val)])
near_clip = max(np.ndarray.min(bds) * .9, 0)
_far = max(np.ndarray.max(bds) * 1., 0)
if args.ndc:
near = 0.
far = 1.
else:
near = 0
far = inward_nearfar_heuristic(poses[i_train, :3, 3])[1]
# print('DEFINING BOUNDS')
# if args.ndc:
# near = 0.
# far = 1.
# else:
# near_clip = max(np.ndarray.min(bds) * .9, 0)
# _far = max(np.ndarray.max(bds) * 1., 0)
# near = 0
# far = inward_nearfar_heuristic(poses[i_train, :3, 3])[1]
# print('near_clip', near_clip)
# print('original far', _far)
# print('NEAR FAR', near, far)
# Cast intrinsics to right types
# H, W, focal = hwf
# H, W = int(H), int(W)
# hwf = [H, W, focal]
HW = np.array([im.shape[:2] for im in images])
irregular_shape = (images.dtype is np.dtype('object'))
# all data should be in [N \times D]
# near_clip = None
near_clip = max(np.ndarray.min(bds) * .9, 0)
data_dict = dict(
hwf=None, HW=HW, Ks=intri,
near=near, far=far, near_clip=near_clip,
i_train=i_train, i_val=i_val, i_test=i_test,
poses=poses, render_poses=render_poses,
images=images, depths=depths,
irregular_shape=irregular_shape,
)
return data_dict
elif args.dataset_type == 'nerfstudio':
images, depths, poses, bds, render_poses, i_test = load_nerfstudio_data(
args.datadir, args.factor, args.width, args.height, recenter=args.recenter, bd_factor=args.bd_factor, dvgohold=args.dvgohold,
spherify=args.spherify, load_depths=args.load_depths, movie_render_kwargs=args.movie_render_kwargs)
hwf = poses[0,:3,-1]
poses = poses[:,:3,:4]
print('Loaded nerfstudio', images.shape, render_poses.shape, hwf, args.datadir)
if not isinstance(i_test, list):
i_test = [i_test]
if args.llffhold > 0:
print('Auto LLFF holdout,', args.llffhold)
i_test = np.arange(images.shape[0])[::args.llffhold]
i_val = i_test
i_train = np.array([i for i in np.arange(int(images.shape[0])) if
(i not in i_test and i not in i_val)])
print('DEFINING BOUNDS')
if args.ndc:
near = 0.
far = 1.
else:
near_clip = max(np.ndarray.min(bds) * .9, 0)
_far = max(np.ndarray.max(bds) * 1., 0)
near = 0
far = inward_nearfar_heuristic(poses[i_train, :3, 3])[1]
elif args.dataset_type == 'blender':
images, poses, render_poses, hwf, i_split = load_blender_data(args.datadir, args.half_res, args.testskip)
print('Loaded blender', images.shape, render_poses.shape, hwf, args.datadir)
i_train, i_val, i_test = i_split
near, far = 2., 6.
if images.shape[-1] == 4:
if args.white_bkgd:
images = images[...,:3]*images[...,-1:] + (1.-images[...,-1:])
else:
images = images[...,:3]*images[...,-1:]
elif args.dataset_type == 'blendedmvs':
images, poses, render_poses, hwf, K, i_split = load_blendedmvs_data(args.datadir)
print('Loaded blendedmvs', images.shape, render_poses.shape, hwf, args.datadir)
i_train, i_val, i_test = i_split
near, far = inward_nearfar_heuristic(poses[i_train, :3, 3])
assert images.shape[-1] == 3
elif args.dataset_type == 'tankstemple':
images, poses, render_poses, hwf, K, i_split = load_tankstemple_data(
args.datadir, movie_render_kwargs=args.movie_render_kwargs)
print('Loaded tankstemple', images.shape, render_poses.shape, hwf, args.datadir)
i_train, i_val, i_test = i_split
# near, far = inward_nearfar_heuristic(poses[i_train, :3, 3], ratio=0)
near_clip, far = inward_nearfar_heuristic(poses[i_train, :3, 3], ratio=0.02)
near = 0
if images.shape[-1] == 4:
if args.white_bkgd:
images = images[...,:3]*images[...,-1:] + (1.-images[...,-1:])
else:
images = images[...,:3]*images[...,-1:]
elif args.dataset_type == 'nsvf':
images, poses, render_poses, hwf, i_split = load_nsvf_data(args.datadir)
print('Loaded nsvf', images.shape, render_poses.shape, hwf, args.datadir)
i_train, i_val, i_test = i_split
near, far = inward_nearfar_heuristic(poses[i_train, :3, 3])
if images.shape[-1] == 4:
if args.white_bkgd:
images = images[...,:3]*images[...,-1:] + (1.-images[...,-1:])
else:
images = images[...,:3]*images[...,-1:]
elif args.dataset_type == 'deepvoxels':
images, poses, render_poses, hwf, i_split = load_dv_data(scene=args.scene, basedir=args.datadir, testskip=args.testskip)
print('Loaded deepvoxels', images.shape, render_poses.shape, hwf, args.datadir)
i_train, i_val, i_test = i_split
hemi_R = np.mean(np.linalg.norm(poses[:,:3,-1], axis=-1))
near = hemi_R - 1
far = hemi_R + 1
assert args.white_bkgd
assert images.shape[-1] == 3
elif args.dataset_type == 'co3d':
# each image can be in different shapes and intrinsics
images, masks, poses, render_poses, hwf, K, i_split = load_co3d_data(args)
print('Loaded co3d', args.datadir, args.annot_path, args.sequence_name)
i_train, i_val, i_test = i_split
near, far = inward_nearfar_heuristic(poses[i_train, :3, 3], ratio=0)
for i in range(len(images)):
if args.white_bkgd:
images[i] = images[i] * masks[i][...,None] + (1.-masks[i][...,None])
else:
images[i] = images[i] * masks[i][...,None]
elif args.dataset_type == 'nerfpp':
images, poses, render_poses, hwf, K, i_split = load_nerfpp_data(args.datadir, rerotate=False,
training_ids=training_ids)
print('Loaded nerf_pp', images.shape, hwf, args.datadir)
i_train, i_val, i_test = i_split
# TODO: remove the near clip and far arguments
near_clip, far = inward_nearfar_heuristic(poses[i_train, :3, 3], ratio=0.02)
near = 0
else:
raise NotImplementedError(f'Unknown dataset type {args.dataset_type} exiting')
# Cast intrinsics to right types
H, W, focal = hwf
H, W = int(H), int(W)
hwf = [H, W, focal]
HW = np.array([im.shape[:2] for im in images])
irregular_shape = (images.dtype is np.dtype('object'))
if K is None:
K = np.array([
[focal, 0, 0.5*W],
[0, focal, 0.5*H],
[0, 0, 1]
])
if len(K.shape) == 2:
Ks = K[None].repeat(len(poses), axis=0)
else:
Ks = K
render_poses = render_poses[...,:4]
data_dict = dict(
hwf=hwf, HW=HW, Ks=Ks,
near=near, far=far, near_clip=near_clip,
i_train=i_train, i_val=i_val, i_test=i_test,
poses=poses, render_poses=render_poses,
images=images, depths=depths,
irregular_shape=irregular_shape,
)
return data_dict
def inward_nearfar_heuristic(cam_o, ratio=0.05):
dist = np.linalg.norm(cam_o[:,None] - cam_o, axis=-1)
far = dist.max() # could be too small to exist the scene bbox
# it is only used to determined scene bbox
# lib/dvgo use 1e9 as far
near = far * ratio
return near, far
================================================
FILE: FourierGrid/common_data_loaders/load_deepvoxels.py
================================================
import os
import numpy as np
import imageio
def load_dv_data(scene='cube', basedir='/data/deepvoxels', testskip=1):
def parse_intrinsics(filepath, trgt_sidelength, invert_y=False):
# Get camera intrinsics
with open(filepath, 'r') as file:
f, cx, cy = list(map(float, file.readline().split()))[:3]
grid_barycenter = np.array(list(map(float, file.readline().split())))
near_plane = float(file.readline())
scale = float(file.readline())
height, width = map(float, file.readline().split())
try:
world2cam_poses = int(file.readline())
except ValueError:
world2cam_poses = None
if world2cam_poses is None:
world2cam_poses = False
world2cam_poses = bool(world2cam_poses)
print(cx,cy,f,height,width)
cx = cx / width * trgt_sidelength
cy = cy / height * trgt_sidelength
f = trgt_sidelength / height * f
fx = f
if invert_y:
fy = -f
else:
fy = f
# Build the intrinsic matrices
full_intrinsic = np.array([[fx, 0., cx, 0.],
[0., fy, cy, 0],
[0., 0, 1, 0],
[0, 0, 0, 1]])
return full_intrinsic, grid_barycenter, scale, near_plane, world2cam_poses
def load_pose(filename):
assert os.path.isfile(filename)
nums = open(filename).read().split()
return np.array([float(x) for x in nums]).reshape([4,4]).astype(np.float32)
H = 512
W = 512
deepvoxels_base = '{}/train/{}/'.format(basedir, scene)
full_intrinsic, grid_barycenter, scale, near_plane, world2cam_poses = parse_intrinsics(os.path.join(deepvoxels_base, 'intrinsics.txt'), H)
print(full_intrinsic, grid_barycenter, scale, near_plane, world2cam_poses)
focal = full_intrinsic[0,0]
print(H, W, focal)
def dir2poses(posedir):
poses = np.stack([load_pose(os.path.join(posedir, f)) for f in sorted(os.listdir(posedir)) if f.endswith('txt')], 0)
transf = np.array([
[1,0,0,0],
[0,-1,0,0],
[0,0,-1,0],
[0,0,0,1.],
])
poses = poses @ transf
poses = poses[:,:3,:4].astype(np.float32)
return poses
posedir = os.path.join(deepvoxels_base, 'pose')
poses = dir2poses(posedir)
testposes = dir2poses('{}/test/{}/pose'.format(basedir, scene))
testposes = testposes[::testskip]
valposes = dir2poses('{}/validation/{}/pose'.format(basedir, scene))
valposes = valposes[::testskip]
imgfiles = [f for f in sorted(os.listdir(os.path.join(deepvoxels_base, 'rgb'))) if f.endswith('png')]
imgs = np.stack([imageio.imread(os.path.join(deepvoxels_base, 'rgb', f))/255. for f in imgfiles], 0).astype(np.float32)
testimgd = '{}/test/{}/rgb'.format(basedir, scene)
imgfiles = [f for f in sorted(os.listdir(testimgd)) if f.endswith('png')]
testimgs = np.stack([imageio.imread(os.path.join(testimgd, f))/255. for f in imgfiles[::testskip]], 0).astype(np.float32)
valimgd = '{}/validation/{}/rgb'.format(basedir, scene)
imgfiles = [f for f in sorted(os.listdir(valimgd)) if f.endswith('png')]
valimgs = np.stack([imageio.imread(os.path.join(valimgd, f))/255. for f in imgfiles[::testskip]], 0).astype(np.float32)
all_imgs = [imgs, valimgs, testimgs]
counts = [0] + [x.shape[0] for x in all_imgs]
counts = np.cumsum(counts)
i_split = [np.arange(counts[i], counts[i+1]) for i in range(3)]
imgs = np.concatenate(all_imgs, 0)
poses = np.concatenate([poses, valposes, testposes], 0)
render_poses = testposes
print(poses.shape, imgs.shape)
return imgs, poses, render_poses, [H, W, focal], i_split
================================================
FILE: FourierGrid/common_data_loaders/load_free.py
================================================
import numpy as np
import os, imageio
import torch
import scipy
import cv2
import pdb
from shutil import copy
from subprocess import check_output
from FourierGrid.trajectory_generators.interp_traj import *
########## Slightly modified version of LLFF data loading code
########## see https://github.com/Fyusion/LLFF for original
def imread(f):
if f.endswith('png'):
return imageio.imread(f, ignoregamma=True)
else:
return imageio.imread(f)
def depthread(path):
with open(path, "rb") as fid:
width, height, channels = np.genfromtxt(fid, delimiter="&", max_rows=1,
usecols=(0, 1, 2), dtype=int)
fid.seek(0)
num_delimiter = 0
byte = fid.read(1)
while True:
if byte == b"&":
num_delimiter += 1
if num_delimiter >= 3:
break
byte = fid.read(1)
array = np.fromfile(fid, np.float32)
array = array.reshape((width, height, channels), order="F")
return np.transpose(array, (1, 0, 2)).squeeze()
def _minify(basedir, factors=[], resolutions=[]):
needtoload = False
for r in factors:
imgdir = os.path.join(basedir, 'images_{}'.format(r))
if not os.path.exists(imgdir):
needtoload = True
for r in resolutions:
imgdir = os.path.join(basedir, 'images_{}x{}'.format(r[1], r[0]))
if not os.path.exists(imgdir):
needtoload = True
if not needtoload:
return
imgdir = os.path.join(basedir, 'images')
imgs = [os.path.join(imgdir, f) for f in sorted(os.listdir(imgdir))]
imgs = [f for f in imgs if any([f.endswith(ex) for ex in ['JPG', 'jpg', 'jpeg', 'png', 'jpeg', 'PNG']])]
imgdir_orig = imgdir
wd = os.getcwd()
for r in factors + resolutions:
if isinstance(r, int):
name = 'images_{}'.format(r)
resizearg = '{}%'.format(100./r)
else:
name = 'images_{}x{}'.format(r[1], r[0])
resizearg = '{}x{}'.format(r[1], r[0])
imgdir = os.path.join(basedir, name)
if os.path.exists(imgdir):
print("Image folder exists, do not call the resize function.")
continue
os.makedirs(imgdir, exist_ok=True)
print('Minifying', r, basedir)
# check_output('cp {}/* {}'.format(imgdir_orig, imgdir), shell=True)
ext = imgs[0].split('.')[-1]
for idx, one_img_p in enumerate(imgs):
one_img = cv2.imread(one_img_p)
ori_h, ori_w = one_img.shape[0], one_img.shape[1]
if isinstance(r, int):
target_h, target_w = int(ori_h / r), int(ori_w / r)
else:
target_h, target_w = r[0], r[1]
resized = cv2.resize(one_img, (target_w, target_h), interpolation = cv2.INTER_AREA)
target_img_p = one_img_p.replace(imgdir_orig, imgdir)
cv2.imwrite(target_img_p, resized)
# args = ' '.join(['convert mogrify', '-resize', resizearg, '-format', 'png', '*.{}'.format(ext)])
# print(args)
# os.chdir(imgdir)
# check_output(args, shell=True)
# os.chdir(wd)
# if ext != 'png':
# check_output('rm {}/*.{}'.format(imgdir, ext), shell=True)
# print('Removed duplicates')
print('Done')
def normalize_scene(poses, n_images, bounds):
# TODO: vdalidate the effectiveness of this function
cam_pos = poses[:, :, 3].clone()
center_ = cam_pos.mean(dim=0, keepdim=False)
bias = cam_pos - center_.unsqueeze(0)
radius_ = torch.linalg.norm(bias, ord=2, dim=-1, keepdim=False).max().item()
cam_pos = (cam_pos - center_.unsqueeze(0)) / radius_
poses[:, :, 3] = cam_pos
bounds = (bounds / radius_)
return poses, bounds, center_, radius_
def load_images_from_disk(basedir, factor, height, width):
img0 = [os.path.join(basedir, 'images', f) for f in sorted(os.listdir(os.path.join(basedir, 'images'))) \
if f.endswith('JPG') or f.endswith('jpg') or f.endswith('jpeg') or f.endswith('png')][0]
sh = imageio.imread(img0).shape
sfx = ''
if height is not None and width is not None:
_minify(basedir, resolutions=[[height, width]])
sfx = '_{}x{}'.format(width, height)
elif factor is not None and factor != 1:
sfx = '_{}'.format(factor)
_minify(basedir, factors=[factor])
factor = factor
elif height is not None:
factor = sh[0] / float(height)
width = int(sh[1] / factor)
_minify(basedir, resolutions=[[height, width]])
sfx = '_{}x{}'.format(width, height)
elif width is not None:
factor = sh[1] / float(width)
height = int(sh[0] / factor)
_minify(basedir, resolutions=[[height, width]])
sfx = '_{}x{}'.format(width, height)
else:
factor = 1
imgdir = os.path.join(basedir, 'images' + sfx)
print(f'Loading images from {imgdir}')
if not os.path.exists(imgdir):
print(imgdir, 'does not exist, returning' )
import sys; sys.exit()
return
imgfiles = [os.path.join(imgdir, f) for f in sorted(os.listdir(imgdir)) if f.endswith('JPG') or f.endswith('jpg') or f.endswith('png')]
if len(imgfiles) < 3:
print('Too few images...')
import sys; sys.exit()
imgs = [imread(f)[...,:3]/255. for f in imgfiles]
imgs = np.stack(imgs, 0)
return imgs, factor
def normalize(x):
return x / np.linalg.norm(x)
def ptstocam(pts, c2w):
tt = np.matmul(c2w[:3,:3].T, (pts-c2w[:3,3])[...,np.newaxis])[...,0]
return tt
def viewmatrix(z, up, pos):
vec2 = normalize(z)
vec1_avg = up
vec0 = normalize(np.cross(vec1_avg, vec2))
vec1 = normalize(np.cross(vec2, vec0))
m = np.stack([vec0, vec1, vec2, pos], 1)
return m
def render_path_spiral(c2w, up, rads, focal, zdelta, zrate, rots, N):
render_poses = []
rads = np.array(list(rads) + [1.])
hwf = c2w[:,4:5]
for theta in np.linspace(0., 2. * np.pi * rots, N+1)[:-1]:
c = np.dot(c2w[:3,:4], np.array([np.cos(theta), -np.sin(theta), -np.sin(theta*zrate)*zdelta, 1.]) * rads)
z = normalize(c - np.dot(c2w[:3,:4], np.array([0,0,-focal, 1.])))
render_poses.append(np.concatenate([viewmatrix(z, up, c), hwf], 1))
return render_poses
def poses_avg(poses):
hwf = poses[0, :3, -1:]
center = poses[:, :3, 3].mean(0)
vec2 = normalize(poses[:, :3, 2].sum(0))
up = poses[:, :3, 1].sum(0)
c2w = np.concatenate([viewmatrix(vec2, up, center), hwf], 1)
return c2w
def recenter_poses(poses, render_poses):
poses_ = poses + 0
bottom = np.reshape([0,0,0,1.], [1,4])
c2w = poses_avg(poses)
c2w = np.concatenate([c2w[:3,:4], bottom], -2)
bottom = np.tile(np.reshape(bottom, [1,1,4]), [poses.shape[0],1,1])
poses = np.concatenate([poses[:,:3,:4], bottom], -2)
poses = np.linalg.inv(c2w) @ poses
poses_[:,:3,:4] = poses[:,:3,:4]
poses = poses_
# apply c2w to render poses
render_poses_ = render_poses + 0
bottom = np.reshape([0,0,0,1.], [1,4])
bottom = np.tile(np.reshape(bottom, [1,1,4]), [render_poses.shape[0],1,1])
render_poses = np.concatenate([render_poses[:,:3,:4], bottom], -2)
render_poses = np.linalg.inv(c2w) @ render_poses
render_poses_[:, :3, :4] = render_poses[:, :3, :4]
render_poses = render_poses_
return poses, render_poses
def rerotate_poses(poses):
poses = np.copy(poses)
centroid = poses[:,:3,3].mean(0)
poses[:,:3,3] = poses[:,:3,3] - centroid
# Find the minimum pca vector with minimum eigen value
x = poses[:,:,3]
mu = x.mean(0)
cov = np.cov((x-mu).T)
ev , eig = np.linalg.eig(cov)
cams_up = eig[:,np.argmin(ev)]
if cams_up[1] < 0:
cams_up = -cams_up
# Find rotation matrix that align cams_up with [0,1,0]
R = scipy.spatial.transform.Rotation.align_vectors(
[[0,1,0]], cams_up[None])[0].as_matrix()
# Apply rotation and add back the centroid position
poses[:,:3,:3] = R @ poses[:,:3,:3]
poses[:,:3,[3]] = R @ poses[:,:3,[3]]
poses[:,:3,3] = poses[:,:3,3] + centroid
return poses
#####################
def spherify_poses(poses, bds, depths):
p34_to_44 = lambda p : np.concatenate([p, np.tile(np.reshape(np.eye(4)[-1,:], [1,1,4]), [p.shape[0], 1,1])], 1)
rays_d = poses[:,:3,2:3]
rays_o = poses[:,:3,3:4]
def min_line_dist(rays_o, rays_d):
A_i = np.eye(3) - rays_d * np.transpose(rays_d, [0,2,1])
b_i = -A_i @ rays_o
pt_mindist = np.squeeze(-np.linalg.inv((np.transpose(A_i, [0,2,1]) @ A_i).mean(0)) @ (b_i).mean(0))
return pt_mindist
pt_mindist = min_line_dist(rays_o, rays_d)
center = pt_mindist
up = (poses[:,:3,3] - center).mean(0)
vec0 = normalize(up)
vec1 = normalize(np.cross([.1,.2,.3], vec0))
vec2 = normalize(np.cross(vec0, vec1))
pos = center
c2w = np.stack([vec1, vec2, vec0, pos], 1)
poses_reset = np.linalg.inv(p34_to_44(c2w[None])) @ p34_to_44(poses[:,:3,:4])
radius = np.sqrt(np.mean(np.sum(np.square(poses_reset[:,:3,3]), -1)))
sc = 1./radius
poses_reset[:,:3,3] *= sc
bds *= sc
radius *= sc
depths *= sc
poses_reset = np.concatenate([poses_reset[:,:3,:4], np.broadcast_to(poses[0,:3,-1:], poses_reset[:,:3,-1:].shape)], -1)
return poses_reset, radius, bds, depths
def load_free_data(args, basedir, factor=8, width=None, height=None,
recenter=True, rerotate=True,
bd_factor=.75, spherify=False, path_zflat=False, load_depths=False,
movie_render_kwargs={}, training_ids=None, generate_render_poses=True, n_out_poses=200, sc=1.0):
# 1. load and parse poses, images, and bounds
meta_pose = torch.tensor(np.load(os.path.join(basedir, 'cams_meta.npy')))
n_images = meta_pose.shape[0]
cam_data = meta_pose.reshape(n_images, 27)
poses = cam_data[:, 0:12].reshape(-1, 3, 4)
intri = cam_data[:, 12:21].reshape(-1, 3, 3)
poses = poses.cpu().numpy()
intri = intri.cpu().numpy()
# 2. Rotation matrix correct, this has been done in colmap2standard
# poses = np.concatenate([poses[:, 1:2, :], -poses[:, 0:1, :], poses[:, 2:, :]], 1)
# poses = np.moveaxis(poses, -1, 0).astype(np.float32)
imgs, factor = load_images_from_disk(basedir, factor, height, width)
intri[..., :2, :3] /= factor
dist_params = cam_data[:, 21:25].reshape(-1, 4)
bounds = cam_data[:, 25:27].reshape(-1, 2)
# 2. normalize scenes
# poses, bounds, center, radius = normalize_scene(poses, n_images, bounds)
assert not load_depths, "do not support loading depths"
assert len(imgs.shape) == 4, "image shape is not correct!"
assert intri[0][0][0] == intri[1][0][0] and intri[1][0][0] == intri[2][0][0], "focal length are varying!"
# filter by training_ids
if training_ids is not None:
poses = np.array([poses[id] for id in training_ids])
intri = np.array([intri[id] for id in training_ids])
imgs = np.array([imgs[id] for id in training_ids])
bounds = bounds[training_ids]
# 3. load render camera poses or generate render poses on the fly
if generate_render_poses:
key_poses_indexs = np.arange(0, poses.shape[0], 5)
key_poses = poses[key_poses_indexs]
render_poses_ = inter_poses(key_poses, n_out_poses)
else:
poses_render_path = os.path.join(basedir, "poses_render.npy")
arr = np.load(poses_render_path)
cam_data = torch.from_numpy(arr.astype(np.float64)).to(torch.float32).cuda()
n_render_poses = arr.shape[0]
cam_data = cam_data.reshape((-1, 3, 4))
cam_data = cam_data[:n_render_poses, :, :]
render_poses_ = cam_data.clone() # [n, 3, 4]
# render_poses_[:, :3, 3] = (render_poses_[:, :3, 3] - center.unsqueeze(0)) / radius #commented out for debugging
render_poses_ = render_poses_.cpu().numpy()
hwf = np.array([[imgs.shape[1], imgs.shape[2], intri[0][0][0]]for _ in range(render_poses_.shape[0])])
render_poses_ = np.concatenate((render_poses_, hwf.reshape((render_poses_.shape[0], 3, 1))), axis=2)
# 4. relax bounds
# bounds_factor = [0.5, 4.0]
bounds = torch.stack([bounds[:, 0], bounds[:, 1]], dim=-1)
bounds.clamp_(1e-2, 1e9)
bounds = bounds.cpu().numpy()
near = bounds.min().item()
# sc = 1 / (near * bd_factor) # 0.12 by default
poses[:,:3,3] *= sc
render_poses_[:, :3, 3] *= sc
hwf = np.array([[imgs.shape[1], imgs.shape[2], intri[0][0][0]]for _ in range(imgs.shape[0])])
poses = np.concatenate((poses, hwf.reshape((imgs.shape[0], 3, 1))), axis=2)
poses, render_poses_ = recenter_poses(poses, render_poses_)
# 5. get test ID, this part is written by DVGO.
if args.llffhold > 0:
print('Auto LLFF holdout,', args.llffhold)
i_test = np.arange(imgs.shape[0])[::args.llffhold]
else:
print("LLFF hold is not used!")
i_test = [0, 1, 2]
return imgs, 0, intri, poses, bounds, render_poses_, i_test
================================================
FILE: FourierGrid/common_data_loaders/load_llff.py
================================================
import numpy as np
import os, imageio
import torch
import scipy
import cv2
import pdb
from shutil import copy
from subprocess import check_output
########## Slightly modified version of LLFF data loading code
########## see https://github.com/Fyusion/LLFF for original
def imread(f):
if f.endswith('png'):
return imageio.imread(f, ignoregamma=True)
else:
return imageio.imread(f)
def depthread(path):
with open(path, "rb") as fid:
width, height, channels = np.genfromtxt(fid, delimiter="&", max_rows=1,
usecols=(0, 1, 2), dtype=int)
fid.seek(0)
num_delimiter = 0
byte = fid.read(1)
while True:
if byte == b"&":
num_delimiter += 1
if num_delimiter >= 3:
break
byte = fid.read(1)
array = np.fromfile(fid, np.float32)
array = array.reshape((width, height, channels), order="F")
return np.transpose(array, (1, 0, 2)).squeeze()
def _minify(basedir, factors=[], resolutions=[]):
needtoload = False
for r in factors:
imgdir = os.path.join(basedir, 'images_{}'.format(r))
if not os.path.exists(imgdir):
needtoload = True
for r in resolutions:
imgdir = os.path.join(basedir, 'images_{}x{}'.format(r[1], r[0]))
if not os.path.exists(imgdir):
needtoload = True
if not needtoload:
return
imgdir = os.path.join(basedir, 'images')
imgs = [os.path.join(imgdir, f) for f in sorted(os.listdir(imgdir))]
imgs = [f for f in imgs if any([f.endswith(ex) for ex in ['JPG', 'jpg', 'jpeg', 'png', 'jpeg', 'PNG']])]
imgdir_orig = imgdir
wd = os.getcwd()
for r in factors + resolutions:
if isinstance(r, int):
name = 'images_{}'.format(r)
resizearg = '{}%'.format(100./r)
else:
name = 'images_{}x{}'.format(r[1], r[0])
resizearg = '{}x{}'.format(r[1], r[0])
imgdir = os.path.join(basedir, name)
if os.path.exists(imgdir):
print("Image folder exists, do not call the resize function.")
continue
os.makedirs(imgdir, exist_ok=True)
print('Minifying', r, basedir)
# check_output('cp {}/* {}'.format(imgdir_orig, imgdir), shell=True)
ext = imgs[0].split('.')[-1]
for idx, one_img_p in enumerate(imgs):
one_img = cv2.imread(one_img_p)
ori_h, ori_w = one_img.shape[0], one_img.shape[1]
if isinstance(r, int):
target_h, target_w = int(ori_h / r), int(ori_w / r)
else:
target_h, target_w = r[0], r[1]
resized = cv2.resize(one_img, (target_w, target_h), interpolation = cv2.INTER_AREA)
target_img_p = one_img_p.replace(imgdir_orig, imgdir)
cv2.imwrite(target_img_p, resized)
# args = ' '.join(['convert mogrify', '-resize', resizearg, '-format', 'png', '*.{}'.format(ext)])
# print(args)
# os.chdir(imgdir)
# check_output(args, shell=True)
# os.chdir(wd)
# if ext != 'png':
# check_output('rm {}/*.{}'.format(imgdir, ext), shell=True)
# print('Removed duplicates')
print('Done')
def _load_data(basedir, factor=None, width=None, height=None, load_imgs=True, load_depths=False):
poses_arr = np.load(os.path.join(basedir, 'poses_bounds.npy'))
if poses_arr.shape[1] == 17:
poses = poses_arr[:, :-2].reshape([-1, 3, 5]).transpose([1,2,0])
elif poses_arr.shape[1] == 14:
poses = poses_arr[:, :-2].reshape([-1, 3, 4]).transpose([1,2,0])
else:
raise NotImplementedError
bds = poses_arr[:, -2:].transpose([1,0])
img0 = [os.path.join(basedir, 'images', f) for f in sorted(os.listdir(os.path.join(basedir, 'images'))) \
if f.endswith('JPG') or f.endswith('jpg') or f.endswith('jpeg') or f.endswith('png')][0]
sh = imageio.imread(img0).shape
sfx = ''
if height is not None and width is not None:
_minify(basedir, resolutions=[[height, width]])
sfx = '_{}x{}'.format(width, height)
elif factor is not None and factor != 1:
sfx = '_{}'.format(factor)
_minify(basedir, factors=[factor])
factor = factor
elif height is not None:
factor = sh[0] / float(height)
width = int(sh[1] / factor)
_minify(basedir, resolutions=[[height, width]])
sfx = '_{}x{}'.format(width, height)
elif width is not None:
factor = sh[1] / float(width)
height = int(sh[0] / factor)
_minify(basedir, resolutions=[[height, width]])
sfx = '_{}x{}'.format(width, height)
else:
factor = 1
imgdir = os.path.join(basedir, 'images' + sfx)
print(f'Loading images from {imgdir}')
if not os.path.exists(imgdir):
print(imgdir, 'does not exist, returning' )
return
imgfiles = [os.path.join(imgdir, f) for f in sorted(os.listdir(imgdir)) if f.endswith('JPG') or f.endswith('jpg') or f.endswith('png')]
if poses.shape[-1] != len(imgfiles):
print()
print( 'Mismatch between imgs {} and poses {} !!!!'.format(len(imgfiles), poses.shape[-1]) )
names = set(name[:-4] for name in np.load(os.path.join(basedir, 'poses_names.npy')))
assert len(names) == poses.shape[-1]
print('Below failed files are skip due to SfM failure:')
new_imgfiles = []
for i in imgfiles:
fname = os.path.split(i)[1][:-4]
if fname in names:
new_imgfiles.append(i)
else:
print('==>', i)
imgfiles = new_imgfiles
if len(imgfiles) < 3:
print('Too few images...')
import sys; sys.exit()
sh = imageio.imread(imgfiles[0]).shape
if poses.shape[1] == 4:
poses = np.concatenate([poses, np.zeros_like(poses[:,[0]])], 1)
poses[2, 4, :] = np.load(os.path.join(basedir, 'hwf_cxcy.npy'))[2]
poses[:2, 4, :] = np.array(sh[:2]).reshape([2, 1])
poses[2, 4, :] = poses[2, 4, :] * 1./factor
if not load_imgs:
return poses, bds
imgs = imgs = [imread(f)[...,:3]/255. for f in imgfiles]
imgs = np.stack(imgs, -1)
print('Loaded image data', imgs.shape, poses[:,-1,0])
if not load_depths:
return poses, bds, imgs
depthdir = os.path.join(basedir, 'stereo', 'depth_maps')
assert os.path.exists(depthdir), f'Dir not found: {depthdir}'
depthfiles = [os.path.join(depthdir, f) for f in sorted(os.listdir(depthdir)) if f.endswith('.geometric.bin')]
assert poses.shape[-1] == len(depthfiles), 'Mismatch between imgs {} and poses {} !!!!'.format(len(depthfiles), poses.shape[-1])
depths = [depthread(f) for f in depthfiles]
depths = np.stack(depths, -1)
print('Loaded depth data', depths.shape)
return poses, bds, imgs, depths
def normalize(x):
return x / np.linalg.norm(x)
def viewmatrix(z, up, pos):
vec2 = normalize(z)
vec1_avg = up
vec0 = normalize(np.cross(vec1_avg, vec2))
vec1 = normalize(np.cross(vec2, vec0))
m = np.stack([vec0, vec1, vec2, pos], 1)
return m
def ptstocam(pts, c2w):
tt = np.matmul(c2w[:3,:3].T, (pts-c2w[:3,3])[...,np.newaxis])[...,0]
return tt
def poses_avg(poses):
assert poses.shape[-1] == 5, "HWF must be given in the pose matrix!"
hwf = poses[0, :3, -1:]
center = poses[:, :3, 3].mean(0)
vec2 = normalize(poses[:, :3, 2].sum(0))
up = poses[:, :3, 1].sum(0)
c2w = np.concatenate([viewmatrix(vec2, up, center), hwf], 1)
return c2w
def render_path_spiral(c2w, up, rads, focal, zdelta, zrate, rots, N):
render_poses = []
rads = np.array(list(rads) + [1.])
hwf = c2w[:,4:5]
for theta in np.linspace(0., 2. * np.pi * rots, N+1)[:-1]:
c = np.dot(c2w[:3,:4], np.array([np.cos(theta), -np.sin(theta), -np.sin(theta*zrate)*zdelta, 1.]) * rads)
z = normalize(c - np.dot(c2w[:3,:4], np.array([0,0,-focal, 1.])))
render_poses.append(np.concatenate([viewmatrix(z, up, c), hwf], 1))
return render_poses
def recenter_poses(poses):
poses_ = poses+0
bottom = np.reshape([0,0,0,1.], [1,4])
c2w = poses_avg(poses)
c2w = np.concatenate([c2w[:3,:4], bottom], -2)
bottom = np.tile(np.reshape(bottom, [1,1,4]), [poses.shape[0],1,1])
poses = np.concatenate([poses[:,:3,:4], bottom], -2)
poses = np.linalg.inv(c2w) @ poses
poses_[:,:3,:4] = poses[:,:3,:4]
poses = poses_
return poses
def rerotate_poses(poses):
poses = np.copy(poses)
centroid = poses[:,:3,3].mean(0)
poses[:,:3,3] = poses[:,:3,3] - centroid
# Find the minimum pca vector with minimum eigen value
x = poses[:,:,3]
mu = x.mean(0)
cov = np.cov((x-mu).T)
ev , eig = np.linalg.eig(cov)
cams_up = eig[:,np.argmin(ev)]
if cams_up[1] < 0:
cams_up = -cams_up
# Find rotation matrix that align cams_up with [0,1,0]
R = scipy.spatial.transform.Rotation.align_vectors(
[[0,1,0]], cams_up[None])[0].as_matrix()
# Apply rotation and add back the centroid position
poses[:,:3,:3] = R @ poses[:,:3,:3]
poses[:,:3,[3]] = R @ poses[:,:3,[3]]
poses[:,:3,3] = poses[:,:3,3] + centroid
return poses
#####################
def spherify_poses(poses, bds, depths):
p34_to_44 = lambda p : np.concatenate([p, np.tile(np.reshape(np.eye(4)[-1,:], [1,1,4]), [p.shape[0], 1,1])], 1)
rays_d = poses[:,:3,2:3]
rays_o = poses[:,:3,3:4]
def min_line_dist(rays_o, rays_d):
A_i = np.eye(3) - rays_d * np.transpose(rays_d, [0,2,1])
b_i = -A_i @ rays_o
pt_mindist = np.squeeze(-np.linalg.inv((np.transpose(A_i, [0,2,1]) @ A_i).mean(0)) @ (b_i).mean(0))
return pt_mindist
pt_mindist = min_line_dist(rays_o, rays_d)
center = pt_mindist
up = (poses[:,:3,3] - center).mean(0)
vec0 = normalize(up)
vec1 = normalize(np.cross([.1,.2,.3], vec0))
vec2 = normalize(np.cross(vec0, vec1))
pos = center
c2w = np.stack([vec1, vec2, vec0, pos], 1)
poses_reset = np.linalg.inv(p34_to_44(c2w[None])) @ p34_to_44(poses[:,:3,:4])
radius = np.sqrt(np.mean(np.sum(np.square(poses_reset[:,:3,3]), -1)))
sc = 1./radius
poses_reset[:,:3,3] *= sc
bds *= sc
radius *= sc
depths *= sc
poses_reset = np.concatenate([poses_reset[:,:3,:4], np.broadcast_to(poses[0,:3,-1:], poses_reset[:,:3,-1:].shape)], -1)
return poses_reset, radius, bds, depths
def load_llff_data(basedir, factor=8, width=None, height=None,
recenter=True, rerotate=True,
bd_factor=.75, spherify=False, path_zflat=False, load_depths=False,
movie_render_kwargs={}):
poses, bds, imgs, *depths = _load_data(basedir, factor=factor, width=width, height=height,
load_depths=load_depths) # factor=8 downsamples original imgs by 8x
# poses: [3, 5, N], bds: [2, N], imgs: [H, W, 3, N], depths: []
print('Loaded', basedir, bds.min(), bds.max())
if load_depths:
depths = depths[0]
else:
depths = 0
# Correct rotation matrix ordering and move variable dim to axis 0
poses = np.concatenate([poses[:, 1:2, :], -poses[:, 0:1, :], poses[:, 2:, :]], 1)
poses = np.moveaxis(poses, -1, 0).astype(np.float32)
imgs = np.moveaxis(imgs, -1, 0).astype(np.float32)
images = imgs
bds = np.moveaxis(bds, -1, 0).astype(np.float32)
# Rescale if bd_factor is provided
if bds.min() < 0 and bd_factor is not None:
print('Found negative z values from SfM sparse points!?')
print('Please try bd_factor=None. This program is terminating now!')
import sys; sys.exit()
sc = 1. if bd_factor is None else 1./(bds.min() * bd_factor)
poses[:,:3,3] *= sc
bds *= sc
depths *= sc
if recenter:
poses = recenter_poses(poses)
if spherify:
poses, radius, bds, depths = spherify_poses(poses, bds, depths)
if rerotate:
poses = rerotate_poses(poses)
### generate spiral poses for rendering fly-through movie
centroid = poses[:,:3,3].mean(0)
radcircle = movie_render_kwargs.get('scale_r', 1) * np.linalg.norm(poses[:,:3,3] - centroid, axis=-1).mean()
centroid[0] += movie_render_kwargs.get('shift_x', 0)
centroid[1] += movie_render_kwargs.get('shift_y', 0)
centroid[2] += movie_render_kwargs.get('shift_z', 0)
new_up_rad = movie_render_kwargs.get('pitch_deg', 0) * np.pi / 180
target_y = radcircle * np.tan(new_up_rad)
render_poses = []
for th in np.linspace(0., 2.*np.pi, 200):
camorigin = np.array([radcircle * np.cos(th), 0, radcircle * np.sin(th)])
if movie_render_kwargs.get('flip_up', False):
up = np.array([0,1.,0])
else:
up = np.array([0,-1.,0])
vec2 = normalize(camorigin)
vec0 = normalize(np.cross(vec2, up))
vec1 = normalize(np.cross(vec2, vec0))
pos = camorigin + centroid
# rotate to align with new pitch rotation
lookat = -vec2
lookat[1] = target_y
lookat = normalize(lookat)
vec2 = -lookat
vec1 = normalize(np.cross(vec2, vec0))
p = np.stack([vec0, vec1, vec2, pos], 1)
render_poses.append(p)
render_poses = np.stack(render_poses, 0)
render_poses = np.concatenate([render_poses, np.broadcast_to(poses[0,:3,-1:], render_poses[:,:3,-1:].shape)], -1)
else:
c2w = poses_avg(poses)
print('recentered', c2w.shape)
print(c2w[:3,:4])
## Get spiral
# Get average pose
up = normalize(poses[:, :3, 1].sum(0))
# Find a reasonable "focus depth" for this dataset
close_depth, inf_depth = bds.min()*.9, bds.max()*5.
dt = .75
mean_dz = 1./(((1.-dt)/close_depth + dt/inf_depth))
focal = mean_dz * movie_render_kwargs.get('scale_f', 1)
# Get radii for spiral path
zdelta = movie_render_kwargs.get('zdelta', 0.5)
zrate = movie_render_kwargs.get('zrate', 1.0)
tt = poses[:,:3,3] # ptstocam(poses[:3,3,:].T, c2w).T
rads = np.percentile(np.abs(tt), 90, 0) * movie_render_kwargs.get('scale_r', 1)
c2w_path = c2w
N_views = 120
N_rots = movie_render_kwargs.get('N_rots', 1)
if path_zflat:
# zloc = np.percentile(tt, 10, 0)[2]
zloc = -close_depth * .1
c2w_path[:3,3] = c2w_path[:3,3] + zloc * c2w_path[:3,2]
rads[2] = 0.
N_rots = 1
N_views/=2
# Generate poses for spiral path
render_poses = render_path_spiral(c2w_path, up, rads, focal, zdelta, zrate=zrate, rots=N_rots, N=N_views)
render_poses = torch.Tensor(render_poses)
c2w = poses_avg(poses)
print('Data:')
print(poses.shape, images.shape, bds.shape)
# this part is written by DVGO.
dists = np.sum(np.square(c2w[:3,3] - poses[:,:3,3]), -1)
i_test = np.argmin(dists)
print('HOLDOUT view is', i_test)
images = images.astype(np.float32)
poses = poses.astype(np.float32)
return images, depths, poses, bds, render_poses, i_test
================================================
FILE: FourierGrid/common_data_loaders/load_nerfpp.py
================================================
'''
Modify from
https://github.com/Kai-46/nerfplusplus/blob/master/data_loader_split.py
'''
import os
import pdb
import glob
import scipy
import imageio
import numpy as np
import torch
########################################################################################################################
# camera coordinate system: x-->right, y-->down, z-->scene (opencv/colmap convention)
# poses is camera-to-world
########################################################################################################################
def find_files(dir, exts):
if os.path.isdir(dir):
files_grabbed = []
for ext in exts:
files_grabbed.extend(glob.glob(os.path.join(dir, ext)))
if len(files_grabbed) > 0:
files_grabbed = sorted(files_grabbed)
return files_grabbed
else:
return []
def load_data_split(split_dir, skip=1, try_load_min_depth=True, only_img_files=False,
training_ids=None):
def parse_txt(filename):
assert os.path.isfile(filename)
nums = open(filename).read().split()
return np.array([float(x) for x in nums]).reshape([4, 4]).astype(np.float32)
if only_img_files:
img_files = find_files('{}/rgb'.format(split_dir), exts=['*.png', '*.jpg'])
return img_files
# camera parameters files
intrinsics_files = find_files('{}/intrinsics'.format(split_dir), exts=['*.txt'])
pose_files = find_files('{}/pose'.format(split_dir), exts=['*.txt'])
intrinsics_files = intrinsics_files[::skip]
pose_files = pose_files[::skip]
cam_cnt = len(pose_files)
# img files
img_files = find_files('{}/rgb'.format(split_dir), exts=['*.png', '*.jpg'])
if len(img_files) > 0:
img_files = img_files[::skip]
assert(len(img_files) == cam_cnt)
else:
img_files = [None, ] * cam_cnt
# mask files
mask_files = find_files('{}/mask'.format(split_dir), exts=['*.png', '*.jpg'])
if len(mask_files) > 0:
mask_files = mask_files[::skip]
assert(len(mask_files) == cam_cnt)
else:
mask_files = [None, ] * cam_cnt
# min depth files
mindepth_files = find_files('{}/min_depth'.format(split_dir), exts=['*.png', '*.jpg'])
if try_load_min_depth and len(mindepth_files) > 0:
mindepth_files = mindepth_files[::skip]
assert(len(mindepth_files) == cam_cnt)
else:
mindepth_files = [None, ] * cam_cnt
# sample by training ids
if training_ids is not None:
final_training_ids = []
for idx, ele in enumerate(intrinsics_files):
if int(ele.split("/")[-1].replace(".txt", "")) in training_ids:
final_training_ids.append(idx)
training_ids = final_training_ids
training_ids = [id - 1 for id in training_ids] # image id start with 1
intrinsics_files = [intrinsics_files[id] for id in training_ids]
pose_files = [pose_files[id] for id in training_ids]
img_files = [img_files[id] for id in training_ids]
mask_files = [mask_files[id] for id in training_ids]
mindepth_files = [mindepth_files[id] for id in training_ids]
return intrinsics_files, pose_files, img_files, mask_files, mindepth_files
def rerotate_poses(poses, render_poses):
poses = np.copy(poses)
centroid = poses[:,:3,3].mean(0)
poses[:,:3,3] = poses[:,:3,3] - centroid
# Find the minimum pca vector with minimum eigen value
x = poses[:,:3,3]
mu = x.mean(0)
cov = np.cov((x-mu).T)
ev , eig = np.linalg.eig(cov)
cams_up = eig[:,np.argmin(ev)]
if cams_up[1] < 0:
cams_up = -cams_up
# Find rotation matrix that align cams_up with [0,1,0]
R = scipy.spatial.transform.Rotation.align_vectors(
[[0,-1,0]], cams_up[None])[0].as_matrix()
# Apply rotation and add back the centroid position
poses[:,:3,:3] = R @ poses[:,:3,:3]
poses[:,:3,[3]] = R @ poses[:,:3,[3]]
poses[:,:3,3] = poses[:,:3,3] + centroid
render_poses = np.copy(render_poses)
render_poses[:,:3,3] = render_poses[:,:3,3] - centroid
render_poses[:,:3,:3] = R @ render_poses[:,:3,:3]
render_poses[:,:3,[3]] = R @ render_poses[:,:3,[3]]
render_poses[:,:3,3] = render_poses[:,:3,3] + centroid
return poses, render_poses
def load_nerfpp_data(basedir, rerotate=True, training_ids=None):
tr_K, tr_c2w, tr_im_path = load_data_split(os.path.join(basedir, 'train'), training_ids=training_ids)[:3]
assert len(tr_im_path) > 0, f"Images are not found in {basedir}"
te_K, te_c2w, te_im_path = load_data_split(os.path.join(basedir, 'test'))[:3]
assert len(tr_K) == len(tr_c2w) and len(tr_K) == len(tr_im_path)
assert len(te_K) == len(te_c2w) and len(te_K) == len(te_im_path)
# Determine split id list
i_split = [[], []]
i = 0
for _ in tr_c2w:
i_split[0].append(i)
i += 1
for _ in te_c2w:
i_split[1].append(i)
i += 1
# Load camera intrinsics. Assume all images share a intrinsic.
K_flatten = np.loadtxt(tr_K[0])
for path in tr_K:
assert np.allclose(np.loadtxt(path), K_flatten)
for path in te_K:
assert np.allclose(np.loadtxt(path), K_flatten)
K = K_flatten.reshape(4,4)[:3,:3]
# Load camera poses
poses = []
for path in tr_c2w:
poses.append(np.loadtxt(path).reshape(4,4))
for path in te_c2w:
poses.append(np.loadtxt(path).reshape(4,4))
# Load images
imgs = []
for path in tr_im_path:
imgs.append(imageio.imread(path) / 255.)
for path in te_im_path:
imgs.append(imageio.imread(path) / 255.)
# Bundle all data
imgs = np.stack(imgs, 0)
poses = np.stack(poses, 0)
i_split.append(i_split[1])
H, W = imgs.shape[1:3]
focal = K[[0,1], [0,1]].mean()
# Generate movie trajectory
render_poses_path = sorted(glob.glob(os.path.join(basedir, 'camera_path', 'pose', '*txt')))
render_poses = []
for path in render_poses_path:
render_poses.append(np.loadtxt(path).reshape(4,4))
render_poses = np.array(render_poses)
render_K = np.loadtxt(glob.glob(os.path.join(basedir, 'camera_path', 'intrinsics', '*txt'))[0]).reshape(4,4)[:3,:3]
render_poses[:,:,0] *= K[0,0] / render_K[0,0]
render_poses[:,:,1] *= K[1,1] / render_K[1,1]
if rerotate:
poses, render_poses = rerotate_poses(poses, render_poses)
render_poses = torch.Tensor(render_poses)
return imgs, poses, render_poses, [H, W, focal], K, i_split
================================================
FILE: FourierGrid/common_data_loaders/load_nsvf.py
================================================
import os
import glob
import torch
import numpy as np
import imageio
import json
import torch.nn.functional as F
import cv2
trans_t = lambda t : torch.Tensor([
[1,0,0,0],
[0,1,0,0],
[0,0,1,t],
[0,0,0,1]]).float()
rot_phi = lambda phi : torch.Tensor([
[1,0,0,0],
[0,np.cos(phi),-np.sin(phi),0],
[0,np.sin(phi), np.cos(phi),0],
[0,0,0,1]]).float()
rot_theta = lambda th : torch.Tensor([
[np.cos(th),0,-np.sin(th),0],
[0,1,0,0],
[np.sin(th),0, np.cos(th),0],
[0,0,0,1]]).float()
def pose_spherical(theta, phi, radius):
c2w = trans_t(radius)
c2w = rot_phi(phi/180.*np.pi) @ c2w
c2w = rot_theta(theta/180.*np.pi) @ c2w
c2w = torch.Tensor(np.array([[-1,0,0,0],[0,0,1,0],[0,1,0,0],[0,0,0,1]])) @ c2w
c2w[:,[1,2]] *= -1
return c2w
def load_nsvf_data(basedir):
pose_paths = sorted(glob.glob(os.path.join(basedir, 'pose', '*txt')))
rgb_paths = sorted(glob.glob(os.path.join(basedir, 'rgb', '*png')))
all_poses = []
all_imgs = []
i_split = [[], [], []]
for i, (pose_path, rgb_path) in enumerate(zip(pose_paths, rgb_paths)):
i_set = int(os.path.split(rgb_path)[-1][0])
all_imgs.append((imageio.imread(rgb_path) / 255.).astype(np.float32))
all_poses.append(np.loadtxt(pose_path).astype(np.float32))
i_split[i_set].append(i)
imgs = np.stack(all_imgs, 0)
poses = np.stack(all_poses, 0)
H, W = imgs[0].shape[:2]
with open(os.path.join(basedir, 'intrinsics.txt')) as f:
focal = float(f.readline().split()[0])
R = np.sqrt((poses[...,:3,3]**2).sum(-1)).mean()
render_poses = torch.stack([pose_spherical(angle, -30.0, R) for angle in np.linspace(-180,180,200+1)[:-1]], 0)
return imgs, poses, render_poses, [H, W, focal], i_split
================================================
FILE: FourierGrid/common_data_loaders/load_tankstemple.py
================================================
import os
import glob
import torch
import numpy as np
import imageio
import json
import torch.nn.functional as F
import cv2
def normalize(x):
return x / np.linalg.norm(x)
def load_tankstemple_data(basedir, movie_render_kwargs={}):
pose_paths = sorted(glob.glob(os.path.join(basedir, 'pose', '*txt')))
rgb_paths = sorted(glob.glob(os.path.join(basedir, 'rgb', '*png')))
all_poses = []
all_imgs = []
i_split = [[], []]
for i, (pose_path, rgb_path) in enumerate(zip(pose_paths, rgb_paths)):
i_set = int(os.path.split(rgb_path)[-1][0])
all_poses.append(np.loadtxt(pose_path).astype(np.float32))
all_imgs.append((imageio.imread(rgb_path) / 255.).astype(np.float32))
i_split[i_set].append(i)
imgs = np.stack(all_imgs, 0)
poses = np.stack(all_poses, 0)
i_split.append(i_split[-1])
path_intrinsics = os.path.join(basedir, 'intrinsics.txt')
H, W = imgs[0].shape[:2]
K = np.loadtxt(path_intrinsics)
focal = float(K[0,0])
### generate spiral poses for rendering fly-through movie
centroid = poses[:,:3,3].mean(0)
radcircle = movie_render_kwargs.get('scale_r', 1.0) * np.linalg.norm(poses[:,:3,3] - centroid, axis=-1).mean()
centroid[0] += movie_render_kwargs.get('shift_x', 0)
centroid[1] += movie_render_kwargs.get('shift_y', 0)
centroid[2] += movie_render_kwargs.get('shift_z', 0)
new_up_rad = movie_render_kwargs.get('pitch_deg', 0) * np.pi / 180
target_y = radcircle * np.tan(new_up_rad)
render_poses = []
for th in np.linspace(0., 2.*np.pi, 200):
camorigin = np.array([radcircle * np.cos(th), 0, radcircle * np.sin(th)])
if movie_render_kwargs.get('flip_up_vec', False):
up = np.array([0,-1.,0])
else:
up = np.array([0,1.,0])
vec2 = normalize(camorigin)
vec0 = normalize(np.cross(vec2, up))
vec1 = normalize(np.cross(vec2, vec0))
pos = camorigin + centroid
# rotate to align with new pitch rotation
lookat = -vec2
lookat[1] = target_y
lookat = normalize(lookat)
lookat *= -1
vec2 = -lookat
vec1 = normalize(np.cross(vec2, vec0))
p = np.stack([vec0, vec1, vec2, pos], 1)
render_poses.append(p)
render_poses = np.stack(render_poses, 0)
render_poses = np.concatenate([render_poses, np.broadcast_to(poses[0,:3,-1:], render_poses[:,:3,-1:].shape)], -1)
return imgs, poses, render_poses, [H, W, focal], K, i_split
================================================
FILE: FourierGrid/configs/blendedmvs/Character.py
================================================
_base_ = '../default.py'
expname = 'dvgo_Character'
basedir = './logs/blended_mvs'
data = dict(
datadir='./data/BlendedMVS/Character/',
dataset_type='blendedmvs',
inverse_y=True,
white_bkgd=True,
)
================================================
FILE: FourierGrid/configs/blendedmvs/Fountain.py
================================================
_base_ = '../default.py'
expname = 'dvgo_Fountain'
basedir = './logs/blended_mvs'
data = dict(
datadir='./data/BlendedMVS/Fountain/',
dataset_type='blendedmvs',
inverse_y=True,
white_bkgd=False,
)
================================================
FILE: FourierGrid/configs/blendedmvs/Jade.py
================================================
_base_ = '../default.py'
expname = 'dvgo_Jade'
basedir = './logs/blended_mvs'
data = dict(
datadir='./data/BlendedMVS/Jade/',
dataset_type='blendedmvs',
inverse_y=True,
white_bkgd=False,
)
================================================
FILE: FourierGrid/configs/blendedmvs/Statues.py
================================================
_base_ = '../default.py'
expname = 'dvgo_Statues'
basedir = './logs/blended_mvs'
data = dict(
datadir='./data/BlendedMVS/Statues/',
dataset_type='blendedmvs',
inverse_y=True,
white_bkgd=True,
)
================================================
FILE: FourierGrid/configs/co3d/donut_369_40208_78816.py
================================================
_base_ = '../default.py'
expname = 'dvgo_donut_369_40208_78816'
basedir = './logs/co3d'
data = dict(
datadir='./data/co3d/',
dataset_type='co3d',
annot_path='./data/co3d/donut/frame_annotations.jgz',
split_path='./data/co3d/donut/set_lists.json',
sequence_name='369_40208_78816',
flip_x=True,
flip_y=True,
inverse_y=True,
white_bkgd=False,
)
coarse_train = dict(
ray_sampler='flatten',
)
================================================
FILE: FourierGrid/configs/custom/Madoka.py
================================================
_base_ = './default_forward_facing.py'
expname = 'Madoka'
data = dict(
datadir='data/Madoka/dense',
factor=2,
movie_render_kwargs={
'scale_r': 1.0,
'scale_f': 0.8,
'zrate': 2.0,
'zdelta': 0.5,
}
)
fine_train = dict(
N_iters=300000,
)
================================================
FILE: FourierGrid/configs/custom/Madoka_long.py
================================================
_base_ = './default_forward_facing.py'
expname = 'Madoka'
data = dict(
datadir='data/Madoka/dense',
factor=2,
movie_render_kwargs={
'scale_r': 1.0,
'scale_f': 0.8,
'zrate': 2.0,
'zdelta': 0.5,
}
)
fine_train = dict(
N_iters=300000,
pg_scale=[50000, 100000, 150000, 200000],
)
================================================
FILE: FourierGrid/configs/custom/Otobai.py
================================================
_base_ = './default_forward_facing.py'
expname = 'Otobai'
data = dict(
datadir='./data/Otobai/dense',
factor=2,
movie_render_kwargs={
'scale_r': 0.8,
'scale_f': 10.0,
'zrate': 6.0,
'zdelta': 0.5,
}
)
fine_train = dict(
N_iters=300000,
pg_scale=[50000, 100000, 150000, 200000],
)
================================================
FILE: FourierGrid/configs/custom/default_forward_facing.py
================================================
_base_ = '../default.py'
basedir = './logs/custom'
data = dict(
dataset_type='llff',
load2gpu_on_the_fly=True,
ndc=True,
llffhold=0,
rand_bkgd=True,
movie_render_kwargs={
'scale_r': 1.0, # circling radius
'scale_f': 1.0, # the distance to the looking point of foucs
'zdelta': 0.5, # amplitude of forward motion
'zrate': 1.0, # frequency of forward motion
'N_rots': 1, # number of rotation in 120 frames
}
)
coarse_train = dict(
N_iters=0,
)
fine_train = dict(
N_iters=30000,
N_rand=4096,
weight_distortion=0.01,
pg_scale=[2000,4000,6000,8000],
decay_after_scale=0.1,
ray_sampler='flatten',
tv_before=1e9,
tv_dense_before=10000,
weight_tv_density=1e-5,
weight_tv_k0=1e-6,
)
_mpi_depth = 256
_stepsize = 1.0
fine_model_and_render = dict(
num_voxels=384*384*_mpi_depth,
mpi_depth=_mpi_depth,
stepsize=_stepsize,
rgbnet_dim=9,
rgbnet_width=64,
world_bound_scale=1,
fast_color_thres=_stepsize/_mpi_depth/5,
)
================================================
FILE: FourierGrid/configs/custom/default_ubd_inward_facing.py
================================================
_base_ = '../default.py'
basedir = './logs/custom'
data = dict(
dataset_type='llff',
spherify=True,
llffhold=0,
bd_factor=None,
white_bkgd=True,
rand_bkgd=True,
unbounded_inward=True,
load2gpu_on_the_fly=True,
)
coarse_train = dict(N_iters=0)
fine_train = dict(
N_iters=40000,
N_rand=4096,
lrate_decay=80,
ray_sampler='flatten',
weight_nearclip=0.0,
weight_distortion=0.01,
pg_scale=[2000,4000,6000,8000,10000,12000,14000,16000],
tv_before=20000,
tv_dense_before=20000,
weight_tv_density=1e-6,
weight_tv_k0=1e-7,
)
alpha_init = 1e-4
stepsize = 0.5
fine_model_and_render = dict(
num_voxels=320**3,
num_voxels_base=320**3,
alpha_init=alpha_init,
stepsize=stepsize,
fast_color_thres={
'_delete_': True,
0 : alpha_init*stepsize/10,
1500: min(alpha_init, 1e-4)*stepsize/5,
2500: min(alpha_init, 1e-4)*stepsize/2,
3500: min(alpha_init, 1e-4)*stepsize/1.5,
4500: min(alpha_init, 1e-4)*stepsize,
5500: min(alpha_init, 1e-4),
6500: 1e-4,
},
world_bound_scale=1,
)
================================================
FILE: FourierGrid/configs/custom/sm01_desktop.py
================================================
_base_ = './default_forward_facing.py'
expname = 'sm01_desktop'
data = dict(
datadir='./data/sm01_desktop/dense',
factor=2,
movie_render_kwargs={
'scale_r': 0.5,
'scale_f': 1.0,
'zrate': 1.0,
'zdelta': 0.5,
}
)
================================================
FILE: FourierGrid/configs/custom/sm02_multiple_desktop.py
================================================
_base_ = './default_forward_facing.py'
expname = 'sm02_multiple_desktop'
data = dict(
datadir='./data/sm02_multiple_desktop/dense',
factor=2,
movie_render_kwargs={
'scale_r': 0.5,
'scale_f': 1.0,
'zrate': 1.0,
'zdelta': 0.5,
}
)
================================================
FILE: FourierGrid/configs/custom/sm03_meeting.py
================================================
_base_ = './default_forward_facing.py'
expname = 'sm03_meeting'
data = dict(
datadir='./data/sm03_meeting/dense',
factor=2,
movie_render_kwargs={
'scale_r': 0.5,
'scale_f': 1.0,
'zrate': 1.0,
'zdelta': 0.5,
}
)
================================================
FILE: FourierGrid/configs/deepvoxels/armchair.py
================================================
_base_ = '../default.py'
expname = 'dvgo_armchair'
basedir = './logs/deepvoxels'
data = dict(
datadir='./data/deepvoxels/',
dataset_type='deepvoxels',
scene='armchair',
white_bkgd=True,
)
================================================
FILE: FourierGrid/configs/deepvoxels/cube.py
================================================
_base_ = '../default.py'
expname = 'dvgo_cube'
basedir = './logs/deepvoxels'
data = dict(
datadir='./data/deepvoxels/',
dataset_type='deepvoxels',
scene='cube',
white_bkgd=True,
)
================================================
FILE: FourierGrid/configs/deepvoxels/greek.py
================================================
_base_ = '../default.py'
expname = 'dvgo_greek'
basedir = './logs/deepvoxels'
data = dict(
datadir='./data/deepvoxels/',
dataset_type='deepvoxels',
scene='greek',
white_bkgd=True,
)
================================================
FILE: FourierGrid/configs/deepvoxels/vase.py
================================================
_base_ = '../default.py'
expname = 'dvgo_vase'
basedir = './logs/deepvoxels'
data = dict(
datadir='./data/deepvoxels/',
dataset_type='deepvoxels',
scene='vase',
white_bkgd=True,
)
================================================
FILE: FourierGrid/configs/default.py
================================================
from copy import deepcopy
expname = None # experiment name
basedir = './logs/' # where to store ckpts and logs
model = None
''' Template of data options
'''
data = dict(
datadir=None, # path to dataset root folder
dataset_type=None, # blender | nsvf | blendedmvs | tankstemple | deepvoxels | co3d
inverse_y=False, # intrinsict mode (to support blendedmvs, nsvf, tankstemple)
flip_x=False, # to support co3d
flip_y=False, # to support co3d
annot_path='', # to support co3d
split_path='', # to support co3d
sequence_name='', # to support co3d
load2gpu_on_the_fly=False, # do not load all images into gpu (to save gpu memory)
testskip=1, # subsample testset to preview results
white_bkgd=False, # use white background (note that some dataset don't provide alpha and with blended bg color)
rand_bkgd=False, # use random background during training
half_res=False, # [TODO]
bd_factor=.75, # background
movie_render_kwargs=dict(),
# Below are forward-facing llff specific settings.
ndc=False, # use ndc coordinate (only for forward-facing; not support yet)
spherify=False, # inward-facing
factor=4, # [TODO]
width=None, # enforce image width
height=None, # enforce image height
llffhold=8, # test split via llff
dvgohold=8, # test split via dvgo
load_depths=False, # load depth
# Below are unbounded inward-facing specific settings.
unbounded_inward=False,
unbounded_inner_r=1.0,
boundary_ratio = 0.01, # used in FourierGrid model.
)
''' Template of training options
'''
coarse_train = dict(
N_iters=5000, # number of optimization steps
N_rand=8192, # batch size (number of random rays per optimization step)
lrate_density=1e-1, # lr of density voxel grid
lrate_k0=1e-1, # lr of color/feature voxel grid
lrate_rgbnet=1e-3, # lr of the mlp to preduct view-dependent color
lrate_decay=20, # lr decay by 0.1 after every lrate_decay*1000 steps
pervoxel_lr=True, # view-count-based lr
pervoxel_lr_downrate=1, # downsampled image for computing view-count-based lr
ray_sampler='random', # ray sampling strategies
weight_main=1.0, # weight of photometric loss
weight_entropy_last=0.01, # weight of background entropy loss
weight_nearclip=0,
weight_distortion=0,
weight_rgbper=0.1, # weight of per-point rgb loss
weight_freq=0, # weight of fourier mse loss
tv_every=1, # count total variation loss every tv_every step
tv_after=0, # count total variation loss from tv_from step
tv_before=0, # count total variation before the given number of iterations
tv_dense_before=0, # count total variation densely before the given number of iterations
weight_tv_density=0.0, # weight of total variation loss of density voxel grid
weight_tv_k0=0.0, # weight of total variation loss of color/feature voxel grid
pg_scale=[], # checkpoints for progressive scaling
decay_after_scale=1.0, # decay act_shift after scaling
skip_zero_grad_fields=[], # the variable name to skip optimizing parameters w/ zero grad in each iteration
maskout_lt_nviews=0,
)
fine_train = deepcopy(coarse_train)
fine_train.update(dict(
N_iters=20000,
pervoxel_lr=False,
ray_sampler='in_maskcache',
weight_entropy_last=0.001,
weight_rgbper=0.01,
pg_scale=[1000, 2000, 3000, 4000],
skip_zero_grad_fields=['density', 'k0'],
))
''' Template of model and rendering options
'''
coarse_model_and_render = dict(
num_voxels_rgb=1024000, # expected number of voxel
num_voxels_density=1024000, # expected number of voxel
num_voxels_viewdir=-1, # expected number of voxel
num_voxels_base_density=1024000, # to rescale delta distance
num_voxels_base_rgb=1024000, # to rescale delta distance
density_type='DenseGrid', # DenseGrid, TensoRFGrid
k0_type='DenseGrid', # DenseGrid, TensoRFGrid
density_config=dict(),
k0_config=dict(),
mpi_depth=128, # the number of planes in Multiplane Image (work when ndc=True)
nearest=False, # nearest interpolation
pre_act_density=False, # pre-activated trilinear interpolation
in_act_density=False, # in-activated trilinear interpolation
bbox_thres=1e-3, # threshold to determine known free-space in the fine stage
mask_cache_thres=1e-3, # threshold to determine a tighten BBox in the fine stage
rgbnet_dim=0, # feature voxel grid dim
rgbnet_full_implicit=False, # let the colors MLP ignore feature voxel grid
rgbnet_direct=True, # set to False to treat the first 3 dim of feature voxel grid as diffuse rgb
rgbnet_depth=3, # depth of the colors MLP (there are rgbnet_depth-1 intermediate features)
rgbnet_width=128, # width of the colors MLP
alpha_init=1e-6, # set the alpha values everywhere at the begin of training
fast_color_thres=1e-7, # threshold of alpha value to skip the fine stage sampled point
maskout_near_cam_vox=True, # maskout grid points that between cameras and their near planes
world_bound_scale=1, # rescale the BBox enclosing the scene
stepsize=0.5, # sampling stepsize in volume rendering
)
fine_model_and_render = deepcopy(coarse_model_and_render)
fine_model_and_render.update(dict(
num_voxels_density=160**3,
num_voxels_rgb=160**3,
num_voxels_base_density=160**3,
num_voxels_base_rgb=160**3,
rgbnet_dim=12,
alpha_init=1e-2,
fast_color_thres=1e-4,
maskout_near_cam_vox=False,
world_bound_scale=1.05,
fourier_freq_num=3,
))
vis = dict(
height_rate = 0.6 # camera direction frustrum height
)
del deepcopy
================================================
FILE: FourierGrid/configs/free_dataset/grass.py
================================================
_base_ = '../default.py'
expname = 'grass_may31_'
vis = dict(
height_rate = 0.6 # camera direction frustrum height
)
model='FourierGrid'
basedir = './logs/free_dataset'
data = dict(
datadir='./data/free_dataset/grass',
dataset_type='free',
ndc=False,
training_ids=[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, \
22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, \
46, 47, 48, 49, 50],
pose_scale=0.75, # scale of pose
factor=1,
# inverse_y=True,
# load2gpu_on_the_fly=True,
# white_bkgd=True,
# rand_bkgd=True,
# movie_render_kwargs={'pitch_deg': 20},
)
coarse_train = dict(
N_iters = 0,
pervoxel_lr_downrate=2,
pervoxel_lr=True, # DVGO default is True
)
fine_train = dict(
N_iters=100000,
N_rand=4096,
weight_distortion=0.0,
pg_scale=[2000,4000,6000,8000],
ray_sampler='flatten',
tv_before=1e9,
tv_dense_before=10000,
weight_tv_density=1e-5,
weight_tv_k0=1e-6,
)
voxel_size_density = 250 # default 400
voxel_size_rgb = 250 # default 320
fine_model_and_render = dict(
num_voxels=256**3,
num_voxels_density=voxel_size_density**3,
num_voxels_base_density=voxel_size_density**3,
num_voxels_rgb=voxel_size_rgb**3,
num_voxels_base_rgb=voxel_size_rgb**3,
mpi_depth=128,
rgbnet_dim=9,
rgbnet_width=64,
world_bound_scale=1,
fast_color_thres=1e-3,
)
================================================
FILE: FourierGrid/configs/free_dataset/lab.py
================================================
_base_ = '../default.py'
expname = 'lab_may30_'
vis = dict(
height_rate = 0.6 # camera direction frustrum height
)
model='DVGO'
basedir = './logs/free_dataset'
data = dict(
datadir='./data/free_dataset/lab',
dataset_type='free',
# ndc=True,
inverse_y=True,
load2gpu_on_the_fly=True,
white_bkgd=True,
rand_bkgd=True,
# movie_render_kwargs={'pitch_deg': 20},
)
coarse_train = dict(
N_iters = 0,
pervoxel_lr_downrate=2,
pervoxel_lr=True, # DVGO default is True
)
fine_train = dict(
N_iters=30000,
N_rand=4096,
weight_distortion=0.0,
pg_scale=[2000,4000,6000,8000],
ray_sampler='flatten',
tv_before=1e9,
tv_dense_before=10000,
weight_tv_density=1e-5,
weight_tv_k0=1e-6,
)
voxel_size_density = 250 # default 400
voxel_size_rgb = 250 # default 320
fine_model_and_render = dict(
# num_voxels=256**3,
num_voxels_density=voxel_size_density**3,
num_voxels_base_density=voxel_size_density**3,
num_voxels_rgb=voxel_size_rgb**3,
num_voxels_base_rgb=voxel_size_rgb**3,
mpi_depth=128,
rgbnet_dim=9,
rgbnet_width=64,
world_bound_scale=1,
fast_color_thres=1e-3,
)
# _base_ = '../default.py'
# expname = 'train_may29_'
# vis = dict(
# height_rate = 0.6 # camera direction frustrum height
# )
# model='FourierGrid'
# # model='DVGO'
# basedir = './logs/free_dataset'
# visualize_poses = False
# alpha_init = 1e-4
# stepsize = 0.5
# _mpi_depth = 256
# if visualize_poses: # for debugging
# unbounded_inward = True
# coarse_iter = 3000
# fast_color_thres=stepsize/_mpi_depth/5
# maskout_near_cam_vox = False
# pervoxel_lr = False
# weight_distortion = 0.0
# else:
# unbounded_inward = True
# coarse_iter = 0
# fast_color_thres={ # default
# '_delete_': True, # to ignore the base config
# 0 : alpha_init*stepsize/10, # 0.5e-5
# 1500: min(alpha_init, 1e-4)*stepsize/5, # 1e-5
# 2500: min(alpha_init, 1e-4)*stepsize/2, # 2.5e-5
# 3500: min(alpha_init, 1e-4)*stepsize/1.5,
# 4500: min(alpha_init, 1e-4)*stepsize,
# 5500: min(alpha_init, 1e-4),
# 6500: 1e-4,
# }
# maskout_near_cam_vox = False
# pervoxel_lr = False
# weight_distortion = 0.01
# data = dict(
# dataset_type='free',
# inverse_y=True,
# white_bkgd=True,
# rand_bkgd=True,
# unbounded_inward=unbounded_inward,
# load2gpu_on_the_fly=True,
# datadir='./data/free_dataset/grass',
# unbounded_inner_r=1.0,
# ndc=False,
# )
# coarse_train = dict(
# N_iters = coarse_iter,
# pervoxel_lr = pervoxel_lr,
# )
# fine_train = dict(
# N_iters=10000,
# # N_rand=2048, # reduce this to fit into memory
# N_rand=4096, # default
# ray_sampler='flatten',
# # ray_sampler='random',
# weight_distortion=weight_distortion,
# # pg_scale=[10,],
# pg_scale=[1000, 2000, 3000, 4000, 5000, 6000, 7000],
# tv_before=1e9, # always use tv
# tv_dense_before=10000,
# tv_after=0, # start from beginning
# tv_every=1,
# weight_tv_density=1e-6,
# weight_tv_k0=1e-7,
# pervoxel_lr=False,
# lrate_decay=20, # default
# lrate_density=1e-1, # default lr of density voxel grid
# lrate_k0=1e-1, # lr of color/feature voxel grid
# lrate_rgbnet=1e-3, # default lr of the mlp to preduct view-dependent color
# weight_entropy_last=1e-3, # default
# weight_rgbper=1e-2, # default
# weight_nearclip=0,
# weight_main=1.0, # default = 1
# weight_freq=0.0,
# )
# coarse_model_and_render = dict(
# maskout_near_cam_vox = maskout_near_cam_vox,
# )
# voxel_size_density = 250 # default 400
# voxel_size_rgb = 250 # default 320
# voxel_size_viewdir = -1
# # voxel_size_viewdir = 64
# fine_model_and_render = dict(
# num_voxels_density=voxel_size_density**3,
# num_voxels_base_density=voxel_size_density**3,
# num_voxels_rgb=voxel_size_rgb**3,
# num_voxels_base_rgb=voxel_size_rgb**3,
# num_voxels_viewdir=voxel_size_viewdir**3,
# alpha_init=alpha_init,
# stepsize=stepsize,
# fast_color_thres=fast_color_thres,
# world_bound_scale=1,
# # contracted_norm='l2', # default
# rgbnet_dim=12, # default
# # rgbnet_depth=3, # default
# viewbase_pe=4, # default=4
# bbox_thres=0.001,
# fourier_freq_num=3,
# maskout_near_cam_vox=False,
# bg_len=0.2, # default=0.2
# )
================================================
FILE: FourierGrid/configs/lf/africa.py
================================================
_base_ = './lf_default.py'
expname = 'dvgo_Africa_unbounded'
data = dict(
datadir='./data/lf_data/africa',
)
================================================
FILE: FourierGrid/configs/lf/basket.py
================================================
_base_ = './lf_default.py'
expname = 'dvgo_Basket_unbounded'
data = dict(
datadir='./data/lf_data/basket',
)
================================================
FILE: FourierGrid/configs/lf/lf_default.py
================================================
_base_ = '../default.py'
basedir = './logs/lf'
data = dict(
dataset_type='nerfpp',
inverse_y=True,
white_bkgd=False,
rand_bkgd=True,
unbounded_inward=True,
)
coarse_train = dict(N_iters=0)
fine_train = dict(
N_iters=25000,
N_rand=4096,
ray_sampler='flatten',
weight_distortion=1e-2,
pg_scale=[1000,2000,3000,4000,5000,6000],
decay_after_scale=1.0,
tv_before=1e9,
tv_dense_before=10000,
weight_tv_density=1e-6,
weight_tv_k0=1e-7,
)
alpha_init = 1e-4
stepsize = 0.5
fine_model_and_render = dict(
num_voxels=256**3,
num_voxels_base=256**3,
alpha_init=alpha_init,
stepsize=stepsize,
fast_color_thres={
'_delete_': True,
0 : alpha_init*stepsize/10,
1500: min(alpha_init, 1e-4)*stepsize/5,
2500: min(alpha_init, 1e-4)*stepsize/2,
3500: min(alpha_init, 1e-4)*stepsize/1.5,
4500: min(alpha_init, 1e-4)*stepsize,
5500: min(alpha_init, 1e-4),
6500: 1e-4,
},
world_bound_scale=1,
)
================================================
FILE: FourierGrid/configs/lf/ship.py
================================================
_base_ = './lf_default.py'
expname = 'dvgo_Ship_unbounded'
data = dict(
datadir='./data/lf_data/ship',
)
================================================
FILE: FourierGrid/configs/lf/statue.py
================================================
_base_ = './lf_default.py'
expname = 'dvgo_Statue_unbounded'
data = dict(
datadir='./data/lf_data/statue',
)
================================================
FILE: FourierGrid/configs/lf/torch.py
================================================
_base_ = './lf_default.py'
expname = 'dvgo_Torch_unbounded'
data = dict(
datadir='./data/lf_data/torch',
)
================================================
FILE: FourierGrid/configs/linemod/ape.py
================================================
_base_ = '../default.py'
seq_name = 'ape'
seq_id = 1
expname = f'{seq_name}_nov11_'
pose_expname = 'bayes_nerf_v2_4'
basedir = 'logs/linemod'
data = dict(
datadir='./data/linemod',
dataset_type='linemod',
white_bkgd=True,
seq_name=seq_name,
seq_id=1,
width_max=90,
height_max=90,
load2gpu_on_the_fly=True,
)
fine_train = dict(
N_iters=1*(10**4),
)
# voxel_num=32**3
# coarse_model_and_render = dict(
# num_voxels_rgb=voxel_num,
# num_voxels_base_rgb=voxel_num,
# num_voxels_density=voxel_num,
# num_voxels_base_density=voxel_num,
# )
# fine_model_and_render = dict(
# num_voxels_rgb=voxel_num,
# num_voxels_base_rgb=voxel_num,
# num_voxels_density=voxel_num,
# num_voxels_base_density=voxel_num,
# # rgbnet_dim=0,
# )
================================================
FILE: FourierGrid/configs/linemod/benchvise.py
================================================
_base_ = '../default.py'
seq_name = 'benchvise'
expname = f'{seq_name}_nov9_'
basedir = './logs/linemod'
data = dict(
datadir='./data/linemod',
dataset_type='linemod',
white_bkgd=True,
seq_name=seq_name,
width_max=230,
height_max=230,
#198, 224
)
# fine_train = dict(
# ray_sampler='flatten',
# )
================================================
FILE: FourierGrid/configs/linemod/camera.py
================================================
_base_ = '../default.py'
seq_name = 'camera'
expname = f'{seq_name}_nov9_'
basedir = './logs/linemod'
data = dict(
datadir='./data/linemod',
dataset_type='linemod',
white_bkgd=True,
seq_name=seq_name,
width_max=150,
height_max=150
# 142, 137
)
# fine_train = dict(
# ray_sampler='flatten',
# )
================================================
FILE: FourierGrid/configs/linemod/can.py
================================================
_base_ = '../default.py'
seq_name = 'can'
expname = f'{seq_name}_nov8'
basedir = './logs/linemod'
data = dict(
datadir='./data/linemod',
dataset_type='linemod',
white_bkgd=True,
seq_name=seq_name,
width_max=190,
height_max=190
# 162, 180
)
# fine_train = dict(
# ray_sampler='flatten',
# )
================================================
FILE: FourierGrid/configs/linemod/cat.py
================================================
_base_ = '../default.py'
seq_name = 'cat'
expname = f'{seq_name}_nov8_'
basedir = './logs/linemod'
data = dict(
datadir='./data/linemod',
dataset_type='linemod',
white_bkgd=True,
seq_name=seq_name,
width_max=130,
height_max=130,
)
# fine_train = dict(
# ray_sampler='flatten',
# )
================================================
FILE: FourierGrid/configs/linemod/driller.py
================================================
_base_ = '../default.py'
seq_name = 'driller'
expname = f'{seq_name}_nov8'
basedir = './logs/linemod'
data = dict(
datadir='./data/linemod',
dataset_type='linemod',
white_bkgd=True,
seq_name=seq_name,
width_max=250,
height_max=250,
# 240, 237
)
# fine_train = dict(
# ray_sampler='flatten',
# )
================================================
FILE: FourierGrid/configs/linemod/duck.py
================================================
_base_ = '../default.py'
seq_name = 'duck'
expname = f'{seq_name}_nov8'
basedir = './logs/linemod'
data = dict(
datadir='./data/linemod',
dataset_type='linemod',
white_bkgd=True,
seq_name=seq_name,
width_max=90,
height_max=90
# 85, 85
)
# fine_train = dict(
# ray_sampler='flatten',
# )
================================================
FILE: FourierGrid/configs/linemod/eggbox.py
================================================
_base_ = '../default.py'
seq_name = 'eggbox'
expname = f'{seq_name}_nov8'
basedir = './logs/linemod'
data = dict(
datadir='./data/linemod',
dataset_type='linemod',
white_bkgd=True,
seq_name=seq_name,
width_max=140,
height_max=140
# 131, 132
)
# fine_train = dict(
# ray_sampler='flatten',
# )
================================================
FILE: FourierGrid/configs/linemod/glue.py
================================================
_base_ = '../default.py'
seq_name = 'glue'
expname = f'{seq_name}_nov8'
basedir = './logs/linemod'
data = dict(
datadir='./data/linemod',
dataset_type='linemod',
white_bkgd=True,
seq_name=seq_name,
width_max=150,
height_max=150
# 111, 147
)
# fine_train = dict(
# ray_sampler='flatten',
# )
================================================
FILE: FourierGrid/configs/linemod/holepuncher.py
================================================
_base_ = '../default.py'
seq_name = 'holepuncher'
expname = f'{seq_name}_nov8'
basedir = './logs/linemod'
data = dict(
datadir='./data/linemod',
dataset_type='linemod',
white_bkgd=True,
seq_name=seq_name,
width_max=120,
height_max=120
# 120, 120
)
# fine_train = dict(
# ray_sampler='flatten',
# )
================================================
FILE: FourierGrid/configs/linemod/iron.py
================================================
_base_ = '../default.py'
seq_name = 'iron'
expname = f'{seq_name}_nov8'
basedir = './logs/linemod'
data = dict(
datadir='./data/linemod',
dataset_type='linemod',
white_bkgd=True,
seq_name=seq_name,
width_max=240,
height_max=240
# 233, 224
)
# fine_train = dict(
# ray_sampler='flatten',
# )
================================================
FILE: FourierGrid/configs/linemod/lamp.py
================================================
_base_ = '../default.py'
seq_name = 'lamp'
expname = f'{seq_name}_nov8'
basedir = './logs/linemod'
data = dict(
datadir='./data/linemod',
dataset_type='linemod',
white_bkgd=True,
seq_name=seq_name,
width_max=260,
height_max=260
# 232, 250
)
# fine_train = dict(
# ray_sampler='flatten',
# )
================================================
FILE: FourierGrid/configs/linemod/phone.py
================================================
_base_ = '../default.py'
seq_name = 'phone'
expname = f'{seq_name}_nov8'
basedir = './logs/linemod'
data = dict(
datadir='./data/linemod',
dataset_type='linemod',
white_bkgd=True,
seq_name=seq_name,
width_max=190,
height_max=190
# 159, 187
)
# fine_train = dict(
# ray_sampler='flatten',
# )
================================================
FILE: FourierGrid/configs/llff/fern.py
================================================
_base_ = './llff_default.py'
expname = 'fern'
data = dict(
datadir='./data/nerf_llff_data/fern',
)
================================================
FILE: FourierGrid/configs/llff/fern_lg.py
================================================
_base_ = './llff_default_lg.py'
expname = 'fern_lg'
data = dict(
datadir='./data/nerf_llff_data/fern',
)
================================================
FILE: FourierGrid/configs/llff/flower.py
================================================
_base_ = './llff_default.py'
expname = 'flower'
data = dict(
datadir='./data/nerf_llff_data/flower',
)
================================================
FILE: FourierGrid/configs/llff/flower_lg.py
================================================
_base_ = './llff_default_lg.py'
expname = 'flower_lg'
data = dict(
datadir='./data/nerf_llff_data/flower',
)
================================================
FILE: FourierGrid/configs/llff/fortress.py
================================================
_base_ = './llff_default.py'
expname = 'fortress'
data = dict(
datadir='./data/nerf_llff_data/fortress',
)
================================================
FILE: FourierGrid/configs/llff/fortress_lg.py
================================================
_base_ = './llff_default_lg.py'
expname = 'fortress_lg'
data = dict(
datadir='./data/nerf_llff_data/fortress',
)
================================================
FILE: FourierGrid/configs/llff/horns.py
================================================
_base_ = './llff_default.py'
expname = 'horns'
data = dict(
datadir='./data/nerf_llff_data/horns',
# dataset_type='free',
)
================================================
FILE: FourierGrid/configs/llff/horns_lg.py
================================================
_base_ = './llff_default_lg.py'
expname = 'horns_lg'
data = dict(
datadir='./data/nerf_llff_data/horns',
)
================================================
FILE: FourierGrid/configs/llff/leaves.py
================================================
_base_ = './llff_default.py'
model = 'DVGO'
expname = 'leaves'
basedir = './logs/llff'
data = dict(
datadir='./data/nerf_llff_data/leaves',
)
================================================
FILE: FourierGrid/configs/llff/leaves_lg.py
================================================
_base_ = './llff_default_lg.py'
expname = 'leaves_lg'
data = dict(
datadir='./data/nerf_llff_data/leaves',
)
================================================
FILE: FourierGrid/configs/llff/llff_default.py
================================================
_base_ = '../default.py'
basedir = './logs/llff'
data = dict(
dataset_type='llff',
ndc=True,
width=1008,
height=756,
)
coarse_train = dict(
N_iters=0,
)
fine_train = dict(
N_iters=30000,
N_rand=4096,
weight_distortion=0.01,
pg_scale=[2000,4000,6000,8000],
ray_sampler='flatten',
tv_before=1e9,
tv_dense_before=10000,
weight_tv_density=1e-5,
weight_tv_k0=1e-6,
)
fine_model_and_render = dict(
num_voxels=256**3,
mpi_depth=128,
rgbnet_dim=9,
rgbnet_width=64,
world_bound_scale=1,
fast_color_thres=1e-3,
)
================================================
FILE: FourierGrid/configs/llff/llff_default_lg.py
================================================
_base_ = '../default.py'
basedir = './logs/llff'
data = dict(
dataset_type='llff',
ndc=True,
width=1008,
height=756,
rand_bkgd=True,
)
coarse_train = dict(
N_iters=0,
)
fine_train = dict(
N_iters=30000,
N_rand=4096,
weight_distortion=0.01,
pg_scale=[2000,4000,6000,8000],
decay_after_scale=0.1,
ray_sampler='flatten',
tv_before=1e9,
tv_dense_before=10000,
weight_tv_density=1e-5,
weight_tv_k0=1e-6,
)
_mpi_depth = 256
_stepsize = 1.0
fine_model_and_render = dict(
num_voxels=384*384*_mpi_depth,
mpi_depth=_mpi_depth,
stepsize=_stepsize,
rgbnet_dim=9,
rgbnet_width=64,
world_bound_scale=1,
fast_color_thres=_stepsize/_mpi_depth/5,
)
================================================
FILE: FourierGrid/configs/llff/orchids.py
================================================
_base_ = './llff_default.py'
expname = 'orchids'
data = dict(
datadir='./data/nerf_llff_data/orchids',
)
================================================
FILE: FourierGrid/configs/llff/orchids_lg.py
================================================
_base_ = './llff_default_lg.py'
expname = 'orchids_lg'
data = dict(
datadir='./data/nerf_llff_data/orchids',
)
================================================
FILE: FourierGrid/configs/llff/room.py
================================================
_base_ = './llff_default.py'
expname = 'room'
data = dict(
datadir='./data/nerf_llff_data/room',
)
================================================
FILE: FourierGrid/configs/llff/room_lg.py
================================================
_base_ = './llff_default_lg.py'
expname = 'room_lg'
data = dict(
datadir='./data/nerf_llff_data/room',
)
================================================
FILE: FourierGrid/configs/llff/trex.py
================================================
_base_ = './llff_default.py'
expname = 'trex'
data = dict(
datadir='./data/nerf_llff_data/trex',
)
================================================
FILE: FourierGrid/configs/llff/trex_lg.py
================================================
_base_ = './llff_default_lg.py'
expname = 'trex_lg'
data = dict(
datadir='./data/nerf_llff_data/trex',
)
================================================
FILE: FourierGrid/configs/mega/building.py
================================================
_base_ = '../default.py'
model='FourierGrid'
data_name = 'building'
basedir = f'./logs/mega/{data_name}'
visualize_poses = False
alpha_init = 1e-2
stepsize = 0.5
_mpi_depth = 256
maskout_near_cam_vox = False # changed
pervoxel_lr = False
unbounded_inward = True
mega_dataset_root = "data/oct9_mega"
expname = f'oct9_mega_{data_name}'
if visualize_poses: # for debugging only
coarse_iter = 600
fast_color_thres=stepsize/_mpi_depth/5
weight_distortion = 0.0
else:
coarse_iter = 0
# fast_color_thres={
# '_delete_': True,
# 0 : alpha_init*stepsize/10,
# 1500: min(alpha_init, 1e-4)*stepsize/5,
# 2500: min(alpha_init, 1e-4)*stepsize/2,
# 3500: min(alpha_init, 1e-4)*stepsize/1.5,
# 4500: min(alpha_init, 1e-4)*stepsize,
# 5500: min(alpha_init, 1e-4),
# 6500: 1e-4,
# }
fast_color_thres=1e-3
weight_distortion = 0.01
data = dict(
dataset_type='mega',
inverse_y=True,
white_bkgd=True, # almost no effect when rand_bkgd=True
rand_bkgd=False, # random background
unbounded_inward=unbounded_inward,
load2gpu_on_the_fly=False,
datadir=f'data/oct9_mega/{data_name}',
factor=2,
near_clip = 0.1,
near = 0.1,
far = 0.01,
test_rotate_angle=50, # rotate angle in testing phase
sample_interval=1,
num_per_block=5, # run this num in block
)
coarse_train = dict(
N_iters=coarse_iter,
pervoxel_lr = pervoxel_lr,
ray_sampler='flatten',
)
fine_train = dict(
N_iters=40000, # 40k for whole training procedure
N_rand=4096,
ray_sampler='flatten',
weight_distortion=weight_distortion,
pg_scale=[],
# pg_scale=[500],
# pg_scale=[1000, 2000, 3000, 4000, 5000,],
tv_before=1e9,
tv_dense_before=10000,
weight_tv_density=1e-6,
weight_tv_k0=1e-7,
)
coarse_model_and_render = dict(
maskout_near_cam_vox = maskout_near_cam_vox,
bbox_thres=1e-10, # display all the bboxes
)
voxel_size = 400
# voxel_size = 320 # default
fine_model_and_render = dict(
num_voxels=voxel_size**3,
num_voxels_base=voxel_size**3,
alpha_init=alpha_init,
stepsize=stepsize,
fast_color_thres=fast_color_thres,
world_bound_scale=1,
contracted_norm='l2',
rgbnet_dim=12, # default
rgbnet_direct=True,
density_type='DenseGrid',
k0_type='DenseGrid',
bg_len=0.2, # default
viewbase_pe=8,
# maskout_near_cam_vox=True,
maskout_near_cam_vox=True, # changed to False because it might often trigger OOM error
# # TensorRF settings
# density_type='TensoRFGrid',
# k0_type='TensoRFGrid',
# density_config=dict(n_comp=8),
# k0_config=dict(n_comp=24),
)
vis = dict(
height_rate = 0.6 # camera direction frustrum height
)
# artistic radiance fields
arf = dict(
style_root = mega_dataset_root + "/styles",
style_id = 34,
)
================================================
FILE: FourierGrid/configs/mega/building_no_block.py
================================================
_base_ = '../default.py'
data_name = 'building'
model='FourierGrid'
basedir = f'./logs/mega/{data_name}'
visualize_poses = False
alpha_init = 1e-4
stepsize = 0.5
_mpi_depth = 256
maskout_near_cam_vox = False # changed
pervoxel_lr = False
unbounded_inward = True
expname = f'oct9_mega_{data_name}'
if visualize_poses: # for debugging only
coarse_iter = 600
fast_color_thres=stepsize/_mpi_depth/5
weight_distortion = 0.0
else:
coarse_iter = 0
fast_color_thres={
'_delete_': True,
0 : alpha_init*stepsize/10,
1500: min(alpha_init, 1e-4)*stepsize/5,
2500: min(alpha_init, 1e-4)*stepsize/2,
3500: min(alpha_init, 1e-4)*stepsize/1.5,
4500: min(alpha_init, 1e-4)*stepsize,
5500: min(alpha_init, 1e-4),
6500: 1e-4,
}
weight_distortion = -1
data = dict(
dataset_type='mega',
inverse_y=True,
white_bkgd=True, # almost no effect when rand_bkgd=True
rand_bkgd=True, # random background
unbounded_inward=unbounded_inward,
load2gpu_on_the_fly=True,
datadir=f'data/oct9_mega/{data_name}',
factor=2,
near_clip = 0.1,
near = 0.1,
far = 0.01,
test_rotate_angle=50, # rotate angle in testing phase
sample_interval=1,
num_per_block=-1, # run this num in block
unbounded_inner_r=1.0,
boundary_ratio=0.0,
# training_ids=['000517', '000520', '000524', ],
)
nerf_em = dict(
sample_num = 3,
pos_x_range = 0.1,
pos_y_range = 0.1,
pos_z_range = 0.1,
)
coarse_train = dict(
N_iters=coarse_iter,
pervoxel_lr = pervoxel_lr,
ray_sampler='flatten',
)
fine_train = dict(
N_iters_m_step=3000,
# N_iters=10*(10**4),
N_iters=3000,
N_rand=4096,
ray_sampler='flatten',
weight_distortion=weight_distortion,
# pg_scale=[1000, 2000, 3000, 4000, 5000, 6000, 7000], # default
pg_scale=[2000, 4000, 6000, 7000],
# pg_scale=[], # used for model size testing
tv_before=1e9,
tv_dense_before=10000,
weight_tv_density=1e-6,
weight_tv_k0=1e-7,
# added
pervoxel_lr=False,
lrate_decay=20, # default
lrate_density=1e-1, # default lr of density voxel grid
lrate_k0=1e-1, # lr of color/feature voxel grid
lrate_rgbnet=1e-3, # default lr of the mlp to preduct view-dependent color
weight_entropy_last=1e-3, # default
weight_rgbper=1e-2, # default
weight_nearclip=0,
weight_main=3.0, # default = 1
weight_freq=1.0,
)
coarse_model_and_render = dict(
maskout_near_cam_vox = maskout_near_cam_vox,
bbox_thres=1e-10, # display all the bboxes
)
voxel_size_density = 300 # default 400
voxel_size_rgb = 300 # default 320
voxel_size_viewdir = -1
# voxel_size = 320 # default
fine_model_and_render = dict(
num_voxels_density=voxel_size_density**3,
num_voxels_base_density=voxel_size_density**3,
num_voxels_rgb=voxel_size_rgb**3,
num_voxels_base_rgb=voxel_size_rgb**3,
num_voxels_viewdir=voxel_size_viewdir**3,
alpha_init=alpha_init,
stepsize=stepsize,
fast_color_thres=fast_color_thres,
world_bound_scale=1,
contracted_norm='l2',
rgbnet_dim=3, # default
rgbnet_direct=True,
density_type='DenseGrid',
k0_type='DenseGrid',
# bg_len=0.2, # default
bg_len=0.25, # default
viewbase_pe=8,
maskout_near_cam_vox=False,
)
vis = dict(
height_rate = 0.6 # camera direction frustrum height
)
================================================
FILE: FourierGrid/configs/mega/quad.py
================================================
_base_ = '../default.py'
model='FourierGrid'
data_name = 'quad'
basedir = f'./logs/mega/{data_name}'
visualize_poses = False
alpha_init = 1e-2
stepsize = 0.5
_mpi_depth = 256
maskout_near_cam_vox = False # changed
pervoxel_lr = False
unbounded_inward = True
expname = f'oct12_mega_{data_name}'
if visualize_poses: # for debugging only
coarse_iter = 600
fast_color_thres=stepsize/_mpi_depth/5
weight_distortion = 0.0
else:
coarse_iter = 0
# fast_color_thres={
# '_delete_': True,
# 0 : alpha_init*stepsize/10,
# 1500: min(alpha_init, 1e-4)*stepsize/5,
# 2500: min(alpha_init, 1e-4)*stepsize/2,
# 3500: min(alpha_init, 1e-4)*stepsize/1.5,
# 4500: min(alpha_init, 1e-4)*stepsize,
# 5500: min(alpha_init, 1e-4),
# 6500: 1e-4,
# }
fast_color_thres=1e-3
weight_distortion = 0.01
data = dict(
dataset_type='mega',
inverse_y=True,
white_bkgd=True, # almost no effect when rand_bkgd=True
rand_bkgd=False, # random background
unbounded_inward=unbounded_inward,
load2gpu_on_the_fly=False,
datadir=f'data/oct9_mega/{data_name}',
boundary_ratio=1.0,
factor=2, # not used
near_clip = 0.1,
near = 0.1,
far = 0.01,
test_rotate_angle=50, # rotate angle in testing phase
sample_interval=1,
)
coarse_train = dict(
N_iters=coarse_iter,
pervoxel_lr = pervoxel_lr,
ray_sampler='flatten',
)
fine_train = dict(
N_iters=20000, # 40k for whole training procedure
N_rand=4096,
ray_sampler='flatten',
weight_distortion=weight_distortion,
pg_scale=[500],
# pg_scale=[1000, 2000, 3000, 4000, 5000,],
tv_before=1e9,
tv_dense_before=10000,
weight_tv_density=1e-6,
weight_tv_k0=1e-7,
)
coarse_model_and_render = dict(
maskout_near_cam_vox = maskout_near_cam_vox,
bbox_thres=1e-10, # display all the bboxes
)
voxel_size = 400
# voxel_size = 320 # default
fine_model_and_render = dict(
num_voxels=voxel_size**3,
num_voxels_base=voxel_size**3,
alpha_init=alpha_init,
stepsize=stepsize,
fast_color_thres=fast_color_thres,
world_bound_scale=1,
contracted_norm='l2',
rgbnet_dim=12, # default
rgbnet_direct=True,
density_type='DenseGrid',
k0_type='DenseGrid',
bg_len=0.2, # default
viewbase_pe=8,
maskout_near_cam_vox=True,
# # TensorRF settings
# density_type='TensoRFGrid',
# k0_type='TensoRFGrid',
# density_config=dict(n_comp=8),
# k0_config=dict(n_comp=24),
)
vis = dict(
height_rate = 0.6 # camera direction frustrum height
)
================================================
FILE: FourierGrid/configs/mega/rubble.py
================================================
_base_ = '../default.py'
model='FourierGrid'
data_name = 'rubble'
basedir = f'./logs/mega/{data_name}'
visualize_poses = False
alpha_init = 1e-2
stepsize = 0.5
_mpi_depth = 256
maskout_near_cam_vox = False # changed
pervoxel_lr = False
unbounded_inward = True
expname = f'oct11_mega_{data_name}'
mega_dataset_root = "data/oct9_mega"
if visualize_poses: # for debugging only
coarse_iter = 600
fast_color_thres=stepsize/_mpi_depth/5
weight_distortion = 0.0
else:
coarse_iter = 0
# fast_color_thres={
# '_delete_': True,
# 0 : alpha_init*stepsize/10,
# 1500: min(alpha_init, 1e-4)*stepsize/5,
# 2500: min(alpha_init, 1e-4)*stepsize/2,
# 3500: min(alpha_init, 1e-4)*stepsize/1.5,
# 4500: min(alpha_init, 1e-4)*stepsize,
# 5500: min(alpha_init, 1e-4),
# 6500: 1e-4,
# }
fast_color_thres=1e-3
weight_distortion = 0.01
data = dict(
dataset_type='mega',
inverse_y=True,
white_bkgd=True, # almost no effect when rand_bkgd=True
rand_bkgd=False, # random background
unbounded_inward=unbounded_inward,
load2gpu_on_the_fly=False,
datadir=mega_dataset_root + f'/{data_name}',
boundary_ratio=1.0,
factor=2, # not used
near_clip = 0.1,
near = 0.1,
far = 0.01,
test_rotate_angle=50, # rotate angle in testing phase
sample_interval=1,
)
coarse_train = dict(
N_iters=coarse_iter,
pervoxel_lr = pervoxel_lr,
ray_sampler='flatten',
)
fine_train = dict(
N_iters=40000, # 40k for whole training procedure
N_rand=4096,
ray_sampler='flatten',
weight_distortion=weight_distortion,
pg_scale=[500],
# pg_scale=[1000, 2000, 3000, 4000, 5000,],
tv_before=1e9,
tv_dense_before=10000,
weight_tv_density=1e-6,
weight_tv_k0=1e-7,
)
coarse_model_and_render = dict(
maskout_near_cam_vox = maskout_near_cam_vox,
bbox_thres=1e-10, # display all the bboxes
)
voxel_size = 400
# voxel_size = 320 # default
fine_model_and_render = dict(
num_voxels=voxel_size**3,
num_voxels_base=voxel_size**3,
alpha_init=alpha_init,
stepsize=stepsize,
fast_color_thres=fast_color_thres,
world_bound_scale=1,
contracted_norm='l2',
rgbnet_dim=12, # default
rgbnet_direct=True,
density_type='DenseGrid',
k0_type='DenseGrid',
bg_len=0.2, # default
viewbase_pe=8,
maskout_near_cam_vox=True,
# # TensorRF settings
# density_type='TensoRFGrid',
# k0_type='TensoRFGrid',
# density_config=dict(n_comp=8),
# k0_config=dict(n_comp=24),
)
vis = dict(
height_rate = 0.6 # camera direction frustrum height
)
# artistic radiance fields
arf = dict(
style_root = mega_dataset_root + "/styles",
style_id = 5,
)
================================================
FILE: FourierGrid/configs/nerf/chair.py
================================================
_base_ = '../default.py'
expname = 'dvgo_chair'
basedir = './logs/nerf_synthetic'
data = dict(
datadir='./data/nerf_synthetic/chair',
dataset_type='blender',
white_bkgd=True,
)
================================================
FILE: FourierGrid/configs/nerf/drums.py
================================================
_base_ = '../default.py'
expname = 'dvgo_drums'
basedir = './logs/nerf_synthetic'
data = dict(
datadir='./data/nerf_synthetic/drums',
dataset_type='blender',
white_bkgd=True,
)
================================================
FILE: FourierGrid/configs/nerf/ficus.py
================================================
_base_ = '../default.py'
expname = 'dvgo_ficus'
basedir = './logs/nerf_synthetic'
data = dict(
datadir='./data/nerf_synthetic/ficus',
dataset_type='blender',
white_bkgd=True,
)
================================================
FILE: FourierGrid/configs/nerf/hotdog.py
================================================
_base_ = '../default.py'
expname = 'dvgo_hotdog'
basedir = './logs/nerf_synthetic'
data = dict(
datadir='./data/nerf_synthetic/hotdog',
dataset_type='blender',
white_bkgd=True,
)
================================================
FILE: FourierGrid/configs/nerf/lego.py
================================================
_base_ = '../default.py'
expname = 'dvgo_lego'
basedir = './logs/nerf_synthetic'
data = dict(
datadir='./data/nerf_synthetic/lego',
dataset_type='blender',
white_bkgd=True,
)
================================================
FILE: FourierGrid/configs/nerf/materials.py
================================================
_base_ = '../default.py'
expname = 'dvgo_materials'
basedir = './logs/nerf_synthetic'
data = dict(
datadir='./data/nerf_synthetic/materials',
dataset_type='blender',
white_bkgd=True,
)
================================================
FILE: FourierGrid/configs/nerf/mic.py
================================================
_base_ = '../default.py'
expname = 'dvgo_mic'
basedir = './logs/nerf_synthetic'
data = dict(
datadir='./data/nerf_synthetic/mic',
dataset_type='blender',
white_bkgd=True,
)
================================================
FILE: FourierGrid/configs/nerf/ship.py
================================================
_base_ = '../default.py'
expname = 'dvgo_ship'
basedir = './logs/nerf_synthetic'
data = dict(
datadir='./data/nerf_synthetic/ship',
dataset_type='blender',
white_bkgd=True,
)
================================================
FILE: FourierGrid/configs/nerf/ship.tensorf.py
================================================
_base_ = '../default.py'
expname = 'dvgo_ship_tensorf'
basedir = './logs/nerf_synthetic'
data = dict(
datadir='./data/nerf_synthetic/ship',
dataset_type='blender',
white_bkgd=True,
)
fine_train = dict(
lrate_density=0.02,
lrate_k0=0.02,
pg_scale=[1000,2000,3000,4000,5000,6000],
)
fine_model_and_render = dict(
num_voxels=384**3,
density_type='TensoRFGrid',
density_config=dict(n_comp=8),
k0_type='TensoRFGrid',
k0_config=dict(n_comp=24),
)
================================================
FILE: FourierGrid/configs/nerf_studio/Giannini_Hall.py
================================================
_base_ = './nerf_studio_default.py'
expname = 'Giannini-Hall_mar16_'
vis = dict(
height_rate = 0.6 # camera direction frustrum height
)
model='FourierGrid'
basedir = './logs/Giannini-Hall'
alpha_init = 1e-4
stepsize = 0.5
_mpi_depth = 256
coarse_iter = 0
fast_color_thres={ # default
'_delete_': True, # to ignore the base config
0 : alpha_init*stepsize/10, # 0.5e-5
1500: min(alpha_init, 1e-4)*stepsize/5, # 1e-5
2500: min(alpha_init, 1e-4)*stepsize/2, # 2.5e-5
3500: min(alpha_init, 1e-4)*stepsize/1.5,
4500: min(alpha_init, 1e-4)*stepsize,
5500: min(alpha_init, 1e-4),
6500: 1e-4,
}
maskout_near_cam_vox = False
pervoxel_lr = False
# weight_distortion = 0.01
weight_distortion = 0.02
data = dict(
dataset_type='nerfstudio',
spherify=True, # default: True
factor=8,
llffhold=-1,
dvgohold=8,
white_bkgd=True,
rand_bkgd=True,
unbounded_inward=True,
load2gpu_on_the_fly=True,
datadir='./data/nerfstudio_data/Giannini-Hall',
movie_render_kwargs=dict(
shift_x=0.0, # positive right
shift_y=-0.3, # negative down
shift_z=0,
scale_r=0.2,
pitch_deg=-40, # negative look downward
),
)
coarse_train = dict(N_iters=0)
fine_train = dict(
N_iters=100000,
N_rand=2048,
lrate_decay=80,
ray_sampler='flatten',
weight_nearclip=1.0,
weight_distortion=weight_distortion,
# pg_scale=[2000,4000,6000,8000,10000,12000,14000,16000],
pg_scale=[1000, 2000, 3000, 4000, 5000, 6000, 7000],
tv_before=20000,
tv_dense_before=20000,
weight_tv_density=1e-6,
weight_tv_k0=1e-7,
weight_main=1.0,
# weight_freq=0.1,
)
voxel_size_density = 200 # default 400
voxel_size_rgb = 200 # default 320
voxel_size_viewdir = -1
fine_model_and_render = dict(
num_voxels_density=voxel_size_density**3,
num_voxels_base_density=voxel_size_density**3,
num_voxels_rgb=voxel_size_rgb**3,
num_voxels_base_rgb=voxel_size_rgb**3,
num_voxels_viewdir=voxel_size_viewdir**3,
alpha_init=alpha_init,
stepsize=stepsize,
fast_color_thres={
'_delete_': True,
0 : alpha_init*stepsize/10,
1500: min(alpha_init, 1e-4)*stepsize/5,
2500: min(alpha_init, 1e-4)*stepsize/2,
3500: min(alpha_init, 1e-4)*stepsize/1.5,
4500: min(alpha_init, 1e-4)*stepsize,
5500: min(alpha_init, 1e-4),
6500: 1e-4,
},
world_bound_scale=1,
)
================================================
FILE: FourierGrid/configs/nerf_studio/nerf_studio_default.py
================================================
_base_ = '../default.py'
basedir = './logs/nerf_unbounded'
data = dict(
dataset_type='nerfstudio',
spherify=True,
factor=4,
llffhold=-1,
dvgohold=-1,
white_bkgd=True,
rand_bkgd=True,
unbounded_inward=True,
load2gpu_on_the_fly=True,
)
coarse_train = dict(N_iters=0)
fine_train = dict(
N_iters=40000,
N_rand=4096,
lrate_decay=80,
ray_sampler='flatten',
weight_nearclip=1.0,
weight_distortion=0.01,
pg_scale=[2000,4000,6000,8000,10000,12000,14000,16000],
tv_before=20000,
tv_dense_before=20000,
weight_tv_density=1e-6,
weight_tv_k0=1e-7,
)
alpha_init = 1e-4
stepsize = 0.5
fine_model_and_render = dict(
num_voxels=320**3,
num_voxels_base=320**3,
alpha_init=alpha_init,
stepsize=stepsize,
fast_color_thres={
'_delete_': True,
0 : alpha_init*stepsize/10,
1500: min(alpha_init, 1e-4)*stepsize/5,
2500: min(alpha_init, 1e-4)*stepsize/2,
3500: min(alpha_init, 1e-4)*stepsize/1.5,
4500: min(alpha_init, 1e-4)*stepsize,
5500: min(alpha_init, 1e-4),
6500: 1e-4,
},
world_bound_scale=1,
)
================================================
FILE: FourierGrid/configs/nerf_studio/stump.py
================================================
_base_ = './nerf_studio_default.py'
expname = 'stump_mar16_'
vis = dict(
height_rate = 0.6 # camera direction frustrum height
)
model='FourierGrid'
basedir = './logs/stump'
alpha_init = 1e-4
stepsize = 0.5
_mpi_depth = 256
coarse_iter = 0
fast_color_thres={ # default
'_delete_': True, # to ignore the base config
0 : alpha_init*stepsize/10, # 0.5e-5
1500: min(alpha_init, 1e-4)*stepsize/5, # 1e-5
2500: min(alpha_init, 1e-4)*stepsize/2, # 2.5e-5
3500: min(alpha_init, 1e-4)*stepsize/1.5,
4500: min(alpha_init, 1e-4)*stepsize,
5500: min(alpha_init, 1e-4),
6500: 1e-4,
}
maskout_near_cam_vox = False
pervoxel_lr = False
weight_distortion = 0.02
data = dict(
dataset_type='nerfstudio',
spherify=False,
recenter=False,
factor=8,
llffhold=-1,
dvgohold=8,
unbounded_inner_r=1.0,
white_bkgd=True,
rand_bkgd=True,
unbounded_inward=True,
load2gpu_on_the_fly=True,
bd_factor=None,
datadir='./data/nerfstudio_data/stump',
movie_render_kwargs=dict(
shift_x=0.0, # positive right
shift_y=-0.3, # negative down
shift_z=0,
scale_r=0.2,
pitch_deg=-40, # negative look downward
),
)
coarse_train = dict(N_iters=0)
fine_train = dict(
N_iters=10000, # zelin working
# N_iters=100000,
N_rand=2048,
lrate_decay=80,
ray_sampler='flatten',
weight_nearclip=1.0,
weight_distortion=weight_distortion,
# pg_scale=[2000,4000,6000,8000,10000,12000,14000,16000],
pg_scale=[1000, 2000, 3000, 4000, 5000, 6000, 7000], # zelin working
tv_before=20000,
tv_dense_before=20000,
weight_tv_density=1e-6,
weight_tv_k0=1e-7,
weight_main=1.0,
# weight_freq=0.1,
)
voxel_size_density = 200 # default 400
voxel_size_rgb = 200 # default 320
voxel_size_viewdir = -1
fine_model_and_render = dict(
num_voxels_density=voxel_size_density**3,
num_voxels_base_density=voxel_size_density**3,
num_voxels_rgb=voxel_size_rgb**3,
num_voxels_base_rgb=voxel_size_rgb**3,
num_voxels_viewdir=voxel_size_viewdir**3,
alpha_init=alpha_init,
stepsize=stepsize,
fast_color_thres={
'_delete_': True,
0 : alpha_init*stepsize/10,
1500: min(alpha_init, 1e-4)*stepsize/5,
2500: min(alpha_init, 1e-4)*stepsize/2,
3500: min(alpha_init, 1e-4)*stepsize/1.5,
4500: min(alpha_init, 1e-4)*stepsize,
5500: min(alpha_init, 1e-4),
6500: 1e-4,
},
world_bound_scale=1,
)
================================================
FILE: FourierGrid/configs/nerf_unbounded/bicycle.py
================================================
_base_ = './nerf_unbounded_default.py'
expname = 'dvgo_bicycle_unbounded'
data = dict(
datadir='./data/360_v2/bicycle',
factor=4, # 1237x822
movie_render_kwargs=dict(
shift_x=0.0, # positive right
shift_y=0, # negative down
shift_z=0,
scale_r=1.0,
pitch_deg=-10, # negative look downward
),
)
================================================
FILE: FourierGrid/configs/nerf_unbounded/bicycle_single.py
================================================
_base_ = './nerf_unbounded_default.py'
expname = 'bicycle_nov29_'
vis = dict(
height_rate = 0.6 # camera direction frustrum height
)
model='FourierGrid'
basedir = './logs/360'
alpha_init = 1e-4
stepsize = 0.5
_mpi_depth = 256
coarse_iter = 0
# fast_color_thres={ # default
# '_delete_': True, # to ignore the base config
# 0 : alpha_init*stepsize/10, # 0.5e-5
# 1500: min(alpha_init, 1e-4)*stepsize/5, # 1e-5
# 2500: min(alpha_init, 1e-4)*stepsize/2, # 2.5e-5
# 3500: min(alpha_init, 1e-4)*stepsize/1.5,
# 4500: min(alpha_init, 1e-4)*stepsize,
# 5500: min(alpha_init, 1e-4),
# 6500: 1e-4,
# }
fast_color_thres = 1e-4
maskout_near_cam_vox = False
pervoxel_lr = False
weight_distortion = 0.05
data = dict(
dataset_type='llff',
datadir='./data/360_v2/bicycle',
factor=16, # 1237x822
# width=320,
# height=240,
movie_render_kwargs=dict(
shift_x=0.0, # positive right
shift_y=0, # negative down
shift_z=0,
scale_r=1.0,
pitch_deg=-10, # negative look downward
),
)
coarse_train = dict(N_iters=0)
fine_train = dict(
N_iters=100000,
N_rand=2048,
lrate_decay=80,
ray_sampler='flatten',
weight_nearclip=1.0,
weight_distortion=weight_distortion,
# pg_scale=[2000,4000,6000,8000,10000,12000,14000,16000],
pg_scale=[1000, 2000, 3000, 4000, 5000, 6000, 7000],
tv_before=20000,
tv_dense_before=20000,
weight_tv_density=1e-6,
weight_tv_k0=1e-7,
weight_main=1.0,
weight_freq=5.0,
)
voxel_size_for_all = 200 # default 220
voxel_size_density = voxel_size_for_all
voxel_size_rgb = voxel_size_for_all
voxel_size_viewdir = -1
fine_model_and_render = dict(
num_voxels_density=voxel_size_density**3,
num_voxels_base_density=voxel_size_density**3,
num_voxels_rgb=voxel_size_rgb**3,
num_voxels_base_rgb=voxel_size_rgb**3,
num_voxels_viewdir=voxel_size_viewdir**3,
alpha_init=alpha_init,
stepsize=stepsize,
fast_color_thres=fast_color_thres,
world_bound_scale=1,
rgbnet_dim=12, # default
rgbnet_depth=3, # default
bbox_thres=-1,
maskout_near_cam_vox=False,
bg_len=0.2,
)
================================================
FILE: FourierGrid/configs/nerf_unbounded/bonsai.py
================================================
_base_ = './nerf_unbounded_default.py'
expname = 'dvgo_bonsai_unbounded'
data = dict(
datadir='./data/360_v2/bonsai',
factor=2, # 1559x1039
movie_render_kwargs=dict(
shift_x=0.0, # positive right
shift_y=0, # negative down
shift_z=0,
scale_r=1.0,
pitch_deg=-30, # negative look downward
),
)
================================================
FILE: FourierGrid/configs/nerf_unbounded/bonsai_single.py
================================================
_base_ = './nerf_unbounded_default.py'
expname = 'bonsai_nov29_'
vis = dict(
height_rate = 0.6 # camera direction frustrum height
)
model='FourierGrid'
basedir = './logs/360'
alpha_init = 1e-4
stepsize = 0.5
_mpi_depth = 256
coarse_iter = 0
fast_color_thres={ # default
'_delete_': True, # to ignore the base config
0 : alpha_init*stepsize/10, # 0.5e-5
1500: min(alpha_init, 1e-4)*stepsize/5, # 1e-5
2500: min(alpha_init, 1e-4)*stepsize/2, # 2.5e-5
3500: min(alpha_init, 1e-4)*stepsize/1.5,
4500: min(alpha_init, 1e-4)*stepsize,
5500: min(alpha_init, 1e-4),
6500: 1e-4,
}
maskout_near_cam_vox = False
pervoxel_lr = False
weight_distortion = 0.01
data = dict(
dataset_type='llff',
datadir='./data/360_v2/bonsai',
# factor=2, # 1559x1039
factor=8, # 1237x822
movie_render_kwargs=dict(
shift_x=0.0, # positive right
shift_y=0, # negative down
shift_z=0,
scale_r=1.0,
pitch_deg=-30, # negative look downward
),
)
coarse_train = dict(N_iters=0)
fine_train = dict(
N_iters=100000,
N_rand=2048,
lrate_decay=80,
ray_sampler='flatten',
weight_nearclip=1.0,
weight_distortion=weight_distortion,
pg_scale=[2000,4000,6000,8000,10000,12000,14000,16000],
tv_before=20000,
tv_dense_before=20000,
weight_tv_density=1e-6,
weight_tv_k0=1e-7,
weight_main=1.0,
# weight_freq=0.1,
)
voxel_size_density = 200 # default 400
voxel_size_rgb = 200 # default 320
voxel_size_viewdir = -1
fine_model_and_render = dict(
num_voxels_density=voxel_size_density**3,
num_voxels_base_density=voxel_size_density**3,
num_voxels_rgb=voxel_size_rgb**3,
num_voxels_base_rgb=voxel_size_rgb**3,
num_voxels_viewdir=voxel_size_vie
gitextract_b74j6bbn/ ├── .all-contributorsrc ├── .gitignore ├── FourierGrid/ │ ├── FourierGrid_ckpt_manager.py │ ├── FourierGrid_grid.py │ ├── FourierGrid_model.py │ ├── __init__.py │ ├── arf.py │ ├── bbox_compute.py │ ├── camera_utils.py │ ├── common_data_loaders/ │ │ ├── __init__.py │ │ ├── load_blendedmvs.py │ │ ├── load_blender.py │ │ ├── load_co3d.py │ │ ├── load_common_data.py │ │ ├── load_deepvoxels.py │ │ ├── load_free.py │ │ ├── load_llff.py │ │ ├── load_nerfpp.py │ │ ├── load_nsvf.py │ │ └── load_tankstemple.py │ ├── configs/ │ │ ├── blendedmvs/ │ │ │ ├── Character.py │ │ │ ├── Fountain.py │ │ │ ├── Jade.py │ │ │ └── Statues.py │ │ ├── co3d/ │ │ │ └── donut_369_40208_78816.py │ │ ├── custom/ │ │ │ ├── Madoka.py │ │ │ ├── Madoka_long.py │ │ │ ├── Otobai.py │ │ │ ├── default_forward_facing.py │ │ │ ├── default_ubd_inward_facing.py │ │ │ ├── sm01_desktop.py │ │ │ ├── sm02_multiple_desktop.py │ │ │ └── sm03_meeting.py │ │ ├── deepvoxels/ │ │ │ ├── armchair.py │ │ │ ├── cube.py │ │ │ ├── greek.py │ │ │ └── vase.py │ │ ├── default.py │ │ ├── free_dataset/ │ │ │ ├── grass.py │ │ │ └── lab.py │ │ ├── lf/ │ │ │ ├── africa.py │ │ │ ├── basket.py │ │ │ ├── lf_default.py │ │ │ ├── ship.py │ │ │ ├── statue.py │ │ │ └── torch.py │ │ ├── linemod/ │ │ │ ├── ape.py │ │ │ ├── benchvise.py │ │ │ ├── camera.py │ │ │ ├── can.py │ │ │ ├── cat.py │ │ │ ├── driller.py │ │ │ ├── duck.py │ │ │ ├── eggbox.py │ │ │ ├── glue.py │ │ │ ├── holepuncher.py │ │ │ ├── iron.py │ │ │ ├── lamp.py │ │ │ └── phone.py │ │ ├── llff/ │ │ │ ├── fern.py │ │ │ ├── fern_lg.py │ │ │ ├── flower.py │ │ │ ├── flower_lg.py │ │ │ ├── fortress.py │ │ │ ├── fortress_lg.py │ │ │ ├── horns.py │ │ │ ├── horns_lg.py │ │ │ ├── leaves.py │ │ │ ├── leaves_lg.py │ │ │ ├── llff_default.py │ │ │ ├── llff_default_lg.py │ │ │ ├── orchids.py │ │ │ ├── orchids_lg.py │ │ │ ├── room.py │ │ │ ├── room_lg.py │ │ │ ├── trex.py │ │ │ └── trex_lg.py │ │ ├── mega/ │ │ │ ├── building.py │ │ │ ├── building_no_block.py │ │ │ ├── quad.py │ │ │ └── rubble.py │ │ ├── nerf/ │ │ │ ├── chair.py │ │ │ ├── drums.py │ │ │ ├── ficus.py │ │ │ ├── hotdog.py │ │ │ ├── lego.py │ │ │ ├── materials.py │ │ │ ├── mic.py │ │ │ ├── ship.py │ │ │ └── ship.tensorf.py │ │ ├── nerf_studio/ │ │ │ ├── Giannini_Hall.py │ │ │ ├── nerf_studio_default.py │ │ │ └── stump.py │ │ ├── nerf_unbounded/ │ │ │ ├── bicycle.py │ │ │ ├── bicycle_single.py │ │ │ ├── bonsai.py │ │ │ ├── bonsai_single.py │ │ │ ├── counter.py │ │ │ ├── counter_single.py │ │ │ ├── garden.py │ │ │ ├── garden_single.py │ │ │ ├── kitchen.py │ │ │ ├── kitchen_single.py │ │ │ ├── nerf_unbounded_default.py │ │ │ ├── room.py │ │ │ ├── room_single.py │ │ │ ├── stump.py │ │ │ └── stump_single.py │ │ ├── nsvf/ │ │ │ ├── Bike.py │ │ │ ├── Lifestyle.py │ │ │ ├── Palace.py │ │ │ ├── Robot.py │ │ │ ├── Spaceship.py │ │ │ ├── Steamtrain.py │ │ │ ├── Toad.py │ │ │ └── Wineholder.py │ │ ├── tankstemple/ │ │ │ ├── Barn.py │ │ │ ├── Barn_lg.py │ │ │ ├── Caterpillar.py │ │ │ ├── Caterpillar_lg.py │ │ │ ├── Family.py │ │ │ ├── Family_lg.py │ │ │ ├── Ignatius.py │ │ │ ├── Ignatius_lg.py │ │ │ ├── Truck.py │ │ │ ├── Truck_lg.py │ │ │ ├── barn_single.py │ │ │ ├── caterpillar_single.py │ │ │ └── family_single.py │ │ ├── tankstemple_unbounded/ │ │ │ ├── M60.py │ │ │ ├── Playground.py │ │ │ ├── Train.py │ │ │ ├── Truck.py │ │ │ ├── m60_single.py │ │ │ ├── playground_single.py │ │ │ ├── train_single.py │ │ │ ├── truck_single.py │ │ │ └── tt_default.py │ │ └── waymo/ │ │ ├── block_0_llff.py │ │ ├── block_0_tt.py │ │ ├── waymo_base.py │ │ ├── waymo_block.py │ │ └── waymo_no_block.py │ ├── cuda/ │ │ ├── adam_upd.cpp │ │ ├── adam_upd_kernel.cu │ │ ├── render_utils.cpp │ │ ├── render_utils_kernel.cu │ │ ├── setup.py │ │ ├── total_variation.cpp │ │ ├── total_variation_kernel.cu │ │ ├── ub360_utils.cpp │ │ └── ub360_utils_kernel.cu │ ├── dcvgo.py │ ├── dmpigo.py │ ├── dvgo.py │ ├── grid.py │ ├── load_everything.py │ ├── load_mega.py │ ├── load_waymo.py │ ├── masked_adam.py │ ├── pose_utils/ │ │ ├── __init__.py │ │ ├── image_operators.py │ │ ├── linemod_constants.py │ │ ├── linemod_evaluator.py │ │ ├── model_operations.py │ │ ├── pose_operators.py │ │ ├── projection.py │ │ └── visualization.py │ ├── pycolmap/ │ │ ├── LICENSE.txt │ │ ├── README.md │ │ ├── __init__.py │ │ ├── pycolmap/ │ │ │ ├── __init__.py │ │ │ ├── camera.py │ │ │ ├── database.py │ │ │ ├── image.py │ │ │ ├── rotation.py │ │ │ └── scene_manager.py │ │ └── tools/ │ │ ├── colmap_to_nvm.py │ │ ├── delete_images.py │ │ ├── impute_missing_cameras.py │ │ ├── save_cameras_as_ply.py │ │ ├── transform_model.py │ │ ├── write_camera_track_to_bundler.py │ │ └── write_depthmap_to_ply.py │ ├── run_colmap2standard.py │ ├── run_export_bbox.py │ ├── run_export_coarse.py │ ├── run_gen_cam_paths.py │ ├── run_gtk_analysis.py │ ├── run_render.py │ ├── run_train.py │ ├── tools/ │ │ ├── __init__.py │ │ ├── colmap_utils/ │ │ │ ├── __init__.py │ │ │ ├── colmap_read_model.py │ │ │ ├── colmap_wrapper.py │ │ │ └── pose_utils.py │ │ ├── imgs2poses.py │ │ ├── vis_train.py │ │ └── vis_volume.py │ ├── trajectory_generators/ │ │ ├── __init__.py │ │ ├── interp_traj.py │ │ ├── mega_traj.py │ │ └── waymo_traj.py │ └── utils.py ├── LICENSE ├── README.md ├── block_nerf/ │ ├── __init__.py │ ├── block_nerf_lightning.py │ ├── block_nerf_model.py │ ├── block_visualize.py │ ├── learning_utils.py │ ├── metrics.py │ ├── rendering.py │ └── waymo_dataset.py ├── data_preprocess/ │ ├── __init__.py │ ├── comvog_organize_by_cam_pos.py │ ├── download_waymo.sh │ ├── fetch_data_from_tf_record.py │ ├── gdown_download_folder.py │ ├── get_one_block_meta_and_visualize.py │ ├── process_mega.sh │ ├── split_block.py │ ├── vis.py │ └── visualize_cameras.py ├── docs/ │ ├── classified_weekly_nerf/ │ │ ├── dynamic.md │ │ ├── editing.md │ │ ├── fast.md │ │ ├── generalization.md │ │ ├── human.md │ │ ├── lighting.md │ │ ├── others.md │ │ ├── pose-slam.md │ │ ├── reconstruction.md │ │ ├── semantic.md │ │ ├── texture.md │ │ └── video.md │ ├── classified_weekly_nerf_cn/ │ │ ├── dynamic.md │ │ ├── editing.md │ │ ├── fast.md │ │ ├── generalization.md │ │ ├── human.md │ │ ├── lighting.md │ │ ├── others.md │ │ ├── pose-slam.md │ │ ├── reconstruction.md │ │ ├── semantic.md │ │ ├── texture.md │ │ └── video.md │ ├── contribute_weekly_nerf.md │ ├── get_pytorch_waymo_dataset.md │ ├── parse_markdown.py │ ├── render_docs_from_csv.py │ ├── sample_logs/ │ │ ├── create_cluster_mask.txt │ │ └── merge_sub_modules.txt │ ├── weekly_nerf.md │ ├── weekly_nerf_cn.md │ └── weekly_nerf_meta_data.xlsx ├── eval_block_nerf.py ├── requirements.txt ├── run_FourierGrid.py ├── scripts/ │ ├── block_nerf_eval.sh │ ├── block_nerf_train.sh │ ├── create_cluster_mask.sh │ ├── download_nerfstudio.sh │ ├── gen_path_FourierGrid.sh │ ├── merge_sub_modules.sh │ ├── one_block_train.sh │ ├── render_FourierGrid.sh │ ├── sfm_FourierGrid.sh │ ├── train_FourierGrid.sh │ └── visualize_FourierGrid.sh ├── setup.py └── train_block_nerf.py
SYMBOL INDEX (613 symbols across 81 files)
FILE: FourierGrid/FourierGrid_ckpt_manager.py
class FourierGridCheckpointManager (line 11) | class FourierGridCheckpointManager:
method __init__ (line 12) | def __init__(self, args, cfg) -> None:
method load_all_info (line 17) | def load_all_info(self, model, optimizer, ckpt_path, no_reload_optimiz...
method load_existing_model (line 25) | def load_existing_model(self, args, cfg, cfg_train, reload_ckpt_path, ...
method save_model (line 44) | def save_model(self, global_step, model, optimizer, save_path):
method load_model (line 53) | def load_model(self, model_class, ckpt_path):
method merge_blocks (line 61) | def merge_blocks(self, args, cfg, device):
FILE: FourierGrid/FourierGrid_grid.py
function create_grid (line 14) | def create_grid(type, **kwargs):
class NeRFPosEmbedding (line 21) | class NeRFPosEmbedding(nn.Module):
method __init__ (line 22) | def __init__(self, num_freqs: int, logscale=True):
method forward (line 32) | def forward(self, x: torch.Tensor) -> torch.Tensor:
class FourierGrid (line 42) | class FourierGrid(nn.Module):
method __init__ (line 43) | def __init__(self, channels, world_size, xyz_min, xyz_max, use_nerf_po...
method forward (line 60) | def forward(self, xyz):
method scale_volume_grid (line 80) | def scale_volume_grid(self, new_world_size):
method total_variation_add_grad (line 87) | def total_variation_add_grad(self, wx, wy, wz, dense_mode):
method get_dense_grid (line 92) | def get_dense_grid(self):
method __isub__ (line 96) | def __isub__(self, val):
method extra_repr (line 100) | def extra_repr(self):
function compute_tensorf_feat (line 104) | def compute_tensorf_feat(xy_plane, xz_plane, yz_plane, x_vec, y_vec, z_v...
function compute_tensorf_val (line 122) | def compute_tensorf_val(xy_plane, xz_plane, yz_plane, x_vec, y_vec, z_ve...
class MaskGrid (line 138) | class MaskGrid(nn.Module):
method __init__ (line 139) | def __init__(self, path=None, mask_cache_thres=None, mask=None, xyz_mi...
method forward (line 160) | def forward(self, xyz):
method extra_repr (line 170) | def extra_repr(self):
FILE: FourierGrid/FourierGrid_model.py
function get_rays (line 21) | def get_rays(H, W, K, c2w, inverse_y, flip_x, flip_y, mode='center'):
function ndc_rays (line 52) | def ndc_rays(H, W, focal, near, rays_o, rays_d):
function get_rays_of_a_view (line 72) | def get_rays_of_a_view(H, W, K, c2w, ndc, inverse_y, flip_x, flip_y, mod...
class NeRFPosEmbedding (line 80) | class NeRFPosEmbedding(nn.Module):
method __init__ (line 81) | def __init__(self, num_freqs: int, logscale=True):
method forward (line 92) | def forward(self, x: torch.Tensor) -> torch.Tensor:
class FourierMSELoss (line 100) | class FourierMSELoss(nn.Module):
method __init__ (line 101) | def __init__(self,):
method forward (line 104) | def forward(self, pred, gt):
method __init__ (line 115) | def __init__(self, num_freqs=7, logscale=True):
method forward (line 119) | def forward(self, pred, gt):
class FourierMSELoss (line 114) | class FourierMSELoss(nn.Module):
method __init__ (line 101) | def __init__(self,):
method forward (line 104) | def forward(self, pred, gt):
method __init__ (line 115) | def __init__(self, num_freqs=7, logscale=True):
method forward (line 119) | def forward(self, pred, gt):
class FourierGridModel (line 134) | class FourierGridModel(nn.Module):
method __init__ (line 135) | def __init__(self, xyz_min, xyz_max, num_voxels_density=0, num_voxels_...
method FourierGrid_get_training_rays (line 264) | def FourierGrid_get_training_rays(self, rgb_tr_ori, train_poses, HW, K...
method gather_training_rays (line 297) | def gather_training_rays(self, data_dict, images, cfg, i_train, cfg_tr...
method _set_grid_resolution (line 335) | def _set_grid_resolution(self, num_voxels_density, num_voxels_rgb):
method get_kwargs (line 351) | def get_kwargs(self):
method maskout_near_cam_vox (line 376) | def maskout_near_cam_vox(self, cam_o, near_clip):
method voxel_count_views (line 390) | def voxel_count_views(self, rays_o_tr, rays_d_tr, imsz, near, far, ste...
method scale_volume_grid (line 423) | def scale_volume_grid(self, num_voxels_density, num_voxels_rgb):
method update_occupancy_cache (line 443) | def update_occupancy_cache(self):
method update_occupancy_cache_lt_nviews (line 458) | def update_occupancy_cache_lt_nviews(self, rays_o_tr, rays_d_tr, imsz,...
method density_total_variation_add_grad (line 482) | def density_total_variation_add_grad(self, weight, dense_mode):
method k0_total_variation_add_grad (line 486) | def k0_total_variation_add_grad(self, weight, dense_mode):
method activate_density (line 490) | def activate_density(self, density, interval=None):
method hit_coarse_geo (line 495) | def hit_coarse_geo(self, rays_o, rays_d, near, far, stepsize, **render...
method sample_ray (line 509) | def sample_ray(self, ori_rays_o, ori_rays_d, stepsize, is_train=False,...
method forward (line 554) | def forward(self, rays_o, rays_d, viewdirs, global_step=None, is_train...
method export_geometry_for_visualize (line 674) | def export_geometry_for_visualize(self, save_path):
class DistortionLoss (line 684) | class DistortionLoss(torch.autograd.Function):
method forward (line 686) | def forward(ctx, w, s, n_max, ray_id):
method backward (line 698) | def backward(ctx, grad_back):
FILE: FourierGrid/arf.py
class ARF (line 9) | class ARF:
method __init__ (line 10) | def __init__(self, cfg, data_dict, device):
method load_style_img (line 25) | def load_style_img(self, style_img_p):
method match_colors_for_image_set (line 51) | def match_colors_for_image_set(self, image_set, train_save_dir): # co...
FILE: FourierGrid/bbox_compute.py
function _compute_bbox_by_cam_frustrm_unbounded (line 10) | def _compute_bbox_by_cam_frustrm_unbounded(cfg, HW, Ks, poses, i_train, ...
function FourierGrid_compute_bbox_by_cam_frustrm_nerfpp (line 29) | def FourierGrid_compute_bbox_by_cam_frustrm_nerfpp(cfg, HW, Ks, poses, i...
function FourierGrid_compute_bbox_by_cam_frustrm_waymo (line 48) | def FourierGrid_compute_bbox_by_cam_frustrm_waymo(cfg, HW, Ks, poses, i_...
function FourierGrid_compute_bbox_by_cam_frustrm_mega (line 75) | def FourierGrid_compute_bbox_by_cam_frustrm_mega(cfg, HW, Ks, poses, i_t...
function _compute_bbox_by_cam_frustrm_bounded (line 96) | def _compute_bbox_by_cam_frustrm_bounded(cfg, HW, Ks, poses, i_train, ne...
function compute_bbox_by_cam_frustrm (line 113) | def compute_bbox_by_cam_frustrm(args, cfg, HW, Ks, poses, i_train, near,...
function compute_bbox_by_coarse_geo (line 137) | def compute_bbox_by_coarse_geo(model_class, model_path, thres, device, a...
FILE: FourierGrid/camera_utils.py
class Rays (line 10) | class Rays:
function split_rays (line 19) | def split_rays(rays, batch_size):
function intrinsic_matrix (line 32) | def intrinsic_matrix(fx: float,
class ProjectionType (line 44) | class ProjectionType(enum.Enum):
function convert_to_ndc (line 50) | def convert_to_ndc(origins: Tensor,
function pixels_to_rays (line 118) | def pixels_to_rays(
function _compute_residual_and_jacobian (line 232) | def _compute_residual_and_jacobian(
function _radial_and_tangential_undistort (line 282) | def _radial_and_tangential_undistort(
FILE: FourierGrid/common_data_loaders/load_blendedmvs.py
function load_blendedmvs_data (line 11) | def load_blendedmvs_data(basedir):
FILE: FourierGrid/common_data_loaders/load_blender.py
function pose_spherical (line 30) | def pose_spherical(theta, phi, radius):
function load_blender_data (line 38) | def load_blender_data(basedir, half_res=False, testskip=1):
FILE: FourierGrid/common_data_loaders/load_co3d.py
function load_co3d_data (line 12) | def load_co3d_data(cfg):
FILE: FourierGrid/common_data_loaders/load_common_data.py
function load_common_data (line 14) | def load_common_data(args):
function inward_nearfar_heuristic (line 248) | def inward_nearfar_heuristic(cam_o, ratio=0.05):
FILE: FourierGrid/common_data_loaders/load_deepvoxels.py
function load_dv_data (line 6) | def load_dv_data(scene='cube', basedir='/data/deepvoxels', testskip=1):
FILE: FourierGrid/common_data_loaders/load_free.py
function imread (line 14) | def imread(f):
function depthread (line 20) | def depthread(path):
function _minify (line 38) | def _minify(basedir, factors=[], resolutions=[]):
function normalize_scene (line 95) | def normalize_scene(poses, n_images, bounds):
function load_images_from_disk (line 107) | def load_images_from_disk(basedir, factor, height, width):
function normalize (line 146) | def normalize(x):
function ptstocam (line 149) | def ptstocam(pts, c2w):
function viewmatrix (line 153) | def viewmatrix(z, up, pos):
function render_path_spiral (line 162) | def render_path_spiral(c2w, up, rads, focal, zdelta, zrate, rots, N):
function poses_avg (line 174) | def poses_avg(poses):
function recenter_poses (line 182) | def recenter_poses(poses, render_poses):
function rerotate_poses (line 205) | def rerotate_poses(poses):
function spherify_poses (line 233) | def spherify_poses(poses, bds, depths):
function load_free_data (line 272) | def load_free_data(args, basedir, factor=8, width=None, height=None,
FILE: FourierGrid/common_data_loaders/load_llff.py
function imread (line 12) | def imread(f):
function depthread (line 18) | def depthread(path):
function _minify (line 36) | def _minify(basedir, factors=[], resolutions=[]):
function _load_data (line 93) | def _load_data(basedir, factor=None, width=None, height=None, load_imgs=...
function normalize (line 182) | def normalize(x):
function viewmatrix (line 185) | def viewmatrix(z, up, pos):
function ptstocam (line 193) | def ptstocam(pts, c2w):
function poses_avg (line 197) | def poses_avg(poses):
function render_path_spiral (line 207) | def render_path_spiral(c2w, up, rads, focal, zdelta, zrate, rots, N):
function recenter_poses (line 219) | def recenter_poses(poses):
function rerotate_poses (line 233) | def rerotate_poses(poses):
function spherify_poses (line 261) | def spherify_poses(poses, bds, depths):
function load_llff_data (line 300) | def load_llff_data(basedir, factor=8, width=None, height=None,
FILE: FourierGrid/common_data_loaders/load_nerfpp.py
function find_files (line 17) | def find_files(dir, exts):
function load_data_split (line 29) | def load_data_split(split_dir, skip=1, try_load_min_depth=True, only_img...
function rerotate_poses (line 89) | def rerotate_poses(poses, render_poses):
function load_nerfpp_data (line 120) | def load_nerfpp_data(basedir, rerotate=True, training_ids=None):
FILE: FourierGrid/common_data_loaders/load_nsvf.py
function pose_spherical (line 30) | def pose_spherical(theta, phi, radius):
function load_nsvf_data (line 39) | def load_nsvf_data(basedir):
FILE: FourierGrid/common_data_loaders/load_tankstemple.py
function normalize (line 11) | def normalize(x):
function load_tankstemple_data (line 14) | def load_tankstemple_data(basedir, movie_render_kwargs={}):
FILE: FourierGrid/cuda/adam_upd.cpp
function adam_upd (line 36) | void adam_upd(
function masked_adam_upd (line 50) | void masked_adam_upd(
function adam_upd_with_perlr (line 64) | void adam_upd_with_perlr(
function PYBIND11_MODULE (line 79) | PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
FILE: FourierGrid/cuda/render_utils.cpp
function infer_t_minmax (line 50) | std::vector<torch::Tensor> infer_t_minmax(
function infer_n_samples (line 60) | torch::Tensor infer_n_samples(torch::Tensor rays_d, torch::Tensor t_min,...
function infer_ray_start_dir (line 67) | std::vector<torch::Tensor> infer_ray_start_dir(torch::Tensor rays_o, tor...
function sample_pts_on_rays (line 74) | std::vector<torch::Tensor> sample_pts_on_rays(
function sample_ndc_pts_on_rays (line 87) | std::vector<torch::Tensor> sample_ndc_pts_on_rays(
function sample_bg_pts_on_rays (line 100) | torch::Tensor sample_bg_pts_on_rays(
function maskcache_lookup (line 109) | torch::Tensor maskcache_lookup(torch::Tensor world, torch::Tensor xyz, t...
function raw2alpha (line 120) | std::vector<torch::Tensor> raw2alpha(torch::Tensor density, const float ...
function raw2alpha_nonuni (line 125) | std::vector<torch::Tensor> raw2alpha_nonuni(torch::Tensor density, const...
function raw2alpha_backward (line 131) | torch::Tensor raw2alpha_backward(torch::Tensor exp, torch::Tensor grad_b...
function raw2alpha_nonuni_backward (line 136) | torch::Tensor raw2alpha_nonuni_backward(torch::Tensor exp, torch::Tensor...
function alpha2weight (line 142) | std::vector<torch::Tensor> alpha2weight(torch::Tensor alpha, torch::Tens...
function alpha2weight_backward (line 151) | torch::Tensor alpha2weight_backward(
function PYBIND11_MODULE (line 170) | PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
FILE: FourierGrid/cuda/setup.py
function setup_sources (line 7) | def setup_sources(name, sources):
FILE: FourierGrid/cuda/total_variation.cpp
function total_variation_add_grad (line 16) | void total_variation_add_grad(torch::Tensor param, torch::Tensor grad, f...
function PYBIND11_MODULE (line 22) | PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
FILE: FourierGrid/cuda/ub360_utils.cpp
function cumdist_thres (line 15) | torch::Tensor cumdist_thres(torch::Tensor dist, float thres) {
function PYBIND11_MODULE (line 20) | PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
FILE: FourierGrid/dcvgo.py
class DirectContractedVoxGO (line 28) | class DirectContractedVoxGO(nn.Module):
method __init__ (line 29) | def __init__(self, xyz_min, xyz_max,
method _set_grid_resolution (line 127) | def _set_grid_resolution(self, num_voxels):
method get_kwargs (line 139) | def get_kwargs(self):
method scale_volume_grid (line 158) | def scale_volume_grid(self, num_voxels):
method update_occupancy_cache (line 181) | def update_occupancy_cache(self):
method update_occupancy_cache_lt_nviews (line 195) | def update_occupancy_cache_lt_nviews(self, rays_o_tr, rays_d_tr, imsz,...
method density_total_variation_add_grad (line 215) | def density_total_variation_add_grad(self, weight, dense_mode):
method k0_total_variation_add_grad (line 219) | def k0_total_variation_add_grad(self, weight, dense_mode):
method activate_density (line 223) | def activate_density(self, density, interval=None):
method sample_ray (line 228) | def sample_ray(self, ori_rays_o, ori_rays_d, stepsize, is_train=False,...
method forward (line 264) | def forward(self, rays_o, rays_d, viewdirs, global_step=None, is_train...
class DistortionLoss (line 387) | class DistortionLoss(torch.autograd.Function):
method forward (line 389) | def forward(ctx, w, s, n_max, ray_id):
method backward (line 401) | def backward(ctx, grad_back):
FILE: FourierGrid/dmpigo.py
class DirectMPIGO (line 18) | class DirectMPIGO(torch.nn.Module):
method __init__ (line 19) | def __init__(self, xyz_min, xyz_max,
method _set_grid_resolution (line 120) | def _set_grid_resolution(self, num_voxels, mpi_depth):
method get_kwargs (line 132) | def get_kwargs(self):
method scale_volume_grid (line 151) | def scale_volume_grid(self, num_voxels, mpi_depth):
method update_occupancy_cache (line 175) | def update_occupancy_cache(self):
method update_occupancy_cache_lt_nviews (line 189) | def update_occupancy_cache_lt_nviews(self, rays_o_tr, rays_d_tr, imsz,...
method density_total_variation_add_grad (line 209) | def density_total_variation_add_grad(self, weight, dense_mode):
method k0_total_variation_add_grad (line 214) | def k0_total_variation_add_grad(self, weight, dense_mode):
method activate_density (line 219) | def activate_density(self, density, interval=None):
method sample_ray (line 224) | def sample_ray(self, rays_o, rays_d, near, far, stepsize, **render_kwa...
method forward (line 251) | def forward(self, rays_o, rays_d, viewdirs, global_step=None, **render...
function create_full_step_id (line 344) | def create_full_step_id(shape):
FILE: FourierGrid/dvgo.py
class DirectVoxGO (line 26) | class DirectVoxGO(torch.nn.Module):
method __init__ (line 27) | def __init__(self, xyz_min, xyz_max,
method _set_grid_resolution (line 154) | def _set_grid_resolution(self, num_voxels):
method get_kwargs (line 165) | def get_kwargs(self):
method maskout_near_cam_vox (line 185) | def maskout_near_cam_vox(self, cam_o, near_clip):
method scale_volume_grid (line 213) | def scale_volume_grid(self, num_voxels):
method update_occupancy_cache (line 236) | def update_occupancy_cache(self):
method voxel_count_views (line 247) | def voxel_count_views(self, rays_o_tr, rays_d_tr, imsz, near, far, ste...
method density_total_variation_add_grad (line 279) | def density_total_variation_add_grad(self, weight, dense_mode):
method k0_total_variation_add_grad (line 283) | def k0_total_variation_add_grad(self, weight, dense_mode):
method activate_density (line 287) | def activate_density(self, density, interval=None):
method hit_coarse_geo (line 292) | def hit_coarse_geo(self, rays_o, rays_d, near, far, stepsize, **render...
method sample_ray (line 306) | def sample_ray(self, rays_o, rays_d, near, far, stepsize, **render_kwa...
method forward (line 330) | def forward(self, rays_o, rays_d, viewdirs, global_step=None, **render...
class Raw2Alpha (line 430) | class Raw2Alpha(torch.autograd.Function):
method forward (line 432) | def forward(ctx, density, shift, interval):
method backward (line 447) | def backward(ctx, grad_back):
class Raw2Alpha_nonuni (line 456) | class Raw2Alpha_nonuni(torch.autograd.Function):
method forward (line 458) | def forward(ctx, density, shift, interval):
method backward (line 467) | def backward(ctx, grad_back):
class Alphas2Weights (line 472) | class Alphas2Weights(torch.autograd.Function):
method forward (line 474) | def forward(ctx, alpha, ray_id, N):
method backward (line 483) | def backward(ctx, grad_weights, grad_last):
function get_rays (line 493) | def get_rays(H, W, K, c2w, inverse_y, flip_x, flip_y, mode='center'):
function get_rays_np (line 524) | def get_rays_np(H, W, K, c2w):
function ndc_rays (line 534) | def ndc_rays(H, W, focal, near, rays_o, rays_d):
function get_rays_of_a_view (line 554) | def get_rays_of_a_view(H, W, K, c2w, ndc, inverse_y, flip_x, flip_y, mod...
function get_training_rays (line 563) | def get_training_rays(rgb_tr, train_poses, HW, Ks, ndc, inverse_y, flip_...
function get_training_rays_flatten (line 588) | def get_training_rays_flatten(rgb_tr_ori, train_poses, HW, Ks, ndc, inve...
function get_training_rays_in_maskcache_sampling (line 620) | def get_training_rays_in_maskcache_sampling(rgb_tr_ori, train_poses, HW,...
function batch_indices_generator (line 660) | def batch_indices_generator(N, BS):
FILE: FourierGrid/grid.py
function create_grid (line 30) | def create_grid(type, **kwargs):
class DenseGrid (line 41) | class DenseGrid(nn.Module):
method __init__ (line 42) | def __init__(self, channels, world_size, xyz_min, xyz_max, **kwargs):
method forward (line 50) | def forward(self, xyz):
method scale_volume_grid (line 63) | def scale_volume_grid(self, new_world_size):
method total_variation_add_grad (line 70) | def total_variation_add_grad(self, wx, wy, wz, dense_mode):
method get_dense_grid (line 75) | def get_dense_grid(self):
method __isub__ (line 79) | def __isub__(self, val):
method extra_repr (line 83) | def extra_repr(self):
class TensoRFGrid (line 90) | class TensoRFGrid(nn.Module):
method __init__ (line 91) | def __init__(self, channels, world_size, xyz_min, xyz_max, config):
method forward (line 111) | def forward(self, xyz):
method scale_volume_grid (line 131) | def scale_volume_grid(self, new_world_size):
method total_variation_add_grad (line 142) | def total_variation_add_grad(self, wx, wy, wz, dense_mode):
method get_dense_grid (line 156) | def get_dense_grid(self):
method extra_repr (line 171) | def extra_repr(self):
function compute_tensorf_feat (line 174) | def compute_tensorf_feat(xy_plane, xz_plane, yz_plane, x_vec, y_vec, z_v...
function compute_tensorf_val (line 191) | def compute_tensorf_val(xy_plane, xz_plane, yz_plane, x_vec, y_vec, z_ve...
class MaskGrid (line 207) | class MaskGrid(nn.Module):
method __init__ (line 208) | def __init__(self, path=None, mask_cache_thres=None, mask=None, xyz_mi...
method forward (line 231) | def forward(self, xyz):
method extra_repr (line 241) | def extra_repr(self):
FILE: FourierGrid/load_everything.py
function load_everything (line 9) | def load_everything(args, cfg):
function load_existing_model (line 41) | def load_existing_model(args, cfg, cfg_train, reload_ckpt_path, device):
FILE: FourierGrid/load_mega.py
function find_files (line 24) | def find_files(dir, exts):
function waymo_load_img_list (line 36) | def waymo_load_img_list(split_dir, skip=1):
function sample_list_by_idx (line 47) | def sample_list_by_idx(one_list, idxs):
function sample_metadata_by_cam (line 52) | def sample_metadata_by_cam(metadata, cam_idx):
function find_most_freq_ele (line 63) | def find_most_freq_ele(one_list):
function sample_metadata_by_shape (line 69) | def sample_metadata_by_shape(metadata):
function sample_metadata_by_idxs (line 82) | def sample_metadata_by_idxs(metadata, sample_idxs):
function sort_metadata_by_pos (line 91) | def sort_metadata_by_pos(metadata):
function normalize (line 109) | def normalize(x):
function viewmatrix (line 113) | def viewmatrix(z, up, pos):
function poses_avg (line 122) | def poses_avg(poses):
function sample_metadata_by_training_ids (line 131) | def sample_metadata_by_training_ids(metadata, training_ids, assign_pos, ...
function load_mega (line 163) | def load_mega(args, cfg, ):
function inward_nearfar_heuristic (line 248) | def inward_nearfar_heuristic(cam_o, ratio=0.05):
function load_mega_data (line 257) | def load_mega_data(args, cfg):
FILE: FourierGrid/load_waymo.py
function find_files (line 25) | def find_files(dir, exts):
function waymo_load_img_list (line 37) | def waymo_load_img_list(split_dir, skip=1):
function rerotate_poses (line 47) | def rerotate_poses(poses, render_poses):
function sample_list_by_idx (line 78) | def sample_list_by_idx(one_list, idxs):
function sample_metadata_by_cam (line 83) | def sample_metadata_by_cam(metadata, cam_idx):
function sample_metadata_by_idxs (line 94) | def sample_metadata_by_idxs(metadata, sample_idxs, val_num=5):
function sample_metadata_by_training_ids (line 110) | def sample_metadata_by_training_ids(metadata, training_ids, assign_pos, ...
function sort_metadata_by_pos (line 142) | def sort_metadata_by_pos(metadata):
function normalize (line 153) | def normalize(x):
function viewmatrix (line 157) | def viewmatrix(z, up, pos):
function ptstocam (line 166) | def ptstocam(pts, c2w):
function poses_avg (line 171) | def poses_avg(poses):
function recenter_poses (line 180) | def recenter_poses(poses):
function find_most_freq_ele (line 194) | def find_most_freq_ele(one_list):
function save_training_imgs_to_disk (line 200) | def save_training_imgs_to_disk(args, cfg, metadata):
function resize_img (line 213) | def resize_img(train_HW, val_HW, imgs, tr_K, val_K):
function find_rotations_from_meta (line 254) | def find_rotations_from_meta(metadata):
function load_waymo (line 263) | def load_waymo(args, cfg, ):
function inward_nearfar_heuristic (line 371) | def inward_nearfar_heuristic(cam_o, ratio=0.05):
function load_waymo_data (line 380) | def load_waymo_data(args, cfg):
FILE: FourierGrid/masked_adam.py
class MaskedAdam (line 21) | class MaskedAdam(torch.optim.Optimizer):
method __init__ (line 23) | def __init__(self, params, lr=1e-3, betas=(0.9, 0.99), eps=1e-8):
method __setstate__ (line 36) | def __setstate__(self, state):
method set_pervoxel_lr (line 39) | def set_pervoxel_lr(self, count):
method step (line 44) | def step(self):
FILE: FourierGrid/pose_utils/image_operators.py
function get_bbox_from_img (line 7) | def get_bbox_from_img(image, color_thre=1e-2):
function change_background_from_black_to_white (line 19) | def change_background_from_black_to_white(image, color_thresh=1e-2):
function get_bbox_from_mask (line 25) | def get_bbox_from_mask(label_img):
function apply_mask_on_img (line 33) | def apply_mask_on_img(one_img, label_img):
function image_normalization_for_pose (line 41) | def image_normalization_for_pose(image):
FILE: FourierGrid/pose_utils/linemod_evaluator.py
function rotation_angle_euler (line 9) | def rotation_angle_euler(R1, R2):
function project (line 26) | def project(xyz, K, RT):
class LineMODEvaluator (line 38) | class LineMODEvaluator:
method __init__ (line 39) | def __init__(self, class_name, obj_m, icp_refine=False):
method projection_2d (line 71) | def projection_2d(self, pose_pred, pose_targets, K, icp=False, thresho...
method projection_2d_sym (line 81) | def projection_2d_sym(self, pose_pred, pose_targets, K, threshold=5):
method add2_metric (line 89) | def add2_metric(self, pose_pred, pose_targets, icp=False, syn=False, p...
method add5_metric (line 110) | def add5_metric(self, pose_pred, pose_targets, icp=False, syn=False, p...
method add_metric (line 130) | def add_metric(self, pose_pred, pose_targets, icp=False, syn=False, pe...
method cm_degree_5_metric (line 162) | def cm_degree_5_metric(self, pose_pred, pose_targets, icp=False):
method mask_iou (line 175) | def mask_iou(self, output, batch):
method icp_refine (line 182) | def icp_refine(self, pose_pred, anno, output, K):
method icp_refine_ (line 200) | def icp_refine_(self, pose, anno, output):
method summarize (line 241) | def summarize(self):
method evaluate_proposals (line 283) | def evaluate_proposals(self, pose_gt, pose_proposals, cam_k):
method evaluate_linemod (line 303) | def evaluate_linemod(self, pose_gt, pose_pred, cam_k): # sample_corres...
FILE: FourierGrid/pose_utils/model_operations.py
function get_bb8_of_model (line 5) | def get_bb8_of_model(obj_m):
FILE: FourierGrid/pose_utils/pose_operators.py
function chordal_distance (line 6) | def chordal_distance(R1,R2):
function rotation_angle_chordal (line 10) | def rotation_angle_chordal(R1, R2):
function cal_pose_rot_diff (line 14) | def cal_pose_rot_diff(pose1, pose2):
function rot_diff_to_norm_angle (line 19) | def rot_diff_to_norm_angle(rotation_difference):
function cal_one_add (line 27) | def cal_one_add(model_points, pose_pred, pose_targets, syn=False):
function pose_rot_interpolation (line 42) | def pose_rot_interpolation(pose_a, pose_b, inter_num=100):
function pose_to_blender (line 67) | def pose_to_blender(pose):
FILE: FourierGrid/pose_utils/projection.py
function get_projected_points (line 6) | def get_projected_points(cam_pose, cam_k, obj_m, one_img=None, save_root...
FILE: FourierGrid/pose_utils/visualization.py
function visualize_2d_points (line 9) | def visualize_2d_points(points_2d, bg_image, save_root=None, pre_str="",...
function get_projected_points (line 29) | def get_projected_points(cam_pose, cam_k, obj_m, one_img=None, save_root...
function draw_bbox_8_2D (line 34) | def draw_bbox_8_2D(draw_img, bbox_8_2D, color = (0, 255, 0), thickness =...
function visualize_pose_prediction (line 67) | def visualize_pose_prediction(pose_a, pose_b, cam_k, obj_bb8, bg_img, sa...
FILE: FourierGrid/pycolmap/pycolmap/camera.py
function simple_radial_distortion (line 14) | def simple_radial_distortion(camera, x):
function radial_distortion (line 17) | def radial_distortion(camera, x):
function opencv_distortion (line 21) | def opencv_distortion(camera, x):
class Camera (line 38) | class Camera:
method GetNumParams (line 40) | def GetNumParams(type_):
method GetNameFromType (line 71) | def GetNameFromType(type_):
method __init__ (line 89) | def __init__(self, type_, width_, height_, params):
method __str__ (line 128) | def __str__(self):
method get_params (line 152) | def get_params(self):
method get_camera_matrix (line 168) | def get_camera_matrix(self):
method get_inverse_camera_matrix (line 172) | def get_inverse_camera_matrix(self):
method K (line 179) | def K(self):
method K_inv (line 183) | def K_inv(self):
method get_inv_camera_matrix (line 189) | def get_inv_camera_matrix(self):
method get_image_grid (line 199) | def get_image_grid(self):
method distort_points (line 213) | def distort_points(self, x, normalized=True, denormalize=True):
method undistort_points (line 237) | def undistort_points(self, x, normalized=False, denormalize=True):
FILE: FourierGrid/pycolmap/pycolmap/database.py
function array_to_blob (line 9) | def array_to_blob(arr):
function blob_to_array (line 12) | def blob_to_array(blob, dtype, shape=(-1,)):
function get_pair_id (line 21) | def get_pair_id(image_id1, image_id2):
function get_image_ids_from_pair_id (line 27) | def get_image_ids_from_pair_id(pair_id):
function add_camera (line 98) | def add_camera(db, model, width, height, params, prior_focal_length=False,
function add_descriptors (line 107) | def add_descriptors(db, image_id, descriptors):
function add_image (line 113) | def add_image(db, name, camera_id, prior_q=np.zeros(4), prior_t=np.zeros...
function add_inlier_matches (line 121) | def add_inlier_matches(db, image_id1, image_id2, matches, config=2, F=None,
function add_keypoints (line 142) | def add_keypoints(db, image_id, keypoints):
function add_matches (line 152) | def add_matches(db, image_id1, image_id2, matches):
class COLMAPDatabase (line 168) | class COLMAPDatabase(sqlite3.Connection):
method connect (line 170) | def connect(database_path):
method __init__ (line 174) | def __init__(self, *args, **kwargs):
function main (line 205) | def main(args):
FILE: FourierGrid/pycolmap/pycolmap/image.py
class Image (line 11) | class Image:
method __init__ (line 12) | def __init__(self, name_, camera_id_, q_, tvec_):
method R (line 23) | def R(self):
method C (line 28) | def C(self):
method t (line 34) | def t(self):
FILE: FourierGrid/pycolmap/pycolmap/rotation.py
function cross_prod_matrix (line 12) | def cross_prod_matrix(v):
function axis_angle_to_rotation_matrix (line 20) | def axis_angle_to_rotation_matrix(axis, angle=None):
function rotation_matrix_to_axis_angle (line 34) | def rotation_matrix_to_axis_angle(R):
class Quaternion (line 43) | class Quaternion:
method FromR (line 47) | def FromR(R):
method FromAxisAngle (line 79) | def FromAxisAngle(axis, angle=None):
method __init__ (line 92) | def __init__(self, q=np.array((1., 0., 0., 0.))):
method __add__ (line 105) | def __add__(self, other):
method __iadd__ (line 108) | def __iadd__(self, other):
method __invert__ (line 113) | def __invert__(self):
method __mul__ (line 119) | def __mul__(self, other):
method __rmul__ (line 133) | def __rmul__(self, other):
method __imul__ (line 136) | def __imul__(self, other):
method __irmul__ (line 140) | def __irmul__(self, other):
method __neg__ (line 144) | def __neg__(self):
method __sub__ (line 147) | def __sub__(self, other):
method __isub__ (line 150) | def __isub__(self, other):
method __str__ (line 154) | def __str__(self):
method copy (line 157) | def copy(self):
method dot (line 160) | def dot(self, other):
method inverse (line 164) | def inverse(self):
method norm (line 167) | def norm(self):
method normalize (line 170) | def normalize(self):
method rotate_points (line 175) | def rotate_points(self, x):
method ToR (line 180) | def ToR(self):
method ToAxisAngle (line 193) | def ToAxisAngle(self):
method ToEulerAngles (line 220) | def ToEulerAngles(self):
class DualQuaternion (line 243) | class DualQuaternion:
method FromQT (line 246) | def FromQT(q, t):
method __init__ (line 249) | def __init__(self, q0=np.array((1., 0., 0., 0.)), qe=np.zeros(4)):
method __add__ (line 252) | def __add__(self, other):
method __iadd__ (line 255) | def __iadd__(self, other):
method __invert__ (line 261) | def __invert__(self):
method __mul__ (line 264) | def __mul__(self, other):
method __rmul__ (line 276) | def __rmul__(self, other):
method __imul__ (line 279) | def __imul__(self, other):
method __neg__ (line 284) | def __neg__(self):
method __sub__ (line 287) | def __sub__(self, other):
method __isub__ (line 290) | def __isub__(self, other):
method inverse (line 297) | def inverse(self):
method norm (line 305) | def norm(self):
method normalize (line 310) | def normalize(self):
method getT (line 320) | def getT(self):
method ToQT (line 323) | def ToQT(self):
FILE: FourierGrid/pycolmap/pycolmap/scene_manager.py
class SceneManager (line 21) | class SceneManager:
method __init__ (line 24) | def __init__(self, colmap_results_folder, image_path=None):
method load_colmap_project_file (line 56) | def load_colmap_project_file(self, project_file=None, image_path=None):
method load (line 79) | def load(self):
method load_cameras (line 86) | def load_cameras(self, input_file=None):
method _load_cameras_bin (line 98) | def _load_cameras_bin(self, input_file):
method _load_cameras_txt (line 111) | def _load_cameras_txt(self, input_file):
method load_images (line 127) | def load_images(self, input_file=None):
method _load_images_bin (line 139) | def _load_images_bin(self, input_file):
method _load_images_txt (line 181) | def _load_images_txt(self, input_file):
method load_points3D (line 217) | def load_points3D(self, input_file=None):
method _load_points3D_bin (line 229) | def _load_points3D_bin(self, input_file):
method _load_points3D_txt (line 257) | def _load_points3D_txt(self, input_file):
method save (line 290) | def save(self, output_folder, binary=True):
method save_cameras (line 297) | def save_cameras(self, output_folder, output_file=None, binary=True):
method _save_cameras_bin (line 311) | def _save_cameras_bin(self, output_file):
method _save_cameras_txt (line 323) | def _save_cameras_txt(self, output_file):
method save_images (line 334) | def save_images(self, output_folder, output_file=None, binary=True):
method _save_images_bin (line 348) | def _save_images_bin(self, output_file):
method _save_images_txt (line 363) | def _save_images_txt(self, output_file):
method save_points3D (line 387) | def save_points3D(self, output_folder, output_file=None, binary=True):
method _save_points3D_bin (line 401) | def _save_points3D_bin(self, output_file):
method _save_points3D_txt (line 424) | def _save_points3D_txt(self, output_file):
method get_image_from_name (line 455) | def get_image_from_name(self, image_name):
method get_camera (line 461) | def get_camera(self, camera_id):
method get_points3D (line 466) | def get_points3D(self, image_id, return_points2D=True, return_colors=F...
method point3D_valid (line 489) | def point3D_valid(self, point3D_id):
method get_filtered_points3D (line 495) | def get_filtered_points3D(self, return_colors=False):
method get_shared_points3D (line 509) | def get_shared_points3D(self, image_id1, image_id2):
method get_viewed_points (line 524) | def get_viewed_points(self, image_id):
method add_camera (line 554) | def add_camera(self, camera):
method add_image (line 561) | def add_image(self, image):
method delete_images (line 568) | def delete_images(self, image_list):
method filter_points3D (line 599) | def filter_points3D(self,
method build_scene_graph (line 660) | def build_scene_graph(self):
FILE: FourierGrid/pycolmap/tools/colmap_to_nvm.py
function main (line 12) | def main(args):
FILE: FourierGrid/pycolmap/tools/delete_images.py
function main (line 11) | def main(args):
FILE: FourierGrid/pycolmap/tools/impute_missing_cameras.py
function interpolate_linear (line 16) | def interpolate_linear(images, camera_id, file_format):
function interpolate_hermite (line 54) | def interpolate_hermite(images, camera_id, file_format):
function main (line 137) | def main(args):
FILE: FourierGrid/pycolmap/tools/save_cameras_as_ply.py
function save_camera_ply (line 19) | def save_camera_ply(ply_file, images, scale):
function main (line 64) | def main(args):
FILE: FourierGrid/pycolmap/tools/transform_model.py
function main (line 11) | def main(args):
FILE: FourierGrid/pycolmap/tools/write_camera_track_to_bundler.py
function main (line 11) | def main(args):
FILE: FourierGrid/pycolmap/tools/write_depthmap_to_ply.py
function main (line 15) | def main(args):
FILE: FourierGrid/run_colmap2standard.py
class NeRFSceneManager (line 14) | class NeRFSceneManager(SceneManager):
method __init__ (line 21) | def __init__(self, data_dir):
method process (line 24) | def process(
class Dataset (line 115) | class Dataset:
method __init__ (line 116) | def __init__(self, data_dir):
method export (line 179) | def export(self, data_dir, out_mode):
function main (line 218) | def main(data_dir, out_mode):
FILE: FourierGrid/run_export_bbox.py
function run_export_bbox_cams (line 9) | def run_export_bbox_cams(args, cfg, data_dict, save_path=None):
FILE: FourierGrid/run_export_coarse.py
function run_export_coarse (line 7) | def run_export_coarse(args, cfg, device, save_path=None):
FILE: FourierGrid/run_gen_cam_paths.py
function select_k_nearest_points (line 15) | def select_k_nearest_points(idx, positions, k):
function move_idxs_to_folder (line 23) | def move_idxs_to_folder(data_dict, sampled_idxs, save_path, data_root="d...
function render_idxs (line 33) | def render_idxs(data_dict, sampled_idxs, save_path, data_root="data/sep1...
function get_rotation_kp_2d (line 44) | def get_rotation_kp_2d(args, cfg, data_dict, sample_idxs):
function run_export_bbox_cams (line 63) | def run_export_bbox_cams(args, cfg, data_dict, sample_idxs, save_path):
function run_gen_cam_paths (line 87) | def run_gen_cam_paths(args, cfg, data_dict, core_cam=None, straight_leng...
FILE: FourierGrid/run_gtk_analysis.py
class VoxelGrid (line 18) | class VoxelGrid(nn.Module):
method __init__ (line 20) | def __init__(self, grid_len=1000, data_point_num=100):
method forward (line 39) | def forward(self,): # calculate GTK
method one_d_regress (line 54) | def one_d_regress(self, x_train, x_test, y_train, y_test_gt):
class FourierGrid (line 84) | class FourierGrid(nn.Module):
method __init__ (line 86) | def __init__(self, grid_len=1000, band_num=10, data_point_num=100):
method gamma_x_i (line 106) | def gamma_x_i(self, x, i):
method forward (line 114) | def forward(self,): # calculate GTK
method one_d_regress (line 130) | def one_d_regress(self, x_train, x_test, y_train, y_test_gt):
function train_model (line 170) | def train_model(one_model):
function get_fg_gtk_spectrum_by_band_num (line 184) | def get_fg_gtk_spectrum_by_band_num(band_num):
function sample_random_signal (line 263) | def sample_random_signal(key, decay_vec):
function sample_random_powerlaw (line 272) | def sample_random_powerlaw(key, N, power):
function get_sine_signal (line 280) | def get_sine_signal():
function get_bessel_signal (line 284) | def get_bessel_signal():
function calculate_Delta (line 333) | def calculate_Delta(gtk, y1_data, y2_data):
FILE: FourierGrid/run_render.py
function render_viewpoints (line 15) | def render_viewpoints(cfg, model, render_poses, HW, Ks, ndc, render_kwargs,
function run_render (line 117) | def run_render(args, cfg, data_dict, device, debug=True, add_info="", vi...
FILE: FourierGrid/run_train.py
function create_new_model (line 19) | def create_new_model(args, cfg, cfg_model, cfg_train, xyz_min, xyz_max, ...
function gather_training_rays (line 59) | def gather_training_rays(data_dict, images, cfg, i_train, cfg_train, pos...
function scene_rep_reconstruction (line 98) | def scene_rep_reconstruction(args, cfg, cfg_model, cfg_train, xyz_min, x...
function run_train (line 336) | def run_train(args, cfg, data_dict, export_cam=False, export_geometry=Fa...
FILE: FourierGrid/tools/colmap_utils/colmap_read_model.py
class Image (line 49) | class Image(BaseImage):
method qvec2rotmat (line 50) | def qvec2rotmat(self):
function read_next_bytes (line 71) | def read_next_bytes(fid, num_bytes, format_char_sequence, endian_charact...
function read_cameras_text (line 83) | def read_cameras_text(path):
function read_cameras_binary (line 109) | def read_cameras_binary(path_to_model_file):
function read_images_text (line 138) | def read_images_text(path):
function read_images_binary (line 169) | def read_images_binary(path_to_model_file):
function read_points3D_text (line 204) | def read_points3D_text(path):
function read_points3d_binary (line 231) | def read_points3d_binary(path_to_model_file):
function read_model (line 261) | def read_model(path, ext):
function qvec2rotmat (line 273) | def qvec2rotmat(qvec):
function rotmat2qvec (line 286) | def rotmat2qvec(R):
function main (line 300) | def main():
FILE: FourierGrid/tools/colmap_utils/colmap_wrapper.py
function run_colmap (line 24) | def run_colmap(basedir, match_type):
FILE: FourierGrid/tools/colmap_utils/pose_utils.py
function load_colmap_data_nerfstudio (line 14) | def load_colmap_data_nerfstudio(basedir):
function load_colmap_data (line 64) | def load_colmap_data(realdir):
function save_poses (line 110) | def save_poses(basedir, poses, pts3d, perm, names):
function minify (line 146) | def minify(basedir, factors=[], resolutions=[]):
function gen_poses (line 199) | def gen_poses(basedir, match_type, factors=None):
FILE: FourierGrid/tools/imgs2poses.py
function check_structure (line 9) | def check_structure(scenedir):
FILE: FourierGrid/tools/vis_volume.py
function change_background_to_black (line 67) | def change_background_to_black(vis):
FILE: FourierGrid/trajectory_generators/interp_traj.py
function inter_two_poses (line 11) | def inter_two_poses(pose_a, pose_b, alpha):
function inter_poses (line 24) | def inter_poses(key_poses, n_out_poses, sigma=1.):
function hello (line 46) | def hello(data_dir, n_out_poses, key_poses):
FILE: FourierGrid/trajectory_generators/mega_traj.py
function all_equal (line 7) | def all_equal(iterable):
function rotate_rot_matrix_by_degree (line 12) | def rotate_rot_matrix_by_degree(rot, rot_degree, axis='y'):
function gen_dummy_trajs (line 18) | def gen_dummy_trajs(metadata, tr_c2w, train_HW, tr_K, test_num=100):
function gen_straight_trajs (line 28) | def gen_straight_trajs(metadata, tr_c2w, train_HW, tr_K, tr_cam_idx, tra...
function gen_rotational_trajs (line 44) | def gen_rotational_trajs(args, cfg, metadata, tr_c2w, train_HW, tr_K, ro...
FILE: FourierGrid/trajectory_generators/waymo_traj.py
function all_equal (line 7) | def all_equal(iterable):
function rotate_rot_matrix_by_degree (line 12) | def rotate_rot_matrix_by_degree(rot, rot_degree, axis='y'):
function gen_dummy_trajs (line 18) | def gen_dummy_trajs(metadata, tr_c2w, train_HW, tr_K, test_num=100):
function gen_straight_trajs (line 28) | def gen_straight_trajs(metadata, tr_c2w, train_HW, tr_K, tr_cam_idx, tra...
function gen_rotational_trajs (line 44) | def gen_rotational_trajs(args, cfg, metadata, tr_c2w, train_HW, tr_K, tr...
FILE: FourierGrid/utils.py
function resize_and_to_8b (line 21) | def resize_and_to_8b(input_images, res):
function create_optimizer_or_freeze_model (line 26) | def create_optimizer_or_freeze_model(model, cfg_train, global_step, verb...
function load_checkpoint (line 61) | def load_checkpoint(model, optimizer, ckpt_path, no_reload_optimizer):
function load_model (line 70) | def load_model(model_class, ckpt_path):
function rgb_ssim (line 79) | def rgb_ssim(img0, img1, max_val,
function init_lpips (line 129) | def init_lpips(net_name, device):
function rgb_lpips (line 135) | def rgb_lpips(np_gt, np_im, net_name, device):
FILE: block_nerf/block_nerf_lightning.py
class Block_NeRF_System (line 13) | class Block_NeRF_System(LightningModule):
method __init__ (line 14) | def __init__(self, hparams):
method forward (line 45) | def forward(self, rays, ts):
method setup (line 71) | def setup(self, stage):
method configure_optimizers (line 85) | def configure_optimizers(self):
method train_dataloader (line 90) | def train_dataloader(self):
method val_dataloader (line 97) | def val_dataloader(self):
method training_step (line 104) | def training_step(self, batch, batch_nb):
method validation_step (line 121) | def validation_step(self, batch, batch_nb): # validate at each epoch
method validation_epoch_end (line 147) | def validation_epoch_end(self, outputs):
FILE: block_nerf/block_nerf_model.py
class BlockNeRFLoss (line 5) | class BlockNeRFLoss(nn.Module):
method __init__ (line 6) | def __init__(self, lambda_mu=0.01, Visi_loss=1e-2):
method forward (line 11) | def forward(self, inputs, targets):
class InterPosEmbedding (line 25) | class InterPosEmbedding(nn.Module):
method __init__ (line 26) | def __init__(self, N_freqs=10):
method forward (line 36) | def forward(self, mu, diagE):
class PosEmbedding (line 57) | class PosEmbedding(nn.Module):
method __init__ (line 58) | def __init__(self, N_freqs):
method forward (line 69) | def forward(self, x):
class Block_NeRF (line 77) | class Block_NeRF(nn.Module):
method __init__ (line 78) | def __init__(self, D=8, W=256, skips=[4],
method forward (line 124) | def forward(self, x, sigma_only=False):
class Visibility (line 156) | class Visibility(nn.Module):
method __init__ (line 157) | def __init__(self,
method forward (line 172) | def forward(self, x):
FILE: block_nerf/block_visualize.py
function visualize_depth (line 6) | def visualize_depth(depth, cmap=cv2.COLORMAP_JET):
FILE: block_nerf/learning_utils.py
class GradualWarmupScheduler (line 12) | class GradualWarmupScheduler(_LRScheduler):
method __init__ (line 22) | def __init__(self, optimizer, multiplier, total_epoch, after_scheduler...
method get_lr (line 31) | def get_lr(self):
method step_ReduceLROnPlateau (line 42) | def step_ReduceLROnPlateau(self, metrics, epoch=None):
method step (line 56) | def step(self, epoch=None, metrics=None):
function get_parameters (line 69) | def get_parameters(models):
function get_optimizer (line 83) | def get_optimizer(hparams, models):
function get_scheduler (line 103) | def get_scheduler(hparams, optimizer):
function get_learning_rate (line 122) | def get_learning_rate(optimizer):
function extract_model_state_dict (line 126) | def extract_model_state_dict(ckpt_path, model_name='model', prefixes_to_...
function load_ckpt (line 143) | def load_ckpt(model, ckpt_path, model_name='model', prefixes_to_ignore=[]):
FILE: block_nerf/metrics.py
function mse (line 4) | def mse(image_pred, image_gt, valid_mask=None, reduction='mean'):
function psnr (line 12) | def psnr(image_pred, image_gt, valid_mask=None, reduction='mean'):
function ssim (line 15) | def ssim(image_pred, image_gt, reduction='mean'):
FILE: block_nerf/rendering.py
function get_cone_mean_conv (line 8) | def get_cone_mean_conv(
function sample_pdf (line 42) | def sample_pdf(
function volume_rendering (line 91) | def volume_rendering(
function render_rays (line 129) | def render_rays(
FILE: block_nerf/waymo_dataset.py
function get_ray_directions (line 13) | def get_ray_directions(H, W, K):
function get_rays (line 27) | def get_rays(directions, c2w):
function find_idx_name (line 39) | def find_idx_name(elements, img_name):
function find_nearest_idx (line 46) | def find_nearest_idx(img_source,block_elements,train_meta):
class WaymoDataset (line 62) | class WaymoDataset(Dataset):
method __init__ (line 63) | def __init__(self, root_dir, split='train', block='block_0',
method read_json (line 83) | def read_json(self):
method __len__ (line 197) | def __len__(self):
method __getitem__ (line 208) | def __getitem__(self, idx):
FILE: data_preprocess/comvog_organize_by_cam_pos.py
function get_pix2cam (line 17) | def get_pix2cam(focals, width, height):
function reorder_meta (line 48) | def reorder_meta(meta_dict):
function form_unified_dict (line 69) | def form_unified_dict(old_to_new, metas, save_prefix='images_train', spl...
FILE: data_preprocess/fetch_data_from_tf_record.py
function test_rays_dir_radii (line 18) | def test_rays_dir_radii(ray_dirs):
function decode_fn (line 28) | def decode_fn(record_bytes):
function get_cam_rays (line 45) | def get_cam_rays(H, W, K):
function get_rotate_one_image (line 57) | def get_rotate_one_image(cam_ray_dir, world_ray_dir):
function handle_one_record (line 80) | def handle_one_record(tfrecord, train_index, val_index):
FILE: data_preprocess/get_one_block_meta_and_visualize.py
function get_pix2cam (line 12) | def get_pix2cam(focals, width, height):
function form_unified_dict (line 53) | def form_unified_dict(images, metas, save_prefix='images_train', split_p...
FILE: data_preprocess/split_block.py
function get_hparams (line 15) | def get_hparams():
function extract_imgname_origins (line 30) | def extract_imgname_origins(meta):
function resort_origins (line 40) | def resort_origins(img_origins, positions):
function get_the_distance (line 54) | def get_the_distance(r=2, overlap=0.5):
function get_each_block_element_train (line 76) | def get_each_block_element_train(img_train_origins, centroid, radius):
function extract_img_base_camidx (line 89) | def extract_img_base_camidx(cam_idx, train_meta):
function get_block_idx (line 98) | def get_block_idx(img_name, split_train_results):
function get_val_block_index (line 107) | def get_val_block_index(img_val_origins, train_meta, val_meta, img_train...
function split_dataset (line 137) | def split_dataset(train_meta, val_meta, radius=0.5, overlap=0.5):
function extract_origins (line 195) | def extract_origins(meta):
function extract_cam_idx (line 208) | def extract_cam_idx(train_meta):
function extract_img_each_idx (line 217) | def extract_img_each_idx(idx,train_meta,train_split_meta):
function transfer_pt_to_json (line 227) | def transfer_pt_to_json(pt_meta):
FILE: data_preprocess/vis.py
function normalize (line 13) | def normalize(v):
function vis_lr (line 18) | def vis_lr(lr_init: float = 5e-4,
function save_image_tensor (line 47) | def save_image_tensor(image: torch.tensor, height: int, width: int, save...
function save_images (line 67) | def save_images(rgb, dist, acc, path, idx):
function visualize_depth (line 76) | def visualize_depth(depth, cmap=cv2.COLORMAP_JET):
function gen_render_path (line 93) | def gen_render_path(c2ws, N_views=30):
function create_spiral_poses (line 125) | def create_spiral_poses(radii, focus_depth, n_poses=120):
function create_spheric_poses (line 160) | def create_spheric_poses(radius, n_poses=120):
function stack_rgb (line 202) | def stack_rgb(rgb_gt, coarse_rgb, fine_rgb):
FILE: data_preprocess/visualize_cameras.py
function get_camera_frustum (line 14) | def get_camera_frustum(img_size, K, W2C, frustum_length=0.5, color=[0.0,...
function frustums2lineset (line 51) | def frustums2lineset(frustums):
function visualize_cameras (line 70) | def visualize_cameras(
function convert_pose (line 118) | def convert_pose(C2W):
function read_single_scale_cam (line 126) | def read_single_scale_cam(path):
function read_multi_scale_cam (line 158) | def read_multi_scale_cam(path):
function create_spheric_cam (line 217) | def create_spheric_cam(focal, img_size, radius, camera_size=1):
FILE: docs/parse_markdown.py
class MyHTMLParser (line 9) | class MyHTMLParser(HTMLParser):
method __init__ (line 10) | def __init__(self, *, convert_charrefs: bool = ...) -> None:
method handle_starttag (line 14) | def handle_starttag(self, tag, attrs):
method handle_endtag (line 25) | def handle_endtag(self, tag):
method handle_data (line 29) | def handle_data(self, data):
FILE: docs/render_docs_from_csv.py
function check_nan (line 10) | def check_nan(item):
function title_by_cls (line 16) | def title_by_cls(cls_str, cn):
function write_item_to_md (line 23) | def write_item_to_md(md, data, idx, cn=False):
function render_main_doc (line 46) | def render_main_doc(meta_data_path="docs/weekly_nerf_meta_data.xlsx", cn...
FILE: eval_block_nerf.py
function get_hparams (line 17) | def get_hparams():
function batched_inference (line 45) | def batched_inference(
function filter_cam_info_by_index (line 78) | def filter_cam_info_by_index(index, cam_infos):
function filter_Block (line 86) | def filter_Block(begin, blocks):
function DistanceWeight (line 95) | def DistanceWeight(point, centroid, p=4):
function Inverse_Interpolation (line 101) | def Inverse_Interpolation(model_result, W_H):
FILE: run_FourierGrid.py
function config_parser (line 15) | def config_parser():
function seed_everything (line 69) | def seed_everything():
FILE: train_block_nerf.py
function get_opts (line 19) | def get_opts():
function main (line 101) | def main(hparams):
Condensed preview — 273 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (3,038K chars).
[
{
"path": ".all-contributorsrc",
"chars": 1802,
"preview": "{\n \"files\": [\n \"README.md\"\n ],\n \"imageSize\": 100,\n \"commit\": false,\n \"contributors\": [\n {\n \"login\": \"sjt"
},
{
"path": ".gitignore",
"chars": 1875,
"preview": "# Byte-compiled / optimized / DLL files\n__pycache__/\n*.py[cod]\n*$py.class\n\n# C extensions\n*.so\n\n# Distribution / packagi"
},
{
"path": "FourierGrid/FourierGrid_ckpt_manager.py",
"chars": 4332,
"preview": "from FourierGrid.FourierGrid_model import FourierGridModel\nfrom FourierGrid import utils, dvgo, dcvgo, dmpigo\nimport tor"
},
{
"path": "FourierGrid/FourierGrid_grid.py",
"chars": 7305,
"preview": "import os\nimport time\nimport functools\nimport numpy as np\nfrom einops import rearrange\nimport torch\nimport torch.nn as n"
},
{
"path": "FourierGrid/FourierGrid_model.py",
"chars": 34183,
"preview": "import os\nimport time\nimport functools\nimport numpy as np\nimport pdb\nimport torch\nimport torch.nn as nn\nimport torch.nn."
},
{
"path": "FourierGrid/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "FourierGrid/arf.py",
"chars": 3415,
"preview": "import numpy as np\nimport torch\nimport torch.nn as nn\nimport imageio\nimport cv2\nimport pdb\nimport os\n\nclass ARF:\n def"
},
{
"path": "FourierGrid/bbox_compute.py",
"chars": 7365,
"preview": "import torch\nimport numpy as np\nfrom FourierGrid import utils, dvgo\nimport time\nfrom FourierGrid.load_everything import "
},
{
"path": "FourierGrid/camera_utils.py",
"chars": 12410,
"preview": "import numpy as np\nimport enum\nfrom dataclasses import dataclass\nfrom typing import List, Mapping, Optional, Text, Tuple"
},
{
"path": "FourierGrid/common_data_loaders/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "FourierGrid/common_data_loaders/load_blendedmvs.py",
"chars": 1114,
"preview": "import os\nimport glob\nimport torch\nimport numpy as np\nimport imageio\nimport json\nimport torch.nn.functional as F\nimport "
},
{
"path": "FourierGrid/common_data_loaders/load_blender.py",
"chars": 2583,
"preview": "import os\nimport torch\nimport numpy as np\nimport imageio\nimport pdb\nimport json\nimport torch.nn.functional as F\nimport c"
},
{
"path": "FourierGrid/common_data_loaders/load_co3d.py",
"chars": 3135,
"preview": "import os\nimport json\nimport gzip\nimport glob\nimport torch\nimport numpy as np\nimport imageio\nimport torch.nn.functional "
},
{
"path": "FourierGrid/common_data_loaders/load_common_data.py",
"chars": 10216,
"preview": "import numpy as np\nimport pdb\nfrom .load_llff import load_llff_data\nfrom .load_free import load_free_data\nfrom .load_ble"
},
{
"path": "FourierGrid/common_data_loaders/load_deepvoxels.py",
"chars": 3844,
"preview": "import os\nimport numpy as np\nimport imageio\n\n\ndef load_dv_data(scene='cube', basedir='/data/deepvoxels', testskip=1):\n\n "
},
{
"path": "FourierGrid/common_data_loaders/load_free.py",
"chars": 13134,
"preview": "import numpy as np\nimport os, imageio\nimport torch\nimport scipy\nimport cv2\nimport pdb\nfrom shutil import copy\nfrom subpr"
},
{
"path": "FourierGrid/common_data_loaders/load_llff.py",
"chars": 15354,
"preview": "import numpy as np\nimport os, imageio\nimport torch\nimport scipy\nimport cv2\nimport pdb\nfrom shutil import copy\nfrom subpr"
},
{
"path": "FourierGrid/common_data_loaders/load_nerfpp.py",
"chars": 6520,
"preview": "'''\nModify from\nhttps://github.com/Kai-46/nerfplusplus/blob/master/data_loader_split.py\n'''\nimport os\nimport pdb\nimport "
},
{
"path": "FourierGrid/common_data_loaders/load_nsvf.py",
"chars": 1787,
"preview": "import os\nimport glob\nimport torch\nimport numpy as np\nimport imageio\nimport json\nimport torch.nn.functional as F\nimport "
},
{
"path": "FourierGrid/common_data_loaders/load_tankstemple.py",
"chars": 2508,
"preview": "import os\nimport glob\nimport torch\nimport numpy as np\nimport imageio\nimport json\nimport torch.nn.functional as F\nimport "
},
{
"path": "FourierGrid/configs/blendedmvs/Character.py",
"chars": 217,
"preview": "_base_ = '../default.py'\n\nexpname = 'dvgo_Character'\nbasedir = './logs/blended_mvs'\n\ndata = dict(\n datadir='./data/Bl"
},
{
"path": "FourierGrid/configs/blendedmvs/Fountain.py",
"chars": 216,
"preview": "_base_ = '../default.py'\n\nexpname = 'dvgo_Fountain'\nbasedir = './logs/blended_mvs'\n\ndata = dict(\n datadir='./data/Ble"
},
{
"path": "FourierGrid/configs/blendedmvs/Jade.py",
"chars": 208,
"preview": "_base_ = '../default.py'\n\nexpname = 'dvgo_Jade'\nbasedir = './logs/blended_mvs'\n\ndata = dict(\n datadir='./data/Blended"
},
{
"path": "FourierGrid/configs/blendedmvs/Statues.py",
"chars": 213,
"preview": "_base_ = '../default.py'\n\nexpname = 'dvgo_Statues'\nbasedir = './logs/blended_mvs'\n\ndata = dict(\n datadir='./data/Blen"
},
{
"path": "FourierGrid/configs/co3d/donut_369_40208_78816.py",
"chars": 432,
"preview": "_base_ = '../default.py'\n\nexpname = 'dvgo_donut_369_40208_78816'\nbasedir = './logs/co3d'\n\ndata = dict(\n datadir='./da"
},
{
"path": "FourierGrid/configs/custom/Madoka.py",
"chars": 288,
"preview": "_base_ = './default_forward_facing.py'\n\nexpname = 'Madoka'\n\ndata = dict(\n datadir='data/Madoka/dense',\n factor=2,\n"
},
{
"path": "FourierGrid/configs/custom/Madoka_long.py",
"chars": 334,
"preview": "_base_ = './default_forward_facing.py'\n\nexpname = 'Madoka'\n\ndata = dict(\n datadir='data/Madoka/dense',\n factor=2,\n"
},
{
"path": "FourierGrid/configs/custom/Otobai.py",
"chars": 337,
"preview": "_base_ = './default_forward_facing.py'\n\nexpname = 'Otobai'\n\ndata = dict(\n datadir='./data/Otobai/dense',\n factor=2"
},
{
"path": "FourierGrid/configs/custom/default_forward_facing.py",
"chars": 1055,
"preview": "_base_ = '../default.py'\n\nbasedir = './logs/custom'\n\ndata = dict(\n dataset_type='llff',\n load2gpu_on_the_fly=True,"
},
{
"path": "FourierGrid/configs/custom/default_ubd_inward_facing.py",
"chars": 1132,
"preview": "_base_ = '../default.py'\n\nbasedir = './logs/custom'\n\ndata = dict(\n dataset_type='llff',\n spherify=True,\n llffho"
},
{
"path": "FourierGrid/configs/custom/sm01_desktop.py",
"chars": 261,
"preview": "_base_ = './default_forward_facing.py'\n\nexpname = 'sm01_desktop'\n\ndata = dict(\n datadir='./data/sm01_desktop/dense',\n"
},
{
"path": "FourierGrid/configs/custom/sm02_multiple_desktop.py",
"chars": 279,
"preview": "_base_ = './default_forward_facing.py'\n\nexpname = 'sm02_multiple_desktop'\n\ndata = dict(\n datadir='./data/sm02_multipl"
},
{
"path": "FourierGrid/configs/custom/sm03_meeting.py",
"chars": 261,
"preview": "_base_ = './default_forward_facing.py'\n\nexpname = 'sm03_meeting'\n\ndata = dict(\n datadir='./data/sm03_meeting/dense',\n"
},
{
"path": "FourierGrid/configs/deepvoxels/armchair.py",
"chars": 207,
"preview": "_base_ = '../default.py'\n\nexpname = 'dvgo_armchair'\nbasedir = './logs/deepvoxels'\n\ndata = dict(\n datadir='./data/deep"
},
{
"path": "FourierGrid/configs/deepvoxels/cube.py",
"chars": 199,
"preview": "_base_ = '../default.py'\n\nexpname = 'dvgo_cube'\nbasedir = './logs/deepvoxels'\n\ndata = dict(\n datadir='./data/deepvoxe"
},
{
"path": "FourierGrid/configs/deepvoxels/greek.py",
"chars": 201,
"preview": "_base_ = '../default.py'\n\nexpname = 'dvgo_greek'\nbasedir = './logs/deepvoxels'\n\ndata = dict(\n datadir='./data/deepvox"
},
{
"path": "FourierGrid/configs/deepvoxels/vase.py",
"chars": 199,
"preview": "_base_ = '../default.py'\n\nexpname = 'dvgo_vase'\nbasedir = './logs/deepvoxels'\n\ndata = dict(\n datadir='./data/deepvoxe"
},
{
"path": "FourierGrid/configs/default.py",
"chars": 6430,
"preview": "from copy import deepcopy\n\nexpname = None # experiment name\nbasedir = './logs/' # where"
},
{
"path": "FourierGrid/configs/free_dataset/grass.py",
"chars": 1493,
"preview": "_base_ = '../default.py'\nexpname = 'grass_may31_'\nvis = dict(\n height_rate = 0.6 # camera direction frustrum height\n)"
},
{
"path": "FourierGrid/configs/free_dataset/lab.py",
"chars": 4615,
"preview": "_base_ = '../default.py'\nexpname = 'lab_may30_'\nvis = dict(\n height_rate = 0.6 # camera direction frustrum height\n)\nm"
},
{
"path": "FourierGrid/configs/lf/africa.py",
"chars": 116,
"preview": "_base_ = './lf_default.py'\n\nexpname = 'dvgo_Africa_unbounded'\n\ndata = dict(\n datadir='./data/lf_data/africa',\n)\n\n"
},
{
"path": "FourierGrid/configs/lf/basket.py",
"chars": 116,
"preview": "_base_ = './lf_default.py'\n\nexpname = 'dvgo_Basket_unbounded'\n\ndata = dict(\n datadir='./data/lf_data/basket',\n)\n\n"
},
{
"path": "FourierGrid/configs/lf/lf_default.py",
"chars": 1032,
"preview": "_base_ = '../default.py'\n\nbasedir = './logs/lf'\n\ndata = dict(\n dataset_type='nerfpp',\n inverse_y=True,\n white_b"
},
{
"path": "FourierGrid/configs/lf/ship.py",
"chars": 112,
"preview": "_base_ = './lf_default.py'\n\nexpname = 'dvgo_Ship_unbounded'\n\ndata = dict(\n datadir='./data/lf_data/ship',\n)\n\n"
},
{
"path": "FourierGrid/configs/lf/statue.py",
"chars": 116,
"preview": "_base_ = './lf_default.py'\n\nexpname = 'dvgo_Statue_unbounded'\n\ndata = dict(\n datadir='./data/lf_data/statue',\n)\n\n"
},
{
"path": "FourierGrid/configs/lf/torch.py",
"chars": 114,
"preview": "_base_ = './lf_default.py'\n\nexpname = 'dvgo_Torch_unbounded'\n\ndata = dict(\n datadir='./data/lf_data/torch',\n)\n\n"
},
{
"path": "FourierGrid/configs/linemod/ape.py",
"chars": 792,
"preview": "_base_ = '../default.py'\nseq_name = 'ape'\nseq_id = 1\nexpname = f'{seq_name}_nov11_'\npose_expname = 'bayes_nerf_v2_4'\nbas"
},
{
"path": "FourierGrid/configs/linemod/benchvise.py",
"chars": 331,
"preview": "_base_ = '../default.py'\nseq_name = 'benchvise'\nexpname = f'{seq_name}_nov9_'\nbasedir = './logs/linemod'\n\ndata = dict(\n "
},
{
"path": "FourierGrid/configs/linemod/camera.py",
"chars": 328,
"preview": "_base_ = '../default.py'\nseq_name = 'camera'\nexpname = f'{seq_name}_nov9_'\nbasedir = './logs/linemod'\n\ndata = dict(\n "
},
{
"path": "FourierGrid/configs/linemod/can.py",
"chars": 324,
"preview": "_base_ = '../default.py'\nseq_name = 'can'\nexpname = f'{seq_name}_nov8'\nbasedir = './logs/linemod'\n\ndata = dict(\n data"
},
{
"path": "FourierGrid/configs/linemod/cat.py",
"chars": 311,
"preview": "_base_ = '../default.py'\nseq_name = 'cat'\nexpname = f'{seq_name}_nov8_'\nbasedir = './logs/linemod'\n\ndata = dict(\n dat"
},
{
"path": "FourierGrid/configs/linemod/driller.py",
"chars": 329,
"preview": "_base_ = '../default.py'\nseq_name = 'driller'\nexpname = f'{seq_name}_nov8'\nbasedir = './logs/linemod'\n\ndata = dict(\n "
},
{
"path": "FourierGrid/configs/linemod/duck.py",
"chars": 321,
"preview": "_base_ = '../default.py'\nseq_name = 'duck'\nexpname = f'{seq_name}_nov8'\nbasedir = './logs/linemod'\n\ndata = dict(\n dat"
},
{
"path": "FourierGrid/configs/linemod/eggbox.py",
"chars": 327,
"preview": "_base_ = '../default.py'\nseq_name = 'eggbox'\nexpname = f'{seq_name}_nov8'\nbasedir = './logs/linemod'\n\ndata = dict(\n d"
},
{
"path": "FourierGrid/configs/linemod/glue.py",
"chars": 325,
"preview": "_base_ = '../default.py'\nseq_name = 'glue'\nexpname = f'{seq_name}_nov8'\nbasedir = './logs/linemod'\n\ndata = dict(\n dat"
},
{
"path": "FourierGrid/configs/linemod/holepuncher.py",
"chars": 332,
"preview": "_base_ = '../default.py'\nseq_name = 'holepuncher'\nexpname = f'{seq_name}_nov8'\nbasedir = './logs/linemod'\n\ndata = dict(\n"
},
{
"path": "FourierGrid/configs/linemod/iron.py",
"chars": 325,
"preview": "_base_ = '../default.py'\nseq_name = 'iron'\nexpname = f'{seq_name}_nov8'\nbasedir = './logs/linemod'\n\ndata = dict(\n dat"
},
{
"path": "FourierGrid/configs/linemod/lamp.py",
"chars": 325,
"preview": "_base_ = '../default.py'\nseq_name = 'lamp'\nexpname = f'{seq_name}_nov8'\nbasedir = './logs/linemod'\n\ndata = dict(\n dat"
},
{
"path": "FourierGrid/configs/linemod/phone.py",
"chars": 326,
"preview": "_base_ = '../default.py'\nseq_name = 'phone'\nexpname = f'{seq_name}_nov8'\nbasedir = './logs/linemod'\n\ndata = dict(\n da"
},
{
"path": "FourierGrid/configs/llff/fern.py",
"chars": 106,
"preview": "_base_ = './llff_default.py'\n\nexpname = 'fern'\n\ndata = dict(\n datadir='./data/nerf_llff_data/fern',\n)\n\n"
},
{
"path": "FourierGrid/configs/llff/fern_lg.py",
"chars": 112,
"preview": "_base_ = './llff_default_lg.py'\n\nexpname = 'fern_lg'\n\ndata = dict(\n datadir='./data/nerf_llff_data/fern',\n)\n\n"
},
{
"path": "FourierGrid/configs/llff/flower.py",
"chars": 110,
"preview": "_base_ = './llff_default.py'\n\nexpname = 'flower'\n\ndata = dict(\n datadir='./data/nerf_llff_data/flower',\n)\n\n"
},
{
"path": "FourierGrid/configs/llff/flower_lg.py",
"chars": 116,
"preview": "_base_ = './llff_default_lg.py'\n\nexpname = 'flower_lg'\n\ndata = dict(\n datadir='./data/nerf_llff_data/flower',\n)\n\n"
},
{
"path": "FourierGrid/configs/llff/fortress.py",
"chars": 114,
"preview": "_base_ = './llff_default.py'\n\nexpname = 'fortress'\n\ndata = dict(\n datadir='./data/nerf_llff_data/fortress',\n)\n\n"
},
{
"path": "FourierGrid/configs/llff/fortress_lg.py",
"chars": 120,
"preview": "_base_ = './llff_default_lg.py'\n\nexpname = 'fortress_lg'\n\ndata = dict(\n datadir='./data/nerf_llff_data/fortress',\n)\n\n"
},
{
"path": "FourierGrid/configs/llff/horns.py",
"chars": 134,
"preview": "_base_ = './llff_default.py'\n\nexpname = 'horns'\n\ndata = dict(\n datadir='./data/nerf_llff_data/horns',\n # dataset_t"
},
{
"path": "FourierGrid/configs/llff/horns_lg.py",
"chars": 114,
"preview": "_base_ = './llff_default_lg.py'\n\nexpname = 'horns_lg'\n\ndata = dict(\n datadir='./data/nerf_llff_data/horns',\n)\n\n"
},
{
"path": "FourierGrid/configs/llff/leaves.py",
"chars": 147,
"preview": "_base_ = './llff_default.py'\nmodel = 'DVGO'\nexpname = 'leaves'\nbasedir = './logs/llff'\n\ndata = dict(\n datadir='./data"
},
{
"path": "FourierGrid/configs/llff/leaves_lg.py",
"chars": 116,
"preview": "_base_ = './llff_default_lg.py'\n\nexpname = 'leaves_lg'\n\ndata = dict(\n datadir='./data/nerf_llff_data/leaves',\n)\n\n"
},
{
"path": "FourierGrid/configs/llff/llff_default.py",
"chars": 589,
"preview": "_base_ = '../default.py'\n\nbasedir = './logs/llff'\n\ndata = dict(\n dataset_type='llff',\n ndc=True,\n width=1008,\n "
},
{
"path": "FourierGrid/configs/llff/llff_default_lg.py",
"chars": 731,
"preview": "_base_ = '../default.py'\n\nbasedir = './logs/llff'\n\ndata = dict(\n dataset_type='llff',\n ndc=True,\n width=1008,\n "
},
{
"path": "FourierGrid/configs/llff/orchids.py",
"chars": 112,
"preview": "_base_ = './llff_default.py'\n\nexpname = 'orchids'\n\ndata = dict(\n datadir='./data/nerf_llff_data/orchids',\n)\n\n"
},
{
"path": "FourierGrid/configs/llff/orchids_lg.py",
"chars": 118,
"preview": "_base_ = './llff_default_lg.py'\n\nexpname = 'orchids_lg'\n\ndata = dict(\n datadir='./data/nerf_llff_data/orchids',\n)\n\n"
},
{
"path": "FourierGrid/configs/llff/room.py",
"chars": 106,
"preview": "_base_ = './llff_default.py'\n\nexpname = 'room'\n\ndata = dict(\n datadir='./data/nerf_llff_data/room',\n)\n\n"
},
{
"path": "FourierGrid/configs/llff/room_lg.py",
"chars": 112,
"preview": "_base_ = './llff_default_lg.py'\n\nexpname = 'room_lg'\n\ndata = dict(\n datadir='./data/nerf_llff_data/room',\n)\n\n"
},
{
"path": "FourierGrid/configs/llff/trex.py",
"chars": 106,
"preview": "_base_ = './llff_default.py'\n\nexpname = 'trex'\n\ndata = dict(\n datadir='./data/nerf_llff_data/trex',\n)\n\n"
},
{
"path": "FourierGrid/configs/llff/trex_lg.py",
"chars": 112,
"preview": "_base_ = './llff_default_lg.py'\n\nexpname = 'trex_lg'\n\ndata = dict(\n datadir='./data/nerf_llff_data/trex',\n)\n\n"
},
{
"path": "FourierGrid/configs/mega/building.py",
"chars": 2916,
"preview": "_base_ = '../default.py'\nmodel='FourierGrid'\ndata_name = 'building'\nbasedir = f'./logs/mega/{data_name}'\nvisualize_poses"
},
{
"path": "FourierGrid/configs/mega/building_no_block.py",
"chars": 3533,
"preview": "_base_ = '../default.py'\ndata_name = 'building'\nmodel='FourierGrid'\nbasedir = f'./logs/mega/{data_name}'\nvisualize_poses"
},
{
"path": "FourierGrid/configs/mega/quad.py",
"chars": 2643,
"preview": "_base_ = '../default.py'\nmodel='FourierGrid'\ndata_name = 'quad'\nbasedir = f'./logs/mega/{data_name}'\nvisualize_poses = F"
},
{
"path": "FourierGrid/configs/mega/rubble.py",
"chars": 2798,
"preview": "_base_ = '../default.py'\nmodel='FourierGrid'\ndata_name = 'rubble'\nbasedir = f'./logs/mega/{data_name}'\nvisualize_poses ="
},
{
"path": "FourierGrid/configs/nerf/chair.py",
"chars": 192,
"preview": "_base_ = '../default.py'\n\nexpname = 'dvgo_chair'\nbasedir = './logs/nerf_synthetic'\n\ndata = dict(\n datadir='./data/ner"
},
{
"path": "FourierGrid/configs/nerf/drums.py",
"chars": 192,
"preview": "_base_ = '../default.py'\n\nexpname = 'dvgo_drums'\nbasedir = './logs/nerf_synthetic'\n\ndata = dict(\n datadir='./data/ner"
},
{
"path": "FourierGrid/configs/nerf/ficus.py",
"chars": 192,
"preview": "_base_ = '../default.py'\n\nexpname = 'dvgo_ficus'\nbasedir = './logs/nerf_synthetic'\n\ndata = dict(\n datadir='./data/ner"
},
{
"path": "FourierGrid/configs/nerf/hotdog.py",
"chars": 194,
"preview": "_base_ = '../default.py'\n\nexpname = 'dvgo_hotdog'\nbasedir = './logs/nerf_synthetic'\n\ndata = dict(\n datadir='./data/ne"
},
{
"path": "FourierGrid/configs/nerf/lego.py",
"chars": 189,
"preview": "_base_ = '../default.py'\n\nexpname = 'dvgo_lego'\nbasedir = './logs/nerf_synthetic'\n\ndata = dict(\n datadir='./data/nerf"
},
{
"path": "FourierGrid/configs/nerf/materials.py",
"chars": 200,
"preview": "_base_ = '../default.py'\n\nexpname = 'dvgo_materials'\nbasedir = './logs/nerf_synthetic'\n\ndata = dict(\n datadir='./data"
},
{
"path": "FourierGrid/configs/nerf/mic.py",
"chars": 188,
"preview": "_base_ = '../default.py'\n\nexpname = 'dvgo_mic'\nbasedir = './logs/nerf_synthetic'\n\ndata = dict(\n datadir='./data/nerf_"
},
{
"path": "FourierGrid/configs/nerf/ship.py",
"chars": 190,
"preview": "_base_ = '../default.py'\n\nexpname = 'dvgo_ship'\nbasedir = './logs/nerf_synthetic'\n\ndata = dict(\n datadir='./data/nerf"
},
{
"path": "FourierGrid/configs/nerf/ship.tensorf.py",
"chars": 490,
"preview": "_base_ = '../default.py'\n\nexpname = 'dvgo_ship_tensorf'\nbasedir = './logs/nerf_synthetic'\n\ndata = dict(\n datadir='./d"
},
{
"path": "FourierGrid/configs/nerf_studio/Giannini_Hall.py",
"chars": 2537,
"preview": "_base_ = './nerf_studio_default.py'\nexpname = 'Giannini-Hall_mar16_'\nvis = dict(\n height_rate = 0.6 # camera directio"
},
{
"path": "FourierGrid/configs/nerf_studio/nerf_studio_default.py",
"chars": 1158,
"preview": "_base_ = '../default.py'\n\nbasedir = './logs/nerf_unbounded'\n\ndata = dict(\n dataset_type='nerfstudio',\n spherify=Tr"
},
{
"path": "FourierGrid/configs/nerf_studio/stump.py",
"chars": 2591,
"preview": "_base_ = './nerf_studio_default.py'\nexpname = 'stump_mar16_'\nvis = dict(\n height_rate = 0.6 # camera direction frustr"
},
{
"path": "FourierGrid/configs/nerf_unbounded/bicycle.py",
"chars": 352,
"preview": "_base_ = './nerf_unbounded_default.py'\nexpname = 'dvgo_bicycle_unbounded'\n\ndata = dict(\n datadir='./data/360_v2/bicyc"
},
{
"path": "FourierGrid/configs/nerf_unbounded/bicycle_single.py",
"chars": 2255,
"preview": "_base_ = './nerf_unbounded_default.py'\nexpname = 'bicycle_nov29_'\nvis = dict(\n height_rate = 0.6 # camera direction f"
},
{
"path": "FourierGrid/configs/nerf_unbounded/bonsai.py",
"chars": 352,
"preview": "_base_ = './nerf_unbounded_default.py'\n\nexpname = 'dvgo_bonsai_unbounded'\n\ndata = dict(\n datadir='./data/360_v2/bonsa"
},
{
"path": "FourierGrid/configs/nerf_unbounded/bonsai_single.py",
"chars": 2285,
"preview": "_base_ = './nerf_unbounded_default.py'\nexpname = 'bonsai_nov29_'\nvis = dict(\n height_rate = 0.6 # camera direction fr"
},
{
"path": "FourierGrid/configs/nerf_unbounded/counter.py",
"chars": 357,
"preview": "_base_ = './nerf_unbounded_default.py'\n\nexpname = 'dvgo_counter_unbounded'\n\ndata = dict(\n datadir='./data/360_v2/coun"
},
{
"path": "FourierGrid/configs/nerf_unbounded/counter_single.py",
"chars": 2348,
"preview": "_base_ = './nerf_unbounded_default.py'\nexpname = 'counter_may25_'\nvis = dict(\n height_rate = 0.6 # camera direction f"
},
{
"path": "FourierGrid/configs/nerf_unbounded/garden.py",
"chars": 329,
"preview": "_base_ = './nerf_unbounded_default.py'\n\nexpname = 'dvgo_garden_unbounded'\n\ndata = dict(\n datadir='./data/360_v2/garde"
},
{
"path": "FourierGrid/configs/nerf_unbounded/garden_single.py",
"chars": 2261,
"preview": "_base_ = './nerf_unbounded_default.py'\nexpname = 'garden_may24_'\nvis = dict(\n height_rate = 0.6 # camera direction fr"
},
{
"path": "FourierGrid/configs/nerf_unbounded/kitchen.py",
"chars": 258,
"preview": "_base_ = './nerf_unbounded_default.py'\n\nexpname = 'dvgo_kitchen_unbounded'\n\ndata = dict(\n datadir='./data/360_v2/kitc"
},
{
"path": "FourierGrid/configs/nerf_unbounded/kitchen_single.py",
"chars": 2192,
"preview": "_base_ = './nerf_unbounded_default.py'\nexpname = 'kitchen_nov29_'\nvis = dict(\n height_rate = 0.6 # camera direction f"
},
{
"path": "FourierGrid/configs/nerf_unbounded/nerf_unbounded_default.py",
"chars": 1134,
"preview": "_base_ = '../default.py'\n\nbasedir = './logs/nerf_unbounded'\n\ndata = dict(\n dataset_type='llff',\n spherify=True,\n "
},
{
"path": "FourierGrid/configs/nerf_unbounded/room.py",
"chars": 350,
"preview": "_base_ = './nerf_unbounded_default.py'\n\nexpname = 'dvgo_room_unbounded'\n\ndata = dict(\n datadir='./data/360_v2/room',\n"
},
{
"path": "FourierGrid/configs/nerf_unbounded/room_single.py",
"chars": 2404,
"preview": "_base_ = './nerf_unbounded_default.py'\nexpname = 'room_nov25_'\nvis = dict(\n height_rate = 0.6 # camera direction frus"
},
{
"path": "FourierGrid/configs/nerf_unbounded/stump.py",
"chars": 341,
"preview": "_base_ = './nerf_unbounded_default.py'\n\nexpname = 'dvgo_stump_unbounded'\n\ndata = dict(\n datadir='./data/360_v2/stump'"
},
{
"path": "FourierGrid/configs/nerf_unbounded/stump_single.py",
"chars": 2063,
"preview": "_base_ = './nerf_unbounded_default.py'\nexpname = 'stump_may23_'\nvis = dict(\n height_rate = 0.6 # camera direction fru"
},
{
"path": "FourierGrid/configs/nsvf/Bike.py",
"chars": 207,
"preview": "_base_ = '../default.py'\n\nexpname = 'dvgo_Bike'\nbasedir = './logs/nsvf_synthetic'\n\ndata = dict(\n datadir='./data/Synt"
},
{
"path": "FourierGrid/configs/nsvf/Lifestyle.py",
"chars": 217,
"preview": "_base_ = '../default.py'\n\nexpname = 'dvgo_Lifestyle'\nbasedir = './logs/nsvf_synthetic'\n\ndata = dict(\n datadir='./data"
},
{
"path": "FourierGrid/configs/nsvf/Palace.py",
"chars": 211,
"preview": "_base_ = '../default.py'\n\nexpname = 'dvgo_Palace'\nbasedir = './logs/nsvf_synthetic'\n\ndata = dict(\n datadir='./data/Sy"
},
{
"path": "FourierGrid/configs/nsvf/Robot.py",
"chars": 209,
"preview": "_base_ = '../default.py'\n\nexpname = 'dvgo_Robot'\nbasedir = './logs/nsvf_synthetic'\n\ndata = dict(\n datadir='./data/Syn"
},
{
"path": "FourierGrid/configs/nsvf/Spaceship.py",
"chars": 217,
"preview": "_base_ = '../default.py'\n\nexpname = 'dvgo_Spaceship'\nbasedir = './logs/nsvf_synthetic'\n\ndata = dict(\n datadir='./data"
},
{
"path": "FourierGrid/configs/nsvf/Steamtrain.py",
"chars": 219,
"preview": "_base_ = '../default.py'\n\nexpname = 'dvgo_Steamtrain'\nbasedir = './logs/nsvf_synthetic'\n\ndata = dict(\n datadir='./dat"
},
{
"path": "FourierGrid/configs/nsvf/Toad.py",
"chars": 207,
"preview": "_base_ = '../default.py'\n\nexpname = 'dvgo_Toad'\nbasedir = './logs/nsvf_synthetic'\n\ndata = dict(\n datadir='./data/Synt"
},
{
"path": "FourierGrid/configs/nsvf/Wineholder.py",
"chars": 219,
"preview": "_base_ = '../default.py'\n\nexpname = 'dvgo_Wineholder'\nbasedir = './logs/nsvf_synthetic'\n\ndata = dict(\n datadir='./dat"
},
{
"path": "FourierGrid/configs/tankstemple/Barn.py",
"chars": 298,
"preview": "_base_ = '../default.py'\n\nexpname = 'dvgo_Barn'\nbasedir = './logs/tanks_and_temple'\n\ndata = dict(\n datadir='./data/Ta"
},
{
"path": "FourierGrid/configs/tankstemple/Barn_lg.py",
"chars": 457,
"preview": "_base_ = '../default.py'\n\nexpname = 'dvgo_Barn_lg'\nbasedir = './logs/tanks_and_temple'\n\ndata = dict(\n datadir='./data"
},
{
"path": "FourierGrid/configs/tankstemple/Caterpillar.py",
"chars": 312,
"preview": "_base_ = '../default.py'\n\nexpname = 'dvgo_Caterpillar'\nbasedir = './logs/tanks_and_temple'\n\ndata = dict(\n datadir='./"
},
{
"path": "FourierGrid/configs/tankstemple/Caterpillar_lg.py",
"chars": 424,
"preview": "_base_ = '../default.py'\n\nexpname = 'dvgo_Caterpillar_lg'\nbasedir = './logs/tanks_and_temple'\n\ndata = dict(\n datadir="
},
{
"path": "FourierGrid/configs/tankstemple/Family.py",
"chars": 302,
"preview": "_base_ = '../default.py'\n\nexpname = 'dvgo_Family'\nbasedir = './logs/tanks_and_temple'\n\ndata = dict(\n datadir='./data/"
},
{
"path": "FourierGrid/configs/tankstemple/Family_lg.py",
"chars": 763,
"preview": "_base_ = '../default.py'\n# model='FourierGrid'\nmodel='DVGO'\nexpname = 'dvgo_Family_lg'\nbasedir = './logs/tanks_and_templ"
},
{
"path": "FourierGrid/configs/tankstemple/Ignatius.py",
"chars": 306,
"preview": "_base_ = '../default.py'\n\nexpname = 'dvgo_Ignatius'\nbasedir = './logs/tanks_and_temple'\n\ndata = dict(\n datadir='./dat"
},
{
"path": "FourierGrid/configs/tankstemple/Ignatius_lg.py",
"chars": 418,
"preview": "_base_ = '../default.py'\n\nexpname = 'dvgo_Ignatius_lg'\nbasedir = './logs/tanks_and_temple'\n\ndata = dict(\n datadir='./"
},
{
"path": "FourierGrid/configs/tankstemple/Truck.py",
"chars": 300,
"preview": "_base_ = '../default.py'\n\nexpname = 'dvgo_Truck'\nbasedir = './logs/tanks_and_temple'\n\ndata = dict(\n datadir='./data/T"
},
{
"path": "FourierGrid/configs/tankstemple/Truck_lg.py",
"chars": 476,
"preview": "_base_ = '../default.py'\n\nexpname = 'dvgo_Truck_lg'\nbasedir = './logs/tanks_and_temple'\n\ndata = dict(\n datadir='./dat"
},
{
"path": "FourierGrid/configs/tankstemple/barn_single.py",
"chars": 3217,
"preview": "_base_ = '../default.py'\nmodel='FourierGrid'\nexpname = 'Barn_nov25_'\nbasedir = './logs/tanks_and_temple'\nvis = dict(\n "
},
{
"path": "FourierGrid/configs/tankstemple/caterpillar_single.py",
"chars": 3228,
"preview": "_base_ = '../default.py'\nmodel='FourierGrid'\nexpname = 'Caterpillar_nov24_'\nbasedir = './logs/tanks_and_temple'\nvis = di"
},
{
"path": "FourierGrid/configs/tankstemple/family_single.py",
"chars": 3497,
"preview": "_base_ = '../default.py'\nexpname = 'family_may23_'\nmodel='FourierGrid'\nbasedir = './logs/tanks_and_temple'\nvis = dict(\n "
},
{
"path": "FourierGrid/configs/tankstemple_unbounded/M60.py",
"chars": 137,
"preview": "_base_ = './tt_default.py'\n\nexpname = 'dvgo_M60_unbounded'\n\ndata = dict(\n datadir='./data/tanks_and_temples/tat_inter"
},
{
"path": "FourierGrid/configs/tankstemple_unbounded/Playground.py",
"chars": 298,
"preview": "_base_ = './tt_default.py'\n\nexpname = 'oct22_dvgo_Playground_unbounded_baseline'\n\nvis = dict(\n height_rate = 0.6 # ca"
},
{
"path": "FourierGrid/configs/tankstemple_unbounded/Train.py",
"chars": 141,
"preview": "_base_ = './tt_default.py'\n\nexpname = 'dvgo_Train_unbounded'\n\ndata = dict(\n datadir='./data/tanks_and_temples/tat_int"
},
{
"path": "FourierGrid/configs/tankstemple_unbounded/Truck.py",
"chars": 137,
"preview": "_base_ = './tt_default.py'\n\nexpname = 'dvgo_Truck_unbounded'\n\ndata = dict(\n datadir='./data/tanks_and_temples/tat_tra"
},
{
"path": "FourierGrid/configs/tankstemple_unbounded/m60_single.py",
"chars": 4019,
"preview": "_base_ = '../default.py'\nexpname = 'm60_may26_'\nmodel='FourierGrid'\nvis = dict(\n height_rate = 0.6 # camera direction"
},
{
"path": "FourierGrid/configs/tankstemple_unbounded/playground_single.py",
"chars": 4550,
"preview": "_base_ = '../default.py'\nexpname = 'oct29_dvgo_Playground_unbounded_baseline'\nvis = dict(\n height_rate = 0.6 # camera"
},
{
"path": "FourierGrid/configs/tankstemple_unbounded/train_single.py",
"chars": 4179,
"preview": "_base_ = '../default.py'\nexpname = 'train_nov21_'\nvis = dict(\n height_rate = 0.6 # camera direction frustrum height\n)"
},
{
"path": "FourierGrid/configs/tankstemple_unbounded/truck_single.py",
"chars": 3448,
"preview": "_base_ = '../default.py'\nexpname = 'truck_may28_'\nvis = dict(\n height_rate = 0.6 # camera direction frustrum height\n)"
},
{
"path": "FourierGrid/configs/tankstemple_unbounded/tt_default.py",
"chars": 1769,
"preview": "_base_ = '../default.py'\nmodel='FourierGrid'\nbasedir = './logs/tanks_and_temple_unbounded'\nvisualize_poses = False\nalpha"
},
{
"path": "FourierGrid/configs/waymo/block_0_llff.py",
"chars": 309,
"preview": "_base_ = './default_waymo.py'\nmodel='FourierGrid'\nexpname = 'sep13_waymo'\n\ndata = dict(\n datadir='data/sep13_block0/d"
},
{
"path": "FourierGrid/configs/waymo/block_0_tt.py",
"chars": 377,
"preview": "_base_ = './tankstemple_base.py'\nmodel='FourierGrid'\nexpname = 'sep15_waymo_5_images_vis_poses'\n\ndata = dict(\n datadi"
},
{
"path": "FourierGrid/configs/waymo/waymo_base.py",
"chars": 2265,
"preview": "_base_ = '../default.py'\nbasedir = './logs/waymo'\nmodel='FourierGrid'\nvisualize_poses = False\nalpha_init = 1e-2 # defau"
},
{
"path": "FourierGrid/configs/waymo/waymo_block.py",
"chars": 899,
"preview": "_base_ = './waymo_base.py'\nmodel='FourierGrid'\ncam_id = 73\nexpname = f'oct99_waymo_{cam_id}_tt'\nvis = dict(\n height_r"
},
{
"path": "FourierGrid/configs/waymo/waymo_no_block.py",
"chars": 5055,
"preview": "_base_ = '../default.py'\nmodel='FourierGrid'\nbasedir = 'logs/waymo'\nvisualize_poses = False\nalpha_init = 1e-4\nstepsize ="
},
{
"path": "FourierGrid/cuda/adam_upd.cpp",
"chars": 2425,
"preview": "#include <torch/extension.h>\n\n#include <vector>\n\n// CUDA forward declarations\n\nvoid adam_upd_cuda(\n torch::Tensor par"
},
{
"path": "FourierGrid/cuda/adam_upd_kernel.cu",
"chars": 4463,
"preview": "#include <torch/extension.h>\n\n#include <cuda.h>\n#include <cuda_runtime.h>\n\n#include <vector>\n\ntemplate <typename scalar_"
},
{
"path": "FourierGrid/cuda/render_utils.cpp",
"chars": 7763,
"preview": "#include <torch/extension.h>\n\n#include <vector>\n\n// CUDA forward declarations\n\nstd::vector<torch::Tensor> infer_t_minmax"
},
{
"path": "FourierGrid/cuda/render_utils_kernel.cu",
"chars": 24807,
"preview": "#include <torch/extension.h>\n\n#include <cuda.h>\n#include <cuda_runtime.h>\n\n#include <vector>\n\n/*\n Points sampling help"
},
{
"path": "FourierGrid/cuda/setup.py",
"chars": 859,
"preview": "import os\nimport pdb\nfrom setuptools import setup, Extension\nfrom torch.utils import cpp_extension\n\n\ndef setup_sources(n"
},
{
"path": "FourierGrid/cuda/total_variation.cpp",
"chars": 824,
"preview": "#include <torch/extension.h>\n\n#include <vector>\n\n// CUDA forward declarations\n\nvoid total_variation_add_grad_cuda(torch:"
},
{
"path": "FourierGrid/cuda/total_variation_kernel.cu",
"chars": 2433,
"preview": "#include <torch/extension.h>\n\n#include <cuda.h>\n#include <cuda_runtime.h>\n\n#include <vector>\n\ntemplate <typename scalar_"
},
{
"path": "FourierGrid/cuda/ub360_utils.cpp",
"chars": 647,
"preview": "#include <torch/extension.h>\n\n#include <vector>\n\n// CUDA forward declarations\n\ntorch::Tensor cumdist_thres_cuda(torch::T"
},
{
"path": "FourierGrid/cuda/ub360_utils_kernel.cu",
"chars": 1330,
"preview": "#include <torch/extension.h>\n\n#include <cuda.h>\n#include <cuda_runtime.h>\n\n#include <vector>\n\n/*\n helper function to s"
},
{
"path": "FourierGrid/dcvgo.py",
"chars": 18098,
"preview": "import os\nimport time\nimport functools\nimport numpy as np\nimport pdb\nimport torch\nimport torch.nn as nn\nimport torch.nn."
},
{
"path": "FourierGrid/dmpigo.py",
"chars": 15654,
"preview": "import os\nimport time\nimport functools\nimport numpy as np\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional"
},
{
"path": "FourierGrid/dvgo.py",
"chars": 30272,
"preview": "import os\nimport time\nimport functools\nimport numpy as np\nimport pdb\nimport torch\nimport torch.nn as nn\nimport torch.nn."
},
{
"path": "FourierGrid/grid.py",
"chars": 11420,
"preview": "import os\nimport time\nimport functools\nimport numpy as np\nimport pdb\nimport torch\nimport torch.nn as nn\nimport torch.nn."
},
{
"path": "FourierGrid/load_everything.py",
"chars": 2339,
"preview": "import torch\nfrom FourierGrid.common_data_loaders.load_common_data import load_common_data\nfrom FourierGrid.load_waymo i"
},
{
"path": "FourierGrid/load_mega.py",
"chars": 10892,
"preview": "'''\nModify from\nhttps://github.com/Kai-46/nerfplusplus/blob/master/data_loader_split.py\n'''\nimport os\nimport pdb\nimport "
},
{
"path": "FourierGrid/load_waymo.py",
"chars": 15736,
"preview": "'''\nModify from\nhttps://github.com/Kai-46/nerfplusplus/blob/master/data_loader_split.py\n'''\nimport os\nimport pdb\nfrom tk"
},
{
"path": "FourierGrid/masked_adam.py",
"chars": 3430,
"preview": "import os\nimport torch\nimport adam_upd_cuda # requires pre-build adam\n## online installation as follows\n# from torch.ut"
},
{
"path": "FourierGrid/pose_utils/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "FourierGrid/pose_utils/image_operators.py",
"chars": 1798,
"preview": "import cv2\nimport pdb\nimport numpy as np\nfrom scipy import stats\n\n\ndef get_bbox_from_img(image, color_thre=1e-2):\n # "
},
{
"path": "FourierGrid/pose_utils/linemod_constants.py",
"chars": 654,
"preview": "import numpy as np\n\n\ndiameters = {\n 'cat': 15.2633,\n 'ape': 9.74298,\n 'benchvise': 28.6908,\n 'bowl': 17.1185"
},
{
"path": "FourierGrid/pose_utils/linemod_evaluator.py",
"chars": 13701,
"preview": "# this code is from rnnpose\nimport pdb\nimport numpy as np\nfrom FourierGrid.pose_utils.linemod_constants import *\nfrom Fo"
},
{
"path": "FourierGrid/pose_utils/model_operations.py",
"chars": 486,
"preview": "import numpy as np\nimport pdb\n\n\ndef get_bb8_of_model(obj_m):\n xmin, ymin, zmin = obj_m[:, 0].min(), obj_m[:, 1].min()"
},
{
"path": "FourierGrid/pose_utils/pose_operators.py",
"chars": 2906,
"preview": "import pdb\nimport numpy as np\nfrom scipy.spatial.transform import Rotation as R\n\n\ndef chordal_distance(R1,R2):\n retur"
},
{
"path": "FourierGrid/pose_utils/projection.py",
"chars": 962,
"preview": "import torch\nimport numpy as np\nfrom FourierGrid.pose_utils.visualization import *\n\n\ndef get_projected_points(cam_pose, "
},
{
"path": "FourierGrid/pose_utils/visualization.py",
"chars": 3745,
"preview": "import os\nimport cv2\nimport pdb\nimport torch\nimport imageio\nimport numpy as np\n\n\ndef visualize_2d_points(points_2d, bg_i"
},
{
"path": "FourierGrid/pycolmap/LICENSE.txt",
"chars": 1084,
"preview": "MIT License\n\nCopyright (c) 2018 True Price, UNC Chapel Hill\n\nPermission is hereby granted, free of charge, to any person"
},
{
"path": "FourierGrid/pycolmap/README.md",
"chars": 257,
"preview": "# pycolmap\nPython interface for COLMAP reconstructions, plus some convenient scripts for loading/modifying/converting re"
},
{
"path": "FourierGrid/pycolmap/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "FourierGrid/pycolmap/pycolmap/__init__.py",
"chars": 263,
"preview": "from pycolmap.pycolmap.camera import Camera\nfrom pycolmap.pycolmap.database import COLMAPDatabase\nfrom pycolmap.pycolmap"
},
{
"path": "FourierGrid/pycolmap/pycolmap/camera.py",
"chars": 8892,
"preview": "# Author: True Price <jtprice at cs.unc.edu>\n\nimport numpy as np\n\nfrom scipy.optimize import root\n\n\n#-------------------"
},
{
"path": "FourierGrid/pycolmap/pycolmap/database.py",
"chars": 10081,
"preview": "import numpy as np\nimport os\nimport sqlite3\n\n\n#-------------------------------------------------------------------------"
},
{
"path": "FourierGrid/pycolmap/pycolmap/image.py",
"chars": 944,
"preview": "# Author: True Price <jtprice at cs.unc.edu>\n\nimport numpy as np\n\n#-----------------------------------------------------"
},
{
"path": "FourierGrid/pycolmap/pycolmap/rotation.py",
"chars": 11595,
"preview": "# Author: True Price <jtprice at cs.unc.edu>\n\nimport numpy as np\n\n#-----------------------------------------------------"
},
{
"path": "FourierGrid/pycolmap/pycolmap/scene_manager.py",
"chars": 27047,
"preview": "# Author: True Price <jtprice at cs.unc.edu>\n\nimport array\nimport numpy as np\nimport os\nimport struct\n\nfrom collections "
},
{
"path": "FourierGrid/pycolmap/tools/colmap_to_nvm.py",
"chars": 2499,
"preview": "import itertools\nimport sys\nsys.path.append(\"..\")\nfloat32\nimport numpy as np\n\nfrom pycolmap import Quaternion, SceneMana"
},
{
"path": "FourierGrid/pycolmap/tools/delete_images.py",
"chars": 943,
"preview": "import sys\nsys.path.append(\"..\")\n\nimport numpy as np\n\nfrom pycolmap import DualQuaternion, Image, SceneManager\n\n\n#------"
},
{
"path": "FourierGrid/pycolmap/tools/impute_missing_cameras.py",
"chars": 5659,
"preview": "import sys\nsys.path.append(\"..\")\n\nimport numpy as np\n\nfrom pycolmap import DualQuaternion, Image, SceneManager\n\n\n#------"
},
{
"path": "FourierGrid/pycolmap/tools/save_cameras_as_ply.py",
"chars": 2821,
"preview": "import sys\nsys.path.append(\"..\")\n\nimport numpy as np\nimport os\n\nfrom pycolmap import SceneManager\n\n\n#-------------------"
},
{
"path": "FourierGrid/pycolmap/tools/transform_model.py",
"chars": 1455,
"preview": "import sys\nsys.path.append(\"..\")\n\nimport numpy as np\n\nfrom pycolmap import Quaternion, SceneManager\n\n\n#-----------------"
},
{
"path": "FourierGrid/pycolmap/tools/write_camera_track_to_bundler.py",
"chars": 1688,
"preview": "import sys\nsys.path.append(\"..\")\n\nimport numpy as np\n\nfrom pycolmap import SceneManager\n\n\n#-----------------------------"
},
{
"path": "FourierGrid/pycolmap/tools/write_depthmap_to_ply.py",
"chars": 4983,
"preview": "import sys\nsys.path.append(\"..\")\n\nimport imageio\nimport numpy as np\nimport os\n\nfrom plyfile import PlyData, PlyElement\nf"
},
{
"path": "FourierGrid/run_colmap2standard.py",
"chars": 8969,
"preview": "from os.path import join as pjoin\nfrom copy import deepcopy\nfrom glob import glob\nimport click\nimport pdb\nimport numpy a"
},
{
"path": "FourierGrid/run_export_bbox.py",
"chars": 1511,
"preview": "import numpy as np\nimport os\nimport pdb\nfrom pathlib import Path\nfrom FourierGrid import utils, dvgo, dcvgo, dmpigo\nfrom"
},
{
"path": "FourierGrid/run_export_coarse.py",
"chars": 877,
"preview": "import torch\nimport os\nimport numpy as np\nfrom FourierGrid.load_everything import load_existing_model\n\n\ndef run_export_c"
},
{
"path": "FourierGrid/run_gen_cam_paths.py",
"chars": 7394,
"preview": "import pdb, os, cv2\nfrom turtle import onkey\nfrom random import sample\nimport imageio\nfrom FourierGrid import utils\nimpo"
},
{
"path": "FourierGrid/run_gtk_analysis.py",
"chars": 20168,
"preview": "import matplotlib\nimport matplotlib.pylab as pylab\nimport matplotlib.pyplot as plt\nfrom matplotlib.lines import Line2D\nf"
},
{
"path": "FourierGrid/run_render.py",
"chars": 15951,
"preview": "import os\nimport pdb\nimport imageio\nimport torch\nfrom tqdm import tqdm, trange\nimport numpy as np\nfrom FourierGrid impor"
},
{
"path": "FourierGrid/run_train.py",
"chars": 20091,
"preview": "import time\nimport torch\nimport torch.nn.functional as F\nimport os,pdb\nimport copy\nimport numpy as np\nfrom tqdm import t"
},
{
"path": "FourierGrid/tools/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "FourierGrid/tools/colmap_utils/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "FourierGrid/tools/colmap_utils/colmap_read_model.py",
"chars": 13315,
"preview": "# Source: https://github.com/Fyusion/LLFF\n# Copyright (c) 2018, ETH Zurich and UNC Chapel Hill.\n# All rights reserved.\n#"
},
{
"path": "FourierGrid/tools/colmap_utils/colmap_wrapper.py",
"chars": 3258,
"preview": "# Source: https://github.com/Fyusion/LLFF\nimport os\nimport subprocess\nimport pdb\n\n\n# $ DATASET_PATH=/path/to/dataset\n\n# "
},
{
"path": "FourierGrid/tools/colmap_utils/pose_utils.py",
"chars": 8325,
"preview": "# Source: https://github.com/Fyusion/LLFF\nimport numpy as np\nimport os\nimport sys\nimport pdb\nimport json\nimport imageio\n"
},
{
"path": "FourierGrid/tools/imgs2poses.py",
"chars": 1365,
"preview": "# Modified from https://github.com/Fyusion/LLFF\nimport os\nimport sys\nimport glob\n\nfrom colmap_utils.pose_utils import ge"
},
{
"path": "FourierGrid/tools/vis_train.py",
"chars": 1778,
"preview": "import argparse\nimport numpy as np\nimport open3d as o3d\n\nparser = argparse.ArgumentParser(formatter_class=argparse.Argum"
},
{
"path": "FourierGrid/tools/vis_volume.py",
"chars": 2905,
"preview": "import argparse\nimport numpy as np\nimport open3d as o3d\n\nparser = argparse.ArgumentParser(formatter_class=argparse.Argum"
},
{
"path": "FourierGrid/trajectory_generators/__init__.py",
"chars": 0,
"preview": ""
}
]
// ... and 73 more files (download for full content)
About this extraction
This page contains the full source code of the dvlab-research/BlockNeRFPytorch GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 273 files (2.3 MB), approximately 614.5k tokens, and a symbol index with 613 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.