Showing preview only (522K chars total). Download the full file or copy to clipboard to get everything.
Repository: CyberAgentAILab/SuperNormal
Branch: main
Commit: 09e26150f7e8
Files: 135
Total size: 479.3 KB
Directory structure:
gitextract_10qul1w_/
├── .gitignore
├── LICENSE
├── README.md
├── __init__.py
├── config/
│ ├── diligent.conf
│ └── own_objects.conf
├── create_env.sh
├── data_capture_and_preprocessing/
│ ├── README.md
│ ├── gather_and_convert_normal_map.py
│ ├── iPhone_mvps_data_preprocessing.py
│ ├── metashape2neus.py
│ ├── metashape2neus2_json_and_images.py
│ └── sam_mvps.py
├── download_data.sh
├── exp_runner.py
├── models/
│ ├── cd_and_fscore.py
│ ├── dataset_loader.py
│ ├── fields.py
│ └── renderer.py
├── run_diligent.sh
├── run_own_object.sh
├── third_parties/
│ └── nerfacc-0.3.5/
│ └── nerfacc-0.3.5/
│ ├── .github/
│ │ └── workflows/
│ │ ├── building.yml
│ │ ├── code_checks.yml
│ │ ├── cuda/
│ │ │ ├── cu101-Linux-env.sh
│ │ │ ├── cu101-Linux.sh
│ │ │ ├── cu101-Windows-env.sh
│ │ │ ├── cu101-Windows.sh
│ │ │ ├── cu102-Linux-env.sh
│ │ │ ├── cu102-Linux.sh
│ │ │ ├── cu102-Windows-env.sh
│ │ │ ├── cu102-Windows.sh
│ │ │ ├── cu111-Linux-env.sh
│ │ │ ├── cu111-Linux.sh
│ │ │ ├── cu111-Windows-env.sh
│ │ │ ├── cu111-Windows.sh
│ │ │ ├── cu113-Linux-env.sh
│ │ │ ├── cu113-Linux.sh
│ │ │ ├── cu113-Windows-env.sh
│ │ │ ├── cu113-Windows.sh
│ │ │ ├── cu115-Linux-env.sh
│ │ │ ├── cu115-Linux.sh
│ │ │ ├── cu115-Windows-env.sh
│ │ │ ├── cu115-Windows.sh
│ │ │ ├── cu116-Linux-env.sh
│ │ │ ├── cu116-Linux.sh
│ │ │ ├── cu116-Windows-env.sh
│ │ │ ├── cu116-Windows.sh
│ │ │ ├── cu117-Linux-env.sh
│ │ │ ├── cu117-Linux.sh
│ │ │ ├── cu117-Windows-env.sh
│ │ │ └── cu117-Windows.sh
│ │ └── publish.yml
│ ├── .gitignore
│ ├── .gitmodules
│ ├── .pre-commit-config.yaml
│ ├── .readthedocs.yaml
│ ├── CMakeLists.txt
│ ├── LICENSE
│ ├── MANIFEST.in
│ ├── README.md
│ ├── docs/
│ │ ├── Makefile
│ │ ├── requirements.txt
│ │ └── source/
│ │ ├── _static/
│ │ │ └── css/
│ │ │ └── readthedocs.css
│ │ ├── apis/
│ │ │ ├── generated/
│ │ │ │ ├── nerfacc.accumulate_along_rays.rst
│ │ │ │ ├── nerfacc.pack_data.rst
│ │ │ │ ├── nerfacc.ray_aabb_intersect.rst
│ │ │ │ ├── nerfacc.ray_resampling.rst
│ │ │ │ ├── nerfacc.render_transmittance_from_alpha.rst
│ │ │ │ ├── nerfacc.render_transmittance_from_density.rst
│ │ │ │ ├── nerfacc.render_visibility.rst
│ │ │ │ ├── nerfacc.render_weight_from_alpha.rst
│ │ │ │ ├── nerfacc.render_weight_from_density.rst
│ │ │ │ ├── nerfacc.unpack_data.rst
│ │ │ │ └── nerfacc.unpack_info.rst
│ │ │ ├── grid.rst
│ │ │ ├── rendering.rst
│ │ │ └── utils.rst
│ │ ├── conf.py
│ │ ├── examples/
│ │ │ ├── dnerf.rst
│ │ │ ├── ngp.rst
│ │ │ ├── unbounded.rst
│ │ │ └── vanilla.rst
│ │ └── index.rst
│ ├── examples/
│ │ ├── datasets/
│ │ │ ├── __init__.py
│ │ │ ├── dnerf_synthetic.py
│ │ │ ├── nerf_360_v2.py
│ │ │ ├── nerf_synthetic.py
│ │ │ └── utils.py
│ │ ├── radiance_fields/
│ │ │ ├── __init__.py
│ │ │ ├── mlp.py
│ │ │ └── ngp.py
│ │ ├── requirements.txt
│ │ ├── train_mlp_dnerf.py
│ │ ├── train_mlp_nerf.py
│ │ ├── train_ngp_nerf.py
│ │ └── utils.py
│ ├── nerfacc/
│ │ ├── __init__.py
│ │ ├── cdf.py
│ │ ├── contraction.py
│ │ ├── cuda/
│ │ │ ├── __init__.py
│ │ │ ├── _backend.py
│ │ │ └── csrc/
│ │ │ ├── cdf.cu
│ │ │ ├── contraction.cu
│ │ │ ├── include/
│ │ │ │ ├── helpers_contraction.h
│ │ │ │ ├── helpers_cuda.h
│ │ │ │ └── helpers_math.h
│ │ │ ├── intersection.cu
│ │ │ ├── pack.cu
│ │ │ ├── pybind.cu
│ │ │ ├── ray_marching.cu
│ │ │ ├── render_transmittance.cu
│ │ │ ├── render_transmittance_cub.cu
│ │ │ └── render_weight.cu
│ │ ├── grid.py
│ │ ├── intersection.py
│ │ ├── losses.py
│ │ ├── pack.py
│ │ ├── ray_marching.py
│ │ ├── sampling.py
│ │ ├── version.py
│ │ └── vol_rendering.py
│ ├── scripts/
│ │ ├── run_aws_listing.py
│ │ ├── run_dev_checks.py
│ │ └── run_profiler.py
│ ├── setup.cfg
│ ├── setup.py
│ └── tests/
│ ├── test_contraction.py
│ ├── test_grid.py
│ ├── test_intersection.py
│ ├── test_loss.py
│ ├── test_pack.py
│ ├── test_ray_marching.py
│ ├── test_rendering.py
│ └── test_resampling.py
└── utilities/
└── utils.py
================================================
FILE CONTENTS
================================================
================================================
FILE: .gitignore
================================================
data/
exp/
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
cover/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
.pybuilder/
target/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# pyenv
# For a library or package, you might want to ignore these files since the code is
# intended to run in multiple environments; otherwise, check them in:
# .python-version
# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
#Pipfile.lock
# poetry
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
# This is especially recommended for binary packages to ensure reproducibility, and is more
# commonly ignored for libraries.
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
#poetry.lock
# pdm
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
#pdm.lock
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
# in version control.
# https://pdm.fming.dev/#use-with-ide
.pdm.toml
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
__pypackages__/
# Celery stuff
celerybeat-schedule
celerybeat.pid
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
# Pyre type checker
.pyre/
# pytype static type analyzer
.pytype/
# Cython debug symbols
cython_debug/
# PyCharm
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
# and can be added to the global gitignore or merged into this file. For a more nuclear
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
.idea/
================================================
FILE: LICENSE
================================================
MIT License
Copyright (c) 2024 CyberAgent AI Lab
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
================================================
FILE: README.md
================================================
<h2 align="center">SuperNormal: Neural Surface Reconstruction via Multi-View Normal Integration</h2>
<h4 align="center">
<a href="https://xucao-42.github.io/homepage/"><strong>Xu Cao</strong></a>
·
<a href="https://taketomitakafumi.sakura.ne.jp/web/en/"><strong>Takafumi Taketomi</strong></a>
<br>
CyberAgent </h4>
<h4 align="center"><a href="https://cvpr.thecvf.com/">CVPR 2024 </a></h3>
<p align="center">
<br>
<a href="https://arxiv.org/abs/2312.04803">
<img src='https://img.shields.io/badge/arXiv-Paper-981E32?style=for-the-badge&Color=B31B1B' alt='arXiv PDF'>
</a>
[//]: # ( <a href='https://xucao-42.github.io/mvas_homepage/'>)
[//]: # ( <img src='https://img.shields.io/badge/MVAS-Project Page-5468FF?style=for-the-badge' alt='Project Page'></a>)
</p>
### Update
- **2024/09/30**: Real-world raw data and step-by-step data pre-processing instructions are available. See [here](./data_capture_and_preprocessing/README.md).
<div align="center">
<img src="./media/teaser.png" alt="Teaser" width="100%">
Fast and fine-grained 3D reconstruction from multi-view surface normal maps.
</div>
### Quick Start
Code was tested on Ubuntu 18.04 (WSL2) using Python 3.8, PyTorch 2.1.0, and CUDA 11.8 on an Nvidia RTX4090Ti (24GB).
**Before started, please ensure CUDA is installed in your environment ([11.8 can be found here](https://developer.nvidia.com/cuda-11-8-0-download-archive?target_os=Linux&target_arch=x86_64&Distribution=WSL-Ubuntu&target_version=2.0&target_type=deb_local)).**
It is required by [tiny-cuda-nn](https://github.com/NVlabs/tiny-cuda-nn).
<details><summary> You should see something like the following after typing `nvcc --version` </summary>
```commandline
nvcc: NVIDIA (R) Cuda compiler driver
Copyright (c) 2005-2022 NVIDIA Corporation
Built on Wed_Sep_21_10:33:58_PDT_2022
Cuda compilation tools, release 11.8, V11.8.89
Build cuda_11.8.r11.8/compiler.31833905_0
```
</details>
Clone the repository and prepare the conda environment:
```commandline
git clone https://github.com/CyberAgentAILab/SuperNormal.git
cd SuperNormal
. ./create_env.sh
```
Download data (~1.8GB):
```commandline
./download_data.sh
```
Run on the DiLiGenT-MV benchmark objects or on our captured objects:
```commandline
./run_diligent.sh # Training should take about 50 seconds per object
```
or
```commandline
./run_own_object.sh # Training should take about 5 minutes per object
```
Results are saved under `./exp`.
NOTE: If RuntimeError like below occurs, `apt install ninja-build` may resolve the error.
```
RuntimeError: Ninja is required to load C++ extensions
```
### Hyperparameter tuning tips
Training hyperparameters are defined in `./configs/*.conf`.
Some important hyperparameters are:
- `dataset.normal_dir`: You can choose normal maps estimated by different methods as input for DiLiGenT-MV benchmark objects.
- `train.end_iter`: The number of iterations for training. Should be adjusted according to the number of views and normal map resolutions.
- `train.increase_bindwidth_every`: A strategy used in [Neuralangelo](https://research.nvidia.com/labs/dir/neuralangelo/) to progressively activate finer hash grid during training. Less than `end_iter`/`model.encoding.n_levels` should be fine.
- `train.batch_size`: Number of patches in each batch for training. Should be adjusted according to the GPU memory.
- `train.patch_size`: Better to be fixed to 3, i.e., each patch is 3x3. Large patch size will cause inaccurate volume rendering results for boudary pixels in a patch.
### Modifications to NerfAcc
We add several functions to the original [NerfAcc](https://www.nerfacc.com) to adapt it to patch-based volume rendering.
The key new functions (which are indicated by `patch_based` in function name) are in
[third_parties/nerfacc-0.3.5/nerfacc-0.3.5/nerfacc/cuda/csrc/render_weight.cu/](./third_parties/nerfacc-0.3.5/nerfacc-0.3.5/nerfacc/cuda/csrc/render_weight.cu)
and [third_parties/nerfacc-0.3.5/nerfacc-0.3.5/nerfacc/vol_rendering.py](./third_parties/nerfacc-0.3.5/nerfacc-0.3.5/nerfacc/vol_rendering.py).
### Acknowledgement
This repo is built up on [NeuS](https://github.com/Totoro97/NeuS) and benefits from the amazing [tiny-cuda-nn](https://github.com/NVlabs/tiny-cuda-nn) and [NerfAcc](https://www.nerfacc.com).
We also learned a lot from [instant-nsr-pl](https://github.com/bennyguo/instant-nsr-pl).
### Citation
If you find our work useful in your research, please consider citing:
```bibtex
@inproceedings{supernormal2024cao,
title={SuperNormal: {N}eural Surface Reconstruction via Multi-View Normal Integration},
author={Cao Xu and Taketomi Takafumi},
booktitle={CVPR},
year={2024}
}
```
================================================
FILE: __init__.py
================================================
================================================
FILE: config/diligent.conf
================================================
general {
dataset_class = models.dataset_loader.Dataset
renderer_class = models.renderer.NeuSRenderer
base_exp_dir = ./exp/diligent_mv/CASE_NAME
recording = [
./,
./models
]
}
dataset {
data_dir = data/diligent_mv_normals/CASE_NAME/
normal_dir = normal_world_space_sdmunips # choose normal maps estimated by different methods, should be in the world space
cameras_name = cameras_sphere.npz
exclude_views = [0, 4, 8, 12, 16] # index of views to exclude for test purpose, 0-based
upsample_factor = 1
}
train {
learning_rate = 5e-4
learning_rate_alpha = 0.05
end_iter = 5000
increase_bindwidth_every = 350 # following neuralangelo's strategy
gradient_method = dfd # dfd or fd or ad, for directional finite difference, finite difference, and auto-differentiation
batch_size = 2048
patch_size = 3 # i.e., each training step samples 2048 patches of 3x3 pixels
warm_up_end = 50
use_white_bkgd = False
loss_type = l2 # for normal loss
normal_weight = 1
eikonal_weight = 1
mask_weight = 1
}
val {
save_freq = 1000
val_normal_freq = 5001
val_normal_resolution_level = 1
gradient_method = dfd # dfd or fd or ad, can be different from training
val_mesh_freq = 10000
val_mesh_res = 512
report_freq = 100
eval_metric_freq = 5000
}
model {
sdf_network {
d_out = 1
d_in = 3
d_hidden = 64
n_layers = 1
skip_in = [-1] # -1 for no skip connection
bias = 0.6
geometric_init = True
weight_norm = True
input_concat = True # concat input positions and encoded features
}
variance_network {
init_val = 0.5
}
ray_marching {
start_step_size = 1e-2
end_step_size = 1e-3
occ_threshold = 0.1
occ_sigmoid_k = 80.0
occ_resolution = 128
occ_update_freq = 8 # batches
}
encoding{
otype=HashGrid,
n_levels=14
n_features_per_level=2
log2_hashmap_size=19
base_resolution=32
per_level_scale=1.3195079107728942
}
}
================================================
FILE: config/own_objects.conf
================================================
general {
dataset_class = models.dataset_loader.Dataset
renderer_class = models.renderer.NeuSRenderer
base_exp_dir = ./exp/own_objects/CASE_NAME
recording = [
./,
./models
]
}
dataset {
data_dir = data/own_objects_normals/CASE_NAME/
normal_dir = normal_world_space_sdmunips
cameras_name = cameras_sphere.npz
exclude_views = [] # index of views to exclude, 0-based
upsample_factor = 1
}
train {
learning_rate = 5e-4
learning_rate_alpha = 0.05
end_iter = 30000
increase_bindwidth_every = 2000 # following neuralangelo's strategy
gradient_method = dfd # dfd or fd or ad, for directional finite difference, finite difference, and auto-differentiation
batch_size = 2048
patch_size = 3 # i.e., each training step samples 2048 patches of 3x3 pixels
warm_up_end = 500
use_white_bkgd = False
loss_type = l2 # for normal loss
normal_weight = 1
eikonal_weight = 1
mask_weight = 1
}
val {
save_freq = 10000
val_normal_freq = 30000
val_normal_resolution_level = 2
gradient_method = dfd # dfd or fd or ad, can be different from training
val_mesh_freq = 30000
val_mesh_res = 1024
report_freq = 100
eval_metric_freq = 30000
}
model {
sdf_network {
d_out = 1
d_in = 3
d_hidden = 64
n_layers = 1
skip_in = [-1]
bias = 0.8
geometric_init = True
weight_norm = True
input_concat = True # concat input positions and encoded features
}
variance_network {
init_val = 0.5
}
ray_marching
{
start_step_size = 1e-2
end_step_size = 1e-3
occ_threshold = 0.1
occ_sigmoid_k = 80.0
occ_resolution = 128
occ_update_freq = 8 # batches
}
encoding{
otype=HashGrid,
n_levels=14
n_features_per_level=2
log2_hashmap_size=19
base_resolution=32
per_level_scale=1.3195079107728942
}
}
================================================
FILE: create_env.sh
================================================
conda deactivate
conda remove -y -n sn --all
conda create -y -n sn python=3.8
conda activate sn
pip install torch==2.1.0 torchvision==0.16.0 torchaudio==2.1.0 --index-url https://download.pytorch.org/whl/cu118
# install tiny-cuda-nn
export PATH="/usr/local/cuda/bin:$PATH"
export LIBRARY_PATH="/usr/local/cuda/lib64/stubs:$LIBRARY_PATH"
pip install git+https://github.com/NVlabs/tiny-cuda-nn/@2ec562e853e6f482b5d09168705205f46358fb39#subdirectory=bindings/torch
pip install -e ./third_parties/nerfacc-0.3.5/nerfacc-0.3.5/
pip install opencv-python==4.8.1.78 trimesh==3.23.5 open3d==0.17 pyvista==0.42.3 scipy==1.10.1 scikit-image==0.21.0 pyhocon==0.3.59 pyexr==0.3.10 tensorboard==2.14.0 icecream==2.1.3 PyMCubes==0.1.4 pyembree==0.2.11
================================================
FILE: data_capture_and_preprocessing/README.md
================================================
This is a step-by-step guide to preprocess the raw images captured by an iPhone for the MVPS task.
You can download our raw images using the following command *(~6 GB per object)*.
```
gdown 'https://drive.google.com/file/d/1BcCuZR0C-snmCNf8iGhkFgkQ6arfcQ-L/view?usp=sharing' --fuzzy
unzip flower_girl.zip
rm flower_girl.zip
gdown 'https://drive.google.com/file/d/12QzgRbOjBSx295BS4zihnOjcdYh7ZaP9/view?usp=sharing' --fuzzy
unzip lion.zip
rm lion.zip
gdown 'https://drive.google.com/file/d/1cvKbI5VvDhsuA4a06rYqqoAtQd8GtyeI/view?usp=sharing' --fuzzy
unzip dog.zip
rm dog.zip
```
## File structure
You should have the following file structure under each object's folder:
```
- RAW
- mask
- cameras.xml
```
The `RAW` folder contains all the DNG images captured by an iPhone.
The `mask` folder contains the foreground masks for each view.
The `cameras.xml` contains the calibrated camera parameters using [Metashape](https://oakcorp.net/agisoft/download/).
## Step-by-step data pre-processing
First we convert the DNG images to PNG file format.
```
# pip install rawpy
python iPhone_mvps_data_preprocessing.py --data_dir <path/to/obj_folder>
```
Now the file structure looks like this
```
- RAW
- mvps_png_full
- sfm_png_full
- mask
- cameras.xml
```
The `mvps_png_full` folder contains the pre-processed images for photometric stereo, and the `sfm_png_full` folder contains the images for camera calibration using Structure from Motion.
In each view, we first take an image in ambient light and then additionally illuminate the object with an active light source.
So the first image in each view is collected in `sfm_png_full`.
### Mask preparation
Now we prepare the foreground masks for each view.
We used SAM to interactively segment the foreground objects.
Please install SAM according to the [official instructions](https://github.com/facebookresearch/segment-anything).
After installation, run the following command to segment the foreground objects for all views:
```
python sam_mvps.py --data_dir <path/to/obj_folder/mvps_png_full> --checkpoint <path/to/sam_vit_h_4b8939.pth>
```
This will pop up a window where you can interactively segment the foreground objects.
Select points on the object to segment the foreground object, and press `Esc` to check the intermediate results.
Continue to select points until you are satisfied with the segmentation results, and press `Enter` to save the mask.
The process will be repeated for all views.
The same mask will be saved in two places: `obj_folder/mask` and the corresponding folder containing the image from the same viewpoint.
The latter will be used for normal map estimation.
### Camera calibration
In [MetaShape](https://oakcorp.net/agisoft/download/), import the images in the `sfm_png_full` folder and run the camera calibration process.
```
[Workflow] -> [Add Folder] -> select `sfm_png_full` -> select single cameras -> [Workflow] -> [Align Photos]
```
After camera calibration, export the camera parameters to `cameras.xml`.
```
[File] -> [Export] -> [Export Cameras]
```
The resulting `cameras.xml` file is what we have put in the object folder.
### Normal map estimation
Install [SDM-UniPS](https://github.com/satoshi-ikehata/SDM-UniPS-CVPR2023) and run the following command to generate the normal maps for each view:
```
python <path/to/sdm_unips/main.py> --session_name YOUR_SESSION_NAME --test_dir <path/to/obj_folder/mvps_png_full> --checkpoint <path/to/sdm_unips_checkpoint_dir> --scalable --target normal
```
Tips: Prepare the mask for each view to improve the normal estimation results. This should be done when you have completed the previous mask segmentation step.
The original SDM-UniPS code outputs normal maps in the PNG format. You can instead get EXR format by replacing [this line](https://github.com/satoshi-ikehata/SDM-UniPS-CVPR2023/blob/96e68f353173c2ae85bfe609e4728a19a2f8c92e/sdm_unips/modules/builder/builder.py#L162) with the following one:
```
pyexr.write(f'{testdata.data.data_workspace}/normal.exr', nout)
```
Remember to install the [pyexr](https://github.com/tvogels/pyexr) package and import it in the file.
After normal estimation, we collect the normal maps in the same folder.
Since SDM-UniPS estimates normal maps in camera space, we also convert them to the world space using the camera parameters from the previous step.
```
python gather_and_convert_normal_map.py --data_dir <path/to/obj_folder> --sdm_unips_result_dir <path/to/YOUR_SESSION_NAME/results>
```
The file structure is now as follows:
```
- RAW
- mvps_png_full
- sfm_png_full
- mask
- normal_camera_space_sdmunips
- normal_world_space_sdmunips
- cameras.xml
- results # if your SDM-UniPS output is in this folder
```
### Convert camera parameters to NeuS format
The last step is to convert the camera parameters to the NeuS format.
```
python metashape2neus.py --xml_path <path/to/obj_folder/cameras.xml>
```
This will create a `cameras_sphere.npz` file in the same folder as `cameras.xml`.
We also provide the converter to NeuS2 format. Check `metashape2neus2_json_and_images.py` for more details.
## Tips for capturing your own data
We used the iPhone's built-in camera app to take the images. Here are some tips for successful reconstruction:
- Use a tripod to stabilize the camera.
- Use a remote shutter release to avoid camera shake.
- Keep the same focus point in each view. On iPhone, you can press and hold the screen to lock the focus point.
- Use a white/black background to simplify the segmentation process.
- Use a turntable to capture the object from different angles.
- Place the object on a textured surface to help the Structure from Motion process.
- Place the object in the center of the image.
- We used a [video light](https://www.ulanzi.com/collections/lighting/products/mini-led-video-light-ulanzi-vl49-1672) to illuminate the object from different angles in each view. Other light sources like a ring light/flashlight may also work.
- In each view, vary the light source's position sufficiently around the camera. We used 12 different light positions in our setup.
- Reduce the exposure if the captured images are overexposed.
The above capture process can be done with off-the-shelf equipment, but it is tedious.
It would be more convenient if you could build a custom rig to automate the capture process, such as [this example](https://youtu.be/zyEw-1QUlkU?si=8RvYC23emoP8TXrU).
================================================
FILE: data_capture_and_preprocessing/gather_and_convert_normal_map.py
================================================
import os
import cv2
import pyexr
from glob import glob
import numpy as np
import shutil
from bs4 import BeautifulSoup # $ pip install beautifulsoup4 lxml
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("--sdm_unips_result_dir", type=str, default="../../SDM-UniPS-CVPR2023/flower_girl/results")
parser.add_argument("--data_dir", type=str, default="./flower_girl")
args = parser.parse_args()
xml_path = os.path.join(args.data_dir, "cameras.xml")
obj_name = os.path.basename(args.data_dir)
num_views = len(glob(os.path.join(args.sdm_unips_result_dir, "view_*.data")))
normal_map_camera_dir = os.path.join(args.data_dir, "normal_camera_space_sdmunips")
normal_map_world_dir = os.path.join(args.data_dir, "normal_world_space_sdmunips")
# create directories
os.makedirs(normal_map_camera_dir, exist_ok=True)
os.makedirs(normal_map_world_dir, exist_ok=True)
with open(xml_path, "r") as f:
xml_data = f.read()
bs_data = BeautifulSoup(xml_data, "xml")
b_unique = bs_data.find_all('camera')
for tag in b_unique:
img_name = tag.get("label")
view_idx = int(img_name.split("_")[-1])
# camera to world transform
C2W = np.array([float(i) for i in tag.find("transform").text.split(" ")]).reshape((4, 4))
normal_map_all = []
normal_map_path_all = []
for i in range(num_views):
view_dir = os.path.join(args.sdm_unips_result_dir, f"view_{i:02d}.data")
for tag in b_unique:
img_name = tag.get("label")
view_idx = int(img_name.split("_")[-1])
# camera to world transform
if view_idx == i:
C2W = np.array([float(i) for i in tag.find("transform").text.split(" ")]).reshape((4, 4))
R = C2W[:3, :3]
break
if os.path.exists(view_dir):
# copy normal map
normal_map_file = os.path.join(view_dir, "normal.exr")
new_normal_map_file = os.path.join(normal_map_camera_dir, f"{i:02d}.exr")
shutil.copy(normal_map_file, new_normal_map_file)
# convert normal map to world space
normal_map_camera = pyexr.read(new_normal_map_file)
normal_map_camera[..., [1, 2]] *= -1 # revert y and z axis to match opencv conversion, X right, Y down, Z front
H, W = normal_map_camera.shape[:2]
normal_world = (R @ normal_map_camera.reshape(-1, 3).T).T.reshape([H, W, 3])
pyexr.write(os.path.join(normal_map_world_dir, f"{i:02d}.exr"), normal_world)
================================================
FILE: data_capture_and_preprocessing/iPhone_mvps_data_preprocessing.py
================================================
import rawpy, os
from glob import glob
import cv2
import numpy as np
import os
from tqdm import tqdm
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("--data_dir", type=str, default="./flower_girl")
parser.add_argument("--num_img_per_view", type=int, default=13)
arg = parser.parse_args()
dng_list = glob(os.path.join(arg.data_dir, "RAW", "*.DNG"))
dng_list.sort()
num_image_per_view = arg.num_img_per_view
num_view = len(dng_list) // num_image_per_view
resize_factor = 1 # resize the png image to 1/2, 1/4, or 1
if resize_factor == 1/2:
sfm_data_dir = os.path.join(arg.data_dir, "sfm_png_half")
mvps_data_dir = os.path.join(arg.data_dir, "mvps_png_half")
elif resize_factor == 1/4:
sfm_data_dir = os.path.join(arg.data_dir, "sfm_png_quarter")
mvps_data_dir = os.path.join(arg.data_dir, "mvps_png_quarter")
elif resize_factor == 1:
mvps_data_dir = os.path.join(arg.data_dir, "mvps_png_full")
sfm_data_dir = os.path.join(arg.data_dir, "sfm_png_full")
os.makedirs(sfm_data_dir, exist_ok=True)
os.makedirs(mvps_data_dir, exist_ok=True)
for view_idx in tqdm(range(num_view)):
view_dir = os.path.join(mvps_data_dir, f"view_{view_idx:02d}.data")
if os.path.exists(view_dir):
continue
os.makedirs(view_dir, exist_ok=True)
view_dng_list = dng_list[view_idx * num_image_per_view: (view_idx + 1) * num_image_per_view]
for dng_idx, dng_path in enumerate(view_dng_list):
with rawpy.imread(dng_path) as raw:
rgb = raw.postprocess(no_auto_bright=True, output_bps=16)[..., ::-1].astype(np.float32)
rgb = rgb.astype(np.uint16)
rgb_resized = cv2.resize(rgb, (0, 0), fx=resize_factor, fy=resize_factor)
# choose the first image in each view for SfM
if dng_idx == 0:
cv2.imwrite(os.path.join(sfm_data_dir, f"{view_idx:02d}.png"), rgb_resized)
cv2.imwrite(os.path.join(view_dir, f"L{dng_idx:02d}.png"), rgb_resized)
================================================
FILE: data_capture_and_preprocessing/metashape2neus.py
================================================
import os.path
import xml
from bs4 import BeautifulSoup # pip install beautifulsoup4 lxml
import numpy as np
# details of camera normalization can be found in Sec. C.3 in https://openaccess.thecvf.com/content/CVPR2023/supplemental/Cao_Multi-View_Azimuth_Stereo_CVPR_2023_supplemental.pdf
def normalize_camera(R_list, t_list, camera2object_ratio=3):
A_camera_normalize = 0
b_camera_normalize = 0
camera_center_list = []
for view_idx in range(len(R_list)):
R = R_list[view_idx]
t = t_list[view_idx]
camera_center = - R.T @ t # in world coordinate
camera_center_list.append(camera_center)
vi = R[2][:, None] # the camera's principal axis in the world coordinates
Vi = vi @ vi.T
A_camera_normalize += np.eye(3) - Vi
b_camera_normalize += camera_center.T @ (np.eye(3) - Vi)
offset = np.linalg.lstsq(A_camera_normalize, np.squeeze(b_camera_normalize), rcond=None)[0]
camera_center_dist_list = [np.sqrt(np.sum((np.squeeze(c) - offset) ** 2))
for c in camera_center_list]
scale = np.max(camera_center_dist_list) / camera2object_ratio
return offset, scale
def make4x4(P):
assert P.shape[-1] == 4 or P.shape[-1] == 3
assert len(P.shape) == 2
assert P.shape[0] == 3 or P.shape[0] == 4
ret = np.eye(4)
ret[:P.shape[0], :P.shape[1]] = P
return ret
class MetashapePoseLoader:
def __init__(self, xml_path, camera2object_ratio):
with open(xml_path, "r") as f:
xml_data = f.read()
bs_data = BeautifulSoup(xml_data, "xml")
c_unique = bs_data.find_all('resolution')
img_width = int(c_unique[0].get("width"))
img_height = int(c_unique[0].get("height"))
c_intrinsics = bs_data.find_all('calibration')
f = float(c_intrinsics[0].find("f").text)
cx_offset = float(c_intrinsics[0].find("cx").text)
cy_offset = float(c_intrinsics[0].find("cy").text)
K = np.array([[f, 0, (img_width-1)/2 + cx_offset],
[0, f, (img_height-1)/2 + cy_offset],
[0, 0, 1]])
b_unique = bs_data.find_all('camera')
R_list = []
t_list = []
C2W_list = []
camera_sphere = dict()
for tag in b_unique:
img_name = tag.get("label")
view_idx = int(img_name.split("_")[-1])
# camera to world transform
C2W = np.array([float(i) for i in tag.find("transform").text.split(" ")]).reshape((4, 4))
C2W_list.append(C2W)
assert int(img_name) == view_idx
W2C = np.linalg.inv(C2W)
R_list.append(W2C[:3, :3])
t_list.append(W2C[:3, 3])
camera_sphere[f"world_mat_{view_idx}"] = make4x4(K) @ W2C
offset, scale = normalize_camera(R_list, t_list, camera2object_ratio=camera2object_ratio)
print("offset", offset, "scale", scale)
num_views = len(C2W_list)
scale_mat = np.eye(4)
scale_mat[:3, :3] *= scale
scale_mat[:3, 3] = offset
for im_idx in range(num_views):
camera_sphere[f"scale_mat_{im_idx}"] = scale_mat
data_dir = os.path.dirname(xml_path)
np.savez(os.path.join(data_dir, 'cameras_sphere.npz'), **camera_sphere)
if __name__=="__main__":
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("--xml_path", type=str, required=True)
parser.add_argument("--ratio", type=float, default=10)
args = parser.parse_args()
MetashapePoseLoader(args.xml_path, camera2object_ratio=args.ratio)
================================================
FILE: data_capture_and_preprocessing/metashape2neus2_json_and_images.py
================================================
from glob import glob
import os
import numpy as np
import cv2
from bs4 import BeautifulSoup
from metashape2neus import normalize_camera, make4x4
import json
import argparse
def create_json_file(data, filename):
with open(filename, 'w') as f:
json.dump(data, f, indent=4)
parser = argparse.ArgumentParser()
parser.add_argument('--data_dir', type=str, default="./flower_girl")
arg = parser.parse_args()
data_dir = os.path.join(arg.data_dir, "sfm_png_full")
mask_dir = os.path.join(arg.data_dir, "mask")
xml_path = os.path.join(arg.data_dir, "cameras.xml")
obj_name = os.path.basename(arg.data_dir)
target_dir = os.path.join(arg.data_dir, "neus2_input", "images")
os.makedirs(target_dir, exist_ok=True)
# load images and masks and save them as rgba images
img_list = glob(os.path.join(data_dir, "*.png"))
img_list.sort()
num_view = len(img_list)
print(num_view)
img_h, img_w = cv2.imread(img_list[0]).shape[:2]
for i in range(num_view):
img_path = img_list[i]
mask_path = os.path.join(mask_dir, f"{i:02d}.png")
img = cv2.imread(img_path)
mask = cv2.imread(mask_path)
img = cv2.cvtColor(img, cv2.COLOR_BGR2BGRA)
img[..., 3] = mask[..., 0]
new_img_path = os.path.join(target_dir, f"{i:02d}.png")
cv2.imwrite(new_img_path, img)
print(f"Saved {new_img_path}")
data = {
"from_na": True,
"w": img_w,
"h": img_h,
"aabb_scale": 1.0,
"frames": [],
"scale": 1,
"offset": [1, 1, 1],
}
with open(xml_path, "r") as f:
xml_data = f.read()
bs_data = BeautifulSoup(xml_data, "xml")
c_unique = bs_data.find_all('resolution')
img_width = int(c_unique[0].get("width"))
img_height = int(c_unique[0].get("height"))
c_intrinsics = bs_data.find_all('calibration')
f = float(c_intrinsics[0].find("f").text)
cx_offset = float(c_intrinsics[0].find("cx").text)
cy_offset = float(c_intrinsics[0].find("cy").text)
K = np.array([[f, 0, (img_width - 1) / 2 + cx_offset],
[0, f, (img_height - 1) / 2 + cy_offset],
[0, 0, 1]])
b_unique = bs_data.find_all('camera')
R_list = []
t_list = []
C2W_list = []
camera_sphere = dict()
for tag in b_unique:
img_name = tag.get("label")
view_idx = int(img_name.split("_")[-1])
# camera to world transform
C2W = np.array([float(i) for i in tag.find("transform").text.split(" ")]).reshape((4, 4))
C2W_list.append(C2W)
print(img_name, view_idx)
W2C = np.linalg.inv(C2W)
R_list.append(W2C[:3, :3])
t_list.append(W2C[:3, 3])
camera_sphere[f"world_mat_{view_idx}"] = make4x4(K) @ W2C
print(img_name)
data["frames"].append({
"file_path": f"images/{img_name}.png",
"transform_matrix": C2W.tolist(),
"intrinsic_matrix": make4x4(K).tolist()
})
offset, scale = normalize_camera(R_list, t_list, camera2object_ratio=10)
data["scale"] = scale
data["offset"] = list((-offset*scale + 0.5))
create_json_file(data, os.path.join(arg.data_dir, "neus2_input", 'transform.json'))
================================================
FILE: data_capture_and_preprocessing/sam_mvps.py
================================================
import os.path
from glob import glob
import argparse
import torch.cuda
from segment_anything import SamPredictor, sam_model_registry
parser = argparse.ArgumentParser()
parser.add_argument("--checkpoint", type=str, default=None)
parser.add_argument("--data_dir", type=str, default="./")
args = parser.parse_args()
sam = sam_model_registry["vit_h"](checkpoint=args.checkpoint)
sam.to(device="cuda")
predictor = SamPredictor(sam)
import cv2
import numpy as np
import matplotlib.pyplot as plt
import time
from IPython.display import display, clear_output
obj_dir = os.listdir(args.data_dir)
obj_dir = [os.path.join(args.data_dir, obj) for obj in obj_dir if ".data" in obj]
mask_dir = os.path.join(os.path.dirname(os.path.dirname(args.data_dir)), "mask")
os.makedirs(mask_dir, exist_ok=True)
def pick_point(event, x, y, flags, param):
if event == cv2.EVENT_LBUTTONDOWN:
print(f'You selected point ({x}, {y})')
points.append(np.array([[x, y]]))
def show_mask(mask, ax, random_color=False):
if random_color:
color = np.concatenate([np.random.random(3), np.array([0.6])], axis=0)
else:
color = np.array([30 / 255, 144 / 255, 255 / 255, 0.6])
h, w = mask.shape[-2:]
mask_image = mask.reshape(h, w, 1) * color.reshape(1, 1, -1)
ax.imshow(mask_image)
def show_points(coords, labels, ax, marker_size=375):
pos_points = coords[labels == 1]
neg_points = coords[labels == 0]
ax.scatter(pos_points[:, 0], pos_points[:, 1], color='green', marker='*', s=marker_size, edgecolor='white',
linewidth=1.25)
ax.scatter(neg_points[:, 0], neg_points[:, 1], color='red', marker='*', s=marker_size, edgecolor='white',
linewidth=1.25)
def show_box(box, ax):
x0, y0 = box[0], box[1]
w, h = box[2] - box[0], box[3] - box[1]
ax.add_patch(plt.Rectangle((x0, y0), w, h, edgecolor='green', facecolor=(0, 0, 0, 0), lw=2))
for obj_dir_path in obj_dir:
mask_path = os.path.join(obj_dir_path, "mask.png")
if os.path.exists(mask_path):
continue
# randomly pick an image from the object directory
img_list = glob(os.path.join(obj_dir_path, "*.png")) + glob(os.path.join(obj_dir_path, "*.jpg"))
img_test_path = img_list[0]
img_test = cv2.imread(img_test_path)
predictor.set_image(img_test)
torch.cuda.synchronize()
points = []
while True:
# Create a window
cv2.namedWindow('image', cv2.WINDOW_NORMAL)
# Bind the callback function to the window
cv2.setMouseCallback('image', pick_point)
while(1):
cv2.imshow('image', img_test)
if cv2.waitKey(20) & 0xFF == 27: # Break the loop when 'ESC' is pressed
break
cv2.destroyAllWindows()
print(f'Selected points: {points}')
input_point = np.concatenate(points, axis=0).reshape(-1, 2)
input_label = np.ones(input_point.shape[0], dtype=np.int64)
print(f'Input point: {input_point}')
masks, scores, logits = predictor.predict(
point_coords=input_point,
point_labels=input_label,
multimask_output=False,
)
for i, (mask, score) in enumerate(zip(masks, scores)):
plt.figure(figsize=(10,10))
plt.imshow(img_test[:, :, ::-1])
show_mask(mask, plt.gca())
show_points(input_point, input_label, plt.gca())
plt.title(f"Mask {i+1}, Score: {score:.3f}", fontsize=18)
plt.axis('off')
plt.show(block=False)
plt.pause(3)
plt.close()
value = input("Press enter to save the mask, or c to continue selecting points: ")
if value == "c":
continue
elif value == "":
break
# save the mask
base_dir = os.path.dirname(img_test_path)
view_idx = int(base_dir.split("/")[-1].split(".")[0].split("_")[-1])
mask_path1 = os.path.join(base_dir, "mask.png")
mask_path2 = os.path.join(mask_dir, f"{view_idx:02d}.png")
cv2.imwrite(mask_path1, mask.astype(np.uint8) * 255)
cv2.imwrite(mask_path2, mask.astype(np.uint8) * 255)
print(f"Mask saved at {mask_path1} and {mask_path2}")
================================================
FILE: download_data.sh
================================================
pip install gdown==5.1.0
gdown 'https://drive.google.com/file/d/1Y3-v5jo-IRyTsPh8srZxIc2v5WZdPly_/view?usp=sharing' --fuzzy
unzip data.zip
rm data.zip
================================================
FILE: exp_runner.py
================================================
import os
import logging
import argparse
import numpy as np
import cv2 as cv
import trimesh
import torch
import torch.nn.functional as F
from torch.utils.tensorboard import SummaryWriter
from shutil import copyfile
from tqdm.auto import tqdm
from pyhocon import ConfigFactory
from models.fields import SDFNetwork, SingleVarianceNetwork
import pyexr
import time
from utilities.utils import crop_image_by_mask, toRGBA
import open3d as o3d
import pyvista as pv
pv.set_plot_theme("document")
pv.global_theme.transparent_background = True
from models.cd_and_fscore import chamfer_distance_and_f1_score
import csv
from collections import OrderedDict
def get_class(kls):
parts = kls.split('.')
module = ".".join(parts[:-1])
m = __import__(module)
for comp in parts[1:]:
m = getattr(m, comp)
return m
class Runner:
def __init__(self, conf_text, mode='train', is_continue=False, datadir=None):
self.device = torch.device('cuda')
self.conf_text = conf_text
if not is_continue:
exp_time = str(time.strftime('%Y_%m_%d_%H_%M_%S', time.localtime(time.time())))
exp_time_dir = f"exp_{exp_time}"
self.conf = ConfigFactory.parse_string(conf_text)
self.base_exp_dir = os.path.join(self.conf['general.base_exp_dir'], exp_time_dir)
os.makedirs(self.base_exp_dir, exist_ok=True)
self.dataset = get_class(self.conf['general.dataset_class'])(self.conf['dataset'])
self.iter_step = 0
# Training parameters
self.end_iter = self.conf.get_int('train.end_iter')
self.batch_size = self.conf.get_int('train.batch_size')
self.patch_size = self.conf.get_int('train.patch_size', default=3)
self.learning_rate = self.conf.get_float('train.learning_rate')
self.learning_rate_alpha = self.conf.get_float('train.learning_rate_alpha')
self.use_white_bkgd = self.conf.get_bool('train.use_white_bkgd')
self.warm_up_end = self.conf.get_float('train.warm_up_end', default=0.0)
self.loss_type = self.conf.get('train.loss_type', 'l1')
self.normal_weight = self.conf.get_float('train.normal_weight')
self.eikonal_weight = self.conf.get_float('train.eikonal_weight')
self.mask_weight = self.conf.get_float('train.mask_weight')
self.increase_bindwidth_every = self.conf.get_int('train.increase_bindwidth_every', default=350)
# validation parameters
self.val_normal_freq = self.conf.get_int('val.val_normal_freq')
self.val_normal_resolution_level = self.conf.get_int('val.val_normal_resolution_level')
self.val_gradient_method = self.conf.get('val.gradient_method', 'dfd')
self.val_mesh_freq = self.conf.get_int('val.val_mesh_freq')
self.val_mesh_res = self.conf.get_int('val.val_mesh_res')
self.eval_metric_freq = self.conf.get_int('val.eval_metric_freq')
self.report_freq = self.conf.get_int('val.report_freq')
self.save_freq = self.conf.get_int('val.save_freq')
# Ray marching parameters
self.start_step_size = self.conf.get_float('model.ray_marching.start_step_size', default=1e-2)
self.end_step_size = self.conf.get_float('model.ray_marching.end_step_size', default=5e-4)
self.slop_step = (np.log10(self.start_step_size) - np.log10(self.end_step_size)) / self.end_iter
# Networks
params_to_train = []
self.sdf_network = SDFNetwork(**self.conf['model.sdf_network'], encoding_config=self.conf['model.encoding']).to(self.device)
self.deviation_network = SingleVarianceNetwork(**self.conf['model.variance_network']).to(self.device)
params_to_train += list(self.sdf_network.parameters())
params_to_train += list(self.deviation_network.parameters())
self.renderer = get_class(self.conf['general.renderer_class'])(self.sdf_network,
self.deviation_network,
self.conf["train"]["gradient_method"])
self.optimizer = torch.optim.Adam(params_to_train, lr=self.learning_rate)
self.is_continue = is_continue
self.mode = mode
# Load checkpoint
latest_model_name = None
if is_continue:
model_list_raw = os.listdir(os.path.join(self.base_exp_dir, 'checkpoints'))
model_list = []
for model_name in model_list_raw:
if model_name[-3:] == 'pth' and int(model_name[5:-4]) <= self.end_iter:
model_list.append(model_name)
model_list.sort()
latest_model_name = model_list[-1]
if latest_model_name is not None:
logging.info('Find checkpoint: {}'.format(latest_model_name))
self.load_checkpoint(latest_model_name)
# Backup codes and configs for debug
if self.mode[:5] == 'train':
self.file_backup()
def train(self):
print("Start training...")
self.writer = SummaryWriter(log_dir=os.path.join(self.base_exp_dir, 'logs'))
self.writer.add_graph(self.sdf_network, verbose=False, input_to_model=torch.randn(1, 3))
self.update_learning_rate()
# create a csv file to save the evaluation metrics
csv_file_name = f"eval_metrics.csv"
csv_file_path = os.path.join(self.base_exp_dir, csv_file_name)
if not os.path.exists(csv_file_path):
with open(csv_file_path, 'w') as f:
writer = csv.writer(f)
if len(self.dataset.exclude_view_list)>0:
writer.writerow(['iter',
'mae_all_view',
'mae_test_view',
'CD',
'fscore'])
else:
writer.writerow(['iter',
'mae_all_view',
'CD',
'fscore'])
res_step = self.end_iter - self.iter_step
pbar = tqdm(range(res_step))
for iter_i in pbar:
# update ray marching step size
self.renderer.sampling_step_size = 10 ** (np.log10(self.start_step_size) - self.slop_step*iter_i)
# update occupancy grid
self.renderer.occupancy_grid.every_n_step(step=iter_i,
occ_eval_fn=self.renderer.occ_eval_fn,
occ_thre=self.conf["model.ray_marching"]["occ_threshold"],
n=self.conf["model.ray_marching"]["occ_update_freq"])
# following neuralangelo, gradually increase ingp bandwidth
if self.iter_step % self.increase_bindwidth_every == 0:
self.renderer.sdf_network.increase_bandwidth()
# sample patches of pixels for training
rays_o_patch_all, rays_d_patch_all, marching_plane_normal, V_inverse_patch_all, true_normal, mask = \
self.dataset.gen_random_patches(self.batch_size, patch_H=self.patch_size, patch_W=self.patch_size)
rays_o_patch_center = rays_o_patch_all[:, self.patch_size // 2, self.patch_size // 2] # (num_patch, 3)
rays_d_patch_center = rays_d_patch_all[:, self.patch_size // 2, self.patch_size// 2] # (num_patch, 3)
near, far = self.dataset.near_far_from_sphere(rays_o_patch_center, rays_d_patch_center)
if self.mask_weight > 0.0:
mask = (mask > 0.5).float()
else:
mask = torch.ones_like(mask)
mask_sum = mask.sum() + 1e-5
# forward rendering
render_out = self.renderer.render(rays_o_patch_all,
rays_d_patch_all,
marching_plane_normal,
near, far, V_inverse_patch_all)
if render_out['gradients'] is None: # all rays are in the zero region of the occupancy grid
self.update_learning_rate()
continue
comp_normal = render_out['comp_normal'] # rendered normal at pixels
gradients = render_out['gradients'] # gradients at all sampled 3D points
comp_mask = render_out['weight_sum'] # rendered occupancy at pixels
samples_per_ray = render_out['samples_per_ray']
normal_error = (comp_normal - true_normal) * mask
if self.loss_type == 'l1':
normal_loss = F.l1_loss(normal_error, torch.zeros_like(normal_error), reduction='sum') / mask_sum
elif self.loss_type == 'l2':
normal_loss = F.mse_loss(normal_error, torch.zeros_like(normal_error), reduction='sum') / mask_sum
gradients_norm = torch.linalg.norm(gradients, ord=2, dim=-1)
eikonal_loss = F.mse_loss(gradients_norm, torch.ones_like(gradients_norm), reduction='mean')
mask_loss = F.binary_cross_entropy(comp_mask.clip(1e-5, 1.0 - 1e-5), mask)
loss = self.normal_weight * normal_loss + \
self.mask_weight * mask_loss + \
self.eikonal_weight * eikonal_loss
self.optimizer.zero_grad()
loss.backward()
self.optimizer.step()
self.iter_step += 1
self.update_learning_rate()
if self.iter_step % self.report_freq == 0:
message_postfix = OrderedDict(loss=f"{loss:.3e}",
s=f"{self.deviation_network.variance.item():.3e}",
rm_step=f"{self.renderer.sampling_step_size.item():.3e}",
samples_per_ray=f"{samples_per_ray:.1f}")
pbar.set_postfix(ordered_dict=message_postfix)
if self.iter_step % self.save_freq == 0:
self.save_checkpoint()
if self.iter_step % self.val_mesh_freq == 0:
self.validate_mesh(resolution=self.val_mesh_res)
if self.iter_step % self.val_normal_freq == 0:
for val_idx in range(self.dataset.n_images):
self.validate_normal_patch_based(idx=val_idx, resolution_level=self.val_normal_resolution_level,
gradient_method=self.val_gradient_method)
if self.iter_step % self.eval_metric_freq == 0:
# no gt mesh, skip the evaluation
if self.dataset.mesh_gt is None:
continue
# remove invisible faces in the gt mesh
if self.dataset.mesh_gt is not None and self.dataset.points_gt is None:
self.dataset.mesh_gt.vertices = o3d.utility.Vector3dVector(
(np.asarray(self.dataset.mesh_gt.vertices) -
self.dataset.scale_mats_np[0][:3, 3][None]) /
self.dataset.scale_mats_np[0][0, 0])
mesh = trimesh.Trimesh(np.asarray(self.dataset.mesh_gt.vertices),
np.asarray(self.dataset.mesh_gt.triangles), process=False)
self.dataset.points_gt = self.find_visible_points(mesh) * self.dataset.scale_mats_np[0][0, 0] + \
self.dataset.scale_mats_np[0][:3, 3][None]
cd, fscore = self.eval_geo(resolution=512)
print(f'iter: {self.iter_step} cd: {cd:.3e}, fscore: {fscore:.3e}')
if len(self.dataset.exclude_view_list)>0:
mae_allview, mae_test_view = self.eval_mae(gradient_method=self.val_gradient_method)
print('MAE (all views) {0}: {1:.5f}'.format(self.val_gradient_method, mae_allview))
print('MAE (test views) {0}: {1:.5f}'.format(self.val_gradient_method, mae_test_view))
with open(csv_file_path, 'a') as f:
writer = csv.writer(f)
writer.writerow([self.iter_step,
mae_allview,
mae_test_view,
cd, fscore])
else:
mae_allview = self.eval_mae(gradient_method="dfd")
# write to csv file
with open(csv_file_path, 'a') as f:
writer = csv.writer(f)
writer.writerow([self.iter_step,
mae_allview,
cd, fscore])
def update_learning_rate(self):
if self.iter_step < self.warm_up_end:
learning_factor = self.iter_step / self.warm_up_end
else:
alpha = self.learning_rate_alpha
progress = (self.iter_step - self.warm_up_end) / (self.end_iter - self.warm_up_end)
learning_factor = (np.cos(np.pi * progress) + 1.0) * 0.5 * (1 - alpha) + alpha
for g in self.optimizer.param_groups:
g['lr'] = self.learning_rate * learning_factor
def file_backup(self):
dir_lis = self.conf['general.recording']
os.makedirs(os.path.join(self.base_exp_dir, 'recording'), exist_ok=True)
for dir_name in dir_lis:
cur_dir = os.path.join(self.base_exp_dir, 'recording', dir_name)
os.makedirs(cur_dir, exist_ok=True)
files = os.listdir(dir_name)
for f_name in files:
if f_name[-3:] == '.py':
copyfile(os.path.join(dir_name, f_name), os.path.join(cur_dir, f_name))
try:
copyfile(self.conf_path, os.path.join(self.base_exp_dir, 'recording', 'config.conf'))
except:
# save conf_text into a txt file
with open(os.path.join(self.base_exp_dir, 'recording', 'config.conf'), 'w') as f:
f.write(self.conf_text)
def load_checkpoint(self, checkpoint_name):
checkpoint = torch.load(os.path.join(self.base_exp_dir, 'checkpoints', checkpoint_name), map_location=self.device)
self.sdf_network.load_state_dict(checkpoint['sdf_network_fine'])
self.deviation_network.load_state_dict(checkpoint['variance_network_fine'])
self.optimizer.load_state_dict(checkpoint['optimizer'])
self.iter_step = checkpoint['iter_step']
logging.info('End')
def save_checkpoint(self):
checkpoint = {
'sdf_network_fine': self.sdf_network.state_dict(),
'variance_network_fine': self.deviation_network.state_dict(),
'optimizer': self.optimizer.state_dict(),
'iter_step': self.iter_step,
}
os.makedirs(os.path.join(self.base_exp_dir, 'checkpoints'), exist_ok=True)
torch.save(checkpoint, os.path.join(self.base_exp_dir, 'checkpoints', 'ckpt_{:0>6d}.pth'.format(self.iter_step)))
def validate_normal_pixel_based(self, idx=-1, resolution_level=-1):
if idx < 0:
idx = np.random.randint(self.dataset.n_images)
print('Validate: iter: {}, camera: {}'.format(self.iter_step, idx))
if resolution_level < 0:
resolution_level = self.validate_resolution_level
rays_o, rays_d = self.dataset.gen_rays_at(idx, resolution_level=resolution_level, within_mask=False)
H, W, _ = rays_o.shape
rays_o = rays_o.reshape(-1, 3).split(8192)
rays_d = rays_d.reshape(-1, 3).split(8192)
out_normal_fine = []
out_depth_fine = []
mask_np = self.dataset.masks_np[idx].astype(bool)[..., 0]
mask_np = cv.resize(mask_np.astype(np.uint8),
((int(W), int(H))),
interpolation=cv.INTER_NEAREST).astype(bool)
for rays_o_batch, rays_d_batch in tqdm(zip(rays_o, rays_d)):
near, far = self.dataset.near_far_from_sphere(rays_o_batch, rays_d_batch)
# background_rgb = torch.ones([1, 3]) if self.use_white_bkgd else None
batch_normal, batch_depth = self.renderer.render_normal_pixel_based(rays_o_batch,
rays_d_batch,
near,
far)
out_normal_fine.append(batch_normal.detach().cpu().numpy())
out_depth_fine.append(batch_depth.detach().cpu().numpy())
if len(out_normal_fine) > 0:
normal_img = np.concatenate(out_normal_fine, axis=0)
rot = np.linalg.inv(self.dataset.pose_all[idx, :3, :3].detach().cpu().numpy()) # W2C rotation
# normal_img_world = (normal_img.reshape([H, W, 3]) * 128 + 128).clip(0, 255)
normal_img = np.matmul(rot[None, :, :], normal_img[:, :, None]).reshape([H, W, 3, -1])
normal_img[:,:, [1, 2]] *= -1
normal_img_norm = np.linalg.norm(np.squeeze(normal_img), axis=2, keepdims=True)
normal_img_normalized = np.squeeze(normal_img) / (normal_img_norm+1e-7)
# normal_img = ((np.squeeze(normal_img)/normal_img_norm) * 128 + 128).clip(0, 255)
normal_img = (np.squeeze(normal_img) * 128 + 128).clip(0, 255)
normal_img_normalized = (np.squeeze(normal_img_normalized) * 128 + 128).clip(0, 255)
depth_img = np.concatenate(out_depth_fine, axis=0).reshape([H, W])
os.makedirs(os.path.join(self.base_exp_dir, 'normals'), exist_ok=True)
os.makedirs(os.path.join(self.base_exp_dir, "depth"), exist_ok=True)
normal_img_norm[~mask_np] = np.nan
depth_img[~mask_np] = np.nan
normal_img_norm = np.squeeze(normal_img_norm.clip(0.8, 1.2))
normal_img_norm = (normal_img_norm - np.nanmin(normal_img_norm)) / (np.nanmax(normal_img_norm) - np.nanmin(normal_img_norm))
normal_img_norm = np.nan_to_num(normal_img_norm)
normal_img_norm = (normal_img_norm * 255).astype(np.uint8)
normal_img_norm = cv.applyColorMap(normal_img_norm, cv.COLORMAP_JET)
normal_img_norm[~mask_np] = 0
cv.imwrite(os.path.join(self.base_exp_dir,
'normals',
'{:0>8d}_{}_{}_norm.png'.format(self.iter_step, 0, idx)),
normal_img_norm[..., ::-1])
cv.imwrite(os.path.join(self.base_exp_dir,
'normals',
'{:0>8d}_{}_{}.png'.format(self.iter_step, 0, idx)),
normal_img[..., ::-1])
cv.imwrite(os.path.join(self.base_exp_dir,
'normals',
'{:0>8d}_{}_{}_normalized.png'.format(self.iter_step, 0, idx)),
normal_img_normalized[..., ::-1])
np.save(os.path.join(self.base_exp_dir,
'depth',
'{:0>8d}_{}_{}.npy'.format(self.iter_step, 0, idx)),
depth_img)
return idx, (normal_img - 128) / 128.
def validate_normal_patch_based(self, idx=-1, resolution_level=-1, gradient_method="dfd"):
if idx < 0:
idx = np.random.randint(self.dataset.n_images)
print('Rendering normal maps... iter: {}, camera: {}'.format(self.iter_step, idx))
if resolution_level < 0:
resolution_level = self.validate_resolution_level
rays_o_patch_center, \
rays_d_patch_center, \
rays_o_patches_all, \
rays_v_patches_all, \
rays_ez, \
rays_A_inverse, horizontal_num_patch, vertical_num_patch = self.dataset.gen_patches_at(idx, resolution_level=resolution_level,
patch_H=self.patch_size,
patch_W=self.patch_size)
mask_np = self.dataset.masks_np[idx].astype(bool) # (H, W)
img_w = horizontal_num_patch * self.patch_size
img_h = vertical_num_patch * self.patch_size
# resize mask to the size of the image
mask_np = cv.resize(mask_np.astype(np.uint8),
((int(img_w), int(img_h))),
interpolation=cv.INTER_NEAREST).astype(bool)
num_patches = rays_o_patches_all.shape[0]
eval_patch_size = 1024
comp_normal_map = np.zeros([img_h, img_w, 3])
comp_normal_list = []
for patch_idx in range(0, num_patches, eval_patch_size):
rays_o_patch_center_batch = rays_o_patch_center[patch_idx:patch_idx+eval_patch_size]
rays_d_patch_center_batch = rays_d_patch_center[patch_idx:patch_idx+eval_patch_size]
rays_o_patches_all_batch = rays_o_patches_all[patch_idx:patch_idx+eval_patch_size]
rays_v_patches_all_batch = rays_v_patches_all[patch_idx:patch_idx+eval_patch_size]
rays_ez_batch = rays_ez[patch_idx:patch_idx+eval_patch_size]
rays_A_inverse_batch = rays_A_inverse[patch_idx:patch_idx+eval_patch_size]
near, far = self.dataset.near_far_from_sphere(rays_o_patch_center_batch,
rays_d_patch_center_batch)
render_out = self.renderer.render(rays_o_patches_all_batch,
rays_v_patches_all_batch,
rays_ez_batch,
near, far,
rays_A_inverse_batch, gradient_method, mode='eval')
comp_normal = render_out['comp_normal']
comp_normal = comp_normal.detach().cpu().numpy()
comp_normal_list.append(comp_normal)
comp_normal_list = np.concatenate(comp_normal_list, axis=0)
count = 0
for i in range(0, img_h, self.patch_size):
for j in range(0, img_w, self.patch_size):
comp_normal_map[i:i+self.patch_size, j:j+self.patch_size] = comp_normal_list[count]
count += 1
normal_img_world = comp_normal_map
rot = np.linalg.inv(self.dataset.pose_all[idx, :3, :3].detach().cpu().numpy()) # W2C rotation
normal_img = np.matmul(rot, normal_img_world[..., None]).squeeze()
normal_img[..., [1, 2]] *= -1
normal_img_png = (np.squeeze(normal_img) * 128 + 128).clip(0, 255)
normal_img_norm = np.linalg.norm(np.squeeze(normal_img), axis=2, keepdims=True)
normal_dir = os.path.join(self.base_exp_dir, f'normals_validation_{gradient_method}', 'iter_{:0>6d}'.format(self.iter_step))
os.makedirs(normal_dir, exist_ok=True)
normal_img_normalized = np.squeeze(normal_img) / (normal_img_norm + 1e-7)
normal_img_normalized = (np.squeeze(normal_img_normalized) * 128 + 128).clip(0, 255)
normal_eval = np.zeros((img_h, img_w, 3))
normal_eval[:normal_img_png.shape[0], :normal_img_png.shape[1]] = normal_img_png
normal_eval_normalized = np.zeros((img_h, img_w, 3))
normal_eval_normalized[:normal_img_normalized.shape[0], :normal_img_normalized.shape[1]] = normal_img_normalized
normal_img_normalized = crop_image_by_mask(toRGBA(normal_eval_normalized.astype(np.uint8)[...,::-1], mask_np), mask_np)
cv.imwrite(os.path.join(normal_dir, '{:0>8d}_{}_{}_rendered.png'.format(self.iter_step, 0, idx)),
normal_eval[..., ::-1])
cv.imwrite(os.path.join(normal_dir, '{:0>8d}_{}_{}_normalized.png'.format(self.iter_step, 0, idx)),
normal_img_normalized)
return normal_img_world, normal_dir
def validate_mesh(self, world_space=True, resolution=256, threshold=0.0):
print('Extracting mesh... iter: {}'.format(self.iter_step))
bound_min = torch.tensor(self.dataset.object_bbox_min, dtype=torch.float32)
bound_max = torch.tensor(self.dataset.object_bbox_max, dtype=torch.float32)
vertices, triangles =\
self.renderer.extract_geometry(bound_min, bound_max, resolution=resolution, threshold=threshold)
mesh = trimesh.Trimesh(vertices, triangles)
vertices, triangles = mesh.vertices, mesh.faces
save_dir = os.path.join(self.base_exp_dir, 'meshes_validation')
os.makedirs(save_dir, exist_ok=True)
if world_space:
vertices = vertices * self.dataset.scale_mats_np[0][0, 0] + self.dataset.scale_mats_np[0][:3, 3][None]
self.writer.add_mesh('mesh_eval', vertices=vertices[None,...], faces=triangles[None,...], global_step=self.iter_step)
mesh = self.remove_isolated_clusters(trimesh.Trimesh(vertices, triangles))
mesh_path = os.path.join(save_dir, 'iter_{:0>8d}.ply'.format(self.iter_step))
o3d.io.write_triangle_mesh((mesh_path), mesh)
print(f'Mesh saved at {mesh_path}')
def remove_isolated_clusters(self, mesh):
# cleaning the marching cube extracted mesh
import copy
mesh = mesh.as_open3d
# with o3d.utility.VerbosityContextManager(
# o3d.utility.VerbosityLevel.Debug) as cm:
triangle_clusters, cluster_n_triangles, cluster_area = (
mesh.cluster_connected_triangles())
triangle_clusters = np.asarray(triangle_clusters)
cluster_n_triangles = np.asarray(cluster_n_triangles)
mesh_eval = copy.deepcopy(mesh)
largest_cluster_idx = cluster_n_triangles.argmax()
triangles_to_remove = triangle_clusters != largest_cluster_idx
mesh_eval.remove_triangles_by_mask(triangles_to_remove)
mesh_eval.remove_unreferenced_vertices()
return mesh_eval
@torch.no_grad()
def eval_mae(self, gradient_method):
print("Computing mean angular errors...")
normal_gt_dir = os.path.join(self.dataset.data_dir, "normal_world_space_GT")
ae_map_list = []
normal_map_eval_list = []
ae_map_eval_list = []
ae_map_test_list = []
for idx in range(self.dataset.n_images):
normal_gt = pyexr.read(os.path.join(normal_gt_dir, "{:02d}.exr".format(idx)))[..., :3]
mask_np = self.dataset.masks_np[idx].astype(bool)
normal_map_world, save_dir = self.validate_normal_patch_based(idx, resolution_level=self.val_normal_resolution_level, gradient_method=gradient_method)
normal_map_world = normal_map_world / (1e-10 + np.linalg.norm(normal_map_world, axis=-1, keepdims=True))
normal_eval = np.zeros((self.dataset.H, self.dataset.W, 3))
normal_eval[:normal_map_world.shape[0], :normal_map_world.shape[1]] = normal_map_world
normal_eval[~mask_np] = np.nan
normal_map_eval_list.append(normal_eval)
# self.writer.add_image(step=self.iter_step, data=(normal_eval + 1) / 2, name=("normal_eval_{:02d}".format(idx)))
# pyexr.write(os.path.join(normal_save_dir, "{:02d}.exr".format(idx)), normal_img)
angular_error_map = np.rad2deg(np.arccos(np.clip(np.sum(normal_gt * normal_eval, axis=-1), -1, 1)))
# save angular error map
ae_map_list.append(angular_error_map.copy())
if idx in self.dataset.exclude_view_list:
ae_map_test_list.append(angular_error_map.copy())
# apply jet to angular error map
angular_error_map[~mask_np] = 0
angular_error_map_jet = cv.applyColorMap((angular_error_map / 20 * 255).clip(0, 255).astype(np.uint8),
cv.COLORMAP_JET)
angular_error_map_jet[~mask_np] = 255
angular_error_map_jet = crop_image_by_mask(toRGBA(angular_error_map_jet, mask_np), mask_np)
cv.imwrite(os.path.join(save_dir, '{:0>8d}_{}_{}_ae_up_{}.png'.format(self.iter_step, 0, idx, 20)), angular_error_map_jet)
ae_map_eval_list.append(angular_error_map_jet)
mae = np.nanmean(np.stack(ae_map_list, axis=0))
self.writer.add_scalar('Statistics/mae_allview', mae, self.iter_step)
if len(ae_map_test_list) > 0:
mae_test = np.nanmean(np.stack(ae_map_test_list, axis=0))
self.writer.add_scalar('Statistics/mae_testview', mae_test, self.iter_step)
return mae, mae_test
return mae
@torch.no_grad()
def eval_geo(self, resolution=1024):
# save the mesh
save_dir = os.path.join(self.base_exp_dir, 'points_val')
os.makedirs(save_dir, exist_ok=True)
# save gt points
pcd_gt = o3d.geometry.PointCloud()
pcd_gt.points = o3d.utility.Vector3dVector(self.dataset.points_gt)
if not os.path.exists(os.path.join(save_dir, f"pcd_gt.ply")):
o3d.io.write_point_cloud(os.path.join(save_dir, f"pcd_gt.ply"), pcd_gt)
# marching cubes
bound_min = torch.tensor(self.dataset.object_bbox_min, dtype=torch.float32)
bound_max = torch.tensor(self.dataset.object_bbox_max, dtype=torch.float32)
vertices, triangles = \
self.renderer.extract_geometry(bound_min, bound_max, resolution=resolution, threshold=0)
# vertices = vertices * self.dataset.scale_mats_np[0][0, 0] + self.dataset.scale_mats_np[0][:3, 3][None]
mesh = trimesh.Trimesh(np.asarray(vertices), np.asarray(triangles), process=False)
vertices_world = vertices * self.dataset.scale_mats_np[0][0, 0] + self.dataset.scale_mats_np[0][:3, 3][None]
mesh_world = trimesh.Trimesh(np.asarray(vertices_world), np.asarray(triangles), process=False)
mesh_world_path = os.path.join(save_dir, f"{self.iter_step}_world.obj")
mesh_world.export(mesh_world_path)
points_eval = self.find_visible_points(mesh)*self.dataset.scale_mats_np[0][0, 0] + self.dataset.scale_mats_np[0][:3, 3][None]
# save the sampled points
sampled_points_path = os.path.join(save_dir, f"{self.iter_step}_points_eval.ply")
pcd_eval = o3d.geometry.PointCloud()
pcd_eval.points = o3d.utility.Vector3dVector(points_eval)
o3d.io.write_point_cloud(sampled_points_path, pcd_eval)
cd, fscore = chamfer_distance_and_f1_score(points_eval, self.dataset.points_gt)
self.writer.add_scalar('Statistics/cd', cd, self.iter_step)
self.writer.add_scalar('Statistics/fscore', fscore, self.iter_step)
return cd, fscore
def find_visible_points(self, mesh):
num_view = self.dataset.n_images
points_list = []
for view_idx in range(num_view):
rays_o, rays_v = self.dataset.gen_rays_at(view_idx, resolution_level=1, within_mask=True)
rays_o, rays_v = rays_o.cpu().detach().numpy(), rays_v.cpu().detach().numpy()
rays_v = rays_v / np.linalg.norm(rays_v, axis=-1, keepdims=True)
locations, index_ray, index_tri = mesh.ray.intersects_location(
ray_origins=rays_o,
ray_directions=rays_v,
multiple_hits=False)
points_list.append(locations)
return np.concatenate(points_list, axis=0)
if __name__ == '__main__':
import warnings
warnings.filterwarnings("ignore")
torch.set_default_tensor_type('torch.cuda.FloatTensor')
parser = argparse.ArgumentParser()
parser.add_argument('--conf', type=str, default='./confs/base.conf')
parser.add_argument('--mode', type=str, default='eval_normal')
parser.add_argument('--mcube_threshold', type=float, default=0.0)
parser.add_argument('--is_continue', default=False, action="store_true")
parser.add_argument('--gpu', type=int, default=0)
parser.add_argument('--obj_name', type=str, default='')
args = parser.parse_args()
torch.cuda.set_device(args.gpu)
print(f'Running on the object: {args.obj_name}')
f = open(args.conf)
conf_text = f.read()
conf_text = conf_text.replace('CASE_NAME', args.obj_name)
runner = Runner(conf_text, args.mode, args.is_continue)
runner.train()
================================================
FILE: models/cd_and_fscore.py
================================================
from scipy.spatial import KDTree
import numpy as np
def chamfer_distance_and_f1_score(ref_points, eval_points, f_threshold=0.5):
"""
This function calculates the chamfer distance and f1 score between two sets of points.
Parameters:
ref_points (numpy.ndarray): Reference points. A (p, 3) array representing points in the world space.
eval_points (numpy.ndarray): Points to be evaluated. A (p, 3) array representing points in the world space.
f_threshold (float, optional): Threshold for f1 score calculation. Default is 0.5mm.
Returns:
chamfer_dist (float): The chamfer distance between gt_points and eval_points.
f_score (float): The f1 score between gt_points and eval_points.
"""
print("computing chamfer distance and f1 score...")
distance_eval2gt, _ = KDTree(ref_points).query(eval_points, k=1, p=2) # p=2 for Euclidean distance
distance_gt2eval, _ = KDTree(eval_points).query(ref_points, k=1, p=2)
# following Uncertainty-aware deep multi-view photometric stereo
chamfer_dist = (np.mean(distance_eval2gt) + np.mean(distance_gt2eval))/2
precision = np.mean(distance_eval2gt < f_threshold)
recall = np.mean(distance_gt2eval < f_threshold)
f_score = 2 * precision * recall / (precision + recall)
return chamfer_dist, f_score
================================================
FILE: models/dataset_loader.py
================================================
import torch
import torch.nn.functional as F
import cv2 as cv
import numpy as np
import os
from glob import glob
from icecream import ic
import pyexr
import open3d as o3d
import time
from concurrent.futures import ThreadPoolExecutor
def load_K_Rt_from_P(filename, P=None):
# This function is borrowed from IDR: https://github.com/lioryariv/idr
if P is None:
lines = open(filename).read().splitlines()
if len(lines) == 4:
lines = lines[1:]
lines = [[x[0], x[1], x[2], x[3]] for x in (x.split(" ") for x in lines)]
P = np.asarray(lines).astype(np.float32).squeeze()
K, R, t, *_ = cv.decomposeProjectionMatrix(P)
# CAUTION: R is the W2C rotation matrix but t is the camera position in world coordinate.
K = K / K[2, 2]
intrinsics = np.eye(4)
intrinsics[:3, :3] = K
C2W = np.eye(4, dtype=np.float32)
C2W[:3, :3] = R.T
C2W[:3, 3] = (t[:3] / t[3])[:, 0]
return intrinsics, C2W
class Dataset:
def __init__(self, conf):
super(Dataset, self).__init__()
print('Load data: Begin')
self.device = torch.device('cuda')
self.conf = conf
normal_dir = conf.get_string('normal_dir')
self.data_dir = conf.get_string('data_dir')
self.cameras_name = conf.get_string('cameras_name')
self.exclude_view_list = conf['exclude_views'] # list of views to exclude from training. Used in novel-view normal synthesis evaluation.
self.upsample_factor = conf.get_int('upsample_factor', default=1)
ic(self.exclude_view_list)
# load the GT mesh for evaluation if any
mesh_path = os.path.join(self.data_dir, 'mesh_Gt.ply')
if os.path.exists(mesh_path):
self.mesh_gt = o3d.io.read_triangle_mesh(mesh_path)
else:
self.mesh_gt = None
self.points_gt = None # will be computed from the mesh at evaluation time
camera_dict = np.load(os.path.join(self.data_dir, self.cameras_name))
self.camera_dict = camera_dict
self.normal_lis = sorted(glob(os.path.join(self.data_dir, normal_dir, '*.exr')))
self.n_images = len(self.normal_lis)
self.train_images = set(range(self.n_images)) - set(self.exclude_view_list)
self.img_idx_list = [int(os.path.basename(x).split('.')[0]) for x in self.normal_lis]
print("loading normal maps...")
with ThreadPoolExecutor(max_workers=min(64, os.cpu_count()*5)) as executor:
def read_normal(im_name):
return pyexr.read(im_name)[..., :3]
self.normal_np = np.stack(list(executor.map(read_normal, self.normal_lis)))
if self.upsample_factor > 1:
# resize normal maps
self.normal_np = F.interpolate(torch.from_numpy(self.normal_np).permute(0, 3, 1, 2), scale_factor=self.upsample_factor, mode='bilinear', align_corners=False).permute(0, 2, 3, 1).numpy()
self.normals = torch.from_numpy(self.normal_np.astype(np.float32)).to(self.device) # [n_images, H, W, 3]
print("loading normal maps done.")
self.masks_lis = sorted(glob(os.path.join(self.data_dir, 'mask/*.png')))
with ThreadPoolExecutor(max_workers=min(64, os.cpu_count()*5)) as executor:
def read_mask(im_name):
return cv.imread(im_name)
self.masks_np = np.stack(list(executor.map(read_mask, self.masks_lis))) / 255.0
if self.upsample_factor > 1:
# resize mask
self.masks_np = F.interpolate(torch.from_numpy(self.masks_np).permute(0, 3, 1, 2), scale_factor=self.upsample_factor, mode='nearest').permute(0, 2, 3, 1).numpy()
self.masks_np = self.masks_np[..., 0]
self.total_pixel = np.sum(self.masks_np)
# set background of normal map to 0
self.normal_np[self.masks_np == 0] = 0
# world_mat is a projection matrix from world to image
self.world_mats_np = [camera_dict['world_mat_%d' % idx].astype(np.float32) for idx in self.img_idx_list]
self.scale_mats_np = []
# scale_mat: used for coordinate normalization, we assume the scene to render is inside a unit sphere at origin.
self.scale_mats_np = [camera_dict['scale_mat_%d' % idx].astype(np.float32) for idx in self.img_idx_list]
self.intrinsics_all = []
self.pose_all = []
self.V_inverse_all = []
self.H, self.W = self.normal_np.shape[1], self.normal_np.shape[2]
for scale_mat, world_mat, normal_map, mask in zip(self.scale_mats_np, self.world_mats_np, self.normals, self.masks_np):
P = world_mat @ scale_mat
P = P[:3, :4]
intrinsics, C2W = load_K_Rt_from_P(None, P)
if self.upsample_factor > 1:
# resize intrinsics
intrinsics[0, 0] *= self.upsample_factor
intrinsics[1, 1] *= self.upsample_factor
intrinsics[0, 2] *= self.upsample_factor
intrinsics[1, 2] *= self.upsample_factor
self.intrinsics_all.append(torch.from_numpy(intrinsics).float())
self.pose_all.append(torch.from_numpy(C2W).float())
intrinsics_inverse = torch.inverse(torch.from_numpy(intrinsics).float())
pose = torch.from_numpy(C2W).float()
# compute the V_inverse
tx = torch.linspace(0, self.W - 1, int(self.W))
ty = torch.linspace(0, self.H - 1, int(self.H))
pixels_x, pixels_y = torch.meshgrid(tx, ty)
p = torch.stack([pixels_x, pixels_y, torch.ones_like(pixels_y)], dim=-1).to(intrinsics_inverse.device) # W, H, 3
p = torch.matmul(intrinsics_inverse[None, None, :3, :3],
p[:, :, :, None]).squeeze() # W, H, 3
rays_v = p / torch.linalg.norm(p, ord=2, dim=-1, keepdim=True) # W, H, 3
rays_v = torch.matmul(pose[None, None, :3, :3],
rays_v[:, :, :, None]).squeeze() # W, H, 3
rays_v = rays_v.transpose(0, 1).to(self.device) # H, W, 3
# the axis direction of the camera coordinate system in the world coordinate system
rays_right = pose[None, :3, 0].expand(rays_v.shape).to(self.device) # H, W, 3
rays_down = pose[None, :3, 1].expand(rays_v.shape).to(self.device) # H, W, 3
V_concat = torch.cat([rays_v[..., None, :],
rays_right[..., None, :],
rays_down[..., None, :]], dim=-2) # (H, W, 3, 3)
# computing the inverse may take a while if the resolution is high
# For 512x612, it takes about 0.8ms
V_inverse = torch.inverse(V_concat) # (H, W, 3, 3)
self.V_inverse_all.append(V_inverse)
self.masks = torch.from_numpy(self.masks_np.astype(np.float32)).to(self.device) # [n_images, H, W, 3]
self.intrinsics_all = torch.stack(self.intrinsics_all).to(self.device) # [n_images, 4, 4]
self.intrinsics_all_inv = torch.inverse(self.intrinsics_all) # [n_images, 4, 4]
self.focal_length = self.intrinsics_all[0][0, 0]
self.pose_all = torch.stack(self.pose_all).to(self.device) # [n_images, 4, 4]
self.image_pixels = self.H * self.W
self.V_inverse_all = torch.stack(self.V_inverse_all).to(self.device) # [n_images, H, W, 3, 3]
# for mesh extraction
self.object_bbox_min = np.array([-1., -1., -1.])
self.object_bbox_max = np.array([1., 1., 1.])
print('Load data: End')
def gen_rays_at(self, img_idx, resolution_level=1, within_mask=False):
"""
Generate all rays at world space from one camera.
"""
mask_np = self.masks_np[img_idx].astype(bool)
# resize the mask using resolution_level
mask_np = cv.resize(mask_np.astype(np.uint8)*255, (int(self.W // resolution_level), int(self.H // resolution_level)), interpolation=cv.INTER_NEAREST).astype(bool)
l = resolution_level
tx = torch.linspace(0, self.W - 1, int(self.W // l))
ty = torch.linspace(0, self.H - 1, int(self.H // l))
pixels_x, pixels_y = torch.meshgrid(tx, ty)
p = torch.stack([pixels_x, pixels_y, torch.ones_like(pixels_y)], dim=-1) # W, H, 3
p = torch.matmul(self.intrinsics_all_inv[img_idx, None, None, :3, :3], p[:, :, :, None]).squeeze() # W, H, 3
rays_v = p / torch.linalg.norm(p, ord=2, dim=-1, keepdim=True) # W, H, 3
rays_v = torch.matmul(self.pose_all[img_idx, None, None, :3, :3], rays_v[:, :, :, None]).squeeze() # W, H, 3
rays_o = self.pose_all[img_idx, None, None, :3, 3].expand(rays_v.shape) # W, H, 3
rays_o = rays_o.transpose(0, 1)
rays_v = rays_v.transpose(0, 1)
if within_mask:
return rays_o[mask_np], rays_v[mask_np]
else:
return rays_o, rays_v
def gen_patches_at(self, img_idx, resolution_level=1, patch_H=3, patch_W=3):
tx = torch.linspace(0, self.W - 1, int(self.W // resolution_level))
ty = torch.linspace(0, self.H - 1, int(self.H // resolution_level))
pixels_y, pixels_x = torch.meshgrid(ty, tx)
p = torch.stack([pixels_x, pixels_y, torch.ones_like(pixels_y)], dim=-1) # H, W, 3
p = torch.matmul(self.intrinsics_all_inv[img_idx, :3, :3], p[..., None]).squeeze() # H, W, 3
rays_v = p / torch.linalg.norm(p, ord=2, dim=-1, keepdim=True) # W, H, 3
rays_v = torch.matmul(self.pose_all[img_idx, :3, :3], rays_v[:, :, :, None]).squeeze() # H, W, 3
# split rays_v into non-overlapping patches
height, width, _ = rays_v.shape
horizontal_num_patch = width // patch_W
vertical_num_patch = height // patch_H
rays_v_patches_all = []
rays_V_inverse_patches_all = []
rays_ez_patches_all = []
mask_value = []
for i in range(0, height-patch_H//2-1, patch_H):
for j in range(0, width-patch_W//2-1, patch_W):
rays_v_patch = rays_v[i:i + patch_H, j:j + patch_W]
rays_v_patches_all.append(rays_v_patch)
rays_V_inverse_patch = self.V_inverse_all[img_idx][i:i + patch_H, j:j + patch_W]
rays_V_inverse_patches_all.append(rays_V_inverse_patch)
rays_ez_patch = self.normals[img_idx][i + patch_H//2, j + patch_W//2]
rays_ez_patches_all.append(rays_ez_patch)
mask_value.append(self.masks_np[img_idx][i + patch_H//2, j + patch_W//2].astype(bool))
rays_v_patches_all = torch.stack(rays_v_patches_all, dim=0) # (num_patch, patch_H, patch_W, 3)
rays_V_inverse_patches_all = torch.stack(rays_V_inverse_patches_all, dim=0) # (num_patch, patch_H, patch_W, 3, 3)
rays_o_patches_all = self.pose_all[img_idx, :3, 3].expand(rays_v_patches_all.shape) # (num_patch, patch_H, patch_W, 3)
rays_o_patch_center = rays_o_patches_all[:, patch_H//2, patch_W//2] # (num_patch, 3)
rays_d_patch_center = rays_v_patches_all[:, patch_H//2, patch_W//2] # (num_patch, 3)
marching_plane_normal_patches_all = self.pose_all[img_idx, :3, 2].expand(rays_d_patch_center.shape) # (num_patch, 3)
return rays_o_patch_center, \
rays_d_patch_center, \
rays_o_patches_all, \
rays_v_patches_all, \
marching_plane_normal_patches_all, \
rays_V_inverse_patches_all, horizontal_num_patch, vertical_num_patch
def gen_random_patches(self, num_patch, patch_H=3, patch_W=3):
"""
Generate random patches of rays at world space from all viewpoints.
X-axis right, Y-axis down
Parameters:
num_patch (int): The number of patches to generate.
patch_H (int, optional): The height of the patches. Default is 3.
patch_W (int, optional): The width of the patches. Default is 3.
Returns:
rays_o_patch_all (torch.Tensor): The origins of the rays in each patch. A tensor of shape (num_patch, patch_H, patch_W, 3).
rays_d_patch_all (torch.Tensor): The directions of the rays in each patch. A tensor of shape (num_patch, patch_H, patch_W, 3).
marching_plane_normal (torch.Tensor): The normal direction of the image/marching plane.
Since we randomly sample patches from all viewpoints, this normal is only identical for each patch. A tensor of shape (num_patch, 3).
V_inverse_patch_all (torch.Tensor): The inverse of the V matrix at patches of pixels. A tensor of shape (num_patch, patch_H, patch_W, 3, 3).
normal (torch.Tensor): The normals at patches of pixels. A tensor of shape (num_patch, patch_H, patch_W, 3).
mask (torch.Tensor): The mask values at patches of pixels. A tensor of shape (num_patch, patch_H, patch_W, 1).
"""
# randomly sample center pixel locations of patches
# assume all images have the same resolution
patch_center_x = torch.randint(low=0+patch_W//2, high=self.W-1-patch_W//2, size=[num_patch], device=self.device) # (num_patch, )
patch_center_y = torch.randint(low=0+patch_H//2, high=self.H-1-patch_H//2, size=[num_patch], device=self.device) # (num_patch, )
# compute all pixel locations within the patches given patch size (patch_H, patch_W)
patch_center_x_all = patch_center_x[:, None, None] + torch.arange(-patch_W//2+1, patch_W//2+1, device=self.device).repeat(patch_H, 1) # (num_patch, patch_H, patch_W)
patch_center_y_all = patch_center_y[:, None, None] + torch.arange(-patch_H//2+1, patch_H//2+1, device=self.device).reshape(-1, 1).repeat(1, patch_W) # (num_patch, patch_H, patch_W)
# randomly sample viewpoints
img_idx = np.random.choice(list(self.train_images), size=[num_patch]) # (num_patch, )
img_idx = torch.tensor(img_idx, device=self.device)
img_idx_expand = img_idx.view(-1, 1, 1).expand_as(patch_center_x_all) # (num_patch, patch_H, patch_W)
# input normals and mask values for supervision
normal = self.normals[img_idx_expand, patch_center_y_all, patch_center_x_all] # (num_patch, patch_H, patch_W, 3)
V_inverse_patch_all = self.V_inverse_all[img_idx_expand, patch_center_y_all, patch_center_x_all] # (num_patch, patch_H, patch_W, 3, 3)
mask = self.masks[img_idx_expand, patch_center_y_all, patch_center_x_all].unsqueeze(-1)#[..., :1] # (num_patch, patch_H, patch_W)
# compute all ray directions within patches
p_all = torch.stack([patch_center_x_all, patch_center_y_all, torch.ones_like(patch_center_y_all)], dim=-1).float().to(self.device) # (num_patch, patch_H, patch_W, 3)
p_all = torch.matmul(self.intrinsics_all_inv[img_idx_expand, :3, :3], p_all[..., None])[..., 0] # (num_patch, patch_H, patch_W, 3)
p_norm_all = torch.linalg.norm(p_all, ord=2, dim=-1, keepdim=True) # (num_patch, patch_H, patch_W, 1)
rays_d_patch_all = p_all / p_norm_all # (num_patch, patch_H, patch_W, 3)
rays_d_patch_all = torch.matmul(self.pose_all[img_idx, None, None, :3, :3], rays_d_patch_all[..., None])[..., 0] # (num_patch, patch_H, patch_W, 3)
rays_o_patch_all = self.pose_all[img_idx, None, None, :3, 3].expand(rays_d_patch_all.shape) # (num_patch, patch_H, patch_W, 3)
# the normal direction of the image/marching plane is the 3rd column of world2camera transformation
marching_plane_normal = self.pose_all[img_idx, :3, 2].expand((num_patch, 3)) # (num_patch, 3)
return rays_o_patch_all, \
rays_d_patch_all, \
marching_plane_normal, \
V_inverse_patch_all, \
normal,\
mask
def near_far_from_sphere(self, rays_o, rays_d):
"""
This function calculates the near and far intersection points of rays with a unit sphere.
Parameters:
rays_o (torch.Tensor): Origin of the rays. A tensor of shape (N, 3) where N is the number of rays.
rays_d (torch.Tensor): Direction of the rays. A tensor of shape (N, 3) where N is the number of rays.
Returns:
near (torch.Tensor): Near intersection points of the rays with the unit sphere. A tensor of shape (N, ).
far (torch.Tensor): Far intersection points of the rays with the unit sphere. A tensor of shape (N, ).
"""
a = torch.sum(rays_d**2, dim=-1, keepdim=True)
b = 2.0 * torch.sum(rays_o * rays_d, dim=-1, keepdim=True)
c = torch.sum(rays_o**2, dim=-1, keepdim=True) - 1.0
mid = 0.5 * (-b) / a
near = mid - torch.sqrt(b ** 2 - 4 * a * c) / (2 * a)
far = mid + torch.sqrt(b ** 2 - 4 * a * c) / (2 * a)
return near[..., 0], far[..., 0]
def image_at(self, idx, resolution_level):
img = cv.imread(self.images_lis[idx])
return (cv.resize(img, (self.W // resolution_level, self.H // resolution_level))).clip(0, 255)
================================================
FILE: models/fields.py
================================================
import torch
import torch.nn as nn
import numpy as np
import tinycudann as tcnn
from icecream import ic
class SDFNetwork(nn.Module):
def __init__(self,
d_in,
d_out,
d_hidden,
n_layers,
skip_in=(4,),
bias=0.5,
geometric_init=True,
weight_norm=True,
inside_outside=False,
encoding_config=None,
input_concat=False):
super(SDFNetwork, self).__init__()
self.input_concat = input_concat
dims = [d_in] + [d_hidden for _ in range(n_layers)] + [d_out]
if encoding_config is not None:
self.encoding = tcnn.Encoding(d_in, encoding_config).to(torch.float32)
dims[0] = self.encoding.n_output_dims
if input_concat:
dims[0] += d_in
else:
self.encoding = None
self.num_layers = len(dims)
self.skip_in = skip_in
self.bindwidth = 0
self.enc_dim = self.encoding.n_output_dims
for l in range(0, self.num_layers - 1):
if l + 1 in self.skip_in:
out_dim = dims[l + 1] - dims[0]
else:
out_dim = dims[l + 1]
lin = nn.Linear(dims[l], out_dim)
if geometric_init:
if l == self.num_layers - 2:
if not inside_outside:
torch.nn.init.normal_(lin.weight, mean=np.sqrt(np.pi) / np.sqrt(dims[l]), std=0.0001)
torch.nn.init.constant_(lin.bias, -bias)
else:
torch.nn.init.normal_(lin.weight, mean=-np.sqrt(np.pi) / np.sqrt(dims[l]), std=0.0001)
torch.nn.init.constant_(lin.bias, bias)
elif self.encoding is not None and l == 0:
torch.nn.init.constant_(lin.bias, 0.0)
torch.nn.init.constant_(lin.weight[:, 3:], 0.0)
torch.nn.init.normal_(lin.weight[:, :3], 0.0, np.sqrt(2) / np.sqrt(out_dim))
elif self.encoding is not None and l in self.skip_in:
torch.nn.init.constant_(lin.bias, 0.0)
torch.nn.init.normal_(lin.weight, 0.0, np.sqrt(2) / np.sqrt(out_dim))
torch.nn.init.constant_(lin.weight[:, -(dims[0] - 3):], 0.0)
else:
torch.nn.init.constant_(lin.bias, 0.0)
torch.nn.init.normal_(lin.weight, 0.0, np.sqrt(2) / np.sqrt(out_dim))
if weight_norm:
lin = nn.utils.weight_norm(lin)
setattr(self, "lin" + str(l), lin)
self.activation = nn.Softplus(beta=100)
# self.activation = nn.ReLU()
def increase_bandwidth(self):
self.bindwidth += 1
def forward(self, inputs):
if self.encoding is not None:
encoded = self.encoding(inputs).to(torch.float32)
# set the dimension of the encoding to 0 if the input is outside the bandwidth
enc_mask = torch.ones(self.enc_dim, dtype=torch.bool, device=encoded.device, requires_grad=False)
enc_mask[self.bindwidth*2:] = 0
encoded = encoded * enc_mask
if self.input_concat:
inputs = torch.cat([inputs, encoded], dim=1)
x = inputs
for l in range(0, self.num_layers - 1):
lin = getattr(self, "lin" + str(l))
if l in self.skip_in:
x = torch.cat([x, inputs], 1) / np.sqrt(2)
x = lin(x)
if l < self.num_layers - 2:
x = self.activation(x)
return x
def sdf(self, x):
return self.forward(x)[:, :1]
def sdf_hidden_appearance(self, x):
return self.forward(x)
@torch.enable_grad()
def gradient(self, x):
x.requires_grad_(True)
y = self.sdf(x)
d_output = torch.ones_like(y, requires_grad=False, device=y.device)
gradients = torch.autograd.grad(
outputs=y,
inputs=x,
grad_outputs=d_output,
create_graph=True,
retain_graph=True,
only_inputs=True)[0]
return gradients.unsqueeze(1)
@torch.enable_grad()
def divergence(self, y, x):
div = 0.
for i in range(y.shape[-1]):
div += torch.autograd.grad(y[..., i], x, torch.ones_like(y[..., i]), create_graph=True)[0][..., i:i + 1]
return div
@torch.enable_grad()
def laplace(self, x):
return self.divergence(self.gradient(x), x)
class SingleVarianceNetwork(nn.Module):
def __init__(self, init_val):
super(SingleVarianceNetwork, self).__init__()
self.register_parameter('variance', nn.Parameter(torch.tensor(init_val)))
def forward(self, x):
return torch.ones([len(x), 1]) * torch.exp(self.variance * 10.0)
================================================
FILE: models/renderer.py
================================================
import torch
import numpy as np
import mcubes
from tqdm import tqdm
from nerfacc import ContractionType, OccupancyGrid, ray_marching, \
render_weight_from_alpha_patch_based, accumulate_along_rays_patch_based, \
render_weight_from_alpha, accumulate_along_rays
def extract_fields(bound_min, bound_max, resolution, query_func):
N = 64
X = torch.linspace(bound_min[0], bound_max[0], resolution).split(N)
Y = torch.linspace(bound_min[1], bound_max[1], resolution).split(N)
Z = torch.linspace(bound_min[2], bound_max[2], resolution).split(N)
u = np.zeros([resolution, resolution, resolution], dtype=np.float32)
with torch.no_grad():
for xi, xs in tqdm(enumerate(X)):
for yi, ys in enumerate(Y):
for zi, zs in enumerate(Z):
xx, yy, zz = torch.meshgrid(xs, ys, zs)
pts = torch.cat([xx.reshape(-1, 1), yy.reshape(-1, 1), zz.reshape(-1, 1)], dim=-1)
val = query_func(pts).reshape(len(xs), len(ys), len(zs)).detach().cpu().numpy()
u[xi * N: xi * N + len(xs), yi * N: yi * N + len(ys), zi * N: zi * N + len(zs)] = val
return u
def extract_geometry(bound_min, bound_max, resolution, threshold, query_func):
u = extract_fields(bound_min, bound_max, resolution, query_func)
vertices, triangles = mcubes.marching_cubes(u, threshold)
b_max_np = bound_max.detach().cpu().numpy()
b_min_np = bound_min.detach().cpu().numpy()
vertices = vertices / (resolution - 1.0) * (b_max_np - b_min_np)[None, :] + b_min_np[None, :]
return vertices, triangles
class NeuSRenderer:
def __init__(self, sdf_network, deviation_network,
gradient_method="dfd"):
self.sdf_network = sdf_network
self.deviation_network = deviation_network
# define the occ grid, see NerfAcc for more details
self.scene_aabb = torch.as_tensor([-1., -1., -1., 1., 1., 1.], dtype=torch.float32)
# define the contraction_type for scene contraction
self.contraction_type = ContractionType.AABB
# create Occupancy Grid
self.occupancy_grid = OccupancyGrid(
roi_aabb=self.scene_aabb,
resolution=128, # if res is different along different axis, use [256,128,64]
contraction_type=self.contraction_type).to("cuda")
self.sampling_step_size = 0.01 # ray marching step size, will be modified during training
self.gradient_method = gradient_method # dfd or fd or ad
def occ_eval_fn(self, x):
# function for updating the occ grid given the current sdf
sdf = self.sdf_network(x)[..., :1]
alpha = torch.sigmoid(- sdf * 80) # occ grids with alpha below the occ threshold will be set as 0
return alpha
def render(self, rays_o_patch_all, # (num_patch, patch_H, patch_W, 3)
rays_d_patch_all, # (num_patch, patch_H, patch_W, 3)
marching_plane_normal, # (num_patch, 3)
near, # (num_patch,)
far, # (num_patch,)
V_inverse_patch_all, # (num_patch, patch_H, patch_W, 3, 3)
val_gradient_method='dfd',
mode='train'):
# patch size, should be odd
patch_H = rays_o_patch_all.shape[1]
patch_W = rays_o_patch_all.shape[2]
num_patch = rays_o_patch_all.shape[0]
# extract camera location and ray direction of the patches' center pixels
rays_o_patch_center = rays_o_patch_all[:, patch_H//2, patch_W//2] # (num_patch, 3)
rays_d_patch_center = rays_d_patch_all[:, patch_H//2, patch_W//2] # (num_patch, 3)
def alpha_fn_patch_center(t_starts, t_ends, ray_indices, ret_sdf=False):
# the function used in ray marching
ray_indices = ray_indices.long()
t_origins = rays_o_patch_center[ray_indices]
t_dirs = rays_d_patch_center[ray_indices]
positions_starts = t_origins + t_dirs * t_starts
positions_ends = t_origins + t_dirs * t_ends
t_starts_shift_left = t_starts[1:]
# attach the last element of t_ends to the end of t_starts_shift_left
t_starts_shift_left = torch.cat([t_starts_shift_left, t_starts[-1:]], 0)
# compute the diff mask between t_ends and t_starts_shift_left
diff_mask = ((t_ends - t_starts_shift_left) != 0).squeeze()
# if the diff maks is empty, return
positions_ends_diff = positions_ends[diff_mask].reshape(-1, 3)
positions_all = torch.cat([positions_starts, positions_ends_diff], 0)
sdf_all = self.sdf_network(positions_all)
sdf_start = sdf_all[:positions_starts.shape[0]]
sdf_end_diff = sdf_all[positions_starts.shape[0]:]
sdf_start_shift_left = sdf_start[1:]
sdf_start_shift_left = torch.cat([sdf_start_shift_left, sdf_start[-1:]], 0)
sdf_start_shift_left[diff_mask] = sdf_end_diff
inv_s = self.deviation_network(torch.zeros([1, 3]))[:, :1].clip(1e-6, 1e6) # Single parameter
inv_s = inv_s.expand(sdf_start.shape[0], 1)
prev_cdf = torch.sigmoid(sdf_start * inv_s)
next_cdf = torch.sigmoid(sdf_start_shift_left * inv_s)
p = prev_cdf - next_cdf
c = prev_cdf
alpha = ((p + 1e-5) / (c + 1e-5)).view(-1).clip(0.0, 1.0)
alpha = alpha.reshape(-1, 1)
if ret_sdf:
return alpha, sdf_start, sdf_start_shift_left
else:
return alpha
with torch.no_grad():
patch_indices, t_starts_patch_center, t_ends_patch_center = ray_marching(
rays_o_patch_center, rays_d_patch_center,
t_min=near,
t_max=far,
grid=self.occupancy_grid,
render_step_size=self.sampling_step_size,
stratified=True,
cone_angle=0.0,
early_stop_eps=1e-8,
alpha_fn=alpha_fn_patch_center,
)
samples_per_ray = patch_indices.shape[0] / num_patch
if patch_indices.shape[0] == 0: # all patch center rays are within the zero region of the occ grid. skip this iteration.
return {
"comp_normal": torch.zeros([num_patch, patch_H, patch_W, 3], device=rays_o_patch_center.device)
}
num_samples = patch_indices.shape[0]
patch_indices = patch_indices.long()
# compute the sampling distance on remaining rays
t_starts_patch_all = t_starts_patch_center[:, None, None, :] * (rays_d_patch_center * marching_plane_normal).sum(-1, keepdim=True)[patch_indices][:, None, None, :] \
/(rays_d_patch_all * marching_plane_normal[:, None, None, :]).sum(-1, keepdim=True)[patch_indices]
t_ends_patch_all = t_ends_patch_center[:, None, None, :] * (rays_d_patch_center * marching_plane_normal).sum(-1, keepdim=True)[patch_indices][:, None, None, :] \
/(rays_d_patch_all * marching_plane_normal[:, None, None, :]).sum(-1, keepdim=True)[patch_indices]
t_starts_patch_center_shift_left = t_starts_patch_center[1:]
t_starts_patch_center_shift_left = torch.cat([t_starts_patch_center_shift_left, t_starts_patch_center[-1:]], 0)
diff_mask = ((t_ends_patch_center - t_starts_patch_center_shift_left) != 0)[..., 0]
positions_starts_patch_all = rays_o_patch_all[patch_indices] + rays_d_patch_all[patch_indices] * t_starts_patch_all
positions_ends_patch_all = rays_o_patch_all[patch_indices] + rays_d_patch_all[patch_indices] * t_ends_patch_all # (num_samples, patch_H, patch_W, 3)
positions_ends_diff = positions_ends_patch_all[diff_mask]
positions_all = torch.cat([positions_starts_patch_all, positions_ends_diff], 0)
positions_all_flat = positions_all.reshape(-1, 3)
sdf_all = self.sdf_network(positions_all_flat)
sdf_all = sdf_all.reshape(*positions_all.shape[:-1], 1)
sdf_starts_patch_all = sdf_all[:positions_starts_patch_all.shape[0]]
sdf_end_diff = sdf_all[positions_starts_patch_all.shape[0]:]
sdf_ends_patch_all = sdf_starts_patch_all[1:]
sdf_ends_patch_all = torch.cat([sdf_ends_patch_all, sdf_starts_patch_all[-1:]], 0)
sdf_ends_patch_all[diff_mask] = sdf_end_diff
inv_s = self.deviation_network(torch.zeros([1, 3]))[:, :1].clip(1e-6, 1e6) # Single parameter
prev_cdf = torch.sigmoid(sdf_starts_patch_all * inv_s) # (num_samples, patch_H, patch_W, 1)
next_cdf = torch.sigmoid(sdf_ends_patch_all * inv_s) # (num_samples, patch_H, patch_W, 1)
p = prev_cdf - next_cdf
c = prev_cdf
alpha = ((p + 1e-5) / (c + 1e-5)).clip(0.0, 1.0) # (num_samples, patch_H, patch_W, 1)
weights_cuda = render_weight_from_alpha_patch_based(alpha.reshape(num_samples, patch_H*patch_W, 1), patch_indices) # (num_samples, patch_H, patch_W, 1)
if mode == 'train':
gradient_method = self.gradient_method
elif mode == 'eval':
gradient_method = val_gradient_method
if gradient_method == "dfd":
with torch.no_grad():
# distance between neighboring points on the same marching plane
dist_x = torch.norm(positions_starts_patch_all[:, :, 1:, :] -
positions_starts_patch_all[:, :, :-1, :], dim=-1, keepdim=True) # (num_samples, patch_H, patch_W-1, 1)
dist_y = torch.norm(positions_starts_patch_all[:, 1:, :, :] -
positions_starts_patch_all[:, :-1, :, :], dim=-1, keepdim=True) # (num_samples, patch_H-1, patch_W, 1)
# directional derivatives along the ray direction
# forward difference
df_dt = (sdf_ends_patch_all - sdf_starts_patch_all) / (t_ends_patch_all - t_starts_patch_all) # (num_samples, patch_H, patch_W, 1)
# directional derivatives along the image's x-direction
# central difference
df_dx = (sdf_starts_patch_all[:, :, 2:] - sdf_starts_patch_all[:, :, :-2]) / (dist_x[:, :, :-1] + dist_x[:, :, 1:] ) # (num_samples, patch_H, patch_W-2, 1)
# directional derivatives along the image's y-direction
# central difference
df_dy = (sdf_starts_patch_all[:, 2:, :] - sdf_starts_patch_all[:, :-2, :]) / (dist_y[:, 1:, :] + dist_y[:, :-1, :]) # (num_samples, patch_H-2, patch_W, 1)
# for points only have one-side neighbor point,
# we use forward or backward difference correspondingly
df_dx_left_boundary = (sdf_starts_patch_all[:, :, 1:2] - sdf_starts_patch_all[:, :, 0:1]) / dist_x[:, :, 0:1] # (num_samples, patch_H, 1)
df_dx_right_boundary = (sdf_starts_patch_all[:, :, -1:] - sdf_starts_patch_all[:, :, -2:-1]) / dist_x[:, :, -1:] # (num_samples, patch_H, 1)
df_dy_top_boundary = (sdf_starts_patch_all[:, 1:2, :] - sdf_starts_patch_all[:, 0:1, :]) / dist_y[:, 0:1, :] # (num_samples, 1, patch_W)
df_dy_bottom_boundary = (sdf_starts_patch_all[:, -1:, :] - sdf_starts_patch_all[:, -2:-1, :]) / dist_y[:, -1:, :] # (num_samples, 1, patch_W)
# concat the directional derivatives for boundary points and central points
df_dx = torch.cat([df_dx_left_boundary, df_dx, df_dx_right_boundary], dim=2) # (num_samples, patch_H, patch_W, 1)
df_dy = torch.cat([df_dy_top_boundary, df_dy, df_dy_bottom_boundary], dim=1) # (num_samples, patch_H, patch_W, 1)
# concat the directional partial derivatives in three directions
projected_gradients = torch.cat([df_dt,
df_dx,
df_dy], dim=-1) # (num_patches, patch_H, patch_W, 3)
# recover the gradients from directional partial derivatives using the inverse of known directions
V_inverse = V_inverse_patch_all[patch_indices] # (num_patches, patch_H, patch_W, 3, 3)
gradients = (V_inverse @ projected_gradients[..., None])[..., 0] # (num_samples, patch_H, patch_W, 3)
elif gradient_method == "ad":
gradients = self.sdf_network.gradient(positions_starts_patch_all.reshape(-1, 3)).reshape(num_samples, patch_H, patch_W, 3)
elif gradient_method == "fd":
# 6-point finite difference
self.fd_epsilon = 1e-3
positions_xn = positions_starts_patch_all + torch.tensor([[[[-self.fd_epsilon, 0, 0]]]], device=positions_starts_patch_all.device).expand(
positions_starts_patch_all.shape)
positions_xp = positions_starts_patch_all + torch.tensor([[[[self.fd_epsilon, 0, 0]]]], device=positions_starts_patch_all.device).expand(
positions_starts_patch_all.shape)
positions_yn = positions_starts_patch_all + torch.tensor([[[[0, -self.fd_epsilon, 0]]]], device=positions_starts_patch_all.device).expand(
positions_starts_patch_all.shape)
positions_yp = positions_starts_patch_all + torch.tensor([[[[0, self.fd_epsilon, 0]]]], device=positions_starts_patch_all.device).expand(
positions_starts_patch_all.shape)
positions_zn = positions_starts_patch_all + torch.tensor([[[[0, 0, -self.fd_epsilon]]]], device=positions_starts_patch_all.device).expand(
positions_starts_patch_all.shape)
positions_zp = positions_starts_patch_all + torch.tensor([[[[0, 0, self.fd_epsilon]]]], device=positions_starts_patch_all.device).expand(
positions_starts_patch_all.shape)
positions_concat = torch.cat(
[positions_xn, positions_xp, positions_yn, positions_yp, positions_zn, positions_zp], 0).to(
torch.float32).reshape(-1, 3)
sdf_concat = self.sdf_network(positions_concat).reshape(-1, patch_H, patch_W, 1)
num_samples = positions_starts_patch_all.shape[0]
sdf_xn = sdf_concat[:num_samples].reshape(num_samples, patch_H, patch_W, 1)
sdf_xp = sdf_concat[num_samples:2 * num_samples].reshape(num_samples, patch_H, patch_W, 1)
sdf_yn = sdf_concat[2 * num_samples:3 * num_samples].reshape(num_samples, patch_H, patch_W, 1)
sdf_yp = sdf_concat[3 * num_samples:4 * num_samples].reshape(num_samples, patch_H, patch_W, 1)
sdf_zn = sdf_concat[4 * num_samples:5 * num_samples].reshape(num_samples, patch_H, patch_W, 1)
sdf_zp = sdf_concat[5 * num_samples:].reshape(num_samples, patch_H, patch_W, 1)
df_dx = (sdf_xp - sdf_xn) / (2 * self.fd_epsilon)
df_dy = (sdf_yp - sdf_yn) / (2 * self.fd_epsilon)
df_dz = (sdf_zp - sdf_zn) / (2 * self.fd_epsilon)
gradients = torch.stack([df_dx, df_dy, df_dz], -1)
weights_sum_cuda = accumulate_along_rays_patch_based(weights_cuda, patch_indices, n_patches=num_patch) # (num_samples, patch_H, patch_W, 1)
weights_sum = weights_sum_cuda.reshape(num_patch, patch_H, patch_W, 1)
comp_normals_cuda = accumulate_along_rays_patch_based(weights_cuda, patch_indices, values=gradients.reshape(num_samples,patch_H * patch_W, 3),n_patches=num_patch) # (num_samples, patch_H, patch_W, 3)
comp_normal = comp_normals_cuda.reshape(num_patch, patch_H, patch_W, 3)
inv_s = self.deviation_network(torch.zeros([1, 3]))[:, :1].clip(1e-6, 1e6) # Single parameter
return {
's_val': 1/inv_s,
'weight_sum': weights_sum,
'gradients': gradients,
"comp_normal": comp_normal,
"samples_per_ray": samples_per_ray,
}
@torch.no_grad()
def render_normal_pixel_based(self, rays_o, rays_d, near, far):
def alpha_fn(t_starts, t_ends, ray_indices, ret_sdf=False):
ray_indices = ray_indices.long()
t_origins = rays_o[ray_indices]
t_dirs = rays_d[ray_indices]
positions_starts = t_origins + t_dirs * t_starts
positions_ends = t_origins + t_dirs * t_ends
t_starts_shift_left = t_starts[1:]
# attach the last element of t_ends to the end of t_starts_shift_left
t_starts_shift_left = torch.cat([t_starts_shift_left, t_starts[-1:]], 0)
# compute the diff mask between t_ends and t_starts_shift_left
diff_mask = ((t_ends - t_starts_shift_left) != 0).squeeze()
# if the diff maks is empty, return
positions_ends_diff = positions_ends[diff_mask].reshape(-1, 3)
# ic(diff_mask.shape, positions_ends_diff.shape, positions_starts.shape)
positions_all = torch.cat([positions_starts, positions_ends_diff], 0)
sdf_all = self.sdf_network(positions_all)
sdf_start = sdf_all[:positions_starts.shape[0]]
sdf_end_diff = sdf_all[positions_starts.shape[0]:]
sdf_start_shift_left = sdf_start[1:]
sdf_start_shift_left = torch.cat([sdf_start_shift_left, sdf_start[-1:]], 0)
sdf_start_shift_left[diff_mask] = sdf_end_diff
inv_s = self.deviation_network(torch.zeros([1, 3]))[:, :1].clip(1e-6, 1e6) # Single parameter
inv_s = inv_s.expand(sdf_start.shape[0], 1)
prev_cdf = torch.sigmoid(sdf_start * inv_s)
next_cdf = torch.sigmoid(sdf_start_shift_left * inv_s)
p = prev_cdf - next_cdf
c = prev_cdf
alpha = ((p + 1e-5) / (c + 1e-5)).view(-1).clip(0.0, 1.0)
alpha = alpha.reshape(-1, 1)
if ret_sdf:
return alpha, sdf_start, sdf_start_shift_left
else:
return alpha
ray_indices, t_starts, t_ends = ray_marching(
rays_o, rays_d,
t_min=near.squeeze(),
t_max=far.squeeze(),
grid=self.occupancy_grid,
render_step_size=self.sampling_step_size,
stratified=True,
cone_angle=0.0,
alpha_thre=0.0,
early_stop_eps=1e-3,
alpha_fn=alpha_fn,
)
alpha = alpha_fn(t_starts, t_ends, ray_indices)
ray_indices = ray_indices.long()
t_origins = rays_o[ray_indices]
t_dirs = rays_d[ray_indices]
midpoints = (t_starts + t_ends) / 2.
positions = t_origins + t_dirs * midpoints
gradients = self.sdf_network.gradient(positions).reshape(-1, 3)
n_rays = rays_o.shape[0]
weights = render_weight_from_alpha(alpha, ray_indices=ray_indices, n_rays=n_rays) # [n_samples, 1]
comp_normal = accumulate_along_rays(weights, ray_indices, values=gradients, n_rays=n_rays)
comp_depth = accumulate_along_rays(weights, ray_indices, values=midpoints, n_rays=n_rays)
return comp_normal, comp_depth
def extract_geometry(self, bound_min, bound_max, resolution, threshold=0.0):
return extract_geometry(bound_min,
bound_max,
resolution=resolution,
threshold=threshold,
query_func=lambda pts: -self.sdf_network.sdf(pts))
================================================
FILE: run_diligent.sh
================================================
for obj_name in buddha pot2 reading bear cow; do
python exp_runner.py --conf config/diligent.conf --obj_name $obj_name
done
================================================
FILE: run_own_object.sh
================================================
for obj_name in lion dog1 woman; do
python exp_runner.py --conf config/own_objects.conf --obj_name $obj_name
done
================================================
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.github/workflows/building.yml
================================================
name: Building Wheels
on: [workflow_dispatch]
jobs:
wheel:
runs-on: ${{ matrix.os }}
environment: production
strategy:
fail-fast: false
matrix:
os: [ubuntu-18.04, windows-2019]
python-version: ['3.7', '3.8', '3.9']
torch-version: [1.10.0, 1.11.0, 1.12.0, 1.13.0]
cuda-version: ['cu102', 'cu113', 'cu116', 'cu117']
# os: [ubuntu-18.04]
# python-version: ['3.9']
# torch-version: [1.10.0]
# cuda-version: ['cu102']
exclude:
- torch-version: 1.10.0
cuda-version: 'cu116'
- torch-version: 1.10.0
cuda-version: 'cu117'
- torch-version: 1.11.0
cuda-version: 'cu116'
- torch-version: 1.11.0
cuda-version: 'cu117'
- torch-version: 1.12.0
cuda-version: 'cu117'
- torch-version: 1.13.0
cuda-version: 'cu102'
- torch-version: 1.13.0
cuda-version: 'cu113'
- os: windows-2019
torch-version: 1.11.0
cuda-version: 'cu102'
- os: windows-2019
torch-version: 1.12.0
cuda-version: 'cu102'
# - os: macos-10.15
# cuda-version: 'cu102'
# - os: macos-10.15
# cuda-version: 'cu113'
# - os: macos-10.15
# cuda-version: 'cu116'
# - os: macos-10.15
# cuda-version: 'cu117'
steps:
- uses: actions/checkout@v2
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Upgrade pip
run: |
pip install --upgrade setuptools
pip install ninja
- name: Free up disk space
if: ${{ runner.os == 'Linux' }}
run: |
sudo rm -rf /usr/share/dotnet
- name: Install CUDA ${{ matrix.cuda-version }}
if: ${{ matrix.cuda-version != 'cpu' }}
run: |
bash .github/workflows/cuda/${{ matrix.cuda-version }}-${{ runner.os }}.sh
- name: Install PyTorch ${{ matrix.torch-version }}+${{ matrix.cuda-version }}
run: |
pip install torch==${{ matrix.torch-version }} --extra-index-url https://download.pytorch.org/whl/${{ matrix.cuda-version }}
python -c "import torch; print('PyTorch:', torch.__version__)"
python -c "import torch; print('CUDA:', torch.version.cuda)"
python -c "import torch; print('CUDA Available:', torch.cuda.is_available())"
- name: Patch PyTorch static constexpr on Windows
if: ${{ runner.os == 'Windows' }}
run: |
Torch_DIR=`python -c 'import os; import torch; print(os.path.dirname(torch.__file__))'`
sed -i '31,38c\
TORCH_API void lazy_init_num_threads();' ${Torch_DIR}/include/ATen/Parallel.h
shell: bash
- name: Set version
if: ${{ runner.os != 'macOS' }}
run: |
VERSION=`sed -n 's/^__version__ = "\(.*\)"/\1/p' nerfacc/version.py`
TORCH_VERSION=`echo "pt${{ matrix.torch-version }}" | sed "s/..$//" | sed "s/\.//g"`
CUDA_VERSION=`echo ${{ matrix.cuda-version }}`
echo "New version name: $VERSION+$TORCH_VERSION$CUDA_VERSION"
sed -i "s/$VERSION/$VERSION+$TORCH_VERSION$CUDA_VERSION/" nerfacc/version.py
shell:
bash
- name: Install main package for CPU
if: ${{ matrix.cuda-version == 'cpu' }}
run: |
FORCE_ONLY_CPU=1 pip install -e .
shell:
bash
- name: Install main package for GPU
if: ${{ matrix.cuda-version != 'cpu' }}
run: |
source .github/workflows/cuda/${{ matrix.cuda-version }}-${{ runner.os }}-env.sh
pip install .
shell:
bash
- name: Test installation
run: |
python -c "import nerfacc; print('nerfacc:', nerfacc.__version__)"
- name: Build wheel
run: |
pip install wheel
source .github/workflows/cuda/${{ matrix.cuda-version }}-${{ runner.os }}-env.sh
python setup.py bdist_wheel --dist-dir=dist
shell: bash
- name: Configure AWS
uses: aws-actions/configure-aws-credentials@v1
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: us-west-2
- name: Upload wheel
run: |
aws s3 sync dist s3://nerfacc-bucket/whl/torch-${{ matrix.torch-version }}_${{ matrix.cuda-version }} --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers
update_aws_listing:
needs: [wheel]
runs-on: ubuntu-latest
environment: production
steps:
- uses: actions/checkout@v2
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: 3.9
- name: Upgrade pip
run: |
pip install --upgrade setuptools
pip install boto3
- name: Configure AWS
uses: aws-actions/configure-aws-credentials@v1
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: us-west-2
- name: Update AWS listing
run: |
python scripts/run_aws_listing.py \
--access_key_id=${{ secrets.AWS_ACCESS_KEY_ID }} \
--secret_access_key=${{ secrets.AWS_SECRET_ACCESS_KEY }} \
--bucket="nerfacc-bucket" \
--region="us-west-2"
================================================
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.github/workflows/code_checks.yml
================================================
name: Core Tests.
on:
push:
branches: [master]
pull_request:
branches: [master]
permissions:
contents: read
jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Set up Python 3.8.12
uses: actions/setup-python@v4
with:
python-version: "3.8.12"
- name: Install dependencies
run: |
pip install isort==5.10.1 black[jupyter]==22.3.0
- name: Run isort
run: isort docs/ nerfacc/ scripts/ examples/ tests/ --profile black --skip examples/pycolmap --line-length 80 --check
- name: Run Black
run: black docs/ nerfacc/ scripts/ examples/ tests/ --exclude examples/pycolmap --line-length 80 --check
# - name: Python Pylint
# run: |
# pylint nerfacc/ tests/ scripts/ examples/
================================================
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.github/workflows/cuda/cu101-Linux-env.sh
================================================
#!/bin/bash
CUDA_HOME=/usr/local/cuda-10.1
LD_LIBRARY_PATH=${CUDA_HOME}/lib64:${LD_LIBRARY_PATH}
PATH=${CUDA_HOME}/bin:${PATH}
export FORCE_CUDA=1
export TORCH_CUDA_ARCH_LIST="3.5;5.0+PTX;6.0;7.0;7.5"
================================================
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.github/workflows/cuda/cu101-Linux.sh
================================================
#!/bin/bash
OS=ubuntu1804
wget -nv https://developer.download.nvidia.com/compute/cuda/repos/${OS}/x86_64/cuda-${OS}.pin
sudo mv cuda-${OS}.pin /etc/apt/preferences.d/cuda-repository-pin-600
wget -nv https://developer.download.nvidia.com/compute/cuda/10.1/Prod/local_installers/cuda-repo-${OS}-10-1-local-10.1.243-418.87.00_1.0-1_amd64.deb
sudo dpkg -i cuda-repo-${OS}-10-1-local-10.1.243-418.87.00_1.0-1_amd64.deb
sudo apt-key add /var/cuda-repo-10-1-local-10.1.243-418.87.00/7fa2af80.pub
sudo apt-get -qq update
sudo apt install -y cuda-nvcc-10-1 cuda-libraries-dev-10-1
sudo apt clean
rm -f https://developer.download.nvidia.com/compute/cuda/10.1/Prod/local_installers/cuda-repo-${OS}-10-1-local-10.1.243-418.87.00_1.0-1_amd64.deb
================================================
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.github/workflows/cuda/cu101-Windows-env.sh
================================================
#!/bin/bash
CUDA_HOME=/c/Program\ Files/NVIDIA\ GPU\ Computing\ Toolkit/CUDA/v10.1
PATH=${CUDA_HOME}/bin:$PATH
PATH=/c/Program\ Files\ \(x86\)/Microsoft\ Visual\ Studio/2017/BuildTools/MSBuild/15.0/Bin:$PATH
export FORCE_CUDA=1
export TORCH_CUDA_ARCH_LIST="3.5;5.0+PTX;6.0;7.0;7.5"
================================================
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.github/workflows/cuda/cu101-Windows.sh
================================================
#!/bin/bash
# Install NVIDIA drivers, see:
# https://github.com/pytorch/vision/blob/master/packaging/windows/internal/cuda_install.bat#L99-L102
curl -k -L "https://drive.google.com/u/0/uc?id=1injUyo3lnarMgWyRcXqKg4UGnN0ysmuq&export=download" --output "/tmp/gpu_driver_dlls.zip"
7z x "/tmp/gpu_driver_dlls.zip" -o"/c/Windows/System32"
export CUDA_SHORT=10.1
export CUDA_URL=https://developer.download.nvidia.com/compute/cuda/${CUDA_SHORT}/Prod/local_installers/
export CUDA_FILE=cuda_${CUDA_SHORT}.243_426.00_win10.exe
# Install CUDA:
curl -k -L "${CUDA_URL}/${CUDA_FILE}" --output "${CUDA_FILE}"
echo ""
echo "Installing from ${CUDA_FILE}..."
PowerShell -Command "Start-Process -FilePath \"${CUDA_FILE}\" -ArgumentList \"-s nvcc_${CUDA_SHORT} cuobjdump_${CUDA_SHORT} nvprune_${CUDA_SHORT} cupti_${CUDA_SHORT} cublas_dev_${CUDA_SHORT} cudart_${CUDA_SHORT} cufft_dev_${CUDA_SHORT} curand_dev_${CUDA_SHORT} cusolver_dev_${CUDA_SHORT} cusparse_dev_${CUDA_SHORT} npp_dev_${CUDA_SHORT} nvrtc_dev_${CUDA_SHORT} nvml_dev_${CUDA_SHORT}\" -Wait -NoNewWindow"
echo "Done!"
rm -f "${CUDA_FILE}"
================================================
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.github/workflows/cuda/cu102-Linux-env.sh
================================================
#!/bin/bash
CUDA_HOME=/usr/local/cuda-10.2
LD_LIBRARY_PATH=${CUDA_HOME}/lib64:${LD_LIBRARY_PATH}
PATH=${CUDA_HOME}/bin:${PATH}
export FORCE_CUDA=1
export TORCH_CUDA_ARCH_LIST="3.5;5.0+PTX;6.0;7.0;7.5"
================================================
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.github/workflows/cuda/cu102-Linux.sh
================================================
#!/bin/bash
OS=ubuntu1804
wget -nv https://developer.download.nvidia.com/compute/cuda/repos/${OS}/x86_64/cuda-${OS}.pin
sudo mv cuda-${OS}.pin /etc/apt/preferences.d/cuda-repository-pin-600
wget -nv https://developer.download.nvidia.com/compute/cuda/10.2/Prod/local_installers/cuda-repo-${OS}-10-2-local-10.2.89-440.33.01_1.0-1_amd64.deb
sudo dpkg -i cuda-repo-${OS}-10-2-local-10.2.89-440.33.01_1.0-1_amd64.deb
sudo apt-key add /var/cuda-repo-10-2-local-10.2.89-440.33.01/7fa2af80.pub
sudo apt-get -qq update
sudo apt install -y cuda-nvcc-10-2 cuda-libraries-dev-10-2
sudo apt clean
rm -f https://developer.download.nvidia.com/compute/cuda/10.2/Prod/local_installers/cuda-repo-${OS}-10-2-local-10.2.89-440.33.01_1.0-1_amd64.deb
================================================
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.github/workflows/cuda/cu102-Windows-env.sh
================================================
#!/bin/bash
CUDA_HOME=/c/Program\ Files/NVIDIA\ GPU\ Computing\ Toolkit/CUDA/v10.2
PATH=${CUDA_HOME}/bin:$PATH
PATH=/c/Program\ Files\ \(x86\)/Microsoft\ Visual\ Studio/2017/BuildTools/MSBuild/15.0/Bin:$PATH
export FORCE_CUDA=1
export TORCH_CUDA_ARCH_LIST="3.5;5.0+PTX;6.0;7.0;7.5"
================================================
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.github/workflows/cuda/cu102-Windows.sh
================================================
#!/bin/bash
# Install NVIDIA drivers, see:
# https://github.com/pytorch/vision/blob/master/packaging/windows/internal/cuda_install.bat#L99-L102
curl -k -L "https://drive.google.com/u/0/uc?id=1injUyo3lnarMgWyRcXqKg4UGnN0ysmuq&export=download" --output "/tmp/gpu_driver_dlls.zip"
7z x "/tmp/gpu_driver_dlls.zip" -o"/c/Windows/System32"
export CUDA_SHORT=10.2
export CUDA_URL=https://developer.download.nvidia.com/compute/cuda/${CUDA_SHORT}/Prod/local_installers
export CUDA_FILE=cuda_${CUDA_SHORT}.89_441.22_win10.exe
# Install CUDA:
curl -k -L "${CUDA_URL}/${CUDA_FILE}" --output "${CUDA_FILE}"
echo ""
echo "Installing from ${CUDA_FILE}..."
PowerShell -Command "Start-Process -FilePath \"${CUDA_FILE}\" -ArgumentList \"-s nvcc_${CUDA_SHORT} cuobjdump_${CUDA_SHORT} nvprune_${CUDA_SHORT} cupti_${CUDA_SHORT} cublas_dev_${CUDA_SHORT} cudart_${CUDA_SHORT} cufft_dev_${CUDA_SHORT} curand_dev_${CUDA_SHORT} cusolver_dev_${CUDA_SHORT} cusparse_dev_${CUDA_SHORT} npp_dev_${CUDA_SHORT} nvrtc_dev_${CUDA_SHORT} nvml_dev_${CUDA_SHORT}\" -Wait -NoNewWindow"
echo "Done!"
rm -f "${CUDA_FILE}"
================================================
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.github/workflows/cuda/cu111-Linux-env.sh
================================================
#!/bin/bash
CUDA_HOME=/usr/local/cuda-11.1
LD_LIBRARY_PATH=${CUDA_HOME}/lib64:${LD_LIBRARY_PATH}
PATH=${CUDA_HOME}/bin:${PATH}
export FORCE_CUDA=1
export TORCH_CUDA_ARCH_LIST="3.5;5.0+PTX;6.0;7.0;7.5;8.0;8.6"
================================================
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.github/workflows/cuda/cu111-Linux.sh
================================================
#!/bin/bash
OS=ubuntu1804
wget -nv https://developer.download.nvidia.com/compute/cuda/repos/${OS}/x86_64/cuda-${OS}.pin
sudo mv cuda-${OS}.pin /etc/apt/preferences.d/cuda-repository-pin-600
wget -nv https://developer.download.nvidia.com/compute/cuda/11.1.1/local_installers/cuda-repo-${OS}-11-1-local_11.1.1-455.32.00-1_amd64.deb
sudo dpkg -i cuda-repo-${OS}-11-1-local_11.1.1-455.32.00-1_amd64.deb
sudo apt-key add /var/cuda-repo-${OS}-11-1-local/7fa2af80.pub
sudo apt-get -qq update
sudo apt install -y cuda-nvcc-11-1 cuda-libraries-dev-11-1
sudo apt clean
rm -f https://developer.download.nvidia.com/compute/cuda/11.1.1/local_installers/cuda-repo-${OS}-11-1-local_11.1.1-455.32.00-1_amd64.deb
================================================
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.github/workflows/cuda/cu111-Windows-env.sh
================================================
#!/bin/bash
CUDA_HOME=/c/Program\ Files/NVIDIA\ GPU\ Computing\ Toolkit/CUDA/v11.1
PATH=${CUDA_HOME}/bin:$PATH
PATH=/c/Program\ Files\ \(x86\)/Microsoft\ Visual\ Studio/2017/BuildTools/MSBuild/15.0/Bin:$PATH
export FORCE_CUDA=1
export TORCH_CUDA_ARCH_LIST="6.0+PTX"
================================================
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.github/workflows/cuda/cu111-Windows.sh
================================================
#!/bin/bash
# Install NVIDIA drivers, see:
# https://github.com/pytorch/vision/blob/master/packaging/windows/internal/cuda_install.bat#L99-L102
curl -k -L "https://drive.google.com/u/0/uc?id=1injUyo3lnarMgWyRcXqKg4UGnN0ysmuq&export=download" --output "/tmp/gpu_driver_dlls.zip"
7z x "/tmp/gpu_driver_dlls.zip" -o"/c/Windows/System32"
export CUDA_SHORT=11.1
export CUDA_URL=https://developer.download.nvidia.com/compute/cuda/${CUDA_SHORT}.1/local_installers
export CUDA_FILE=cuda_${CUDA_SHORT}.1_456.81_win10.exe
# Install CUDA:
curl -k -L "${CUDA_URL}/${CUDA_FILE}" --output "${CUDA_FILE}"
echo ""
echo "Installing from ${CUDA_FILE}..."
PowerShell -Command "Start-Process -FilePath \"${CUDA_FILE}\" -ArgumentList \"-s nvcc_${CUDA_SHORT} cuobjdump_${CUDA_SHORT} nvprune_${CUDA_SHORT} cupti_${CUDA_SHORT} cublas_dev_${CUDA_SHORT} cudart_${CUDA_SHORT} cufft_dev_${CUDA_SHORT} curand_dev_${CUDA_SHORT} cusolver_dev_${CUDA_SHORT} cusparse_dev_${CUDA_SHORT} npp_dev_${CUDA_SHORT} nvrtc_dev_${CUDA_SHORT} nvml_dev_${CUDA_SHORT}\" -Wait -NoNewWindow"
echo "Done!"
rm -f "${CUDA_FILE}"
================================================
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.github/workflows/cuda/cu113-Linux-env.sh
================================================
#!/bin/bash
CUDA_HOME=/usr/local/cuda-11.3
LD_LIBRARY_PATH=${CUDA_HOME}/lib64:${LD_LIBRARY_PATH}
PATH=${CUDA_HOME}/bin:${PATH}
export FORCE_CUDA=1
export TORCH_CUDA_ARCH_LIST="3.5;5.0+PTX;6.0;7.0;7.5;8.0;8.6"
================================================
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.github/workflows/cuda/cu113-Linux.sh
================================================
#!/bin/bash
OS=ubuntu1804
wget -nv https://developer.download.nvidia.com/compute/cuda/repos/${OS}/x86_64/cuda-${OS}.pin
sudo mv cuda-${OS}.pin /etc/apt/preferences.d/cuda-repository-pin-600
wget -nv https://developer.download.nvidia.com/compute/cuda/11.3.0/local_installers/cuda-repo-${OS}-11-3-local_11.3.0-465.19.01-1_amd64.deb
sudo dpkg -i cuda-repo-${OS}-11-3-local_11.3.0-465.19.01-1_amd64.deb
sudo apt-key add /var/cuda-repo-${OS}-11-3-local/7fa2af80.pub
sudo apt-get -qq update
sudo apt install -y cuda-nvcc-11-3 cuda-libraries-dev-11-3
sudo apt clean
rm -f https://developer.download.nvidia.com/compute/cuda/11.3.0/local_installers/cuda-repo-${OS}-11-3-local_11.3.0-465.19.01-1_amd64.deb
================================================
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.github/workflows/cuda/cu113-Windows-env.sh
================================================
#!/bin/bash
CUDA_HOME=/c/Program\ Files/NVIDIA\ GPU\ Computing\ Toolkit/CUDA/v11.3
PATH=${CUDA_HOME}/bin:$PATH
PATH=/c/Program\ Files\ \(x86\)/Microsoft\ Visual\ Studio/2017/BuildTools/MSBuild/15.0/Bin:$PATH
export FORCE_CUDA=1
export TORCH_CUDA_ARCH_LIST="6.0+PTX"
================================================
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.github/workflows/cuda/cu113-Windows.sh
================================================
#!/bin/bash
# Install NVIDIA drivers, see:
# https://github.com/pytorch/vision/blob/master/packaging/windows/internal/cuda_install.bat#L99-L102
curl -k -L "https://drive.google.com/u/0/uc?id=1injUyo3lnarMgWyRcXqKg4UGnN0ysmuq&export=download" --output "/tmp/gpu_driver_dlls.zip"
7z x "/tmp/gpu_driver_dlls.zip" -o"/c/Windows/System32"
export CUDA_SHORT=11.3
export CUDA_URL=https://developer.download.nvidia.com/compute/cuda/${CUDA_SHORT}.0/local_installers
export CUDA_FILE=cuda_${CUDA_SHORT}.0_465.89_win10.exe
# Install CUDA:
curl -k -L "${CUDA_URL}/${CUDA_FILE}" --output "${CUDA_FILE}"
echo ""
echo "Installing from ${CUDA_FILE}..."
PowerShell -Command "Start-Process -FilePath \"${CUDA_FILE}\" -ArgumentList \"-s nvcc_${CUDA_SHORT} cuobjdump_${CUDA_SHORT} nvprune_${CUDA_SHORT} cupti_${CUDA_SHORT} cublas_dev_${CUDA_SHORT} cudart_${CUDA_SHORT} cufft_dev_${CUDA_SHORT} curand_dev_${CUDA_SHORT} cusolver_dev_${CUDA_SHORT} cusparse_dev_${CUDA_SHORT} thrust_${CUDA_SHORT} npp_dev_${CUDA_SHORT} nvrtc_dev_${CUDA_SHORT} nvml_dev_${CUDA_SHORT}\" -Wait -NoNewWindow"
echo "Done!"
rm -f "${CUDA_FILE}"
================================================
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.github/workflows/cuda/cu115-Linux-env.sh
================================================
#!/bin/bash
CUDA_HOME=/usr/local/cuda-11.5
LD_LIBRARY_PATH=${CUDA_HOME}/lib64:${LD_LIBRARY_PATH}
PATH=${CUDA_HOME}/bin:${PATH}
export FORCE_CUDA=1
export TORCH_CUDA_ARCH_LIST="3.5;5.0+PTX;6.0;7.0;7.5;8.0;8.6"
================================================
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.github/workflows/cuda/cu115-Linux.sh
================================================
#!/bin/bash
OS=ubuntu1804
wget -nv https://developer.download.nvidia.com/compute/cuda/repos/${OS}/x86_64/cuda-${OS}.pin
sudo mv cuda-${OS}.pin /etc/apt/preferences.d/cuda-repository-pin-600
wget -nv https://developer.download.nvidia.com/compute/cuda/11.5.2/local_installers/cuda-repo-${OS}-11-5-local_11.5.2-495.29.05-1_amd64.deb
sudo dpkg -i cuda-repo-${OS}-11-5-local_11.5.2-495.29.05-1_amd64.deb
sudo apt-key add /var/cuda-repo-${OS}-11-5-local/7fa2af80.pub
sudo apt-get -qq update
sudo apt install -y cuda-nvcc-11-5 cuda-libraries-dev-11-5
sudo apt clean
rm -f https://developer.download.nvidia.com/compute/cuda/11.5.2/local_installers/cuda-repo-${OS}-11-5-local_11.5.2-495.29.05-1_amd64.deb
================================================
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.github/workflows/cuda/cu115-Windows-env.sh
================================================
#!/bin/bash
CUDA_HOME=/c/Program\ Files/NVIDIA\ GPU\ Computing\ Toolkit/CUDA/v11.3
PATH=${CUDA_HOME}/bin:$PATH
PATH=/c/Program\ Files\ \(x86\)/Microsoft\ Visual\ Studio/2017/BuildTools/MSBuild/15.0/Bin:$PATH
export FORCE_CUDA=1
export TORCH_CUDA_ARCH_LIST="6.0+PTX"
================================================
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.github/workflows/cuda/cu115-Windows.sh
================================================
#!/bin/bash
# TODO We currently use CUDA 11.3 to build CUDA 11.5 Windows wheels
# Install NVIDIA drivers, see:
# https://github.com/pytorch/vision/blob/master/packaging/windows/internal/cuda_install.bat#L99-L102
curl -k -L "https://drive.google.com/u/0/uc?id=1injUyo3lnarMgWyRcXqKg4UGnN0ysmuq&export=download" --output "/tmp/gpu_driver_dlls.zip"
7z x "/tmp/gpu_driver_dlls.zip" -o"/c/Windows/System32"
export CUDA_SHORT=11.3
export CUDA_URL=https://developer.download.nvidia.com/compute/cuda/${CUDA_SHORT}.0/local_installers
export CUDA_FILE=cuda_${CUDA_SHORT}.0_465.89_win10.exe
# Install CUDA:
curl -k -L "${CUDA_URL}/${CUDA_FILE}" --output "${CUDA_FILE}"
echo ""
echo "Installing from ${CUDA_FILE}..."
PowerShell -Command "Start-Process -FilePath \"${CUDA_FILE}\" -ArgumentList \"-s nvcc_${CUDA_SHORT} cuobjdump_${CUDA_SHORT} nvprune_${CUDA_SHORT} cupti_${CUDA_SHORT} cublas_dev_${CUDA_SHORT} cudart_${CUDA_SHORT} cufft_dev_${CUDA_SHORT} curand_dev_${CUDA_SHORT} cusolver_dev_${CUDA_SHORT} cusparse_dev_${CUDA_SHORT} thrust_${CUDA_SHORT} npp_dev_${CUDA_SHORT} nvrtc_dev_${CUDA_SHORT} nvml_dev_${CUDA_SHORT}\" -Wait -NoNewWindow"
echo "Done!"
rm -f "${CUDA_FILE}"
================================================
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.github/workflows/cuda/cu116-Linux-env.sh
================================================
#!/bin/bash
CUDA_HOME=/usr/local/cuda-11.6
LD_LIBRARY_PATH=${CUDA_HOME}/lib64:${LD_LIBRARY_PATH}
PATH=${CUDA_HOME}/bin:${PATH}
export FORCE_CUDA=1
export TORCH_CUDA_ARCH_LIST="3.5;5.0+PTX;6.0;7.0;7.5;8.0;8.6"
================================================
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.github/workflows/cuda/cu116-Linux.sh
================================================
#!/bin/bash
OS=ubuntu1804
wget -nv https://developer.download.nvidia.com/compute/cuda/repos/${OS}/x86_64/cuda-${OS}.pin
sudo mv cuda-${OS}.pin /etc/apt/preferences.d/cuda-repository-pin-600
wget -nv https://developer.download.nvidia.com/compute/cuda/11.6.2/local_installers/cuda-repo-${OS}-11-6-local_11.6.2-510.47.03-1_amd64.deb
sudo dpkg -i cuda-repo-${OS}-11-6-local_11.6.2-510.47.03-1_amd64.deb
sudo apt-key add /var/cuda-repo-${OS}-11-6-local/7fa2af80.pub
sudo apt-get -qq update
sudo apt install -y cuda-nvcc-11-6 cuda-libraries-dev-11-6
sudo apt clean
rm -f https://developer.download.nvidia.com/compute/cuda/11.5.2/local_installers/cuda-repo-${OS}-11-6-local_11.6.2-510.47.03-1_amd64.deb
================================================
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.github/workflows/cuda/cu116-Windows-env.sh
================================================
#!/bin/bash
CUDA_HOME=/c/Program\ Files/NVIDIA\ GPU\ Computing\ Toolkit/CUDA/v11.3
PATH=${CUDA_HOME}/bin:$PATH
PATH=/c/Program\ Files\ \(x86\)/Microsoft\ Visual\ Studio/2017/BuildTools/MSBuild/15.0/Bin:$PATH
export FORCE_CUDA=1
export TORCH_CUDA_ARCH_LIST="6.0+PTX"
================================================
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.github/workflows/cuda/cu116-Windows.sh
================================================
#!/bin/bash
# TODO We currently use CUDA 11.3 to build CUDA 11.6 Windows wheels
# Install NVIDIA drivers, see:
# https://github.com/pytorch/vision/blob/master/packaging/windows/internal/cuda_install.bat#L99-L102
curl -k -L "https://drive.google.com/u/0/uc?id=1injUyo3lnarMgWyRcXqKg4UGnN0ysmuq&export=download" --output "/tmp/gpu_driver_dlls.zip"
7z x "/tmp/gpu_driver_dlls.zip" -o"/c/Windows/System32"
export CUDA_SHORT=11.3
export CUDA_URL=https://developer.download.nvidia.com/compute/cuda/${CUDA_SHORT}.0/local_installers
export CUDA_FILE=cuda_${CUDA_SHORT}.0_465.89_win10.exe
# Install CUDA:
curl -k -L "${CUDA_URL}/${CUDA_FILE}" --output "${CUDA_FILE}"
echo ""
echo "Installing from ${CUDA_FILE}..."
PowerShell -Command "Start-Process -FilePath \"${CUDA_FILE}\" -ArgumentList \"-s nvcc_${CUDA_SHORT} cuobjdump_${CUDA_SHORT} nvprune_${CUDA_SHORT} cupti_${CUDA_SHORT} cublas_dev_${CUDA_SHORT} cudart_${CUDA_SHORT} cufft_dev_${CUDA_SHORT} curand_dev_${CUDA_SHORT} cusolver_dev_${CUDA_SHORT} cusparse_dev_${CUDA_SHORT} thrust_${CUDA_SHORT} npp_dev_${CUDA_SHORT} nvrtc_dev_${CUDA_SHORT} nvml_dev_${CUDA_SHORT}\" -Wait -NoNewWindow"
echo "Done!"
rm -f "${CUDA_FILE}"
================================================
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.github/workflows/cuda/cu117-Linux-env.sh
================================================
#!/bin/bash
CUDA_HOME=/usr/local/cuda-11.7
LD_LIBRARY_PATH=${CUDA_HOME}/lib64:${LD_LIBRARY_PATH}
PATH=${CUDA_HOME}/bin:${PATH}
export FORCE_CUDA=1
export TORCH_CUDA_ARCH_LIST="3.5;5.0+PTX;6.0;7.0;7.5;8.0;8.6"
================================================
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.github/workflows/cuda/cu117-Linux.sh
================================================
#!/bin/bash
OS=ubuntu1804
wget -nv https://developer.download.nvidia.com/compute/cuda/repos/${OS}/x86_64/cuda-${OS}.pin
sudo mv cuda-${OS}.pin /etc/apt/preferences.d/cuda-repository-pin-600
wget -nv https://developer.download.nvidia.com/compute/cuda/11.7.1/local_installers/cuda-repo-${OS}-11-7-local_11.7.1-515.65.01-1_amd64.deb
sudo dpkg -i cuda-repo-${OS}-11-7-local_11.7.1-515.65.01-1_amd64.deb
sudo cp /var/cuda-repo-${OS}-11-7-local/cuda-*-keyring.gpg /usr/share/keyrings/
sudo apt-get -qq update
sudo apt install -y cuda-nvcc-11-7 cuda-libraries-dev-11-7
sudo apt clean
rm -f https://developer.download.nvidia.com/compute/cuda/11.7.1/local_installers/cuda-repo-${OS}-11-7-local_11.7.1-515.65.01-1_amd64.deb
================================================
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.github/workflows/cuda/cu117-Windows-env.sh
================================================
#!/bin/bash
CUDA_HOME=/c/Program\ Files/NVIDIA\ GPU\ Computing\ Toolkit/CUDA/v11.3
PATH=${CUDA_HOME}/bin:$PATH
PATH=/c/Program\ Files\ \(x86\)/Microsoft\ Visual\ Studio/2017/BuildTools/MSBuild/15.0/Bin:$PATH
export FORCE_CUDA=1
export TORCH_CUDA_ARCH_LIST="6.0+PTX"
================================================
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.github/workflows/cuda/cu117-Windows.sh
================================================
#!/bin/bash
# TODO We currently use CUDA 11.3 to build CUDA 11.7 Windows wheels
# Install NVIDIA drivers, see:
# https://github.com/pytorch/vision/blob/master/packaging/windows/internal/cuda_install.bat#L99-L102
curl -k -L "https://drive.google.com/u/0/uc?id=1injUyo3lnarMgWyRcXqKg4UGnN0ysmuq&export=download" --output "/tmp/gpu_driver_dlls.zip"
7z x "/tmp/gpu_driver_dlls.zip" -o"/c/Windows/System32"
export CUDA_SHORT=11.3
export CUDA_URL=https://developer.download.nvidia.com/compute/cuda/${CUDA_SHORT}.0/local_installers
export CUDA_FILE=cuda_${CUDA_SHORT}.0_465.89_win10.exe
# Install CUDA:
curl -k -L "${CUDA_URL}/${CUDA_FILE}" --output "${CUDA_FILE}"
echo ""
echo "Installing from ${CUDA_FILE}..."
PowerShell -Command "Start-Process -FilePath \"${CUDA_FILE}\" -ArgumentList \"-s nvcc_${CUDA_SHORT} cuobjdump_${CUDA_SHORT} nvprune_${CUDA_SHORT} cupti_${CUDA_SHORT} cublas_dev_${CUDA_SHORT} cudart_${CUDA_SHORT} cufft_dev_${CUDA_SHORT} curand_dev_${CUDA_SHORT} cusolver_dev_${CUDA_SHORT} cusparse_dev_${CUDA_SHORT} thrust_${CUDA_SHORT} npp_dev_${CUDA_SHORT} nvrtc_dev_${CUDA_SHORT} nvml_dev_${CUDA_SHORT}\" -Wait -NoNewWindow"
echo "Done!"
rm -f "${CUDA_FILE}"
================================================
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.github/workflows/publish.yml
================================================
# This workflows will upload a Python Package using twine when a release is created
# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
name: Upload Python Package
on:
release:
types: [created]
branches: [master]
jobs:
deploy:
runs-on: ubuntu-latest
environment: production
steps:
- uses: actions/checkout@v2
- name: Set up Python
uses: actions/setup-python@v1
with:
python-version: '3.7'
- name: Install dependencies
run: |
python -m pip install build twine
- name: Strip unsupported tags in README
run: |
sed -i '/<!-- pypi-strip -->/,/<!-- \/pypi-strip -->/d' README.md
- name: Build and publish
env:
PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
run: |
BUILD_NO_CUDA=1 python -m build
twine upload --username __token__ --password $PYPI_TOKEN dist/*
================================================
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.gitignore
================================================
# Visual Studio Code configs.
.vscode/
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
# lib/
lib64/
parts/
sdist/
var/
wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
.hypothesis/
.pytest_cache/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
target/
# Jupyter Notebook
.ipynb_checkpoints
# pyenv
.python-version
# celery beat schedule file
celerybeat-schedule
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
.DS_Store
# Direnv config.
.envrc
# line_profiler
*.lprof
# vscode
.vsocde
benchmarks/
outputs/
================================================
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.gitmodules
================================================
[submodule "examples/pycolmap"]
path = examples/pycolmap
url = https://github.com/rmbrualla/pycolmap.git
================================================
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.pre-commit-config.yaml
================================================
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v2.3.0
hooks:
- id: end-of-file-fixer
- id: trailing-whitespace
- id: check-yaml
- id: check-merge-conflict
- id: requirements-txt-fixer
- repo: https://github.com/psf/black
rev: 22.10.0
hooks:
- id: black
language_version: python3.8.12
args: # arguments to configure black
- --line-length=80
- repo: https://github.com/pycqa/isort
rev: 5.10.1
hooks:
- id: isort
================================================
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.readthedocs.yaml
================================================
version: 2
build:
os: ubuntu-20.04
tools:
python: "3.9"
sphinx:
fail_on_warning: true
configuration: docs/source/conf.py
python:
install:
# Equivalent to 'pip install .'
- method: pip
path: .
# Equivalent to 'pip install -r docs/requirements.txt'
- requirements: docs/requirements.txt
================================================
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/CMakeLists.txt
================================================
# cmake_minimum_required(VERSION 3.3)
# project(nerfacc LANGUAGES CXX CUDA)
# find_package(pybind11 REQUIRED)
# find_package(Torch REQUIRED)
# set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${TORCH_CXX_FLAGS}")
# set(SOURCE_DIR nerfacc/cuda/csrc)
# set(INCLUDE_DIR nerfacc/cuda/csrc/include)
# file(GLOB SOURCES ${SOURCE_DIR}/*.cu)
# pybind11_add_module(${PROJECT_NAME} SHARED ${SOURCES})
# target_link_libraries(${PROJECT_NAME} PRIVATE "${TORCH_LIBRARIES}")
# target_include_directories(${PROJECT_NAME} PRIVATE "${INCLUDE_DIR}")
# # message(STATUS "CUDA enabled")
# # set( CMAKE_CUDA_STANDARD 14 )
# # set( CMAKE_CUDA_STANDARD_REQUIRED ON)
# # find_package(pybind11 REQUIRED)
# # # find_package(Python3 REQUIRED COMPONENTS Development)
# # # target_link_libraries(${PROJECT_NAME} PRIVATE Python3::Python)
# # find_package(Torch REQUIRED)
# # set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${TORCH_CXX_FLAGS}")
# # target_link_libraries(${PROJECT_NAME} PRIVATE ${TORCH_LIBRARIES})
# # set(CSRC nerfacc/cuda/csrc)
# # file(GLOB_RECURSE ALL_SOURCES ${ALL_SOURCES} ${CSRC}/*.cu)
# # file(GLOB_RECURSE ALL_HEADERS ${CSRC}/include/*.h)
# # add_library(${PROJECT_NAME} SHARED ${ALL_SOURCES})
# # target_include_directories(${PROJECT_NAME} PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}")
# # set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -O0")
# # message("-- CMAKE_BUILD_TYPE: ${CMAKE_BUILD_TYPE}")
# # message("-- CMAKE_CXX_FLAGS_DEBUG: ${CMAKE_CXX_FLAGS_DEBUG}")
# # message("-- CMAKE_CXX_FLAGS_RELEASE: ${CMAKE_CXX_FLAGS_RELEASE}")
# # set_target_properties(${PROJECT_NAME} PROPERTIES
# # EXPORT_NAME nerfacc
# # INSTALL_RPATH ${TORCH_INSTALL_PREFIX}/lib)
# # Cmake creates *.dylib by default, but python expects *.so by default
# # if (APPLE)
# # set_property(TARGET ${PROJECT_NAME} PROPERTY SUFFIX .so)
# # endif()
================================================
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/LICENSE
================================================
MIT License
Copyright (c) 2022 Ruilong Li
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
================================================
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/MANIFEST.in
================================================
include nerfacc/cuda/csrc/include/*
include nerfacc/cuda/csrc/*
================================================
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/README.md
================================================
<p>
<!-- pypi-strip -->
<picture>
<source media="(prefers-color-scheme: dark)" srcset="https://user-images.githubusercontent.com/3310961/199083722-881a2372-62c1-4255-8521-31a95a721851.png" />
<source media="(prefers-color-scheme: light)" srcset="https://user-images.githubusercontent.com/3310961/199084143-0d63eb40-3f35-48d2-a9d5-78d1d60b7d66.png" />
<!-- /pypi-strip -->
<img alt="nerfacc logo" src="https://user-images.githubusercontent.com/3310961/199084143-0d63eb40-3f35-48d2-a9d5-78d1d60b7d66.png" width="350px" />
<!-- pypi-strip -->
</picture>
<!-- /pypi-strip -->
</p>
[](https://github.com/KAIR-BAIR/nerfacc/actions/workflows/code_checks.yml)
[](https://www.nerfacc.com/en/latest/?badge=latest)
[](https://pepy.tech/project/nerfacc)
https://www.nerfacc.com/
NerfAcc is a PyTorch Nerf acceleration toolbox for both training and inference. It focuses on efficient volumetric rendering of radiance fields, which is universal and plug-and-play for most of the NeRFs.
Using NerfAcc,
- The `vanilla NeRF` model with 8-layer MLPs can be trained to *better quality* (+~0.5 PNSR)
in *1 hour* rather than *days* as in the paper.
- The `Instant-NGP NeRF` model can be trained to *equal quality* in *4.5 minutes*,
comparing to the official pure-CUDA implementation.
- The `D-NeRF` model for *dynamic* objects can also be trained in *1 hour*
rather than *2 days* as in the paper, and with *better quality* (+~2.5 PSNR).
- Both *bounded* and *unbounded* scenes are supported.
**And it is a pure Python interface with flexible APIs!**
## Installation
**Dependence**: Please install [Pytorch](https://pytorch.org/get-started/locally/) first.
The easist way is to install from PyPI. In this way it will build the CUDA code **on the first run** (JIT).
```
pip install nerfacc
```
Or install from source. In this way it will build the CUDA code during installation.
```
pip install git+https://github.com/KAIR-BAIR/nerfacc.git
```
We also provide pre-built wheels covering major combinations of Pytorch + CUDA supported by [official Pytorch](https://pytorch.org/get-started/previous-versions/).
```
# e.g., torch 1.13.0 + cu117
pip install nerfacc -f https://nerfacc-bucket.s3.us-west-2.amazonaws.com/whl/torch-1.13.0_cu117.html
```
| Windows & Linux | `cu102` | `cu113` | `cu116` | `cu117` |
|-----------------|---------|---------|---------|---------|
| torch 1.10.0 | ✅ | ✅ | | |
| torch 1.11.0 | ✅* | ✅ | | |
| torch 1.12.0 | ✅* | ✅ | ✅ | |
| torch 1.13.0 | | | ✅ | ✅ |
\* Pytorch does not support Windows pre-built wheels for those combinations thus we do not support as well.
## Usage
The idea of NerfAcc is to perform efficient ray marching and volumetric rendering. So NerfAcc can work with any user-defined radiance field. To plug the NerfAcc rendering pipeline into your code and enjoy the acceleration, you only need to define two functions with your radiance field.
- `sigma_fn`: Compute density at each sample. It will be used by `nerfacc.ray_marching()` to skip the empty and occluded space during ray marching, which is where the major speedup comes from.
- `rgb_sigma_fn`: Compute color and density at each sample. It will be used by `nerfacc.rendering()` to conduct differentiable volumetric rendering. This function will receive gradients to update your network.
A simple example is like this:
``` python
import torch
from torch import Tensor
import nerfacc
radiance_field = ... # network: a NeRF model
rays_o: Tensor = ... # ray origins. (n_rays, 3)
rays_d: Tensor = ... # ray normalized directions. (n_rays, 3)
optimizer = ... # optimizer
def sigma_fn(
t_starts: Tensor, t_ends:Tensor, ray_indices: Tensor
) -> Tensor:
""" Query density values from a user-defined radiance field.
:params t_starts: Start of the sample interval along the ray. (n_samples, 1).
:params t_ends: End of the sample interval along the ray. (n_samples, 1).
:params ray_indices: Ray indices that each sample belongs to. (n_samples,).
:returns The post-activation density values. (n_samples, 1).
"""
t_origins = rays_o[ray_indices] # (n_samples, 3)
t_dirs = rays_d[ray_indices] # (n_samples, 3)
positions = t_origins + t_dirs * (t_starts + t_ends) / 2.0
sigmas = radiance_field.query_density(positions)
return sigmas # (n_samples, 1)
def rgb_sigma_fn(
t_starts: Tensor, t_ends: Tensor, ray_indices: Tensor
) -> Tuple[Tensor, Tensor]:
""" Query rgb and density values from a user-defined radiance field.
:params t_starts: Start of the sample interval along the ray. (n_samples, 1).
:params t_ends: End of the sample interval along the ray. (n_samples, 1).
:params ray_indices: Ray indices that each sample belongs to. (n_samples,).
:returns The post-activation rgb and density values.
(n_samples, 3), (n_samples, 1).
"""
t_origins = rays_o[ray_indices] # (n_samples, 3)
t_dirs = rays_d[ray_indices] # (n_samples, 3)
positions = t_origins + t_dirs * (t_starts + t_ends) / 2.0
rgbs, sigmas = radiance_field(positions, condition=t_dirs)
return rgbs, sigmas # (n_samples, 3), (n_samples, 1)
# Efficient Raymarching: Skip empty and occluded space, pack samples from all rays.
# ray_indices: (n_samples,). t_starts: (n_samples, 1). t_ends: (n_samples, 1).
with torch.no_grad():
ray_indices, t_starts, t_ends = nerfacc.ray_marching(
rays_o, rays_d, sigma_fn=sigma_fn, near_plane=0.2, far_plane=1.0,
early_stop_eps=1e-4, alpha_thre=1e-2,
)
# Differentiable Volumetric Rendering.
# colors: (n_rays, 3). opaicity: (n_rays, 1). depth: (n_rays, 1).
color, opacity, depth = nerfacc.rendering(
t_starts, t_ends, ray_indices, n_rays=rays_o.shape[0], rgb_sigma_fn=rgb_sigma_fn
)
# Optimize: Both the network and rays will receive gradients
optimizer.zero_grad()
loss = F.mse_loss(color, color_gt)
loss.backward()
optimizer.step()
```
## Examples:
Before running those example scripts, please check the script about which dataset it is needed, and download the dataset first.
```bash
# clone the repo with submodules.
git clone --recursive git://github.com/KAIR-BAIR/nerfacc/
```
``` bash
# Instant-NGP NeRF in 4.5 minutes with reproduced performance!
# See results at here: https://www.nerfacc.com/en/latest/examples/ngp.html
python examples/train_ngp_nerf.py --train_split train --scene lego
```
``` bash
# Vanilla MLP NeRF in 1 hour with better performance!
# See results at here: https://www.nerfacc.com/en/latest/examples/vanilla.html
python examples/train_mlp_nerf.py --train_split train --scene lego
```
```bash
# D-NeRF for Dynamic objects in 1 hour with better performance!
# See results at here: https://www.nerfacc.com/en/latest/examples/dnerf.html
python examples/train_mlp_dnerf.py --train_split train --scene lego
```
```bash
# Instant-NGP on unbounded scenes in 20 minutes!
# See results at here: https://www.nerfacc.com/en/latest/examples/unbounded.html
python examples/train_ngp_nerf.py --train_split train --scene garden --auto_aabb --unbounded --cone_angle=0.004
```
Used by:
- [nerfstudio](https://github.com/nerfstudio-project/nerfstudio): A collaboration friendly studio for NeRFs.
- [instant-nsr-pl](https://github.com/bennyguo/instant-nsr-pl): NeuS in 10 minutes.
## Common Installation Issues
<details>
<summary>ImportError: .../csrc.so: undefined symbol</summary>
If you are installing a pre-built wheel, make sure the Pytorch and CUDA version matchs with the nerfacc version (nerfacc.__version__).
</details>
## Citation
```bibtex
@article{li2022nerfacc,
title={NerfAcc: A General NeRF Accleration Toolbox.},
author={Li, Ruilong and Tancik, Matthew and Kanazawa, Angjoo},
journal={arXiv preprint arXiv:2210.04847},
year={2022}
}
```
================================================
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/docs/Makefile
================================================
# Minimal makefile for Sphinx documentation
#
# You can set these variables from the command line, and also
# from the environment for the first two.
SPHINXOPTS ?=
SPHINXBUILD ?= sphinx-build
SOURCEDIR = source
BUILDDIR = build
# Put it first so that "make" without argument is like "make help".
help:
@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
.PHONY: help Makefile
# Catch-all target: route all unknown targets to Sphinx using the new
# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
%: Makefile
@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
================================================
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/docs/requirements.txt
================================================
pytorch_sphinx_theme @ git+https://github.com/liruilong940607/pytorch_sphinx_theme.git#egg=pytorch_sphinx_theme
sphinx==5.2.1
sphinx-copybutton==0.5.0
sphinx-design==0.2.0
================================================
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/docs/source/_static/css/readthedocs.css
================================================
.header-logo {
background-image: url("../images/logo4x.png");
background-size: 156px 35px;
height: 35px;
width: 156px;
}
code {
word-break: normal;
}
================================================
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/docs/source/apis/generated/nerfacc.accumulate_along_rays.rst
================================================
nerfacc.accumulate\_along\_rays
===============================
.. currentmodule:: nerfacc
.. autofunction:: accumulate_along_rays
================================================
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/docs/source/apis/generated/nerfacc.pack_data.rst
================================================
nerfacc.pack\_data
==================
.. currentmodule:: nerfacc
.. autofunction:: pack_data
================================================
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/docs/source/apis/generated/nerfacc.ray_aabb_intersect.rst
================================================
nerfacc.ray\_aabb\_intersect
============================
.. currentmodule:: nerfacc
.. autofunction:: ray_aabb_intersect
================================================
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/docs/source/apis/generated/nerfacc.ray_resampling.rst
================================================
nerfacc.ray\_resampling
=======================
.. currentmodule:: nerfacc
.. autofunction:: ray_resampling
================================================
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/docs/source/apis/generated/nerfacc.render_transmittance_from_alpha.rst
================================================
nerfacc.render\_transmittance\_from\_alpha
==========================================
.. currentmodule:: nerfacc
.. autofunction:: render_transmittance_from_alpha
================================================
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/docs/source/apis/generated/nerfacc.render_transmittance_from_density.rst
================================================
nerfacc.render\_transmittance\_from\_density
============================================
.. currentmodule:: nerfacc
.. autofunction:: render_transmittance_from_density
================================================
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/docs/source/apis/generated/nerfacc.render_visibility.rst
================================================
nerfacc.render\_visibility
==========================
.. currentmodule:: nerfacc
.. autofunction:: render_visibility
================================================
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/docs/source/apis/generated/nerfacc.render_weight_from_alpha.rst
================================================
nerfacc.render\_weight\_from\_alpha
===================================
.. currentmodule:: nerfacc
.. autofunction:: render_weight_from_alpha
================================================
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/docs/source/apis/generated/nerfacc.render_weight_from_density.rst
================================================
nerfacc.render\_weight\_from\_density
=====================================
.. currentmodule:: nerfacc
.. autofunction:: render_weight_from_density
================================================
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/docs/source/apis/generated/nerfacc.unpack_data.rst
================================================
nerfacc.unpack\_data
====================
.. currentmodule:: nerfacc
.. autofunction:: unpack_data
================================================
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/docs/source/apis/generated/nerfacc.unpack_info.rst
================================================
nerfacc.unpack\_info
====================
.. currentmodule:: nerfacc
.. autofunction:: unpack_info
================================================
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/docs/source/apis/grid.rst
================================================
.. _`Occupancy Grid`:
Occupancy Grid
===================================
.. currentmodule:: nerfacc
.. autoclass:: ContractionType
:members:
.. autoclass:: Grid
:members:
.. autoclass:: OccupancyGrid
:members:
================================================
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/docs/source/apis/rendering.rst
================================================
Volumetric Rendering
===================================
In `nerfacc`, the volumetric rendering pipeline is broken down into 2 steps:
1. **Raymarching**: This is the process of shooting a ray through the scene and
generate samples along the way. To perform efficient volumetric rendering, here we aim
at skipping as many areas as possible. The emtpy space is skipped by using the cached
occupancy grid (see :class:`nerfacc.OccupancyGrid`), and the invisible space is skipped by
checking the transmittance of the ray while marching. Almost in all cases, those skipping
won't result in a noticeable loss of quality as they would contribute very little to the
final rendered image. But they will bring a significant speedup.
2. **Rendering**: This is the process of accumulating samples along the rays into final image.
In this step we also need to query the attributes (a.k.a. color and density) of those samples
generated by raymarching. Early stoping is supported in this step.
|
.. currentmodule:: nerfacc
.. autofunction:: ray_marching
.. autofunction:: rendering
================================================
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/docs/source/apis/utils.rst
================================================
Utils
===================================
.. currentmodule:: nerfacc
.. autosummary::
:nosignatures:
:toctree: generated/
ray_aabb_intersect
unpack_info
accumulate_along_rays
render_transmittance_from_density
render_transmittance_from_alpha
render_weight_from_density
render_weight_from_alpha
render_visibility
ray_resampling
pack_data
unpack_data
================================================
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/docs/source/conf.py
================================================
import pytorch_sphinx_theme
__version__ = None
exec(open("../../nerfacc/version.py", "r").read())
# -- Project information
project = "nerfacc"
copyright = "2022, Ruilong"
author = "Ruilong"
release = __version__
# -- General configuration
extensions = [
"sphinx.ext.napoleon",
"sphinx.ext.duration",
"sphinx.ext.doctest",
"sphinx.ext.autodoc",
"sphinx.ext.autosummary",
"sphinx.ext.intersphinx",
]
intersphinx_mapping = {
"python": ("https://docs.python.org/3/", None),
"sphinx": ("https://www.sphinx-doc.org/en/master/", None),
}
intersphinx_disabled_domains = ["std"]
templates_path = ["_templates"]
# -- Options for HTML output
# html_theme = "furo"
html_theme = "pytorch_sphinx_theme"
html_theme_path = [pytorch_sphinx_theme.get_html_theme_path()]
html_static_path = ["_static"]
html_css_files = ["css/readthedocs.css"]
# Ignore >>> when copying code
copybutton_prompt_text = r">>> |\.\.\. "
copybutton_prompt_is_regexp = True
# Theme options are theme-specific and customize the look and feel of a theme
# further. For a list of options available for each theme, see the
# documentation.
html_theme_options = {
# The target url that the logo directs to. Unset to do nothing
"logo_url": "https://www.nerfacc.com/en/latest/index.html",
# "menu" is a list of dictionaries where you can specify the content and the
# behavior of each item in the menu. Each item can either be a link or a
# dropdown menu containing a list of links.
"menu": [
# A link
{"name": "GitHub", "url": "https://github.com/KAIR-BAIR/nerfacc"},
# A dropdown menu
# {
# "name": "Projects",
# "children": [
# # A vanilla dropdown item
# {
# "name": "nerfstudio",
# "url": "https://docs.nerf.studio/",
# "description": "The all-in-one repo for NeRFs",
# },
# ],
# # Optional, determining whether this dropdown menu will always be
# # highlighted.
# # "active": True,
# },
],
}
# html_theme_options = {
# "canonical_url": "",
# "analytics_id": "",
# "logo_only": False,
# "display_version": True,
# "prev_next_buttons_location": "bottom",
# "style_external_links": False,
# # Toc options
# "collapse_navigation": True,
# "sticky_navigation": True,
# "navigation_depth": 4,
# "includehidden": True,
# "titles_only": False
# }
# -- Options for EPUB output
epub_show_urls = "footnote"
# typehints
autodoc_typehints = "description"
================================================
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/docs/source/examples/dnerf.rst
================================================
Dynamic Scene
====================
See code `examples/train_mlp_dnerf.py` at our `github repository`_ for details.
Benchmarks
------------
*updated on 2022-10-08*
Here we trained a 8-layer-MLP for the radiance field and a 4-layer-MLP for the warping field,
(similar to the T-Nerf model in the `D-Nerf`_ paper) on the `D-Nerf dataset`_. We used train
split for training and test split for evaluation. Our experiments are conducted on a
single NVIDIA TITAN RTX GPU. The training memory footprint is about 11GB.
.. note::
The :ref:`Occupancy Grid` used in this example is shared by all the frames. In other words,
instead of using it to indicate the opacity of an area at a single timestamp,
Here we use it to indicate the `maximum` opacity at this area `over all the timestamps`.
It is not optimal but still makes the rendering very efficient.
+----------------------+----------+---------+-------+---------+-------+--------+---------+-------+-------+
| PSNR | bouncing | hell | hook | jumping | lego | mutant | standup | trex | MEAN |
| | balls | warrior | | jacks | | | | | |
+======================+==========+=========+=======+=========+=======+========+=========+=======+=======+
| D-Nerf (~ days) | 32.80 | 25.02 | 29.25 | 32.80 | 21.64 | 31.29 | 32.79 | 31.75 | 29.67 |
+----------------------+----------+---------+-------+---------+-------+--------+---------+-------+-------+
| Ours (~ 1 hr) | 39.49 | 25.58 | 31.86 | 32.73 | 24.32 | 35.55 | 35.90 | 32.33 | 32.22 |
+----------------------+----------+---------+-------+---------+-------+--------+---------+-------+-------+
| Ours (Training time)| 37min | 52min | 69min | 64min | 44min | 79min | 79min | 39min | 58min |
+----------------------+----------+---------+-------+---------+-------+--------+---------+-------+-------+
.. _`D-Nerf`: https://arxiv.org/abs/2011.13961
.. _`D-Nerf dataset`: https://www.dropbox.com/s/0bf6fl0ye2vz3vr/data.zip?dl=0
.. _`github repository`: https://github.com/KAIR-BAIR/nerfacc/tree/76c0f9817da4c9c8b5ccf827eb069ee2ce854b75
================================================
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/docs/source/examples/ngp.rst
================================================
.. _`Instant-NGP Example`:
Instant-NGP
====================
See code `examples/train_ngp_nerf.py` at our `github repository`_ for details.
Benchmarks
------------
*updated on 2022-10-12*
Here we trained a `Instant-NGP Nerf`_ model on the `Nerf-Synthetic dataset`_. We follow the same
settings with the Instant-NGP paper, which uses train split for training and test split for
evaluation. All experiments are conducted on a single NVIDIA TITAN RTX GPU. The training
memory footprint is about 3GB.
.. note::
The Instant-NGP paper makes use of the alpha channel in the images to apply random background
augmentation during training. For fair comparision, we rerun their code with a constant white
background during both training and testing. Also it is worth to mention that we didn't strictly
follow the training receipe in the Instant-NGP paper, such as the learning rate schedule etc, as
the purpose of this benchmark is to showcase instead of reproducing the paper.
+-----------------------+-------+-------+---------+-------+-------+-------+-------+-------+-------+
| PSNR | Lego | Mic |Materials| Chair |Hotdog | Ficus | Drums | Ship | MEAN |
| | | | | | | | | | |
+=======================+=======+=======+=========+=======+=======+=======+=======+=======+=======+
|Instant-NGP 35k steps | 35.87 | 36.22 | 29.08 | 35.10 | 37.48 | 30.61 | 23.85 | 30.62 | 32.35 |
+-----------------------+-------+-------+---------+-------+-------+-------+-------+-------+-------+
|(training time) | 309s | 258s | 256s | 316s | 292s | 207s | 218s | 250s | 263s |
+-----------------------+-------+-------+---------+-------+-------+-------+-------+-------+-------+
|Ours 20k steps | 35.50 | 36.16 | 29.14 | 35.23 | 37.15 | 31.71 | 24.88 | 29.91 | 32.46 |
+-----------------------+-------+-------+---------+-------+-------+-------+-------+-------+-------+
|(training time) | 287s | 274s | 269s | 317s | 269s | 244s | 249s | 257s | 271s |
+-----------------------+-------+-------+---------+-------+-------+-------+-------+-------+-------+
.. _`Instant-NGP Nerf`: https://github.com/NVlabs/instant-ngp/tree/51e4107edf48338e9ab0316d56a222e0adf87143
.. _`github repository`: https://github.com/KAIR-BAIR/nerfacc/tree/76c0f9817da4c9c8b5ccf827eb069ee2ce854b75
.. _`Nerf-Synthetic dataset`: https://drive.google.com/drive/folders/1JDdLGDruGNXWnM1eqY1FNL9PlStjaKWi
================================================
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/docs/source/examples/unbounded.rst
================================================
Unbounded Scene
====================
See code `examples/train_ngp_nerf.py` at our `github repository`_ for details.
Benchmarks
------------
*updated on 2022-11-07*
Here we trained a `Instant-NGP Nerf`_ on the `MipNerf360`_ dataset. We used train
split for training and test split for evaluation. Our experiments are conducted on a
single NVIDIA TITAN RTX GPU. The training memory footprint is about 6-9GB.
The main difference between working with unbounded scenes and bounded scenes, is that
a contraction method is needed to map the infinite space to a finite :ref:`Occupancy Grid`.
We have difference options provided for this (see :ref:`Occupancy Grid`). The experiments
here is basically the Instant-NGP experiments (see :ref:`Instant-NGP Example`) with a contraction method
that takes from `MipNerf360`_.
.. note::
Even though we are comparing with `Nerf++`_ and `MipNerf360`_, the model and everything are
totally different with them. There are plenty of ideas from those papers that would be very
helpful for the performance, but we didn't adopt them. As this is just a simple example to
show how to use the library, we didn't want to make it too complicated.
+----------------------+-------+-------+-------+-------+-------+-------+-------+-------+
| PSNR |Garden |Bicycle|Bonsai |Counter|Kitchen| Room | Stump | MEAN |
| | | | | | | | | |
+======================+=======+=======+=======+=======+=======+=======+=======+=======+
| Nerf++ (~days) | 24.32 | 22.64 | 29.15 | 26.38 | 27.80 | 28.87 | 24.34 | 26.21 |
+----------------------+-------+-------+-------+-------+-------+-------+-------+-------+
| MipNerf360 (~days) | 26.98 | 24.37 | 33.46 | 29.55 | 32.23 | 31.63 | 26.40 | 29.23 |
+----------------------+-------+-------+-------+-------+-------+-------+-------+-------+
| Ours (~20 mins) | 25.41 | 22.97 | 30.71 | 27.34 | 30.32 | 31.00 | 23.43 | 27.31 |
+----------------------+-------+-------+-------+-------+-------+-------+-------+-------+
| Ours (Training time) | 25min | 17min | 19min | 23min | 28min | 20min | 17min | 21min |
+----------------------+-------+-------+-------+-------+-------+-------+-------+-------+
.. _`Instant-NGP Nerf`: https://arxiv.org/abs/2201.05989
.. _`MipNerf360`: https://arxiv.org/abs/2111.12077
.. _`Nerf++`: https://arxiv.org/abs/2010.07492
.. _`github repository`: https://github.com/KAIR-BAIR/nerfacc/tree/76c0f9817da4c9c8b5ccf827eb069ee2ce854b75
================================================
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/docs/source/examples/vanilla.rst
================================================
Vanilla Nerf
====================
See code `examples/train_mlp_nerf.py` at our `github repository`_ for details.
Benchmarks
------------
*updated on 2022-10-08*
Here we trained a 8-layer-MLP for the radiance field as in the `vanilla Nerf`_. We used the
train split for training and test split for evaluation as in the Nerf paper. Our experiments are
conducted on a single NVIDIA TITAN RTX GPU. The training memory footprint is about 10GB.
.. note::
The vanilla Nerf paper uses two MLPs for course-to-fine sampling. Instead here we only use a
single MLP with more samples (1024). Both ways share the same spirit to do dense sampling
around the surface. Our fast rendering inheritly skip samples away from the surface
so we can simplly increase the number of samples with a single MLP, to achieve the same goal
with the coarse-to-fine sampling, without runtime or memory issue.
+----------------------+-------+-------+---------+-------+-------+-------+-------+-------+-------+
| PSNR | Lego | Mic |Materials| Chair |Hotdog | Ficus | Drums | Ship | MEAN |
| | | | | | | | | | |
+======================+=======+=======+=========+=======+=======+=======+=======+=======+=======+
| NeRF (~ days) | 32.54 | 32.91 | 29.62 | 33.00 | 36.18 | 30.13 | 25.01 | 28.65 | 31.00 |
+----------------------+-------+-------+---------+-------+-------+-------+-------+-------+-------+
| Ours (~ 50min) | 33.69 | 33.76 | 29.73 | 33.32 | 35.80 | 32.52 | 25.39 | 28.18 | 31.55 |
+----------------------+-------+-------+---------+-------+-------+-------+-------+-------+-------+
| Ours (Training time)| 58min | 53min | 46min | 62min | 56min | 42min | 52min | 49min | 52min |
+----------------------+-------+-------+---------+-------+-------+-------+-------+-------+-------+
.. _`github repository`: : https://github.com/KAIR-BAIR/nerfacc/tree/76c0f9817da4c9c8b5ccf827eb069ee2ce854b75
.. _`vanilla Nerf`: https://arxiv.org/abs/2003.08934
================================================
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/docs/source/index.rst
================================================
NerfAcc Documentation
===================================
NerfAcc is a PyTorch Nerf acceleration toolbox for both training and inference. It focus on
efficient volumetric rendering of radiance fields, which is universal and plug-and-play for most of the NeRFs.
Using NerfAcc,
- The `vanilla Nerf`_ model with 8-layer MLPs can be trained to *better quality* (+~0.5 PNSR) \
in *1 hour* rather than *1~2 days* as in the paper.
- The `Instant-NGP Nerf`_ model can be trained to *equal quality* in *4.5 minutes*, \
comparing to the official pure-CUDA implementation.
- The `D-Nerf`_ model for *dynamic* objects can also be trained in *1 hour* \
rather than *2 days* as in the paper, and with *better quality* (+~2.5 PSNR).
- Both *bounded* and *unbounded* scenes are supported.
**And it is pure Python interface with flexible APIs!**
| Github: https://github.com/KAIR-BAIR/nerfacc
| Paper: https://arxiv.org/pdf/2210.04847.pdf
| Authors: `Ruilong Li`_, `Matthew Tancik`_, `Angjoo Kanazawa`_
.. note::
This repo is focusing on the single scene situation. Generalizable Nerfs across
multiple scenes is currently out of the scope of this repo. But you may still find
some useful tricks in this repo. :)
Installation:
-------------
.. code-block:: console
$ pip install nerfacc
Usage:
-------------
The idea of NerfAcc is to perform efficient ray marching and volumetric rendering.
So NerfAcc can work with any user-defined radiance field. To plug the NerfAcc rendering
pipeline into your code and enjoy the acceleration, you only need to define two functions
with your radience field.
- `sigma_fn`: Compute density at each sample. It will be used by :func:`nerfacc.ray_marching` to skip the empty and occluded space during ray marching, which is where the major speedup comes from.
- `rgb_sigma_fn`: Compute color and density at each sample. It will be used by :func:`nerfacc.rendering` to conduct differentiable volumetric rendering. This function will receive gradients to update your network.
An simple example is like this:
.. code-block:: python
import torch
from torch import Tensor
import nerfacc
radiance_field = ... # network: a NeRF model
rays_o: Tensor = ... # ray origins. (n_rays, 3)
rays_d: Tensor = ... # ray normalized directions. (n_rays, 3)
optimizer = ... # optimizer
def sigma_fn(
t_starts: Tensor, t_ends:Tensor, ray_indices: Tensor
) -> Tensor:
""" Query density values from a user-defined radiance field.
:params t_starts: Start of the sample interval along the ray. (n_samples, 1).
:params t_ends: End of the sample interval along the ray. (n_samples, 1).
:params ray_indices: Ray indices that each sample belongs to. (n_samples,).
:returns The post-activation density values. (n_samples, 1).
"""
t_origins = rays_o[ray_indices] # (n_samples, 3)
t_dirs = rays_d[ray_indices] # (n_samples, 3)
positions = t_origins + t_dirs * (t_starts + t_ends) / 2.0
sigmas = radiance_field.query_density(positions)
return sigmas # (n_samples, 1)
def rgb_sigma_fn(
t_starts: Tensor, t_ends: Tensor, ray_indices: Tensor
) -> Tuple[Tensor, Tensor]:
""" Query rgb and density values from a user-defined radiance field.
:params t_starts: Start of the sample interval along the ray. (n_samples, 1).
:params t_ends: End of the sample interval along the ray. (n_samples, 1).
:params ray_indices: Ray indices that each sample belongs to. (n_samples,).
:returns The post-activation rgb and density values.
(n_samples, 3), (n_samples, 1).
"""
t_origins = rays_o[ray_indices] # (n_samples, 3)
t_dirs = rays_d[ray_indices] # (n_samples, 3)
positions = t_origins + t_dirs * (t_starts + t_ends) / 2.0
rgbs, sigmas = radiance_field(positions, condition=t_dirs)
return rgbs, sigmas # (n_samples, 3), (n_samples, 1)
# Efficient Raymarching: Skip empty and occluded space, pack samples from all rays.
# ray_indices: (n_samples,). t_starts: (n_samples, 1). t_ends: (n_samples, 1).
with torch.no_grad():
ray_indices, t_starts, t_ends = nerfacc.ray_marching(
rays_o, rays_d, sigma_fn=sigma_fn, near_plane=0.2, far_plane=1.0,
early_stop_eps=1e-4, alpha_thre=1e-2,
)
# Differentiable Volumetric Rendering.
# colors: (n_rays, 3). opaicity: (n_rays, 1). depth: (n_rays, 1).
color, opacity, depth = nerfacc.rendering(
t_starts, t_ends, ray_indices, n_rays=rays_o.shape[0], rgb_sigma_fn=rgb_sigma_fn
)
# Optimize: Both the network and rays will receive gradients
optimizer.zero_grad()
loss = F.mse_loss(color, color_gt)
loss.backward()
optimizer.step()
Links:
-------------
.. toctree::
:glob:
:maxdepth: 1
:caption: Python API
apis/*
.. toctree::
:glob:
:maxdepth: 1
:caption: Example Usages
examples/*
.. toctree::
:maxdepth: 1
:caption: Projects
nerfstudio <https://docs.nerf.studio/>
.. _`vanilla Nerf`: https://arxiv.org/abs/2003.08934
.. _`Instant-NGP Nerf`: https://arxiv.org/abs/2201.05989
.. _`D-Nerf`: https://arxiv.org/abs/2011.13961
.. _`MipNerf360`: https://arxiv.org/abs/2111.12077
.. _`pixel-Nerf`: https://arxiv.org/abs/2012.02190
.. _`Nerf++`: https://arxiv.org/abs/2010.07492
.. _`Ruilong Li`: https://www.liruilong.cn/
.. _`Matthew Tancik`: https://www.matthewtancik.com/
.. _`Angjoo Kanazawa`: https://people.eecs.berkeley.edu/~kanazawa/
================================================
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/examples/datasets/__init__.py
================================================
================================================
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/examples/datasets/dnerf_synthetic.py
================================================
"""
Copyright (c) 2022 Ruilong Li, UC Berkeley.
"""
import json
import os
import imageio.v2 as imageio
import numpy as np
import torch
import torch.nn.functional as F
from .utils import Rays
def _load_renderings(root_fp: str, subject_id: str, split: str):
"""Load images from disk."""
if not root_fp.startswith("/"):
# allow relative path. e.g., "./data/dnerf_synthetic/"
root_fp = os.path.join(
os.path.dirname(os.path.abspath(__file__)),
"..",
"..",
root_fp,
)
data_dir = os.path.join(root_fp, subject_id)
with open(
os.path.join(data_dir, "transforms_{}.json".format(split)), "r"
) as fp:
meta = json.load(fp)
images = []
camtoworlds = []
timestamps = []
for i in range(len(meta["frames"])):
frame = meta["frames"][i]
fname = os.path.join(data_dir, frame["file_path"] + ".png")
rgba = imageio.imread(fname)
timestamp = (
frame["time"]
if "time" in frame
else float(i) / (len(meta["frames"]) - 1)
)
timestamps.append(timestamp)
camtoworlds.append(frame["transform_matrix"])
images.append(rgba)
images = np.stack(images, axis=0)
camtoworlds = np.stack(camtoworlds, axis=0)
timestamps = np.stack(timestamps, axis=0)
h, w = images.shape[1:3]
camera_angle_x = float(meta["camera_angle_x"])
focal = 0.5 * w / np.tan(0.5 * camera_angle_x)
return images, camtoworlds, focal, timestamps
class SubjectLoader(torch.utils.data.Dataset):
"""Single subject data loader for training and evaluation."""
SPLITS = ["train", "val", "test"]
SUBJECT_IDS = [
"bouncingballs",
"hellwarrior",
"hook",
"jumpingjacks",
"lego",
"mutant",
"standup",
"trex",
]
WIDTH, HEIGHT = 800, 800
NEAR, FAR = 2.0, 6.0
OPENGL_CAMERA = True
def __init__(
self,
subject_id: str,
root_fp: str,
split: str,
color_bkgd_aug: str = "white",
num_rays: int = None,
near: float = None,
far: float = None,
batch_over_images: bool = True,
):
super().__init__()
assert split in self.SPLITS, "%s" % split
assert subject_id in self.SUBJECT_IDS, "%s" % subject_id
assert color_bkgd_aug in ["white", "black", "random"]
self.split = split
self.num_rays = num_rays
self.near = self.NEAR if near is None else near
self.far = self.FAR if far is None else far
self.training = (num_rays is not None) and (
split in ["train", "trainval"]
)
self.color_bkgd_aug = color_bkgd_aug
self.batch_over_images = batch_over_images
(
self.images,
self.camtoworlds,
self.focal,
self.timestamps,
) = _load_renderings(root_fp, subject_id, split)
self.images = torch.from_numpy(self.images).to(torch.uint8)
self.camtoworlds = torch.from_numpy(self.camtoworlds).to(torch.float32)
self.timestamps = torch.from_numpy(self.timestamps).to(torch.float32)[
:, None
]
self.K = torch.tensor(
[
[self.focal, 0, self.WIDTH / 2.0],
[0, self.focal, self.HEIGHT / 2.0],
[0, 0, 1],
],
dtype=torch.float32,
) # (3, 3)
assert self.images.shape[1:3] == (self.HEIGHT, self.WIDTH)
def __len__(self):
return len(self.images)
@torch.no_grad()
def __getitem__(self, index):
data = self.fetch_data(index)
data = self.preprocess(data)
return data
def preprocess(self, data):
"""Process the fetched / cached data with randomness."""
rgba, rays = data["rgba"], data["rays"]
pixels, alpha = torch.split(rgba, [3, 1], dim=-1)
if self.training:
if self.color_bkgd_aug == "random":
color_bkgd = torch.rand(3, device=self.images.device)
elif self.color_bkgd_aug == "white":
color_bkgd = torch.ones(3, device=self.images.device)
elif self.color_bkgd_aug == "black":
color_bkgd = torch.zeros(3, device=self.images.device)
else:
# just use white during inference
color_bkgd = torch.ones(3, device=self.images.device)
pixels = pixels * alpha + color_bkgd * (1.0 - alpha)
return {
"pixels": pixels, # [n_rays, 3] or [h, w, 3]
"rays": rays, # [n_rays,] or [h, w]
"color_bkgd": color_bkgd, # [3,]
**{k: v for k, v in data.items() if k not in ["rgba", "rays"]},
}
def update_num_rays(self, num_rays):
self.num_rays = num_rays
def fetch_data(self, index):
"""Fetch the data (it maybe cached for multiple batches)."""
num_rays = self.num_rays
if self.training:
if self.batch_over_images:
image_id = torch.randint(
0,
len(self.images),
size=(num_rays,),
device=self.images.device,
)
else:
image_id = [index]
x = torch.randint(
0, self.WIDTH, size=(num_rays,), device=self.images.device
)
y = torch.randint(
0, self.HEIGHT, size=(num_rays,), device=self.images.device
)
else:
image_id = [index]
x, y = torch.meshgrid(
torch.arange(self.WIDTH, device=self.images.device),
torch.arange(self.HEIGHT, device=self.images.device),
indexing="xy",
)
x = x.flatten()
y = y.flatten()
# generate rays
rgba = self.images[image_id, y, x] / 255.0 # (num_rays, 4)
c2w = self.camtoworlds[image_id] # (num_rays, 3, 4)
camera_dirs = F.pad(
torch.stack(
[
(x - self.K[0, 2] + 0.5) / self.K[0, 0],
(y - self.K[1, 2] + 0.5)
/ self.K[1, 1]
* (-1.0 if self.OPENGL_CAMERA else 1.0),
],
dim=-1,
),
(0, 1),
value=(-1.0 if self.OPENGL_CAMERA else 1.0),
) # [num_rays, 3]
# [n_cams, height, width, 3]
directions = (camera_dirs[:, None, :] * c2w[:, :3, :3]).sum(dim=-1)
origins = torch.broadcast_to(c2w[:, :3, -1], directions.shape)
viewdirs = directions / torch.linalg.norm(
directions, dim=-1, keepdims=True
)
if self.training:
origins = torch.reshape(origins, (num_rays, 3))
viewdirs = torch.reshape(viewdirs, (num_rays, 3))
rgba = torch.reshape(rgba, (num_rays, 4))
else:
origins = torch.reshape(origins, (self.HEIGHT, self.WIDTH, 3))
viewdirs = torch.reshape(viewdirs, (self.HEIGHT, self.WIDTH, 3))
rgba = torch.reshape(rgba, (self.HEIGHT, self.WIDTH, 4))
rays = Rays(origins=origins, viewdirs=viewdirs)
timestamps = self.timestamps[image_id]
return {
"rgba": rgba, # [h, w, 4] or [num_rays, 4]
"rays": rays, # [h, w, 3] or [num_rays, 3]
"timestamps": timestamps, # [num_rays, 1]
}
================================================
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/examples/datasets/nerf_360_v2.py
================================================
"""
Copyright (c) 2022 Ruilong Li, UC Berkeley.
"""
import collections
import os
import sys
import imageio
import numpy as np
import torch
import torch.nn.functional as F
import tqdm
from .utils import Rays
_PATH = os.path.abspath(__file__)
sys.path.insert(
0, os.path.join(os.path.dirname(_PATH), "..", "pycolmap", "pycolmap")
)
from scene_manager import SceneManager
def _load_colmap(root_fp: str, subject_id: str, split: str, factor: int = 1):
assert factor in [1, 2, 4, 8]
data_dir = os.path.join(root_fp, subject_id)
colmap_dir = os.path.join(data_dir, "sparse/0/")
manager = SceneManager(colmap_dir)
manager.load_cameras()
manager.load_images()
# Assume shared intrinsics between all cameras.
cam = manager.cameras[1]
fx, fy, cx, cy = cam.fx, cam.fy, cam.cx, cam.cy
K = np.array([[fx, 0, cx], [0, fy, cy], [0, 0, 1]])
K[:2, :] /= factor
# Extract extrinsic matrices in world-to-camera format.
imdata = manager.images
w2c_mats = []
bottom = np.array([0, 0, 0, 1]).reshape(1, 4)
for k in imdata:
im = imdata[k]
rot = im.R()
trans = im.tvec.reshape(3, 1)
w2c = np.concatenate([np.concatenate([rot, trans], 1), bottom], axis=0)
w2c_mats.append(w2c)
w2c_mats = np.stack(w2c_mats, axis=0)
# Convert extrinsics to camera-to-world.
camtoworlds = np.linalg.inv(w2c_mats)
# Image names from COLMAP. No need for permuting the poses according to
# image names anymore.
image_names = [imdata[k].name for k in imdata]
# # Switch from COLMAP (right, down, fwd) to Nerf (right, up, back) frame.
# poses = poses @ np.diag([1, -1, -1, 1])
# Get distortion parameters.
type_ = cam.camera_type
if type_ == 0 or type_ == "SIMPLE_PINHOLE":
params = None
camtype = "perspective"
elif type_ == 1 or type_ == "PINHOLE":
params = None
camtype = "perspective"
if type_ == 2 or type_ == "SIMPLE_RADIAL":
params = {k: 0.0 for k in ["k1", "k2", "k3", "p1", "p2"]}
params["k1"] = cam.k1
camtype = "perspective"
elif type_ == 3 or type_ == "RADIAL":
params = {k: 0.0 for k in ["k1", "k2", "k3", "p1", "p2"]}
params["k1"] = cam.k1
params["k2"] = cam.k2
camtype = "perspective"
elif type_ == 4 or type_ == "OPENCV":
params = {k: 0.0 for k in ["k1", "k2", "k3", "p1", "p2"]}
params["k1"] = cam.k1
params["k2"] = cam.k2
params["p1"] = cam.p1
params["p2"] = cam.p2
camtype = "perspective"
elif type_ == 5 or type_ == "OPENCV_FISHEYE":
params = {k: 0.0 for k in ["k1", "k2", "k3", "k4"]}
params["k1"] = cam.k1
params["k2"] = cam.k2
params["k3"] = cam.k3
params["k4"] = cam.k4
camtype = "fisheye"
assert params is None, "Only support pinhole camera model."
# Previous Nerf results were generated with images sorted by filename,
# ensure metrics are reported on the same test set.
inds = np.argsort(image_names)
image_names = [image_names[i] for i in inds]
camtoworlds = camtoworlds[inds]
# Load images.
if factor > 1:
image_dir_suffix = f"_{factor}"
else:
image_dir_suffix = ""
colmap_image_dir = os.path.join(data_dir, "images")
image_dir = os.path.join(data_dir, "images" + image_dir_suffix)
for d in [image_dir, colmap_image_dir]:
if not os.path.exists(d):
raise ValueError(f"Image folder {d} does not exist.")
# Downsampled images may have different names vs images used for COLMAP,
# so we need to map between the two sorted lists of files.
colmap_files = sorted(os.listdir(colmap_image_dir))
image_files = sorted(os.listdir(image_dir))
colmap_to_image = dict(zip(colmap_files, image_files))
image_paths = [
os.path.join(image_dir, colmap_to_image[f]) for f in image_names
]
print("loading images")
images = [imageio.imread(x) for x in tqdm.tqdm(image_paths)]
images = np.stack(images, axis=0)
# Select the split.
all_indices = np.arange(images.shape[0])
split_indices = {
"test": all_indices[all_indices % 8 == 0],
"train": all_indices[all_indices % 8 != 0],
}
indices = split_indices[split]
# All per-image quantities must be re-indexed using the split indices.
images = images[indices]
camtoworlds = camtoworlds[indices]
return images, camtoworlds, K
class SubjectLoader(torch.utils.data.Dataset):
"""Single subject data loader for training and evaluation."""
SPLITS = ["train", "test"]
SUBJECT_IDS = [
"garden",
"bicycle",
"bonsai",
"counter",
"kitchen",
"room",
"stump",
]
OPENGL_CAMERA = False
def __init__(
self,
subject_id: str,
root_fp: str,
split: str,
color_bkgd_aug: str = "white",
num_rays: int = None,
near: float = None,
far: float = None,
batch_over_images: bool = True,
factor: int = 1,
):
super().__init__()
assert split in self.SPLITS, "%s" % split
assert subject_id in self.SUBJECT_IDS, "%s" % subject_id
assert color_bkgd_aug in ["white", "black", "random"]
self.split = split
self.num_rays = num_rays
self.near = near
self.far = far
self.training = (num_rays is not None) and (
split in ["train", "trainval"]
)
self.color_bkgd_aug = color_bkgd_aug
self.batch_over_images = batch_over_images
self.images, self.camtoworlds, self.K = _load_colmap(
root_fp, subject_id, split, factor
)
self.images = torch.from_numpy(self.images).to(torch.uint8)
self.camtoworlds = torch.from_numpy(self.camtoworlds).to(torch.float32)
self.K = torch.tensor(self.K).to(torch.float32)
self.height, self.width = self.images.shape[1:3]
def __len__(self):
return len(self.images)
@torch.no_grad()
def __getitem__(self, index):
data = self.fetch_data(index)
data = self.preprocess(data)
return data
def preprocess(self, data):
"""Process the fetched / cached data with randomness."""
pixels, rays = data["rgb"], data["rays"]
if self.training:
if self.color_bkgd_aug == "random":
color_bkgd = torch.rand(3, device=self.images.device)
elif self.color_bkgd_aug == "white":
color_bkgd = torch.ones(3, device=self.ima
gitextract_10qul1w_/
├── .gitignore
├── LICENSE
├── README.md
├── __init__.py
├── config/
│ ├── diligent.conf
│ └── own_objects.conf
├── create_env.sh
├── data_capture_and_preprocessing/
│ ├── README.md
│ ├── gather_and_convert_normal_map.py
│ ├── iPhone_mvps_data_preprocessing.py
│ ├── metashape2neus.py
│ ├── metashape2neus2_json_and_images.py
│ └── sam_mvps.py
├── download_data.sh
├── exp_runner.py
├── models/
│ ├── cd_and_fscore.py
│ ├── dataset_loader.py
│ ├── fields.py
│ └── renderer.py
├── run_diligent.sh
├── run_own_object.sh
├── third_parties/
│ └── nerfacc-0.3.5/
│ └── nerfacc-0.3.5/
│ ├── .github/
│ │ └── workflows/
│ │ ├── building.yml
│ │ ├── code_checks.yml
│ │ ├── cuda/
│ │ │ ├── cu101-Linux-env.sh
│ │ │ ├── cu101-Linux.sh
│ │ │ ├── cu101-Windows-env.sh
│ │ │ ├── cu101-Windows.sh
│ │ │ ├── cu102-Linux-env.sh
│ │ │ ├── cu102-Linux.sh
│ │ │ ├── cu102-Windows-env.sh
│ │ │ ├── cu102-Windows.sh
│ │ │ ├── cu111-Linux-env.sh
│ │ │ ├── cu111-Linux.sh
│ │ │ ├── cu111-Windows-env.sh
│ │ │ ├── cu111-Windows.sh
│ │ │ ├── cu113-Linux-env.sh
│ │ │ ├── cu113-Linux.sh
│ │ │ ├── cu113-Windows-env.sh
│ │ │ ├── cu113-Windows.sh
│ │ │ ├── cu115-Linux-env.sh
│ │ │ ├── cu115-Linux.sh
│ │ │ ├── cu115-Windows-env.sh
│ │ │ ├── cu115-Windows.sh
│ │ │ ├── cu116-Linux-env.sh
│ │ │ ├── cu116-Linux.sh
│ │ │ ├── cu116-Windows-env.sh
│ │ │ ├── cu116-Windows.sh
│ │ │ ├── cu117-Linux-env.sh
│ │ │ ├── cu117-Linux.sh
│ │ │ ├── cu117-Windows-env.sh
│ │ │ └── cu117-Windows.sh
│ │ └── publish.yml
│ ├── .gitignore
│ ├── .gitmodules
│ ├── .pre-commit-config.yaml
│ ├── .readthedocs.yaml
│ ├── CMakeLists.txt
│ ├── LICENSE
│ ├── MANIFEST.in
│ ├── README.md
│ ├── docs/
│ │ ├── Makefile
│ │ ├── requirements.txt
│ │ └── source/
│ │ ├── _static/
│ │ │ └── css/
│ │ │ └── readthedocs.css
│ │ ├── apis/
│ │ │ ├── generated/
│ │ │ │ ├── nerfacc.accumulate_along_rays.rst
│ │ │ │ ├── nerfacc.pack_data.rst
│ │ │ │ ├── nerfacc.ray_aabb_intersect.rst
│ │ │ │ ├── nerfacc.ray_resampling.rst
│ │ │ │ ├── nerfacc.render_transmittance_from_alpha.rst
│ │ │ │ ├── nerfacc.render_transmittance_from_density.rst
│ │ │ │ ├── nerfacc.render_visibility.rst
│ │ │ │ ├── nerfacc.render_weight_from_alpha.rst
│ │ │ │ ├── nerfacc.render_weight_from_density.rst
│ │ │ │ ├── nerfacc.unpack_data.rst
│ │ │ │ └── nerfacc.unpack_info.rst
│ │ │ ├── grid.rst
│ │ │ ├── rendering.rst
│ │ │ └── utils.rst
│ │ ├── conf.py
│ │ ├── examples/
│ │ │ ├── dnerf.rst
│ │ │ ├── ngp.rst
│ │ │ ├── unbounded.rst
│ │ │ └── vanilla.rst
│ │ └── index.rst
│ ├── examples/
│ │ ├── datasets/
│ │ │ ├── __init__.py
│ │ │ ├── dnerf_synthetic.py
│ │ │ ├── nerf_360_v2.py
│ │ │ ├── nerf_synthetic.py
│ │ │ └── utils.py
│ │ ├── radiance_fields/
│ │ │ ├── __init__.py
│ │ │ ├── mlp.py
│ │ │ └── ngp.py
│ │ ├── requirements.txt
│ │ ├── train_mlp_dnerf.py
│ │ ├── train_mlp_nerf.py
│ │ ├── train_ngp_nerf.py
│ │ └── utils.py
│ ├── nerfacc/
│ │ ├── __init__.py
│ │ ├── cdf.py
│ │ ├── contraction.py
│ │ ├── cuda/
│ │ │ ├── __init__.py
│ │ │ ├── _backend.py
│ │ │ └── csrc/
│ │ │ ├── cdf.cu
│ │ │ ├── contraction.cu
│ │ │ ├── include/
│ │ │ │ ├── helpers_contraction.h
│ │ │ │ ├── helpers_cuda.h
│ │ │ │ └── helpers_math.h
│ │ │ ├── intersection.cu
│ │ │ ├── pack.cu
│ │ │ ├── pybind.cu
│ │ │ ├── ray_marching.cu
│ │ │ ├── render_transmittance.cu
│ │ │ ├── render_transmittance_cub.cu
│ │ │ └── render_weight.cu
│ │ ├── grid.py
│ │ ├── intersection.py
│ │ ├── losses.py
│ │ ├── pack.py
│ │ ├── ray_marching.py
│ │ ├── sampling.py
│ │ ├── version.py
│ │ └── vol_rendering.py
│ ├── scripts/
│ │ ├── run_aws_listing.py
│ │ ├── run_dev_checks.py
│ │ └── run_profiler.py
│ ├── setup.cfg
│ ├── setup.py
│ └── tests/
│ ├── test_contraction.py
│ ├── test_grid.py
│ ├── test_intersection.py
│ ├── test_loss.py
│ ├── test_pack.py
│ ├── test_ray_marching.py
│ ├── test_rendering.py
│ └── test_resampling.py
└── utilities/
└── utils.py
SYMBOL INDEX (395 symbols across 42 files)
FILE: data_capture_and_preprocessing/metashape2neus.py
function normalize_camera (line 7) | def normalize_camera(R_list, t_list, camera2object_ratio=3):
function make4x4 (line 26) | def make4x4(P):
class MetashapePoseLoader (line 34) | class MetashapePoseLoader:
method __init__ (line 35) | def __init__(self, xml_path, camera2object_ratio):
FILE: data_capture_and_preprocessing/metashape2neus2_json_and_images.py
function create_json_file (line 10) | def create_json_file(data, filename):
FILE: data_capture_and_preprocessing/sam_mvps.py
function pick_point (line 27) | def pick_point(event, x, y, flags, param):
function show_mask (line 32) | def show_mask(mask, ax, random_color=False):
function show_points (line 42) | def show_points(coords, labels, ax, marker_size=375):
function show_box (line 51) | def show_box(box, ax):
FILE: exp_runner.py
function get_class (line 28) | def get_class(kls):
class Runner (line 36) | class Runner:
method __init__ (line 37) | def __init__(self, conf_text, mode='train', is_continue=False, datadir...
method train (line 121) | def train(self):
method update_learning_rate (line 270) | def update_learning_rate(self):
method file_backup (line 281) | def file_backup(self):
method load_checkpoint (line 298) | def load_checkpoint(self, checkpoint_name):
method save_checkpoint (line 306) | def save_checkpoint(self):
method validate_normal_pixel_based (line 317) | def validate_normal_pixel_based(self, idx=-1, resolution_level=-1):
method validate_normal_patch_based (line 397) | def validate_normal_patch_based(self, idx=-1, resolution_level=-1, gra...
method validate_mesh (line 483) | def validate_mesh(self, world_space=True, resolution=256, threshold=0.0):
method remove_isolated_clusters (line 508) | def remove_isolated_clusters(self, mesh):
method eval_mae (line 527) | def eval_mae(self, gradient_method):
method eval_geo (line 580) | def eval_geo(self, resolution=1024):
method find_visible_points (line 618) | def find_visible_points(self, mesh):
FILE: models/cd_and_fscore.py
function chamfer_distance_and_f1_score (line 5) | def chamfer_distance_and_f1_score(ref_points, eval_points, f_threshold=0...
FILE: models/dataset_loader.py
function load_K_Rt_from_P (line 14) | def load_K_Rt_from_P(filename, P=None):
class Dataset (line 37) | class Dataset:
method __init__ (line 38) | def __init__(self, conf):
method gen_rays_at (line 158) | def gen_rays_at(self, img_idx, resolution_level=1, within_mask=False):
method gen_patches_at (line 183) | def gen_patches_at(self, img_idx, resolution_level=1, patch_H=3, patch...
method gen_random_patches (line 229) | def gen_random_patches(self, num_patch, patch_H=3, patch_W=3):
method near_far_from_sphere (line 285) | def near_far_from_sphere(self, rays_o, rays_d):
method image_at (line 305) | def image_at(self, idx, resolution_level):
FILE: models/fields.py
class SDFNetwork (line 7) | class SDFNetwork(nn.Module):
method __init__ (line 8) | def __init__(self,
method increase_bandwidth (line 73) | def increase_bandwidth(self):
method forward (line 76) | def forward(self, inputs):
method sdf (line 101) | def sdf(self, x):
method sdf_hidden_appearance (line 104) | def sdf_hidden_appearance(self, x):
method gradient (line 108) | def gradient(self, x):
method divergence (line 122) | def divergence(self, y, x):
method laplace (line 129) | def laplace(self, x):
class SingleVarianceNetwork (line 133) | class SingleVarianceNetwork(nn.Module):
method __init__ (line 134) | def __init__(self, init_val):
method forward (line 138) | def forward(self, x):
FILE: models/renderer.py
function extract_fields (line 9) | def extract_fields(bound_min, bound_max, resolution, query_func):
function extract_geometry (line 27) | def extract_geometry(bound_min, bound_max, resolution, threshold, query_...
class NeuSRenderer (line 37) | class NeuSRenderer:
method __init__ (line 38) | def __init__(self, sdf_network, deviation_network,
method occ_eval_fn (line 56) | def occ_eval_fn(self, x):
method render (line 63) | def render(self, rays_o_patch_all, # (num_patch, patch_H, patch_W, 3)
method render_normal_pixel_based (line 279) | def render_normal_pixel_based(self, rays_o, rays_d, near, far):
method extract_geometry (line 353) | def extract_geometry(self, bound_min, bound_max, resolution, threshold...
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/examples/datasets/dnerf_synthetic.py
function _load_renderings (line 16) | def _load_renderings(root_fp: str, subject_id: str, split: str):
class SubjectLoader (line 60) | class SubjectLoader(torch.utils.data.Dataset):
method __init__ (line 79) | def __init__(
method __len__ (line 124) | def __len__(self):
method __getitem__ (line 128) | def __getitem__(self, index):
method preprocess (line 133) | def preprocess(self, data):
method update_num_rays (line 157) | def update_num_rays(self, num_rays):
method fetch_data (line 160) | def fetch_data(self, index):
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/examples/datasets/nerf_360_v2.py
function _load_colmap (line 25) | def _load_colmap(root_fp: str, subject_id: str, split: str, factor: int ...
class SubjectLoader (line 145) | class SubjectLoader(torch.utils.data.Dataset):
method __init__ (line 161) | def __init__(
method __len__ (line 194) | def __len__(self):
method __getitem__ (line 198) | def __getitem__(self, index):
method preprocess (line 203) | def preprocess(self, data):
method update_num_rays (line 225) | def update_num_rays(self, num_rays):
method fetch_data (line 228) | def fetch_data(self, index):
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/examples/datasets/nerf_synthetic.py
function _load_renderings (line 17) | def _load_renderings(root_fp: str, subject_id: str, split: str):
class SubjectLoader (line 53) | class SubjectLoader(torch.utils.data.Dataset):
method __init__ (line 72) | def __init__(
method __len__ (line 124) | def __len__(self):
method __getitem__ (line 128) | def __getitem__(self, index):
method preprocess (line 133) | def preprocess(self, data):
method update_num_rays (line 157) | def update_num_rays(self, num_rays):
method fetch_data (line 160) | def fetch_data(self, index):
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/examples/datasets/utils.py
function namedtuple_map (line 10) | def namedtuple_map(fn, tup):
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/examples/radiance_fields/mlp.py
class MLP (line 14) | class MLP(nn.Module):
method __init__ (line 15) | def __init__(
method initialize (line 67) | def initialize(self):
method forward (line 87) | def forward(self, x):
class DenseLayer (line 104) | class DenseLayer(MLP):
method __init__ (line 105) | def __init__(self, input_dim, output_dim, **kwargs):
class NerfMLP (line 114) | class NerfMLP(nn.Module):
method __init__ (line 115) | def __init__(
method query_density (line 148) | def query_density(self, x):
method forward (line 153) | def forward(self, x, condition=None):
class SinusoidalEncoder (line 168) | class SinusoidalEncoder(nn.Module):
method __init__ (line 171) | def __init__(self, x_dim, min_deg, max_deg, use_identity: bool = True):
method latent_dim (line 182) | def latent_dim(self) -> int:
method forward (line 187) | def forward(self, x: torch.Tensor) -> torch.Tensor:
class VanillaNeRFRadianceField (line 206) | class VanillaNeRFRadianceField(nn.Module):
method __init__ (line 207) | def __init__(
method query_opacity (line 228) | def query_opacity(self, x, step_size):
method query_density (line 235) | def query_density(self, x):
method forward (line 240) | def forward(self, x, condition=None):
class DNeRFRadianceField (line 248) | class DNeRFRadianceField(nn.Module):
method __init__ (line 249) | def __init__(self) -> None:
method query_opacity (line 264) | def query_opacity(self, x, timestamps, step_size):
method query_density (line 273) | def query_density(self, x, t):
method forward (line 279) | def forward(self, x, t, condition=None):
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/examples/radiance_fields/ngp.py
class _TruncExp (line 22) | class _TruncExp(Function): # pylint: disable=abstract-method
method forward (line 27) | def forward(ctx, x): # pylint: disable=arguments-differ
method backward (line 33) | def backward(ctx, g): # pylint: disable=arguments-differ
function contract_to_unisphere (line 41) | def contract_to_unisphere(
class NGPradianceField (line 66) | class NGPradianceField(torch.nn.Module):
method __init__ (line 69) | def __init__(
method query_density (line 147) | def query_density(self, x, return_feat: bool = False):
method _query_rgb (line 171) | def _query_rgb(self, dir, embedding):
method forward (line 186) | def forward(
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/examples/train_ngp_nerf.py
function occ_eval_fn (line 190) | def occ_eval_fn(x):
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/examples/utils.py
function set_random_seed (line 15) | def set_random_seed(seed):
function render_image (line 21) | def render_image(
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/nerfacc/__init__.py
function unpack_to_ray_indices (line 30) | def unpack_to_ray_indices(*args, **kwargs):
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/nerfacc/cdf.py
function ray_resampling (line 12) | def ray_resampling(
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/nerfacc/contraction.py
class ContractionType (line 12) | class ContractionType(Enum):
method to_cpp_version (line 55) | def to_cpp_version(self):
function contract (line 66) | def contract(
function contract_inv (line 86) | def contract_inv(
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/nerfacc/cuda/__init__.py
function _make_lazy_cuda_func (line 8) | def _make_lazy_cuda_func(name: str) -> Callable:
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/nerfacc/cuda/_backend.py
function cuda_toolkit_available (line 17) | def cuda_toolkit_available():
function cuda_toolkit_version (line 26) | def cuda_toolkit_version():
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/nerfacc/cuda/csrc/include/helpers_contraction.h
type ContractionType (line 9) | enum ContractionType
function float3 (line 16) | float3 roi_to_unit(
function float3 (line 23) | float3 unit_to_roi(
function float3 (line 30) | float3 inf_to_unit_tanh(
function float3 (line 42) | float3 unit_to_inf_tanh(
function float3 (line 61) | float3 inf_to_unit_sphere(
function float3 (line 81) | float3 unit_sphere_to_inf(
function float3 (line 101) | float3 apply_contraction(
function float3 (line 116) | float3 apply_contraction_inv(
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/nerfacc/cuda/csrc/include/helpers_math.h
type uint (line 45) | typedef unsigned int uint;
type ushort (line 46) | typedef unsigned short ushort;
function fminf (line 59) | inline float fminf(float a, float b)
function fmaxf (line 64) | inline float fmaxf(float a, float b)
function max (line 69) | inline int max(int a, int b)
function min (line 74) | inline int min(int a, int b)
function rsqrtf (line 79) | inline float rsqrtf(float x)
function float2 (line 89) | float2 make_float2(float s)
function float2 (line 93) | float2 make_float2(float3 a)
function float2 (line 97) | float2 make_float2(int2 a)
function float2 (line 101) | float2 make_float2(uint2 a)
function int2 (line 106) | int2 make_int2(int s)
function int2 (line 110) | int2 make_int2(int3 a)
function int2 (line 114) | int2 make_int2(uint2 a)
function int2 (line 118) | int2 make_int2(float2 a)
function uint2 (line 123) | uint2 make_uint2(uint s)
function uint2 (line 127) | uint2 make_uint2(uint3 a)
function uint2 (line 131) | uint2 make_uint2(int2 a)
function float3 (line 136) | float3 make_float3(float s)
function float3 (line 140) | float3 make_float3(float2 a)
function float3 (line 144) | float3 make_float3(float2 a, float s)
function float3 (line 148) | float3 make_float3(float4 a)
function float3 (line 152) | float3 make_float3(int3 a)
function float3 (line 156) | float3 make_float3(uint3 a)
function int3 (line 161) | int3 make_int3(int s)
function int3 (line 165) | int3 make_int3(int2 a)
function int3 (line 169) | int3 make_int3(int2 a, int s)
function int3 (line 173) | int3 make_int3(uint3 a)
function int3 (line 177) | int3 make_int3(float3 a)
function uint3 (line 182) | uint3 make_uint3(uint s)
function uint3 (line 186) | uint3 make_uint3(uint2 a)
function uint3 (line 190) | uint3 make_uint3(uint2 a, uint s)
function uint3 (line 194) | uint3 make_uint3(uint4 a)
function uint3 (line 198) | uint3 make_uint3(int3 a)
function float4 (line 203) | float4 make_float4(float s)
function float4 (line 207) | float4 make_float4(float3 a)
function float4 (line 211) | float4 make_float4(float3 a, float w)
function float4 (line 215) | float4 make_float4(int4 a)
function float4 (line 219) | float4 make_float4(uint4 a)
function int4 (line 224) | int4 make_int4(int s)
function int4 (line 228) | int4 make_int4(int3 a)
function int4 (line 232) | int4 make_int4(int3 a, int w)
function int4 (line 236) | int4 make_int4(uint4 a)
function int4 (line 240) | int4 make_int4(float4 a)
function uint4 (line 245) | uint4 make_uint4(uint s)
function uint4 (line 249) | uint4 make_uint4(uint3 a)
function uint4 (line 253) | uint4 make_uint4(uint3 a, uint w)
function uint4 (line 257) | uint4 make_uint4(int4 a)
function float2 (line 750) | float2 operator*(float2 a, float2 b)
function float2 (line 759) | float2 operator*(float2 a, float b)
function float2 (line 763) | float2 operator*(float b, float2 a)
function int2 (line 773) | int2 operator*(int2 a, int2 b)
function int2 (line 782) | int2 operator*(int2 a, int b)
function int2 (line 786) | int2 operator*(int b, int2 a)
function uint2 (line 796) | uint2 operator*(uint2 a, uint2 b)
function uint2 (line 805) | uint2 operator*(uint2 a, uint b)
function uint2 (line 809) | uint2 operator*(uint b, uint2 a)
function float3 (line 819) | float3 operator*(float3 a, float3 b)
function float3 (line 829) | float3 operator*(float3 a, float b)
function float3 (line 833) | float3 operator*(float b, float3 a)
function int3 (line 844) | int3 operator*(int3 a, int3 b)
function int3 (line 854) | int3 operator*(int3 a, int b)
function int3 (line 858) | int3 operator*(int b, int3 a)
function uint3 (line 869) | uint3 operator*(uint3 a, uint3 b)
function uint3 (line 879) | uint3 operator*(uint3 a, uint b)
function uint3 (line 883) | uint3 operator*(uint b, uint3 a)
function float4 (line 894) | float4 operator*(float4 a, float4 b)
function float4 (line 905) | float4 operator*(float4 a, float b)
function float4 (line 909) | float4 operator*(float b, float4 a)
function int4 (line 921) | int4 operator*(int4 a, int4 b)
function int4 (line 932) | int4 operator*(int4 a, int b)
function int4 (line 936) | int4 operator*(int b, int4 a)
function uint4 (line 948) | uint4 operator*(uint4 a, uint4 b)
function uint4 (line 959) | uint4 operator*(uint4 a, uint b)
function uint4 (line 963) | uint4 operator*(uint b, uint4 a)
function float2 (line 1058) | float2 fminf(float2 a, float2 b)
function float3 (line 1062) | float3 fminf(float3 a, float3 b)
function float4 (line 1066) | float4 fminf(float4 a, float4 b)
function int2 (line 1071) | int2 min(int2 a, int2 b)
function int3 (line 1075) | int3 min(int3 a, int3 b)
function int4 (line 1079) | int4 min(int4 a, int4 b)
function uint2 (line 1084) | uint2 min(uint2 a, uint2 b)
function uint3 (line 1088) | uint3 min(uint3 a, uint3 b)
function uint4 (line 1092) | uint4 min(uint4 a, uint4 b)
function float2 (line 1101) | float2 fmaxf(float2 a, float2 b)
function float3 (line 1105) | float3 fmaxf(float3 a, float3 b)
function float4 (line 1109) | float4 fmaxf(float4 a, float4 b)
function int2 (line 1114) | int2 max(int2 a, int2 b)
function int3 (line 1118) | int3 max(int3 a, int3 b)
function int4 (line 1122) | int4 max(int4 a, int4 b)
function uint2 (line 1127) | uint2 max(uint2 a, uint2 b)
function uint3 (line 1131) | uint3 max(uint3 a, uint3 b)
function uint4 (line 1135) | uint4 max(uint4 a, uint4 b)
function lerp (line 1145) | float lerp(float a, float b, float t)
function float2 (line 1149) | float2 lerp(float2 a, float2 b, float t)
function float3 (line 1153) | float3 lerp(float3 a, float3 b, float t)
function float4 (line 1157) | float4 lerp(float4 a, float4 b, float t)
function clamp (line 1167) | float clamp(float f, float a, float b)
function clamp (line 1171) | int clamp(int f, int a, int b)
function uint (line 1175) | uint clamp(uint f, uint a, uint b)
function float2 (line 1180) | float2 clamp(float2 v, float a, float b)
function float2 (line 1184) | float2 clamp(float2 v, float2 a, float2 b)
function float3 (line 1188) | float3 clamp(float3 v, float a, float b)
function float3 (line 1192) | float3 clamp(float3 v, float3 a, float3 b)
function float4 (line 1196) | float4 clamp(float4 v, float a, float b)
function float4 (line 1200) | float4 clamp(float4 v, float4 a, float4 b)
function int2 (line 1205) | int2 clamp(int2 v, int a, int b)
function int2 (line 1209) | int2 clamp(int2 v, int2 a, int2 b)
function int3 (line 1213) | int3 clamp(int3 v, int a, int b)
function int3 (line 1217) | int3 clamp(int3 v, int3 a, int3 b)
function int4 (line 1221) | int4 clamp(int4 v, int a, int b)
function int4 (line 1225) | int4 clamp(int4 v, int4 a, int4 b)
function uint2 (line 1230) | uint2 clamp(uint2 v, uint a, uint b)
function uint2 (line 1234) | uint2 clamp(uint2 v, uint2 a, uint2 b)
function uint3 (line 1238) | uint3 clamp(uint3 v, uint a, uint b)
function uint3 (line 1242) | uint3 clamp(uint3 v, uint3 a, uint3 b)
function uint4 (line 1246) | uint4 clamp(uint4 v, uint a, uint b)
function uint4 (line 1250) | uint4 clamp(uint4 v, uint4 a, uint4 b)
function dot (line 1259) | float dot(float2 a, float2 b)
function dot (line 1263) | float dot(float3 a, float3 b)
function dot (line 1267) | float dot(float4 a, float4 b)
function dot (line 1272) | int dot(int2 a, int2 b)
function dot (line 1276) | int dot(int3 a, int3 b)
function dot (line 1280) | int dot(int4 a, int4 b)
function uint (line 1285) | uint dot(uint2 a, uint2 b)
function uint (line 1289) | uint dot(uint3 a, uint3 b)
function uint (line 1293) | uint dot(uint4 a, uint4 b)
function length (line 1302) | float length(float2 v)
function length (line 1306) | float length(float3 v)
function length (line 1310) | float length(float4 v)
function float2 (line 1319) | float2 normalize(float2 v)
function float3 (line 1324) | float3 normalize(float3 v)
function float4 (line 1329) | float4 normalize(float4 v)
function float2 (line 1339) | float2 floorf(float2 v)
function float3 (line 1343) | float3 floorf(float3 v)
function float4 (line 1347) | float4 floorf(float4 v)
function fracf (line 1356) | float fracf(float v)
function float2 (line 1360) | float2 fracf(float2 v)
function float3 (line 1364) | float3 fracf(float3 v)
function float4 (line 1368) | float4 fracf(float4 v)
function float2 (line 1377) | float2 fmodf(float2 a, float2 b)
function float3 (line 1381) | float3 fmodf(float3 a, float3 b)
function float4 (line 1385) | float4 fmodf(float4 a, float4 b)
function float2 (line 1394) | float2 fabs(float2 v)
function float3 (line 1398) | float3 fabs(float3 v)
function float4 (line 1402) | float4 fabs(float4 v)
function int2 (line 1407) | int2 abs(int2 v)
function int3 (line 1411) | int3 abs(int3 v)
function int4 (line 1415) | int4 abs(int4 v)
function float3 (line 1426) | float3 reflect(float3 i, float3 n)
function float3 (line 1435) | float3 cross(float3 a, float3 b)
function smoothstep (line 1447) | float smoothstep(float a, float b, float x)
function float2 (line 1452) | float2 smoothstep(float2 a, float2 b, float2 x)
function float3 (line 1457) | float3 smoothstep(float3 a, float3 b, float3 x)
function float4 (line 1462) | float4 smoothstep(float4 a, float4 b, float4 x)
function float3 (line 1471) | float3 sign(float3 a)
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/nerfacc/grid.py
function query_grid (line 19) | def query_grid(
class Grid (line 50) | class Grid(nn.Module):
method __init__ (line 67) | def __init__(self, *args, **kwargs):
method device (line 72) | def device(self) -> torch.device:
method roi_aabb (line 76) | def roi_aabb(self) -> torch.Tensor:
method binary (line 87) | def binary(self) -> torch.Tensor:
method contraction_type (line 99) | def contraction_type(self) -> ContractionType:
class OccupancyGrid (line 113) | class OccupancyGrid(Grid):
method __init__ (line 127) | def __init__(
method _get_all_cells (line 177) | def _get_all_cells(self) -> torch.Tensor:
method _sample_uniform_and_occupied_cells (line 182) | def _sample_uniform_and_occupied_cells(self, n: int) -> torch.Tensor:
method _update (line 197) | def _update(
method every_n_step (line 242) | def every_n_step(
method query_occ (line 280) | def query_occ(self, samples: torch.Tensor) -> torch.Tensor:
function _meshgrid3d (line 297) | def _meshgrid3d(
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/nerfacc/intersection.py
function ray_aabb_intersect (line 14) | def ray_aabb_intersect(
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/nerfacc/losses.py
function distortion (line 6) | def distortion(
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/nerfacc/pack.py
function pack_data (line 12) | def pack_data(data: Tensor, mask: Tensor) -> Tuple[Tensor, Tensor]:
function pack_info (line 47) | def pack_info(ray_indices: Tensor, n_rays: int = None) -> Tensor:
function unpack_info (line 81) | def unpack_info(packed_info: Tensor, n_samples: int) -> Tensor:
function unpack_data (line 124) | def unpack_data(
class _UnpackData (line 170) | class _UnpackData(torch.autograd.Function):
method forward (line 174) | def forward(ctx, packed_info: Tensor, data: Tensor, n_samples: int):
method backward (line 184) | def backward(ctx, grad: Tensor):
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/nerfacc/ray_marching.py
function ray_marching (line 14) | def ray_marching(
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/nerfacc/sampling.py
function sample_along_rays (line 18) | def sample_along_rays(
function sample_along_rays (line 32) | def sample_along_rays(
function sample_along_rays (line 45) | def sample_along_rays(
function proposal_sampling_with_filter (line 102) | def proposal_sampling_with_filter(
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/nerfacc/vol_rendering.py
function rendering (line 15) | def rendering(
function accumulate_along_rays (line 132) | def accumulate_along_rays(
function accumulate_along_rays_importance (line 200) | def accumulate_along_rays_importance(
function accumulate_along_rays_patch_based (line 269) | def accumulate_along_rays_patch_based(
function render_transmittance_from_density (line 338) | def render_transmittance_from_density(
function render_transmittance_from_alpha (line 406) | def render_transmittance_from_alpha(
function render_weight_from_density (line 463) | def render_weight_from_density(
function render_weight_from_alpha_patch_based (line 533) | def render_weight_from_alpha_patch_based(
function render_weight_and_transmittance_from_alpha_patch_based (line 579) | def render_weight_and_transmittance_from_alpha_patch_based(
function render_weight_from_alpha (line 624) | def render_weight_from_alpha(
function render_visibility (line 681) | def render_visibility(
function render_visibility_patch_based (line 752) | def render_visibility_patch_based(
class _RenderingTransmittanceFromDensityCUB (line 821) | class _RenderingTransmittanceFromDensityCUB(torch.autograd.Function):
method forward (line 825) | def forward(ctx, ray_indices, t_starts, t_ends, sigmas):
method backward (line 838) | def backward(ctx, transmittance_grads):
class _RenderingTransmittanceFromDensityNaive (line 847) | class _RenderingTransmittanceFromDensityNaive(torch.autograd.Function):
method forward (line 851) | def forward(ctx, packed_info, t_starts, t_ends, sigmas):
method backward (line 864) | def backward(ctx, transmittance_grads):
class _RenderingTransmittanceFromAlphaCUB (line 873) | class _RenderingTransmittanceFromAlphaCUB(torch.autograd.Function):
method forward (line 877) | def forward(ctx, ray_indices, alphas):
method backward (line 888) | def backward(ctx, transmittance_grads):
class _RenderingTransmittanceFromAlphaNaive (line 897) | class _RenderingTransmittanceFromAlphaNaive(torch.autograd.Function):
method forward (line 901) | def forward(ctx, packed_info, alphas):
method backward (line 912) | def backward(ctx, transmittance_grads):
class _RenderingTransmittanceFromAlphaPatchBasedNaive (line 920) | class _RenderingTransmittanceFromAlphaPatchBasedNaive(torch.autograd.Fun...
method forward (line 924) | def forward(ctx, packed_info, alphas):
method backward (line 933) | def backward(ctx, grad_transmittance):
class _RenderingWeightFromDensityNaive (line 941) | class _RenderingWeightFromDensityNaive(torch.autograd.Function):
method forward (line 945) | def forward(ctx, packed_info, t_starts, t_ends, sigmas):
method backward (line 960) | def backward(ctx, grad_weights):
class _RenderingWeightFromAlphaNaive (line 969) | class _RenderingWeightFromAlphaNaive(torch.autograd.Function):
method forward (line 973) | def forward(ctx, packed_info, alphas):
method backward (line 982) | def backward(ctx, grad_weights):
class _RenderingWeightFromAlphaPatchBasedNaive (line 990) | class _RenderingWeightFromAlphaPatchBasedNaive(torch.autograd.Function):
method forward (line 994) | def forward(ctx, packed_info, alphas):
method backward (line 1004) | def backward(ctx, grad_weights):
class _RenderingWeightAndTransmittanceFromAlphaPatchBasedNaive (line 1013) | class _RenderingWeightAndTransmittanceFromAlphaPatchBasedNaive(torch.aut...
method forward (line 1017) | def forward(ctx, packed_info, alphas):
method backward (line 1027) | def backward(ctx, grad_weights, grad_transmittance):
class _RenderingWeightFromAlphaImportanceSamplingNaive (line 1036) | class _RenderingWeightFromAlphaImportanceSamplingNaive(torch.autograd.Fu...
method forward (line 1040) | def forward(ctx, packed_info, alphas, importance_pdfs):
method backward (line 1050) | def backward(ctx, grad_weights):
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/scripts/run_dev_checks.py
function run_command (line 20) | def run_command(command: str) -> bool:
function run_github_actions_file (line 33) | def run_github_actions_file(filename: str):
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/scripts/run_profiler.py
class Profiler (line 12) | class Profiler:
method __init__ (line 13) | def __init__(self, warmup=10, repeat=1000):
method __call__ (line 17) | def __call__(self, func: Callable):
function main (line 54) | def main():
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/setup.py
function get_ext (line 18) | def get_ext():
function get_extensions (line 26) | def get_extensions():
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/tests/test_contraction.py
function test_ContractionType (line 13) | def test_ContractionType():
function test_identity (line 23) | def test_identity():
function test_aabb (line 33) | def test_aabb():
function test_tanh (line 46) | def test_tanh():
function test_sphere (line 61) | def test_sphere():
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/tests/test_grid.py
function occ_eval_fn (line 10) | def occ_eval_fn(x: torch.Tensor) -> torch.Tensor:
function test_occ_grid (line 16) | def test_occ_grid():
function test_query_grid (line 25) | def test_query_grid():
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/tests/test_intersection.py
function test_intersection (line 12) | def test_intersection():
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/tests/test_loss.py
function test_distortion (line 13) | def test_distortion():
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/tests/test_pack.py
function test_pack_data (line 12) | def test_pack_data():
function test_unpack_info (line 27) | def test_unpack_info():
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/tests/test_ray_marching.py
function test_marching_with_near_far (line 11) | def test_marching_with_near_far():
function test_marching_with_grid (line 27) | def test_marching_with_grid():
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/tests/test_rendering.py
function test_render_visibility (line 19) | def test_render_visibility():
function test_render_weight_from_alpha (line 49) | def test_render_weight_from_alpha():
function test_render_weight_from_density (line 72) | def test_render_weight_from_density():
function test_accumulate_along_rays (line 93) | def test_accumulate_along_rays():
function test_rendering (line 114) | def test_rendering():
function test_grads (line 137) | def test_grads():
FILE: third_parties/nerfacc-0.3.5/nerfacc-0.3.5/tests/test_resampling.py
function test_resampling (line 11) | def test_resampling():
FILE: utilities/utils.py
function crop_a_set_of_images (line 13) | def crop_a_set_of_images(*image_path):
function crop_image_based_on_ref_image (line 36) | def crop_image_based_on_ref_image(ref_img_path, *img_path):
function angular_error_map (line 50) | def angular_error_map(N1, N2):
function crop_mask (line 56) | def crop_mask(mask):
function crop_image_by_mask (line 67) | def crop_image_by_mask(img, mask):
function save_video (line 76) | def save_video(vpath, images, fps):
function toRGBA (line 86) | def toRGBA(img, mask):
Condensed preview — 135 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (523K chars).
[
{
"path": ".gitignore",
"chars": 3250,
"preview": "data/\r\nexp/\r\n\r\n# Byte-compiled / optimized / DLL files\r\n__pycache__/\r\n*.py[cod]\r\n*$py.class\r\n\r\n# C extensions\r\n*.so\r\n\r\n#"
},
{
"path": "LICENSE",
"chars": 1074,
"preview": "MIT License\n\nCopyright (c) 2024 CyberAgent AI Lab\n\nPermission is hereby granted, free of charge, to any person obtaining"
},
{
"path": "README.md",
"chars": 4686,
"preview": "<h2 align=\"center\">SuperNormal: Neural Surface Reconstruction via Multi-View Normal Integration</h2>\n<h4 align=\"center\">"
},
{
"path": "__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "config/diligent.conf",
"chars": 2204,
"preview": "general {\r\n dataset_class = models.dataset_loader.Dataset\r\n renderer_class = models.renderer.NeuSRenderer\r\n\r\n b"
},
{
"path": "config/own_objects.conf",
"chars": 2076,
"preview": "general {\r\n dataset_class = models.dataset_loader.Dataset\r\n renderer_class = models.renderer.NeuSRenderer\r\n\r\n b"
},
{
"path": "create_env.sh",
"chars": 739,
"preview": "conda deactivate\nconda remove -y -n sn --all\nconda create -y -n sn python=3.8\nconda activate sn\n\npip install torch==2.1."
},
{
"path": "data_capture_and_preprocessing/README.md",
"chars": 6452,
"preview": "\nThis is a step-by-step guide to preprocess the raw images captured by an iPhone for the MVPS task.\nYou can download our"
},
{
"path": "data_capture_and_preprocessing/gather_and_convert_normal_map.py",
"chars": 2411,
"preview": "import os\nimport cv2\nimport pyexr\nfrom glob import glob\nimport numpy as np\nimport shutil\nfrom bs4 import BeautifulSoup "
},
{
"path": "data_capture_and_preprocessing/iPhone_mvps_data_preprocessing.py",
"chars": 1958,
"preview": "import rawpy, os\nfrom glob import glob\nimport cv2\nimport numpy as np\nimport os\nfrom tqdm import tqdm\nimport argparse\n\npa"
},
{
"path": "data_capture_and_preprocessing/metashape2neus.py",
"chars": 3612,
"preview": "import os.path\nimport xml\nfrom bs4 import BeautifulSoup # pip install beautifulsoup4 lxml\nimport numpy as np\n\n# details"
},
{
"path": "data_capture_and_preprocessing/metashape2neus2_json_and_images.py",
"chars": 2957,
"preview": "from glob import glob\nimport os\nimport numpy as np\nimport cv2\nfrom bs4 import BeautifulSoup\nfrom metashape2neus import n"
},
{
"path": "data_capture_and_preprocessing/sam_mvps.py",
"chars": 4196,
"preview": "import os.path\nfrom glob import glob\nimport argparse\nimport torch.cuda\nfrom segment_anything import SamPredictor, sam_mo"
},
{
"path": "download_data.sh",
"chars": 150,
"preview": "pip install gdown==5.1.0\ngdown 'https://drive.google.com/file/d/1Y3-v5jo-IRyTsPh8srZxIc2v5WZdPly_/view?usp=sharing' --fu"
},
{
"path": "exp_runner.py",
"chars": 33001,
"preview": "import os\r\nimport logging\r\nimport argparse\r\nimport numpy as np\r\nimport cv2 as cv\r\nimport trimesh\r\nimport torch\r\nimport t"
},
{
"path": "models/cd_and_fscore.py",
"chars": 1341,
"preview": "from scipy.spatial import KDTree\r\nimport numpy as np\r\n\r\n\r\ndef chamfer_distance_and_f1_score(ref_points, eval_points, f_t"
},
{
"path": "models/dataset_loader.py",
"chars": 17233,
"preview": "import torch\r\nimport torch.nn.functional as F\r\nimport cv2 as cv\r\nimport numpy as np\r\nimport os\r\nfrom glob import glob\r\nf"
},
{
"path": "models/fields.py",
"chars": 5060,
"preview": "import torch\r\nimport torch.nn as nn\r\nimport numpy as np\r\nimport tinycudann as tcnn\r\nfrom icecream import ic\r\n\r\nclass SDF"
},
{
"path": "models/renderer.py",
"chars": 19782,
"preview": "import torch\r\nimport numpy as np\r\nimport mcubes\r\nfrom tqdm import tqdm\r\nfrom nerfacc import ContractionType, OccupancyGr"
},
{
"path": "run_diligent.sh",
"chars": 129,
"preview": "for obj_name in buddha pot2 reading bear cow; do\n python exp_runner.py --conf config/diligent.conf --obj_name $obj_n"
},
{
"path": "run_own_object.sh",
"chars": 118,
"preview": "for obj_name in lion dog1 woman; do\n python exp_runner.py --conf config/own_objects.conf --obj_name $obj_name\ndone"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.github/workflows/building.yml",
"chars": 5650,
"preview": "name: Building Wheels\n\non: [workflow_dispatch]\n\njobs:\n\n wheel:\n runs-on: ${{ matrix.os }}\n environment: productio"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.github/workflows/code_checks.yml",
"chars": 832,
"preview": "name: Core Tests.\n\non:\n push:\n branches: [master]\n pull_request:\n branches: [master]\n\npermissions:\n contents: r"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.github/workflows/cuda/cu101-Linux-env.sh",
"chars": 203,
"preview": "#!/bin/bash\n\nCUDA_HOME=/usr/local/cuda-10.1\nLD_LIBRARY_PATH=${CUDA_HOME}/lib64:${LD_LIBRARY_PATH}\nPATH=${CUDA_HOME}/bin:"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.github/workflows/cuda/cu101-Linux.sh",
"chars": 737,
"preview": "#!/bin/bash\n\nOS=ubuntu1804\n\nwget -nv https://developer.download.nvidia.com/compute/cuda/repos/${OS}/x86_64/cuda-${OS}.pi"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.github/workflows/cuda/cu101-Windows-env.sh",
"chars": 284,
"preview": "#!/bin/bash\n\nCUDA_HOME=/c/Program\\ Files/NVIDIA\\ GPU\\ Computing\\ Toolkit/CUDA/v10.1\nPATH=${CUDA_HOME}/bin:$PATH\nPATH=/c/"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.github/workflows/cuda/cu101-Windows.sh",
"chars": 1086,
"preview": "#!/bin/bash\n\n# Install NVIDIA drivers, see:\n# https://github.com/pytorch/vision/blob/master/packaging/windows/internal/c"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.github/workflows/cuda/cu102-Linux-env.sh",
"chars": 203,
"preview": "#!/bin/bash\n\nCUDA_HOME=/usr/local/cuda-10.2\nLD_LIBRARY_PATH=${CUDA_HOME}/lib64:${LD_LIBRARY_PATH}\nPATH=${CUDA_HOME}/bin:"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.github/workflows/cuda/cu102-Linux.sh",
"chars": 733,
"preview": "#!/bin/bash\n\nOS=ubuntu1804\n\nwget -nv https://developer.download.nvidia.com/compute/cuda/repos/${OS}/x86_64/cuda-${OS}.pi"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.github/workflows/cuda/cu102-Windows-env.sh",
"chars": 284,
"preview": "#!/bin/bash\n\nCUDA_HOME=/c/Program\\ Files/NVIDIA\\ GPU\\ Computing\\ Toolkit/CUDA/v10.2\nPATH=${CUDA_HOME}/bin:$PATH\nPATH=/c/"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.github/workflows/cuda/cu102-Windows.sh",
"chars": 1084,
"preview": "#!/bin/bash\n\n# Install NVIDIA drivers, see:\n# https://github.com/pytorch/vision/blob/master/packaging/windows/internal/c"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.github/workflows/cuda/cu111-Linux-env.sh",
"chars": 211,
"preview": "#!/bin/bash\n\nCUDA_HOME=/usr/local/cuda-11.1\nLD_LIBRARY_PATH=${CUDA_HOME}/lib64:${LD_LIBRARY_PATH}\nPATH=${CUDA_HOME}/bin:"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.github/workflows/cuda/cu111-Linux.sh",
"chars": 700,
"preview": "#!/bin/bash\n\nOS=ubuntu1804\n\nwget -nv https://developer.download.nvidia.com/compute/cuda/repos/${OS}/x86_64/cuda-${OS}.pi"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.github/workflows/cuda/cu111-Windows-env.sh",
"chars": 268,
"preview": "#!/bin/bash\n\nCUDA_HOME=/c/Program\\ Files/NVIDIA\\ GPU\\ Computing\\ Toolkit/CUDA/v11.1\nPATH=${CUDA_HOME}/bin:$PATH\nPATH=/c/"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.github/workflows/cuda/cu111-Windows.sh",
"chars": 1080,
"preview": "#!/bin/bash\n\n# Install NVIDIA drivers, see:\n# https://github.com/pytorch/vision/blob/master/packaging/windows/internal/c"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.github/workflows/cuda/cu113-Linux-env.sh",
"chars": 211,
"preview": "#!/bin/bash\n\nCUDA_HOME=/usr/local/cuda-11.3\nLD_LIBRARY_PATH=${CUDA_HOME}/lib64:${LD_LIBRARY_PATH}\nPATH=${CUDA_HOME}/bin:"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.github/workflows/cuda/cu113-Linux.sh",
"chars": 700,
"preview": "#!/bin/bash\n\nOS=ubuntu1804\n\nwget -nv https://developer.download.nvidia.com/compute/cuda/repos/${OS}/x86_64/cuda-${OS}.pi"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.github/workflows/cuda/cu113-Windows-env.sh",
"chars": 268,
"preview": "#!/bin/bash\n\nCUDA_HOME=/c/Program\\ Files/NVIDIA\\ GPU\\ Computing\\ Toolkit/CUDA/v11.3\nPATH=${CUDA_HOME}/bin:$PATH\nPATH=/c/"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.github/workflows/cuda/cu113-Windows.sh",
"chars": 1101,
"preview": "#!/bin/bash\n\n# Install NVIDIA drivers, see:\n# https://github.com/pytorch/vision/blob/master/packaging/windows/internal/c"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.github/workflows/cuda/cu115-Linux-env.sh",
"chars": 211,
"preview": "#!/bin/bash\n\nCUDA_HOME=/usr/local/cuda-11.5\nLD_LIBRARY_PATH=${CUDA_HOME}/lib64:${LD_LIBRARY_PATH}\nPATH=${CUDA_HOME}/bin:"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.github/workflows/cuda/cu115-Linux.sh",
"chars": 700,
"preview": "#!/bin/bash\n\nOS=ubuntu1804\n\nwget -nv https://developer.download.nvidia.com/compute/cuda/repos/${OS}/x86_64/cuda-${OS}.pi"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.github/workflows/cuda/cu115-Windows-env.sh",
"chars": 268,
"preview": "#!/bin/bash\n\nCUDA_HOME=/c/Program\\ Files/NVIDIA\\ GPU\\ Computing\\ Toolkit/CUDA/v11.3\nPATH=${CUDA_HOME}/bin:$PATH\nPATH=/c/"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.github/workflows/cuda/cu115-Windows.sh",
"chars": 1170,
"preview": "#!/bin/bash\n\n# TODO We currently use CUDA 11.3 to build CUDA 11.5 Windows wheels\n\n# Install NVIDIA drivers, see:\n# https"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.github/workflows/cuda/cu116-Linux-env.sh",
"chars": 211,
"preview": "#!/bin/bash\n\nCUDA_HOME=/usr/local/cuda-11.6\nLD_LIBRARY_PATH=${CUDA_HOME}/lib64:${LD_LIBRARY_PATH}\nPATH=${CUDA_HOME}/bin:"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.github/workflows/cuda/cu116-Linux.sh",
"chars": 700,
"preview": "#!/bin/bash\n\nOS=ubuntu1804\n\nwget -nv https://developer.download.nvidia.com/compute/cuda/repos/${OS}/x86_64/cuda-${OS}.pi"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.github/workflows/cuda/cu116-Windows-env.sh",
"chars": 268,
"preview": "#!/bin/bash\n\nCUDA_HOME=/c/Program\\ Files/NVIDIA\\ GPU\\ Computing\\ Toolkit/CUDA/v11.3\nPATH=${CUDA_HOME}/bin:$PATH\nPATH=/c/"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.github/workflows/cuda/cu116-Windows.sh",
"chars": 1170,
"preview": "#!/bin/bash\n\n# TODO We currently use CUDA 11.3 to build CUDA 11.6 Windows wheels\n\n# Install NVIDIA drivers, see:\n# https"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.github/workflows/cuda/cu117-Linux-env.sh",
"chars": 211,
"preview": "#!/bin/bash\n\nCUDA_HOME=/usr/local/cuda-11.7\nLD_LIBRARY_PATH=${CUDA_HOME}/lib64:${LD_LIBRARY_PATH}\nPATH=${CUDA_HOME}/bin:"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.github/workflows/cuda/cu117-Linux.sh",
"chars": 718,
"preview": "#!/bin/bash\n\nOS=ubuntu1804\n\nwget -nv https://developer.download.nvidia.com/compute/cuda/repos/${OS}/x86_64/cuda-${OS}.pi"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.github/workflows/cuda/cu117-Windows-env.sh",
"chars": 268,
"preview": "#!/bin/bash\n\nCUDA_HOME=/c/Program\\ Files/NVIDIA\\ GPU\\ Computing\\ Toolkit/CUDA/v11.3\nPATH=${CUDA_HOME}/bin:$PATH\nPATH=/c/"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.github/workflows/cuda/cu117-Windows.sh",
"chars": 1170,
"preview": "#!/bin/bash\n\n# TODO We currently use CUDA 11.3 to build CUDA 11.7 Windows wheels\n\n# Install NVIDIA drivers, see:\n# https"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.github/workflows/publish.yml",
"chars": 1017,
"preview": "# This workflows will upload a Python Package using twine when a release is created\n# For more information see: https://"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.gitignore",
"chars": 1345,
"preview": "# Visual Studio Code configs.\n.vscode/\n\n# Byte-compiled / optimized / DLL files\n__pycache__/\n*.py[cod]\n*$py.class\n\n# C e"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.gitmodules",
"chars": 106,
"preview": "[submodule \"examples/pycolmap\"]\n\tpath = examples/pycolmap\n\turl = https://github.com/rmbrualla/pycolmap.git"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.pre-commit-config.yaml",
"chars": 529,
"preview": "repos:\n- repo: https://github.com/pre-commit/pre-commit-hooks\n rev: v2.3.0\n hooks:\n - id: end-of-file-fixer"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/.readthedocs.yaml",
"chars": 323,
"preview": "version: 2\n\nbuild:\n os: ubuntu-20.04\n tools:\n python: \"3.9\"\n\nsphinx:\n fail_on_warning: true\n configuration: docs/"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/CMakeLists.txt",
"chars": 1815,
"preview": "# cmake_minimum_required(VERSION 3.3)\n# project(nerfacc LANGUAGES CXX CUDA)\n\n# find_package(pybind11 REQUIRED)\n# find_pa"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/LICENSE",
"chars": 1067,
"preview": "MIT License\n\nCopyright (c) 2022 Ruilong Li\n\nPermission is hereby granted, free of charge, to any person obtaining a copy"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/MANIFEST.in",
"chars": 64,
"preview": "include nerfacc/cuda/csrc/include/*\ninclude nerfacc/cuda/csrc/*\n"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/README.md",
"chars": 8103,
"preview": "<p>\n <!-- pypi-strip -->\n <picture>\n <source media=\"(prefers-color-scheme: dark)\" srcset=\"https://user-images.githubu"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/docs/Makefile",
"chars": 637,
"preview": "# Minimal makefile for Sphinx documentation\n#\n\n# You can set these variables from the command line, and also\n# from the "
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/docs/requirements.txt",
"chars": 171,
"preview": "pytorch_sphinx_theme @ git+https://github.com/liruilong940607/pytorch_sphinx_theme.git#egg=pytorch_sphinx_theme\nsphinx=="
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/docs/source/_static/css/readthedocs.css",
"chars": 169,
"preview": ".header-logo {\n background-image: url(\"../images/logo4x.png\");\n background-size: 156px 35px;\n height: 35px;\n "
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/docs/source/apis/generated/nerfacc.accumulate_along_rays.rst",
"chars": 133,
"preview": "nerfacc.accumulate\\_along\\_rays\n===============================\n\n.. currentmodule:: nerfacc\n\n.. autofunction:: accumula"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/docs/source/apis/generated/nerfacc.pack_data.rst",
"chars": 95,
"preview": "nerfacc.pack\\_data\n==================\n\n.. currentmodule:: nerfacc\n\n.. autofunction:: pack_data"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/docs/source/apis/generated/nerfacc.ray_aabb_intersect.rst",
"chars": 124,
"preview": "nerfacc.ray\\_aabb\\_intersect\n============================\n\n.. currentmodule:: nerfacc\n\n.. autofunction:: ray_aabb_inter"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/docs/source/apis/generated/nerfacc.ray_resampling.rst",
"chars": 110,
"preview": "nerfacc.ray\\_resampling\n=======================\n\n.. currentmodule:: nerfacc\n\n.. autofunction:: ray_resampling"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/docs/source/apis/generated/nerfacc.render_transmittance_from_alpha.rst",
"chars": 165,
"preview": "nerfacc.render\\_transmittance\\_from\\_alpha\n==========================================\n\n.. currentmodule:: nerfacc\n\n.. a"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/docs/source/apis/generated/nerfacc.render_transmittance_from_density.rst",
"chars": 171,
"preview": "nerfacc.render\\_transmittance\\_from\\_density\n============================================\n\n.. currentmodule:: nerfacc\n\n"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/docs/source/apis/generated/nerfacc.render_visibility.rst",
"chars": 119,
"preview": "nerfacc.render\\_visibility\n==========================\n\n.. currentmodule:: nerfacc\n\n.. autofunction:: render_visibility"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/docs/source/apis/generated/nerfacc.render_weight_from_alpha.rst",
"chars": 144,
"preview": "nerfacc.render\\_weight\\_from\\_alpha\n===================================\n\n.. currentmodule:: nerfacc\n\n.. autofunction:: "
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/docs/source/apis/generated/nerfacc.render_weight_from_density.rst",
"chars": 150,
"preview": "nerfacc.render\\_weight\\_from\\_density\n=====================================\n\n.. currentmodule:: nerfacc\n\n.. autofunctio"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/docs/source/apis/generated/nerfacc.unpack_data.rst",
"chars": 101,
"preview": "nerfacc.unpack\\_data\n====================\n\n.. currentmodule:: nerfacc\n\n.. autofunction:: unpack_data"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/docs/source/apis/generated/nerfacc.unpack_info.rst",
"chars": 101,
"preview": "nerfacc.unpack\\_info\n====================\n\n.. currentmodule:: nerfacc\n\n.. autofunction:: unpack_info"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/docs/source/apis/grid.rst",
"chars": 228,
"preview": ".. _`Occupancy Grid`:\n\nOccupancy Grid\n===================================\n\n.. currentmodule:: nerfacc\n\n.. autoclass:: Co"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/docs/source/apis/rendering.rst",
"chars": 1096,
"preview": "Volumetric Rendering\n===================================\n\nIn `nerfacc`, the volumetric rendering pipeline is broken down"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/docs/source/apis/utils.rst",
"chars": 395,
"preview": "Utils\n===================================\n\n.. currentmodule:: nerfacc\n\n.. autosummary::\n :nosignatures:\n :toctree: g"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/docs/source/conf.py",
"chars": 2642,
"preview": "import pytorch_sphinx_theme\n\n__version__ = None\nexec(open(\"../../nerfacc/version.py\", \"r\").read())\n\n# -- Project informa"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/docs/source/examples/dnerf.rst",
"chars": 2173,
"preview": "Dynamic Scene\n====================\n\nSee code `examples/train_mlp_dnerf.py` at our `github repository`_ for details.\n\nBen"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/docs/source/examples/ngp.rst",
"chars": 2518,
"preview": ".. _`Instant-NGP Example`:\n\nInstant-NGP\n====================\n\nSee code `examples/train_ngp_nerf.py` at our `github repos"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/docs/source/examples/unbounded.rst",
"chars": 2527,
"preview": "Unbounded Scene\n====================\n\nSee code `examples/train_ngp_nerf.py` at our `github repository`_ for details.\n\nBe"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/docs/source/examples/vanilla.rst",
"chars": 2063,
"preview": "Vanilla Nerf \n====================\n\nSee code `examples/train_mlp_nerf.py` at our `github repository`_ for details.\n\nBenc"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/docs/source/index.rst",
"chars": 5494,
"preview": "NerfAcc Documentation\n===================================\n\nNerfAcc is a PyTorch Nerf acceleration toolbox for both train"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/examples/datasets/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/examples/datasets/dnerf_synthetic.py",
"chars": 7502,
"preview": "\"\"\"\nCopyright (c) 2022 Ruilong Li, UC Berkeley.\n\"\"\"\n\nimport json\nimport os\n\nimport imageio.v2 as imageio\nimport numpy as"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/examples/datasets/nerf_360_v2.py",
"chars": 9735,
"preview": "\"\"\"\nCopyright (c) 2022 Ruilong Li, UC Berkeley.\n\"\"\"\n\nimport collections\nimport os\nimport sys\n\nimport imageio\nimport nump"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/examples/datasets/nerf_synthetic.py",
"chars": 7540,
"preview": "\"\"\"\nCopyright (c) 2022 Ruilong Li, UC Berkeley.\n\"\"\"\n\nimport collections\nimport json\nimport os\n\nimport imageio.v2 as imag"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/examples/datasets/utils.py",
"chars": 312,
"preview": "\"\"\"\nCopyright (c) 2022 Ruilong Li, UC Berkeley.\n\"\"\"\n\nimport collections\n\nRays = collections.namedtuple(\"Rays\", (\"origins"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/examples/radiance_fields/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/examples/radiance_fields/mlp.py",
"chars": 9885,
"preview": "\"\"\"\nCopyright (c) 2022 Ruilong Li, UC Berkeley.\n\"\"\"\n\nimport functools\nimport math\nfrom typing import Callable, Optional\n"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/examples/radiance_fields/ngp.py",
"chars": 6392,
"preview": "\"\"\"\nCopyright (c) 2022 Ruilong Li, UC Berkeley.\n\"\"\"\n\nfrom typing import Callable, List, Union\n\nimport torch\nfrom torch.a"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/examples/requirements.txt",
"chars": 110,
"preview": "git+https://github.com/NVlabs/tiny-cuda-nn/#subdirectory=bindings/torch\nopencv-python\nimageio\nnumpy\ntqdm\nscipy"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/examples/train_mlp_dnerf.py",
"chars": 8092,
"preview": "\"\"\"\nCopyright (c) 2022 Ruilong Li, UC Berkeley.\n\"\"\"\n\nimport argparse\nimport math\nimport os\nimport time\n\nimport imageio\ni"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/examples/train_mlp_nerf.py",
"chars": 8621,
"preview": "\"\"\"\nCopyright (c) 2022 Ruilong Li, UC Berkeley.\n\"\"\"\n\nimport argparse\nimport math\nimport os\nimport time\n\nimport imageio\ni"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/examples/train_ngp_nerf.py",
"chars": 10485,
"preview": "\"\"\"\nCopyright (c) 2022 Ruilong Li, UC Berkeley.\n\"\"\"\n\nimport argparse\nimport math\nimport os\nimport time\n\nimport imageio\ni"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/examples/utils.py",
"chars": 3724,
"preview": "\"\"\"\nCopyright (c) 2022 Ruilong Li, UC Berkeley.\n\"\"\"\n\nimport random\nfrom typing import Optional\n\nimport numpy as np\nimpor"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/nerfacc/__init__.py",
"chars": 1870,
"preview": "\"\"\"\nCopyright (c) 2022 Ruilong Li, UC Berkeley.\n\"\"\"\nimport warnings\n\nfrom .cdf import ray_resampling\nfrom .contraction i"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/nerfacc/cdf.py",
"chars": 1412,
"preview": "\"\"\"\nCopyright (c) 2022 Ruilong Li, UC Berkeley.\n\"\"\"\n\nfrom typing import Tuple\n\nfrom torch import Tensor\n\nimport nerfacc."
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/nerfacc/contraction.py",
"chars": 2911,
"preview": "\"\"\"\nCopyright (c) 2022 Ruilong Li, UC Berkeley.\n\"\"\"\n\nfrom enum import Enum\n\nimport torch\n\nimport nerfacc.cuda as _C\n\n\ncl"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/nerfacc/cuda/__init__.py",
"chars": 3319,
"preview": "\"\"\"\nCopyright (c) 2022 Ruilong Li, UC Berkeley.\n\"\"\"\n\nfrom typing import Any, Callable\n\n\ndef _make_lazy_cuda_func(name: s"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/nerfacc/cuda/_backend.py",
"chars": 2768,
"preview": "\"\"\"\nCopyright (c) 2022 Ruilong Li, UC Berkeley.\n\"\"\"\n\nimport glob\nimport json\nimport os\nimport shutil\nfrom subprocess imp"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/nerfacc/cuda/csrc/cdf.cu",
"chars": 7026,
"preview": "/*\n * Copyright (c) 2022 Ruilong Li, UC Berkeley.\n */\n\n#include \"include/helpers_cuda.h\"\n\ntemplate <typename scalar_t>\n_"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/nerfacc/cuda/csrc/contraction.cu",
"chars": 3026,
"preview": "/*\n * Copyright (c) 2022 Ruilong Li, UC Berkeley.\n */\n\n#include \"include/helpers_cuda.h\"\n#include \"include/helpers_math."
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/nerfacc/cuda/csrc/include/helpers_contraction.h",
"chars": 3725,
"preview": "/*\n * Copyright (c) 2022 Ruilong Li, UC Berkeley.\n */\n\n#pragma once\n\n#include \"helpers_math.h\"\n\nenum ContractionType\n{\n "
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/nerfacc/cuda/csrc/include/helpers_cuda.h",
"chars": 2001,
"preview": "/*\n * Copyright (c) 2022 Ruilong Li, UC Berkeley.\n */\n\n#pragma once\n\n#include <torch/extension.h>\n#include <c10/cuda/CUD"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/nerfacc/cuda/csrc/include/helpers_math.h",
"chars": 38908,
"preview": "/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.\n * Modified by Ruilong Li, 2022\n *\n * Redistribution and"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/nerfacc/cuda/csrc/intersection.cu",
"chars": 3623,
"preview": "/*\n * Copyright (c) 2022 Ruilong Li, UC Berkeley.\n */\n\n#include \"include/helpers_cuda.h\"\n\ntemplate <typename scalar_t>\ni"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/nerfacc/cuda/csrc/pack.cu",
"chars": 4297,
"preview": "/*\n * Copyright (c) 2022 Ruilong Li, UC Berkeley.\n */\n\n#include \"include/helpers_cuda.h\"\n\n__global__ void unpack_info_ke"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/nerfacc/cuda/csrc/pybind.cu",
"chars": 7419,
"preview": "/*\n * Copyright (c) 2022 Ruilong Li, UC Berkeley.\n */\n\n#include \"include/helpers_cuda.h\"\n#include \"include/helpers_math."
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/nerfacc/cuda/csrc/ray_marching.cu",
"chars": 10929,
"preview": "/*\n * Copyright (c) 2022 Ruilong Li, UC Berkeley.\n */\n\n#include \"include/helpers_cuda.h\"\n#include \"include/helpers_math."
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/nerfacc/cuda/csrc/render_transmittance.cu",
"chars": 13055,
"preview": "/*\n * Copyright (c) 2022 Ruilong Li, UC Berkeley.\n */\n\n#include \"include/helpers_cuda.h\"\n\n__global__ void transmittance_"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/nerfacc/cuda/csrc/render_transmittance_cub.cu",
"chars": 6297,
"preview": "/*\n * Copyright (c) 2022 Ruilong Li, UC Berkeley.\n */\n// CUB is supported in CUDA >= 11.0\n// ExclusiveScanByKey is suppo"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/nerfacc/cuda/csrc/render_weight.cu",
"chars": 24791,
"preview": "/*\n * Copyright (c) 2022 Ruilong Li, UC Berkeley.\n */\n\n#include \"include/helpers_cuda.h\"\n\n__global__ void weight_from_si"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/nerfacc/grid.py",
"chars": 11066,
"preview": "\"\"\"\nCopyright (c) 2022 Ruilong Li @ UC Berkeley\n\"\"\"\n\nfrom typing import Callable, List, Union\n\nimport torch\nimport torch"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/nerfacc/intersection.py",
"chars": 1457,
"preview": "\"\"\"\nCopyright (c) 2022 Ruilong Li, UC Berkeley.\n\"\"\"\n\nfrom typing import Tuple\n\nimport torch\nfrom torch import Tensor\n\nim"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/nerfacc/losses.py",
"chars": 1058,
"preview": "from torch import Tensor\n\nfrom .pack import unpack_data\n\n\ndef distortion(\n packed_info: Tensor, weights: Tensor, t_st"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/nerfacc/pack.py",
"chars": 6999,
"preview": "\"\"\"\nCopyright (c) 2022 Ruilong Li, UC Berkeley.\n\"\"\"\nfrom typing import Optional, Tuple\n\nimport torch\nfrom torch import T"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/nerfacc/ray_marching.py",
"chars": 9231,
"preview": "from typing import Callable, Optional, Tuple\n\nimport torch\n\nimport nerfacc.cuda as _C\n\nfrom .contraction import Contract"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/nerfacc/sampling.py",
"chars": 6540,
"preview": "import math\nfrom typing import Callable, Optional, Tuple, Union, overload\n\nimport torch\n\nimport nerfacc.cuda as _C\n\nfrom"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/nerfacc/version.py",
"chars": 75,
"preview": "\"\"\"\nCopyright (c) 2022 Ruilong Li, UC Berkeley.\n\"\"\"\n\n__version__ = \"0.3.5\"\n"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/nerfacc/vol_rendering.py",
"chars": 43094,
"preview": "\"\"\"\nCopyright (c) 2022 Ruilong Li, UC Berkeley.\n\"\"\"\n\nfrom typing import Callable, Optional, Tuple\n\nimport torch\nfrom tor"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/scripts/run_aws_listing.py",
"chars": 1546,
"preview": "import argparse\nimport os\n\nfrom boto3 import client\n\nparser = argparse.ArgumentParser()\nparser.add_argument(\"--access_ke"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/scripts/run_dev_checks.py",
"chars": 2568,
"preview": "#!/usr/bin/env python\n\"\"\"Simple yaml debugger\"\"\"\nimport subprocess\n\nimport yaml\nfrom rich.console import Console\nfrom ri"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/scripts/run_profiler.py",
"chars": 3373,
"preview": "from typing import Callable\n\nimport torch\nimport tqdm\n\nimport nerfacc\n\n# timing\n# https://github.com/pytorch/pytorch/com"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/setup.cfg",
"chars": 101,
"preview": "[isort]\nmulti_line_output = 3\nline_length = 80\ninclude_trailing_comma = true\nskip=./examples/pycolmap"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/setup.py",
"chars": 3751,
"preview": "import glob\nimport os\nimport os.path as osp\nimport platform\nimport sys\n\nfrom setuptools import find_packages, setup\n\n__v"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/tests/test_contraction.py",
"chars": 2761,
"preview": "import pytest\nimport torch\n\nimport nerfacc.cuda as _C\nfrom nerfacc import ContractionType, contract, contract_inv\n\ndevic"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/tests/test_grid.py",
"chars": 1139,
"preview": "import pytest\nimport torch\n\nfrom nerfacc import OccupancyGrid\n\ndevice = \"cuda:0\"\n\n\n@pytest.mark.skipif(not torch.cuda.is"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/tests/test_intersection.py",
"chars": 694,
"preview": "import pytest\nimport torch\n\nfrom nerfacc import ray_aabb_intersect\n\ndevice = \"cuda:0\"\nbatch_size = 32\neps = 1e-6\n\n\n@pyte"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/tests/test_loss.py",
"chars": 865,
"preview": "import pytest\nimport torch\n\nfrom nerfacc import pack_info, ray_marching\nfrom nerfacc.losses import distortion\n\ndevice = "
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/tests/test_pack.py",
"chars": 1345,
"preview": "import pytest\nimport torch\n\nfrom nerfacc import pack_data, pack_info, unpack_data, unpack_info\n\ndevice = \"cuda:0\"\nbatch_"
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/tests/test_ray_marching.py",
"chars": 1493,
"preview": "import pytest\nimport torch\n\nfrom nerfacc import OccupancyGrid, ray_marching, unpack_info\n\ndevice = \"cuda:0\"\nbatch_size ="
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/tests/test_rendering.py",
"chars": 7346,
"preview": "import pytest\nimport torch\n\nfrom nerfacc import (\n accumulate_along_rays,\n render_transmittance_from_density,\n "
},
{
"path": "third_parties/nerfacc-0.3.5/nerfacc-0.3.5/tests/test_resampling.py",
"chars": 917,
"preview": "import pytest\nimport torch\n\nfrom nerfacc import pack_info, ray_marching, ray_resampling\n\ndevice = \"cuda:0\"\nbatch_size = "
},
{
"path": "utilities/utils.py",
"chars": 2597,
"preview": "import numpy as np\nimport cv2\nfrom PIL import Image, ImageChops\nimport os\nimport time\nimport torch\nfrom PIL import Image"
}
]
About this extraction
This page contains the full source code of the CyberAgentAILab/SuperNormal GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 135 files (479.3 KB), approximately 138.4k tokens, and a symbol index with 395 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.