diff --git a/Dockerfile b/Dockerfile
new file mode 100644
index 0000000000000000000000000000000000000000..77b1605b2d90f5d17493bed5589825ede7c4a220
--- /dev/null
+++ b/Dockerfile
@@ -0,0 +1,72 @@
+FROM nvidia/cuda:12.1.1-devel-ubuntu22.04 AS builder
+
+WORKDIR /app
+
+COPY . /app/
+
+ENV CUDA_HOME=/usr/local/cuda-12.1
+ENV LD_LIBRARY_PATH=$CUDA_HOME/lib64:$LD_LIBRARY_PATH
+ENV PATH=$CUDA_HOME/bin:$PATH
+
+ENV TORCH_CUDA_ARCH_LIST="7.0 7.5 8.0 8.6 8.9 9.0+PTX"
+
+RUN apt-get update && \
+DEBIAN_FRONTEND=noninteractive apt-get install -y \
+build-essential wget curl nano ninja-build unzip libgl-dev ffmpeg && \
+apt-get clean && \
+rm -rf /var/lib/apt/lists/*
+
+ENV CONDA_DIR=/opt/conda
+
+RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh && \
+/bin/bash ~/miniconda.sh -b -p /opt/conda && \
+rm ~/miniconda.sh
+
+ENV PATH=$CONDA_DIR/bin:$PATH
+
+RUN conda update -n base conda -y && \
+conda install -n base conda-libmamba-solver -y && \
+conda config --set solver libmamba
+
+RUN conda create -y -n edgs python=3.10 pip
+
+SHELL ["conda", "run", "-n", "edgs", "/bin/bash", "-c"]
+
+
+RUN conda install pytorch torchvision torchaudio pytorch-cuda=12.1 -c pytorch -c nvidia -y
+
+RUN pip install -e submodules/gaussian-splatting/submodules/diff-gaussian-rasterization --no-build-isolation && \
+ pip install -e submodules/gaussian-splatting/submodules/simple-knn --no-build-isolation
+
+RUN pip install pycolmap
+
+RUN pip install wandb hydra-core tqdm torchmetrics lpips matplotlib rich plyfile imageio imageio-ffmpeg && \
+ conda install numpy=1.26.4 -y -c conda-forge --override-channels
+
+RUN pip install -e submodules/RoMa
+
+RUN pip install plotly scikit-learn moviepy==2.1.1 ffmpeg fastapi[standard]
+
+# Imagen final
+
+FROM nvidia/cuda:12.1.1-runtime-ubuntu22.04 AS final
+
+WORKDIR /app
+
+RUN apt-get update && \
+ DEBIAN_FRONTEND=noninteractive apt-get install -y \
+ libgl1-mesa-glx libsm6 libxext6 ffmpeg && \
+ apt-get clean && \
+ rm -rf /var/lib/apt/lists/*
+
+COPY --from=builder /opt/conda /opt/conda
+
+COPY --from=builder /app /app
+
+ENV PATH="/opt/conda/bin:/opt/conda/envs/edgs/bin:$PATH"
+
+ENV CUDA_HOME=/usr/local/cuda-12.1
+ENV LD_LIBRARY_PATH=$CUDA_HOME/lib64:$LD_LIBRARY_PATH
+
+CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "7860"]
+
diff --git a/LICENSE.txt b/LICENSE.txt
new file mode 100644
index 0000000000000000000000000000000000000000..d3b17cbb35125414350ef6cebe4ca79b2960b28e
--- /dev/null
+++ b/LICENSE.txt
@@ -0,0 +1,12 @@
+Copyright 2025, Dmytro Kotovenko, Olga Grebenkova, Björn Ommer
+Redistribution and use in source and binary forms, with or without modification, are permitted for non-commercial academic research and/or non-commercial personal use only provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+
+Any use of this software beyond the above specified conditions requires a separate license. Please contact the copyright holders to discuss license terms.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/README.md b/README.md
index 60a1c896ecf4c703b969e46d53a3c500d5b9e358..8c404272cf3a41c558d9c375edbcfb3b621a63c9 100644
--- a/README.md
+++ b/README.md
@@ -1,10 +1,234 @@
+
EDGS: Eliminating Densification for Efficient Convergence of 3DGS
+
+
+ Dmytro Kotovenko * ·
+ Olga Grebenkova * ·
+ Björn Ommer
+
+
+CompVis @ LMU Munich · Munich Center for Machine Learning (MCML)
+* equal contribution
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+3DGS initializes with a sparse set of Gaussians and progressively adds more in under-reconstructed regions. In contrast, EDGS starts with
+a dense initialization from triangulated 2D correspondences across training image pairs,
+requiring only minimal refinement. This leads to faster convergence and higher rendering quality . Our method reaches the original 3DGS LPIPS score in just 25% of the training time and uses only 60% of the splats .
+Renderings become nearly indistinguishable from ground truth after only 3,000 steps — without any densification .
+
+
+3D scene reconstruction using our method in 11 seconds.
+
+
+
+
+
+
+## 📚 Table of Contents
+- [🚀 Quickstart](#sec-quickstart)
+- [🛠️ Installation](#sec-install)
+- [📦 Data](#sec-data)
+
+- [🏋️ Training](#sec-training)
+- [🏗️ Reusing Our Model](#sec-reuse)
+- [📄 Citation](#sec-citation)
+
+
+## 🚀 Quickstart
+The fastest way to try our model is through the [Hugging Face demo](https://huggingface.co/spaces/magistrkoljan/EDGS), which lets you upload images or a video and interactively rotate the resulting 3D scene. For broad accessibility, we currently support only **forward-facing scenes**.
+#### Steps:
+1. Upload a list of photos or a single video.
+2. Click **📸 Preprocess Input** to estimate 3D positions using COLMAP.
+3. Click **🚀 Start Reconstruction** to run the model.
+
+You can also **explore the reconstructed scene in 3D** directly in the browser.
+
+> ⚡ Runtime: EDGS typically takes just **10–20 seconds**, plus **5–10 seconds** for COLMAP processing. Additional time may be needed to save outputs (model, video, 3D preview).
+
+You can also run the same app locally on your machine with command:
+```CUDA_VISIBLE_DEVICES=0 python gradio_demo.py --port 7862 --no_share```
+Without `--no_share` flag you will get the adress for gradio app that you can share with the others allowing others to process their data on your server.
+
+Alternatively, check our [Colab notebook](https://colab.research.google.com/github/CompVis/EDGS/blob/main/notebooks/fit_model_to_scene_full.ipynb).
+
+###
+
+
+
+
+## 🛠️ Installation
+
+You can either run `install.sh` or manually install using the following:
+
+```bash
+git clone git@github.com:CompVis/EDGS.git --recursive
+cd EDGS
+git submodule update --init --recursive
+
+conda create -y -n edgs python=3.10 pip
+conda activate edgs
+
+# Set up path to your CUDA. In our experience similar versions like 12.2 also work well
+export CUDA_HOME=/usr/local/cuda-12.1
+export LD_LIBRARY_PATH=$CUDA_HOME/lib64:$LD_LIBRARY_PATH
+export PATH=$CUDA_HOME/bin:$PATH
+
+conda install pytorch torchvision torchaudio pytorch-cuda=12.1 -c pytorch -c nvidia -y
+conda install nvidia/label/cuda-12.1.0::cuda-toolkit -y
+
+pip install -e submodules/gaussian-splatting/submodules/diff-gaussian-rasterization
+pip install -e submodules/gaussian-splatting/submodules/simple-knn
+
+# For COLMAP and pycolmap
+# Optionally install original colmap but probably pycolmap suffices
+# conda install conda-forge/label/colmap_dev::colmap
+pip install pycolmap
+
+
+pip install wandb hydra-core tqdm torchmetrics lpips matplotlib rich plyfile imageio imageio-ffmpeg
+conda install numpy=1.26.4 -y -c conda-forge --override-channels
+
+pip install -e submodules/RoMa
+conda install anaconda::jupyter --yes
+
+# Stuff necessary for gradio and visualizations
+pip install gradio
+pip install plotly scikit-learn moviepy==2.1.1 ffmpeg
+pip install open3d
+```
+
+
+## 📦 Data
+
+We evaluated on the following datasets:
+
+- **MipNeRF360** — download [here](https://jonbarron.info/mipnerf360/). Unzip "Dataset Pt. 1" and "Dataset Pt. 2", then merge scenes.
+- **Tanks & Temples + Deep Blending** — from the [original 3DGS repo](https://repo-sam.inria.fr/fungraph/3d-gaussian-splatting/datasets/input/tandt_db.zip).
+
+### Using Your Own Dataset
+
+You can use the same data format as the [3DGS project](https://github.com/graphdeco-inria/gaussian-splatting?tab=readme-ov-file#processing-your-own-scenes). Please follow their guide to prepare your scene.
+
+Expected folder structure:
+```
+scene_folder
+|---images
+| |---
+| |---
+| |---...
+|---sparse
+ |---0
+ |---cameras.bin
+ |---images.bin
+ |---points3D.bin
+```
+
+Nerf synthetic format is also acceptable.
+
+You can also use functions provided in our code to convert a collection of images or a sinlge video into a desired format. However, this may requre tweaking and processing time can be large for large collection of images with little overlap.
+
+
+## 🏋️ Training
+
+
+To optimize on a single scene in COLMAP format use this code.
+```bash
+python train.py \
+ train.gs_epochs=30000 \
+ train.no_densify=True \
+ gs.dataset.source_path= \
+ gs.dataset.model_path= \
+ init_wC.matches_per_ref=20000 \
+ init_wC.nns_per_ref=3 \
+ init_wC.num_refs=180
+```
+
+Command Line Arguments for train.py
+
+ * `train.gs_epochs`
+ Number of training iterations (steps) for Gaussian Splatting.
+ * `train.no_densify`
+ Disables densification. True by default.
+ * `gs.dataset.source_path`
+ Path to your input dataset directory. This should follow the same format as the original 3DGS dataset structure.
+ * `gs.dataset.model_path`
+ Output directory where the trained model, logs, and renderings will be saved.
+ * `init_wC.matches_per_ref`
+ Number of 2D feature correspondences to extract per reference view for initialization. More matches leads to more gaussians.
+ * `init_wC.nns_per_ref`
+ Number of nearest neighbor images used per reference during matching.
+ * `init_wC.num_refs`
+ Total number of reference views sampled.
+ * `wandb.mode`
+ Specifies how Weights & Biases (W&B) logging is handled.
+
+ - Default: `"disabled"`
+ - Options:
+ - `"online"` — log to the W&B server in real-time
+ - `"offline"` — save logs locally to sync later
+ - `"disabled"` — turn off W&B logging entirely
+
+ If you want to enable W&B logging, make sure to also configure:
+
+ - `wandb.project` — the name of your W&B project
+ - `wandb.entity` — your W&B username or team name
+
+Example override:
+```bash
+wandb.mode=online wandb.project=EDGS wandb.entity=your_username train.gs_epochs=15_000 init_wC.matches_per_ref=15_000
+```
+
+
+
+To run full evaluation on all datasets:
+
+```bash
+python full_eval.py -m360 -tat -db
+```
+
+## 🏗️ Reusing Our Model
+
+Our model is essentially a better **initialization module** for Gaussian Splatting. You can integrate it into your pipeline by calling:
+
+```python
+source.corr_init.init_gaussians_with_corr(...)
+```
+### Input arguments:
+- A GaussianModel and Scene instance
+- A configuration namespace `cfg.init_wC` to specify parameters like the number of matches, neighbors, and reference views
+- A RoMA model (automatically instantiated if not provided)
+
+
+
+
+## 📄 Citation
+```bibtex
+@misc{kotovenko2025edgseliminatingdensificationefficient,
+ title={EDGS: Eliminating Densification for Efficient Convergence of 3DGS},
+ author={Dmytro Kotovenko and Olga Grebenkova and Björn Ommer},
+ year={2025},
+ eprint={2504.13204},
+ archivePrefix={arXiv},
+ primaryClass={cs.GR},
+ url={https://arxiv.org/abs/2504.13204},
+}
+```
---
-title: Gs Final
-emoji: 🐢
-colorFrom: gray
-colorTo: green
-sdk: docker
-pinned: false
----
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# TODO:
+- [ ] Code for training and processing forward-facing scenes.
+- [ ] More data examples
+
+
+
diff --git a/configs/gs/base.yaml b/configs/gs/base.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..5a74ddca24dc480d9332aea2c67ea7069ddb0307
--- /dev/null
+++ b/configs/gs/base.yaml
@@ -0,0 +1,51 @@
+_target_: source.networks.Warper3DGS
+
+verbose: True
+viewpoint_stack: !!null
+sh_degree: 3
+
+opt:
+ iterations: 30000
+ position_lr_init: 0.00016
+ position_lr_final: 1.6e-06
+ position_lr_delay_mult: 0.01
+ position_lr_max_steps: 30000
+ feature_lr: 0.0025
+ opacity_lr: 0.025
+ scaling_lr: 0.005
+ rotation_lr: 0.001
+ percent_dense: 0.01
+ lambda_dssim: 0.2
+ densification_interval: 100
+ opacity_reset_interval: 30000
+ densify_from_iter: 500
+ densify_until_iter: 15000
+ densify_grad_threshold: 0.0002
+ random_background: false
+ save_iterations: [3000, 7000, 15000, 30000]
+ batch_size: 64
+ exposure_lr_init: 0.01
+ exposure_lr_final: 0.0001
+ exposure_lr_delay_steps: 0
+ exposure_lr_delay_mult: 0.0
+
+ TRAIN_CAM_IDX_TO_LOG: 50
+ TEST_CAM_IDX_TO_LOG: 10
+
+pipe:
+ convert_SHs_python: False
+ compute_cov3D_python: False
+ debug: False
+ antialiasing: False
+
+dataset:
+ densify_until_iter: 15000
+ source_path: '' #path to dataset
+ model_path: '' #path to logs
+ images: images
+ resolution: -1
+ white_background: false
+ data_device: cuda
+ eval: false
+ depths: ""
+ train_test_exp: False
\ No newline at end of file
diff --git a/configs/train.yaml b/configs/train.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..e40ec5be88301016a1c7b4da5d4f3b383b050909
--- /dev/null
+++ b/configs/train.yaml
@@ -0,0 +1,38 @@
+defaults:
+ - gs: base
+ - _self_
+
+seed: 228
+
+wandb:
+ mode: "online" # "disabled" for no logging
+ entity: "3dcorrespondence"
+ project: "Adv3DGS"
+ group: null
+ name: null
+ tag: "debug"
+
+train:
+ gs_epochs: 0 # number of 3dgs iterations
+ reduce_opacity: True
+ no_densify: False # if True, the model will not be densified
+ max_lr: True
+
+load:
+ gs: null #path to 3dgs checkpoint
+ gs_step: null #number of iterations, e.g. 7000
+
+device: "cuda:0"
+verbose: true
+
+init_wC:
+ use: True # use EDGS
+ matches_per_ref: 15_000 # number of matches per reference
+ num_refs: 180 # number of reference images
+ nns_per_ref: 3 # number of nearest neighbors per reference
+ scaling_factor: 0.001
+ proj_err_tolerance: 0.01
+ roma_model: "outdoors" # you can change this to "indoors" or "outdoors"
+ add_SfM_init : False
+
+
diff --git a/extra/archive/rasterizer_impl.h b/extra/archive/rasterizer_impl.h
new file mode 100644
index 0000000000000000000000000000000000000000..b395fa01effd3e315056218d095f3b96d1f1f094
--- /dev/null
+++ b/extra/archive/rasterizer_impl.h
@@ -0,0 +1,75 @@
+/*
+ * Copyright (C) 2023, Inria
+ * GRAPHDECO research group, https://team.inria.fr/graphdeco
+ * All rights reserved.
+ *
+ * This software is free for non-commercial, research and evaluation use
+ * under the terms of the LICENSE.md file.
+ *
+ * For inquiries contact george.drettakis@inria.fr
+ */
+
+#pragma once
+
+#include
+#include
+#include
+#include "rasterizer.h"
+#include
+
+namespace CudaRasterizer
+{
+ template
+ static void obtain(char*& chunk, T*& ptr, std::size_t count, std::size_t alignment)
+ {
+ std::size_t offset = (reinterpret_cast(chunk) + alignment - 1) & ~(alignment - 1);
+ ptr = reinterpret_cast(offset);
+ chunk = reinterpret_cast(ptr + count);
+ }
+
+ struct GeometryState
+ {
+ size_t scan_size;
+ float* depths;
+ char* scanning_space;
+ bool* clamped;
+ int* internal_radii;
+ float2* means2D;
+ float* cov3D;
+ float4* conic_opacity;
+ float* rgb;
+ uint32_t* point_offsets;
+ uint32_t* tiles_touched;
+
+ static GeometryState fromChunk(char*& chunk, size_t P);
+ };
+
+ struct ImageState
+ {
+ uint2* ranges;
+ uint32_t* n_contrib;
+ float* accum_alpha;
+
+ static ImageState fromChunk(char*& chunk, size_t N);
+ };
+
+ struct BinningState
+ {
+ size_t sorting_size;
+ uint64_t* point_list_keys_unsorted;
+ uint64_t* point_list_keys;
+ uint32_t* point_list_unsorted;
+ uint32_t* point_list;
+ char* list_sorting_space;
+
+ static BinningState fromChunk(char*& chunk, size_t P);
+ };
+
+ template
+ size_t required(size_t P)
+ {
+ char* size = nullptr;
+ T::fromChunk(size, P);
+ return ((size_t)size) + 128;
+ }
+};
diff --git a/extra/archive/simple-knn.patch.txt b/extra/archive/simple-knn.patch.txt
new file mode 100644
index 0000000000000000000000000000000000000000..3b69aa4fb434fdeb2af0b5dee91a371cd41241d8
--- /dev/null
+++ b/extra/archive/simple-knn.patch.txt
@@ -0,0 +1,13 @@
+diff --git a/simple_knn.cu b/simple_knn.cu
+index e72e4c9..b2deb1b 100644
+--- a/simple_knn.cu
++++ b/simple_knn.cu
+@@ -11,6 +11,8 @@
+
+ #define BOX_SIZE 1024
+
++#include
++
+ #include "cuda_runtime.h"
+ #include "device_launch_parameters.h"
+ #include "simple_knn.h"
diff --git a/full_eval.py b/full_eval.py
new file mode 100644
index 0000000000000000000000000000000000000000..53425664aed8cef786f71ccf67c49a47633025fb
--- /dev/null
+++ b/full_eval.py
@@ -0,0 +1,102 @@
+#
+# Copyright (C) 2023, Inria
+# GRAPHDECO research group, https://team.inria.fr/graphdeco
+# All rights reserved.
+#
+# This software is free for non-commercial, research and evaluation use
+# under the terms of the LICENSE.md file.
+#
+# For inquiries contact george.drettakis@inria.fr
+#
+
+import os
+from argparse import ArgumentParser
+
+mipnerf360_outdoor_scenes = ["bicycle", "flowers", "garden", "stump", "treehill"]
+mipnerf360_indoor_scenes = ["room", "counter", "kitchen", "bonsai"]
+tanks_and_temples_scenes = ["truck", "train"]
+deep_blending_scenes = ["drjohnson", "playroom"]
+
+parser = ArgumentParser(description="Full evaluation script parameters")
+parser.add_argument("--skip_training", action="store_true")
+parser.add_argument("--skip_rendering", action="store_true")
+parser.add_argument("--skip_metrics", action="store_true")
+parser.add_argument("--output_path", default="./eval")
+args, _ = parser.parse_known_args()
+
+all_scenes = []
+all_scenes.extend(mipnerf360_outdoor_scenes)
+all_scenes.extend(mipnerf360_indoor_scenes)
+all_scenes.extend(tanks_and_temples_scenes)
+all_scenes.extend(deep_blending_scenes)
+
+if not args.skip_training or not args.skip_rendering:
+ parser.add_argument('--mipnerf360', "-m360", required=True, type=str)
+ parser.add_argument("--tanksandtemples", "-tat", required=True, type=str)
+ parser.add_argument("--deepblending", "-db", required=True, type=str)
+ args = parser.parse_args()
+
+if not args.skip_training:
+ name = "EDGS_"
+ common_args = " --quiet --eval --test_iterations -1 "
+ for scene in mipnerf360_outdoor_scenes:
+ source = args.mipnerf360 + "/" + scene
+ experiment = name + scene
+ os.system(f"python train.py verbose=True gs.dataset.source_path={source} gs.dataset.model_path={args.output_path}/mipnerf/{scene} wandb.name={experiment} init_wC.use=True train.gs_epochs=30000 init_wC.matches_per_ref=25_000 init_wC.nns_per_ref=3 gs.dataset.images=images_4 init_wC.num_refs=180 train.no_densify=True")
+ for scene in mipnerf360_indoor_scenes:
+ source = args.mipnerf360 + "/" + scene
+ experiment = name + scene
+ os.system(f"python train.py verbose=True gs.dataset.source_path={source} gs.dataset.model_path={args.output_path}/mipnerf/{scene} wandb.name={experiment} init_wC.use=True train.gs_epochs=30000 init_wC.matches_per_ref=25_000 init_wC.nns_per_ref=3 gs.dataset.images=images_2 init_wC.num_refs=180 train.no_densify=True")
+ for scene in tanks_and_temples_scenes:
+ source = args.tanksandtemples + "/" + scene
+ experiment = name + scene +"_tandt"
+ os.system(f"python train.py verbose=True gs.dataset.source_path={source} gs.dataset.model_path={args.output_path}/mipnerf/{scene} wandb.name={experiment} init_wC.use=True train.gs_epochs=30000 init_wC.matches_per_ref=15_000 init_wC.nns_per_ref=3 init_wC.num_refs=180 train.no_densify=True")
+ for scene in deep_blending_scenes:
+ source = args.deepblending + "/" + scene
+ experiment = name + scene + "_db"
+ os.system(f"python train.py verbose=True gs.dataset.source_path={source} gs.dataset.model_path={args.output_path}/mipnerf/{scene} wandb.name={experiment} init_wC.use=True train.gs_epochs=30000 init_wC.matches_per_ref=15_000 init_wC.nns_per_ref=3 init_wC.num_refs=180 train.no_densify=True")
+
+
+if not args.skip_rendering:
+ all_sources = []
+ for scene in mipnerf360_outdoor_scenes:
+ all_sources.append(args.mipnerf360 + "/" + scene)
+ for scene in mipnerf360_indoor_scenes:
+ all_sources.append(args.mipnerf360 + "/" + scene)
+ for scene in tanks_and_temples_scenes:
+ all_sources.append(args.tanksandtemples + "/" + scene )
+ for scene in deep_blending_scenes:
+ all_sources.append(args.deepblending + "/" + scene)
+
+ all_outputs = []
+ for scene in mipnerf360_outdoor_scenes:
+ all_outputs.append(args.output_path + "/mipnerf/" + scene)
+ for scene in mipnerf360_indoor_scenes:
+ all_outputs.append(args.output_path + "/mipnerf/" + scene)
+ for scene in tanks_and_temples_scenes:
+ all_outputs.append(args.output_path + "/tandt/" + scene)
+ for scene in deep_blending_scenes:
+ all_outputs.append(args.output_path + "/db/" + scene)
+
+
+ common_args = " --quiet --eval --skip_train"
+ for scene, source, output in zip(all_scenes, all_sources, all_outputs):
+ os.system("python ./submodules/gaussian-splatting/render.py --iteration 7000 -s " + source + " -m " + output + common_args)
+ os.system("python ./submodules/gaussian-splatting/render.py --iteration 30000 -s " + source + " -m " + output + common_args)
+
+if not args.skip_metrics:
+ all_outputs = []
+ for scene in mipnerf360_outdoor_scenes:
+ all_outputs.append(args.output_path + "/mipnerf/" + scene)
+ for scene in mipnerf360_indoor_scenes:
+ all_outputs.append(args.output_path + "/mipnerf/" + scene)
+ for scene in tanks_and_temples_scenes:
+ all_outputs.append(args.output_path + "/tandt/" + scene)
+ for scene in deep_blending_scenes:
+ all_outputs.append(args.output_path + "/db/" + scene)
+
+ scenes_string = ""
+ for scene, output in zip(all_scenes, all_outputs):
+ scenes_string += "\"" + output + "\" "
+
+ os.system("python metrics.py -m " + scenes_string)
\ No newline at end of file
diff --git a/gradio_demo.py b/gradio_demo.py
new file mode 100644
index 0000000000000000000000000000000000000000..9c55e44a018f49f05d70bfd9637c2bcf099eb7cd
--- /dev/null
+++ b/gradio_demo.py
@@ -0,0 +1,424 @@
+import torch
+import os
+import shutil
+import tempfile
+import argparse
+import gradio as gr
+import sys
+import io
+from PIL import Image
+import numpy as np
+from source.utils_aux import set_seed
+from source.utils_preprocess import read_video_frames, preprocess_frames, select_optimal_frames, save_frames_to_scene_dir, run_colmap_on_scene
+from source.trainer import EDGSTrainer
+from hydra import initialize, compose
+import hydra
+import time
+from source.visualization import generate_circular_camera_path, save_numpy_frames_as_mp4, generate_fully_smooth_cameras_with_tsp, put_text_on_image
+import contextlib
+import base64
+
+
+# Init RoMA model:
+sys.path.append('../submodules/RoMa')
+from romatch import roma_outdoor, roma_indoor
+
+roma_model = roma_indoor(device="cuda:0")
+roma_model.upsample_preds = False
+roma_model.symmetric = False
+
+STATIC_FILE_SERVING_FOLDER = "./served_files"
+MODEL_PATH = None
+os.makedirs(STATIC_FILE_SERVING_FOLDER, exist_ok=True)
+
+trainer = None
+
+class Tee(io.TextIOBase):
+ def __init__(self, *streams):
+ self.streams = streams
+
+ def write(self, data):
+ for stream in self.streams:
+ stream.write(data)
+ return len(data)
+
+ def flush(self):
+ for stream in self.streams:
+ stream.flush()
+
+def capture_logs(func, *args, **kwargs):
+ log_capture_string = io.StringIO()
+ tee = Tee(sys.__stdout__, log_capture_string)
+ with contextlib.redirect_stdout(tee):
+ result = func(*args, **kwargs)
+ return result, log_capture_string.getvalue()
+
+# Training Pipeline
+def run_training_pipeline(scene_dir,
+ num_ref_views=16,
+ num_corrs_per_view=20000,
+ num_steps=1_000,
+ mode_toggle="Ours (EDGS)"):
+ with initialize(config_path="./configs", version_base="1.1"):
+ cfg = compose(config_name="train")
+
+ scene_name = os.path.basename(scene_dir)
+ model_output_dir = f"./outputs/{scene_name}_trained"
+
+ cfg.wandb.mode = "disabled"
+ cfg.gs.dataset.model_path = model_output_dir
+ cfg.gs.dataset.source_path = scene_dir
+ cfg.gs.dataset.images = "images"
+
+ cfg.gs.opt.TEST_CAM_IDX_TO_LOG = 12
+ cfg.train.gs_epochs = 30000
+
+ if mode_toggle=="Ours (EDGS)":
+ cfg.gs.opt.opacity_reset_interval = 1_000_000
+ cfg.train.reduce_opacity = True
+ cfg.train.no_densify = True
+ cfg.train.max_lr = True
+
+ cfg.init_wC.use = True
+ cfg.init_wC.matches_per_ref = num_corrs_per_view
+ cfg.init_wC.nns_per_ref = 1
+ cfg.init_wC.num_refs = num_ref_views
+ cfg.init_wC.add_SfM_init = False
+ cfg.init_wC.scaling_factor = 0.00077 * 2.
+
+ set_seed(cfg.seed)
+ os.makedirs(cfg.gs.dataset.model_path, exist_ok=True)
+
+ global trainer
+ global MODEL_PATH
+ generator3dgs = hydra.utils.instantiate(cfg.gs, do_train_test_split=False)
+ trainer = EDGSTrainer(GS=generator3dgs, training_config=cfg.gs.opt, device=cfg.device, log_wandb=cfg.wandb.mode != 'disabled')
+
+ # Disable evaluation and saving
+ trainer.saving_iterations = []
+ trainer.evaluate_iterations = []
+
+ # Initialize
+ trainer.timer.start()
+ start_time = time.time()
+ trainer.init_with_corr(cfg.init_wC, roma_model=roma_model)
+ time_for_init = time.time()-start_time
+
+ viewpoint_cams = trainer.GS.scene.getTrainCameras()
+ path_cameras = generate_fully_smooth_cameras_with_tsp(existing_cameras=viewpoint_cams,
+ n_selected=6, # 8
+ n_points_per_segment=30, # 30
+ closed=False)
+ path_cameras = path_cameras + path_cameras[::-1]
+
+ path_renderings = []
+ idx = 0
+ # Visualize after init
+ for _ in range(120):
+ with torch.no_grad():
+ viewpoint_cam = path_cameras[idx]
+ idx = (idx + 1) % len(path_cameras)
+ render_pkg = trainer.GS(viewpoint_cam)
+ image = render_pkg["render"]
+ image_np = np.clip(image.detach().cpu().numpy().transpose(1, 2, 0), 0, 1)
+ image_np = (image_np * 255).astype(np.uint8)
+ path_renderings.append(put_text_on_image(img=image_np,
+ text=f"Init stage.\nTime:{time_for_init:.3f}s. "))
+ path_renderings = path_renderings + [put_text_on_image(img=image_np, text=f"Start fitting.\nTime:{time_for_init:.3f}s. ")]*30
+
+ # Train and save visualizations during training.
+ start_time = time.time()
+ for _ in range(int(num_steps//10)):
+ with torch.no_grad():
+ viewpoint_cam = path_cameras[idx]
+ idx = (idx + 1) % len(path_cameras)
+ render_pkg = trainer.GS(viewpoint_cam)
+ image = render_pkg["render"]
+ image_np = np.clip(image.detach().cpu().numpy().transpose(1, 2, 0), 0, 1)
+ image_np = (image_np * 255).astype(np.uint8)
+ path_renderings.append(put_text_on_image(
+ img=image_np,
+ text=f"Fitting stage.\nTime:{time_for_init + time.time()-start_time:.3f}s. "))
+
+ cfg.train.gs_epochs = 10
+ trainer.train(cfg.train)
+ print(f"Time elapsed: {(time_for_init + time.time()-start_time):.2f}s.")
+ # if (cfg.init_wC.use == False) and (time_for_init + time.time()-start_time) > 60:
+ # break
+ final_time = time.time()
+
+ # Add static frame. To highlight we're done
+ path_renderings += [put_text_on_image(
+ img=image_np, text=f"Done.\nTime:{time_for_init + final_time -start_time:.3f}s. ")]*30
+ # Final rendering at the end.
+ for _ in range(len(path_cameras)):
+ with torch.no_grad():
+ viewpoint_cam = path_cameras[idx]
+ idx = (idx + 1) % len(path_cameras)
+ render_pkg = trainer.GS(viewpoint_cam)
+ image = render_pkg["render"]
+ image_np = np.clip(image.detach().cpu().numpy().transpose(1, 2, 0), 0, 1)
+ image_np = (image_np * 255).astype(np.uint8)
+ path_renderings.append(put_text_on_image(img=image_np,
+ text=f"Final result.\nTime:{time_for_init + final_time -start_time:.3f}s. "))
+
+ trainer.save_model()
+ final_video_path = os.path.join(STATIC_FILE_SERVING_FOLDER, f"{scene_name}_final.mp4")
+ save_numpy_frames_as_mp4(frames=path_renderings, output_path=final_video_path, fps=30, center_crop=0.85)
+ MODEL_PATH = cfg.gs.dataset.model_path
+ ply_path = os.path.join(cfg.gs.dataset.model_path, f"point_cloud/iteration_{trainer.gs_step}/point_cloud.ply")
+ shutil.copy(ply_path, os.path.join(STATIC_FILE_SERVING_FOLDER, "point_cloud_final.ply"))
+
+ return final_video_path, ply_path
+
+# Gradio Interface
+def gradio_interface(input_path, num_ref_views, num_corrs, num_steps):
+ images, scene_dir = run_full_pipeline(input_path, num_ref_views, num_corrs, max_size=1024)
+ shutil.copytree(scene_dir, STATIC_FILE_SERVING_FOLDER+'/scene_colmaped', dirs_exist_ok=True)
+ (final_video_path, ply_path), log_output = capture_logs(run_training_pipeline,
+ scene_dir,
+ num_ref_views,
+ num_corrs,
+ num_steps)
+ images_rgb = [img[:, :, ::-1] for img in images]
+ return images_rgb, final_video_path, scene_dir, ply_path, log_output
+
+# Dummy Render Functions
+def render_all_views(scene_dir):
+ viewpoint_cams = trainer.GS.scene.getTrainCameras()
+ path_cameras = generate_fully_smooth_cameras_with_tsp(existing_cameras=viewpoint_cams,
+ n_selected=8,
+ n_points_per_segment=60,
+ closed=False)
+ path_cameras = path_cameras + path_cameras[::-1]
+
+ path_renderings = []
+ with torch.no_grad():
+ for viewpoint_cam in path_cameras:
+ render_pkg = trainer.GS(viewpoint_cam)
+ image = render_pkg["render"]
+ image_np = np.clip(image.detach().cpu().numpy().transpose(1, 2, 0), 0, 1)
+ image_np = (image_np * 255).astype(np.uint8)
+ path_renderings.append(image_np)
+ save_numpy_frames_as_mp4(frames=path_renderings,
+ output_path=os.path.join(STATIC_FILE_SERVING_FOLDER, "render_all_views.mp4"),
+ fps=30,
+ center_crop=0.85)
+
+ return os.path.join(STATIC_FILE_SERVING_FOLDER, "render_all_views.mp4")
+
+def render_circular_path(scene_dir):
+ viewpoint_cams = trainer.GS.scene.getTrainCameras()
+ path_cameras = generate_circular_camera_path(existing_cameras=viewpoint_cams,
+ N=240,
+ radius_scale=0.65,
+ d=0)
+
+ path_renderings = []
+ with torch.no_grad():
+ for viewpoint_cam in path_cameras:
+ render_pkg = trainer.GS(viewpoint_cam)
+ image = render_pkg["render"]
+ image_np = np.clip(image.detach().cpu().numpy().transpose(1, 2, 0), 0, 1)
+ image_np = (image_np * 255).astype(np.uint8)
+ path_renderings.append(image_np)
+ save_numpy_frames_as_mp4(frames=path_renderings,
+ output_path=os.path.join(STATIC_FILE_SERVING_FOLDER, "render_circular_path.mp4"),
+ fps=30,
+ center_crop=0.85)
+
+ return os.path.join(STATIC_FILE_SERVING_FOLDER, "render_circular_path.mp4")
+
+# Download Functions
+def download_cameras():
+ path = os.path.join(MODEL_PATH, "cameras.json")
+ return f"[📥 Download Cameras.json](file={path})"
+
+def download_model():
+ path = os.path.join(STATIC_FILE_SERVING_FOLDER, "point_cloud_final.ply")
+ return f"[📥 Download Pretrained Model (.ply)](file={path})"
+
+# Full pipeline helpers
+def run_full_pipeline(input_path, num_ref_views, num_corrs, max_size=1024):
+ tmpdirname = tempfile.mkdtemp()
+ scene_dir = os.path.join(tmpdirname, "scene")
+ os.makedirs(scene_dir, exist_ok=True)
+
+ selected_frames = process_input(input_path, num_ref_views, scene_dir, max_size)
+ run_colmap_on_scene(scene_dir)
+
+ return selected_frames, scene_dir
+
+# Preprocess Input
+def process_input(input_path, num_ref_views, output_dir, max_size=1024):
+ if isinstance(input_path, (str, os.PathLike)):
+ if os.path.isdir(input_path):
+ frames = []
+ for img_file in sorted(os.listdir(input_path)):
+ if img_file.lower().endswith(('jpg', 'jpeg', 'png')):
+ img = Image.open(os.path.join(output_dir, img_file)).convert('RGB')
+ img.thumbnail((1024, 1024))
+ frames.append(np.array(img))
+ else:
+ frames = read_video_frames(video_input=input_path, max_size=max_size)
+ else:
+ frames = read_video_frames(video_input=input_path, max_size=max_size)
+
+ frames_scores = preprocess_frames(frames)
+ selected_frames_indices = select_optimal_frames(scores=frames_scores,
+ k=min(num_ref_views, len(frames)))
+ selected_frames = [frames[frame_idx] for frame_idx in selected_frames_indices]
+
+ save_frames_to_scene_dir(frames=selected_frames, scene_dir=output_dir)
+ return selected_frames
+
+def preprocess_input(input_path, num_ref_views, max_size=1024):
+ tmpdirname = tempfile.mkdtemp()
+ scene_dir = os.path.join(tmpdirname, "scene")
+ os.makedirs(scene_dir, exist_ok=True)
+ selected_frames = process_input(input_path, num_ref_views, scene_dir, max_size)
+ run_colmap_on_scene(scene_dir)
+ return selected_frames, scene_dir
+
+def start_training(scene_dir, num_ref_views, num_corrs, num_steps):
+ return capture_logs(run_training_pipeline, scene_dir, num_ref_views, num_corrs, num_steps)
+
+
+# Gradio App
+with gr.Blocks() as demo:
+ with gr.Row():
+ with gr.Column(scale=6):
+ gr.Markdown("""
+ ## 📄 EDGS: Eliminating Densification for Efficient Convergence of 3DGS
+ 🔗 Project Page
+ """, elem_id="header")
+
+ gr.Markdown("""
+ ### 🛠️ How to Use This Demo
+
+ 1. Upload a **front-facing video** or **a folder of images** of a **static** scene.
+ 2. Use the sliders to configure the number of reference views, correspondences, and optimization steps.
+ 3. First press on preprocess Input to extract frames from video(for videos) and COLMAP frames.
+ 4. Then click **🚀 Start Reconstruction** to actually launch the reconstruction pipeline.
+ 5. Watch the training visualization and explore the 3D model.
+ ‼️ **If you see nothing in the 3D model viewer**, try rotating or zooming — sometimes the initial camera orientation is off.
+
+
+ ✅ Best for scenes with small camera motion.
+ ❗ For full 360° or large-scale scenes, we recommend the Colab version (see project page).
+ """, elem_id="quickstart")
+
+
+ scene_dir_state = gr.State()
+ ply_model_state = gr.State()
+
+ with gr.Row():
+ with gr.Column(scale=2):
+ input_file = gr.File(label="Upload Video or Images",
+ file_types=[".mp4", ".avi", ".mov", ".png", ".jpg", ".jpeg"],
+ file_count="multiple")
+ gr.Examples(
+ examples = [
+ [["assets/examples/video_bakery.mp4"]],
+ [["assets/examples/video_flowers.mp4"]],
+ [["assets/examples/video_fruits.mp4"]],
+ [["assets/examples/video_plant.mp4"]],
+ [["assets/examples/video_salad.mp4"]],
+ [["assets/examples/video_tram.mp4"]],
+ [["assets/examples/video_tulips.mp4"]]
+ ],
+ inputs=[input_file],
+ label="🎞️ ALternatively, try an Example Video",
+ examples_per_page=4
+ )
+ ref_slider = gr.Slider(4, 32, value=16, step=1, label="Number of Reference Views")
+ corr_slider = gr.Slider(5000, 30000, value=20000, step=1000, label="Correspondences per Reference View")
+ fit_steps_slider = gr.Slider(100, 5000, value=400, step=100, label="Number of optimization steps")
+ preprocess_button = gr.Button("📸 Preprocess Input")
+ start_button = gr.Button("🚀 Start Reconstruction", interactive=False)
+ gallery = gr.Gallery(label="Selected Reference Views", columns=4, height=300)
+
+ with gr.Column(scale=3):
+ gr.Markdown("### 🏋️ Training Visualization")
+ video_output = gr.Video(label="Training Video", autoplay=True)
+ render_all_views_button = gr.Button("🎥 Render All-Views Path")
+ render_circular_path_button = gr.Button("🎥 Render Circular Path")
+ rendered_video_output = gr.Video(label="Rendered Video", autoplay=True)
+ with gr.Column(scale=5):
+ gr.Markdown("### 🌐 Final 3D Model")
+ model3d_viewer = gr.Model3D(label="3D Model Viewer")
+
+ gr.Markdown("### 📦 Output Files")
+ with gr.Row(height=50):
+ with gr.Column():
+ #gr.Markdown(value=f"[📥 Download .ply](file/point_cloud_final.ply)")
+ download_cameras_button = gr.Button("📥 Download Cameras.json")
+ download_cameras_file = gr.File(label="📄 Cameras.json")
+ with gr.Column():
+ download_model_button = gr.Button("📥 Download Pretrained Model (.ply)")
+ download_model_file = gr.File(label="📄 Pretrained Model (.ply)")
+
+ log_output_box = gr.Textbox(label="🖥️ Log", lines=10, interactive=False)
+
+ def on_preprocess_click(input_file, num_ref_views):
+ images, scene_dir = preprocess_input(input_file, num_ref_views)
+ return gr.update(value=[x[...,::-1] for x in images]), scene_dir, gr.update(interactive=True)
+
+ def on_start_click(scene_dir, num_ref_views, num_corrs, num_steps):
+ (video_path, ply_path), logs = start_training(scene_dir, num_ref_views, num_corrs, num_steps)
+ return video_path, ply_path, logs
+
+ preprocess_button.click(
+ fn=on_preprocess_click,
+ inputs=[input_file, ref_slider],
+ outputs=[gallery, scene_dir_state, start_button]
+ )
+
+ start_button.click(
+ fn=on_start_click,
+ inputs=[scene_dir_state, ref_slider, corr_slider, fit_steps_slider],
+ outputs=[video_output, model3d_viewer, log_output_box]
+ )
+
+ render_all_views_button.click(fn=render_all_views, inputs=[scene_dir_state], outputs=[rendered_video_output])
+ render_circular_path_button.click(fn=render_circular_path, inputs=[scene_dir_state], outputs=[rendered_video_output])
+
+ download_cameras_button.click(fn=lambda: os.path.join(MODEL_PATH, "cameras.json"), inputs=[], outputs=[download_cameras_file])
+ download_model_button.click(fn=lambda: os.path.join(STATIC_FILE_SERVING_FOLDER, "point_cloud_final.ply"), inputs=[], outputs=[download_model_file])
+
+
+ gr.Markdown("""
+ ---
+ ### 📖 Detailed Overview
+
+ If you uploaded a video, it will be automatically cut into a smaller number of frames (default: 16).
+
+ The model pipeline:
+ 1. 🧠 Runs PyCOLMAP to estimate camera intrinsics & poses (~3–7 seconds for <16 images).
+ 2. 🔁 Computes 2D-2D correspondences between views. More correspondences generally improve quality.
+ 3. 🔧 Optimizes a 3D Gaussian Splatting model for several steps.
+
+ ### 🎥 Training Visualization
+ You will see a visualization of the entire training process in the "Training Video" pane.
+
+ ### 🌀 Rendering & 3D Model
+ - Render the scene from a circular path of novel views.
+ - Or from camera views close to the original input.
+
+ The 3D model is shown in the right viewer. You can explore it interactively:
+ - On PC: WASD keys, arrow keys, and mouse clicks
+ - On mobile: pan and pinch to zoom
+
+ 🕒 Note: the 3D viewer takes a few extra seconds (~5s) to display after training ends.
+
+ ---
+ Preloaded models coming soon. (TODO)
+ """, elem_id="details")
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser(description="Launch Gradio demo for EDGS preprocessing and 3D viewing.")
+ parser.add_argument("--port", type=int, default=7860, help="Port to launch the Gradio app on.")
+ parser.add_argument("--no_share", action='store_true', help="Disable Gradio sharing and assume local access (default: share=True)")
+ args = parser.parse_args()
+
+ demo.launch(server_name="0.0.0.0", server_port=args.port, share=not args.no_share)
diff --git a/install.sh b/install.sh
new file mode 100644
index 0000000000000000000000000000000000000000..82578139a8d5f7d6f56d0a8a6ec257a7450d354a
--- /dev/null
+++ b/install.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+git clone git@github.com:CompVis/EDGS.git --recursive
+cd EDGS
+git submodule update --init --recursive
+
+conda create -y -n edgs python=3.10 pip
+conda activate edgs
+
+# Optionally set path to CUDA
+export CUDA_HOME=/usr/local/cuda-12.1
+export LD_LIBRARY_PATH=$CUDA_HOME/lib64:$LD_LIBRARY_PATH
+export PATH=$CUDA_HOME/bin:$PATH
+conda install pytorch torchvision torchaudio pytorch-cuda=12.1 -c pytorch -c nvidia -y
+conda install nvidia/label/cuda-12.1.0::cuda-toolkit -y
+
+pip install -e submodules/gaussian-splatting/submodules/diff-gaussian-rasterization
+pip install -e submodules/gaussian-splatting/submodules/simple-knn
+
+# For COLMAP and pycolmap
+# Optionally install original colmap but probably pycolmap suffices
+# conda install conda-forge/label/colmap_dev::colmap
+pip install pycolmap
+
+
+pip install wandb hydra-core tqdm torchmetrics lpips matplotlib rich plyfile imageio imageio-ffmpeg
+conda install numpy=1.26.4 -y -c conda-forge --override-channels
+
+pip install -e submodules/RoMa
+conda install anaconda::jupyter --yes
+
+# Stuff necessary for gradio and visualizations
+pip install gradio
+pip install plotly scikit-learn moviepy==2.1.1 ffmpeg
+pip install open3d
+
diff --git a/main.py b/main.py
new file mode 100644
index 0000000000000000000000000000000000000000..e7867441520e380e1da8e0d940ccb9f27d15790f
--- /dev/null
+++ b/main.py
@@ -0,0 +1,268 @@
+import torch
+import os
+import shutil
+import tempfile
+import uuid
+import asyncio
+import io
+import time
+import contextlib
+import base64
+from PIL import Image
+import numpy as np
+from fastapi import FastAPI, UploadFile, File, HTTPException, Body
+from fastapi.responses import JSONResponse, StreamingResponse, FileResponse
+from pydantic import BaseModel, Field
+
+try:
+ from source.utils_aux import set_seed
+ from source.utils_preprocess import read_video_frames, preprocess_frames, select_optimal_frames, save_frames_to_scene_dir, run_colmap_on_scene
+ from source.trainer import EDGSTrainer
+ from hydra import initialize, compose
+ import hydra
+ from source.visualization import generate_fully_smooth_cameras_with_tsp, put_text_on_image
+ import sys
+ sys.path.append('../submodules/RoMa') # Ajusta esta ruta si es necesario
+ from romatch import roma_indoor
+except ImportError as e:
+ print(f"Error: No se pudieron importar los módulos del proyecto EDGS. Asegúrate de que las rutas y la instalación son correctas. {e}")
+ sys.exit(1)
+
+# --- Configuración Inicial ---
+# 1. Inicialización de la App FastAPI
+app = FastAPI(
+ title="EDGS Training API",
+ description="Una API para preprocesar videos y entrenar modelos 3DGS con EDGS.",
+ version="1.0.0"
+)
+
+# 2. Variables Globales y Almacenamiento de Estado
+# El modelo se cargará en el evento 'startup'
+roma_model = None
+
+# Base de datos en memoria para gestionar el estado de las tareas entre endpoints
+tasks_db = {}
+
+# 3. Modelos Pydantic para la validación de datos
+class TrainParams(BaseModel):
+ num_corrs_per_view: int = Field(20000, gt=0, description="Correspondencias por vista de referencia.")
+ num_steps: int = Field(1000, gt=0, description="Número de pasos de optimización.")
+
+class PreprocessResponse(BaseModel):
+ task_id: str
+ message: str
+ selected_frames_count: int
+ # Opcional: podrías devolver las imágenes en base64 si el cliente las necesita visualizar
+ # frames: list[str]
+
+# --- Lógica de Negocio (Adaptada del script de Gradio) ---
+
+# Esta función se ejecutará en un hilo separado para no bloquear el servidor
+def run_preprocessing_sync(input_path: str, num_ref_views: int):
+ """
+ Ejecuta el preprocesamiento: selección de frames y ejecución de COLMAP.
+ """
+ tmpdirname = tempfile.mkdtemp()
+ scene_dir = os.path.join(tmpdirname, "scene")
+ os.makedirs(scene_dir, exist_ok=True)
+
+ # 1. Lee y selecciona los mejores frames
+ frames = read_video_frames(video_input=input_path, max_size=1024)
+ frames_scores = preprocess_frames(frames)
+ selected_frames_indices = select_optimal_frames(scores=frames_scores, k=min(num_ref_views, len(frames)))
+ selected_frames = [frames[frame_idx] for frame_idx in selected_frames_indices]
+
+ # 2. Guarda los frames y ejecuta COLMAP
+ save_frames_to_scene_dir(frames=selected_frames, scene_dir=scene_dir)
+ run_colmap_on_scene(scene_dir)
+
+ return scene_dir, selected_frames
+
+async def training_log_generator(scene_dir: str, num_ref_views: int, params: TrainParams, task_id: str):
+ """
+ Un generador asíncrono que ejecuta el entrenamiento. Los logs detallados se muestran
+ en la terminal del servidor, mientras que el cliente recibe un stream de progreso simple.
+ """
+ def training_pipeline():
+ try:
+ # La inicialización y configuración de Hydra se mantienen igual
+ with initialize(config_path="./configs", version_base="1.1"):
+ cfg = compose(config_name="train")
+
+ # --- CONFIGURACIÓN COMPLETA ---
+ scene_name = os.path.basename(scene_dir)
+ model_output_dir = f"./outputs/{scene_name}_trained"
+ cfg.wandb.mode = "disabled"
+ cfg.gs.dataset.model_path = model_output_dir
+ cfg.gs.dataset.source_path = scene_dir
+ cfg.gs.dataset.images = "images"
+ cfg.train.gs_epochs = 30000
+ cfg.gs.opt.opacity_reset_interval = 1_000_000
+ cfg.train.reduce_opacity = True
+ cfg.train.no_densify = True
+ cfg.train.max_lr = True
+ cfg.init_wC.use = True
+ cfg.init_wC.matches_per_ref = params.num_corrs_per_view
+ cfg.init_wC.nns_per_ref = 1
+ cfg.init_wC.num_refs = num_ref_views
+ cfg.init_wC.add_SfM_init = False
+ cfg.init_wC.scaling_factor = 0.00077 * 2.
+
+ set_seed(cfg.seed)
+ os.makedirs(cfg.gs.dataset.model_path, exist_ok=True)
+
+ device = cfg.device
+ generator3dgs = hydra.utils.instantiate(cfg.gs, do_train_test_split=False)
+ trainer = EDGSTrainer(GS=generator3dgs, training_config=cfg.gs.opt, device=device, log_wandb=False)
+ trainer.saving_iterations = []
+ trainer.evaluate_iterations = []
+ trainer.timer.start()
+
+ # Mensaje de progreso para el cliente antes de la inicialización
+ yield "data: Inicializando modelo...\n\n"
+ trainer.init_with_corr(cfg.init_wC, roma_model=roma_model)
+
+ # El bucle de entrenamiento principal
+ for step in range(int(params.num_steps // 10)):
+ cfg.train.gs_epochs = 10
+ # trainer.train() ahora imprimirá sus logs detallados directamente en la terminal
+ trainer.train(cfg.train)
+
+ # --- CAMBIO CLAVE ---
+ # Envía un mensaje de progreso simple al cliente en lugar de los logs capturados.
+ yield f"data: Progreso: {step*10+10}/{params.num_steps} pasos completados.\n\n"
+
+ trainer.save_model()
+ ply_path = os.path.join(cfg.gs.dataset.model_path, f"point_cloud/iteration_{trainer.gs_step}/point_cloud.ply")
+
+ tasks_db[task_id]['result_ply_path'] = ply_path
+
+ final_message = "Entrenamiento completado. El modelo está listo para descargar."
+ yield f"data: {final_message}\n\n"
+
+ except Exception as e:
+ yield f"data: ERROR: {repr(e)}\n\n"
+
+ # El bucle que llama a la pipeline se mantiene igual
+ training_gen = training_pipeline()
+ for log_message in training_gen:
+ yield log_message
+ await asyncio.sleep(0.1)
+
+# --- Eventos de Ciclo de Vida de la App ---
+
+@app.on_event("startup")
+async def startup_event():
+ """
+ Carga el modelo RoMa cuando el servidor se inicia.
+ """
+ global roma_model
+ print("🚀 Iniciando servidor FastAPI...")
+ if torch.cuda.is_available():
+ device = "cuda:0"
+ print("✅ GPU detectada. Usando CUDA.")
+ else:
+ device = "cpu"
+ print("⚠️ No se detectó GPU. Usando CPU (puede ser muy lento).")
+
+ roma_model = roma_indoor(device=device)
+ roma_model.upsample_preds = False
+ roma_model.symmetric = False
+ print("🤖 Modelo RoMa cargado y listo.")
+
+# --- Endpoints de la API ---
+
+@app.post("/preprocess", response_model=PreprocessResponse)
+async def preprocess_video(
+ num_ref_views: int = Body(16, embed=True, description="Número de vistas de referencia a extraer del video."),
+ video: UploadFile = File(..., description="Archivo de video a procesar (.mp4, .mov).")
+):
+ """
+ Recibe un video, lo preprocesa (extrae frames + COLMAP) y prepara para el entrenamiento.
+ """
+ if not video.filename.lower().endswith(('.mp4', '.avi', '.mov')):
+ raise HTTPException(status_code=400, detail="Formato de archivo no soportado. Usa .mp4, .avi, o .mov.")
+
+ # Guarda el video temporalmente para que la librería pueda procesarlo
+ with tempfile.NamedTemporaryFile(delete=False, suffix=video.filename) as tmp_video:
+ shutil.copyfileobj(video.file, tmp_video)
+ tmp_video_path = tmp_video.name
+
+ try:
+ loop = asyncio.get_running_loop()
+ # Ejecuta la función síncrona y bloqueante en un executor para no bloquear el servidor
+ scene_dir, selected_frames = await loop.run_in_executor(
+ None, run_preprocessing_sync, tmp_video_path, num_ref_views
+ )
+
+ # Genera un ID único para esta tarea y guarda la ruta
+ task_id = str(uuid.uuid4())
+ tasks_db[task_id] = {
+ "scene_dir": scene_dir,
+ "num_ref_views": len(selected_frames),
+ "result_ply_path": None
+ }
+
+ return JSONResponse(
+ status_code=200,
+ content={
+ "task_id": task_id,
+ "message": f"Preprocesamiento completado. Se generó el directorio de la escena. Listo para entrenar.",
+ "selected_frames_count": len(selected_frames)
+ }
+ )
+ except Exception as e:
+ raise HTTPException(status_code=500, detail=f"Error durante el preprocesamiento: {e}")
+ finally:
+ os.unlink(tmp_video_path) # Limpia el archivo de video temporal
+
+
+@app.post("/train/{task_id}")
+async def train_model(task_id: str, params: TrainParams):
+ """
+ Inicia el entrenamiento para una tarea preprocesada.
+ Devuelve un stream de logs en tiempo real.
+ """
+ if task_id not in tasks_db:
+ raise HTTPException(status_code=404, detail="Task ID no encontrado. Por favor, ejecuta el preprocesamiento primero.")
+
+ task_info = tasks_db[task_id]
+ scene_dir = task_info["scene_dir"]
+ num_ref_views = task_info["num_ref_views"]
+
+ return StreamingResponse(
+ training_log_generator(scene_dir, num_ref_views, params, task_id),
+ media_type="text/event-stream"
+ )
+
+@app.get("/download/{task_id}")
+async def download_ply_file(task_id: str):
+ """
+ Permite descargar el archivo .ply resultante de un entrenamiento completado.
+ """
+ if task_id not in tasks_db:
+ raise HTTPException(status_code=404, detail="Task ID no encontrado.")
+
+ task_info = tasks_db[task_id]
+ ply_path = task_info.get("result_ply_path")
+
+ if not ply_path:
+ raise HTTPException(status_code=404, detail="El entrenamiento no ha finalizado o el archivo aún no está disponible.")
+
+ if not os.path.exists(ply_path):
+ raise HTTPException(status_code=500, detail="Error: El archivo del modelo no se encuentra en el servidor.")
+
+ # Generamos un nombre de archivo amigable para el usuario
+ file_name = f"model_{task_id[:8]}.ply"
+
+ return FileResponse(
+ path=ply_path,
+ media_type='application/octet-stream',
+ filename=file_name
+ )
+
+if __name__ == "__main__":
+ import uvicorn
+ # Para ejecutar: uvicorn main:app --reload
+ # El flag --reload es para desarrollo. Quítalo en producción.
+ uvicorn.run("main:app", host="0.0.0.0", port=7860, reload=False)
\ No newline at end of file
diff --git a/metrics.py b/metrics.py
new file mode 100644
index 0000000000000000000000000000000000000000..511d1068af2762527ad72f5d03d4bb46afccdc2b
--- /dev/null
+++ b/metrics.py
@@ -0,0 +1,115 @@
+#
+# Copyright (C) 2023, Inria
+# GRAPHDECO research group, https://team.inria.fr/graphdeco
+# All rights reserved.
+#
+# This software is free for non-commercial, research and evaluation use
+# under the terms of the LICENSE.md file.
+#
+# For inquiries contact george.drettakis@inria.fr
+#
+
+from pathlib import Path
+import os
+import sys
+from PIL import Image
+import torch
+import torchvision.transforms.functional as tf
+sys.path.append('./submodules/gaussian-splatting/')
+from utils.loss_utils import ssim
+from lpipsPyTorch import lpips as lpips_3dgs
+import json
+from tqdm import tqdm
+from utils.image_utils import psnr
+from argparse import ArgumentParser
+
+import lpips
+
+def readImages(renders_dir, gt_dir):
+ renders = []
+ gts = []
+ image_names = []
+ for fname in os.listdir(renders_dir):
+ render = Image.open(renders_dir / fname)
+ gt = Image.open(gt_dir / fname)
+ renders.append(tf.to_tensor(render).unsqueeze(0)[:, :3, :, :].cuda())
+ gts.append(tf.to_tensor(gt).unsqueeze(0)[:, :3, :, :].cuda())
+ image_names.append(fname)
+ return renders, gts, image_names
+
+def evaluate(model_paths):
+
+ full_dict = {}
+ per_view_dict = {}
+ full_dict_polytopeonly = {}
+ per_view_dict_polytopeonly = {}
+ print("")
+
+ for scene_dir in model_paths:
+ #try:
+ print("Scene:", scene_dir)
+ full_dict[scene_dir] = {}
+ per_view_dict[scene_dir] = {}
+ full_dict_polytopeonly[scene_dir] = {}
+ per_view_dict_polytopeonly[scene_dir] = {}
+
+ test_dir = Path(scene_dir) / "test"
+
+ for method in os.listdir(test_dir):
+ print("Method:", method)
+
+ full_dict[scene_dir][method] = {}
+ per_view_dict[scene_dir][method] = {}
+ full_dict_polytopeonly[scene_dir][method] = {}
+ per_view_dict_polytopeonly[scene_dir][method] = {}
+
+ method_dir = test_dir / method
+ gt_dir = method_dir/ "gt"
+ renders_dir = method_dir / "renders"
+ renders, gts, image_names = readImages(renders_dir, gt_dir)
+
+ ssims = []
+ psnrs = []
+ lpipss = []
+ lpipss_3dgs = []
+ with torch.no_grad():
+ for idx in tqdm(range(len(renders)), desc="Metric evaluation progress"):
+ ssims.append(ssim(renders[idx], gts[idx]))
+ psnrs.append(psnr(renders[idx], gts[idx]))
+ lpipss.append(lpips_fn(renders[idx], gts[idx]))
+ lpipss_3dgs.append(lpips_3dgs(renders[idx], gts[idx], net_type='vgg'))
+ torch.cuda.empty_cache()
+
+ print(" SSIM : {:>12.7f}".format(torch.tensor(ssims).mean(), ".5"))
+ print(" PSNR : {:>12.7f}".format(torch.tensor(psnrs).mean(), ".5"))
+ print(" LPIPS: {:>12.7f}".format(torch.tensor(lpipss).mean(), ".5"))
+ print(" LPIPS_3dgs: {:>12.7f}".format(torch.tensor(lpipss_3dgs).mean(), ".5"))
+ print("")
+
+ full_dict[scene_dir][method].update({"SSIM": torch.tensor(ssims).mean().item(),
+ "PSNR": torch.tensor(psnrs).mean().item(),
+ "LPIPS": torch.tensor(lpipss).mean().item(),
+ "LPIPS_3dgs": torch.tensor(lpipss_3dgs).mean().item(),
+ })
+ per_view_dict[scene_dir][method].update({"SSIM": {name: ssim for ssim, name in zip(torch.tensor(ssims).tolist(), image_names)},
+ "PSNR": {name: psnr for psnr, name in zip(torch.tensor(psnrs).tolist(), image_names)},
+ "LPIPS": {name: lp for lp, name in zip(torch.tensor(lpipss).tolist(), image_names)},
+ "LPIPS_3dgs": {name: lp for lp, name in zip(torch.tensor(lpipss_3dgs).tolist(), image_names)},
+ })
+
+ with open(scene_dir + "/results.json", 'w') as fp:
+ json.dump(full_dict[scene_dir], fp, indent=True)
+ with open(scene_dir + "/per_view.json", 'w') as fp:
+ json.dump(per_view_dict[scene_dir], fp, indent=True)
+ #except:
+ # print("Unable to compute metrics for model", scene_dir)
+
+if __name__ == "__main__":
+ device = torch.device("cuda:0")
+ torch.cuda.set_device(device)
+ lpips_fn = lpips.LPIPS(net='vgg').to(device)
+ # Set up command line argument parser
+ parser = ArgumentParser(description="Training script parameters")
+ parser.add_argument('--model_paths', '-m', required=True, nargs="+", type=str, default=[])
+ args = parser.parse_args()
+ evaluate(args.model_paths)
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..170703df656cbf9fd6c8b01d37ad13ea971b9e6d
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1 @@
+fastapi
\ No newline at end of file
diff --git a/script.bash b/script.bash
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/source/EDGS.code-workspace b/source/EDGS.code-workspace
new file mode 100644
index 0000000000000000000000000000000000000000..edaa508ccdc6a9c70ca61d91e4ac1f74c34c0c78
--- /dev/null
+++ b/source/EDGS.code-workspace
@@ -0,0 +1,11 @@
+{
+ "folders": [
+ {
+ "path": ".."
+ },
+ {
+ "path": "../../../../.."
+ }
+ ],
+ "settings": {}
+}
\ No newline at end of file
diff --git a/source/__init__.py b/source/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/source/corr_init.py b/source/corr_init.py
new file mode 100644
index 0000000000000000000000000000000000000000..09c94a225fc8c6057999ed95035d13ad751198bb
--- /dev/null
+++ b/source/corr_init.py
@@ -0,0 +1,907 @@
+import sys
+sys.path.append('../')
+sys.path.append("../submodules")
+sys.path.append('../submodules/RoMa')
+
+from matplotlib import pyplot as plt
+from PIL import Image
+import torch
+import numpy as np
+
+#from tqdm import tqdm_notebook as tqdm
+from tqdm import tqdm
+from scipy.cluster.vq import kmeans, vq
+from scipy.spatial.distance import cdist
+
+import torch.nn.functional as F
+from romatch import roma_outdoor, roma_indoor
+from utils.sh_utils import RGB2SH
+from romatch.utils import get_tuple_transform_ops
+
+import time
+from collections import defaultdict
+from tqdm import tqdm
+
+
+def pairwise_distances(matrix):
+ """
+ Computes the pairwise Euclidean distances between all vectors in the input matrix.
+
+ Args:
+ matrix (torch.Tensor): Input matrix of shape [N, D], where N is the number of vectors and D is the dimensionality.
+
+ Returns:
+ torch.Tensor: Pairwise distance matrix of shape [N, N].
+ """
+ # Compute squared pairwise distances
+ squared_diff = torch.cdist(matrix, matrix, p=2)
+ return squared_diff
+
+
+def k_closest_vectors(matrix, k):
+ """
+ Finds the k-closest vectors for each vector in the input matrix based on Euclidean distance.
+
+ Args:
+ matrix (torch.Tensor): Input matrix of shape [N, D], where N is the number of vectors and D is the dimensionality.
+ k (int): Number of closest vectors to return for each vector.
+
+ Returns:
+ torch.Tensor: Indices of the k-closest vectors for each vector, excluding the vector itself.
+ """
+ # Compute pairwise distances
+ distances = pairwise_distances(matrix)
+
+ # For each vector, sort distances and get the indices of the k-closest vectors (excluding itself)
+ # Set diagonal distances to infinity to exclude the vector itself from the nearest neighbors
+ distances.fill_diagonal_(float('inf'))
+
+ # Get the indices of the k smallest distances (k-closest vectors)
+ _, indices = torch.topk(distances, k, largest=False, dim=1)
+
+ return indices
+
+
+def select_cameras_kmeans(cameras, K):
+ """
+ Selects K cameras from a set using K-means clustering.
+
+ Args:
+ cameras: NumPy array of shape (N, 16), representing N cameras with their 4x4 homogeneous matrices flattened.
+ K: Number of clusters (cameras to select).
+
+ Returns:
+ selected_indices: List of indices of the cameras closest to the cluster centers.
+ """
+ # Ensure input is a NumPy array
+ if not isinstance(cameras, np.ndarray):
+ cameras = np.asarray(cameras)
+
+ if cameras.shape[1] != 16:
+ raise ValueError("Each camera must have 16 values corresponding to a flattened 4x4 matrix.")
+
+ # Perform K-means clustering
+ cluster_centers, _ = kmeans(cameras, K)
+
+ # Assign each camera to a cluster and find distances to cluster centers
+ cluster_assignments, _ = vq(cameras, cluster_centers)
+
+ # Find the camera nearest to each cluster center
+ selected_indices = []
+ for k in range(K):
+ cluster_members = cameras[cluster_assignments == k]
+ distances = cdist([cluster_centers[k]], cluster_members)[0]
+ nearest_camera_idx = np.where(cluster_assignments == k)[0][np.argmin(distances)]
+ selected_indices.append(nearest_camera_idx)
+
+ return selected_indices
+
+
+def compute_warp_and_confidence(viewpoint_cam1, viewpoint_cam2, roma_model, device="cuda", verbose=False, output_dict={}):
+ """
+ Computes the warp and confidence between two viewpoint cameras using the roma_model.
+
+ Args:
+ viewpoint_cam1: Source viewpoint camera.
+ viewpoint_cam2: Target viewpoint camera.
+ roma_model: Pre-trained Roma model for correspondence matching.
+ device: Device to run the computation on.
+ verbose: If True, displays the images.
+
+ Returns:
+ certainty: Confidence tensor.
+ warp: Warp tensor.
+ imB: Processed image B as numpy array.
+ """
+ # Prepare images
+ imA = viewpoint_cam1.original_image.detach().cpu().numpy().transpose(1, 2, 0)
+ imB = viewpoint_cam2.original_image.detach().cpu().numpy().transpose(1, 2, 0)
+ imA = Image.fromarray(np.clip(imA * 255, 0, 255).astype(np.uint8))
+ imB = Image.fromarray(np.clip(imB * 255, 0, 255).astype(np.uint8))
+
+ if verbose:
+ fig, ax = plt.subplots(nrows=1, ncols=2, figsize=(16, 8))
+ cax1 = ax[0].imshow(imA)
+ ax[0].set_title("Image 1")
+ cax2 = ax[1].imshow(imB)
+ ax[1].set_title("Image 2")
+ fig.colorbar(cax1, ax=ax[0])
+ fig.colorbar(cax2, ax=ax[1])
+
+ for axis in ax:
+ axis.axis('off')
+ # Save the figure into the dictionary
+ output_dict[f'image_pair'] = fig
+
+ # Transform images
+ ws, hs = roma_model.w_resized, roma_model.h_resized
+ test_transform = get_tuple_transform_ops(resize=(hs, ws), normalize=True)
+ im_A, im_B = test_transform((imA, imB))
+ batch = {"im_A": im_A[None].to(device), "im_B": im_B[None].to(device)}
+
+ # Forward pass through Roma model
+ corresps = roma_model.forward(batch) if not roma_model.symmetric else roma_model.forward_symmetric(batch)
+ finest_scale = 1
+ hs, ws = roma_model.upsample_res if roma_model.upsample_preds else (hs, ws)
+
+ # Process certainty and warp
+ certainty = corresps[finest_scale]["certainty"]
+ im_A_to_im_B = corresps[finest_scale]["flow"]
+ if roma_model.attenuate_cert:
+ low_res_certainty = F.interpolate(
+ corresps[16]["certainty"], size=(hs, ws), align_corners=False, mode="bilinear"
+ )
+ certainty -= 0.5 * low_res_certainty * (low_res_certainty < 0)
+
+ # Upsample predictions if needed
+ if roma_model.upsample_preds:
+ im_A_to_im_B = F.interpolate(
+ im_A_to_im_B, size=(hs, ws), align_corners=False, mode="bilinear"
+ )
+ certainty = F.interpolate(
+ certainty, size=(hs, ws), align_corners=False, mode="bilinear"
+ )
+
+ # Convert predictions to final format
+ im_A_to_im_B = im_A_to_im_B.permute(0, 2, 3, 1)
+ im_A_coords = torch.stack(torch.meshgrid(
+ torch.linspace(-1 + 1 / hs, 1 - 1 / hs, hs, device=device),
+ torch.linspace(-1 + 1 / ws, 1 - 1 / ws, ws, device=device),
+ indexing='ij'
+ ), dim=0).permute(1, 2, 0).unsqueeze(0).expand(im_A_to_im_B.size(0), -1, -1, -1)
+
+ warp = torch.cat((im_A_coords, im_A_to_im_B), dim=-1)
+ certainty = certainty.sigmoid()
+
+ return certainty[0, 0], warp[0], np.array(imB)
+
+
+def resize_batch(tensors_3d, tensors_4d, target_shape):
+ """
+ Resizes a batch of tensors with shapes [B, H, W] and [B, H, W, 4] to the target spatial dimensions.
+
+ Args:
+ tensors_3d: Tensor of shape [B, H, W].
+ tensors_4d: Tensor of shape [B, H, W, 4].
+ target_shape: Tuple (target_H, target_W) specifying the target spatial dimensions.
+
+ Returns:
+ resized_tensors_3d: Tensor of shape [B, target_H, target_W].
+ resized_tensors_4d: Tensor of shape [B, target_H, target_W, 4].
+ """
+ target_H, target_W = target_shape
+
+ # Resize [B, H, W] tensor
+ resized_tensors_3d = F.interpolate(
+ tensors_3d.unsqueeze(1), size=(target_H, target_W), mode="bilinear", align_corners=False
+ ).squeeze(1)
+
+ # Resize [B, H, W, 4] tensor
+ B, _, _, C = tensors_4d.shape
+ resized_tensors_4d = F.interpolate(
+ tensors_4d.permute(0, 3, 1, 2), size=(target_H, target_W), mode="bilinear", align_corners=False
+ ).permute(0, 2, 3, 1)
+
+ return resized_tensors_3d, resized_tensors_4d
+
+
+def aggregate_confidences_and_warps(viewpoint_stack, closest_indices, roma_model, source_idx, verbose=False, output_dict={}):
+ """
+ Aggregates confidences and warps by iterating over the nearest neighbors of the source viewpoint.
+
+ Args:
+ viewpoint_stack: Stack of viewpoint cameras.
+ closest_indices: Indices of the nearest neighbors for each viewpoint.
+ roma_model: Pre-trained Roma model.
+ source_idx: Index of the source viewpoint.
+ verbose: If True, displays intermediate results.
+
+ Returns:
+ certainties_max: Aggregated maximum confidences.
+ warps_max: Aggregated warps corresponding to maximum confidences.
+ certainties_max_idcs: Pixel-wise index of the image from which we taken the best matching.
+ imB_compound: List of the neighboring images.
+ """
+ certainties_all, warps_all, imB_compound = [], [], []
+
+ for nn in tqdm(closest_indices[source_idx]):
+
+ viewpoint_cam1 = viewpoint_stack[source_idx]
+ viewpoint_cam2 = viewpoint_stack[nn]
+
+ certainty, warp, imB = compute_warp_and_confidence(viewpoint_cam1, viewpoint_cam2, roma_model, verbose=verbose, output_dict=output_dict)
+ certainties_all.append(certainty)
+ warps_all.append(warp)
+ imB_compound.append(imB)
+
+ certainties_all = torch.stack(certainties_all, dim=0)
+ target_shape = imB_compound[0].shape[:2]
+ if verbose:
+ print("certainties_all.shape:", certainties_all.shape)
+ print("torch.stack(warps_all, dim=0).shape:", torch.stack(warps_all, dim=0).shape)
+ print("target_shape:", target_shape)
+
+ certainties_all_resized, warps_all_resized = resize_batch(certainties_all,
+ torch.stack(warps_all, dim=0),
+ target_shape
+ )
+
+ if verbose:
+ print("warps_all_resized.shape:", warps_all_resized.shape)
+ for n, cert in enumerate(certainties_all):
+ fig, ax = plt.subplots()
+ cax = ax.imshow(cert.cpu().numpy(), cmap='viridis')
+ fig.colorbar(cax, ax=ax)
+ ax.set_title("Pixel-wise Confidence")
+ output_dict[f'certainty_{n}'] = fig
+
+ for n, warp in enumerate(warps_all):
+ fig, ax = plt.subplots()
+ cax = ax.imshow(warp.cpu().numpy()[:, :, :3], cmap='viridis')
+ fig.colorbar(cax, ax=ax)
+ ax.set_title("Pixel-wise warp")
+ output_dict[f'warp_resized_{n}'] = fig
+
+ for n, cert in enumerate(certainties_all_resized):
+ fig, ax = plt.subplots()
+ cax = ax.imshow(cert.cpu().numpy(), cmap='viridis')
+ fig.colorbar(cax, ax=ax)
+ ax.set_title("Pixel-wise Confidence resized")
+ output_dict[f'certainty_resized_{n}'] = fig
+
+ for n, warp in enumerate(warps_all_resized):
+ fig, ax = plt.subplots()
+ cax = ax.imshow(warp.cpu().numpy()[:, :, :3], cmap='viridis')
+ fig.colorbar(cax, ax=ax)
+ ax.set_title("Pixel-wise warp resized")
+ output_dict[f'warp_resized_{n}'] = fig
+
+ certainties_max, certainties_max_idcs = torch.max(certainties_all_resized, dim=0)
+ H, W = certainties_max.shape
+
+ warps_max = warps_all_resized[certainties_max_idcs, torch.arange(H).unsqueeze(1), torch.arange(W)]
+
+ imA = viewpoint_cam1.original_image.detach().cpu().numpy().transpose(1, 2, 0)
+ imA = np.clip(imA * 255, 0, 255).astype(np.uint8)
+
+ return certainties_max, warps_max, certainties_max_idcs, imA, imB_compound, certainties_all_resized, warps_all_resized
+
+
+
+def extract_keypoints_and_colors(imA, imB_compound, certainties_max, certainties_max_idcs, matches, roma_model,
+ verbose=False, output_dict={}):
+ """
+ Extracts keypoints and corresponding colors from the source image (imA) and multiple target images (imB_compound).
+
+ Args:
+ imA: Source image as a NumPy array (H_A, W_A, C).
+ imB_compound: List of target images as NumPy arrays [(H_B, W_B, C), ...].
+ certainties_max: Tensor of pixel-wise maximum confidences.
+ certainties_max_idcs: Tensor of pixel-wise indices for the best matches.
+ matches: Matches in normalized coordinates.
+ roma_model: Roma model instance for keypoint operations.
+ verbose: if to show intermediate outputs and visualize results
+
+ Returns:
+ kptsA_np: Keypoints in imA in normalized coordinates.
+ kptsB_np: Keypoints in imB in normalized coordinates.
+ kptsA_color: Colors of keypoints in imA.
+ kptsB_color: Colors of keypoints in imB based on certainties_max_idcs.
+ """
+ H_A, W_A, _ = imA.shape
+ H, W = certainties_max.shape
+
+ # Convert matches to pixel coordinates
+ kptsA, kptsB = roma_model.to_pixel_coordinates(
+ matches, W_A, H_A, H, W # W, H
+ )
+
+ kptsA_np = kptsA.detach().cpu().numpy()
+ kptsB_np = kptsB.detach().cpu().numpy()
+ kptsA_np = kptsA_np[:, [1, 0]]
+
+ if verbose:
+ fig, ax = plt.subplots(figsize=(12, 6))
+ cax = ax.imshow(imA)
+ ax.set_title("Reference image, imA")
+ output_dict[f'reference_image'] = fig
+
+ fig, ax = plt.subplots(figsize=(12, 6))
+ cax = ax.imshow(imB_compound[0])
+ ax.set_title("Image to compare to image, imB_compound")
+ output_dict[f'imB_compound'] = fig
+
+ fig, ax = plt.subplots(figsize=(12, 6))
+ cax = ax.imshow(np.flipud(imA))
+ cax = ax.scatter(kptsA_np[:, 0], H_A - kptsA_np[:, 1], s=.03)
+ ax.set_title("Keypoints in imA")
+ ax.set_xlim(0, W_A)
+ ax.set_ylim(0, H_A)
+ output_dict[f'kptsA'] = fig
+
+ fig, ax = plt.subplots(figsize=(12, 6))
+ cax = ax.imshow(np.flipud(imB_compound[0]))
+ cax = ax.scatter(kptsB_np[:, 0], H_A - kptsB_np[:, 1], s=.03)
+ ax.set_title("Keypoints in imB")
+ ax.set_xlim(0, W_A)
+ ax.set_ylim(0, H_A)
+ output_dict[f'kptsB'] = fig
+
+ # Keypoints are in format (row, column) so the first value is alwain in range [0;height] and second is in range[0;width]
+
+ kptsA_np = kptsA.detach().cpu().numpy()
+ kptsB_np = kptsB.detach().cpu().numpy()
+
+ # Extract colors for keypoints in imA (vectorized)
+ # New experimental version
+ kptsA_x = np.round(kptsA_np[:, 0] / 1.).astype(int)
+ kptsA_y = np.round(kptsA_np[:, 1] / 1.).astype(int)
+ kptsA_color = imA[np.clip(kptsA_x, 0, H - 1), np.clip(kptsA_y, 0, W - 1)]
+
+ # Create a composite image from imB_compound
+ imB_compound_np = np.stack(imB_compound, axis=0)
+ H_B, W_B, _ = imB_compound[0].shape
+
+ # Extract colors for keypoints in imB using certainties_max_idcs
+ imB_np = imB_compound_np[
+ certainties_max_idcs.detach().cpu().numpy(),
+ np.arange(H).reshape(-1, 1),
+ np.arange(W)
+ ]
+
+ if verbose:
+ print("imB_np.shape:", imB_np.shape)
+ print("imB_np:", imB_np)
+ fig, ax = plt.subplots(figsize=(12, 6))
+ cax = ax.imshow(np.flipud(imB_np))
+ cax = ax.scatter(kptsB_np[:, 0], H_A - kptsB_np[:, 1], s=.03)
+ ax.set_title("np.flipud(imB_np[0]")
+ ax.set_xlim(0, W_A)
+ ax.set_ylim(0, H_A)
+ output_dict[f'np.flipud(imB_np[0]'] = fig
+
+
+ kptsB_x = np.round(kptsB_np[:, 0]).astype(int)
+ kptsB_y = np.round(kptsB_np[:, 1]).astype(int)
+
+ certainties_max_idcs_np = certainties_max_idcs.detach().cpu().numpy()
+ kptsB_proj_matrices_idx = certainties_max_idcs_np[np.clip(kptsA_x, 0, H - 1), np.clip(kptsA_y, 0, W - 1)]
+ kptsB_color = imB_compound_np[kptsB_proj_matrices_idx, np.clip(kptsB_y, 0, H - 1), np.clip(kptsB_x, 0, W - 1)]
+
+ # Normalize keypoints in both images
+ kptsA_np[:, 0] = kptsA_np[:, 0] / H * 2.0 - 1.0
+ kptsA_np[:, 1] = kptsA_np[:, 1] / W * 2.0 - 1.0
+ kptsB_np[:, 0] = kptsB_np[:, 0] / W_B * 2.0 - 1.0
+ kptsB_np[:, 1] = kptsB_np[:, 1] / H_B * 2.0 - 1.0
+
+ return kptsA_np[:, [1, 0]], kptsB_np, kptsB_proj_matrices_idx, kptsA_color, kptsB_color
+
+def prepare_tensor(input_array, device):
+ """
+ Converts an input array to a torch tensor, clones it, and detaches it for safe computation.
+ Args:
+ input_array (array-like): The input array to convert.
+ device (str or torch.device): The device to move the tensor to.
+ Returns:
+ torch.Tensor: A detached tensor clone of the input array on the specified device.
+ """
+ if not isinstance(input_array, torch.Tensor):
+ return torch.tensor(input_array, dtype=torch.float32).to(device).clone().detach()
+ return input_array.clone().detach().to(device).to(torch.float32)
+
+def triangulate_points(P1, P2, k1_x, k1_y, k2_x, k2_y, device="cuda"):
+ """
+ Solves for a batch of 3D points given batches of projection matrices and corresponding image points.
+
+ Parameters:
+ - P1, P2: Tensors of projection matrices of size (batch_size, 4, 4) or (4, 4)
+ - k1_x, k1_y: Tensors of shape (batch_size,)
+ - k2_x, k2_y: Tensors of shape (batch_size,)
+
+ Returns:
+ - X: A tensor containing the 3D homogeneous coordinates, shape (batch_size, 4)
+ """
+ EPS = 1e-4
+ # Ensure inputs are tensors
+
+ P1 = prepare_tensor(P1, device)
+ P2 = prepare_tensor(P2, device)
+ k1_x = prepare_tensor(k1_x, device)
+ k1_y = prepare_tensor(k1_y, device)
+ k2_x = prepare_tensor(k2_x, device)
+ k2_y = prepare_tensor(k2_y, device)
+ batch_size = k1_x.shape[0]
+
+ # Expand P1 and P2 if they are not batched
+ if P1.ndim == 2:
+ P1 = P1.unsqueeze(0).expand(batch_size, -1, -1)
+ if P2.ndim == 2:
+ P2 = P2.unsqueeze(0).expand(batch_size, -1, -1)
+
+ # Extract columns from P1 and P2
+ P1_0 = P1[:, :, 0] # Shape: (batch_size, 4)
+ P1_1 = P1[:, :, 1]
+ P1_2 = P1[:, :, 2]
+
+ P2_0 = P2[:, :, 0]
+ P2_1 = P2[:, :, 1]
+ P2_2 = P2[:, :, 2]
+
+ # Reshape kx and ky to (batch_size, 1)
+ k1_x = k1_x.view(-1, 1)
+ k1_y = k1_y.view(-1, 1)
+ k2_x = k2_x.view(-1, 1)
+ k2_y = k2_y.view(-1, 1)
+
+ # Construct the equations for each batch
+ # For camera 1
+ A1 = P1_0 - k1_x * P1_2 # Shape: (batch_size, 4)
+ A2 = P1_1 - k1_y * P1_2
+ # For camera 2
+ A3 = P2_0 - k2_x * P2_2
+ A4 = P2_1 - k2_y * P2_2
+
+ # Stack the equations
+ A = torch.stack([A1, A2, A3, A4], dim=1) # Shape: (batch_size, 4, 4)
+
+ # Right-hand side (constants)
+ b = -A[:, :, 3] # Shape: (batch_size, 4)
+ A_reduced = A[:, :, :3] # Coefficients of x, y, z
+
+ # Solve using torch.linalg.lstsq (supports batching)
+ X_xyz = torch.linalg.lstsq(A_reduced, b.unsqueeze(2)).solution.squeeze(2) # Shape: (batch_size, 3)
+
+ # Append 1 to get homogeneous coordinates
+ ones = torch.ones((batch_size, 1), dtype=torch.float32, device=X_xyz.device)
+ X = torch.cat([X_xyz, ones], dim=1) # Shape: (batch_size, 4)
+
+ # Now compute the errors of projections.
+ seeked_splats_proj1 = (X.unsqueeze(1) @ P1).squeeze(1)
+ seeked_splats_proj1 = seeked_splats_proj1 / (EPS + seeked_splats_proj1[:, [3]])
+ seeked_splats_proj2 = (X.unsqueeze(1) @ P2).squeeze(1)
+ seeked_splats_proj2 = seeked_splats_proj2 / (EPS + seeked_splats_proj2[:, [3]])
+ proj1_target = torch.concat([k1_x, k1_y], dim=1)
+ proj2_target = torch.concat([k2_x, k2_y], dim=1)
+ errors_proj1 = torch.abs(seeked_splats_proj1[:, :2] - proj1_target).sum(1).detach().cpu().numpy()
+ errors_proj2 = torch.abs(seeked_splats_proj2[:, :2] - proj2_target).sum(1).detach().cpu().numpy()
+
+ return X, errors_proj1, errors_proj2
+
+
+
+def select_best_keypoints(
+ NNs_triangulated_points, NNs_errors_proj1, NNs_errors_proj2, device="cuda"):
+ """
+ From all the points fitted to keypoints and corresponding colors from the source image (imA) and multiple target images (imB_compound).
+
+ Args:
+ NNs_triangulated_points: torch tensor with keypoints coordinates (num_nns, num_points, dim). dim can be arbitrary,
+ usually 3 or 4(for homogeneous representation).
+ NNs_errors_proj1: numpy array with projection error of the estimated keypoint on the reference frame (num_nns, num_points).
+ NNs_errors_proj2: numpy array with projection error of the estimated keypoint on the neighbor frame (num_nns, num_points).
+ Returns:
+ selected_keypoints: keypoints with the best score.
+ """
+
+ NNs_errors_proj = np.maximum(NNs_errors_proj1, NNs_errors_proj2)
+
+ # Convert indices to PyTorch tensor
+ indices = torch.from_numpy(np.argmin(NNs_errors_proj, axis=0)).long().to(device)
+
+ # Create index tensor for the second dimension
+ n_indices = torch.arange(NNs_triangulated_points.shape[1]).long().to(device)
+
+ # Use advanced indexing to select elements
+ NNs_triangulated_points_selected = NNs_triangulated_points[indices, n_indices, :] # Shape: [N, k]
+
+ return NNs_triangulated_points_selected, np.min(NNs_errors_proj, axis=0)
+
+
+
+def init_gaussians_with_corr(gaussians, scene, cfg, device, verbose = False, roma_model=None):
+ """
+ For a given input gaussians and a scene we instantiate a RoMa model(change to indoors if necessary) and process scene
+ training frames to extract correspondences. Those are used to initialize gaussians
+ Args:
+ gaussians: object gaussians of the class GaussianModel that we need to enrich with gaussians.
+ scene: object of the Scene class.
+ cfg: configuration. Use init_wC
+ Returns:
+ gaussians: inplace transforms object gaussians of the class GaussianModel.
+
+ """
+ if roma_model is None:
+ if cfg.roma_model == "indoors":
+ roma_model = roma_indoor(device=device)
+ else:
+ roma_model = roma_outdoor(device=device)
+ roma_model.upsample_preds = False
+ roma_model.symmetric = False
+ M = cfg.matches_per_ref
+ upper_thresh = roma_model.sample_thresh
+ scaling_factor = cfg.scaling_factor
+ expansion_factor = 1
+ keypoint_fit_error_tolerance = cfg.proj_err_tolerance
+ visualizations = {}
+ viewpoint_stack = scene.getTrainCameras().copy()
+ NUM_REFERENCE_FRAMES = min(cfg.num_refs, len(viewpoint_stack))
+ NUM_NNS_PER_REFERENCE = min(cfg.nns_per_ref , len(viewpoint_stack))
+ # Select cameras using K-means
+ viewpoint_cam_all = torch.stack([x.world_view_transform.flatten() for x in viewpoint_stack], axis=0)
+
+ selected_indices = select_cameras_kmeans(cameras=viewpoint_cam_all.detach().cpu().numpy(), K=NUM_REFERENCE_FRAMES)
+ selected_indices = sorted(selected_indices)
+
+
+ # Find the k-closest vectors for each vector
+ viewpoint_cam_all = torch.stack([x.world_view_transform.flatten() for x in viewpoint_stack], axis=0)
+ closest_indices = k_closest_vectors(viewpoint_cam_all, NUM_NNS_PER_REFERENCE)
+ if verbose: print("Indices of k-closest vectors for each vector:\n", closest_indices)
+
+ closest_indices_selected = closest_indices[:, :].detach().cpu().numpy()
+
+ all_new_xyz = []
+ all_new_features_dc = []
+ all_new_features_rest = []
+ all_new_opacities = []
+ all_new_scaling = []
+ all_new_rotation = []
+
+ # Run roma_model.match once to kinda initialize the model
+ with torch.no_grad():
+ viewpoint_cam1 = viewpoint_stack[0]
+ viewpoint_cam2 = viewpoint_stack[1]
+ imA = viewpoint_cam1.original_image.detach().cpu().numpy().transpose(1, 2, 0)
+ imB = viewpoint_cam2.original_image.detach().cpu().numpy().transpose(1, 2, 0)
+ imA = Image.fromarray(np.clip(imA * 255, 0, 255).astype(np.uint8))
+ imB = Image.fromarray(np.clip(imB * 255, 0, 255).astype(np.uint8))
+ warp, certainty_warp = roma_model.match(imA, imB, device=device)
+ print("Once run full roma_model.match warp.shape:", warp.shape)
+ print("Once run full roma_model.match certainty_warp.shape:", certainty_warp.shape)
+ del warp, certainty_warp
+ torch.cuda.empty_cache()
+
+ for source_idx in tqdm(sorted(selected_indices)):
+ # 1. Compute keypoints and warping for all the neigboring views
+ with torch.no_grad():
+ # Call the aggregation function to get imA and imB_compound
+ certainties_max, warps_max, certainties_max_idcs, imA, imB_compound, certainties_all, warps_all = aggregate_confidences_and_warps(
+ viewpoint_stack=viewpoint_stack,
+ closest_indices=closest_indices_selected,
+ roma_model=roma_model,
+ source_idx=source_idx,
+ verbose=verbose, output_dict=visualizations
+ )
+
+
+ # Triangulate keypoints
+ with torch.no_grad():
+ matches = warps_max
+ certainty = certainties_max
+ certainty = certainty.clone()
+ certainty[certainty > upper_thresh] = 1
+ matches, certainty = (
+ matches.reshape(-1, 4),
+ certainty.reshape(-1),
+ )
+
+ # Select based on certainty elements with high confidence. These are basically all of
+ # kptsA_np.
+ good_samples = torch.multinomial(certainty,
+ num_samples=min(expansion_factor * M, len(certainty)),
+ replacement=False)
+
+ certainties_max, warps_max, certainties_max_idcs, imA, imB_compound, certainties_all, warps_all
+ reference_image_dict = {
+ "ref_image": imA,
+ "NNs_images": imB_compound,
+ "certainties_all": certainties_all,
+ "warps_all": warps_all,
+ "triangulated_points": [],
+ "triangulated_points_errors_proj1": [],
+ "triangulated_points_errors_proj2": []
+
+ }
+ with torch.no_grad():
+ for NN_idx in tqdm(range(len(warps_all))):
+ matches_NN = warps_all[NN_idx].reshape(-1, 4)[good_samples]
+
+ # Extract keypoints and colors
+ kptsA_np, kptsB_np, kptsB_proj_matrices_idcs, kptsA_color, kptsB_color = extract_keypoints_and_colors(
+ imA, imB_compound, certainties_max, certainties_max_idcs, matches_NN, roma_model
+ )
+
+ proj_matrices_A = viewpoint_stack[source_idx].full_proj_transform
+ proj_matrices_B = viewpoint_stack[closest_indices_selected[source_idx, NN_idx]].full_proj_transform
+ triangulated_points, triangulated_points_errors_proj1, triangulated_points_errors_proj2 = triangulate_points(
+ P1=torch.stack([proj_matrices_A] * M, axis=0),
+ P2=torch.stack([proj_matrices_B] * M, axis=0),
+ k1_x=kptsA_np[:M, 0], k1_y=kptsA_np[:M, 1],
+ k2_x=kptsB_np[:M, 0], k2_y=kptsB_np[:M, 1])
+
+ reference_image_dict["triangulated_points"].append(triangulated_points)
+ reference_image_dict["triangulated_points_errors_proj1"].append(triangulated_points_errors_proj1)
+ reference_image_dict["triangulated_points_errors_proj2"].append(triangulated_points_errors_proj2)
+
+ with torch.no_grad():
+ NNs_triangulated_points_selected, NNs_triangulated_points_selected_proj_errors = select_best_keypoints(
+ NNs_triangulated_points=torch.stack(reference_image_dict["triangulated_points"], dim=0),
+ NNs_errors_proj1=np.stack(reference_image_dict["triangulated_points_errors_proj1"], axis=0),
+ NNs_errors_proj2=np.stack(reference_image_dict["triangulated_points_errors_proj2"], axis=0))
+
+ # 4. Save as gaussians
+ viewpoint_cam1 = viewpoint_stack[source_idx]
+ N = len(NNs_triangulated_points_selected)
+ with torch.no_grad():
+ new_xyz = NNs_triangulated_points_selected[:, :-1]
+ all_new_xyz.append(new_xyz) # seeked_splats
+ all_new_features_dc.append(RGB2SH(torch.tensor(kptsA_color.astype(np.float32) / 255.)).unsqueeze(1))
+ all_new_features_rest.append(torch.stack([gaussians._features_rest[-1].clone().detach() * 0.] * N, dim=0))
+ # new version that sets points with large error invisible
+ # TODO: remove those points instead. However it doesn't affect the performance.
+ mask_bad_points = torch.tensor(
+ NNs_triangulated_points_selected_proj_errors > keypoint_fit_error_tolerance,
+ dtype=torch.float32).unsqueeze(1).to(device)
+ all_new_opacities.append(torch.stack([gaussians._opacity[-1].clone().detach()] * N, dim=0) * 0. - mask_bad_points * (1e1))
+
+ dist_points_to_cam1 = torch.linalg.norm(viewpoint_cam1.camera_center.clone().detach() - new_xyz,
+ dim=1, ord=2)
+ #all_new_scaling.append(torch.log(((dist_points_to_cam1) / 1. * scaling_factor).unsqueeze(1).repeat(1, 3)))
+ all_new_scaling.append(gaussians.scaling_inverse_activation((dist_points_to_cam1 * scaling_factor).unsqueeze(1).repeat(1, 3)))
+ all_new_rotation.append(torch.stack([gaussians._rotation[-1].clone().detach()] * N, dim=0))
+
+ all_new_xyz = torch.cat(all_new_xyz, dim=0)
+ all_new_features_dc = torch.cat(all_new_features_dc, dim=0)
+ new_tmp_radii = torch.zeros(all_new_xyz.shape[0])
+ prune_mask = torch.ones(all_new_xyz.shape[0], dtype=torch.bool)
+
+ gaussians.densification_postfix(all_new_xyz[prune_mask].to(device),
+ all_new_features_dc[prune_mask].to(device),
+ torch.cat(all_new_features_rest, dim=0)[prune_mask].to(device),
+ torch.cat(all_new_opacities, dim=0)[prune_mask].to(device),
+ torch.cat(all_new_scaling, dim=0)[prune_mask].to(device),
+ torch.cat(all_new_rotation, dim=0)[prune_mask].to(device),
+ new_tmp_radii[prune_mask].to(device))
+
+ return viewpoint_stack, closest_indices_selected, visualizations
+
+
+
+def extract_keypoints_and_colors_single(imA, imB, matches, roma_model, verbose=False, output_dict={}):
+ """
+ Extracts keypoints and corresponding colors from a source image (imA) and a single target image (imB).
+
+ Args:
+ imA: Source image as a NumPy array (H_A, W_A, C).
+ imB: Target image as a NumPy array (H_B, W_B, C).
+ matches: Matches in normalized coordinates (torch.Tensor).
+ roma_model: Roma model instance for keypoint operations.
+ verbose: If True, outputs intermediate visualizations.
+ Returns:
+ kptsA_np: Keypoints in imA (normalized).
+ kptsB_np: Keypoints in imB (normalized).
+ kptsA_color: Colors of keypoints in imA.
+ kptsB_color: Colors of keypoints in imB.
+ """
+ H_A, W_A, _ = imA.shape
+ H_B, W_B, _ = imB.shape
+
+ # Convert matches to pixel coordinates
+ # Matches format: (B, 4) = (x1_norm, y1_norm, x2_norm, y2_norm)
+ kptsA = matches[:, :2] # [N, 2]
+ kptsB = matches[:, 2:] # [N, 2]
+
+ # Scale normalized coordinates [-1,1] to pixel coordinates
+ kptsA_pix = torch.zeros_like(kptsA)
+ kptsB_pix = torch.zeros_like(kptsB)
+
+ # Important! [Normalized to pixel space]
+ kptsA_pix[:, 0] = (kptsA[:, 0] + 1) * (W_A - 1) / 2
+ kptsA_pix[:, 1] = (kptsA[:, 1] + 1) * (H_A - 1) / 2
+
+ kptsB_pix[:, 0] = (kptsB[:, 0] + 1) * (W_B - 1) / 2
+ kptsB_pix[:, 1] = (kptsB[:, 1] + 1) * (H_B - 1) / 2
+
+ kptsA_np = kptsA_pix.detach().cpu().numpy()
+ kptsB_np = kptsB_pix.detach().cpu().numpy()
+
+ # Extract colors
+ kptsA_x = np.round(kptsA_np[:, 0]).astype(int)
+ kptsA_y = np.round(kptsA_np[:, 1]).astype(int)
+ kptsB_x = np.round(kptsB_np[:, 0]).astype(int)
+ kptsB_y = np.round(kptsB_np[:, 1]).astype(int)
+
+ kptsA_color = imA[np.clip(kptsA_y, 0, H_A-1), np.clip(kptsA_x, 0, W_A-1)]
+ kptsB_color = imB[np.clip(kptsB_y, 0, H_B-1), np.clip(kptsB_x, 0, W_B-1)]
+
+ # Normalize keypoints into [-1, 1] for downstream triangulation
+ kptsA_np_norm = np.zeros_like(kptsA_np)
+ kptsB_np_norm = np.zeros_like(kptsB_np)
+
+ kptsA_np_norm[:, 0] = kptsA_np[:, 0] / (W_A - 1) * 2.0 - 1.0
+ kptsA_np_norm[:, 1] = kptsA_np[:, 1] / (H_A - 1) * 2.0 - 1.0
+
+ kptsB_np_norm[:, 0] = kptsB_np[:, 0] / (W_B - 1) * 2.0 - 1.0
+ kptsB_np_norm[:, 1] = kptsB_np[:, 1] / (H_B - 1) * 2.0 - 1.0
+
+ return kptsA_np_norm, kptsB_np_norm, kptsA_color, kptsB_color
+
+
+
+def init_gaussians_with_corr_fast(gaussians, scene, cfg, device, verbose=False, roma_model=None):
+ timings = defaultdict(list)
+
+ if roma_model is None:
+ if cfg.roma_model == "indoors":
+ roma_model = roma_indoor(device=device)
+ else:
+ roma_model = roma_outdoor(device=device)
+ roma_model.upsample_preds = False
+ roma_model.symmetric = False
+
+ M = cfg.matches_per_ref
+ upper_thresh = roma_model.sample_thresh
+ scaling_factor = cfg.scaling_factor
+ expansion_factor = 1
+ keypoint_fit_error_tolerance = cfg.proj_err_tolerance
+ visualizations = {}
+ viewpoint_stack = scene.getTrainCameras().copy()
+ NUM_REFERENCE_FRAMES = min(cfg.num_refs, len(viewpoint_stack))
+ NUM_NNS_PER_REFERENCE = 1 # Only ONE neighbor now!
+
+ viewpoint_cam_all = torch.stack([x.world_view_transform.flatten() for x in viewpoint_stack], axis=0)
+
+ selected_indices = select_cameras_kmeans(cameras=viewpoint_cam_all.detach().cpu().numpy(), K=NUM_REFERENCE_FRAMES)
+ selected_indices = sorted(selected_indices)
+
+ viewpoint_cam_all = torch.stack([x.world_view_transform.flatten() for x in viewpoint_stack], axis=0)
+ closest_indices = k_closest_vectors(viewpoint_cam_all, NUM_NNS_PER_REFERENCE)
+ closest_indices_selected = closest_indices[:, :].detach().cpu().numpy()
+
+ all_new_xyz = []
+ all_new_features_dc = []
+ all_new_features_rest = []
+ all_new_opacities = []
+ all_new_scaling = []
+ all_new_rotation = []
+
+ # Dummy first pass to initialize model
+ with torch.no_grad():
+ viewpoint_cam1 = viewpoint_stack[0]
+ viewpoint_cam2 = viewpoint_stack[1]
+ imA = viewpoint_cam1.original_image.detach().cpu().numpy().transpose(1, 2, 0)
+ imB = viewpoint_cam2.original_image.detach().cpu().numpy().transpose(1, 2, 0)
+ imA = Image.fromarray(np.clip(imA * 255, 0, 255).astype(np.uint8))
+ imB = Image.fromarray(np.clip(imB * 255, 0, 255).astype(np.uint8))
+ warp, certainty_warp = roma_model.match(imA, imB, device=device)
+ del warp, certainty_warp
+ torch.cuda.empty_cache()
+
+ # Main Loop over source_idx
+ for source_idx in tqdm(sorted(selected_indices), desc="Profiling source frames"):
+
+ # =================== Step 1: Compute Warp and Certainty ===================
+ start = time.time()
+ viewpoint_cam1 = viewpoint_stack[source_idx]
+ NNs=closest_indices_selected.shape[1]
+ viewpoint_cam2 = viewpoint_stack[closest_indices_selected[source_idx, np.random.randint(NNs)]]
+ imA = viewpoint_cam1.original_image.detach().cpu().numpy().transpose(1, 2, 0)
+ imB = viewpoint_cam2.original_image.detach().cpu().numpy().transpose(1, 2, 0)
+ imA = Image.fromarray(np.clip(imA * 255, 0, 255).astype(np.uint8))
+ imB = Image.fromarray(np.clip(imB * 255, 0, 255).astype(np.uint8))
+ warp, certainty_warp = roma_model.match(imA, imB, device=device)
+
+ certainties_max = certainty_warp # New manual sampling
+ timings['aggregation_warp_certainty'].append(time.time() - start)
+
+ # =================== Step 2: Good Samples Selection ===================
+ start = time.time()
+ certainty = certainties_max.reshape(-1).clone()
+ certainty[certainty > upper_thresh] = 1
+ good_samples = torch.multinomial(certainty, num_samples=min(expansion_factor * M, len(certainty)), replacement=False)
+ timings['good_samples_selection'].append(time.time() - start)
+
+ # =================== Step 3: Triangulate Keypoints ===================
+ reference_image_dict = {
+ "triangulated_points": [],
+ "triangulated_points_errors_proj1": [],
+ "triangulated_points_errors_proj2": []
+ }
+
+ start = time.time()
+ matches_NN = warp.reshape(-1, 4)[good_samples]
+
+ # Convert matches to pixel coordinates
+ kptsA_np, kptsB_np, kptsA_color, kptsB_color = extract_keypoints_and_colors_single(
+ np.array(imA).astype(np.uint8),
+ np.array(imB).astype(np.uint8),
+ matches_NN,
+ roma_model
+ )
+
+ proj_matrices_A = viewpoint_stack[source_idx].full_proj_transform
+ proj_matrices_B = viewpoint_stack[closest_indices_selected[source_idx, 0]].full_proj_transform
+
+ triangulated_points, triangulated_points_errors_proj1, triangulated_points_errors_proj2 = triangulate_points(
+ P1=torch.stack([proj_matrices_A] * M, axis=0),
+ P2=torch.stack([proj_matrices_B] * M, axis=0),
+ k1_x=kptsA_np[:M, 0], k1_y=kptsA_np[:M, 1],
+ k2_x=kptsB_np[:M, 0], k2_y=kptsB_np[:M, 1])
+
+ reference_image_dict["triangulated_points"].append(triangulated_points)
+ reference_image_dict["triangulated_points_errors_proj1"].append(triangulated_points_errors_proj1)
+ reference_image_dict["triangulated_points_errors_proj2"].append(triangulated_points_errors_proj2)
+ timings['triangulation_per_NN'].append(time.time() - start)
+
+ # =================== Step 4: Select Best Triangulated Points ===================
+ start = time.time()
+ NNs_triangulated_points_selected, NNs_triangulated_points_selected_proj_errors = select_best_keypoints(
+ NNs_triangulated_points=torch.stack(reference_image_dict["triangulated_points"], dim=0),
+ NNs_errors_proj1=np.stack(reference_image_dict["triangulated_points_errors_proj1"], axis=0),
+ NNs_errors_proj2=np.stack(reference_image_dict["triangulated_points_errors_proj2"], axis=0))
+ timings['select_best_keypoints'].append(time.time() - start)
+
+ # =================== Step 5: Create New Gaussians ===================
+ start = time.time()
+ viewpoint_cam1 = viewpoint_stack[source_idx]
+ N = len(NNs_triangulated_points_selected)
+ new_xyz = NNs_triangulated_points_selected[:, :-1]
+ all_new_xyz.append(new_xyz)
+ all_new_features_dc.append(RGB2SH(torch.tensor(kptsA_color.astype(np.float32) / 255.)).unsqueeze(1))
+ all_new_features_rest.append(torch.stack([gaussians._features_rest[-1].clone().detach() * 0.] * N, dim=0))
+
+ mask_bad_points = torch.tensor(
+ NNs_triangulated_points_selected_proj_errors > keypoint_fit_error_tolerance,
+ dtype=torch.float32).unsqueeze(1).to(device)
+
+ all_new_opacities.append(torch.stack([gaussians._opacity[-1].clone().detach()] * N, dim=0) * 0. - mask_bad_points * (1e1))
+
+ dist_points_to_cam1 = torch.linalg.norm(viewpoint_cam1.camera_center.clone().detach() - new_xyz, dim=1, ord=2)
+ all_new_scaling.append(gaussians.scaling_inverse_activation((dist_points_to_cam1 * scaling_factor).unsqueeze(1).repeat(1, 3)))
+ all_new_rotation.append(torch.stack([gaussians._rotation[-1].clone().detach()] * N, dim=0))
+ timings['save_gaussians'].append(time.time() - start)
+
+ # =================== Final Densification Postfix ===================
+ start = time.time()
+ all_new_xyz = torch.cat(all_new_xyz, dim=0)
+ all_new_features_dc = torch.cat(all_new_features_dc, dim=0)
+ new_tmp_radii = torch.zeros(all_new_xyz.shape[0])
+ prune_mask = torch.ones(all_new_xyz.shape[0], dtype=torch.bool)
+
+ gaussians.densification_postfix(
+ all_new_xyz[prune_mask].to(device),
+ all_new_features_dc[prune_mask].to(device),
+ torch.cat(all_new_features_rest, dim=0)[prune_mask].to(device),
+ torch.cat(all_new_opacities, dim=0)[prune_mask].to(device),
+ torch.cat(all_new_scaling, dim=0)[prune_mask].to(device),
+ torch.cat(all_new_rotation, dim=0)[prune_mask].to(device),
+ new_tmp_radii[prune_mask].to(device)
+ )
+ timings['final_densification_postfix'].append(time.time() - start)
+
+ # =================== Print Profiling Results ===================
+ print("\n=== Profiling Summary (average per frame) ===")
+ for key, times in timings.items():
+ print(f"{key:35s}: {sum(times) / len(times):.4f} sec (total {sum(times):.2f} sec)")
+
+ return viewpoint_stack, closest_indices_selected, visualizations
\ No newline at end of file
diff --git a/source/data_utils.py b/source/data_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..df5a566a9b482e5c5f05ef02b24e325d56f8bcef
--- /dev/null
+++ b/source/data_utils.py
@@ -0,0 +1,28 @@
+def scene_cameras_train_test_split(scene, verbose=False):
+ """
+ Iterate over resolutions in the scene. For each resolution check if this resolution has test_cameras
+ if it doesn't then extract every 8th camera from the train and put it to the test set. This follows the
+ evaluation protocol suggested by Kerbl et al. in the seminal work on 3DGS. All changes to the input
+ object scene are inplace changes.
+ :param scene: Scene Class object from the gaussian-splatting.scene module
+ :param verbose: Print initial and final stage of the function
+ :return: None
+
+ """
+ if verbose: print("Preparing train and test sets split...")
+ for resolution in scene.train_cameras.keys():
+ if len(scene.test_cameras[resolution]) == 0:
+ if verbose:
+ print(f"Found no test_cameras for resolution {resolution}. Move every 8th camera out ouf total "+\
+ f"{len(scene.train_cameras[resolution])} train cameras to the test set now")
+ N = len(scene.train_cameras[resolution])
+ scene.test_cameras[resolution] = [scene.train_cameras[resolution][idx] for idx in range(0, N)
+ if idx % 8 == 0]
+ scene.train_cameras[resolution] = [scene.train_cameras[resolution][idx] for idx in range(0, N)
+ if idx % 8 != 0]
+ if verbose:
+ print(f"Done. Now train and test sets contain each {len(scene.train_cameras[resolution])} and " + \
+ f"{len(scene.test_cameras[resolution])} cameras respectively.")
+
+
+ return
diff --git a/source/losses.py b/source/losses.py
new file mode 100644
index 0000000000000000000000000000000000000000..dcaf4d0dddc779cb734fc1cbc84c4a3950a1d98c
--- /dev/null
+++ b/source/losses.py
@@ -0,0 +1,100 @@
+# Code is copied from the gaussian-splatting/utils/loss_utils.py
+
+import torch
+import torch.nn.functional as F
+from torch.autograd import Variable
+from math import exp
+
+def l1_loss(network_output, gt, mean=True):
+ return torch.abs((network_output - gt)).mean() if mean else torch.abs((network_output - gt))
+
+def l2_loss(network_output, gt):
+ return ((network_output - gt) ** 2).mean()
+
+def gaussian(window_size, sigma):
+ gauss = torch.Tensor([exp(-(x - window_size // 2) ** 2 / float(2 * sigma ** 2)) for x in range(window_size)])
+ return gauss / gauss.sum()
+
+def create_window(window_size, channel):
+ _1D_window = gaussian(window_size, 1.5).unsqueeze(1)
+ _2D_window = _1D_window.mm(_1D_window.t()).float().unsqueeze(0).unsqueeze(0)
+ window = Variable(_2D_window.expand(channel, 1, window_size, window_size).contiguous())
+ return window
+
+def ssim(img1, img2, window_size=11, size_average=True, mask = None):
+ channel = img1.size(-3)
+ window = create_window(window_size, channel)
+
+ if img1.is_cuda:
+ window = window.cuda(img1.get_device())
+ window = window.type_as(img1)
+
+ return _ssim(img1, img2, window, window_size, channel, size_average, mask)
+
+def _ssim(img1, img2, window, window_size, channel, size_average=True, mask = None):
+ mu1 = F.conv2d(img1, window, padding=window_size // 2, groups=channel)
+ mu2 = F.conv2d(img2, window, padding=window_size // 2, groups=channel)
+
+ mu1_sq = mu1.pow(2)
+ mu2_sq = mu2.pow(2)
+ mu1_mu2 = mu1 * mu2
+
+ sigma1_sq = F.conv2d(img1 * img1, window, padding=window_size // 2, groups=channel) - mu1_sq
+ sigma2_sq = F.conv2d(img2 * img2, window, padding=window_size // 2, groups=channel) - mu2_sq
+ sigma12 = F.conv2d(img1 * img2, window, padding=window_size // 2, groups=channel) - mu1_mu2
+
+ C1 = 0.01 ** 2
+ C2 = 0.03 ** 2
+
+ ssim_map = ((2 * mu1_mu2 + C1) * (2 * sigma12 + C2)) / ((mu1_sq + mu2_sq + C1) * (sigma1_sq + sigma2_sq + C2))
+
+ if mask is not None:
+ ssim_map = ssim_map * mask
+
+ if size_average:
+ return ssim_map.mean()
+ else:
+ return ssim_map.mean(1).mean(1).mean(1)
+
+
+def mse(img1, img2):
+ return (((img1 - img2)) ** 2).view(img1.shape[0], -1).mean(1, keepdim=True)
+
+def psnr(img1, img2):
+ """
+ Computes the Peak Signal-to-Noise Ratio (PSNR) between two single images. NOT BATCHED!
+ Args:
+ img1 (torch.Tensor): The first image tensor, with pixel values scaled between 0 and 1.
+ Shape should be (channels, height, width).
+ img2 (torch.Tensor): The second image tensor with the same shape as img1, used for comparison.
+
+ Returns:
+ torch.Tensor: A scalar tensor containing the PSNR value in decibels (dB).
+ """
+ mse = (((img1 - img2)) ** 2).view(img1.shape[0], -1).mean(1, keepdim=True)
+ return 20 * torch.log10(1.0 / torch.sqrt(mse))
+
+
+def tv_loss(image):
+ """
+ Computes the total variation (TV) loss for an image of shape [3, H, W].
+
+ Args:
+ image (torch.Tensor): Input image of shape [3, H, W]
+
+ Returns:
+ torch.Tensor: Scalar value representing the total variation loss.
+ """
+ # Ensure the image has the correct dimensions
+ assert image.ndim == 3 and image.shape[0] == 3, "Input must be of shape [3, H, W]"
+
+ # Compute the difference between adjacent pixels in the x-direction (width)
+ diff_x = torch.abs(image[:, :, 1:] - image[:, :, :-1])
+
+ # Compute the difference between adjacent pixels in the y-direction (height)
+ diff_y = torch.abs(image[:, 1:, :] - image[:, :-1, :])
+
+ # Sum the total variation in both directions
+ tv_loss_value = torch.mean(diff_x) + torch.mean(diff_y)
+
+ return tv_loss_value
\ No newline at end of file
diff --git a/source/networks.py b/source/networks.py
new file mode 100644
index 0000000000000000000000000000000000000000..e8d0477cd95ccf9439ffe11de7663dfd78a53181
--- /dev/null
+++ b/source/networks.py
@@ -0,0 +1,48 @@
+import torch
+
+import sys
+sys.path.append('./submodules/gaussian-splatting/')
+
+from random import randint
+from scene import Scene, GaussianModel
+from gaussian_renderer import render
+from source.data_utils import scene_cameras_train_test_split
+
+class Warper3DGS(torch.nn.Module):
+ def __init__(self, sh_degree, opt, pipe, dataset, viewpoint_stack, verbose,
+ do_train_test_split=True):
+ super(Warper3DGS, self).__init__()
+ """
+ Init Warper using all the objects necessary for rendering gaussian splats.
+ Here we merely link class objects to the objects instantiated outsided the class.
+ """
+ self.gaussians = GaussianModel(sh_degree)
+ self.gaussians.tmp_radii = torch.zeros((self.gaussians.get_xyz.shape[0]), device="cuda")
+ self.render = render
+ self.gs_config_opt = opt
+ bg_color = [1, 1, 1] if dataset.white_background else [0, 0, 0]
+ self.bg = torch.tensor(bg_color, dtype=torch.float32, device="cuda")
+ self.pipe = pipe
+ self.scene = Scene(dataset, self.gaussians, shuffle=False)
+ if do_train_test_split:
+ scene_cameras_train_test_split(self.scene, verbose=verbose)
+
+ self.gaussians.training_setup(opt)
+ self.viewpoint_stack = viewpoint_stack
+ if not self.viewpoint_stack:
+ self.viewpoint_stack = self.scene.getTrainCameras().copy()
+
+ def forward(self, viewpoint_cam=None):
+ """
+ For a provided camera viewpoint_cam we render gaussians from this viewpoint.
+ If no camera provided then we use the self.viewpoint_stack (list of cameras).
+ If the latter is empty we reinitialize it using the self.scene object.
+ """
+ if not viewpoint_cam:
+ if not self.viewpoint_stack:
+ self.viewpoint_stack = self.scene.getTrainCameras().copy()
+ viewpoint_cam = self.viewpoint_stack[randint(0, len(self.viewpoint_stack) - 1)]
+
+ render_pkg = self.render(viewpoint_cam, self.gaussians, self.pipe, self.bg)
+ return render_pkg
+
diff --git a/source/timer.py b/source/timer.py
new file mode 100755
index 0000000000000000000000000000000000000000..c01ff93c1bdc94a07f1c20a07fb0a983dc496e62
--- /dev/null
+++ b/source/timer.py
@@ -0,0 +1,24 @@
+import time
+class Timer:
+ def __init__(self):
+ self.start_time = None
+ self.elapsed = 0
+ self.paused = False
+
+ def start(self):
+ if self.start_time is None:
+ self.start_time = time.time()
+ elif self.paused:
+ self.start_time = time.time() - self.elapsed
+ self.paused = False
+
+ def pause(self):
+ if not self.paused:
+ self.elapsed = time.time() - self.start_time
+ self.paused = True
+
+ def get_elapsed_time(self):
+ if self.paused:
+ return self.elapsed
+ else:
+ return time.time() - self.start_time
\ No newline at end of file
diff --git a/source/trainer.py b/source/trainer.py
new file mode 100644
index 0000000000000000000000000000000000000000..7fbd28afc946b144e15f240f432381f21707b7f2
--- /dev/null
+++ b/source/trainer.py
@@ -0,0 +1,265 @@
+import torch
+from random import randint
+from tqdm.rich import trange
+from tqdm import tqdm as tqdm
+from source.networks import Warper3DGS
+import wandb
+import sys
+
+sys.path.append('./submodules/gaussian-splatting/')
+import lpips
+from source.losses import ssim, l1_loss, psnr
+from rich.console import Console
+from rich.theme import Theme
+
+custom_theme = Theme({
+ "info": "dim cyan",
+ "warning": "magenta",
+ "danger": "bold red"
+})
+
+from source.corr_init import init_gaussians_with_corr, init_gaussians_with_corr_fast
+from source.utils_aux import log_samples
+
+from source.timer import Timer
+
+class EDGSTrainer:
+ def __init__(self,
+ GS: Warper3DGS,
+ training_config,
+ dataset_white_background=False,
+ device=torch.device('cuda'),
+ log_wandb=True,
+ ):
+ self.GS = GS
+ self.scene = GS.scene
+ self.viewpoint_stack = GS.viewpoint_stack
+ self.gaussians = GS.gaussians
+
+ self.training_config = training_config
+ self.GS_optimizer = GS.gaussians.optimizer
+ self.dataset_white_background = dataset_white_background
+
+ self.training_step = 1
+ self.gs_step = 0
+ self.CONSOLE = Console(width=120, theme=custom_theme)
+ self.saving_iterations = training_config.save_iterations
+ self.evaluate_iterations = None
+ self.batch_size = training_config.batch_size
+ self.ema_loss_for_log = 0.0
+
+ # Logs in the format {step:{"loss1":loss1_value, "loss2":loss2_value}}
+ self.logs_losses = {}
+ self.lpips = lpips.LPIPS(net='vgg').to(device)
+ self.device = device
+ self.timer = Timer()
+ self.log_wandb = log_wandb
+
+ def load_checkpoints(self, load_cfg):
+ # Load 3DGS checkpoint
+ if load_cfg.gs:
+ self.gs.gaussians.restore(
+ torch.load(f"{load_cfg.gs}/chkpnt{load_cfg.gs_step}.pth")[0],
+ self.training_config)
+ self.GS_optimizer = self.GS.gaussians.optimizer
+ self.CONSOLE.print(f"3DGS loaded from checkpoint for iteration {load_cfg.gs_step}",
+ style="info")
+ self.training_step += load_cfg.gs_step
+ self.gs_step += load_cfg.gs_step
+
+ def train(self, train_cfg):
+ # 3DGS training
+ self.CONSOLE.print("Train 3DGS for {} iterations".format(train_cfg.gs_epochs), style="info")
+ with trange(self.training_step, self.training_step + train_cfg.gs_epochs, desc="[green]Train gaussians") as progress_bar:
+ for self.training_step in progress_bar:
+ radii = self.train_step_gs(max_lr=train_cfg.max_lr, no_densify=train_cfg.no_densify)
+ with torch.no_grad():
+ if train_cfg.no_densify:
+ self.prune(radii)
+ else:
+ self.densify_and_prune(radii)
+ if train_cfg.reduce_opacity:
+ # Slightly reduce opacity every few steps:
+ if self.gs_step < self.training_config.densify_until_iter and self.gs_step % 10 == 0:
+ opacities_new = torch.log(torch.exp(self.GS.gaussians._opacity.data) * 0.99)
+ self.GS.gaussians._opacity.data = opacities_new
+ self.timer.pause()
+ # Progress bar
+ if self.training_step % 10 == 0:
+ progress_bar.set_postfix({"[red]Loss": f"{self.ema_loss_for_log:.{7}f}"}, refresh=True)
+ # Log and save
+ if self.training_step in self.saving_iterations:
+ self.save_model()
+ if self.evaluate_iterations is not None:
+ if self.training_step in self.evaluate_iterations:
+ self.evaluate()
+ else:
+ if (self.training_step <= 3000 and self.training_step % 500 == 0) or \
+ (self.training_step > 3000 and self.training_step % 1000 == 228) :
+ self.evaluate()
+
+ self.timer.start()
+
+
+ def evaluate(self):
+ torch.cuda.empty_cache()
+ log_gen_images, log_real_images = [], []
+ validation_configs = ({'name': 'test', 'cameras': self.scene.getTestCameras(), 'cam_idx': self.training_config.TEST_CAM_IDX_TO_LOG},
+ {'name': 'train',
+ 'cameras': [self.scene.getTrainCameras()[idx % len(self.scene.getTrainCameras())] for idx in
+ range(0, 150, 5)], 'cam_idx': 10})
+ if self.log_wandb:
+ wandb.log({f"Number of Gaussians": len(self.GS.gaussians._xyz)}, step=self.training_step)
+ for config in validation_configs:
+ if config['cameras'] and len(config['cameras']) > 0:
+ l1_test = 0.0
+ psnr_test = 0.0
+ ssim_test = 0.0
+ lpips_splat_test = 0.0
+ for idx, viewpoint in enumerate(config['cameras']):
+ image = torch.clamp(self.GS(viewpoint)["render"], 0.0, 1.0)
+ gt_image = torch.clamp(viewpoint.original_image.to(self.device), 0.0, 1.0)
+ l1_test += l1_loss(image, gt_image).double()
+ psnr_test += psnr(image.unsqueeze(0), gt_image.unsqueeze(0)).double()
+ ssim_test += ssim(image, gt_image).double()
+ lpips_splat_test += self.lpips(image, gt_image).detach().double()
+ if idx in [config['cam_idx']]:
+ log_gen_images.append(image)
+ log_real_images.append(gt_image)
+ psnr_test /= len(config['cameras'])
+ l1_test /= len(config['cameras'])
+ ssim_test /= len(config['cameras'])
+ lpips_splat_test /= len(config['cameras'])
+ if self.log_wandb:
+ wandb.log({f"{config['name']}/L1": l1_test.item(), f"{config['name']}/PSNR": psnr_test.item(), \
+ f"{config['name']}/SSIM": ssim_test.item(), f"{config['name']}/LPIPS_splat": lpips_splat_test.item()}, step = self.training_step)
+ self.CONSOLE.print("\n[ITER {}], #{} gaussians, Evaluating {}: L1={:.6f}, PSNR={:.6f}, SSIM={:.6f}, LPIPS_splat={:.6f} ".format(
+ self.training_step, len(self.GS.gaussians._xyz), config['name'], l1_test.item(), psnr_test.item(), ssim_test.item(), lpips_splat_test.item()), style="info")
+ if self.log_wandb:
+ with torch.no_grad():
+ log_samples(torch.stack((log_real_images[0],log_gen_images[0])) , [], self.training_step, caption="Real and Generated Samples")
+ wandb.log({"time": self.timer.get_elapsed_time()}, step=self.training_step)
+ torch.cuda.empty_cache()
+
+ def train_step_gs(self, max_lr = False, no_densify = False):
+ self.gs_step += 1
+ if max_lr:
+ self.GS.gaussians.update_learning_rate(max(self.gs_step, 8_000))
+ else:
+ self.GS.gaussians.update_learning_rate(self.gs_step)
+ # Every 1000 its we increase the levels of SH up to a maximum degree
+ if self.gs_step % 1000 == 0:
+ self.GS.gaussians.oneupSHdegree()
+
+ # Pick a random Camera
+ if not self.viewpoint_stack:
+ self.viewpoint_stack = self.scene.getTrainCameras().copy()
+ viewpoint_cam = self.viewpoint_stack.pop(randint(0, len(self.viewpoint_stack) - 1))
+
+ render_pkg = self.GS(viewpoint_cam=viewpoint_cam)
+ image = render_pkg["render"]
+ # Loss
+ gt_image = viewpoint_cam.original_image.to(self.device)
+ L1_loss = l1_loss(image, gt_image)
+
+ ssim_loss = (1.0 - ssim(image, gt_image))
+ loss = (1.0 - self.training_config.lambda_dssim) * L1_loss + \
+ self.training_config.lambda_dssim * ssim_loss
+ self.timer.pause()
+ self.logs_losses[self.training_step] = {"loss": loss.item(),
+ "L1_loss": L1_loss.item(),
+ "ssim_loss": ssim_loss.item()}
+
+ if self.log_wandb:
+ for k, v in self.logs_losses[self.training_step].items():
+ wandb.log({f"train/{k}": v}, step=self.training_step)
+ self.ema_loss_for_log = 0.4 * self.logs_losses[self.training_step]["loss"] + 0.6 * self.ema_loss_for_log
+ self.timer.start()
+ self.GS_optimizer.zero_grad(set_to_none=True)
+ loss.backward()
+ with torch.no_grad():
+ if self.gs_step < self.training_config.densify_until_iter and not no_densify:
+ self.GS.gaussians.max_radii2D[render_pkg["visibility_filter"]] = torch.max(
+ self.GS.gaussians.max_radii2D[render_pkg["visibility_filter"]],
+ render_pkg["radii"][render_pkg["visibility_filter"]])
+ self.GS.gaussians.add_densification_stats(render_pkg["viewspace_points"],
+ render_pkg["visibility_filter"])
+
+ # Optimizer step
+ self.GS_optimizer.step()
+ self.GS_optimizer.zero_grad(set_to_none=True)
+ return render_pkg["radii"]
+
+ def densify_and_prune(self, radii = None):
+ # Densification or pruning
+ if self.gs_step < self.training_config.densify_until_iter:
+ if (self.gs_step > self.training_config.densify_from_iter) and \
+ (self.gs_step % self.training_config.densification_interval == 0):
+ size_threshold = 20 if self.gs_step > self.training_config.opacity_reset_interval else None
+ self.GS.gaussians.densify_and_prune(self.training_config.densify_grad_threshold,
+ 0.005,
+ self.GS.scene.cameras_extent,
+ size_threshold, radii)
+ if self.gs_step % self.training_config.opacity_reset_interval == 0 or (
+ self.dataset_white_background and self.gs_step == self.training_config.densify_from_iter):
+ self.GS.gaussians.reset_opacity()
+
+
+
+ def save_model(self):
+ print("\n[ITER {}] Saving Gaussians".format(self.gs_step))
+ self.scene.save(self.gs_step)
+ print("\n[ITER {}] Saving Checkpoint".format(self.gs_step))
+ torch.save((self.GS.gaussians.capture(), self.gs_step),
+ self.scene.model_path + "/chkpnt" + str(self.gs_step) + ".pth")
+
+
+ def init_with_corr(self, cfg, verbose=False, roma_model=None):
+ """
+ Initializes image with matchings. Also removes SfM init points.
+ Args:
+ cfg: configuration part named init_wC. Check train.yaml
+ verbose: whether you want to print intermediate results. Useful for debug.
+ roma_model: optionally you can pass here preinit RoMA model to avoid reinit
+ it every time.
+ """
+ if not cfg.use:
+ return None
+ N_splats_at_init = len(self.GS.gaussians._xyz)
+ print("N_splats_at_init:", N_splats_at_init)
+ if cfg.nns_per_ref == 1:
+ init_fn = init_gaussians_with_corr_fast
+ else:
+ init_fn = init_gaussians_with_corr
+ camera_set, selected_indices, visualization_dict = init_fn(
+ self.GS.gaussians,
+ self.scene,
+ cfg,
+ self.device,
+ verbose=verbose,
+ roma_model=roma_model)
+
+ # Remove SfM points and leave only matchings inits
+ if not cfg.add_SfM_init:
+ with torch.no_grad():
+ N_splats_after_init = len(self.GS.gaussians._xyz)
+ print("N_splats_after_init:", N_splats_after_init)
+ self.gaussians.tmp_radii = torch.zeros(self.gaussians._xyz.shape[0]).to(self.device)
+ mask = torch.concat([torch.ones(N_splats_at_init, dtype=torch.bool),
+ torch.zeros(N_splats_after_init-N_splats_at_init, dtype=torch.bool)],
+ axis=0)
+ self.GS.gaussians.prune_points(mask)
+ with torch.no_grad():
+ gaussians = self.gaussians
+ gaussians._scaling = gaussians.scaling_inverse_activation(gaussians.scaling_activation(gaussians._scaling)*0.5)
+ return visualization_dict
+
+
+ def prune(self, radii, min_opacity=0.005):
+ self.GS.gaussians.tmp_radii = radii
+ if self.gs_step < self.training_config.densify_until_iter:
+ prune_mask = (self.GS.gaussians.get_opacity < min_opacity).squeeze()
+ self.GS.gaussians.prune_points(prune_mask)
+ torch.cuda.empty_cache()
+ self.GS.gaussians.tmp_radii = None
+
diff --git a/source/utils_aux.py b/source/utils_aux.py
new file mode 100644
index 0000000000000000000000000000000000000000..48fcc752cd2c2ef142086d70b4a079f63e940809
--- /dev/null
+++ b/source/utils_aux.py
@@ -0,0 +1,92 @@
+# Perlin noise code taken from https://gist.github.com/adefossez/0646dbe9ed4005480a2407c62aac8869
+from types import SimpleNamespace
+import random
+import numpy as np
+import torch
+import torchvision
+import wandb
+import random
+import torchvision.transforms as T
+import torchvision.transforms.functional as F
+import torch
+from PIL import Image
+
+def parse_dict_to_namespace(dict_nested):
+ """Turns nested dictionary into nested namespaces"""
+ if type(dict_nested) != dict and type(dict_nested) != list: return dict_nested
+ x = SimpleNamespace()
+ for key, val in dict_nested.items():
+ if type(val) == dict:
+ setattr(x, key, parse_dict_to_namespace(val))
+ elif type(val) == list:
+ setattr(x, key, [parse_dict_to_namespace(v) for v in val])
+ else:
+ setattr(x, key, val)
+ return x
+
+def set_seed(seed=42, cuda=True):
+ random.seed(seed)
+ np.random.seed(seed)
+ torch.manual_seed(seed)
+ if cuda:
+ torch.cuda.manual_seed_all(seed)
+
+
+
+def log_samples(samples, scores, iteration, caption="Real Samples"):
+ # Create a grid of images
+ grid = torchvision.utils.make_grid(samples)
+
+ # Log the images and scores to wandb
+ wandb.log({
+ f"{caption}_images": [wandb.Image(grid, caption=f"{caption}: {scores}")],
+ }, step = iteration)
+
+
+
+def pairwise_distances(matrix):
+ """
+ Computes the pairwise Euclidean distances between all vectors in the input matrix.
+
+ Args:
+ matrix (torch.Tensor): Input matrix of shape [N, D], where N is the number of vectors and D is the dimensionality.
+
+ Returns:
+ torch.Tensor: Pairwise distance matrix of shape [N, N].
+ """
+ # Compute squared pairwise distances
+ squared_diff = torch.cdist(matrix, matrix, p=2)
+ return squared_diff
+
+def k_closest_vectors(matrix, k):
+ """
+ Finds the k-closest vectors for each vector in the input matrix based on Euclidean distance.
+
+ Args:
+ matrix (torch.Tensor): Input matrix of shape [N, D], where N is the number of vectors and D is the dimensionality.
+ k (int): Number of closest vectors to return for each vector.
+
+ Returns:
+ torch.Tensor: Indices of the k-closest vectors for each vector, excluding the vector itself.
+ """
+ # Compute pairwise distances
+ distances = pairwise_distances(matrix)
+
+ # For each vector, sort distances and get the indices of the k-closest vectors (excluding itself)
+ # Set diagonal distances to infinity to exclude the vector itself from the nearest neighbors
+ distances.fill_diagonal_(float('inf'))
+
+ # Get the indices of the k smallest distances (k-closest vectors)
+ _, indices = torch.topk(distances, k, largest=False, dim=1)
+
+ return indices
+
+def process_image(image_tensor):
+ image_np = image_tensor.detach().cpu().numpy().transpose(1, 2, 0)
+ return Image.fromarray(np.clip(image_np * 255, 0, 255).astype(np.uint8))
+
+
+def normalize_keypoints(kpts_np, width, height):
+ kpts_np[:, 0] = kpts_np[:, 0] / width * 2. - 1.
+ kpts_np[:, 1] = kpts_np[:, 1] / height * 2. - 1.
+ return kpts_np
\ No newline at end of file
diff --git a/source/utils_preprocess.py b/source/utils_preprocess.py
new file mode 100644
index 0000000000000000000000000000000000000000..7c6dab31946b10d57c6e9b6a3d948297e7b27227
--- /dev/null
+++ b/source/utils_preprocess.py
@@ -0,0 +1,334 @@
+# This file contains function for video or image collection preprocessing.
+# For video we do the preprocessing and select k sharpest frames.
+# Afterwards scene is constructed
+import cv2
+import numpy as np
+from tqdm import tqdm
+import pycolmap
+import os
+import time
+import tempfile
+from moviepy import VideoFileClip
+from matplotlib import pyplot as plt
+from PIL import Image
+import cv2
+from tqdm import tqdm
+
+WORKDIR = "../outputs/"
+
+
+def get_rotation_moviepy(video_path):
+ clip = VideoFileClip(video_path)
+ rotation = 0
+
+ try:
+ displaymatrix = clip.reader.infos['inputs'][0]['streams'][2]['metadata'].get('displaymatrix', '')
+ if 'rotation of' in displaymatrix:
+ angle = float(displaymatrix.strip().split('rotation of')[-1].split('degrees')[0])
+ rotation = int(angle) % 360
+
+ except Exception as e:
+ print(f"No displaymatrix rotation found: {e}")
+
+ clip.reader.close()
+ #if clip.audio:
+ # clip.audio.reader.close_proc()
+
+ return rotation
+
+def resize_max_side(frame, max_size):
+ h, w = frame.shape[:2]
+ scale = max_size / max(h, w)
+ if scale < 1:
+ frame = cv2.resize(frame, (int(w * scale), int(h * scale)))
+ return frame
+
+def read_video_frames(video_input, k=1, max_size=1024):
+ """
+ Extracts every k-th frame from a video or list of images, resizes to max size, and returns frames as list.
+
+ Parameters:
+ video_input (str, file-like, or list): Path to video file, file-like object, or list of image files.
+ k (int): Interval for frame extraction (every k-th frame).
+ max_size (int): Maximum size for width or height after resizing.
+
+ Returns:
+ frames (list): List of resized frames (numpy arrays).
+ """
+ # Handle list of image files (not single video in a list)
+ if isinstance(video_input, list):
+ # If it's a single video in a list, treat it as video
+ if len(video_input) == 1 and video_input[0].name.endswith(('.mp4', '.avi', '.mov')):
+ video_input = video_input[0] # unwrap single video file
+ else:
+ # Treat as list of images
+ frames = []
+ for img_file in video_input:
+ img = Image.open(img_file.name).convert("RGB")
+ img.thumbnail((max_size, max_size))
+ frames.append(np.array(img)[...,::-1])
+ return frames
+
+ # Handle file-like or path
+ if hasattr(video_input, 'name'):
+ video_path = video_input.name
+ elif isinstance(video_input, (str, os.PathLike)):
+ video_path = str(video_input)
+ else:
+ raise ValueError("Unsupported video input type. Must be a filepath, file-like object, or list of images.")
+
+
+ cap = cv2.VideoCapture(video_path)
+ if not cap.isOpened():
+ raise ValueError(f"Error: Could not open video {video_path}.")
+
+ total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
+ frame_count = 0
+ frames = []
+
+ with tqdm(total=total_frames // k, desc="Processing Video", unit="frame") as pbar:
+ while True:
+ ret, frame = cap.read()
+ if not ret:
+ break
+ if frame_count % k == 0:
+ frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+ h, w = frame.shape[:2]
+ scale = max(h, w) / max_size
+ if scale > 1:
+ frame = cv2.resize(frame, (int(w / scale), int(h / scale)))
+ frames.append(frame[...,[2,1,0]])
+ pbar.update(1)
+ frame_count += 1
+
+ cap.release()
+ return frames
+
+def resize_max_side(frame, max_size):
+ """
+ Resizes the frame so that its largest side equals max_size, maintaining aspect ratio.
+ """
+ height, width = frame.shape[:2]
+ max_dim = max(height, width)
+
+ if max_dim <= max_size:
+ return frame # No need to resize
+
+ scale = max_size / max_dim
+ new_width = int(width * scale)
+ new_height = int(height * scale)
+
+ resized_frame = cv2.resize(frame, (new_width, new_height), interpolation=cv2.INTER_AREA)
+ return resized_frame
+
+
+
+def variance_of_laplacian(image):
+ # compute the Laplacian of the image and then return the focus
+ # measure, which is simply the variance of the Laplacian
+ return cv2.Laplacian(image, cv2.CV_64F).var()
+
+def process_all_frames(IMG_FOLDER = '/scratch/datasets/hq_data/night2_all_frames',
+ to_visualize=False,
+ save_images=True):
+ dict_scores = {}
+ for idx, img_name in tqdm(enumerate(sorted([x for x in os.listdir(IMG_FOLDER) if '.png' in x]))):
+
+ img = cv2.imread(os.path.join(IMG_FOLDER, img_name))#[250:, 100:]
+ gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
+ fm = variance_of_laplacian(gray) + \
+ variance_of_laplacian(cv2.resize(gray, (0,0), fx=0.75, fy=0.75)) + \
+ variance_of_laplacian(cv2.resize(gray, (0,0), fx=0.5, fy=0.5)) + \
+ variance_of_laplacian(cv2.resize(gray, (0,0), fx=0.25, fy=0.25))
+ if to_visualize:
+ plt.figure()
+ plt.title(f"Laplacian score: {fm:.2f}")
+ plt.imshow(img[..., [2,1,0]])
+ plt.show()
+ dict_scores[idx] = {"idx" : idx,
+ "img_name" : img_name,
+ "score" : fm}
+ if save_images:
+ dict_scores[idx]["img"] = img
+
+ return dict_scores
+
+def select_optimal_frames(scores, k):
+ """
+ Selects a minimal subset of frames while ensuring no gaps exceed k.
+
+ Args:
+ scores (list of float): List of scores where index represents frame number.
+ k (int): Maximum allowed gap between selected frames.
+
+ Returns:
+ list of int: Indices of selected frames.
+ """
+ n = len(scores)
+ selected = [0, n-1]
+ i = 0 # Start at the first frame
+
+ while i < n:
+ # Find the best frame to select within the next k frames
+ best_idx = max(range(i, min(i + k + 1, n)), key=lambda x: scores[x], default=None)
+
+ if best_idx is None:
+ break # No more frames left
+
+ selected.append(best_idx)
+ i = best_idx + k + 1 # Move forward, ensuring gaps stay within k
+
+ return sorted(selected)
+
+
+def variance_of_laplacian(image):
+ """
+ Compute the variance of Laplacian as a focus measure.
+ """
+ return cv2.Laplacian(image, cv2.CV_64F).var()
+
+def preprocess_frames(frames, verbose=False):
+ """
+ Compute sharpness scores for a list of frames using multi-scale Laplacian variance.
+
+ Args:
+ frames (list of np.ndarray): List of frames (BGR images).
+ verbose (bool): If True, print scores.
+
+ Returns:
+ list of float: Sharpness scores for each frame.
+ """
+ scores = []
+
+ for idx, frame in enumerate(tqdm(frames, desc="Scoring frames")):
+ gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
+
+ fm = (
+ variance_of_laplacian(gray) +
+ variance_of_laplacian(cv2.resize(gray, (0, 0), fx=0.75, fy=0.75)) +
+ variance_of_laplacian(cv2.resize(gray, (0, 0), fx=0.5, fy=0.5)) +
+ variance_of_laplacian(cv2.resize(gray, (0, 0), fx=0.25, fy=0.25))
+ )
+
+ if verbose:
+ print(f"Frame {idx}: Sharpness Score = {fm:.2f}")
+
+ scores.append(fm)
+
+ return scores
+
+def select_optimal_frames(scores, k):
+ """
+ Selects k frames by splitting into k segments and picking the sharpest frame from each.
+
+ Args:
+ scores (list of float): List of sharpness scores.
+ k (int): Number of frames to select.
+
+ Returns:
+ list of int: Indices of selected frames.
+ """
+ n = len(scores)
+ selected_indices = []
+ segment_size = n // k
+
+ for i in range(k):
+ start = i * segment_size
+ end = (i + 1) * segment_size if i < k - 1 else n # Last chunk may be larger
+ segment_scores = scores[start:end]
+
+ if len(segment_scores) == 0:
+ continue # Safety check if some segment is empty
+
+ best_in_segment = start + np.argmax(segment_scores)
+ selected_indices.append(best_in_segment)
+
+ return sorted(selected_indices)
+
+def save_frames_to_scene_dir(frames, scene_dir):
+ """
+ Saves a list of frames into the target scene directory under 'images/' subfolder.
+
+ Args:
+ frames (list of np.ndarray): List of frames (BGR images) to save.
+ scene_dir (str): Target path where 'images/' subfolder will be created.
+ """
+ images_dir = os.path.join(scene_dir, "images")
+ os.makedirs(images_dir, exist_ok=True)
+
+ for idx, frame in enumerate(frames):
+ filename = os.path.join(images_dir, f"{idx:08d}.png") # 00000000.png, 00000001.png, etc.
+ cv2.imwrite(filename, frame)
+
+ print(f"Saved {len(frames)} frames to {images_dir}")
+
+
+def run_colmap_on_scene(scene_dir):
+ """
+ Runs feature extraction, matching, and mapping on all images inside scene_dir/images using pycolmap.
+
+ Args:
+ scene_dir (str): Path to scene directory containing 'images' folder.
+
+ TODO: if the function hasn't managed to match all the frames either increase image size,
+ increase number of features or just remove those frames from the folder scene_dir/images
+ """
+ start_time = time.time()
+ print(f"Running COLMAP pipeline on all images inside {scene_dir}")
+
+ # Setup paths
+ database_path = os.path.join(scene_dir, "database.db")
+ sparse_path = os.path.join(scene_dir, "sparse")
+ image_dir = os.path.join(scene_dir, "images")
+
+ # Make sure output directories exist
+ os.makedirs(sparse_path, exist_ok=True)
+
+ # Step 1: Feature Extraction
+ pycolmap.extract_features(
+ database_path,
+ image_dir,
+ sift_options={
+ "max_num_features": 512 * 2,
+ "max_image_size": 512 * 1,
+ }
+ )
+ print(f"Finished feature extraction in {(time.time() - start_time):.2f}s.")
+
+ # Step 2: Feature Matching
+ pycolmap.match_exhaustive(database_path)
+ print(f"Finished feature matching in {(time.time() - start_time):.2f}s.")
+
+ # Step 3: Mapping
+ pipeline_options = pycolmap.IncrementalPipelineOptions()
+ pipeline_options.min_num_matches = 15
+ pipeline_options.multiple_models = True
+ pipeline_options.max_num_models = 50
+ pipeline_options.max_model_overlap = 20
+ pipeline_options.min_model_size = 10
+ pipeline_options.extract_colors = True
+ pipeline_options.num_threads = 8
+ pipeline_options.mapper.init_min_num_inliers = 30
+ pipeline_options.mapper.init_max_error = 8.0
+ pipeline_options.mapper.init_min_tri_angle = 5.0
+
+ reconstruction = pycolmap.incremental_mapping(
+ database_path=database_path,
+ image_path=image_dir,
+ output_path=sparse_path,
+ options=pipeline_options,
+ )
+ print(f"Finished incremental mapping in {(time.time() - start_time):.2f}s.")
+
+ # Step 4: Post-process Cameras to SIMPLE_PINHOLE
+ recon_path = os.path.join(sparse_path, "0")
+ reconstruction = pycolmap.Reconstruction(recon_path)
+
+ for cam in reconstruction.cameras.values():
+ cam.model = 'SIMPLE_PINHOLE'
+ cam.params = cam.params[:3] # Keep only [f, cx, cy]
+
+ reconstruction.write(recon_path)
+
+ print(f"Total pipeline time: {(time.time() - start_time):.2f}s.")
+
diff --git a/source/visualization.py b/source/visualization.py
new file mode 100644
index 0000000000000000000000000000000000000000..ed41a1546c07b6031c73c50e3f8c6c4cb5a7ec44
--- /dev/null
+++ b/source/visualization.py
@@ -0,0 +1,1072 @@
+from matplotlib import pyplot as plt
+import numpy as np
+import torch
+
+import numpy as np
+from typing import List
+import sys
+sys.path.append('./submodules/gaussian-splatting/')
+from scene.cameras import Camera
+from PIL import Image
+import imageio
+from scipy.interpolate import splprep, splev
+
+import cv2
+import numpy as np
+import plotly.graph_objects as go
+import numpy as np
+from scipy.spatial.transform import Rotation as R, Slerp
+from scipy.spatial import distance_matrix
+from sklearn.decomposition import PCA
+from scipy.interpolate import splprep, splev
+from typing import List
+from sklearn.mixture import GaussianMixture
+
+def render_gaussians_rgb(generator3DGS, viewpoint_cam, visualize=False):
+ """
+ Simply render gaussians from the generator3DGS from the viewpoint_cam.
+ Args:
+ generator3DGS : instance of the Generator3DGS class from the networks.py file
+ viewpoint_cam : camera instance
+ visualize : boolean flag. If True, will call pyplot function and render image inplace
+ Returns:
+ uint8 numpy array with shape (H, W, 3) representing the image
+ """
+ with torch.no_grad():
+ render_pkg = generator3DGS(viewpoint_cam)
+ image = render_pkg["render"]
+ image_np = image.clone().detach().cpu().numpy().transpose(1, 2, 0)
+
+ # Clip values to be in the range [0, 1]
+ image_np = np.clip(image_np * 255, 0, 255).astype(np.uint8)
+ if visualize:
+ plt.figure(figsize=(12, 8))
+ plt.imshow(image_np)
+ plt.show()
+
+ return image_np
+
+def render_gaussians_D_scores(generator3DGS, viewpoint_cam, mask=None, mask_channel=0, visualize=False):
+ """
+ Simply render D_scores of gaussians from the generator3DGS from the viewpoint_cam.
+ Args:
+ generator3DGS : instance of the Generator3DGS class from the networks.py file
+ viewpoint_cam : camera instance
+ visualize : boolean flag. If True, will call pyplot function and render image inplace
+ mask : optional mask to highlight specific gaussians. Must be of shape (N) where N is the numnber
+ of gaussians in generator3DGS.gaussians. Must be a torch tensor of floats, please scale according
+ to how much color you want to have. Recommended mask value is 10.
+ mask_channel: to which color channel should we add mask
+ Returns:
+ uint8 numpy array with shape (H, W, 3) representing the generator3DGS.gaussians.D_scores rendered as colors
+ """
+ with torch.no_grad():
+ # Visualize D_scores
+ generator3DGS.gaussians._features_dc = generator3DGS.gaussians._features_dc * 1e-4 + \
+ torch.stack([generator3DGS.gaussians.D_scores] * 3, axis=-1)
+ generator3DGS.gaussians._features_rest = generator3DGS.gaussians._features_rest * 1e-4
+ if mask is not None:
+ generator3DGS.gaussians._features_dc[..., mask_channel] += mask.unsqueeze(-1)
+ render_pkg = generator3DGS(viewpoint_cam)
+ image = render_pkg["render"]
+ image_np = image.clone().detach().cpu().numpy().transpose(1, 2, 0)
+
+ # Clip values to be in the range [0, 1]
+ image_np = np.clip(image_np * 255, 0, 255).astype(np.uint8)
+ if visualize:
+ plt.figure(figsize=(12, 8))
+ plt.imshow(image_np)
+ plt.show()
+
+ if mask is not None:
+ generator3DGS.gaussians._features_dc[..., mask_channel] -= mask.unsqueeze(-1)
+
+ generator3DGS.gaussians._features_dc = (generator3DGS.gaussians._features_dc - \
+ torch.stack([generator3DGS.gaussians.D_scores] * 3, axis=-1)) * 1e4
+ generator3DGS.gaussians._features_rest = generator3DGS.gaussians._features_rest * 1e4
+
+ return image_np
+
+
+
+def normalize(v):
+ """
+ Normalize a vector to unit length.
+
+ Parameters:
+ v (np.ndarray): Input vector.
+
+ Returns:
+ np.ndarray: Unit vector in the same direction as `v`.
+ """
+ return v / np.linalg.norm(v)
+
+def look_at_rotation(camera_position: np.ndarray, target: np.ndarray, world_up=np.array([0, 1, 0])):
+ """
+ Compute a rotation matrix for a camera looking at a target point.
+
+ Parameters:
+ camera_position (np.ndarray): The 3D position of the camera.
+ target (np.ndarray): The point the camera should look at.
+ world_up (np.ndarray): A vector that defines the global 'up' direction.
+
+ Returns:
+ np.ndarray: A 3x3 rotation matrix (camera-to-world) with columns [right, up, forward].
+ """
+ z_axis = normalize(target - camera_position) # Forward direction
+ x_axis = normalize(np.cross(world_up, z_axis)) # Right direction
+ y_axis = np.cross(z_axis, x_axis) # Recomputed up
+ return np.stack([x_axis, y_axis, z_axis], axis=1)
+
+
+def generate_circular_camera_path(existing_cameras: List[Camera], N: int = 12, radius_scale: float = 1.0, d: float = 2.0) -> List[Camera]:
+ """
+ Generate a circular path of cameras around an existing camera group,
+ with each new camera oriented to look at the average viewing direction.
+
+ Parameters:
+ existing_cameras (List[Camera]): List of existing camera objects to estimate average orientation and layout.
+ N (int): Number of new cameras to generate along the circular path.
+ radius_scale (float): Scale factor to adjust the radius of the circle.
+ d (float): Distance ahead of each camera used to estimate its look-at point.
+
+ Returns:
+ List[Camera]: A list of newly generated Camera objects forming a circular path and oriented toward a shared view center.
+ """
+ # Step 1: Compute average camera position
+ center = np.mean([cam.T for cam in existing_cameras], axis=0)
+
+ # Estimate where each camera is looking
+ # d denotes how far ahead each camera sees — you can scale this
+ look_targets = [cam.T + cam.R[:, 2] * d for cam in existing_cameras]
+ center_of_view = np.mean(look_targets, axis=0)
+
+ # Step 2: Define circular plane basis using fixed up vector
+ avg_forward = normalize(np.mean([cam.R[:, 2] for cam in existing_cameras], axis=0))
+ up_guess = np.array([0, 1, 0])
+ right = normalize(np.cross(avg_forward, up_guess))
+ up = normalize(np.cross(right, avg_forward))
+
+ # Step 3: Estimate radius
+ avg_radius = np.mean([np.linalg.norm(cam.T - center) for cam in existing_cameras]) * radius_scale
+
+ # Step 4: Create cameras on a circular path
+ angles = np.linspace(0, 2 * np.pi, N, endpoint=False)
+ reference_cam = existing_cameras[0]
+ new_cameras = []
+
+
+ for i, a in enumerate(angles):
+ position = center + avg_radius * (np.cos(a) * right + np.sin(a) * up)
+
+ if d < 1e-5 or radius_scale < 1e-5:
+ # Use same orientation as the first camera
+ R = reference_cam.R.copy()
+ else:
+ # Change orientation
+ R = look_at_rotation(position, center_of_view)
+ new_cameras.append(Camera(
+ R=R,
+ T=position, # New position
+ FoVx=reference_cam.FoVx,
+ FoVy=reference_cam.FoVy,
+ resolution=(reference_cam.image_width, reference_cam.image_height),
+ colmap_id=-1,
+ depth_params=None,
+ image=Image.fromarray(np.zeros((reference_cam.image_height, reference_cam.image_width, 3), dtype=np.uint8)),
+ invdepthmap=None,
+ image_name=f"circular_a={a:.3f}",
+ uid=i
+ ))
+
+ return new_cameras
+
+
+def save_numpy_frames_as_gif(frames, output_path="animation.gif", duration=100):
+ """
+ Save a list of RGB NumPy frames as a looping GIF animation.
+
+ Parameters:
+ frames (List[np.ndarray]): List of RGB images as uint8 NumPy arrays (shape HxWx3).
+ output_path (str): Path to save the output GIF.
+ duration (int): Duration per frame in milliseconds.
+
+ Returns:
+ None
+ """
+ pil_frames = [Image.fromarray(f) for f in frames]
+ pil_frames[0].save(
+ output_path,
+ save_all=True,
+ append_images=pil_frames[1:],
+ duration=duration, # duration per frame in ms
+ loop=0
+ )
+ print(f"GIF saved to: {output_path}")
+
+def center_crop_frame(frame: np.ndarray, crop_fraction: float) -> np.ndarray:
+ """
+ Crop the central region of the frame by the given fraction.
+
+ Parameters:
+ frame (np.ndarray): Input RGB image (H, W, 3).
+ crop_fraction (float): Fraction of the original size to retain (e.g., 0.8 keeps 80%).
+
+ Returns:
+ np.ndarray: Cropped RGB image.
+ """
+ if crop_fraction >= 1.0:
+ return frame
+
+ h, w, _ = frame.shape
+ new_h, new_w = int(h * crop_fraction), int(w * crop_fraction)
+ start_y = (h - new_h) // 2
+ start_x = (w - new_w) // 2
+ return frame[start_y:start_y + new_h, start_x:start_x + new_w, :]
+
+
+
+def generate_smooth_closed_camera_path(existing_cameras: List[Camera], N: int = 120, d: float = 2.0, s=.25) -> List[Camera]:
+ """
+ Generate a smooth, closed path interpolating the positions of existing cameras.
+
+ Parameters:
+ existing_cameras (List[Camera]): List of existing cameras.
+ N (int): Number of points (cameras) to sample along the smooth path.
+ d (float): Distance ahead for estimating the center of view.
+
+ Returns:
+ List[Camera]: A list of smoothly moving Camera objects along a closed loop.
+ """
+ # Step 1: Extract camera positions
+ positions = np.array([cam.T for cam in existing_cameras])
+
+ # Step 2: Estimate center of view
+ look_targets = [cam.T + cam.R[:, 2] * d for cam in existing_cameras]
+ center_of_view = np.mean(look_targets, axis=0)
+
+ # Step 3: Fit a smooth closed spline through the positions
+ positions = np.vstack([positions, positions[0]]) # close the loop
+ tck, u = splprep(positions.T, s=s, per=True) # periodic=True for closed loop
+
+ # Step 4: Sample points along the spline
+ u_fine = np.linspace(0, 1, N)
+ smooth_path = np.stack(splev(u_fine, tck), axis=-1)
+
+ # Step 5: Generate cameras along the smooth path
+ reference_cam = existing_cameras[0]
+ new_cameras = []
+
+ for i, pos in enumerate(smooth_path):
+ R = look_at_rotation(pos, center_of_view)
+ new_cameras.append(Camera(
+ R=R,
+ T=pos,
+ FoVx=reference_cam.FoVx,
+ FoVy=reference_cam.FoVy,
+ resolution=(reference_cam.image_width, reference_cam.image_height),
+ colmap_id=-1,
+ depth_params=None,
+ image=Image.fromarray(np.zeros((reference_cam.image_height, reference_cam.image_width, 3), dtype=np.uint8)),
+ invdepthmap=None,
+ image_name=f"smooth_path_i={i}",
+ uid=i
+ ))
+
+ return new_cameras
+
+
+def save_numpy_frames_as_mp4(frames, output_path="animation.mp4", fps=10, center_crop: float = 1.0):
+ """
+ Save a list of RGB NumPy frames as an MP4 video with optional center cropping.
+
+ Parameters:
+ frames (List[np.ndarray]): List of RGB images as uint8 NumPy arrays (shape HxWx3).
+ output_path (str): Path to save the output MP4.
+ fps (int): Frames per second for playback speed.
+ center_crop (float): Fraction (0 < center_crop <= 1.0) of central region to retain.
+ Use 1.0 for no cropping; 0.8 to crop to 80% center region.
+
+ Returns:
+ None
+ """
+ with imageio.get_writer(output_path, fps=fps, codec='libx264', quality=8) as writer:
+ for frame in frames:
+ cropped = center_crop_frame(frame, center_crop)
+ writer.append_data(cropped)
+ print(f"MP4 saved to: {output_path}")
+
+
+
+def put_text_on_image(img: np.ndarray, text: str) -> np.ndarray:
+ """
+ Draws multiline white text on a copy of the input image, positioned near the bottom
+ and around 80% of the image width. Handles '\n' characters to split text into multiple lines.
+
+ Args:
+ img (np.ndarray): Input image as a (H, W, 3) uint8 numpy array.
+ text (str): Text string to draw on the image. Newlines '\n' are treated as line breaks.
+
+ Returns:
+ np.ndarray: The output image with the text drawn on it.
+
+ Notes:
+ - The function automatically adjusts line spacing and prevents text from going outside the image.
+ - Text is drawn in white with small font size (0.5) for minimal visual impact.
+ """
+ img = img.copy()
+ height, width, _ = img.shape
+
+ font = cv2.FONT_HERSHEY_SIMPLEX
+ font_scale = 1.
+ color = (255, 255, 255)
+ thickness = 2
+ line_spacing = 5 # extra pixels between lines
+
+ lines = text.split('\n')
+
+ # Precompute the maximum text width to adjust starting x
+ max_text_width = max(cv2.getTextSize(line, font, font_scale, thickness)[0][0] for line in lines)
+
+ x = int(0.8 * width)
+ x = min(x, width - max_text_width - 30) # margin on right
+ #x = int(0.03 * width)
+
+ # Start near the bottom, but move up depending on number of lines
+ total_text_height = len(lines) * (cv2.getTextSize('A', font, font_scale, thickness)[0][1] + line_spacing)
+ y_start = int(height*0.9) - total_text_height # 30 pixels from bottom
+
+ for i, line in enumerate(lines):
+ y = y_start + i * (cv2.getTextSize(line, font, font_scale, thickness)[0][1] + line_spacing)
+ cv2.putText(img, line, (x, y), font, font_scale, color, thickness, cv2.LINE_AA)
+
+ return img
+
+
+
+
+def catmull_rom_spline(P0, P1, P2, P3, n_points=20):
+ """
+ Compute Catmull-Rom spline segment between P1 and P2.
+ """
+ t = np.linspace(0, 1, n_points)[:, None]
+
+ M = 0.5 * np.array([
+ [-1, 3, -3, 1],
+ [ 2, -5, 4, -1],
+ [-1, 0, 1, 0],
+ [ 0, 2, 0, 0]
+ ])
+
+ G = np.stack([P0, P1, P2, P3], axis=0)
+ T = np.concatenate([t**3, t**2, t, np.ones_like(t)], axis=1)
+
+ return T @ M @ G
+
+def sort_cameras_pca(existing_cameras: List[Camera]):
+ """
+ Sort cameras along the main PCA axis.
+ """
+ positions = np.array([cam.T for cam in existing_cameras])
+ pca = PCA(n_components=1)
+ scores = pca.fit_transform(positions)
+ sorted_indices = np.argsort(scores[:, 0])
+ return sorted_indices
+
+def generate_fully_smooth_cameras(existing_cameras: List[Camera],
+ n_selected: int = 30,
+ n_points_per_segment: int = 20,
+ d: float = 2.0,
+ closed: bool = False) -> List[Camera]:
+ """
+ Generate a fully smooth camera path using PCA ordering, global Catmull-Rom spline for positions, and global SLERP for orientations.
+
+ Args:
+ existing_cameras (List[Camera]): List of input cameras.
+ n_selected (int): Number of cameras to select after sorting.
+ n_points_per_segment (int): Number of interpolated points per spline segment.
+ d (float): Distance ahead for estimating center of view.
+ closed (bool): Whether to close the path.
+
+ Returns:
+ List[Camera]: List of smoothly moving Camera objects.
+ """
+ # 1. Sort cameras along PCA axis
+ sorted_indices = sort_cameras_pca(existing_cameras)
+ sorted_cameras = [existing_cameras[i] for i in sorted_indices]
+ positions = np.array([cam.T for cam in sorted_cameras])
+
+ # 2. Subsample uniformly
+ idx = np.linspace(0, len(positions) - 1, n_selected).astype(int)
+ sampled_positions = positions[idx]
+ sampled_cameras = [sorted_cameras[i] for i in idx]
+
+ # 3. Prepare for Catmull-Rom
+ if closed:
+ sampled_positions = np.vstack([sampled_positions[-1], sampled_positions, sampled_positions[0], sampled_positions[1]])
+ else:
+ sampled_positions = np.vstack([sampled_positions[0], sampled_positions, sampled_positions[-1], sampled_positions[-1]])
+
+ # 4. Generate smooth path positions
+ path_positions = []
+ for i in range(1, len(sampled_positions) - 2):
+ segment = catmull_rom_spline(sampled_positions[i-1], sampled_positions[i], sampled_positions[i+1], sampled_positions[i+2], n_points_per_segment)
+ path_positions.append(segment)
+ path_positions = np.concatenate(path_positions, axis=0)
+
+ # 5. Global SLERP for rotations
+ rotations = R.from_matrix([cam.R for cam in sampled_cameras])
+ key_times = np.linspace(0, 1, len(rotations))
+ slerp = Slerp(key_times, rotations)
+
+ query_times = np.linspace(0, 1, len(path_positions))
+ interpolated_rotations = slerp(query_times)
+
+ # 6. Generate Camera objects
+ reference_cam = existing_cameras[0]
+ smooth_cameras = []
+
+ for i, pos in enumerate(path_positions):
+ R_interp = interpolated_rotations[i].as_matrix()
+
+ smooth_cameras.append(Camera(
+ R=R_interp,
+ T=pos,
+ FoVx=reference_cam.FoVx,
+ FoVy=reference_cam.FoVy,
+ resolution=(reference_cam.image_width, reference_cam.image_height),
+ colmap_id=-1,
+ depth_params=None,
+ image=Image.fromarray(np.zeros((reference_cam.image_height, reference_cam.image_width, 3), dtype=np.uint8)),
+ invdepthmap=None,
+ image_name=f"fully_smooth_path_i={i}",
+ uid=i
+ ))
+
+ return smooth_cameras
+
+
+def plot_cameras_and_smooth_path_with_orientation(existing_cameras: List[Camera], smooth_cameras: List[Camera], scale: float = 0.1):
+ """
+ Plot input cameras and smooth path cameras with their orientations in 3D.
+
+ Args:
+ existing_cameras (List[Camera]): List of original input cameras.
+ smooth_cameras (List[Camera]): List of smooth path cameras.
+ scale (float): Length of orientation arrows.
+
+ Returns:
+ None
+ """
+ # Input cameras
+ input_positions = np.array([cam.T for cam in existing_cameras])
+
+ # Smooth cameras
+ smooth_positions = np.array([cam.T for cam in smooth_cameras])
+
+ fig = go.Figure()
+
+ # Plot input camera positions
+ fig.add_trace(go.Scatter3d(
+ x=input_positions[:, 0], y=input_positions[:, 1], z=input_positions[:, 2],
+ mode='markers',
+ marker=dict(size=4, color='blue'),
+ name='Input Cameras'
+ ))
+
+ # Plot smooth path positions
+ fig.add_trace(go.Scatter3d(
+ x=smooth_positions[:, 0], y=smooth_positions[:, 1], z=smooth_positions[:, 2],
+ mode='lines+markers',
+ line=dict(color='red', width=3),
+ marker=dict(size=2, color='red'),
+ name='Smooth Path Cameras'
+ ))
+
+ # Plot input camera orientations
+ for cam in existing_cameras:
+ origin = cam.T
+ forward = cam.R[:, 2] # Forward direction
+
+ fig.add_trace(go.Cone(
+ x=[origin[0]], y=[origin[1]], z=[origin[2]],
+ u=[forward[0]], v=[forward[1]], w=[forward[2]],
+ colorscale=[[0, 'blue'], [1, 'blue']],
+ sizemode="absolute",
+ sizeref=scale,
+ anchor="tail",
+ showscale=False,
+ name='Input Camera Direction'
+ ))
+
+ # Plot smooth camera orientations
+ for cam in smooth_cameras:
+ origin = cam.T
+ forward = cam.R[:, 2] # Forward direction
+
+ fig.add_trace(go.Cone(
+ x=[origin[0]], y=[origin[1]], z=[origin[2]],
+ u=[forward[0]], v=[forward[1]], w=[forward[2]],
+ colorscale=[[0, 'red'], [1, 'red']],
+ sizemode="absolute",
+ sizeref=scale,
+ anchor="tail",
+ showscale=False,
+ name='Smooth Camera Direction'
+ ))
+
+ fig.update_layout(
+ scene=dict(
+ xaxis_title='X',
+ yaxis_title='Y',
+ zaxis_title='Z',
+ aspectmode='data'
+ ),
+ title="Input Cameras and Smooth Path with Orientations",
+ margin=dict(l=0, r=0, b=0, t=30)
+ )
+
+ fig.show()
+
+
+def solve_tsp_nearest_neighbor(points: np.ndarray):
+ """
+ Solve TSP approximately using nearest neighbor heuristic.
+
+ Args:
+ points (np.ndarray): (N, 3) array of points.
+
+ Returns:
+ List[int]: Optimal visiting order of points.
+ """
+ N = points.shape[0]
+ dist = distance_matrix(points, points)
+ visited = [0]
+ unvisited = set(range(1, N))
+
+ while unvisited:
+ last = visited[-1]
+ next_city = min(unvisited, key=lambda city: dist[last, city])
+ visited.append(next_city)
+ unvisited.remove(next_city)
+
+ return visited
+
+def solve_tsp_2opt(points: np.ndarray, n_iter: int = 1000) -> np.ndarray:
+ """
+ Solve TSP approximately using Nearest Neighbor + 2-Opt.
+
+ Args:
+ points (np.ndarray): Array of shape (N, D) with points.
+ n_iter (int): Number of 2-opt iterations.
+
+ Returns:
+ np.ndarray: Ordered list of indices.
+ """
+ n_points = points.shape[0]
+
+ # === 1. Start with Nearest Neighbor
+ unvisited = list(range(n_points))
+ current = unvisited.pop(0)
+ path = [current]
+
+ while unvisited:
+ dists = np.linalg.norm(points[unvisited] - points[current], axis=1)
+ next_idx = unvisited[np.argmin(dists)]
+ unvisited.remove(next_idx)
+ path.append(next_idx)
+ current = next_idx
+
+ # === 2. Apply 2-Opt improvements
+ def path_length(path):
+ return np.sum(np.linalg.norm(points[path[i]] - points[path[i+1]], axis=0) for i in range(len(path)-1))
+
+ best_length = path_length(path)
+ improved = True
+
+ for _ in range(n_iter):
+ if not improved:
+ break
+ improved = False
+ for i in range(1, n_points - 2):
+ for j in range(i + 1, n_points):
+ if j - i == 1: continue
+ new_path = path[:i] + path[i:j][::-1] + path[j:]
+ new_length = path_length(new_path)
+ if new_length < best_length:
+ path = new_path
+ best_length = new_length
+ improved = True
+ break
+ if improved:
+ break
+
+ return np.array(path)
+
+def generate_fully_smooth_cameras_with_tsp(existing_cameras: List[Camera],
+ n_selected: int = 30,
+ n_points_per_segment: int = 20,
+ d: float = 2.0,
+ closed: bool = False) -> List[Camera]:
+ """
+ Generate a fully smooth camera path using TSP ordering, global Catmull-Rom spline for positions, and global SLERP for orientations.
+
+ Args:
+ existing_cameras (List[Camera]): List of input cameras.
+ n_selected (int): Number of cameras to select after ordering.
+ n_points_per_segment (int): Number of interpolated points per spline segment.
+ d (float): Distance ahead for estimating center of view.
+ closed (bool): Whether to close the path.
+
+ Returns:
+ List[Camera]: List of smoothly moving Camera objects.
+ """
+ positions = np.array([cam.T for cam in existing_cameras])
+
+ # 1. Solve approximate TSP
+ order = solve_tsp_nearest_neighbor(positions)
+ ordered_cameras = [existing_cameras[i] for i in order]
+ ordered_positions = positions[order]
+
+ # 2. Subsample uniformly
+ idx = np.linspace(0, len(ordered_positions) - 1, n_selected).astype(int)
+ sampled_positions = ordered_positions[idx]
+ sampled_cameras = [ordered_cameras[i] for i in idx]
+
+ # 3. Prepare for Catmull-Rom
+ if closed:
+ sampled_positions = np.vstack([sampled_positions[-1], sampled_positions, sampled_positions[0], sampled_positions[1]])
+ else:
+ sampled_positions = np.vstack([sampled_positions[0], sampled_positions, sampled_positions[-1], sampled_positions[-1]])
+
+ # 4. Generate smooth path positions
+ path_positions = []
+ for i in range(1, len(sampled_positions) - 2):
+ segment = catmull_rom_spline(sampled_positions[i-1], sampled_positions[i], sampled_positions[i+1], sampled_positions[i+2], n_points_per_segment)
+ path_positions.append(segment)
+ path_positions = np.concatenate(path_positions, axis=0)
+
+ # 5. Global SLERP for rotations
+ rotations = R.from_matrix([cam.R for cam in sampled_cameras])
+ key_times = np.linspace(0, 1, len(rotations))
+ slerp = Slerp(key_times, rotations)
+
+ query_times = np.linspace(0, 1, len(path_positions))
+ interpolated_rotations = slerp(query_times)
+
+ # 6. Generate Camera objects
+ reference_cam = existing_cameras[0]
+ smooth_cameras = []
+
+ for i, pos in enumerate(path_positions):
+ R_interp = interpolated_rotations[i].as_matrix()
+
+ smooth_cameras.append(Camera(
+ R=R_interp,
+ T=pos,
+ FoVx=reference_cam.FoVx,
+ FoVy=reference_cam.FoVy,
+ resolution=(reference_cam.image_width, reference_cam.image_height),
+ colmap_id=-1,
+ depth_params=None,
+ image=Image.fromarray(np.zeros((reference_cam.image_height, reference_cam.image_width, 3), dtype=np.uint8)),
+ invdepthmap=None,
+ image_name=f"fully_smooth_path_i={i}",
+ uid=i
+ ))
+
+ return smooth_cameras
+
+from typing import List
+import numpy as np
+from sklearn.mixture import GaussianMixture
+from scipy.spatial.transform import Rotation as R, Slerp
+from PIL import Image
+
+def generate_clustered_smooth_cameras_with_tsp(existing_cameras: List[Camera],
+ n_selected: int = 30,
+ n_points_per_segment: int = 20,
+ d: float = 2.0,
+ n_clusters: int = 5,
+ closed: bool = False) -> List[Camera]:
+ """
+ Generate a fully smooth camera path using clustering + TSP between nearest cluster centers + TSP inside clusters.
+ Positions are normalized before clustering and denormalized before generating final cameras.
+
+ Args:
+ existing_cameras (List[Camera]): List of input cameras.
+ n_selected (int): Number of cameras to select after ordering.
+ n_points_per_segment (int): Number of interpolated points per spline segment.
+ d (float): Distance ahead for estimating center of view.
+ n_clusters (int): Number of GMM clusters.
+ closed (bool): Whether to close the path.
+
+ Returns:
+ List[Camera]: Smooth path of Camera objects.
+ """
+ # Extract positions and rotations
+ positions = np.array([cam.T for cam in existing_cameras])
+ rotations = np.array([R.from_matrix(cam.R).as_quat() for cam in existing_cameras])
+
+ # === Normalize positions
+ mean_pos = np.mean(positions, axis=0)
+ scale_pos = np.std(positions, axis=0)
+ scale_pos[scale_pos == 0] = 1.0 # avoid division by zero
+
+ positions_normalized = (positions - mean_pos) / scale_pos
+
+ # === Features for clustering (only positions, not rotations)
+ features = positions_normalized
+
+ # === 1. GMM clustering
+ gmm = GaussianMixture(n_components=n_clusters, covariance_type='full', random_state=42)
+ cluster_labels = gmm.fit_predict(features)
+
+ clusters = {}
+ cluster_centers = []
+
+ for cluster_id in range(n_clusters):
+ cluster_indices = np.where(cluster_labels == cluster_id)[0]
+ if len(cluster_indices) == 0:
+ continue
+ clusters[cluster_id] = cluster_indices
+ cluster_center = np.mean(features[cluster_indices], axis=0)
+ cluster_centers.append(cluster_center)
+
+ cluster_centers = np.stack(cluster_centers)
+
+ # === 2. Remap cluster centers to nearest existing cameras
+ if False:
+ mapped_centers = []
+ for center in cluster_centers:
+ dists = np.linalg.norm(features - center, axis=1)
+ nearest_idx = np.argmin(dists)
+ mapped_centers.append(features[nearest_idx])
+ mapped_centers = np.stack(mapped_centers)
+ cluster_centers = mapped_centers
+ # === 3. Solve TSP between mapped cluster centers
+ cluster_order = solve_tsp_2opt(cluster_centers)
+
+ # === 4. For each cluster, solve TSP inside cluster
+ final_indices = []
+ for cluster_id in cluster_order:
+ cluster_indices = clusters[cluster_id]
+ cluster_positions = features[cluster_indices]
+
+ if len(cluster_positions) == 1:
+ final_indices.append(cluster_indices[0])
+ continue
+
+ local_order = solve_tsp_nearest_neighbor(cluster_positions)
+ ordered_cluster_indices = cluster_indices[local_order]
+ final_indices.extend(ordered_cluster_indices)
+
+ ordered_cameras = [existing_cameras[i] for i in final_indices]
+ ordered_positions = positions_normalized[final_indices]
+
+ # === 5. Subsample uniformly
+ idx = np.linspace(0, len(ordered_positions) - 1, n_selected).astype(int)
+ sampled_positions = ordered_positions[idx]
+ sampled_cameras = [ordered_cameras[i] for i in idx]
+
+ # === 6. Prepare for Catmull-Rom spline
+ if closed:
+ sampled_positions = np.vstack([sampled_positions[-1], sampled_positions, sampled_positions[0], sampled_positions[1]])
+ else:
+ sampled_positions = np.vstack([sampled_positions[0], sampled_positions, sampled_positions[-1], sampled_positions[-1]])
+
+ # === 7. Smooth path positions
+ path_positions = []
+ for i in range(1, len(sampled_positions) - 2):
+ segment = catmull_rom_spline(sampled_positions[i-1], sampled_positions[i], sampled_positions[i+1], sampled_positions[i+2], n_points_per_segment)
+ path_positions.append(segment)
+ path_positions = np.concatenate(path_positions, axis=0)
+
+ # === 8. Denormalize
+ path_positions = path_positions * scale_pos + mean_pos
+
+ # === 9. SLERP for rotations
+ rotations = R.from_matrix([cam.R for cam in sampled_cameras])
+ key_times = np.linspace(0, 1, len(rotations))
+ slerp = Slerp(key_times, rotations)
+
+ query_times = np.linspace(0, 1, len(path_positions))
+ interpolated_rotations = slerp(query_times)
+
+ # === 10. Generate Camera objects
+ reference_cam = existing_cameras[0]
+ smooth_cameras = []
+
+ for i, pos in enumerate(path_positions):
+ R_interp = interpolated_rotations[i].as_matrix()
+
+ smooth_cameras.append(Camera(
+ R=R_interp,
+ T=pos,
+ FoVx=reference_cam.FoVx,
+ FoVy=reference_cam.FoVy,
+ resolution=(reference_cam.image_width, reference_cam.image_height),
+ colmap_id=-1,
+ depth_params=None,
+ image=Image.fromarray(np.zeros((reference_cam.image_height, reference_cam.image_width, 3), dtype=np.uint8)),
+ invdepthmap=None,
+ image_name=f"clustered_smooth_path_i={i}",
+ uid=i
+ ))
+
+ return smooth_cameras
+
+
+# def generate_clustered_path(existing_cameras: List[Camera],
+# n_points_per_segment: int = 20,
+# d: float = 2.0,
+# n_clusters: int = 5,
+# closed: bool = False) -> List[Camera]:
+# """
+# Generate a smooth camera path using GMM clustering and TSP on cluster centers.
+
+# Args:
+# existing_cameras (List[Camera]): List of input cameras.
+# n_points_per_segment (int): Number of interpolated points per spline segment.
+# d (float): Distance ahead for estimating center of view.
+# n_clusters (int): Number of GMM clusters (zones).
+# closed (bool): Whether to close the path.
+
+# Returns:
+# List[Camera]: Smooth path of Camera objects.
+# """
+# # Extract positions and rotations
+# positions = np.array([cam.T for cam in existing_cameras])
+
+# # === Normalize positions
+# mean_pos = np.mean(positions, axis=0)
+# scale_pos = np.std(positions, axis=0)
+# scale_pos[scale_pos == 0] = 1.0
+
+# positions_normalized = (positions - mean_pos) / scale_pos
+
+# # === 1. GMM clustering (only positions)
+# gmm = GaussianMixture(n_components=n_clusters, covariance_type='full', random_state=42)
+# cluster_labels = gmm.fit_predict(positions_normalized)
+
+# cluster_centers = []
+# for cluster_id in range(n_clusters):
+# cluster_indices = np.where(cluster_labels == cluster_id)[0]
+# if len(cluster_indices) == 0:
+# continue
+# cluster_center = np.mean(positions_normalized[cluster_indices], axis=0)
+# cluster_centers.append(cluster_center)
+
+# cluster_centers = np.stack(cluster_centers)
+
+# # === 2. Solve TSP between cluster centers
+# cluster_order = solve_tsp_2opt(cluster_centers)
+
+# # === 3. Reorder cluster centers
+# ordered_centers = cluster_centers[cluster_order]
+
+# # === 4. Prepare Catmull-Rom spline
+# if closed:
+# ordered_centers = np.vstack([ordered_centers[-1], ordered_centers, ordered_centers[0], ordered_centers[1]])
+# else:
+# ordered_centers = np.vstack([ordered_centers[0], ordered_centers, ordered_centers[-1], ordered_centers[-1]])
+
+# # === 5. Generate smooth path positions
+# path_positions = []
+# for i in range(1, len(ordered_centers) - 2):
+# segment = catmull_rom_spline(ordered_centers[i-1], ordered_centers[i], ordered_centers[i+1], ordered_centers[i+2], n_points_per_segment)
+# path_positions.append(segment)
+# path_positions = np.concatenate(path_positions, axis=0)
+
+# # === 6. Denormalize back
+# path_positions = path_positions * scale_pos + mean_pos
+
+# # === 7. Generate dummy rotations (constant forward facing)
+# reference_cam = existing_cameras[0]
+# default_rotation = R.from_matrix(reference_cam.R)
+
+# # For simplicity, fixed rotation for all
+# smooth_cameras = []
+
+# for i, pos in enumerate(path_positions):
+# R_interp = default_rotation.as_matrix()
+
+# smooth_cameras.append(Camera(
+# R=R_interp,
+# T=pos,
+# FoVx=reference_cam.FoVx,
+# FoVy=reference_cam.FoVy,
+# resolution=(reference_cam.image_width, reference_cam.image_height),
+# colmap_id=-1,
+# depth_params=None,
+# image=Image.fromarray(np.zeros((reference_cam.image_height, reference_cam.image_width, 3), dtype=np.uint8)),
+# invdepthmap=None,
+# image_name=f"cluster_path_i={i}",
+# uid=i
+# ))
+
+# return smooth_cameras
+
+from typing import List
+import numpy as np
+from sklearn.cluster import KMeans
+from scipy.spatial.transform import Rotation as R, Slerp
+from PIL import Image
+
+def generate_clustered_path(existing_cameras: List[Camera],
+ n_points_per_segment: int = 20,
+ d: float = 2.0,
+ n_clusters: int = 5,
+ closed: bool = False) -> List[Camera]:
+ """
+ Generate a smooth camera path using K-Means clustering and TSP on cluster centers.
+
+ Args:
+ existing_cameras (List[Camera]): List of input cameras.
+ n_points_per_segment (int): Number of interpolated points per spline segment.
+ d (float): Distance ahead for estimating center of view.
+ n_clusters (int): Number of KMeans clusters (zones).
+ closed (bool): Whether to close the path.
+
+ Returns:
+ List[Camera]: Smooth path of Camera objects.
+ """
+ # Extract positions
+ positions = np.array([cam.T for cam in existing_cameras])
+
+ # === Normalize positions
+ mean_pos = np.mean(positions, axis=0)
+ scale_pos = np.std(positions, axis=0)
+ scale_pos[scale_pos == 0] = 1.0
+
+ positions_normalized = (positions - mean_pos) / scale_pos
+
+ # === 1. K-Means clustering (only positions)
+ kmeans = KMeans(n_clusters=n_clusters, random_state=42, n_init='auto')
+ cluster_labels = kmeans.fit_predict(positions_normalized)
+
+ cluster_centers = []
+ for cluster_id in range(n_clusters):
+ cluster_indices = np.where(cluster_labels == cluster_id)[0]
+ if len(cluster_indices) == 0:
+ continue
+ cluster_center = np.mean(positions_normalized[cluster_indices], axis=0)
+ cluster_centers.append(cluster_center)
+
+ cluster_centers = np.stack(cluster_centers)
+
+ # === 2. Solve TSP between cluster centers
+ cluster_order = solve_tsp_2opt(cluster_centers)
+
+ # === 3. Reorder cluster centers
+ ordered_centers = cluster_centers[cluster_order]
+
+ # === 4. Prepare Catmull-Rom spline
+ if closed:
+ ordered_centers = np.vstack([ordered_centers[-1], ordered_centers, ordered_centers[0], ordered_centers[1]])
+ else:
+ ordered_centers = np.vstack([ordered_centers[0], ordered_centers, ordered_centers[-1], ordered_centers[-1]])
+
+ # === 5. Generate smooth path positions
+ path_positions = []
+ for i in range(1, len(ordered_centers) - 2):
+ segment = catmull_rom_spline(ordered_centers[i-1], ordered_centers[i], ordered_centers[i+1], ordered_centers[i+2], n_points_per_segment)
+ path_positions.append(segment)
+ path_positions = np.concatenate(path_positions, axis=0)
+
+ # === 6. Denormalize back
+ path_positions = path_positions * scale_pos + mean_pos
+
+ # === 7. Generate dummy rotations (constant forward facing)
+ reference_cam = existing_cameras[0]
+ default_rotation = R.from_matrix(reference_cam.R)
+
+ # For simplicity, fixed rotation for all
+ smooth_cameras = []
+
+ for i, pos in enumerate(path_positions):
+ R_interp = default_rotation.as_matrix()
+
+ smooth_cameras.append(Camera(
+ R=R_interp,
+ T=pos,
+ FoVx=reference_cam.FoVx,
+ FoVy=reference_cam.FoVy,
+ resolution=(reference_cam.image_width, reference_cam.image_height),
+ colmap_id=-1,
+ depth_params=None,
+ image=Image.fromarray(np.zeros((reference_cam.image_height, reference_cam.image_width, 3), dtype=np.uint8)),
+ invdepthmap=None,
+ image_name=f"cluster_path_i={i}",
+ uid=i
+ ))
+
+ return smooth_cameras
+
+
+
+
+def visualize_image_with_points(image, points):
+ """
+ Visualize an image with points overlaid on top. This is useful for correspondences visualizations
+
+ Parameters:
+ - image: PIL Image object
+ - points: Numpy array of shape [N, 2] containing (x, y) coordinates of points
+
+ Returns:
+ - None (displays the visualization)
+ """
+
+ # Convert PIL image to numpy array
+ img_array = np.array(image)
+
+ # Create a figure and axis
+ fig, ax = plt.subplots(figsize=(7,7))
+
+ # Display the image
+ ax.imshow(img_array)
+
+ # Scatter plot the points on top of the image
+ ax.scatter(points[:, 0], points[:, 1], color='red', marker='o', s=1)
+
+ # Show the plot
+ plt.show()
+
+
+def visualize_correspondences(image1, points1, image2, points2):
+ """
+ Visualize two images concatenated horizontally with key points and correspondences.
+
+ Parameters:
+ - image1: PIL Image object (left image)
+ - points1: Numpy array of shape [N, 2] containing (x, y) coordinates of key points for image1
+ - image2: PIL Image object (right image)
+ - points2: Numpy array of shape [N, 2] containing (x, y) coordinates of key points for image2
+
+ Returns:
+ - None (displays the visualization)
+ """
+
+ # Concatenate images horizontally
+ concatenated_image = np.concatenate((np.array(image1), np.array(image2)), axis=1)
+
+ # Create a figure and axis
+ fig, ax = plt.subplots(figsize=(10,10))
+
+ # Display the concatenated image
+ ax.imshow(concatenated_image)
+
+ # Plot key points on the left image
+ ax.scatter(points1[:, 0], points1[:, 1], color='red', marker='o', s=10)
+
+ # Plot key points on the right image
+ ax.scatter(points2[:, 0] + image1.width, points2[:, 1], color='blue', marker='o', s=10)
+
+ # Draw lines connecting corresponding key points
+ for i in range(len(points1)):
+ ax.plot([points1[i, 0], points2[i, 0] + image1.width], [points1[i, 1], points2[i, 1]])#, color='green')
+
+ # Show the plot
+ plt.show()
+
diff --git a/submodules/RoMa/.gitignore b/submodules/RoMa/.gitignore
new file mode 100644
index 0000000000000000000000000000000000000000..7f8801c4b89dd12eeca1e5709e8161f017370a44
--- /dev/null
+++ b/submodules/RoMa/.gitignore
@@ -0,0 +1,11 @@
+*.egg-info*
+*.vscode*
+*__pycache__*
+vis*
+workspace*
+.venv
+.DS_Store
+jobs/*
+*ignore_me*
+*.pth
+wandb*
\ No newline at end of file
diff --git a/submodules/RoMa/LICENSE b/submodules/RoMa/LICENSE
new file mode 100644
index 0000000000000000000000000000000000000000..ca95157052a76debc473afb395bffae0c1329e63
--- /dev/null
+++ b/submodules/RoMa/LICENSE
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Johan Edstedt
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/submodules/RoMa/README.md b/submodules/RoMa/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..8458762622603dc8d0b4a89ba5cb3354bee7deb5
--- /dev/null
+++ b/submodules/RoMa/README.md
@@ -0,0 +1,123 @@
+#
+
+
RoMa 🏛️: Robust Dense Feature Matching ⭐CVPR 2024⭐
+
+ Johan Edstedt
+ ·
+ Qiyu Sun
+ ·
+ Georg Bökman
+ ·
+ Mårten Wadenbäck
+ ·
+ Michael Felsberg
+
+
+ Paper |
+ Project Page
+
+
+
+
+
+
+
+ RoMa is the robust dense feature matcher capable of estimating pixel-dense warps and reliable certainties for almost any image pair.
+
+
+## Setup/Install
+In your python environment (tested on Linux python 3.10), run:
+```bash
+pip install -e .
+```
+## Demo / How to Use
+We provide two demos in the [demos folder](demo).
+Here's the gist of it:
+```python
+from romatch import roma_outdoor
+roma_model = roma_outdoor(device=device)
+# Match
+warp, certainty = roma_model.match(imA_path, imB_path, device=device)
+# Sample matches for estimation
+matches, certainty = roma_model.sample(warp, certainty)
+# Convert to pixel coordinates (RoMa produces matches in [-1,1]x[-1,1])
+kptsA, kptsB = roma_model.to_pixel_coordinates(matches, H_A, W_A, H_B, W_B)
+# Find a fundamental matrix (or anything else of interest)
+F, mask = cv2.findFundamentalMat(
+ kptsA.cpu().numpy(), kptsB.cpu().numpy(), ransacReprojThreshold=0.2, method=cv2.USAC_MAGSAC, confidence=0.999999, maxIters=10000
+)
+```
+
+**New**: You can also match arbitrary keypoints with RoMa. See [match_keypoints](romatch/models/matcher.py) in RegressionMatcher.
+
+## Settings
+
+### Resolution
+By default RoMa uses an initial resolution of (560,560) which is then upsampled to (864,864).
+You can change this at construction (see roma_outdoor kwargs).
+You can also change this later, by changing the roma_model.w_resized, roma_model.h_resized, and roma_model.upsample_res.
+
+### Sampling
+roma_model.sample_thresh controls the thresholding used when sampling matches for estimation. In certain cases a lower or higher threshold may improve results.
+
+
+## Reproducing Results
+The experiments in the paper are provided in the [experiments folder](experiments).
+
+### Training
+1. First follow the instructions provided here: https://github.com/Parskatt/DKM for downloading and preprocessing datasets.
+2. Run the relevant experiment, e.g.,
+```bash
+torchrun --nproc_per_node=4 --nnodes=1 --rdzv_backend=c10d experiments/roma_outdoor.py
+```
+### Testing
+```bash
+python experiments/roma_outdoor.py --only_test --benchmark mega-1500
+```
+## License
+All our code except DINOv2 is MIT license.
+DINOv2 has an Apache 2 license [DINOv2](https://github.com/facebookresearch/dinov2/blob/main/LICENSE).
+
+## Acknowledgement
+Our codebase builds on the code in [DKM](https://github.com/Parskatt/DKM).
+
+## Tiny RoMa
+If you find that RoMa is too heavy, you might want to try Tiny RoMa which is built on top of XFeat.
+```python
+from romatch import tiny_roma_v1_outdoor
+tiny_roma_model = tiny_roma_v1_outdoor(device=device)
+```
+Mega1500:
+| | AUC@5 | AUC@10 | AUC@20 |
+|----------|----------|----------|----------|
+| XFeat | 46.4 | 58.9 | 69.2 |
+| XFeat* | 51.9 | 67.2 | 78.9 |
+| Tiny RoMa v1 | 56.4 | 69.5 | 79.5 |
+| RoMa | - | - | - |
+
+Mega-8-Scenes (See DKM):
+| | AUC@5 | AUC@10 | AUC@20 |
+|----------|----------|----------|----------|
+| XFeat | - | - | - |
+| XFeat* | 50.1 | 64.4 | 75.2 |
+| Tiny RoMa v1 | 57.7 | 70.5 | 79.6 |
+| RoMa | - | - | - |
+
+IMC22 :'):
+| | mAA@10 |
+|----------|----------|
+| XFeat | 42.1 |
+| XFeat* | - |
+| Tiny RoMa v1 | 42.2 |
+| RoMa | - |
+
+## BibTeX
+If you find our models useful, please consider citing our paper!
+```
+@article{edstedt2024roma,
+title={{RoMa: Robust Dense Feature Matching}},
+author={Edstedt, Johan and Sun, Qiyu and Bökman, Georg and Wadenbäck, Mårten and Felsberg, Michael},
+journal={IEEE Conference on Computer Vision and Pattern Recognition},
+year={2024}
+}
+```
diff --git a/submodules/RoMa/data/.gitignore b/submodules/RoMa/data/.gitignore
new file mode 100644
index 0000000000000000000000000000000000000000..c96a04f008ee21e260b28f7701595ed59e2839e3
--- /dev/null
+++ b/submodules/RoMa/data/.gitignore
@@ -0,0 +1,2 @@
+*
+!.gitignore
\ No newline at end of file
diff --git a/submodules/RoMa/demo/demo_3D_effect.py b/submodules/RoMa/demo/demo_3D_effect.py
new file mode 100644
index 0000000000000000000000000000000000000000..ae26caaf92deb884dfabb6eca96aec3406325c3f
--- /dev/null
+++ b/submodules/RoMa/demo/demo_3D_effect.py
@@ -0,0 +1,47 @@
+from PIL import Image
+import torch
+import torch.nn.functional as F
+import numpy as np
+from romatch.utils.utils import tensor_to_pil
+
+from romatch import roma_outdoor
+
+device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+if torch.backends.mps.is_available():
+ device = torch.device('mps')
+
+if __name__ == "__main__":
+ from argparse import ArgumentParser
+ parser = ArgumentParser()
+ parser.add_argument("--im_A_path", default="assets/toronto_A.jpg", type=str)
+ parser.add_argument("--im_B_path", default="assets/toronto_B.jpg", type=str)
+ parser.add_argument("--save_path", default="demo/gif/roma_warp_toronto", type=str)
+
+ args, _ = parser.parse_known_args()
+ im1_path = args.im_A_path
+ im2_path = args.im_B_path
+ save_path = args.save_path
+
+ # Create model
+ roma_model = roma_outdoor(device=device, coarse_res=560, upsample_res=(864, 1152))
+ roma_model.symmetric = False
+
+ H, W = roma_model.get_output_resolution()
+
+ im1 = Image.open(im1_path).resize((W, H))
+ im2 = Image.open(im2_path).resize((W, H))
+
+ # Match
+ warp, certainty = roma_model.match(im1_path, im2_path, device=device)
+ # Sampling not needed, but can be done with model.sample(warp, certainty)
+ x1 = (torch.tensor(np.array(im1)) / 255).to(device).permute(2, 0, 1)
+ x2 = (torch.tensor(np.array(im2)) / 255).to(device).permute(2, 0, 1)
+
+ coords_A, coords_B = warp[...,:2], warp[...,2:]
+ for i, x in enumerate(np.linspace(0,2*np.pi,200)):
+ t = (1 + np.cos(x))/2
+ interp_warp = (1-t)*coords_A + t*coords_B
+ im2_transfer_rgb = F.grid_sample(
+ x2[None], interp_warp[None], mode="bilinear", align_corners=False
+ )[0]
+ tensor_to_pil(im2_transfer_rgb, unnormalize=False).save(f"{save_path}_{i:03d}.jpg")
\ No newline at end of file
diff --git a/submodules/RoMa/demo/demo_fundamental.py b/submodules/RoMa/demo/demo_fundamental.py
new file mode 100644
index 0000000000000000000000000000000000000000..65ea9ccb76525da3e88e4f426bdebdc4fe742161
--- /dev/null
+++ b/submodules/RoMa/demo/demo_fundamental.py
@@ -0,0 +1,34 @@
+from PIL import Image
+import torch
+import cv2
+from romatch import roma_outdoor
+
+device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+if torch.backends.mps.is_available():
+ device = torch.device('mps')
+
+if __name__ == "__main__":
+ from argparse import ArgumentParser
+ parser = ArgumentParser()
+ parser.add_argument("--im_A_path", default="assets/sacre_coeur_A.jpg", type=str)
+ parser.add_argument("--im_B_path", default="assets/sacre_coeur_B.jpg", type=str)
+
+ args, _ = parser.parse_known_args()
+ im1_path = args.im_A_path
+ im2_path = args.im_B_path
+
+ # Create model
+ roma_model = roma_outdoor(device=device)
+
+
+ W_A, H_A = Image.open(im1_path).size
+ W_B, H_B = Image.open(im2_path).size
+
+ # Match
+ warp, certainty = roma_model.match(im1_path, im2_path, device=device)
+ # Sample matches for estimation
+ matches, certainty = roma_model.sample(warp, certainty)
+ kpts1, kpts2 = roma_model.to_pixel_coordinates(matches, H_A, W_A, H_B, W_B)
+ F, mask = cv2.findFundamentalMat(
+ kpts1.cpu().numpy(), kpts2.cpu().numpy(), ransacReprojThreshold=0.2, method=cv2.USAC_MAGSAC, confidence=0.999999, maxIters=10000
+ )
\ No newline at end of file
diff --git a/submodules/RoMa/demo/demo_match.py b/submodules/RoMa/demo/demo_match.py
new file mode 100644
index 0000000000000000000000000000000000000000..582767e19d8b50c6c241ea32f81cabb38f52fce2
--- /dev/null
+++ b/submodules/RoMa/demo/demo_match.py
@@ -0,0 +1,50 @@
+import os
+os.environ['PYTORCH_ENABLE_MPS_FALLBACK'] = '1'
+import torch
+from PIL import Image
+import torch.nn.functional as F
+import numpy as np
+from romatch.utils.utils import tensor_to_pil
+
+from romatch import roma_outdoor
+
+device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+if torch.backends.mps.is_available():
+ device = torch.device('mps')
+
+if __name__ == "__main__":
+ from argparse import ArgumentParser
+ parser = ArgumentParser()
+ parser.add_argument("--im_A_path", default="assets/toronto_A.jpg", type=str)
+ parser.add_argument("--im_B_path", default="assets/toronto_B.jpg", type=str)
+ parser.add_argument("--save_path", default="demo/roma_warp_toronto.jpg", type=str)
+
+ args, _ = parser.parse_known_args()
+ im1_path = args.im_A_path
+ im2_path = args.im_B_path
+ save_path = args.save_path
+
+ # Create model
+ roma_model = roma_outdoor(device=device, coarse_res=560, upsample_res=(864, 1152))
+
+ H, W = roma_model.get_output_resolution()
+
+ im1 = Image.open(im1_path).resize((W, H))
+ im2 = Image.open(im2_path).resize((W, H))
+
+ # Match
+ warp, certainty = roma_model.match(im1_path, im2_path, device=device)
+ # Sampling not needed, but can be done with model.sample(warp, certainty)
+ x1 = (torch.tensor(np.array(im1)) / 255).to(device).permute(2, 0, 1)
+ x2 = (torch.tensor(np.array(im2)) / 255).to(device).permute(2, 0, 1)
+
+ im2_transfer_rgb = F.grid_sample(
+ x2[None], warp[:,:W, 2:][None], mode="bilinear", align_corners=False
+ )[0]
+ im1_transfer_rgb = F.grid_sample(
+ x1[None], warp[:, W:, :2][None], mode="bilinear", align_corners=False
+ )[0]
+ warp_im = torch.cat((im2_transfer_rgb,im1_transfer_rgb),dim=2)
+ white_im = torch.ones((H,2*W),device=device)
+ vis_im = certainty * warp_im + (1 - certainty) * white_im
+ tensor_to_pil(vis_im, unnormalize=False).save(save_path)
\ No newline at end of file
diff --git a/submodules/RoMa/demo/demo_match_opencv_sift.py b/submodules/RoMa/demo/demo_match_opencv_sift.py
new file mode 100644
index 0000000000000000000000000000000000000000..3196fcfaab248f6c4c6247a0afb4db745206aee8
--- /dev/null
+++ b/submodules/RoMa/demo/demo_match_opencv_sift.py
@@ -0,0 +1,43 @@
+from PIL import Image
+import numpy as np
+
+import numpy as np
+import cv2 as cv
+import matplotlib.pyplot as plt
+
+
+
+if __name__ == "__main__":
+ from argparse import ArgumentParser
+ parser = ArgumentParser()
+ parser.add_argument("--im_A_path", default="assets/toronto_A.jpg", type=str)
+ parser.add_argument("--im_B_path", default="assets/toronto_B.jpg", type=str)
+ parser.add_argument("--save_path", default="demo/roma_warp_toronto.jpg", type=str)
+
+ args, _ = parser.parse_known_args()
+ im1_path = args.im_A_path
+ im2_path = args.im_B_path
+ save_path = args.save_path
+
+ img1 = cv.imread(im1_path,cv.IMREAD_GRAYSCALE) # queryImage
+ img2 = cv.imread(im2_path,cv.IMREAD_GRAYSCALE) # trainImage
+ # Initiate SIFT detector
+ sift = cv.SIFT_create()
+ # find the keypoints and descriptors with SIFT
+ kp1, des1 = sift.detectAndCompute(img1,None)
+ kp2, des2 = sift.detectAndCompute(img2,None)
+ # BFMatcher with default params
+ bf = cv.BFMatcher()
+ matches = bf.knnMatch(des1,des2,k=2)
+ # Apply ratio test
+ good = []
+ for m,n in matches:
+ if m.distance < 0.75*n.distance:
+ good.append([m])
+ # cv.drawMatchesKnn expects list of lists as matches.
+ draw_params = dict(matchColor = (255,0,0), # draw matches in red color
+ singlePointColor = None,
+ flags = 2)
+
+ img3 = cv.drawMatchesKnn(img1,kp1,img2,kp2,good,None,**draw_params)
+ Image.fromarray(img3).save("demo/sift_matches.png")
diff --git a/submodules/RoMa/demo/demo_match_tiny.py b/submodules/RoMa/demo/demo_match_tiny.py
new file mode 100644
index 0000000000000000000000000000000000000000..b8e66a4b80a2361e22673ddc59632f48ad653b69
--- /dev/null
+++ b/submodules/RoMa/demo/demo_match_tiny.py
@@ -0,0 +1,77 @@
+import os
+os.environ['PYTORCH_ENABLE_MPS_FALLBACK'] = '1'
+import torch
+from PIL import Image
+import torch.nn.functional as F
+import numpy as np
+from romatch.utils.utils import tensor_to_pil
+
+from romatch import tiny_roma_v1_outdoor
+
+device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+if torch.backends.mps.is_available():
+ device = torch.device('mps')
+
+if __name__ == "__main__":
+ from argparse import ArgumentParser
+ parser = ArgumentParser()
+ parser.add_argument("--im_A_path", default="assets/sacre_coeur_A.jpg", type=str)
+ parser.add_argument("--im_B_path", default="assets/sacre_coeur_B.jpg", type=str)
+ parser.add_argument("--save_A_path", default="demo/tiny_roma_warp_A.jpg", type=str)
+ parser.add_argument("--save_B_path", default="demo/tiny_roma_warp_B.jpg", type=str)
+
+ args, _ = parser.parse_known_args()
+ im1_path = args.im_A_path
+ im2_path = args.im_B_path
+
+ # Create model
+ roma_model = tiny_roma_v1_outdoor(device=device)
+
+ # Match
+ warp, certainty1 = roma_model.match(im1_path, im2_path)
+
+ h1, w1 = warp.shape[:2]
+
+ # maybe im1.size != im2.size
+ im1 = Image.open(im1_path).resize((w1, h1))
+ im2 = Image.open(im2_path)
+ x1 = (torch.tensor(np.array(im1)) / 255).to(device).permute(2, 0, 1)
+ x2 = (torch.tensor(np.array(im2)) / 255).to(device).permute(2, 0, 1)
+
+ h2, w2 = x2.shape[1:]
+ g1_p2x = w2 / 2 * (warp[..., 2] + 1)
+ g1_p2y = h2 / 2 * (warp[..., 3] + 1)
+ g2_p1x = torch.zeros((h2, w2), dtype=torch.float32).to(device) - 2
+ g2_p1y = torch.zeros((h2, w2), dtype=torch.float32).to(device) - 2
+
+ x, y = torch.meshgrid(
+ torch.arange(w1, device=device),
+ torch.arange(h1, device=device),
+ indexing="xy",
+ )
+ g2x = torch.round(g1_p2x[y, x]).long()
+ g2y = torch.round(g1_p2y[y, x]).long()
+ idx_x = torch.bitwise_and(0 <= g2x, g2x < w2)
+ idx_y = torch.bitwise_and(0 <= g2y, g2y < h2)
+ idx = torch.bitwise_and(idx_x, idx_y)
+ g2_p1x[g2y[idx], g2x[idx]] = x[idx].float() * 2 / w1 - 1
+ g2_p1y[g2y[idx], g2x[idx]] = y[idx].float() * 2 / h1 - 1
+
+ certainty2 = F.grid_sample(
+ certainty1[None][None],
+ torch.stack([g2_p1x, g2_p1y], dim=2)[None],
+ mode="bilinear",
+ align_corners=False,
+ )[0]
+
+ white_im1 = torch.ones((h1, w1), device = device)
+ white_im2 = torch.ones((h2, w2), device = device)
+
+ certainty1 = F.avg_pool2d(certainty1[None], kernel_size=5, stride=1, padding=2)[0]
+ certainty2 = F.avg_pool2d(certainty2[None], kernel_size=5, stride=1, padding=2)[0]
+
+ vis_im1 = certainty1 * x1 + (1 - certainty1) * white_im1
+ vis_im2 = certainty2 * x2 + (1 - certainty2) * white_im2
+
+ tensor_to_pil(vis_im1, unnormalize=False).save(args.save_A_path)
+ tensor_to_pil(vis_im2, unnormalize=False).save(args.save_B_path)
\ No newline at end of file
diff --git a/submodules/RoMa/demo/gif/.gitignore b/submodules/RoMa/demo/gif/.gitignore
new file mode 100644
index 0000000000000000000000000000000000000000..c96a04f008ee21e260b28f7701595ed59e2839e3
--- /dev/null
+++ b/submodules/RoMa/demo/gif/.gitignore
@@ -0,0 +1,2 @@
+*
+!.gitignore
\ No newline at end of file
diff --git a/submodules/RoMa/experiments/eval_roma_outdoor.py b/submodules/RoMa/experiments/eval_roma_outdoor.py
new file mode 100644
index 0000000000000000000000000000000000000000..de549ae2952f0efd5c209dc92da5afe811f4ba7a
--- /dev/null
+++ b/submodules/RoMa/experiments/eval_roma_outdoor.py
@@ -0,0 +1,57 @@
+import json
+
+from romatch.benchmarks import MegadepthDenseBenchmark
+from romatch.benchmarks import MegaDepthPoseEstimationBenchmark, HpatchesHomogBenchmark
+from romatch.benchmarks import Mega1500PoseLibBenchmark
+
+def test_mega_8_scenes(model, name):
+ mega_8_scenes_benchmark = MegaDepthPoseEstimationBenchmark("data/megadepth",
+ scene_names=['mega_8_scenes_0019_0.1_0.3.npz',
+ 'mega_8_scenes_0025_0.1_0.3.npz',
+ 'mega_8_scenes_0021_0.1_0.3.npz',
+ 'mega_8_scenes_0008_0.1_0.3.npz',
+ 'mega_8_scenes_0032_0.1_0.3.npz',
+ 'mega_8_scenes_1589_0.1_0.3.npz',
+ 'mega_8_scenes_0063_0.1_0.3.npz',
+ 'mega_8_scenes_0024_0.1_0.3.npz',
+ 'mega_8_scenes_0019_0.3_0.5.npz',
+ 'mega_8_scenes_0025_0.3_0.5.npz',
+ 'mega_8_scenes_0021_0.3_0.5.npz',
+ 'mega_8_scenes_0008_0.3_0.5.npz',
+ 'mega_8_scenes_0032_0.3_0.5.npz',
+ 'mega_8_scenes_1589_0.3_0.5.npz',
+ 'mega_8_scenes_0063_0.3_0.5.npz',
+ 'mega_8_scenes_0024_0.3_0.5.npz'])
+ mega_8_scenes_results = mega_8_scenes_benchmark.benchmark(model, model_name=name)
+ print(mega_8_scenes_results)
+ json.dump(mega_8_scenes_results, open(f"results/mega_8_scenes_{name}.json", "w"))
+
+def test_mega1500(model, name):
+ mega1500_benchmark = MegaDepthPoseEstimationBenchmark("data/megadepth")
+ mega1500_results = mega1500_benchmark.benchmark(model, model_name=name)
+ json.dump(mega1500_results, open(f"results/mega1500_{name}.json", "w"))
+
+def test_mega1500_poselib(model, name):
+ mega1500_benchmark = Mega1500PoseLibBenchmark("data/megadepth")
+ mega1500_results = mega1500_benchmark.benchmark(model, model_name=name)
+ json.dump(mega1500_results, open(f"results/mega1500_{name}.json", "w"))
+
+def test_mega_dense(model, name):
+ megadense_benchmark = MegadepthDenseBenchmark("data/megadepth", num_samples = 1000)
+ megadense_results = megadense_benchmark.benchmark(model)
+ json.dump(megadense_results, open(f"results/mega_dense_{name}.json", "w"))
+
+def test_hpatches(model, name):
+ hpatches_benchmark = HpatchesHomogBenchmark("data/hpatches")
+ hpatches_results = hpatches_benchmark.benchmark(model)
+ json.dump(hpatches_results, open(f"results/hpatches_{name}.json", "w"))
+
+
+if __name__ == "__main__":
+ from romatch import roma_outdoor
+ device = "cuda"
+ model = roma_outdoor(device = device, coarse_res = 672, upsample_res = 1344)
+ experiment_name = "roma_latest"
+ test_mega1500(model, experiment_name)
+ #test_mega1500_poselib(model, experiment_name)
+
diff --git a/submodules/RoMa/experiments/eval_tiny_roma_v1_outdoor.py b/submodules/RoMa/experiments/eval_tiny_roma_v1_outdoor.py
new file mode 100644
index 0000000000000000000000000000000000000000..0541ffce7bcfe92c3016572dd3b450fc3535c90c
--- /dev/null
+++ b/submodules/RoMa/experiments/eval_tiny_roma_v1_outdoor.py
@@ -0,0 +1,84 @@
+import torch
+import os
+from pathlib import Path
+import json
+from romatch.benchmarks import ScanNetBenchmark
+from romatch.benchmarks import Mega1500PoseLibBenchmark, ScanNetPoselibBenchmark
+from romatch.benchmarks import MegaDepthPoseEstimationBenchmark
+
+def test_mega_8_scenes(model, name):
+ mega_8_scenes_benchmark = MegaDepthPoseEstimationBenchmark("data/megadepth",
+ scene_names=['mega_8_scenes_0019_0.1_0.3.npz',
+ 'mega_8_scenes_0025_0.1_0.3.npz',
+ 'mega_8_scenes_0021_0.1_0.3.npz',
+ 'mega_8_scenes_0008_0.1_0.3.npz',
+ 'mega_8_scenes_0032_0.1_0.3.npz',
+ 'mega_8_scenes_1589_0.1_0.3.npz',
+ 'mega_8_scenes_0063_0.1_0.3.npz',
+ 'mega_8_scenes_0024_0.1_0.3.npz',
+ 'mega_8_scenes_0019_0.3_0.5.npz',
+ 'mega_8_scenes_0025_0.3_0.5.npz',
+ 'mega_8_scenes_0021_0.3_0.5.npz',
+ 'mega_8_scenes_0008_0.3_0.5.npz',
+ 'mega_8_scenes_0032_0.3_0.5.npz',
+ 'mega_8_scenes_1589_0.3_0.5.npz',
+ 'mega_8_scenes_0063_0.3_0.5.npz',
+ 'mega_8_scenes_0024_0.3_0.5.npz'])
+ mega_8_scenes_results = mega_8_scenes_benchmark.benchmark(model, model_name=name)
+ print(mega_8_scenes_results)
+ json.dump(mega_8_scenes_results, open(f"results/mega_8_scenes_{name}.json", "w"))
+
+def test_mega1500(model, name):
+ mega1500_benchmark = MegaDepthPoseEstimationBenchmark("data/megadepth")
+ mega1500_results = mega1500_benchmark.benchmark(model, model_name=name)
+ json.dump(mega1500_results, open(f"results/mega1500_{name}.json", "w"))
+
+def test_mega1500_poselib(model, name):
+ #model.exact_softmax = True
+ mega1500_benchmark = Mega1500PoseLibBenchmark("data/megadepth", num_ransac_iter = 1, test_every = 1)
+ mega1500_results = mega1500_benchmark.benchmark(model, model_name=name)
+ json.dump(mega1500_results, open(f"results/mega1500_poselib_{name}.json", "w"))
+
+def test_mega_8_scenes_poselib(model, name):
+ mega1500_benchmark = Mega1500PoseLibBenchmark("data/megadepth", num_ransac_iter = 1, test_every = 1,
+ scene_names=['mega_8_scenes_0019_0.1_0.3.npz',
+ 'mega_8_scenes_0025_0.1_0.3.npz',
+ 'mega_8_scenes_0021_0.1_0.3.npz',
+ 'mega_8_scenes_0008_0.1_0.3.npz',
+ 'mega_8_scenes_0032_0.1_0.3.npz',
+ 'mega_8_scenes_1589_0.1_0.3.npz',
+ 'mega_8_scenes_0063_0.1_0.3.npz',
+ 'mega_8_scenes_0024_0.1_0.3.npz',
+ 'mega_8_scenes_0019_0.3_0.5.npz',
+ 'mega_8_scenes_0025_0.3_0.5.npz',
+ 'mega_8_scenes_0021_0.3_0.5.npz',
+ 'mega_8_scenes_0008_0.3_0.5.npz',
+ 'mega_8_scenes_0032_0.3_0.5.npz',
+ 'mega_8_scenes_1589_0.3_0.5.npz',
+ 'mega_8_scenes_0063_0.3_0.5.npz',
+ 'mega_8_scenes_0024_0.3_0.5.npz'])
+ mega1500_results = mega1500_benchmark.benchmark(model, model_name=name)
+ json.dump(mega1500_results, open(f"results/mega_8_scenes_poselib_{name}.json", "w"))
+
+def test_scannet_poselib(model, name):
+ scannet_benchmark = ScanNetPoselibBenchmark("data/scannet")
+ scannet_results = scannet_benchmark.benchmark(model)
+ json.dump(scannet_results, open(f"results/scannet_{name}.json", "w"))
+
+def test_scannet(model, name):
+ scannet_benchmark = ScanNetBenchmark("data/scannet")
+ scannet_results = scannet_benchmark.benchmark(model)
+ json.dump(scannet_results, open(f"results/scannet_{name}.json", "w"))
+
+if __name__ == "__main__":
+ os.environ["TORCH_CUDNN_V8_API_ENABLED"] = "1" # For BF16 computations
+ os.environ["OMP_NUM_THREADS"] = "16"
+ torch.backends.cudnn.allow_tf32 = True # allow tf32 on cudnn
+ from romatch import tiny_roma_v1_outdoor
+
+ experiment_name = Path(__file__).stem
+ device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+ model = tiny_roma_v1_outdoor(device)
+ #test_mega1500_poselib(model, experiment_name)
+ test_mega_8_scenes_poselib(model, experiment_name)
+
\ No newline at end of file
diff --git a/submodules/RoMa/experiments/roma_indoor.py b/submodules/RoMa/experiments/roma_indoor.py
new file mode 100644
index 0000000000000000000000000000000000000000..c7df478d1a352ad840cb7f2d84f0de22fbe5835e
--- /dev/null
+++ b/submodules/RoMa/experiments/roma_indoor.py
@@ -0,0 +1,320 @@
+import os
+import torch
+from argparse import ArgumentParser
+
+from torch import nn
+from torch.utils.data import ConcatDataset
+import torch.distributed as dist
+from torch.nn.parallel import DistributedDataParallel as DDP
+
+import json
+import wandb
+from tqdm import tqdm
+
+from romatch.benchmarks import MegadepthDenseBenchmark
+from romatch.datasets.megadepth import MegadepthBuilder
+from romatch.datasets.scannet import ScanNetBuilder
+from romatch.losses.robust_loss import RobustLosses
+from romatch.benchmarks import MegadepthDenseBenchmark, ScanNetBenchmark
+from romatch.train.train import train_k_steps
+from romatch.models.matcher import *
+from romatch.models.transformer import Block, TransformerDecoder, MemEffAttention
+from romatch.models.encoders import *
+from romatch.checkpointing import CheckPoint
+
+resolutions = {"low":(448, 448), "medium":(14*8*5, 14*8*5), "high":(14*8*6, 14*8*6)}
+
+def get_model(pretrained_backbone=True, resolution = "medium", **kwargs):
+ gp_dim = 512
+ feat_dim = 512
+ decoder_dim = gp_dim + feat_dim
+ cls_to_coord_res = 64
+ coordinate_decoder = TransformerDecoder(
+ nn.Sequential(*[Block(decoder_dim, 8, attn_class=MemEffAttention) for _ in range(5)]),
+ decoder_dim,
+ cls_to_coord_res**2 + 1,
+ is_classifier=True,
+ amp = True,
+ pos_enc = False,)
+ dw = True
+ hidden_blocks = 8
+ kernel_size = 5
+ displacement_emb = "linear"
+ disable_local_corr_grad = True
+
+ conv_refiner = nn.ModuleDict(
+ {
+ "16": ConvRefiner(
+ 2 * 512+128+(2*7+1)**2,
+ 2 * 512+128+(2*7+1)**2,
+ 2 + 1,
+ kernel_size=kernel_size,
+ dw=dw,
+ hidden_blocks=hidden_blocks,
+ displacement_emb=displacement_emb,
+ displacement_emb_dim=128,
+ local_corr_radius = 7,
+ corr_in_other = True,
+ amp = True,
+ disable_local_corr_grad = disable_local_corr_grad,
+ bn_momentum = 0.01,
+ ),
+ "8": ConvRefiner(
+ 2 * 512+64+(2*3+1)**2,
+ 2 * 512+64+(2*3+1)**2,
+ 2 + 1,
+ kernel_size=kernel_size,
+ dw=dw,
+ hidden_blocks=hidden_blocks,
+ displacement_emb=displacement_emb,
+ displacement_emb_dim=64,
+ local_corr_radius = 3,
+ corr_in_other = True,
+ amp = True,
+ disable_local_corr_grad = disable_local_corr_grad,
+ bn_momentum = 0.01,
+ ),
+ "4": ConvRefiner(
+ 2 * 256+32+(2*2+1)**2,
+ 2 * 256+32+(2*2+1)**2,
+ 2 + 1,
+ kernel_size=kernel_size,
+ dw=dw,
+ hidden_blocks=hidden_blocks,
+ displacement_emb=displacement_emb,
+ displacement_emb_dim=32,
+ local_corr_radius = 2,
+ corr_in_other = True,
+ amp = True,
+ disable_local_corr_grad = disable_local_corr_grad,
+ bn_momentum = 0.01,
+ ),
+ "2": ConvRefiner(
+ 2 * 64+16,
+ 128+16,
+ 2 + 1,
+ kernel_size=kernel_size,
+ dw=dw,
+ hidden_blocks=hidden_blocks,
+ displacement_emb=displacement_emb,
+ displacement_emb_dim=16,
+ amp = True,
+ disable_local_corr_grad = disable_local_corr_grad,
+ bn_momentum = 0.01,
+ ),
+ "1": ConvRefiner(
+ 2 * 9 + 6,
+ 24,
+ 2 + 1,
+ kernel_size=kernel_size,
+ dw=dw,
+ hidden_blocks = hidden_blocks,
+ displacement_emb = displacement_emb,
+ displacement_emb_dim = 6,
+ amp = True,
+ disable_local_corr_grad = disable_local_corr_grad,
+ bn_momentum = 0.01,
+ ),
+ }
+ )
+ kernel_temperature = 0.2
+ learn_temperature = False
+ no_cov = True
+ kernel = CosKernel
+ only_attention = False
+ basis = "fourier"
+ gp16 = GP(
+ kernel,
+ T=kernel_temperature,
+ learn_temperature=learn_temperature,
+ only_attention=only_attention,
+ gp_dim=gp_dim,
+ basis=basis,
+ no_cov=no_cov,
+ )
+ gps = nn.ModuleDict({"16": gp16})
+ proj16 = nn.Sequential(nn.Conv2d(1024, 512, 1, 1), nn.BatchNorm2d(512))
+ proj8 = nn.Sequential(nn.Conv2d(512, 512, 1, 1), nn.BatchNorm2d(512))
+ proj4 = nn.Sequential(nn.Conv2d(256, 256, 1, 1), nn.BatchNorm2d(256))
+ proj2 = nn.Sequential(nn.Conv2d(128, 64, 1, 1), nn.BatchNorm2d(64))
+ proj1 = nn.Sequential(nn.Conv2d(64, 9, 1, 1), nn.BatchNorm2d(9))
+ proj = nn.ModuleDict({
+ "16": proj16,
+ "8": proj8,
+ "4": proj4,
+ "2": proj2,
+ "1": proj1,
+ })
+ displacement_dropout_p = 0.0
+ gm_warp_dropout_p = 0.0
+ decoder = Decoder(coordinate_decoder,
+ gps,
+ proj,
+ conv_refiner,
+ detach=True,
+ scales=["16", "8", "4", "2", "1"],
+ displacement_dropout_p = displacement_dropout_p,
+ gm_warp_dropout_p = gm_warp_dropout_p)
+ h,w = resolutions[resolution]
+ encoder = CNNandDinov2(
+ cnn_kwargs = dict(
+ pretrained=pretrained_backbone,
+ amp = True),
+ amp = True,
+ use_vgg = True,
+ )
+ matcher = RegressionMatcher(encoder, decoder, h=h, w=w, alpha=1, beta=0,**kwargs)
+ return matcher
+
+def train(args):
+ dist.init_process_group('nccl')
+ #torch._dynamo.config.verbose=True
+ gpus = int(os.environ['WORLD_SIZE'])
+ # create model and move it to GPU with id rank
+ rank = dist.get_rank()
+ print(f"Start running DDP on rank {rank}")
+ device_id = rank % torch.cuda.device_count()
+ romatch.LOCAL_RANK = device_id
+ torch.cuda.set_device(device_id)
+
+ resolution = args.train_resolution
+ wandb_log = not args.dont_log_wandb
+ experiment_name = os.path.splitext(os.path.basename(__file__))[0]
+ wandb_mode = "online" if wandb_log and rank == 0 and False else "disabled"
+ wandb.init(project="romatch", entity=args.wandb_entity, name=experiment_name, reinit=False, mode = wandb_mode)
+ checkpoint_dir = "workspace/checkpoints/"
+ h,w = resolutions[resolution]
+ model = get_model(pretrained_backbone=True, resolution=resolution, attenuate_cert = False).to(device_id)
+ # Num steps
+ global_step = 0
+ batch_size = args.gpu_batch_size
+ step_size = gpus*batch_size
+ romatch.STEP_SIZE = step_size
+
+ N = (32 * 250000) # 250k steps of batch size 32
+ # checkpoint every
+ k = 25000 // romatch.STEP_SIZE
+
+ # Data
+ mega = MegadepthBuilder(data_root="data/megadepth", loftr_ignore=True, imc21_ignore = True)
+ use_horizontal_flip_aug = True
+ rot_prob = 0
+ depth_interpolation_mode = "bilinear"
+ megadepth_train1 = mega.build_scenes(
+ split="train_loftr", min_overlap=0.01, shake_t=32, use_horizontal_flip_aug = use_horizontal_flip_aug, rot_prob = rot_prob,
+ ht=h,wt=w,
+ )
+ megadepth_train2 = mega.build_scenes(
+ split="train_loftr", min_overlap=0.35, shake_t=32, use_horizontal_flip_aug = use_horizontal_flip_aug, rot_prob = rot_prob,
+ ht=h,wt=w,
+ )
+ megadepth_train = ConcatDataset(megadepth_train1 + megadepth_train2)
+ mega_ws = mega.weight_scenes(megadepth_train, alpha=0.75)
+
+ scannet = ScanNetBuilder(data_root="data/scannet")
+ scannet_train = scannet.build_scenes(split="train", ht=h, wt=w, use_horizontal_flip_aug = use_horizontal_flip_aug)
+ scannet_train = ConcatDataset(scannet_train)
+ scannet_ws = scannet.weight_scenes(scannet_train, alpha=0.75)
+
+ # Loss and optimizer
+ depth_loss_scannet = RobustLosses(
+ ce_weight=0.0,
+ local_dist={1:4, 2:4, 4:8, 8:8},
+ local_largest_scale=8,
+ depth_interpolation_mode=depth_interpolation_mode,
+ alpha = 0.5,
+ c = 1e-4,)
+ # Loss and optimizer
+ depth_loss_mega = RobustLosses(
+ ce_weight=0.01,
+ local_dist={1:4, 2:4, 4:8, 8:8},
+ local_largest_scale=8,
+ depth_interpolation_mode=depth_interpolation_mode,
+ alpha = 0.5,
+ c = 1e-4,)
+ parameters = [
+ {"params": model.encoder.parameters(), "lr": romatch.STEP_SIZE * 5e-6 / 8},
+ {"params": model.decoder.parameters(), "lr": romatch.STEP_SIZE * 1e-4 / 8},
+ ]
+ optimizer = torch.optim.AdamW(parameters, weight_decay=0.01)
+ lr_scheduler = torch.optim.lr_scheduler.MultiStepLR(
+ optimizer, milestones=[(9*N/romatch.STEP_SIZE)//10])
+ megadense_benchmark = MegadepthDenseBenchmark("data/megadepth", num_samples = 1000, h=h,w=w)
+ checkpointer = CheckPoint(checkpoint_dir, experiment_name)
+ model, optimizer, lr_scheduler, global_step = checkpointer.load(model, optimizer, lr_scheduler, global_step)
+ romatch.GLOBAL_STEP = global_step
+ ddp_model = DDP(model, device_ids=[device_id], find_unused_parameters = False, gradient_as_bucket_view=True)
+ grad_scaler = torch.cuda.amp.GradScaler(growth_interval=1_000_000)
+ grad_clip_norm = 0.01
+ for n in range(romatch.GLOBAL_STEP, N, k * romatch.STEP_SIZE):
+ mega_sampler = torch.utils.data.WeightedRandomSampler(
+ mega_ws, num_samples = batch_size * k, replacement=False
+ )
+ mega_dataloader = iter(
+ torch.utils.data.DataLoader(
+ megadepth_train,
+ batch_size = batch_size,
+ sampler = mega_sampler,
+ num_workers = 8,
+ )
+ )
+ scannet_ws_sampler = torch.utils.data.WeightedRandomSampler(
+ scannet_ws, num_samples=batch_size * k, replacement=False
+ )
+ scannet_dataloader = iter(
+ torch.utils.data.DataLoader(
+ scannet_train,
+ batch_size=batch_size,
+ sampler=scannet_ws_sampler,
+ num_workers=gpus * 8,
+ )
+ )
+ for n_k in tqdm(range(n, n + 2 * k, 2),disable = romatch.RANK > 0):
+ train_k_steps(
+ n_k, 1, mega_dataloader, ddp_model, depth_loss_mega, optimizer, lr_scheduler, grad_scaler, grad_clip_norm = grad_clip_norm, progress_bar=False
+ )
+ train_k_steps(
+ n_k + 1, 1, scannet_dataloader, ddp_model, depth_loss_scannet, optimizer, lr_scheduler, grad_scaler, grad_clip_norm = grad_clip_norm, progress_bar=False
+ )
+ checkpointer.save(model, optimizer, lr_scheduler, romatch.GLOBAL_STEP)
+ wandb.log(megadense_benchmark.benchmark(model), step = romatch.GLOBAL_STEP)
+
+def test_scannet(model, name, resolution, sample_mode):
+ scannet_benchmark = ScanNetBenchmark("data/scannet")
+ scannet_results = scannet_benchmark.benchmark(model)
+ json.dump(scannet_results, open(f"results/scannet_{name}.json", "w"))
+
+if __name__ == "__main__":
+ import warnings
+ warnings.filterwarnings('ignore', category=UserWarning, message='TypedStorage is deprecated')
+ warnings.filterwarnings('ignore')#, category=UserWarning)#, message='WARNING batched routines are designed for small sizes.')
+ os.environ["TORCH_CUDNN_V8_API_ENABLED"] = "1" # For BF16 computations
+ os.environ["OMP_NUM_THREADS"] = "16"
+
+ import romatch
+ parser = ArgumentParser()
+ parser.add_argument("--test", action='store_true')
+ parser.add_argument("--debug_mode", action='store_true')
+ parser.add_argument("--dont_log_wandb", action='store_true')
+ parser.add_argument("--train_resolution", default='medium')
+ parser.add_argument("--gpu_batch_size", default=4, type=int)
+ parser.add_argument("--wandb_entity", required = False)
+
+ args, _ = parser.parse_known_args()
+ romatch.DEBUG_MODE = args.debug_mode
+ if not args.test:
+ train(args)
+ experiment_name = os.path.splitext(os.path.basename(__file__))[0]
+ checkpoint_dir = "workspace/"
+ checkpoint_name = checkpoint_dir + experiment_name + ".pth"
+ test_resolution = "medium"
+ sample_mode = "threshold_balanced"
+ symmetric = True
+ upsample_preds = False
+ attenuate_cert = True
+
+ model = get_model(pretrained_backbone=False, resolution = test_resolution, sample_mode = sample_mode, upsample_preds = upsample_preds, symmetric=symmetric, name=experiment_name, attenuate_cert = attenuate_cert)
+ model = model.cuda()
+ states = torch.load(checkpoint_name)
+ model.load_state_dict(states["model"])
+ test_scannet(model, experiment_name, resolution = test_resolution, sample_mode = sample_mode)
diff --git a/submodules/RoMa/experiments/train_roma_outdoor.py b/submodules/RoMa/experiments/train_roma_outdoor.py
new file mode 100644
index 0000000000000000000000000000000000000000..a47bfca577a1633fd0f3950319b99e88ac2476f1
--- /dev/null
+++ b/submodules/RoMa/experiments/train_roma_outdoor.py
@@ -0,0 +1,307 @@
+import os
+import torch
+from argparse import ArgumentParser
+
+from torch import nn
+from torch.utils.data import ConcatDataset
+import torch.distributed as dist
+from torch.nn.parallel import DistributedDataParallel as DDP
+import json
+import wandb
+
+from romatch.benchmarks import MegadepthDenseBenchmark
+from romatch.datasets.megadepth import MegadepthBuilder
+from romatch.losses.robust_loss import RobustLosses
+from romatch.benchmarks import MegaDepthPoseEstimationBenchmark, MegadepthDenseBenchmark, HpatchesHomogBenchmark
+
+from romatch.train.train import train_k_steps
+from romatch.models.matcher import *
+from romatch.models.transformer import Block, TransformerDecoder, MemEffAttention
+from romatch.models.encoders import *
+from romatch.checkpointing import CheckPoint
+
+resolutions = {"low":(448, 448), "medium":(14*8*5, 14*8*5), "high":(14*8*6, 14*8*6)}
+
+def get_model(pretrained_backbone=True, resolution = "medium", **kwargs):
+ import warnings
+ warnings.filterwarnings('ignore', category=UserWarning, message='TypedStorage is deprecated')
+ gp_dim = 512
+ feat_dim = 512
+ decoder_dim = gp_dim + feat_dim
+ cls_to_coord_res = 64
+ coordinate_decoder = TransformerDecoder(
+ nn.Sequential(*[Block(decoder_dim, 8, attn_class=MemEffAttention) for _ in range(5)]),
+ decoder_dim,
+ cls_to_coord_res**2 + 1,
+ is_classifier=True,
+ amp = True,
+ pos_enc = False,)
+ dw = True
+ hidden_blocks = 8
+ kernel_size = 5
+ displacement_emb = "linear"
+ disable_local_corr_grad = True
+
+ conv_refiner = nn.ModuleDict(
+ {
+ "16": ConvRefiner(
+ 2 * 512+128+(2*7+1)**2,
+ 2 * 512+128+(2*7+1)**2,
+ 2 + 1,
+ kernel_size=kernel_size,
+ dw=dw,
+ hidden_blocks=hidden_blocks,
+ displacement_emb=displacement_emb,
+ displacement_emb_dim=128,
+ local_corr_radius = 7,
+ corr_in_other = True,
+ amp = True,
+ disable_local_corr_grad = disable_local_corr_grad,
+ bn_momentum = 0.01,
+ ),
+ "8": ConvRefiner(
+ 2 * 512+64+(2*3+1)**2,
+ 2 * 512+64+(2*3+1)**2,
+ 2 + 1,
+ kernel_size=kernel_size,
+ dw=dw,
+ hidden_blocks=hidden_blocks,
+ displacement_emb=displacement_emb,
+ displacement_emb_dim=64,
+ local_corr_radius = 3,
+ corr_in_other = True,
+ amp = True,
+ disable_local_corr_grad = disable_local_corr_grad,
+ bn_momentum = 0.01,
+ ),
+ "4": ConvRefiner(
+ 2 * 256+32+(2*2+1)**2,
+ 2 * 256+32+(2*2+1)**2,
+ 2 + 1,
+ kernel_size=kernel_size,
+ dw=dw,
+ hidden_blocks=hidden_blocks,
+ displacement_emb=displacement_emb,
+ displacement_emb_dim=32,
+ local_corr_radius = 2,
+ corr_in_other = True,
+ amp = True,
+ disable_local_corr_grad = disable_local_corr_grad,
+ bn_momentum = 0.01,
+ ),
+ "2": ConvRefiner(
+ 2 * 64+16,
+ 128+16,
+ 2 + 1,
+ kernel_size=kernel_size,
+ dw=dw,
+ hidden_blocks=hidden_blocks,
+ displacement_emb=displacement_emb,
+ displacement_emb_dim=16,
+ amp = True,
+ disable_local_corr_grad = disable_local_corr_grad,
+ bn_momentum = 0.01,
+ ),
+ "1": ConvRefiner(
+ 2 * 9 + 6,
+ 24,
+ 2 + 1,
+ kernel_size=kernel_size,
+ dw=dw,
+ hidden_blocks = hidden_blocks,
+ displacement_emb = displacement_emb,
+ displacement_emb_dim = 6,
+ amp = True,
+ disable_local_corr_grad = disable_local_corr_grad,
+ bn_momentum = 0.01,
+ ),
+ }
+ )
+ kernel_temperature = 0.2
+ learn_temperature = False
+ no_cov = True
+ kernel = CosKernel
+ only_attention = False
+ basis = "fourier"
+ gp16 = GP(
+ kernel,
+ T=kernel_temperature,
+ learn_temperature=learn_temperature,
+ only_attention=only_attention,
+ gp_dim=gp_dim,
+ basis=basis,
+ no_cov=no_cov,
+ )
+ gps = nn.ModuleDict({"16": gp16})
+ proj16 = nn.Sequential(nn.Conv2d(1024, 512, 1, 1), nn.BatchNorm2d(512))
+ proj8 = nn.Sequential(nn.Conv2d(512, 512, 1, 1), nn.BatchNorm2d(512))
+ proj4 = nn.Sequential(nn.Conv2d(256, 256, 1, 1), nn.BatchNorm2d(256))
+ proj2 = nn.Sequential(nn.Conv2d(128, 64, 1, 1), nn.BatchNorm2d(64))
+ proj1 = nn.Sequential(nn.Conv2d(64, 9, 1, 1), nn.BatchNorm2d(9))
+ proj = nn.ModuleDict({
+ "16": proj16,
+ "8": proj8,
+ "4": proj4,
+ "2": proj2,
+ "1": proj1,
+ })
+ displacement_dropout_p = 0.0
+ gm_warp_dropout_p = 0.0
+ decoder = Decoder(coordinate_decoder,
+ gps,
+ proj,
+ conv_refiner,
+ detach=True,
+ scales=["16", "8", "4", "2", "1"],
+ displacement_dropout_p = displacement_dropout_p,
+ gm_warp_dropout_p = gm_warp_dropout_p)
+ h,w = resolutions[resolution]
+ encoder = CNNandDinov2(
+ cnn_kwargs = dict(
+ pretrained=pretrained_backbone,
+ amp = True),
+ amp = True,
+ use_vgg = True,
+ )
+ matcher = RegressionMatcher(encoder, decoder, h=h, w=w,**kwargs)
+ return matcher
+
+def train(args):
+ dist.init_process_group('nccl')
+ #torch._dynamo.config.verbose=True
+ gpus = int(os.environ['WORLD_SIZE'])
+ # create model and move it to GPU with id rank
+ rank = dist.get_rank()
+ print(f"Start running DDP on rank {rank}")
+ device_id = rank % torch.cuda.device_count()
+ romatch.LOCAL_RANK = device_id
+ torch.cuda.set_device(device_id)
+
+ resolution = args.train_resolution
+ wandb_log = not args.dont_log_wandb
+ experiment_name = os.path.splitext(os.path.basename(__file__))[0]
+ wandb_mode = "online" if wandb_log and rank == 0 else "disabled"
+ wandb.init(project="romatch", entity=args.wandb_entity, name=experiment_name, reinit=False, mode = wandb_mode)
+ checkpoint_dir = "workspace/checkpoints/"
+ h,w = resolutions[resolution]
+ model = get_model(pretrained_backbone=True, resolution=resolution, attenuate_cert = False).to(device_id)
+ # Num steps
+ global_step = 0
+ batch_size = args.gpu_batch_size
+ step_size = gpus*batch_size
+ romatch.STEP_SIZE = step_size
+
+ N = (32 * 250000) # 250k steps of batch size 32
+ # checkpoint every
+ k = 25000 // romatch.STEP_SIZE
+
+ # Data
+ mega = MegadepthBuilder(data_root="data/megadepth", loftr_ignore=True, imc21_ignore = True)
+ use_horizontal_flip_aug = True
+ rot_prob = 0
+ depth_interpolation_mode = "bilinear"
+ megadepth_train1 = mega.build_scenes(
+ split="train_loftr", min_overlap=0.01, shake_t=32, use_horizontal_flip_aug = use_horizontal_flip_aug, rot_prob = rot_prob,
+ ht=h,wt=w,
+ )
+ megadepth_train2 = mega.build_scenes(
+ split="train_loftr", min_overlap=0.35, shake_t=32, use_horizontal_flip_aug = use_horizontal_flip_aug, rot_prob = rot_prob,
+ ht=h,wt=w,
+ )
+ megadepth_train = ConcatDataset(megadepth_train1 + megadepth_train2)
+ mega_ws = mega.weight_scenes(megadepth_train, alpha=0.75)
+ # Loss and optimizer
+ depth_loss = RobustLosses(
+ ce_weight=0.01,
+ local_dist={1:4, 2:4, 4:8, 8:8},
+ local_largest_scale=8,
+ depth_interpolation_mode=depth_interpolation_mode,
+ alpha = 0.5,
+ c = 1e-4,)
+ parameters = [
+ {"params": model.encoder.parameters(), "lr": romatch.STEP_SIZE * 5e-6 / 8},
+ {"params": model.decoder.parameters(), "lr": romatch.STEP_SIZE * 1e-4 / 8},
+ ]
+ optimizer = torch.optim.AdamW(parameters, weight_decay=0.01)
+ lr_scheduler = torch.optim.lr_scheduler.MultiStepLR(
+ optimizer, milestones=[(9*N/romatch.STEP_SIZE)//10])
+ megadense_benchmark = MegadepthDenseBenchmark("data/megadepth", num_samples = 1000, h=h,w=w)
+ checkpointer = CheckPoint(checkpoint_dir, experiment_name)
+ model, optimizer, lr_scheduler, global_step = checkpointer.load(model, optimizer, lr_scheduler, global_step)
+ romatch.GLOBAL_STEP = global_step
+ ddp_model = DDP(model, device_ids=[device_id], find_unused_parameters = False, gradient_as_bucket_view=True)
+ grad_scaler = torch.cuda.amp.GradScaler(growth_interval=1_000_000)
+ grad_clip_norm = 0.01
+ for n in range(romatch.GLOBAL_STEP, N, k * romatch.STEP_SIZE):
+ mega_sampler = torch.utils.data.WeightedRandomSampler(
+ mega_ws, num_samples = batch_size * k, replacement=False
+ )
+ mega_dataloader = iter(
+ torch.utils.data.DataLoader(
+ megadepth_train,
+ batch_size = batch_size,
+ sampler = mega_sampler,
+ num_workers = 8,
+ )
+ )
+ train_k_steps(
+ n, k, mega_dataloader, ddp_model, depth_loss, optimizer, lr_scheduler, grad_scaler, grad_clip_norm = grad_clip_norm,
+ )
+ checkpointer.save(model, optimizer, lr_scheduler, romatch.GLOBAL_STEP)
+ wandb.log(megadense_benchmark.benchmark(model), step = romatch.GLOBAL_STEP)
+
+def test_mega_8_scenes(model, name):
+ mega_8_scenes_benchmark = MegaDepthPoseEstimationBenchmark("data/megadepth",
+ scene_names=['mega_8_scenes_0019_0.1_0.3.npz',
+ 'mega_8_scenes_0025_0.1_0.3.npz',
+ 'mega_8_scenes_0021_0.1_0.3.npz',
+ 'mega_8_scenes_0008_0.1_0.3.npz',
+ 'mega_8_scenes_0032_0.1_0.3.npz',
+ 'mega_8_scenes_1589_0.1_0.3.npz',
+ 'mega_8_scenes_0063_0.1_0.3.npz',
+ 'mega_8_scenes_0024_0.1_0.3.npz',
+ 'mega_8_scenes_0019_0.3_0.5.npz',
+ 'mega_8_scenes_0025_0.3_0.5.npz',
+ 'mega_8_scenes_0021_0.3_0.5.npz',
+ 'mega_8_scenes_0008_0.3_0.5.npz',
+ 'mega_8_scenes_0032_0.3_0.5.npz',
+ 'mega_8_scenes_1589_0.3_0.5.npz',
+ 'mega_8_scenes_0063_0.3_0.5.npz',
+ 'mega_8_scenes_0024_0.3_0.5.npz'])
+ mega_8_scenes_results = mega_8_scenes_benchmark.benchmark(model, model_name=name)
+ print(mega_8_scenes_results)
+ json.dump(mega_8_scenes_results, open(f"results/mega_8_scenes_{name}.json", "w"))
+
+def test_mega1500(model, name):
+ mega1500_benchmark = MegaDepthPoseEstimationBenchmark("data/megadepth")
+ mega1500_results = mega1500_benchmark.benchmark(model, model_name=name)
+ json.dump(mega1500_results, open(f"results/mega1500_{name}.json", "w"))
+
+def test_mega_dense(model, name):
+ megadense_benchmark = MegadepthDenseBenchmark("data/megadepth", num_samples = 1000)
+ megadense_results = megadense_benchmark.benchmark(model)
+ json.dump(megadense_results, open(f"results/mega_dense_{name}.json", "w"))
+
+def test_hpatches(model, name):
+ hpatches_benchmark = HpatchesHomogBenchmark("data/hpatches")
+ hpatches_results = hpatches_benchmark.benchmark(model)
+ json.dump(hpatches_results, open(f"results/hpatches_{name}.json", "w"))
+
+
+if __name__ == "__main__":
+ os.environ["TORCH_CUDNN_V8_API_ENABLED"] = "1" # For BF16 computations
+ os.environ["OMP_NUM_THREADS"] = "16"
+ torch.backends.cudnn.allow_tf32 = True # allow tf32 on cudnn
+ import romatch
+ parser = ArgumentParser()
+ parser.add_argument("--only_test", action='store_true')
+ parser.add_argument("--debug_mode", action='store_true')
+ parser.add_argument("--dont_log_wandb", action='store_true')
+ parser.add_argument("--train_resolution", default='medium')
+ parser.add_argument("--gpu_batch_size", default=8, type=int)
+ parser.add_argument("--wandb_entity", required = False)
+
+ args, _ = parser.parse_known_args()
+ romatch.DEBUG_MODE = args.debug_mode
+ if not args.only_test:
+ train(args)
diff --git a/submodules/RoMa/experiments/train_tiny_roma_v1_outdoor.py b/submodules/RoMa/experiments/train_tiny_roma_v1_outdoor.py
new file mode 100644
index 0000000000000000000000000000000000000000..8fbb97452b7744792b1fa08fb9e2fd23b9566684
--- /dev/null
+++ b/submodules/RoMa/experiments/train_tiny_roma_v1_outdoor.py
@@ -0,0 +1,498 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import os
+import torch
+from argparse import ArgumentParser
+from pathlib import Path
+import math
+import numpy as np
+
+from torch import nn
+from torch.utils.data import ConcatDataset
+import torch.distributed as dist
+from torch.nn.parallel import DistributedDataParallel as DDP
+import json
+import wandb
+from PIL import Image
+from torchvision.transforms import ToTensor
+
+from romatch.benchmarks import MegadepthDenseBenchmark, ScanNetBenchmark
+from romatch.benchmarks import Mega1500PoseLibBenchmark, ScanNetPoselibBenchmark
+from romatch.datasets.megadepth import MegadepthBuilder
+from romatch.losses.robust_loss_tiny_roma import RobustLosses
+from romatch.benchmarks import MegaDepthPoseEstimationBenchmark, MegadepthDenseBenchmark, HpatchesHomogBenchmark
+from romatch.train.train import train_k_steps
+from romatch.checkpointing import CheckPoint
+
+resolutions = {"low":(448, 448), "medium":(14*8*5, 14*8*5), "high":(14*8*6, 14*8*6), "xfeat": (600,800), "big": (768, 1024)}
+
+def kde(x, std = 0.1):
+ # use a gaussian kernel to estimate density
+ x = x.half() # Do it in half precision TODO: remove hardcoding
+ scores = (-torch.cdist(x,x)**2/(2*std**2)).exp()
+ density = scores.sum(dim=-1)
+ return density
+
+class BasicLayer(nn.Module):
+ """
+ Basic Convolutional Layer: Conv2d -> BatchNorm -> ReLU
+ """
+ def __init__(self, in_channels, out_channels, kernel_size=3, stride=1, padding=1, dilation=1, bias=False, relu = True):
+ super().__init__()
+ self.layer = nn.Sequential(
+ nn.Conv2d( in_channels, out_channels, kernel_size, padding = padding, stride=stride, dilation=dilation, bias = bias),
+ nn.BatchNorm2d(out_channels, affine=False),
+ nn.ReLU(inplace = True) if relu else nn.Identity()
+ )
+
+ def forward(self, x):
+ return self.layer(x)
+
+class XFeatModel(nn.Module):
+ """
+ Implementation of architecture described in
+ "XFeat: Accelerated Features for Lightweight Image Matching, CVPR 2024."
+ """
+
+ def __init__(self, xfeat = None,
+ freeze_xfeat = True,
+ sample_mode = "threshold_balanced",
+ symmetric = False,
+ exact_softmax = False):
+ super().__init__()
+ if xfeat is None:
+ xfeat = torch.hub.load('verlab/accelerated_features', 'XFeat', pretrained = True, top_k = 4096).net
+ del xfeat.heatmap_head, xfeat.keypoint_head, xfeat.fine_matcher
+ if freeze_xfeat:
+ xfeat.train(False)
+ self.xfeat = [xfeat]# hide params from ddp
+ else:
+ self.xfeat = nn.ModuleList([xfeat])
+ self.freeze_xfeat = freeze_xfeat
+ match_dim = 256
+ self.coarse_matcher = nn.Sequential(
+ BasicLayer(64+64+2, match_dim,),
+ BasicLayer(match_dim, match_dim,),
+ BasicLayer(match_dim, match_dim,),
+ BasicLayer(match_dim, match_dim,),
+ nn.Conv2d(match_dim, 3, kernel_size=1, bias=True, padding=0))
+ fine_match_dim = 64
+ self.fine_matcher = nn.Sequential(
+ BasicLayer(24+24+2, fine_match_dim,),
+ BasicLayer(fine_match_dim, fine_match_dim,),
+ BasicLayer(fine_match_dim, fine_match_dim,),
+ BasicLayer(fine_match_dim, fine_match_dim,),
+ nn.Conv2d(fine_match_dim, 3, kernel_size=1, bias=True, padding=0),)
+ self.sample_mode = sample_mode
+ self.sample_thresh = 0.2
+ self.symmetric = symmetric
+ self.exact_softmax = exact_softmax
+
+ @property
+ def device(self):
+ return self.fine_matcher[-1].weight.device
+
+ def preprocess_tensor(self, x):
+ """ Guarantee that image is divisible by 32 to avoid aliasing artifacts. """
+ H, W = x.shape[-2:]
+ _H, _W = (H//32) * 32, (W//32) * 32
+ rh, rw = H/_H, W/_W
+
+ x = F.interpolate(x, (_H, _W), mode='bilinear', align_corners=False)
+ return x, rh, rw
+
+ def forward_single(self, x):
+ with torch.inference_mode(self.freeze_xfeat or not self.training):
+ xfeat = self.xfeat[0]
+ with torch.no_grad():
+ x = x.mean(dim=1, keepdim = True)
+ x = xfeat.norm(x)
+
+ #main backbone
+ x1 = xfeat.block1(x)
+ x2 = xfeat.block2(x1 + xfeat.skip1(x))
+ x3 = xfeat.block3(x2)
+ x4 = xfeat.block4(x3)
+ x5 = xfeat.block5(x4)
+ x4 = F.interpolate(x4, (x3.shape[-2], x3.shape[-1]), mode='bilinear')
+ x5 = F.interpolate(x5, (x3.shape[-2], x3.shape[-1]), mode='bilinear')
+ feats = xfeat.block_fusion( x3 + x4 + x5 )
+ if self.freeze_xfeat:
+ return x2.clone(), feats.clone()
+ return x2, feats
+
+ def to_pixel_coordinates(self, coords, H_A, W_A, H_B = None, W_B = None):
+ if coords.shape[-1] == 2:
+ return self._to_pixel_coordinates(coords, H_A, W_A)
+
+ if isinstance(coords, (list, tuple)):
+ kpts_A, kpts_B = coords[0], coords[1]
+ else:
+ kpts_A, kpts_B = coords[...,:2], coords[...,2:]
+ return self._to_pixel_coordinates(kpts_A, H_A, W_A), self._to_pixel_coordinates(kpts_B, H_B, W_B)
+
+ def _to_pixel_coordinates(self, coords, H, W):
+ kpts = torch.stack((W/2 * (coords[...,0]+1), H/2 * (coords[...,1]+1)),axis=-1)
+ return kpts
+
+ def pos_embed(self, corr_volume: torch.Tensor):
+ B, H1, W1, H0, W0 = corr_volume.shape
+ grid = torch.stack(
+ torch.meshgrid(
+ torch.linspace(-1+1/W1,1-1/W1, W1),
+ torch.linspace(-1+1/H1,1-1/H1, H1),
+ indexing = "xy"),
+ dim = -1).float().to(corr_volume).reshape(H1*W1, 2)
+ down = 4
+ if not self.training and not self.exact_softmax:
+ grid_lr = torch.stack(
+ torch.meshgrid(
+ torch.linspace(-1+down/W1,1-down/W1, W1//down),
+ torch.linspace(-1+down/H1,1-down/H1, H1//down),
+ indexing = "xy"),
+ dim = -1).float().to(corr_volume).reshape(H1*W1 //down**2, 2)
+ cv = corr_volume
+ best_match = cv.reshape(B,H1*W1,H0,W0).amax(dim=1) # B, HW, H, W
+ P_lowres = torch.cat((cv[:,::down,::down].reshape(B,H1*W1 // down**2,H0,W0), best_match[:,None]),dim=1).softmax(dim=1)
+ pos_embeddings = torch.einsum('bchw,cd->bdhw', P_lowres[:,:-1], grid_lr)
+ pos_embeddings += P_lowres[:,-1] * grid[best_match].permute(0,3,1,2)
+ else:
+ P = corr_volume.reshape(B,H1*W1,H0,W0).softmax(dim=1) # B, HW, H, W
+ pos_embeddings = torch.einsum('bchw,cd->bdhw', P, grid)
+ return pos_embeddings
+
+ def visualize_warp(self, warp, certainty, im_A = None, im_B = None,
+ im_A_path = None, im_B_path = None, symmetric = True, save_path = None, unnormalize = False):
+ device = warp.device
+ H,W2,_ = warp.shape
+ W = W2//2 if symmetric else W2
+ if im_A is None:
+ from PIL import Image
+ im_A, im_B = Image.open(im_A_path).convert("RGB"), Image.open(im_B_path).convert("RGB")
+ if not isinstance(im_A, torch.Tensor):
+ im_A = im_A.resize((W,H))
+ im_B = im_B.resize((W,H))
+ x_B = (torch.tensor(np.array(im_B)) / 255).to(device).permute(2, 0, 1)
+ if symmetric:
+ x_A = (torch.tensor(np.array(im_A)) / 255).to(device).permute(2, 0, 1)
+ else:
+ if symmetric:
+ x_A = im_A
+ x_B = im_B
+ im_A_transfer_rgb = F.grid_sample(
+ x_B[None], warp[:,:W, 2:][None], mode="bilinear", align_corners=False
+ )[0]
+ if symmetric:
+ im_B_transfer_rgb = F.grid_sample(
+ x_A[None], warp[:, W:, :2][None], mode="bilinear", align_corners=False
+ )[0]
+ warp_im = torch.cat((im_A_transfer_rgb,im_B_transfer_rgb),dim=2)
+ white_im = torch.ones((H,2*W),device=device)
+ else:
+ warp_im = im_A_transfer_rgb
+ white_im = torch.ones((H, W), device = device)
+ vis_im = certainty * warp_im + (1 - certainty) * white_im
+ if save_path is not None:
+ from romatch.utils import tensor_to_pil
+ tensor_to_pil(vis_im, unnormalize=unnormalize).save(save_path)
+ return vis_im
+
+ def corr_volume(self, feat0, feat1):
+ """
+ input:
+ feat0 -> torch.Tensor(B, C, H, W)
+ feat1 -> torch.Tensor(B, C, H, W)
+ return:
+ corr_volume -> torch.Tensor(B, H, W, H, W)
+ """
+ B, C, H0, W0 = feat0.shape
+ B, C, H1, W1 = feat1.shape
+ feat0 = feat0.view(B, C, H0*W0)
+ feat1 = feat1.view(B, C, H1*W1)
+ corr_volume = torch.einsum('bci,bcj->bji', feat0, feat1).reshape(B, H1, W1, H0 , W0)/math.sqrt(C) #16*16*16
+ return corr_volume
+
+ @torch.inference_mode()
+ def match_from_path(self, im0_path, im1_path):
+ device = self.device
+ im0 = ToTensor()(Image.open(im0_path))[None].to(device)
+ im1 = ToTensor()(Image.open(im1_path))[None].to(device)
+ return self.match(im0, im1, batched = False)
+
+ @torch.inference_mode()
+ def match(self, im0, im1, *args, batched = True):
+ # stupid
+ if isinstance(im0, (str, Path)):
+ return self.match_from_path(im0, im1)
+ elif isinstance(im0, Image.Image):
+ batched = False
+ device = self.device
+ im0 = ToTensor()(im0)[None].to(device)
+ im1 = ToTensor()(im1)[None].to(device)
+
+ B,C,H0,W0 = im0.shape
+ B,C,H1,W1 = im1.shape
+ self.train(False)
+ corresps = self.forward({"im_A":im0, "im_B":im1})
+ #return 1,1
+ flow = F.interpolate(
+ corresps[4]["flow"],
+ size = (H0, W0),
+ mode = "bilinear", align_corners = False).permute(0,2,3,1).reshape(B,H0,W0,2)
+ grid = torch.stack(
+ torch.meshgrid(
+ torch.linspace(-1+1/W0,1-1/W0, W0),
+ torch.linspace(-1+1/H0,1-1/H0, H0),
+ indexing = "xy"),
+ dim = -1).float().to(flow.device).expand(B, H0, W0, 2)
+
+ certainty = F.interpolate(corresps[4]["certainty"], size = (H0,W0), mode = "bilinear", align_corners = False)
+ warp, cert = torch.cat((grid, flow), dim = -1), certainty[:,0].sigmoid()
+ if batched:
+ return warp, cert
+ else:
+ return warp[0], cert[0]
+
+ def sample(
+ self,
+ matches,
+ certainty,
+ num=10000,
+ ):
+ if "threshold" in self.sample_mode:
+ upper_thresh = self.sample_thresh
+ certainty = certainty.clone()
+ certainty[certainty > upper_thresh] = 1
+ matches, certainty = (
+ matches.reshape(-1, 4),
+ certainty.reshape(-1),
+ )
+ expansion_factor = 4 if "balanced" in self.sample_mode else 1
+ good_samples = torch.multinomial(certainty,
+ num_samples = min(expansion_factor*num, len(certainty)),
+ replacement=False)
+ good_matches, good_certainty = matches[good_samples], certainty[good_samples]
+ if "balanced" not in self.sample_mode:
+ return good_matches, good_certainty
+ density = kde(good_matches, std=0.1)
+ p = 1 / (density+1)
+ p[density < 10] = 1e-7 # Basically should have at least 10 perfect neighbours, or around 100 ok ones
+ balanced_samples = torch.multinomial(p,
+ num_samples = min(num,len(good_certainty)),
+ replacement=False)
+ return good_matches[balanced_samples], good_certainty[balanced_samples]
+
+ def forward(self, batch):
+ """
+ input:
+ x -> torch.Tensor(B, C, H, W) grayscale or rgb images
+ return:
+
+ """
+ im0 = batch["im_A"]
+ im1 = batch["im_B"]
+ corresps = {}
+ im0, rh0, rw0 = self.preprocess_tensor(im0)
+ im1, rh1, rw1 = self.preprocess_tensor(im1)
+ B, C, H0, W0 = im0.shape
+ B, C, H1, W1 = im1.shape
+ to_normalized = torch.tensor((2/W1, 2/H1, 1)).to(im0.device)[None,:,None,None]
+
+ if im0.shape[-2:] == im1.shape[-2:]:
+ x = torch.cat([im0, im1], dim=0)
+ x = self.forward_single(x)
+ feats_x0_c, feats_x1_c = x[1].chunk(2)
+ feats_x0_f, feats_x1_f = x[0].chunk(2)
+ else:
+ feats_x0_f, feats_x0_c = self.forward_single(im0)
+ feats_x1_f, feats_x1_c = self.forward_single(im1)
+ corr_volume = self.corr_volume(feats_x0_c, feats_x1_c)
+ coarse_warp = self.pos_embed(corr_volume)
+ coarse_matches = torch.cat((coarse_warp, torch.zeros_like(coarse_warp[:,-1:])), dim=1)
+ feats_x1_c_warped = F.grid_sample(feats_x1_c, coarse_matches.permute(0, 2, 3, 1)[...,:2], mode = 'bilinear', align_corners = False)
+ coarse_matches_delta = self.coarse_matcher(torch.cat((feats_x0_c, feats_x1_c_warped, coarse_warp), dim=1))
+ coarse_matches = coarse_matches + coarse_matches_delta * to_normalized
+ corresps[8] = {"flow": coarse_matches[:,:2], "certainty": coarse_matches[:,2:]}
+ coarse_matches_up = F.interpolate(coarse_matches, size = feats_x0_f.shape[-2:], mode = "bilinear", align_corners = False)
+ coarse_matches_up_detach = coarse_matches_up.detach()#note the detach
+ feats_x1_f_warped = F.grid_sample(feats_x1_f, coarse_matches_up_detach.permute(0, 2, 3, 1)[...,:2], mode = 'bilinear', align_corners = False)
+ fine_matches_delta = self.fine_matcher(torch.cat((feats_x0_f, feats_x1_f_warped, coarse_matches_up_detach[:,:2]), dim=1))
+ fine_matches = coarse_matches_up_detach+fine_matches_delta * to_normalized
+ corresps[4] = {"flow": fine_matches[:,:2], "certainty": fine_matches[:,2:]}
+ return corresps
+
+
+
+
+
+def train(args):
+ rank = 0
+ gpus = 1
+ device_id = rank % torch.cuda.device_count()
+ romatch.LOCAL_RANK = 0
+ torch.cuda.set_device(device_id)
+
+ resolution = "big"
+ wandb_log = not args.dont_log_wandb
+ experiment_name = Path(__file__).stem
+ wandb_mode = "online" if wandb_log and rank == 0 else "disabled"
+ wandb.init(project="romatch", entity=args.wandb_entity, name=experiment_name, reinit=False, mode = wandb_mode)
+ checkpoint_dir = "workspace/checkpoints/"
+ h,w = resolutions[resolution]
+ model = XFeatModel(freeze_xfeat = False).to(device_id)
+ # Num steps
+ global_step = 0
+ batch_size = args.gpu_batch_size
+ step_size = gpus*batch_size
+ romatch.STEP_SIZE = step_size
+
+ N = 2_000_000 # 2M pairs
+ # checkpoint every
+ k = 25000 // romatch.STEP_SIZE
+
+ # Data
+ mega = MegadepthBuilder(data_root="data/megadepth", loftr_ignore=True, imc21_ignore = True)
+ use_horizontal_flip_aug = True
+ normalize = False # don't imgnet normalize
+ rot_prob = 0
+ depth_interpolation_mode = "bilinear"
+ megadepth_train1 = mega.build_scenes(
+ split="train_loftr", min_overlap=0.01, shake_t=32, use_horizontal_flip_aug = use_horizontal_flip_aug, rot_prob = rot_prob,
+ ht=h,wt=w, normalize = normalize
+ )
+ megadepth_train2 = mega.build_scenes(
+ split="train_loftr", min_overlap=0.35, shake_t=32, use_horizontal_flip_aug = use_horizontal_flip_aug, rot_prob = rot_prob,
+ ht=h,wt=w, normalize = normalize
+ )
+ megadepth_train = ConcatDataset(megadepth_train1 + megadepth_train2)
+ mega_ws = mega.weight_scenes(megadepth_train, alpha=0.75)
+ # Loss and optimizer
+ depth_loss = RobustLosses(
+ ce_weight=0.01,
+ local_dist={4:4},
+ depth_interpolation_mode=depth_interpolation_mode,
+ alpha = {4:0.15, 8:0.15},
+ c = 1e-4,
+ epe_mask_prob_th = 0.001,
+ )
+ parameters = [
+ {"params": model.parameters(), "lr": romatch.STEP_SIZE * 1e-4 / 8},
+ ]
+ optimizer = torch.optim.AdamW(parameters, weight_decay=0.01)
+ lr_scheduler = torch.optim.lr_scheduler.MultiStepLR(
+ optimizer, milestones=[(9*N/romatch.STEP_SIZE)//10])
+ #megadense_benchmark = MegadepthDenseBenchmark("data/megadepth", num_samples = 1000, h=h,w=w)
+ mega1500_benchmark = Mega1500PoseLibBenchmark("data/megadepth", num_ransac_iter = 1, test_every = 30)
+
+ checkpointer = CheckPoint(checkpoint_dir, experiment_name)
+ model, optimizer, lr_scheduler, global_step = checkpointer.load(model, optimizer, lr_scheduler, global_step)
+ romatch.GLOBAL_STEP = global_step
+ grad_scaler = torch.cuda.amp.GradScaler(growth_interval=1_000_000)
+ grad_clip_norm = 0.01
+ #megadense_benchmark.benchmark(model)
+ for n in range(romatch.GLOBAL_STEP, N, k * romatch.STEP_SIZE):
+ mega_sampler = torch.utils.data.WeightedRandomSampler(
+ mega_ws, num_samples = batch_size * k, replacement=False
+ )
+ mega_dataloader = iter(
+ torch.utils.data.DataLoader(
+ megadepth_train,
+ batch_size = batch_size,
+ sampler = mega_sampler,
+ num_workers = 8,
+ )
+ )
+ train_k_steps(
+ n, k, mega_dataloader, model, depth_loss, optimizer, lr_scheduler, grad_scaler, grad_clip_norm = grad_clip_norm,
+ )
+ checkpointer.save(model, optimizer, lr_scheduler, romatch.GLOBAL_STEP)
+ wandb.log(mega1500_benchmark.benchmark(model, model_name=experiment_name), step = romatch.GLOBAL_STEP)
+
+def test_mega_8_scenes(model, name):
+ mega_8_scenes_benchmark = MegaDepthPoseEstimationBenchmark("data/megadepth",
+ scene_names=['mega_8_scenes_0019_0.1_0.3.npz',
+ 'mega_8_scenes_0025_0.1_0.3.npz',
+ 'mega_8_scenes_0021_0.1_0.3.npz',
+ 'mega_8_scenes_0008_0.1_0.3.npz',
+ 'mega_8_scenes_0032_0.1_0.3.npz',
+ 'mega_8_scenes_1589_0.1_0.3.npz',
+ 'mega_8_scenes_0063_0.1_0.3.npz',
+ 'mega_8_scenes_0024_0.1_0.3.npz',
+ 'mega_8_scenes_0019_0.3_0.5.npz',
+ 'mega_8_scenes_0025_0.3_0.5.npz',
+ 'mega_8_scenes_0021_0.3_0.5.npz',
+ 'mega_8_scenes_0008_0.3_0.5.npz',
+ 'mega_8_scenes_0032_0.3_0.5.npz',
+ 'mega_8_scenes_1589_0.3_0.5.npz',
+ 'mega_8_scenes_0063_0.3_0.5.npz',
+ 'mega_8_scenes_0024_0.3_0.5.npz'])
+ mega_8_scenes_results = mega_8_scenes_benchmark.benchmark(model, model_name=name)
+ print(mega_8_scenes_results)
+ json.dump(mega_8_scenes_results, open(f"results/mega_8_scenes_{name}.json", "w"))
+
+def test_mega1500(model, name):
+ mega1500_benchmark = MegaDepthPoseEstimationBenchmark("data/megadepth")
+ mega1500_results = mega1500_benchmark.benchmark(model, model_name=name)
+ json.dump(mega1500_results, open(f"results/mega1500_{name}.json", "w"))
+
+def test_mega1500_poselib(model, name):
+ mega1500_benchmark = Mega1500PoseLibBenchmark("data/megadepth", num_ransac_iter = 1, test_every = 1)
+ mega1500_results = mega1500_benchmark.benchmark(model, model_name=name)
+ json.dump(mega1500_results, open(f"results/mega1500_poselib_{name}.json", "w"))
+
+def test_mega_8_scenes_poselib(model, name):
+ mega1500_benchmark = Mega1500PoseLibBenchmark("data/megadepth", num_ransac_iter = 1, test_every = 1,
+ scene_names=['mega_8_scenes_0019_0.1_0.3.npz',
+ 'mega_8_scenes_0025_0.1_0.3.npz',
+ 'mega_8_scenes_0021_0.1_0.3.npz',
+ 'mega_8_scenes_0008_0.1_0.3.npz',
+ 'mega_8_scenes_0032_0.1_0.3.npz',
+ 'mega_8_scenes_1589_0.1_0.3.npz',
+ 'mega_8_scenes_0063_0.1_0.3.npz',
+ 'mega_8_scenes_0024_0.1_0.3.npz',
+ 'mega_8_scenes_0019_0.3_0.5.npz',
+ 'mega_8_scenes_0025_0.3_0.5.npz',
+ 'mega_8_scenes_0021_0.3_0.5.npz',
+ 'mega_8_scenes_0008_0.3_0.5.npz',
+ 'mega_8_scenes_0032_0.3_0.5.npz',
+ 'mega_8_scenes_1589_0.3_0.5.npz',
+ 'mega_8_scenes_0063_0.3_0.5.npz',
+ 'mega_8_scenes_0024_0.3_0.5.npz'])
+ mega1500_results = mega1500_benchmark.benchmark(model, model_name=name)
+ json.dump(mega1500_results, open(f"results/mega_8_scenes_poselib_{name}.json", "w"))
+
+def test_scannet_poselib(model, name):
+ scannet_benchmark = ScanNetPoselibBenchmark("data/scannet")
+ scannet_results = scannet_benchmark.benchmark(model)
+ json.dump(scannet_results, open(f"results/scannet_{name}.json", "w"))
+
+def test_scannet(model, name):
+ scannet_benchmark = ScanNetBenchmark("data/scannet")
+ scannet_results = scannet_benchmark.benchmark(model)
+ json.dump(scannet_results, open(f"results/scannet_{name}.json", "w"))
+
+if __name__ == "__main__":
+ os.environ["TORCH_CUDNN_V8_API_ENABLED"] = "1" # For BF16 computations
+ os.environ["OMP_NUM_THREADS"] = "16"
+ torch.backends.cudnn.allow_tf32 = True # allow tf32 on cudnn
+ import romatch
+ parser = ArgumentParser()
+ parser.add_argument("--only_test", action='store_true')
+ parser.add_argument("--debug_mode", action='store_true')
+ parser.add_argument("--dont_log_wandb", action='store_true')
+ parser.add_argument("--train_resolution", default='medium')
+ parser.add_argument("--gpu_batch_size", default=8, type=int)
+ parser.add_argument("--wandb_entity", required = False)
+
+ args, _ = parser.parse_known_args()
+ romatch.DEBUG_MODE = args.debug_mode
+ if not args.only_test:
+ train(args)
+
+ experiment_name = "tiny_roma_v1_outdoor"#Path(__file__).stem
+ device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+ model = XFeatModel(freeze_xfeat=False, exact_softmax=False).to(device)
+ model.load_state_dict(torch.load(f"{experiment_name}.pth"))
+ test_mega1500_poselib(model, experiment_name)
+
\ No newline at end of file
diff --git a/submodules/RoMa/requirements.txt b/submodules/RoMa/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..81360b228245bc573dba770458e4dbeb76d0dd9f
--- /dev/null
+++ b/submodules/RoMa/requirements.txt
@@ -0,0 +1,14 @@
+torch
+einops
+torchvision
+opencv-python
+kornia
+albumentations
+loguru
+tqdm
+matplotlib
+h5py
+wandb
+timm
+poselib
+#xformers # Optional, used for memefficient attention
\ No newline at end of file
diff --git a/submodules/RoMa/romatch/__init__.py b/submodules/RoMa/romatch/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..45c2aff7bb510b1d8dfcdc9c4d91ffc3aa5021f6
--- /dev/null
+++ b/submodules/RoMa/romatch/__init__.py
@@ -0,0 +1,8 @@
+import os
+from .models import roma_outdoor, tiny_roma_v1_outdoor, roma_indoor
+
+DEBUG_MODE = False
+RANK = int(os.environ.get('RANK', default = 0))
+GLOBAL_STEP = 0
+STEP_SIZE = 1
+LOCAL_RANK = -1
\ No newline at end of file
diff --git a/submodules/RoMa/romatch/benchmarks/__init__.py b/submodules/RoMa/romatch/benchmarks/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..af32a46ba4a48d719e3ad38f9b2355a13fe6cc44
--- /dev/null
+++ b/submodules/RoMa/romatch/benchmarks/__init__.py
@@ -0,0 +1,6 @@
+from .hpatches_sequences_homog_benchmark import HpatchesHomogBenchmark
+from .scannet_benchmark import ScanNetBenchmark
+from .megadepth_pose_estimation_benchmark import MegaDepthPoseEstimationBenchmark
+from .megadepth_dense_benchmark import MegadepthDenseBenchmark
+from .megadepth_pose_estimation_benchmark_poselib import Mega1500PoseLibBenchmark
+#from .scannet_benchmark_poselib import ScanNetPoselibBenchmark
\ No newline at end of file
diff --git a/submodules/RoMa/romatch/benchmarks/hpatches_sequences_homog_benchmark.py b/submodules/RoMa/romatch/benchmarks/hpatches_sequences_homog_benchmark.py
new file mode 100644
index 0000000000000000000000000000000000000000..5972361f80d4f4e5cafd8fd359c87c0433a0a5a5
--- /dev/null
+++ b/submodules/RoMa/romatch/benchmarks/hpatches_sequences_homog_benchmark.py
@@ -0,0 +1,113 @@
+from PIL import Image
+import numpy as np
+
+import os
+
+from tqdm import tqdm
+from romatch.utils import pose_auc
+import cv2
+
+
+class HpatchesHomogBenchmark:
+ """Hpatches grid goes from [0,n-1] instead of [0.5,n-0.5]"""
+
+ def __init__(self, dataset_path) -> None:
+ seqs_dir = "hpatches-sequences-release"
+ self.seqs_path = os.path.join(dataset_path, seqs_dir)
+ self.seq_names = sorted(os.listdir(self.seqs_path))
+ # Ignore seqs is same as LoFTR.
+ self.ignore_seqs = set(
+ [
+ "i_contruction",
+ "i_crownnight",
+ "i_dc",
+ "i_pencils",
+ "i_whitebuilding",
+ "v_artisans",
+ "v_astronautis",
+ "v_talent",
+ ]
+ )
+
+ def convert_coordinates(self, im_A_coords, im_A_to_im_B, wq, hq, wsup, hsup):
+ offset = 0.5 # Hpatches assumes that the center of the top-left pixel is at [0,0] (I think)
+ im_A_coords = (
+ np.stack(
+ (
+ wq * (im_A_coords[..., 0] + 1) / 2,
+ hq * (im_A_coords[..., 1] + 1) / 2,
+ ),
+ axis=-1,
+ )
+ - offset
+ )
+ im_A_to_im_B = (
+ np.stack(
+ (
+ wsup * (im_A_to_im_B[..., 0] + 1) / 2,
+ hsup * (im_A_to_im_B[..., 1] + 1) / 2,
+ ),
+ axis=-1,
+ )
+ - offset
+ )
+ return im_A_coords, im_A_to_im_B
+
+ def benchmark(self, model, model_name = None):
+ n_matches = []
+ homog_dists = []
+ for seq_idx, seq_name in tqdm(
+ enumerate(self.seq_names), total=len(self.seq_names)
+ ):
+ im_A_path = os.path.join(self.seqs_path, seq_name, "1.ppm")
+ im_A = Image.open(im_A_path)
+ w1, h1 = im_A.size
+ for im_idx in range(2, 7):
+ im_B_path = os.path.join(self.seqs_path, seq_name, f"{im_idx}.ppm")
+ im_B = Image.open(im_B_path)
+ w2, h2 = im_B.size
+ H = np.loadtxt(
+ os.path.join(self.seqs_path, seq_name, "H_1_" + str(im_idx))
+ )
+ dense_matches, dense_certainty = model.match(
+ im_A_path, im_B_path
+ )
+ good_matches, _ = model.sample(dense_matches, dense_certainty, 5000)
+ pos_a, pos_b = self.convert_coordinates(
+ good_matches[:, :2], good_matches[:, 2:], w1, h1, w2, h2
+ )
+ try:
+ H_pred, inliers = cv2.findHomography(
+ pos_a,
+ pos_b,
+ method = cv2.RANSAC,
+ confidence = 0.99999,
+ ransacReprojThreshold = 3 * min(w2, h2) / 480,
+ )
+ except:
+ H_pred = None
+ if H_pred is None:
+ H_pred = np.zeros((3, 3))
+ H_pred[2, 2] = 1.0
+ corners = np.array(
+ [[0, 0, 1], [0, h1 - 1, 1], [w1 - 1, 0, 1], [w1 - 1, h1 - 1, 1]]
+ )
+ real_warped_corners = np.dot(corners, np.transpose(H))
+ real_warped_corners = (
+ real_warped_corners[:, :2] / real_warped_corners[:, 2:]
+ )
+ warped_corners = np.dot(corners, np.transpose(H_pred))
+ warped_corners = warped_corners[:, :2] / warped_corners[:, 2:]
+ mean_dist = np.mean(
+ np.linalg.norm(real_warped_corners - warped_corners, axis=1)
+ ) / (min(w2, h2) / 480.0)
+ homog_dists.append(mean_dist)
+
+ n_matches = np.array(n_matches)
+ thresholds = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
+ auc = pose_auc(np.array(homog_dists), thresholds)
+ return {
+ "hpatches_homog_auc_3": auc[2],
+ "hpatches_homog_auc_5": auc[4],
+ "hpatches_homog_auc_10": auc[9],
+ }
diff --git a/submodules/RoMa/romatch/benchmarks/megadepth_dense_benchmark.py b/submodules/RoMa/romatch/benchmarks/megadepth_dense_benchmark.py
new file mode 100644
index 0000000000000000000000000000000000000000..09d0a297f2937afc609eed3a74aa0c3c4c7ccebc
--- /dev/null
+++ b/submodules/RoMa/romatch/benchmarks/megadepth_dense_benchmark.py
@@ -0,0 +1,106 @@
+import torch
+import numpy as np
+import tqdm
+from romatch.datasets import MegadepthBuilder
+from romatch.utils import warp_kpts
+from torch.utils.data import ConcatDataset
+import romatch
+
+class MegadepthDenseBenchmark:
+ def __init__(self, data_root="data/megadepth", h = 384, w = 512, num_samples = 2000) -> None:
+ mega = MegadepthBuilder(data_root=data_root)
+ self.dataset = ConcatDataset(
+ mega.build_scenes(split="test_loftr", ht=h, wt=w)
+ ) # fixed resolution of 384,512
+ self.num_samples = num_samples
+
+ def geometric_dist(self, depth1, depth2, T_1to2, K1, K2, dense_matches):
+ b, h1, w1, d = dense_matches.shape
+ with torch.no_grad():
+ x1 = dense_matches[..., :2].reshape(b, h1 * w1, 2)
+ mask, x2 = warp_kpts(
+ x1.double(),
+ depth1.double(),
+ depth2.double(),
+ T_1to2.double(),
+ K1.double(),
+ K2.double(),
+ )
+ x2 = torch.stack(
+ (w1 * (x2[..., 0] + 1) / 2, h1 * (x2[..., 1] + 1) / 2), dim=-1
+ )
+ prob = mask.float().reshape(b, h1, w1)
+ x2_hat = dense_matches[..., 2:]
+ x2_hat = torch.stack(
+ (w1 * (x2_hat[..., 0] + 1) / 2, h1 * (x2_hat[..., 1] + 1) / 2), dim=-1
+ )
+ gd = (x2_hat - x2.reshape(b, h1, w1, 2)).norm(dim=-1)
+ gd = gd[prob == 1]
+ pck_1 = (gd < 1.0).float().mean()
+ pck_3 = (gd < 3.0).float().mean()
+ pck_5 = (gd < 5.0).float().mean()
+ return gd, pck_1, pck_3, pck_5, prob
+
+ def benchmark(self, model, batch_size=8):
+ model.train(False)
+ with torch.no_grad():
+ gd_tot = 0.0
+ pck_1_tot = 0.0
+ pck_3_tot = 0.0
+ pck_5_tot = 0.0
+ sampler = torch.utils.data.WeightedRandomSampler(
+ torch.ones(len(self.dataset)), replacement=False, num_samples=self.num_samples
+ )
+ B = batch_size
+ dataloader = torch.utils.data.DataLoader(
+ self.dataset, batch_size=B, num_workers=batch_size, sampler=sampler
+ )
+ for idx, data in tqdm.tqdm(enumerate(dataloader), disable = romatch.RANK > 0):
+ im_A, im_B, depth1, depth2, T_1to2, K1, K2 = (
+ data["im_A"].cuda(),
+ data["im_B"].cuda(),
+ data["im_A_depth"].cuda(),
+ data["im_B_depth"].cuda(),
+ data["T_1to2"].cuda(),
+ data["K1"].cuda(),
+ data["K2"].cuda(),
+ )
+ matches, certainty = model.match(im_A, im_B, batched=True)
+ gd, pck_1, pck_3, pck_5, prob = self.geometric_dist(
+ depth1, depth2, T_1to2, K1, K2, matches
+ )
+ if romatch.DEBUG_MODE:
+ from romatch.utils.utils import tensor_to_pil
+ import torch.nn.functional as F
+ path = "vis"
+ H, W = model.get_output_resolution()
+ white_im = torch.ones((B,1,H,W),device="cuda")
+ im_B_transfer_rgb = F.grid_sample(
+ im_B.cuda(), matches[:,:,:W, 2:], mode="bilinear", align_corners=False
+ )
+ warp_im = im_B_transfer_rgb
+ c_b = certainty[:,None]#(certainty*0.9 + 0.1*torch.ones_like(certainty))[:,None]
+ vis_im = c_b * warp_im + (1 - c_b) * white_im
+ for b in range(B):
+ import os
+ os.makedirs(f"{path}/{model.name}/{idx}_{b}_{H}_{W}",exist_ok=True)
+ tensor_to_pil(vis_im[b], unnormalize=True).save(
+ f"{path}/{model.name}/{idx}_{b}_{H}_{W}/warp.jpg")
+ tensor_to_pil(im_A[b].cuda(), unnormalize=True).save(
+ f"{path}/{model.name}/{idx}_{b}_{H}_{W}/im_A.jpg")
+ tensor_to_pil(im_B[b].cuda(), unnormalize=True).save(
+ f"{path}/{model.name}/{idx}_{b}_{H}_{W}/im_B.jpg")
+
+
+ gd_tot, pck_1_tot, pck_3_tot, pck_5_tot = (
+ gd_tot + gd.mean(),
+ pck_1_tot + pck_1,
+ pck_3_tot + pck_3,
+ pck_5_tot + pck_5,
+ )
+ return {
+ "epe": gd_tot.item() / len(dataloader),
+ "mega_pck_1": pck_1_tot.item() / len(dataloader),
+ "mega_pck_3": pck_3_tot.item() / len(dataloader),
+ "mega_pck_5": pck_5_tot.item() / len(dataloader),
+ }
diff --git a/submodules/RoMa/romatch/benchmarks/megadepth_pose_estimation_benchmark.py b/submodules/RoMa/romatch/benchmarks/megadepth_pose_estimation_benchmark.py
new file mode 100644
index 0000000000000000000000000000000000000000..36f293f9556d919643f6f39156314a5b402d9082
--- /dev/null
+++ b/submodules/RoMa/romatch/benchmarks/megadepth_pose_estimation_benchmark.py
@@ -0,0 +1,118 @@
+import numpy as np
+import torch
+from romatch.utils import *
+from PIL import Image
+from tqdm import tqdm
+import torch.nn.functional as F
+import romatch
+import kornia.geometry.epipolar as kepi
+
+class MegaDepthPoseEstimationBenchmark:
+ def __init__(self, data_root="data/megadepth", scene_names = None) -> None:
+ if scene_names is None:
+ self.scene_names = [
+ "0015_0.1_0.3.npz",
+ "0015_0.3_0.5.npz",
+ "0022_0.1_0.3.npz",
+ "0022_0.3_0.5.npz",
+ "0022_0.5_0.7.npz",
+ ]
+ else:
+ self.scene_names = scene_names
+ self.scenes = [
+ np.load(f"{data_root}/{scene}", allow_pickle=True)
+ for scene in self.scene_names
+ ]
+ self.data_root = data_root
+
+ def benchmark(self, model, model_name = None):
+ with torch.no_grad():
+ data_root = self.data_root
+ tot_e_t, tot_e_R, tot_e_pose = [], [], []
+ thresholds = [5, 10, 20]
+ for scene_ind in range(len(self.scenes)):
+ import os
+ scene_name = os.path.splitext(self.scene_names[scene_ind])[0]
+ scene = self.scenes[scene_ind]
+ pairs = scene["pair_infos"]
+ intrinsics = scene["intrinsics"]
+ poses = scene["poses"]
+ im_paths = scene["image_paths"]
+ pair_inds = range(len(pairs))
+ for pairind in tqdm(pair_inds):
+ idx1, idx2 = pairs[pairind][0]
+ K1 = intrinsics[idx1].copy()
+ T1 = poses[idx1].copy()
+ R1, t1 = T1[:3, :3], T1[:3, 3]
+ K2 = intrinsics[idx2].copy()
+ T2 = poses[idx2].copy()
+ R2, t2 = T2[:3, :3], T2[:3, 3]
+ R, t = compute_relative_pose(R1, t1, R2, t2)
+ T1_to_2 = np.concatenate((R,t[:,None]), axis=-1)
+ im_A_path = f"{data_root}/{im_paths[idx1]}"
+ im_B_path = f"{data_root}/{im_paths[idx2]}"
+ dense_matches, dense_certainty = model.match(
+ im_A_path, im_B_path, K1.copy(), K2.copy(), T1_to_2.copy()
+ )
+ sparse_matches,_ = model.sample(
+ dense_matches, dense_certainty, 5_000
+ )
+
+ im_A = Image.open(im_A_path)
+ w1, h1 = im_A.size
+ im_B = Image.open(im_B_path)
+ w2, h2 = im_B.size
+ if True: # Note: we keep this true as it was used in DKM/RoMa papers. There is very little difference compared to setting to False.
+ scale1 = 1200 / max(w1, h1)
+ scale2 = 1200 / max(w2, h2)
+ w1, h1 = scale1 * w1, scale1 * h1
+ w2, h2 = scale2 * w2, scale2 * h2
+ K1, K2 = K1.copy(), K2.copy()
+ K1[:2] = K1[:2] * scale1
+ K2[:2] = K2[:2] * scale2
+
+ kpts1, kpts2 = model.to_pixel_coordinates(sparse_matches, h1, w1, h2, w2)
+ kpts1, kpts2 = kpts1.cpu().numpy(), kpts2.cpu().numpy()
+ for _ in range(5):
+ shuffling = np.random.permutation(np.arange(len(kpts1)))
+ kpts1 = kpts1[shuffling]
+ kpts2 = kpts2[shuffling]
+ try:
+ threshold = 0.5
+ norm_threshold = threshold / (np.mean(np.abs(K1[:2, :2])) + np.mean(np.abs(K2[:2, :2])))
+ R_est, t_est, mask = estimate_pose(
+ kpts1,
+ kpts2,
+ K1,
+ K2,
+ norm_threshold,
+ conf=0.99999,
+ )
+ T1_to_2_est = np.concatenate((R_est, t_est), axis=-1) #
+ e_t, e_R = compute_pose_error(T1_to_2_est, R, t)
+ e_pose = max(e_t, e_R)
+ except Exception as e:
+ print(repr(e))
+ e_t, e_R = 90, 90
+ e_pose = max(e_t, e_R)
+ tot_e_t.append(e_t)
+ tot_e_R.append(e_R)
+ tot_e_pose.append(e_pose)
+ tot_e_pose = np.array(tot_e_pose)
+ auc = pose_auc(tot_e_pose, thresholds)
+ acc_5 = (tot_e_pose < 5).mean()
+ acc_10 = (tot_e_pose < 10).mean()
+ acc_15 = (tot_e_pose < 15).mean()
+ acc_20 = (tot_e_pose < 20).mean()
+ map_5 = acc_5
+ map_10 = np.mean([acc_5, acc_10])
+ map_20 = np.mean([acc_5, acc_10, acc_15, acc_20])
+ print(f"{model_name} auc: {auc}")
+ return {
+ "auc_5": auc[0],
+ "auc_10": auc[1],
+ "auc_20": auc[2],
+ "map_5": map_5,
+ "map_10": map_10,
+ "map_20": map_20,
+ }
diff --git a/submodules/RoMa/romatch/benchmarks/megadepth_pose_estimation_benchmark_poselib.py b/submodules/RoMa/romatch/benchmarks/megadepth_pose_estimation_benchmark_poselib.py
new file mode 100644
index 0000000000000000000000000000000000000000..4732ccf2af5b50e6db60831d7c63c5bf70ec727c
--- /dev/null
+++ b/submodules/RoMa/romatch/benchmarks/megadepth_pose_estimation_benchmark_poselib.py
@@ -0,0 +1,119 @@
+import numpy as np
+import torch
+from romatch.utils import *
+from PIL import Image
+from tqdm import tqdm
+import torch.nn.functional as F
+import romatch
+import kornia.geometry.epipolar as kepi
+
+# wrap cause pyposelib is still in dev
+# will add in deps later
+import poselib
+
+class Mega1500PoseLibBenchmark:
+ def __init__(self, data_root="data/megadepth", scene_names = None, num_ransac_iter = 5, test_every = 1) -> None:
+ if scene_names is None:
+ self.scene_names = [
+ "0015_0.1_0.3.npz",
+ "0015_0.3_0.5.npz",
+ "0022_0.1_0.3.npz",
+ "0022_0.3_0.5.npz",
+ "0022_0.5_0.7.npz",
+ ]
+ else:
+ self.scene_names = scene_names
+ self.scenes = [
+ np.load(f"{data_root}/{scene}", allow_pickle=True)
+ for scene in self.scene_names
+ ]
+ self.data_root = data_root
+ self.num_ransac_iter = num_ransac_iter
+ self.test_every = test_every
+
+ def benchmark(self, model, model_name = None):
+ with torch.no_grad():
+ data_root = self.data_root
+ tot_e_t, tot_e_R, tot_e_pose = [], [], []
+ thresholds = [5, 10, 20]
+ for scene_ind in range(len(self.scenes)):
+ import os
+ scene_name = os.path.splitext(self.scene_names[scene_ind])[0]
+ scene = self.scenes[scene_ind]
+ pairs = scene["pair_infos"]
+ intrinsics = scene["intrinsics"]
+ poses = scene["poses"]
+ im_paths = scene["image_paths"]
+ pair_inds = range(len(pairs))[::self.test_every]
+ for pairind in (pbar := tqdm(pair_inds, desc = "Current AUC: ?")):
+ idx1, idx2 = pairs[pairind][0]
+ K1 = intrinsics[idx1].copy()
+ T1 = poses[idx1].copy()
+ R1, t1 = T1[:3, :3], T1[:3, 3]
+ K2 = intrinsics[idx2].copy()
+ T2 = poses[idx2].copy()
+ R2, t2 = T2[:3, :3], T2[:3, 3]
+ R, t = compute_relative_pose(R1, t1, R2, t2)
+ T1_to_2 = np.concatenate((R,t[:,None]), axis=-1)
+ im_A_path = f"{data_root}/{im_paths[idx1]}"
+ im_B_path = f"{data_root}/{im_paths[idx2]}"
+ dense_matches, dense_certainty = model.match(
+ im_A_path, im_B_path, K1.copy(), K2.copy(), T1_to_2.copy()
+ )
+ sparse_matches,_ = model.sample(
+ dense_matches, dense_certainty, 5_000
+ )
+
+ im_A = Image.open(im_A_path)
+ w1, h1 = im_A.size
+ im_B = Image.open(im_B_path)
+ w2, h2 = im_B.size
+ kpts1, kpts2 = model.to_pixel_coordinates(sparse_matches, h1, w1, h2, w2)
+ kpts1, kpts2 = kpts1.cpu().numpy(), kpts2.cpu().numpy()
+ for _ in range(self.num_ransac_iter):
+ shuffling = np.random.permutation(np.arange(len(kpts1)))
+ kpts1 = kpts1[shuffling]
+ kpts2 = kpts2[shuffling]
+ try:
+ threshold = 1
+ camera1 = {'model': 'PINHOLE', 'width': w1, 'height': h1, 'params': K1[[0,1,0,1], [0,1,2,2]]}
+ camera2 = {'model': 'PINHOLE', 'width': w2, 'height': h2, 'params': K2[[0,1,0,1], [0,1,2,2]]}
+ relpose, res = poselib.estimate_relative_pose(
+ kpts1,
+ kpts2,
+ camera1,
+ camera2,
+ ransac_opt = {"max_reproj_error": 2*threshold, "max_epipolar_error": threshold, "min_inliers": 8, "max_iterations": 10_000},
+ )
+ Rt_est = relpose.Rt
+ R_est, t_est = Rt_est[:3,:3], Rt_est[:3,3:]
+ mask = np.array(res['inliers']).astype(np.float32)
+ T1_to_2_est = np.concatenate((R_est, t_est), axis=-1) #
+ e_t, e_R = compute_pose_error(T1_to_2_est, R, t)
+ e_pose = max(e_t, e_R)
+ except Exception as e:
+ print(repr(e))
+ e_t, e_R = 90, 90
+ e_pose = max(e_t, e_R)
+ tot_e_t.append(e_t)
+ tot_e_R.append(e_R)
+ tot_e_pose.append(e_pose)
+ pbar.set_description(f"Current AUC: {pose_auc(tot_e_pose, thresholds)}")
+ tot_e_pose = np.array(tot_e_pose)
+ auc = pose_auc(tot_e_pose, thresholds)
+ acc_5 = (tot_e_pose < 5).mean()
+ acc_10 = (tot_e_pose < 10).mean()
+ acc_15 = (tot_e_pose < 15).mean()
+ acc_20 = (tot_e_pose < 20).mean()
+ map_5 = acc_5
+ map_10 = np.mean([acc_5, acc_10])
+ map_20 = np.mean([acc_5, acc_10, acc_15, acc_20])
+ print(f"{model_name} auc: {auc}")
+ return {
+ "auc_5": auc[0],
+ "auc_10": auc[1],
+ "auc_20": auc[2],
+ "map_5": map_5,
+ "map_10": map_10,
+ "map_20": map_20,
+ }
diff --git a/submodules/RoMa/romatch/benchmarks/scannet_benchmark.py b/submodules/RoMa/romatch/benchmarks/scannet_benchmark.py
new file mode 100644
index 0000000000000000000000000000000000000000..8c6b3c0eeb1c211d224edde84974529c66b52460
--- /dev/null
+++ b/submodules/RoMa/romatch/benchmarks/scannet_benchmark.py
@@ -0,0 +1,143 @@
+import os.path as osp
+import numpy as np
+import torch
+from romatch.utils import *
+from PIL import Image
+from tqdm import tqdm
+
+
+class ScanNetBenchmark:
+ def __init__(self, data_root="data/scannet") -> None:
+ self.data_root = data_root
+
+ def benchmark(self, model, model_name = None):
+ model.train(False)
+ with torch.no_grad():
+ data_root = self.data_root
+ tmp = np.load(osp.join(data_root, "test.npz"))
+ pairs, rel_pose = tmp["name"], tmp["rel_pose"]
+ tot_e_t, tot_e_R, tot_e_pose = [], [], []
+ pair_inds = np.random.choice(
+ range(len(pairs)), size=len(pairs), replace=False
+ )
+ for pairind in tqdm(pair_inds, smoothing=0.9):
+ scene = pairs[pairind]
+ scene_name = f"scene0{scene[0]}_00"
+ im_A_path = osp.join(
+ self.data_root,
+ "scans_test",
+ scene_name,
+ "color",
+ f"{scene[2]}.jpg",
+ )
+ im_A = Image.open(im_A_path)
+ im_B_path = osp.join(
+ self.data_root,
+ "scans_test",
+ scene_name,
+ "color",
+ f"{scene[3]}.jpg",
+ )
+ im_B = Image.open(im_B_path)
+ T_gt = rel_pose[pairind].reshape(3, 4)
+ R, t = T_gt[:3, :3], T_gt[:3, 3]
+ K = np.stack(
+ [
+ np.array([float(i) for i in r.split()])
+ for r in open(
+ osp.join(
+ self.data_root,
+ "scans_test",
+ scene_name,
+ "intrinsic",
+ "intrinsic_color.txt",
+ ),
+ "r",
+ )
+ .read()
+ .split("\n")
+ if r
+ ]
+ )
+ w1, h1 = im_A.size
+ w2, h2 = im_B.size
+ K1 = K.copy()
+ K2 = K.copy()
+ dense_matches, dense_certainty = model.match(im_A_path, im_B_path)
+ sparse_matches, sparse_certainty = model.sample(
+ dense_matches, dense_certainty, 5000
+ )
+ scale1 = 480 / min(w1, h1)
+ scale2 = 480 / min(w2, h2)
+ w1, h1 = scale1 * w1, scale1 * h1
+ w2, h2 = scale2 * w2, scale2 * h2
+ K1 = K1 * scale1
+ K2 = K2 * scale2
+
+ offset = 0.5
+ kpts1 = sparse_matches[:, :2]
+ kpts1 = (
+ np.stack(
+ (
+ w1 * (kpts1[:, 0] + 1) / 2 - offset,
+ h1 * (kpts1[:, 1] + 1) / 2 - offset,
+ ),
+ axis=-1,
+ )
+ )
+ kpts2 = sparse_matches[:, 2:]
+ kpts2 = (
+ np.stack(
+ (
+ w2 * (kpts2[:, 0] + 1) / 2 - offset,
+ h2 * (kpts2[:, 1] + 1) / 2 - offset,
+ ),
+ axis=-1,
+ )
+ )
+ for _ in range(5):
+ shuffling = np.random.permutation(np.arange(len(kpts1)))
+ kpts1 = kpts1[shuffling]
+ kpts2 = kpts2[shuffling]
+ try:
+ norm_threshold = 0.5 / (
+ np.mean(np.abs(K1[:2, :2])) + np.mean(np.abs(K2[:2, :2])))
+ R_est, t_est, mask = estimate_pose(
+ kpts1,
+ kpts2,
+ K1,
+ K2,
+ norm_threshold,
+ conf=0.99999,
+ )
+ T1_to_2_est = np.concatenate((R_est, t_est), axis=-1) #
+ e_t, e_R = compute_pose_error(T1_to_2_est, R, t)
+ e_pose = max(e_t, e_R)
+ except Exception as e:
+ print(repr(e))
+ e_t, e_R = 90, 90
+ e_pose = max(e_t, e_R)
+ tot_e_t.append(e_t)
+ tot_e_R.append(e_R)
+ tot_e_pose.append(e_pose)
+ tot_e_t.append(e_t)
+ tot_e_R.append(e_R)
+ tot_e_pose.append(e_pose)
+ tot_e_pose = np.array(tot_e_pose)
+ thresholds = [5, 10, 20]
+ auc = pose_auc(tot_e_pose, thresholds)
+ acc_5 = (tot_e_pose < 5).mean()
+ acc_10 = (tot_e_pose < 10).mean()
+ acc_15 = (tot_e_pose < 15).mean()
+ acc_20 = (tot_e_pose < 20).mean()
+ map_5 = acc_5
+ map_10 = np.mean([acc_5, acc_10])
+ map_20 = np.mean([acc_5, acc_10, acc_15, acc_20])
+ return {
+ "auc_5": auc[0],
+ "auc_10": auc[1],
+ "auc_20": auc[2],
+ "map_5": map_5,
+ "map_10": map_10,
+ "map_20": map_20,
+ }
diff --git a/submodules/RoMa/romatch/checkpointing/__init__.py b/submodules/RoMa/romatch/checkpointing/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..22f5afe727aa6f6e8fffa9ecf5be69cbff686577
--- /dev/null
+++ b/submodules/RoMa/romatch/checkpointing/__init__.py
@@ -0,0 +1 @@
+from .checkpoint import CheckPoint
diff --git a/submodules/RoMa/romatch/checkpointing/checkpoint.py b/submodules/RoMa/romatch/checkpointing/checkpoint.py
new file mode 100644
index 0000000000000000000000000000000000000000..ab5a5131322241475323f1b992d9d3f7b21dbdac
--- /dev/null
+++ b/submodules/RoMa/romatch/checkpointing/checkpoint.py
@@ -0,0 +1,60 @@
+import os
+import torch
+from torch.nn.parallel.data_parallel import DataParallel
+from torch.nn.parallel.distributed import DistributedDataParallel
+from loguru import logger
+import gc
+
+import romatch
+
+class CheckPoint:
+ def __init__(self, dir=None, name="tmp"):
+ self.name = name
+ self.dir = dir
+ os.makedirs(self.dir, exist_ok=True)
+
+ def save(
+ self,
+ model,
+ optimizer,
+ lr_scheduler,
+ n,
+ ):
+ if romatch.RANK == 0:
+ assert model is not None
+ if isinstance(model, (DataParallel, DistributedDataParallel)):
+ model = model.module
+ states = {
+ "model": model.state_dict(),
+ "n": n,
+ "optimizer": optimizer.state_dict(),
+ "lr_scheduler": lr_scheduler.state_dict(),
+ }
+ torch.save(states, self.dir + self.name + f"_latest.pth")
+ logger.info(f"Saved states {list(states.keys())}, at step {n}")
+
+ def load(
+ self,
+ model,
+ optimizer,
+ lr_scheduler,
+ n,
+ ):
+ if os.path.exists(self.dir + self.name + f"_latest.pth") and romatch.RANK == 0:
+ states = torch.load(self.dir + self.name + f"_latest.pth")
+ if "model" in states:
+ model.load_state_dict(states["model"])
+ if "n" in states:
+ n = states["n"] if states["n"] else n
+ if "optimizer" in states:
+ try:
+ optimizer.load_state_dict(states["optimizer"])
+ except Exception as e:
+ print(f"Failed to load states for optimizer, with error {e}")
+ if "lr_scheduler" in states:
+ lr_scheduler.load_state_dict(states["lr_scheduler"])
+ print(f"Loaded states {list(states.keys())}, at step {n}")
+ del states
+ gc.collect()
+ torch.cuda.empty_cache()
+ return model, optimizer, lr_scheduler, n
\ No newline at end of file
diff --git a/submodules/RoMa/romatch/datasets/__init__.py b/submodules/RoMa/romatch/datasets/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..b60c709926a4a7bd019b73eac10879063a996c90
--- /dev/null
+++ b/submodules/RoMa/romatch/datasets/__init__.py
@@ -0,0 +1,2 @@
+from .megadepth import MegadepthBuilder
+from .scannet import ScanNetBuilder
\ No newline at end of file
diff --git a/submodules/RoMa/romatch/datasets/megadepth.py b/submodules/RoMa/romatch/datasets/megadepth.py
new file mode 100644
index 0000000000000000000000000000000000000000..88f775ef412f7bd062cc9a1d67d95a030e7a15dd
--- /dev/null
+++ b/submodules/RoMa/romatch/datasets/megadepth.py
@@ -0,0 +1,232 @@
+import os
+from PIL import Image
+import h5py
+import numpy as np
+import torch
+import torchvision.transforms.functional as tvf
+import kornia.augmentation as K
+from romatch.utils import get_depth_tuple_transform_ops, get_tuple_transform_ops
+import romatch
+from romatch.utils import *
+import math
+
+class MegadepthScene:
+ def __init__(
+ self,
+ data_root,
+ scene_info,
+ ht=384,
+ wt=512,
+ min_overlap=0.0,
+ max_overlap=1.0,
+ shake_t=0,
+ rot_prob=0.0,
+ normalize=True,
+ max_num_pairs = 100_000,
+ scene_name = None,
+ use_horizontal_flip_aug = False,
+ use_single_horizontal_flip_aug = False,
+ colorjiggle_params = None,
+ random_eraser = None,
+ use_randaug = False,
+ randaug_params = None,
+ randomize_size = False,
+ ) -> None:
+ self.data_root = data_root
+ self.scene_name = os.path.splitext(scene_name)[0]+f"_{min_overlap}_{max_overlap}"
+ self.image_paths = scene_info["image_paths"]
+ self.depth_paths = scene_info["depth_paths"]
+ self.intrinsics = scene_info["intrinsics"]
+ self.poses = scene_info["poses"]
+ self.pairs = scene_info["pairs"]
+ self.overlaps = scene_info["overlaps"]
+ threshold = (self.overlaps > min_overlap) & (self.overlaps < max_overlap)
+ self.pairs = self.pairs[threshold]
+ self.overlaps = self.overlaps[threshold]
+ if len(self.pairs) > max_num_pairs:
+ pairinds = np.random.choice(
+ np.arange(0, len(self.pairs)), max_num_pairs, replace=False
+ )
+ self.pairs = self.pairs[pairinds]
+ self.overlaps = self.overlaps[pairinds]
+ if randomize_size:
+ area = ht * wt
+ s = int(16 * (math.sqrt(area)//16))
+ sizes = ((ht,wt), (s,s), (wt,ht))
+ choice = romatch.RANK % 3
+ ht, wt = sizes[choice]
+ # counts, bins = np.histogram(self.overlaps,20)
+ # print(counts)
+ self.im_transform_ops = get_tuple_transform_ops(
+ resize=(ht, wt), normalize=normalize, colorjiggle_params = colorjiggle_params,
+ )
+ self.depth_transform_ops = get_depth_tuple_transform_ops(
+ resize=(ht, wt)
+ )
+ self.wt, self.ht = wt, ht
+ self.shake_t = shake_t
+ self.random_eraser = random_eraser
+ if use_horizontal_flip_aug and use_single_horizontal_flip_aug:
+ raise ValueError("Can't both flip both images and only flip one")
+ self.use_horizontal_flip_aug = use_horizontal_flip_aug
+ self.use_single_horizontal_flip_aug = use_single_horizontal_flip_aug
+ self.use_randaug = use_randaug
+
+ def load_im(self, im_path):
+ im = Image.open(im_path)
+ return im
+
+ def horizontal_flip(self, im_A, im_B, depth_A, depth_B, K_A, K_B):
+ im_A = im_A.flip(-1)
+ im_B = im_B.flip(-1)
+ depth_A, depth_B = depth_A.flip(-1), depth_B.flip(-1)
+ flip_mat = torch.tensor([[-1, 0, self.wt],[0,1,0],[0,0,1.]]).to(K_A.device)
+ K_A = flip_mat@K_A
+ K_B = flip_mat@K_B
+
+ return im_A, im_B, depth_A, depth_B, K_A, K_B
+
+ def load_depth(self, depth_ref, crop=None):
+ depth = np.array(h5py.File(depth_ref, "r")["depth"])
+ return torch.from_numpy(depth)
+
+ def __len__(self):
+ return len(self.pairs)
+
+ def scale_intrinsic(self, K, wi, hi):
+ sx, sy = self.wt / wi, self.ht / hi
+ sK = torch.tensor([[sx, 0, 0], [0, sy, 0], [0, 0, 1]])
+ return sK @ K
+
+ def rand_shake(self, *things):
+ t = np.random.choice(range(-self.shake_t, self.shake_t + 1), size=2)
+ return [
+ tvf.affine(thing, angle=0.0, translate=list(t), scale=1.0, shear=[0.0, 0.0])
+ for thing in things
+ ], t
+
+ def __getitem__(self, pair_idx):
+ # read intrinsics of original size
+ idx1, idx2 = self.pairs[pair_idx]
+ K1 = torch.tensor(self.intrinsics[idx1].copy(), dtype=torch.float).reshape(3, 3)
+ K2 = torch.tensor(self.intrinsics[idx2].copy(), dtype=torch.float).reshape(3, 3)
+
+ # read and compute relative poses
+ T1 = self.poses[idx1]
+ T2 = self.poses[idx2]
+ T_1to2 = torch.tensor(np.matmul(T2, np.linalg.inv(T1)), dtype=torch.float)[
+ :4, :4
+ ] # (4, 4)
+
+ # Load positive pair data
+ im_A, im_B = self.image_paths[idx1], self.image_paths[idx2]
+ depth1, depth2 = self.depth_paths[idx1], self.depth_paths[idx2]
+ im_A_ref = os.path.join(self.data_root, im_A)
+ im_B_ref = os.path.join(self.data_root, im_B)
+ depth_A_ref = os.path.join(self.data_root, depth1)
+ depth_B_ref = os.path.join(self.data_root, depth2)
+ im_A = self.load_im(im_A_ref)
+ im_B = self.load_im(im_B_ref)
+ K1 = self.scale_intrinsic(K1, im_A.width, im_A.height)
+ K2 = self.scale_intrinsic(K2, im_B.width, im_B.height)
+
+ if self.use_randaug:
+ im_A, im_B = self.rand_augment(im_A, im_B)
+
+ depth_A = self.load_depth(depth_A_ref)
+ depth_B = self.load_depth(depth_B_ref)
+ # Process images
+ im_A, im_B = self.im_transform_ops((im_A, im_B))
+ depth_A, depth_B = self.depth_transform_ops(
+ (depth_A[None, None], depth_B[None, None])
+ )
+
+ [im_A, im_B, depth_A, depth_B], t = self.rand_shake(im_A, im_B, depth_A, depth_B)
+ K1[:2, 2] += t
+ K2[:2, 2] += t
+
+ im_A, im_B = im_A[None], im_B[None]
+ if self.random_eraser is not None:
+ im_A, depth_A = self.random_eraser(im_A, depth_A)
+ im_B, depth_B = self.random_eraser(im_B, depth_B)
+
+ if self.use_horizontal_flip_aug:
+ if np.random.rand() > 0.5:
+ im_A, im_B, depth_A, depth_B, K1, K2 = self.horizontal_flip(im_A, im_B, depth_A, depth_B, K1, K2)
+ if self.use_single_horizontal_flip_aug:
+ if np.random.rand() > 0.5:
+ im_B, depth_B, K2 = self.single_horizontal_flip(im_B, depth_B, K2)
+
+ if romatch.DEBUG_MODE:
+ tensor_to_pil(im_A[0], unnormalize=True).save(
+ f"vis/im_A.jpg")
+ tensor_to_pil(im_B[0], unnormalize=True).save(
+ f"vis/im_B.jpg")
+
+ data_dict = {
+ "im_A": im_A[0],
+ "im_A_identifier": self.image_paths[idx1].split("/")[-1].split(".jpg")[0],
+ "im_B": im_B[0],
+ "im_B_identifier": self.image_paths[idx2].split("/")[-1].split(".jpg")[0],
+ "im_A_depth": depth_A[0, 0],
+ "im_B_depth": depth_B[0, 0],
+ "K1": K1,
+ "K2": K2,
+ "T_1to2": T_1to2,
+ "im_A_path": im_A_ref,
+ "im_B_path": im_B_ref,
+
+ }
+ return data_dict
+
+
+class MegadepthBuilder:
+ def __init__(self, data_root="data/megadepth", loftr_ignore=True, imc21_ignore = True) -> None:
+ self.data_root = data_root
+ self.scene_info_root = os.path.join(data_root, "prep_scene_info")
+ self.all_scenes = os.listdir(self.scene_info_root)
+ self.test_scenes = ["0017.npy", "0004.npy", "0048.npy", "0013.npy"]
+ # LoFTR did the D2-net preprocessing differently than we did and got more ignore scenes, can optionially ignore those
+ self.loftr_ignore_scenes = set(['0121.npy', '0133.npy', '0168.npy', '0178.npy', '0229.npy', '0349.npy', '0412.npy', '0430.npy', '0443.npy', '1001.npy', '5014.npy', '5015.npy', '5016.npy'])
+ self.imc21_scenes = set(['0008.npy', '0019.npy', '0021.npy', '0024.npy', '0025.npy', '0032.npy', '0063.npy', '1589.npy'])
+ self.test_scenes_loftr = ["0015.npy", "0022.npy"]
+ self.loftr_ignore = loftr_ignore
+ self.imc21_ignore = imc21_ignore
+
+ def build_scenes(self, split="train", min_overlap=0.0, scene_names = None, **kwargs):
+ if split == "train":
+ scene_names = set(self.all_scenes) - set(self.test_scenes)
+ elif split == "train_loftr":
+ scene_names = set(self.all_scenes) - set(self.test_scenes_loftr)
+ elif split == "test":
+ scene_names = self.test_scenes
+ elif split == "test_loftr":
+ scene_names = self.test_scenes_loftr
+ elif split == "custom":
+ scene_names = scene_names
+ else:
+ raise ValueError(f"Split {split} not available")
+ scenes = []
+ for scene_name in scene_names:
+ if self.loftr_ignore and scene_name in self.loftr_ignore_scenes:
+ continue
+ if self.imc21_ignore and scene_name in self.imc21_scenes:
+ continue
+ if ".npy" not in scene_name:
+ continue
+ scene_info = np.load(
+ os.path.join(self.scene_info_root, scene_name), allow_pickle=True
+ ).item()
+ scenes.append(
+ MegadepthScene(
+ self.data_root, scene_info, min_overlap=min_overlap,scene_name = scene_name, **kwargs
+ )
+ )
+ return scenes
+
+ def weight_scenes(self, concat_dataset, alpha=0.5):
+ ns = []
+ for d in concat_dataset.datasets:
+ ns.append(len(d))
+ ws = torch.cat([torch.ones(n) / n**alpha for n in ns])
+ return ws
diff --git a/submodules/RoMa/romatch/datasets/scannet.py b/submodules/RoMa/romatch/datasets/scannet.py
new file mode 100644
index 0000000000000000000000000000000000000000..e03261557147cd3449c76576a5e5e22c0ae288e9
--- /dev/null
+++ b/submodules/RoMa/romatch/datasets/scannet.py
@@ -0,0 +1,160 @@
+import os
+import random
+from PIL import Image
+import cv2
+import h5py
+import numpy as np
+import torch
+from torch.utils.data import (
+ Dataset,
+ DataLoader,
+ ConcatDataset)
+
+import torchvision.transforms.functional as tvf
+import kornia.augmentation as K
+import os.path as osp
+import matplotlib.pyplot as plt
+import romatch
+from romatch.utils import get_depth_tuple_transform_ops, get_tuple_transform_ops
+from romatch.utils.transforms import GeometricSequential
+from tqdm import tqdm
+
+class ScanNetScene:
+ def __init__(self, data_root, scene_info, ht = 384, wt = 512, min_overlap=0., shake_t = 0, rot_prob=0.,use_horizontal_flip_aug = False,
+) -> None:
+ self.scene_root = osp.join(data_root,"scans","scans_train")
+ self.data_names = scene_info['name']
+ self.overlaps = scene_info['score']
+ # Only sample 10s
+ valid = (self.data_names[:,-2:] % 10).sum(axis=-1) == 0
+ self.overlaps = self.overlaps[valid]
+ self.data_names = self.data_names[valid]
+ if len(self.data_names) > 10000:
+ pairinds = np.random.choice(np.arange(0,len(self.data_names)),10000,replace=False)
+ self.data_names = self.data_names[pairinds]
+ self.overlaps = self.overlaps[pairinds]
+ self.im_transform_ops = get_tuple_transform_ops(resize=(ht, wt), normalize=True)
+ self.depth_transform_ops = get_depth_tuple_transform_ops(resize=(ht, wt), normalize=False)
+ self.wt, self.ht = wt, ht
+ self.shake_t = shake_t
+ self.H_generator = GeometricSequential(K.RandomAffine(degrees=90, p=rot_prob))
+ self.use_horizontal_flip_aug = use_horizontal_flip_aug
+
+ def load_im(self, im_B, crop=None):
+ im = Image.open(im_B)
+ return im
+
+ def load_depth(self, depth_ref, crop=None):
+ depth = cv2.imread(str(depth_ref), cv2.IMREAD_UNCHANGED)
+ depth = depth / 1000
+ depth = torch.from_numpy(depth).float() # (h, w)
+ return depth
+
+ def __len__(self):
+ return len(self.data_names)
+
+ def scale_intrinsic(self, K, wi, hi):
+ sx, sy = self.wt / wi, self.ht / hi
+ sK = torch.tensor([[sx, 0, 0],
+ [0, sy, 0],
+ [0, 0, 1]])
+ return sK@K
+
+ def horizontal_flip(self, im_A, im_B, depth_A, depth_B, K_A, K_B):
+ im_A = im_A.flip(-1)
+ im_B = im_B.flip(-1)
+ depth_A, depth_B = depth_A.flip(-1), depth_B.flip(-1)
+ flip_mat = torch.tensor([[-1, 0, self.wt],[0,1,0],[0,0,1.]]).to(K_A.device)
+ K_A = flip_mat@K_A
+ K_B = flip_mat@K_B
+
+ return im_A, im_B, depth_A, depth_B, K_A, K_B
+ def read_scannet_pose(self,path):
+ """ Read ScanNet's Camera2World pose and transform it to World2Camera.
+
+ Returns:
+ pose_w2c (np.ndarray): (4, 4)
+ """
+ cam2world = np.loadtxt(path, delimiter=' ')
+ world2cam = np.linalg.inv(cam2world)
+ return world2cam
+
+
+ def read_scannet_intrinsic(self,path):
+ """ Read ScanNet's intrinsic matrix and return the 3x3 matrix.
+ """
+ intrinsic = np.loadtxt(path, delimiter=' ')
+ return torch.tensor(intrinsic[:-1, :-1], dtype = torch.float)
+
+ def __getitem__(self, pair_idx):
+ # read intrinsics of original size
+ data_name = self.data_names[pair_idx]
+ scene_name, scene_sub_name, stem_name_1, stem_name_2 = data_name
+ scene_name = f'scene{scene_name:04d}_{scene_sub_name:02d}'
+
+ # read the intrinsic of depthmap
+ K1 = K2 = self.read_scannet_intrinsic(osp.join(self.scene_root,
+ scene_name,
+ 'intrinsic', 'intrinsic_color.txt'))#the depth K is not the same, but doesnt really matter
+ # read and compute relative poses
+ T1 = self.read_scannet_pose(osp.join(self.scene_root,
+ scene_name,
+ 'pose', f'{stem_name_1}.txt'))
+ T2 = self.read_scannet_pose(osp.join(self.scene_root,
+ scene_name,
+ 'pose', f'{stem_name_2}.txt'))
+ T_1to2 = torch.tensor(np.matmul(T2, np.linalg.inv(T1)), dtype=torch.float)[:4, :4] # (4, 4)
+
+ # Load positive pair data
+ im_A_ref = os.path.join(self.scene_root, scene_name, 'color', f'{stem_name_1}.jpg')
+ im_B_ref = os.path.join(self.scene_root, scene_name, 'color', f'{stem_name_2}.jpg')
+ depth_A_ref = os.path.join(self.scene_root, scene_name, 'depth', f'{stem_name_1}.png')
+ depth_B_ref = os.path.join(self.scene_root, scene_name, 'depth', f'{stem_name_2}.png')
+
+ im_A = self.load_im(im_A_ref)
+ im_B = self.load_im(im_B_ref)
+ depth_A = self.load_depth(depth_A_ref)
+ depth_B = self.load_depth(depth_B_ref)
+
+ # Recompute camera intrinsic matrix due to the resize
+ K1 = self.scale_intrinsic(K1, im_A.width, im_A.height)
+ K2 = self.scale_intrinsic(K2, im_B.width, im_B.height)
+ # Process images
+ im_A, im_B = self.im_transform_ops((im_A, im_B))
+ depth_A, depth_B = self.depth_transform_ops((depth_A[None,None], depth_B[None,None]))
+ if self.use_horizontal_flip_aug:
+ if np.random.rand() > 0.5:
+ im_A, im_B, depth_A, depth_B, K1, K2 = self.horizontal_flip(im_A, im_B, depth_A, depth_B, K1, K2)
+
+ data_dict = {'im_A': im_A,
+ 'im_B': im_B,
+ 'im_A_depth': depth_A[0,0],
+ 'im_B_depth': depth_B[0,0],
+ 'K1': K1,
+ 'K2': K2,
+ 'T_1to2':T_1to2,
+ }
+ return data_dict
+
+
+class ScanNetBuilder:
+ def __init__(self, data_root = 'data/scannet') -> None:
+ self.data_root = data_root
+ self.scene_info_root = os.path.join(data_root,'scannet_indices')
+ self.all_scenes = os.listdir(self.scene_info_root)
+
+ def build_scenes(self, split = 'train', min_overlap=0., **kwargs):
+ # Note: split doesn't matter here as we always use same scannet_train scenes
+ scene_names = self.all_scenes
+ scenes = []
+ for scene_name in tqdm(scene_names, disable = romatch.RANK > 0):
+ scene_info = np.load(os.path.join(self.scene_info_root,scene_name), allow_pickle=True)
+ scenes.append(ScanNetScene(self.data_root, scene_info, min_overlap=min_overlap, **kwargs))
+ return scenes
+
+ def weight_scenes(self, concat_dataset, alpha=.5):
+ ns = []
+ for d in concat_dataset.datasets:
+ ns.append(len(d))
+ ws = torch.cat([torch.ones(n)/n**alpha for n in ns])
+ return ws
diff --git a/submodules/RoMa/romatch/losses/__init__.py b/submodules/RoMa/romatch/losses/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..2e08abacfc0f83d7de0f2ddc0583766a80bf53cf
--- /dev/null
+++ b/submodules/RoMa/romatch/losses/__init__.py
@@ -0,0 +1 @@
+from .robust_loss import RobustLosses
\ No newline at end of file
diff --git a/submodules/RoMa/romatch/losses/robust_loss.py b/submodules/RoMa/romatch/losses/robust_loss.py
new file mode 100644
index 0000000000000000000000000000000000000000..80d430069666fabe2471ec7eda2fa6e9c996f041
--- /dev/null
+++ b/submodules/RoMa/romatch/losses/robust_loss.py
@@ -0,0 +1,161 @@
+from einops.einops import rearrange
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from romatch.utils.utils import get_gt_warp
+import wandb
+import romatch
+import math
+
+class RobustLosses(nn.Module):
+ def __init__(
+ self,
+ robust=False,
+ center_coords=False,
+ scale_normalize=False,
+ ce_weight=0.01,
+ local_loss=True,
+ local_dist=4.0,
+ local_largest_scale=8,
+ smooth_mask = False,
+ depth_interpolation_mode = "bilinear",
+ mask_depth_loss = False,
+ relative_depth_error_threshold = 0.05,
+ alpha = 1.,
+ c = 1e-3,
+ ):
+ super().__init__()
+ self.robust = robust # measured in pixels
+ self.center_coords = center_coords
+ self.scale_normalize = scale_normalize
+ self.ce_weight = ce_weight
+ self.local_loss = local_loss
+ self.local_dist = local_dist
+ self.local_largest_scale = local_largest_scale
+ self.smooth_mask = smooth_mask
+ self.depth_interpolation_mode = depth_interpolation_mode
+ self.mask_depth_loss = mask_depth_loss
+ self.relative_depth_error_threshold = relative_depth_error_threshold
+ self.avg_overlap = dict()
+ self.alpha = alpha
+ self.c = c
+
+ def gm_cls_loss(self, x2, prob, scale_gm_cls, gm_certainty, scale):
+ with torch.no_grad():
+ B, C, H, W = scale_gm_cls.shape
+ device = x2.device
+ cls_res = round(math.sqrt(C))
+ G = torch.meshgrid(*[torch.linspace(-1+1/cls_res, 1 - 1/cls_res, steps = cls_res,device = device) for _ in range(2)], indexing='ij')
+ G = torch.stack((G[1], G[0]), dim = -1).reshape(C,2)
+ GT = (G[None,:,None,None,:]-x2[:,None]).norm(dim=-1).min(dim=1).indices
+ cls_loss = F.cross_entropy(scale_gm_cls, GT, reduction = 'none')[prob > 0.99]
+ certainty_loss = F.binary_cross_entropy_with_logits(gm_certainty[:,0], prob)
+ if not torch.any(cls_loss):
+ cls_loss = (certainty_loss * 0.0) # Prevent issues where prob is 0 everywhere
+
+ losses = {
+ f"gm_certainty_loss_{scale}": certainty_loss.mean(),
+ f"gm_cls_loss_{scale}": cls_loss.mean(),
+ }
+ wandb.log(losses, step = romatch.GLOBAL_STEP)
+ return losses
+
+ def delta_cls_loss(self, x2, prob, flow_pre_delta, delta_cls, certainty, scale, offset_scale):
+ with torch.no_grad():
+ B, C, H, W = delta_cls.shape
+ device = x2.device
+ cls_res = round(math.sqrt(C))
+ G = torch.meshgrid(*[torch.linspace(-1+1/cls_res, 1 - 1/cls_res, steps = cls_res,device = device) for _ in range(2)])
+ G = torch.stack((G[1], G[0]), dim = -1).reshape(C,2) * offset_scale
+ GT = (G[None,:,None,None,:] + flow_pre_delta[:,None] - x2[:,None]).norm(dim=-1).min(dim=1).indices
+ cls_loss = F.cross_entropy(delta_cls, GT, reduction = 'none')[prob > 0.99]
+ certainty_loss = F.binary_cross_entropy_with_logits(certainty[:,0], prob)
+ if not torch.any(cls_loss):
+ cls_loss = (certainty_loss * 0.0) # Prevent issues where prob is 0 everywhere
+ losses = {
+ f"delta_certainty_loss_{scale}": certainty_loss.mean(),
+ f"delta_cls_loss_{scale}": cls_loss.mean(),
+ }
+ wandb.log(losses, step = romatch.GLOBAL_STEP)
+ return losses
+
+ def regression_loss(self, x2, prob, flow, certainty, scale, eps=1e-8, mode = "delta"):
+ epe = (flow.permute(0,2,3,1) - x2).norm(dim=-1)
+ if scale == 1:
+ pck_05 = (epe[prob > 0.99] < 0.5 * (2/512)).float().mean()
+ wandb.log({"train_pck_05": pck_05}, step = romatch.GLOBAL_STEP)
+
+ ce_loss = F.binary_cross_entropy_with_logits(certainty[:, 0], prob)
+ a = self.alpha[scale] if isinstance(self.alpha, dict) else self.alpha
+ cs = self.c * scale
+ x = epe[prob > 0.99]
+ reg_loss = cs**a * ((x/(cs))**2 + 1**2)**(a/2)
+ if not torch.any(reg_loss):
+ reg_loss = (ce_loss * 0.0) # Prevent issues where prob is 0 everywhere
+ losses = {
+ f"{mode}_certainty_loss_{scale}": ce_loss.mean(),
+ f"{mode}_regression_loss_{scale}": reg_loss.mean(),
+ }
+ wandb.log(losses, step = romatch.GLOBAL_STEP)
+ return losses
+
+ def forward(self, corresps, batch):
+ scales = list(corresps.keys())
+ tot_loss = 0.0
+ # scale_weights due to differences in scale for regression gradients and classification gradients
+ scale_weights = {1:1, 2:1, 4:1, 8:1, 16:1}
+ for scale in scales:
+ scale_corresps = corresps[scale]
+ scale_certainty, flow_pre_delta, delta_cls, offset_scale, scale_gm_cls, scale_gm_certainty, flow, scale_gm_flow = (
+ scale_corresps["certainty"],
+ scale_corresps.get("flow_pre_delta"),
+ scale_corresps.get("delta_cls"),
+ scale_corresps.get("offset_scale"),
+ scale_corresps.get("gm_cls"),
+ scale_corresps.get("gm_certainty"),
+ scale_corresps["flow"],
+ scale_corresps.get("gm_flow"),
+
+ )
+ if flow_pre_delta is not None:
+ flow_pre_delta = rearrange(flow_pre_delta, "b d h w -> b h w d")
+ b, h, w, d = flow_pre_delta.shape
+ else:
+ # _ = 1
+ b, _, h, w = scale_certainty.shape
+ gt_warp, gt_prob = get_gt_warp(
+ batch["im_A_depth"],
+ batch["im_B_depth"],
+ batch["T_1to2"],
+ batch["K1"],
+ batch["K2"],
+ H=h,
+ W=w,
+ )
+ x2 = gt_warp.float()
+ prob = gt_prob
+
+ if self.local_largest_scale >= scale:
+ prob = prob * (
+ F.interpolate(prev_epe[:, None], size=(h, w), mode="nearest-exact")[:, 0]
+ < (2 / 512) * (self.local_dist[scale] * scale))
+
+ if scale_gm_cls is not None:
+ gm_cls_losses = self.gm_cls_loss(x2, prob, scale_gm_cls, scale_gm_certainty, scale)
+ gm_loss = self.ce_weight * gm_cls_losses[f"gm_certainty_loss_{scale}"] + gm_cls_losses[f"gm_cls_loss_{scale}"]
+ tot_loss = tot_loss + scale_weights[scale] * gm_loss
+ elif scale_gm_flow is not None:
+ gm_flow_losses = self.regression_loss(x2, prob, scale_gm_flow, scale_gm_certainty, scale, mode = "gm")
+ gm_loss = self.ce_weight * gm_flow_losses[f"gm_certainty_loss_{scale}"] + gm_flow_losses[f"gm_regression_loss_{scale}"]
+ tot_loss = tot_loss + scale_weights[scale] * gm_loss
+
+ if delta_cls is not None:
+ delta_cls_losses = self.delta_cls_loss(x2, prob, flow_pre_delta, delta_cls, scale_certainty, scale, offset_scale)
+ delta_cls_loss = self.ce_weight * delta_cls_losses[f"delta_certainty_loss_{scale}"] + delta_cls_losses[f"delta_cls_loss_{scale}"]
+ tot_loss = tot_loss + scale_weights[scale] * delta_cls_loss
+ else:
+ delta_regression_losses = self.regression_loss(x2, prob, flow, scale_certainty, scale)
+ reg_loss = self.ce_weight * delta_regression_losses[f"delta_certainty_loss_{scale}"] + delta_regression_losses[f"delta_regression_loss_{scale}"]
+ tot_loss = tot_loss + scale_weights[scale] * reg_loss
+ prev_epe = (flow.permute(0,2,3,1) - x2).norm(dim=-1).detach()
+ return tot_loss
diff --git a/submodules/RoMa/romatch/losses/robust_loss_tiny_roma.py b/submodules/RoMa/romatch/losses/robust_loss_tiny_roma.py
new file mode 100644
index 0000000000000000000000000000000000000000..a17c24678b093ca843d16c1a17ea16f19fa594d5
--- /dev/null
+++ b/submodules/RoMa/romatch/losses/robust_loss_tiny_roma.py
@@ -0,0 +1,160 @@
+from einops.einops import rearrange
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from romatch.utils.utils import get_gt_warp
+import wandb
+import romatch
+import math
+
+# This is slightly different than regular romatch due to significantly worse corresps
+# The confidence loss is quite tricky here //Johan
+
+class RobustLosses(nn.Module):
+ def __init__(
+ self,
+ robust=False,
+ center_coords=False,
+ scale_normalize=False,
+ ce_weight=0.01,
+ local_loss=True,
+ local_dist=None,
+ smooth_mask = False,
+ depth_interpolation_mode = "bilinear",
+ mask_depth_loss = False,
+ relative_depth_error_threshold = 0.05,
+ alpha = 1.,
+ c = 1e-3,
+ epe_mask_prob_th = None,
+ cert_only_on_consistent_depth = False,
+ ):
+ super().__init__()
+ if local_dist is None:
+ local_dist = {}
+ self.robust = robust # measured in pixels
+ self.center_coords = center_coords
+ self.scale_normalize = scale_normalize
+ self.ce_weight = ce_weight
+ self.local_loss = local_loss
+ self.local_dist = local_dist
+ self.smooth_mask = smooth_mask
+ self.depth_interpolation_mode = depth_interpolation_mode
+ self.mask_depth_loss = mask_depth_loss
+ self.relative_depth_error_threshold = relative_depth_error_threshold
+ self.avg_overlap = dict()
+ self.alpha = alpha
+ self.c = c
+ self.epe_mask_prob_th = epe_mask_prob_th
+ self.cert_only_on_consistent_depth = cert_only_on_consistent_depth
+
+ def corr_volume_loss(self, mnn:torch.Tensor, corr_volume:torch.Tensor, scale):
+ b, h,w, h,w = corr_volume.shape
+ inv_temp = 10
+ corr_volume = corr_volume.reshape(-1, h*w, h*w)
+ nll = -(inv_temp*corr_volume).log_softmax(dim = 1) - (inv_temp*corr_volume).log_softmax(dim = 2)
+ corr_volume_loss = nll[mnn[:,0], mnn[:,1], mnn[:,2]].mean()
+
+ losses = {
+ f"gm_corr_volume_loss_{scale}": corr_volume_loss.mean(),
+ }
+ wandb.log(losses, step = romatch.GLOBAL_STEP)
+ return losses
+
+
+
+ def regression_loss(self, x2, prob, flow, certainty, scale, eps=1e-8, mode = "delta"):
+ epe = (flow.permute(0,2,3,1) - x2).norm(dim=-1)
+ if scale in self.local_dist:
+ prob = prob * (epe < (2 / 512) * (self.local_dist[scale] * scale)).float()
+ if scale == 1:
+ pck_05 = (epe[prob > 0.99] < 0.5 * (2/512)).float().mean()
+ wandb.log({"train_pck_05": pck_05}, step = romatch.GLOBAL_STEP)
+ if self.epe_mask_prob_th is not None:
+ # if too far away from gt, certainty should be 0
+ gt_cert = prob * (epe < scale * self.epe_mask_prob_th)
+ else:
+ gt_cert = prob
+ if self.cert_only_on_consistent_depth:
+ ce_loss = F.binary_cross_entropy_with_logits(certainty[:, 0][prob > 0], gt_cert[prob > 0])
+ else:
+ ce_loss = F.binary_cross_entropy_with_logits(certainty[:, 0], gt_cert)
+ a = self.alpha[scale] if isinstance(self.alpha, dict) else self.alpha
+ cs = self.c * scale
+ x = epe[prob > 0.99]
+ reg_loss = cs**a * ((x/(cs))**2 + 1**2)**(a/2)
+ if not torch.any(reg_loss):
+ reg_loss = (ce_loss * 0.0) # Prevent issues where prob is 0 everywhere
+ losses = {
+ f"{mode}_certainty_loss_{scale}": ce_loss.mean(),
+ f"{mode}_regression_loss_{scale}": reg_loss.mean(),
+ }
+ wandb.log(losses, step = romatch.GLOBAL_STEP)
+ return losses
+
+ def forward(self, corresps, batch):
+ scales = list(corresps.keys())
+ tot_loss = 0.0
+ # scale_weights due to differences in scale for regression gradients and classification gradients
+ for scale in scales:
+ scale_corresps = corresps[scale]
+ scale_certainty, flow_pre_delta, delta_cls, offset_scale, scale_gm_corr_volume, scale_gm_certainty, flow, scale_gm_flow = (
+ scale_corresps["certainty"],
+ scale_corresps.get("flow_pre_delta"),
+ scale_corresps.get("delta_cls"),
+ scale_corresps.get("offset_scale"),
+ scale_corresps.get("corr_volume"),
+ scale_corresps.get("gm_certainty"),
+ scale_corresps["flow"],
+ scale_corresps.get("gm_flow"),
+
+ )
+ if flow_pre_delta is not None:
+ flow_pre_delta = rearrange(flow_pre_delta, "b d h w -> b h w d")
+ b, h, w, d = flow_pre_delta.shape
+ else:
+ # _ = 1
+ b, _, h, w = scale_certainty.shape
+ gt_warp, gt_prob = get_gt_warp(
+ batch["im_A_depth"],
+ batch["im_B_depth"],
+ batch["T_1to2"],
+ batch["K1"],
+ batch["K2"],
+ H=h,
+ W=w,
+ )
+ x2 = gt_warp.float()
+ prob = gt_prob
+
+ if scale_gm_corr_volume is not None:
+ gt_warp_back, _ = get_gt_warp(
+ batch["im_B_depth"],
+ batch["im_A_depth"],
+ batch["T_1to2"].inverse(),
+ batch["K2"],
+ batch["K1"],
+ H=h,
+ W=w,
+ )
+ grid = torch.stack(torch.meshgrid(torch.linspace(-1+1/w, 1-1/w, w), torch.linspace(-1+1/h, 1-1/h, h), indexing='xy'), dim =-1).to(gt_warp.device)
+ #fwd_bck = F.grid_sample(gt_warp_back.permute(0,3,1,2), gt_warp, align_corners=False, mode = 'bilinear').permute(0,2,3,1)
+ #diff = (fwd_bck - grid).norm(dim = -1)
+ with torch.no_grad():
+ D_B = torch.cdist(gt_warp.float().reshape(-1,h*w,2), grid.reshape(-1,h*w,2))
+ D_A = torch.cdist(grid.reshape(-1,h*w,2), gt_warp_back.float().reshape(-1,h*w,2))
+ inds = torch.nonzero((D_B == D_B.min(dim=-1, keepdim = True).values)
+ * (D_A == D_A.min(dim=-2, keepdim = True).values)
+ * (D_B < 0.01)
+ * (D_A < 0.01))
+
+ gm_cls_losses = self.corr_volume_loss(inds, scale_gm_corr_volume, scale)
+ gm_loss = gm_cls_losses[f"gm_corr_volume_loss_{scale}"]
+ tot_loss = tot_loss + gm_loss
+ elif scale_gm_flow is not None:
+ gm_flow_losses = self.regression_loss(x2, prob, scale_gm_flow, scale_gm_certainty, scale, mode = "gm")
+ gm_loss = self.ce_weight * gm_flow_losses[f"gm_certainty_loss_{scale}"] + gm_flow_losses[f"gm_regression_loss_{scale}"]
+ tot_loss = tot_loss + gm_loss
+ delta_regression_losses = self.regression_loss(x2, prob, flow, scale_certainty, scale)
+ reg_loss = self.ce_weight * delta_regression_losses[f"delta_certainty_loss_{scale}"] + delta_regression_losses[f"delta_regression_loss_{scale}"]
+ tot_loss = tot_loss + reg_loss
+ return tot_loss
diff --git a/submodules/RoMa/romatch/models/__init__.py b/submodules/RoMa/romatch/models/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..7650c9c7480920905e27578f175fcb5f995cc8ba
--- /dev/null
+++ b/submodules/RoMa/romatch/models/__init__.py
@@ -0,0 +1 @@
+from .model_zoo import roma_outdoor, tiny_roma_v1_outdoor, roma_indoor
\ No newline at end of file
diff --git a/submodules/RoMa/romatch/models/encoders.py b/submodules/RoMa/romatch/models/encoders.py
new file mode 100644
index 0000000000000000000000000000000000000000..84fb54395139a2ca21860ce2c18d033ad0afb19f
--- /dev/null
+++ b/submodules/RoMa/romatch/models/encoders.py
@@ -0,0 +1,122 @@
+from typing import Optional, Union
+import torch
+from torch import device
+import torch.nn as nn
+import torch.nn.functional as F
+import torchvision.models as tvm
+import gc
+from romatch.utils.utils import get_autocast_params
+
+
+class ResNet50(nn.Module):
+ def __init__(self, pretrained=False, high_res = False, weights = None,
+ dilation = None, freeze_bn = True, anti_aliased = False, early_exit = False, amp = False, amp_dtype = torch.float16) -> None:
+ super().__init__()
+ if dilation is None:
+ dilation = [False,False,False]
+ if anti_aliased:
+ pass
+ else:
+ if weights is not None:
+ self.net = tvm.resnet50(weights = weights,replace_stride_with_dilation=dilation)
+ else:
+ self.net = tvm.resnet50(pretrained=pretrained,replace_stride_with_dilation=dilation)
+
+ self.high_res = high_res
+ self.freeze_bn = freeze_bn
+ self.early_exit = early_exit
+ self.amp = amp
+ self.amp_dtype = amp_dtype
+
+ def forward(self, x, **kwargs):
+ autocast_device, autocast_enabled, autocast_dtype = get_autocast_params(x.device, self.amp, self.amp_dtype)
+ with torch.autocast(autocast_device, enabled=autocast_enabled, dtype = autocast_dtype):
+ net = self.net
+ feats = {1:x}
+ x = net.conv1(x)
+ x = net.bn1(x)
+ x = net.relu(x)
+ feats[2] = x
+ x = net.maxpool(x)
+ x = net.layer1(x)
+ feats[4] = x
+ x = net.layer2(x)
+ feats[8] = x
+ if self.early_exit:
+ return feats
+ x = net.layer3(x)
+ feats[16] = x
+ x = net.layer4(x)
+ feats[32] = x
+ return feats
+
+ def train(self, mode=True):
+ super().train(mode)
+ if self.freeze_bn:
+ for m in self.modules():
+ if isinstance(m, nn.BatchNorm2d):
+ m.eval()
+ pass
+
+class VGG19(nn.Module):
+ def __init__(self, pretrained=False, amp = False, amp_dtype = torch.float16) -> None:
+ super().__init__()
+ self.layers = nn.ModuleList(tvm.vgg19_bn(pretrained=pretrained).features[:40])
+ self.amp = amp
+ self.amp_dtype = amp_dtype
+
+ def forward(self, x, **kwargs):
+ autocast_device, autocast_enabled, autocast_dtype = get_autocast_params(x.device, self.amp, self.amp_dtype)
+ with torch.autocast(device_type=autocast_device, enabled=autocast_enabled, dtype = autocast_dtype):
+ feats = {}
+ scale = 1
+ for layer in self.layers:
+ if isinstance(layer, nn.MaxPool2d):
+ feats[scale] = x
+ scale = scale*2
+ x = layer(x)
+ return feats
+
+class CNNandDinov2(nn.Module):
+ def __init__(self, cnn_kwargs = None, amp = False, use_vgg = False, dinov2_weights = None, amp_dtype = torch.float16):
+ super().__init__()
+ if dinov2_weights is None:
+ dinov2_weights = torch.hub.load_state_dict_from_url("https://dl.fbaipublicfiles.com/dinov2/dinov2_vitl14/dinov2_vitl14_pretrain.pth", map_location="cpu")
+ from .transformer import vit_large
+ vit_kwargs = dict(img_size= 518,
+ patch_size= 14,
+ init_values = 1.0,
+ ffn_layer = "mlp",
+ block_chunks = 0,
+ )
+
+ dinov2_vitl14 = vit_large(**vit_kwargs).eval()
+ dinov2_vitl14.load_state_dict(dinov2_weights)
+ cnn_kwargs = cnn_kwargs if cnn_kwargs is not None else {}
+ if not use_vgg:
+ self.cnn = ResNet50(**cnn_kwargs)
+ else:
+ self.cnn = VGG19(**cnn_kwargs)
+ self.amp = amp
+ self.amp_dtype = amp_dtype
+ if self.amp:
+ dinov2_vitl14 = dinov2_vitl14.to(self.amp_dtype)
+ self.dinov2_vitl14 = [dinov2_vitl14] # ugly hack to not show parameters to DDP
+
+
+ def train(self, mode: bool = True):
+ return self.cnn.train(mode)
+
+ def forward(self, x, upsample = False):
+ B,C,H,W = x.shape
+ feature_pyramid = self.cnn(x)
+
+ if not upsample:
+ with torch.no_grad():
+ if self.dinov2_vitl14[0].device != x.device:
+ self.dinov2_vitl14[0] = self.dinov2_vitl14[0].to(x.device).to(self.amp_dtype)
+ dinov2_features_16 = self.dinov2_vitl14[0].forward_features(x.to(self.amp_dtype))
+ features_16 = dinov2_features_16['x_norm_patchtokens'].permute(0,2,1).reshape(B,1024,H//14, W//14)
+ del dinov2_features_16
+ feature_pyramid[16] = features_16
+ return feature_pyramid
\ No newline at end of file
diff --git a/submodules/RoMa/romatch/models/matcher.py b/submodules/RoMa/romatch/models/matcher.py
new file mode 100644
index 0000000000000000000000000000000000000000..36c7427cc6a22f6b54b7ce8f9f55e1736cfb7494
--- /dev/null
+++ b/submodules/RoMa/romatch/models/matcher.py
@@ -0,0 +1,760 @@
+import os
+import math
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from einops import rearrange
+import warnings
+from warnings import warn
+from PIL import Image
+
+from romatch.utils import get_tuple_transform_ops
+from romatch.utils.local_correlation import local_correlation
+from romatch.utils.utils import cls_to_flow_refine, get_autocast_params
+from romatch.utils.kde import kde
+
+class ConvRefiner(nn.Module):
+ def __init__(
+ self,
+ in_dim=6,
+ hidden_dim=16,
+ out_dim=2,
+ dw=False,
+ kernel_size=5,
+ hidden_blocks=3,
+ displacement_emb = None,
+ displacement_emb_dim = None,
+ local_corr_radius = None,
+ corr_in_other = None,
+ no_im_B_fm = False,
+ amp = False,
+ concat_logits = False,
+ use_bias_block_1 = True,
+ use_cosine_corr = False,
+ disable_local_corr_grad = False,
+ is_classifier = False,
+ sample_mode = "bilinear",
+ norm_type = nn.BatchNorm2d,
+ bn_momentum = 0.1,
+ amp_dtype = torch.float16,
+ ):
+ super().__init__()
+ self.bn_momentum = bn_momentum
+ self.block1 = self.create_block(
+ in_dim, hidden_dim, dw=dw, kernel_size=kernel_size, bias = use_bias_block_1,
+ )
+ self.hidden_blocks = nn.Sequential(
+ *[
+ self.create_block(
+ hidden_dim,
+ hidden_dim,
+ dw=dw,
+ kernel_size=kernel_size,
+ norm_type=norm_type,
+ )
+ for hb in range(hidden_blocks)
+ ]
+ )
+ self.hidden_blocks = self.hidden_blocks
+ self.out_conv = nn.Conv2d(hidden_dim, out_dim, 1, 1, 0)
+ if displacement_emb:
+ self.has_displacement_emb = True
+ self.disp_emb = nn.Conv2d(2,displacement_emb_dim,1,1,0)
+ else:
+ self.has_displacement_emb = False
+ self.local_corr_radius = local_corr_radius
+ self.corr_in_other = corr_in_other
+ self.no_im_B_fm = no_im_B_fm
+ self.amp = amp
+ self.concat_logits = concat_logits
+ self.use_cosine_corr = use_cosine_corr
+ self.disable_local_corr_grad = disable_local_corr_grad
+ self.is_classifier = is_classifier
+ self.sample_mode = sample_mode
+ self.amp_dtype = amp_dtype
+
+ def create_block(
+ self,
+ in_dim,
+ out_dim,
+ dw=False,
+ kernel_size=5,
+ bias = True,
+ norm_type = nn.BatchNorm2d,
+ ):
+ num_groups = 1 if not dw else in_dim
+ if dw:
+ assert (
+ out_dim % in_dim == 0
+ ), "outdim must be divisible by indim for depthwise"
+ conv1 = nn.Conv2d(
+ in_dim,
+ out_dim,
+ kernel_size=kernel_size,
+ stride=1,
+ padding=kernel_size // 2,
+ groups=num_groups,
+ bias=bias,
+ )
+ norm = norm_type(out_dim, momentum = self.bn_momentum) if norm_type is nn.BatchNorm2d else norm_type(num_channels = out_dim)
+ relu = nn.ReLU(inplace=True)
+ conv2 = nn.Conv2d(out_dim, out_dim, 1, 1, 0)
+ return nn.Sequential(conv1, norm, relu, conv2)
+
+ def forward(self, x, y, flow, scale_factor = 1, logits = None):
+ b,c,hs,ws = x.shape
+ autocast_device, autocast_enabled, autocast_dtype = get_autocast_params(x.device, enabled=self.amp, dtype=self.amp_dtype)
+ with torch.autocast(autocast_device, enabled=autocast_enabled, dtype = autocast_dtype):
+ x_hat = F.grid_sample(y, flow.permute(0, 2, 3, 1), align_corners=False, mode = self.sample_mode)
+ if self.has_displacement_emb:
+ im_A_coords = torch.meshgrid(
+ (
+ torch.linspace(-1 + 1 / hs, 1 - 1 / hs, hs, device=x.device),
+ torch.linspace(-1 + 1 / ws, 1 - 1 / ws, ws, device=x.device),
+ ), indexing='ij'
+ )
+ im_A_coords = torch.stack((im_A_coords[1], im_A_coords[0]))
+ im_A_coords = im_A_coords[None].expand(b, 2, hs, ws)
+ in_displacement = flow-im_A_coords
+ emb_in_displacement = self.disp_emb(40/32 * scale_factor * in_displacement)
+ if self.local_corr_radius:
+ if self.corr_in_other:
+ # Corr in other means take a kxk grid around the predicted coordinate in other image
+ local_corr = local_correlation(x,y,local_radius=self.local_corr_radius,flow = flow,
+ sample_mode = self.sample_mode)
+ else:
+ raise NotImplementedError("Local corr in own frame should not be used.")
+ if self.no_im_B_fm:
+ x_hat = torch.zeros_like(x)
+ d = torch.cat((x, x_hat, emb_in_displacement, local_corr), dim=1)
+ else:
+ d = torch.cat((x, x_hat, emb_in_displacement), dim=1)
+ else:
+ if self.no_im_B_fm:
+ x_hat = torch.zeros_like(x)
+ d = torch.cat((x, x_hat), dim=1)
+ if self.concat_logits:
+ d = torch.cat((d, logits), dim=1)
+ d = self.block1(d)
+ d = self.hidden_blocks(d)
+ d = self.out_conv(d.float())
+ displacement, certainty = d[:, :-1], d[:, -1:]
+ return displacement, certainty
+
+class CosKernel(nn.Module): # similar to softmax kernel
+ def __init__(self, T, learn_temperature=False):
+ super().__init__()
+ self.learn_temperature = learn_temperature
+ if self.learn_temperature:
+ self.T = nn.Parameter(torch.tensor(T))
+ else:
+ self.T = T
+
+ def __call__(self, x, y, eps=1e-6):
+ c = torch.einsum("bnd,bmd->bnm", x, y) / (
+ x.norm(dim=-1)[..., None] * y.norm(dim=-1)[:, None] + eps
+ )
+ if self.learn_temperature:
+ T = self.T.abs() + 0.01
+ else:
+ T = torch.tensor(self.T, device=c.device)
+ K = ((c - 1.0) / T).exp()
+ return K
+
+class GP(nn.Module):
+ def __init__(
+ self,
+ kernel,
+ T=1,
+ learn_temperature=False,
+ only_attention=False,
+ gp_dim=64,
+ basis="fourier",
+ covar_size=5,
+ only_nearest_neighbour=False,
+ sigma_noise=0.1,
+ no_cov=False,
+ predict_features = False,
+ ):
+ super().__init__()
+ self.K = kernel(T=T, learn_temperature=learn_temperature)
+ self.sigma_noise = sigma_noise
+ self.covar_size = covar_size
+ self.pos_conv = torch.nn.Conv2d(2, gp_dim, 1, 1)
+ self.only_attention = only_attention
+ self.only_nearest_neighbour = only_nearest_neighbour
+ self.basis = basis
+ self.no_cov = no_cov
+ self.dim = gp_dim
+ self.predict_features = predict_features
+
+ def get_local_cov(self, cov):
+ K = self.covar_size
+ b, h, w, h, w = cov.shape
+ hw = h * w
+ cov = F.pad(cov, 4 * (K // 2,)) # pad v_q
+ delta = torch.stack(
+ torch.meshgrid(
+ torch.arange(-(K // 2), K // 2 + 1), torch.arange(-(K // 2), K // 2 + 1),
+ indexing = 'ij'),
+ dim=-1,
+ )
+ positions = torch.stack(
+ torch.meshgrid(
+ torch.arange(K // 2, h + K // 2), torch.arange(K // 2, w + K // 2),
+ indexing = 'ij'),
+ dim=-1,
+ )
+ neighbours = positions[:, :, None, None, :] + delta[None, :, :]
+ points = torch.arange(hw)[:, None].expand(hw, K**2)
+ local_cov = cov.reshape(b, hw, h + K - 1, w + K - 1)[
+ :,
+ points.flatten(),
+ neighbours[..., 0].flatten(),
+ neighbours[..., 1].flatten(),
+ ].reshape(b, h, w, K**2)
+ return local_cov
+
+ def reshape(self, x):
+ return rearrange(x, "b d h w -> b (h w) d")
+
+ def project_to_basis(self, x):
+ if self.basis == "fourier":
+ return torch.cos(8 * math.pi * self.pos_conv(x))
+ elif self.basis == "linear":
+ return self.pos_conv(x)
+ else:
+ raise ValueError(
+ "No other bases other than fourier and linear currently im_Bed in public release"
+ )
+
+ def get_pos_enc(self, y):
+ b, c, h, w = y.shape
+ coarse_coords = torch.meshgrid(
+ (
+ torch.linspace(-1 + 1 / h, 1 - 1 / h, h, device=y.device),
+ torch.linspace(-1 + 1 / w, 1 - 1 / w, w, device=y.device),
+ ),
+ indexing = 'ij'
+ )
+
+ coarse_coords = torch.stack((coarse_coords[1], coarse_coords[0]), dim=-1)[
+ None
+ ].expand(b, h, w, 2)
+ coarse_coords = rearrange(coarse_coords, "b h w d -> b d h w")
+ coarse_embedded_coords = self.project_to_basis(coarse_coords)
+ return coarse_embedded_coords
+
+ def forward(self, x, y, **kwargs):
+ b, c, h1, w1 = x.shape
+ b, c, h2, w2 = y.shape
+ f = self.get_pos_enc(y)
+ b, d, h2, w2 = f.shape
+ x, y, f = self.reshape(x.float()), self.reshape(y.float()), self.reshape(f)
+ K_xx = self.K(x, x)
+ K_yy = self.K(y, y)
+ K_xy = self.K(x, y)
+ K_yx = K_xy.permute(0, 2, 1)
+ sigma_noise = self.sigma_noise * torch.eye(h2 * w2, device=x.device)[None, :, :]
+ with warnings.catch_warnings():
+ K_yy_inv = torch.linalg.inv(K_yy + sigma_noise)
+
+ mu_x = K_xy.matmul(K_yy_inv.matmul(f))
+ mu_x = rearrange(mu_x, "b (h w) d -> b d h w", h=h1, w=w1)
+ if not self.no_cov:
+ cov_x = K_xx - K_xy.matmul(K_yy_inv.matmul(K_yx))
+ cov_x = rearrange(cov_x, "b (h w) (r c) -> b h w r c", h=h1, w=w1, r=h1, c=w1)
+ local_cov_x = self.get_local_cov(cov_x)
+ local_cov_x = rearrange(local_cov_x, "b h w K -> b K h w")
+ gp_feats = torch.cat((mu_x, local_cov_x), dim=1)
+ else:
+ gp_feats = mu_x
+ return gp_feats
+
+class Decoder(nn.Module):
+ def __init__(
+ self, embedding_decoder, gps, proj, conv_refiner, detach=False, scales="all", pos_embeddings = None,
+ num_refinement_steps_per_scale = 1, warp_noise_std = 0.0, displacement_dropout_p = 0.0, gm_warp_dropout_p = 0.0,
+ flow_upsample_mode = "bilinear", amp_dtype = torch.float16,
+ ):
+ super().__init__()
+ self.embedding_decoder = embedding_decoder
+ self.num_refinement_steps_per_scale = num_refinement_steps_per_scale
+ self.gps = gps
+ self.proj = proj
+ self.conv_refiner = conv_refiner
+ self.detach = detach
+ if pos_embeddings is None:
+ self.pos_embeddings = {}
+ else:
+ self.pos_embeddings = pos_embeddings
+ if scales == "all":
+ self.scales = ["32", "16", "8", "4", "2", "1"]
+ else:
+ self.scales = scales
+ self.warp_noise_std = warp_noise_std
+ self.refine_init = 4
+ self.displacement_dropout_p = displacement_dropout_p
+ self.gm_warp_dropout_p = gm_warp_dropout_p
+ self.flow_upsample_mode = flow_upsample_mode
+ self.amp_dtype = amp_dtype
+
+ def get_placeholder_flow(self, b, h, w, device):
+ coarse_coords = torch.meshgrid(
+ (
+ torch.linspace(-1 + 1 / h, 1 - 1 / h, h, device=device),
+ torch.linspace(-1 + 1 / w, 1 - 1 / w, w, device=device),
+ ),
+ indexing = 'ij'
+ )
+ coarse_coords = torch.stack((coarse_coords[1], coarse_coords[0]), dim=-1)[
+ None
+ ].expand(b, h, w, 2)
+ coarse_coords = rearrange(coarse_coords, "b h w d -> b d h w")
+ return coarse_coords
+
+ def get_positional_embedding(self, b, h ,w, device):
+ coarse_coords = torch.meshgrid(
+ (
+ torch.linspace(-1 + 1 / h, 1 - 1 / h, h, device=device),
+ torch.linspace(-1 + 1 / w, 1 - 1 / w, w, device=device),
+ ),
+ indexing = 'ij'
+ )
+
+ coarse_coords = torch.stack((coarse_coords[1], coarse_coords[0]), dim=-1)[
+ None
+ ].expand(b, h, w, 2)
+ coarse_coords = rearrange(coarse_coords, "b h w d -> b d h w")
+ coarse_embedded_coords = self.pos_embedding(coarse_coords)
+ return coarse_embedded_coords
+
+ def forward(self, f1, f2, gt_warp = None, gt_prob = None, upsample = False, flow = None, certainty = None, scale_factor = 1):
+ coarse_scales = self.embedding_decoder.scales()
+ all_scales = self.scales if not upsample else ["8", "4", "2", "1"]
+ sizes = {scale: f1[scale].shape[-2:] for scale in f1}
+ h, w = sizes[1]
+ b = f1[1].shape[0]
+ device = f1[1].device
+ coarsest_scale = int(all_scales[0])
+ old_stuff = torch.zeros(
+ b, self.embedding_decoder.hidden_dim, *sizes[coarsest_scale], device=f1[coarsest_scale].device
+ )
+ corresps = {}
+ if not upsample:
+ flow = self.get_placeholder_flow(b, *sizes[coarsest_scale], device)
+ certainty = 0.0
+ else:
+ flow = F.interpolate(
+ flow,
+ size=sizes[coarsest_scale],
+ align_corners=False,
+ mode="bilinear",
+ )
+ certainty = F.interpolate(
+ certainty,
+ size=sizes[coarsest_scale],
+ align_corners=False,
+ mode="bilinear",
+ )
+ displacement = 0.0
+ for new_scale in all_scales:
+ ins = int(new_scale)
+ corresps[ins] = {}
+ f1_s, f2_s = f1[ins], f2[ins]
+ if new_scale in self.proj:
+ autocast_device, autocast_enabled, autocast_dtype = get_autocast_params(f1_s.device, str(f1_s)=='cuda', self.amp_dtype)
+ with torch.autocast(autocast_device, enabled=autocast_enabled, dtype = autocast_dtype):
+ if not autocast_enabled:
+ f1_s, f2_s = f1_s.to(torch.float32), f2_s.to(torch.float32)
+ f1_s, f2_s = self.proj[new_scale](f1_s), self.proj[new_scale](f2_s)
+
+ if ins in coarse_scales:
+ old_stuff = F.interpolate(
+ old_stuff, size=sizes[ins], mode="bilinear", align_corners=False
+ )
+ gp_posterior = self.gps[new_scale](f1_s, f2_s)
+ gm_warp_or_cls, certainty, old_stuff = self.embedding_decoder(
+ gp_posterior, f1_s, old_stuff, new_scale
+ )
+
+ if self.embedding_decoder.is_classifier:
+ flow = cls_to_flow_refine(
+ gm_warp_or_cls,
+ ).permute(0,3,1,2)
+ corresps[ins].update({"gm_cls": gm_warp_or_cls,"gm_certainty": certainty,}) if self.training else None
+ else:
+ corresps[ins].update({"gm_flow": gm_warp_or_cls,"gm_certainty": certainty,}) if self.training else None
+ flow = gm_warp_or_cls.detach()
+
+ if new_scale in self.conv_refiner:
+ corresps[ins].update({"flow_pre_delta": flow}) if self.training else None
+ delta_flow, delta_certainty = self.conv_refiner[new_scale](
+ f1_s, f2_s, flow, scale_factor = scale_factor, logits = certainty,
+ )
+ corresps[ins].update({"delta_flow": delta_flow,}) if self.training else None
+ displacement = ins*torch.stack((delta_flow[:, 0].float() / (self.refine_init * w),
+ delta_flow[:, 1].float() / (self.refine_init * h),),dim=1,)
+ flow = flow + displacement
+ certainty = (
+ certainty + delta_certainty
+ ) # predict both certainty and displacement
+ corresps[ins].update({
+ "certainty": certainty,
+ "flow": flow,
+ })
+ if new_scale != "1":
+ flow = F.interpolate(
+ flow,
+ size=sizes[ins // 2],
+ mode=self.flow_upsample_mode,
+ )
+ certainty = F.interpolate(
+ certainty,
+ size=sizes[ins // 2],
+ mode=self.flow_upsample_mode,
+ )
+ if self.detach:
+ flow = flow.detach()
+ certainty = certainty.detach()
+ #torch.cuda.empty_cache()
+ return corresps
+
+
+class RegressionMatcher(nn.Module):
+ def __init__(
+ self,
+ encoder,
+ decoder,
+ h=448,
+ w=448,
+ sample_mode = "threshold_balanced",
+ upsample_preds = False,
+ symmetric = False,
+ name = None,
+ attenuate_cert = None,
+ ):
+ super().__init__()
+ self.attenuate_cert = attenuate_cert
+ self.encoder = encoder
+ self.decoder = decoder
+ self.name = name
+ self.w_resized = w
+ self.h_resized = h
+ self.og_transforms = get_tuple_transform_ops(resize=None, normalize=True)
+ self.sample_mode = sample_mode
+ self.upsample_preds = upsample_preds
+ self.upsample_res = (14*16*6, 14*16*6)
+ self.symmetric = symmetric
+ self.sample_thresh = 0.05
+
+ def get_output_resolution(self):
+ if not self.upsample_preds:
+ return self.h_resized, self.w_resized
+ else:
+ return self.upsample_res
+
+ def extract_backbone_features(self, batch, batched = True, upsample = False):
+ x_q = batch["im_A"]
+ x_s = batch["im_B"]
+ if batched:
+ X = torch.cat((x_q, x_s), dim = 0)
+ feature_pyramid = self.encoder(X, upsample = upsample)
+ else:
+ feature_pyramid = self.encoder(x_q, upsample = upsample), self.encoder(x_s, upsample = upsample)
+ return feature_pyramid
+
+ def sample(
+ self,
+ matches,
+ certainty,
+ num=10000,
+ ):
+ if "threshold" in self.sample_mode:
+ upper_thresh = self.sample_thresh
+ certainty = certainty.clone()
+ certainty[certainty > upper_thresh] = 1
+ matches, certainty = (
+ matches.reshape(-1, 4),
+ certainty.reshape(-1),
+ )
+ expansion_factor = 4 if "balanced" in self.sample_mode else 1
+ good_samples = torch.multinomial(certainty,
+ num_samples = min(expansion_factor*num, len(certainty)),
+ replacement=False)
+ good_matches, good_certainty = matches[good_samples], certainty[good_samples]
+ if "balanced" not in self.sample_mode:
+ return good_matches, good_certainty
+ density = kde(good_matches, std=0.1)
+ p = 1 / (density+1)
+ p[density < 10] = 1e-7 # Basically should have at least 10 perfect neighbours, or around 100 ok ones
+ balanced_samples = torch.multinomial(p,
+ num_samples = min(num,len(good_certainty)),
+ replacement=False)
+ return good_matches[balanced_samples], good_certainty[balanced_samples]
+
+ def forward(self, batch, batched = True, upsample = False, scale_factor = 1):
+ feature_pyramid = self.extract_backbone_features(batch, batched=batched, upsample = upsample)
+ if batched:
+ f_q_pyramid = {
+ scale: f_scale.chunk(2)[0] for scale, f_scale in feature_pyramid.items()
+ }
+ f_s_pyramid = {
+ scale: f_scale.chunk(2)[1] for scale, f_scale in feature_pyramid.items()
+ }
+ else:
+ f_q_pyramid, f_s_pyramid = feature_pyramid
+ corresps = self.decoder(f_q_pyramid,
+ f_s_pyramid,
+ upsample = upsample,
+ **(batch["corresps"] if "corresps" in batch else {}),
+ scale_factor=scale_factor)
+
+ return corresps
+
+ def forward_symmetric(self, batch, batched = True, upsample = False, scale_factor = 1):
+ feature_pyramid = self.extract_backbone_features(batch, batched = batched, upsample = upsample)
+ f_q_pyramid = feature_pyramid
+ f_s_pyramid = {
+ scale: torch.cat((f_scale.chunk(2)[1], f_scale.chunk(2)[0]), dim = 0)
+ for scale, f_scale in feature_pyramid.items()
+ }
+ corresps = self.decoder(f_q_pyramid,
+ f_s_pyramid,
+ upsample = upsample,
+ **(batch["corresps"] if "corresps" in batch else {}),
+ scale_factor=scale_factor)
+ return corresps
+
+ def conf_from_fb_consistency(self, flow_forward, flow_backward, th = 2):
+ # assumes that flow forward is of shape (..., H, W, 2)
+ has_batch = False
+ if len(flow_forward.shape) == 3:
+ flow_forward, flow_backward = flow_forward[None], flow_backward[None]
+ else:
+ has_batch = True
+ H,W = flow_forward.shape[-3:-1]
+ th_n = 2 * th / max(H,W)
+ coords = torch.stack(torch.meshgrid(
+ torch.linspace(-1 + 1 / W, 1 - 1 / W, W),
+ torch.linspace(-1 + 1 / H, 1 - 1 / H, H), indexing = "xy"),
+ dim = -1).to(flow_forward.device)
+ coords_fb = F.grid_sample(
+ flow_backward.permute(0, 3, 1, 2),
+ flow_forward,
+ align_corners=False, mode="bilinear").permute(0, 2, 3, 1)
+ diff = (coords - coords_fb).norm(dim=-1)
+ in_th = (diff < th_n).float()
+ if not has_batch:
+ in_th = in_th[0]
+ return in_th
+
+ def to_pixel_coordinates(self, coords, H_A, W_A, H_B = None, W_B = None):
+ if coords.shape[-1] == 2:
+ return self._to_pixel_coordinates(coords, H_A, W_A)
+
+ if isinstance(coords, (list, tuple)):
+ kpts_A, kpts_B = coords[0], coords[1]
+ else:
+ kpts_A, kpts_B = coords[...,:2], coords[...,2:]
+ return self._to_pixel_coordinates(kpts_A, H_A, W_A), self._to_pixel_coordinates(kpts_B, H_B, W_B)
+
+ def _to_pixel_coordinates(self, coords, H, W):
+ kpts = torch.stack((W/2 * (coords[...,0]+1), H/2 * (coords[...,1]+1)),axis=-1)
+ return kpts
+
+ def to_normalized_coordinates(self, coords, H_A, W_A, H_B, W_B):
+ if isinstance(coords, (list, tuple)):
+ kpts_A, kpts_B = coords[0], coords[1]
+ else:
+ kpts_A, kpts_B = coords[...,:2], coords[...,2:]
+ kpts_A = torch.stack((2/W_A * kpts_A[...,0] - 1, 2/H_A * kpts_A[...,1] - 1),axis=-1)
+ kpts_B = torch.stack((2/W_B * kpts_B[...,0] - 1, 2/H_B * kpts_B[...,1] - 1),axis=-1)
+ return kpts_A, kpts_B
+
+ def match_keypoints(self, x_A, x_B, warp, certainty, return_tuple = True, return_inds = False):
+ x_A_to_B = F.grid_sample(warp[...,-2:].permute(2,0,1)[None], x_A[None,None], align_corners = False, mode = "bilinear")[0,:,0].mT
+ cert_A_to_B = F.grid_sample(certainty[None,None,...], x_A[None,None], align_corners = False, mode = "bilinear")[0,0,0]
+ D = torch.cdist(x_A_to_B, x_B)
+ inds_A, inds_B = torch.nonzero((D == D.min(dim=-1, keepdim = True).values) * (D == D.min(dim=-2, keepdim = True).values) * (cert_A_to_B[:,None] > self.sample_thresh), as_tuple = True)
+
+ if return_tuple:
+ if return_inds:
+ return inds_A, inds_B
+ else:
+ return x_A[inds_A], x_B[inds_B]
+ else:
+ if return_inds:
+ return torch.cat((inds_A, inds_B),dim=-1)
+ else:
+ return torch.cat((x_A[inds_A], x_B[inds_B]),dim=-1)
+
+ @torch.inference_mode()
+ def match(
+ self,
+ im_A_input,
+ im_B_input,
+ *args,
+ batched=False,
+ device=None,
+ ):
+ if device is None:
+ device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+
+ # Check if inputs are file paths or already loaded images
+ if isinstance(im_A_input, (str, os.PathLike)):
+ im_A = Image.open(im_A_input).convert("RGB")
+ else:
+ im_A = im_A_input
+
+ if isinstance(im_B_input, (str, os.PathLike)):
+ im_B = Image.open(im_B_input).convert("RGB")
+ else:
+ im_B = im_B_input
+
+ symmetric = self.symmetric
+ self.train(False)
+ with torch.no_grad():
+ if not batched:
+ b = 1
+ w, h = im_A.size
+ w2, h2 = im_B.size
+ # Get images in good format
+ ws = self.w_resized
+ hs = self.h_resized
+
+ test_transform = get_tuple_transform_ops(
+ resize=(hs, ws), normalize=True, clahe=False
+ )
+ im_A, im_B = test_transform((im_A, im_B))
+ batch = {"im_A": im_A[None].to(device), "im_B": im_B[None].to(device)}
+ else:
+ b, c, h, w = im_A.shape
+ b, c, h2, w2 = im_B.shape
+ assert w == w2 and h == h2, "For batched images we assume same size"
+ batch = {"im_A": im_A.to(device), "im_B": im_B.to(device)}
+ if h != self.h_resized or self.w_resized != w:
+ warn("Model resolution and batch resolution differ, may produce unexpected results")
+ hs, ws = h, w
+ finest_scale = 1
+ # Run matcher
+ if symmetric:
+ corresps = self.forward_symmetric(batch)
+ else:
+ corresps = self.forward(batch, batched=True)
+
+ if self.upsample_preds:
+ hs, ws = self.upsample_res
+
+ if self.attenuate_cert:
+ low_res_certainty = F.interpolate(
+ corresps[16]["certainty"], size=(hs, ws), align_corners=False, mode="bilinear"
+ )
+ cert_clamp = 0
+ factor = 0.5
+ low_res_certainty = factor * low_res_certainty * (low_res_certainty < cert_clamp)
+
+ if self.upsample_preds:
+ finest_corresps = corresps[finest_scale]
+ torch.cuda.empty_cache()
+ test_transform = get_tuple_transform_ops(
+ resize=(hs, ws), normalize=True
+ )
+ if isinstance(im_A_input, (str, os.PathLike)):
+ im_A, im_B = test_transform(
+ (Image.open(im_A_input).convert('RGB'), Image.open(im_B_input).convert('RGB')))
+ else:
+ im_A, im_B = test_transform((im_A_input, im_B_input))
+
+ im_A, im_B = im_A[None].to(device), im_B[None].to(device)
+ scale_factor = math.sqrt(self.upsample_res[0] * self.upsample_res[1] / (self.w_resized * self.h_resized))
+ batch = {"im_A": im_A, "im_B": im_B, "corresps": finest_corresps}
+ if symmetric:
+ corresps = self.forward_symmetric(batch, upsample=True, batched=True, scale_factor=scale_factor)
+ else:
+ corresps = self.forward(batch, batched=True, upsample=True, scale_factor=scale_factor)
+
+ im_A_to_im_B = corresps[finest_scale]["flow"]
+ certainty = corresps[finest_scale]["certainty"] - (low_res_certainty if self.attenuate_cert else 0)
+ if finest_scale != 1:
+ im_A_to_im_B = F.interpolate(
+ im_A_to_im_B, size=(hs, ws), align_corners=False, mode="bilinear"
+ )
+ certainty = F.interpolate(
+ certainty, size=(hs, ws), align_corners=False, mode="bilinear"
+ )
+ im_A_to_im_B = im_A_to_im_B.permute(
+ 0, 2, 3, 1
+ )
+ # Create im_A meshgrid
+ im_A_coords = torch.meshgrid(
+ (
+ torch.linspace(-1 + 1 / hs, 1 - 1 / hs, hs, device=device),
+ torch.linspace(-1 + 1 / ws, 1 - 1 / ws, ws, device=device),
+ ),
+ indexing='ij'
+ )
+ im_A_coords = torch.stack((im_A_coords[1], im_A_coords[0]))
+ im_A_coords = im_A_coords[None].expand(b, 2, hs, ws)
+ certainty = certainty.sigmoid() # logits -> probs
+ im_A_coords = im_A_coords.permute(0, 2, 3, 1)
+ if (im_A_to_im_B.abs() > 1).any() and True:
+ wrong = (im_A_to_im_B.abs() > 1).sum(dim=-1) > 0
+ certainty[wrong[:, None]] = 0
+ im_A_to_im_B = torch.clamp(im_A_to_im_B, -1, 1)
+ if symmetric:
+ A_to_B, B_to_A = im_A_to_im_B.chunk(2)
+ q_warp = torch.cat((im_A_coords, A_to_B), dim=-1)
+ im_B_coords = im_A_coords
+ s_warp = torch.cat((B_to_A, im_B_coords), dim=-1)
+ warp = torch.cat((q_warp, s_warp), dim=2)
+ certainty = torch.cat(certainty.chunk(2), dim=3)
+ else:
+ warp = torch.cat((im_A_coords, im_A_to_im_B), dim=-1)
+ if batched:
+ return (
+ warp,
+ certainty[:, 0]
+ )
+ else:
+ return (
+ warp[0],
+ certainty[0, 0],
+ )
+
+ def visualize_warp(self, warp, certainty, im_A = None, im_B = None,
+ im_A_path = None, im_B_path = None, device = "cuda", symmetric = True, save_path = None, unnormalize = False):
+ #assert symmetric == True, "Currently assuming bidirectional warp, might update this if someone complains ;)"
+ H,W2,_ = warp.shape
+ W = W2//2 if symmetric else W2
+ if im_A is None:
+ from PIL import Image
+ im_A, im_B = Image.open(im_A_path).convert("RGB"), Image.open(im_B_path).convert("RGB")
+ if not isinstance(im_A, torch.Tensor):
+ im_A = im_A.resize((W,H))
+ im_B = im_B.resize((W,H))
+ x_B = (torch.tensor(np.array(im_B)) / 255).to(device).permute(2, 0, 1)
+ if symmetric:
+ x_A = (torch.tensor(np.array(im_A)) / 255).to(device).permute(2, 0, 1)
+ else:
+ if symmetric:
+ x_A = im_A
+ x_B = im_B
+ im_A_transfer_rgb = F.grid_sample(
+ x_B[None], warp[:,:W, 2:][None], mode="bilinear", align_corners=False
+ )[0]
+ if symmetric:
+ im_B_transfer_rgb = F.grid_sample(
+ x_A[None], warp[:, W:, :2][None], mode="bilinear", align_corners=False
+ )[0]
+ warp_im = torch.cat((im_A_transfer_rgb,im_B_transfer_rgb),dim=2)
+ white_im = torch.ones((H,2*W),device=device)
+ else:
+ warp_im = im_A_transfer_rgb
+ white_im = torch.ones((H, W), device = device)
+ vis_im = certainty * warp_im + (1 - certainty) * white_im
+ if save_path is not None:
+ from romatch.utils import tensor_to_pil
+ tensor_to_pil(vis_im, unnormalize=unnormalize).save(save_path)
+ return vis_im
diff --git a/submodules/RoMa/romatch/models/model_zoo/__init__.py b/submodules/RoMa/romatch/models/model_zoo/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..d0470ca3f0c3b8064b1b2f01663dfb13742d7a10
--- /dev/null
+++ b/submodules/RoMa/romatch/models/model_zoo/__init__.py
@@ -0,0 +1,73 @@
+from typing import Union
+import torch
+from .roma_models import roma_model, tiny_roma_v1_model
+
+weight_urls = {
+ "romatch": {
+ "outdoor": "https://github.com/Parskatt/storage/releases/download/roma/roma_outdoor.pth",
+ "indoor": "https://github.com/Parskatt/storage/releases/download/roma/roma_indoor.pth",
+ },
+ "tiny_roma_v1": {
+ "outdoor": "https://github.com/Parskatt/storage/releases/download/roma/tiny_roma_v1_outdoor.pth",
+ },
+ "dinov2": "https://dl.fbaipublicfiles.com/dinov2/dinov2_vitl14/dinov2_vitl14_pretrain.pth", #hopefully this doesnt change :D
+}
+
+def tiny_roma_v1_outdoor(device, weights = None, xfeat = None):
+ if weights is None:
+ weights = torch.hub.load_state_dict_from_url(
+ weight_urls["tiny_roma_v1"]["outdoor"],
+ map_location=device)
+ if xfeat is None:
+ xfeat = torch.hub.load(
+ 'verlab/accelerated_features',
+ 'XFeat',
+ pretrained = True,
+ top_k = 4096).net
+
+ return tiny_roma_v1_model(weights = weights, xfeat = xfeat).to(device)
+
+def roma_outdoor(device, weights=None, dinov2_weights=None, coarse_res: Union[int,tuple[int,int]] = 560, upsample_res: Union[int,tuple[int,int]] = 864, amp_dtype: torch.dtype = torch.float16):
+ if isinstance(coarse_res, int):
+ coarse_res = (coarse_res, coarse_res)
+ if isinstance(upsample_res, int):
+ upsample_res = (upsample_res, upsample_res)
+
+ if str(device) == 'cpu':
+ amp_dtype = torch.float32
+
+ assert coarse_res[0] % 14 == 0, "Needs to be multiple of 14 for backbone"
+ assert coarse_res[1] % 14 == 0, "Needs to be multiple of 14 for backbone"
+
+ if weights is None:
+ weights = torch.hub.load_state_dict_from_url(weight_urls["romatch"]["outdoor"],
+ map_location=device)
+ if dinov2_weights is None:
+ dinov2_weights = torch.hub.load_state_dict_from_url(weight_urls["dinov2"],
+ map_location=device)
+ model = roma_model(resolution=coarse_res, upsample_preds=True,
+ weights=weights,dinov2_weights = dinov2_weights,device=device, amp_dtype=amp_dtype)
+ model.upsample_res = upsample_res
+ print(f"Using coarse resolution {coarse_res}, and upsample res {model.upsample_res}")
+ return model
+
+def roma_indoor(device, weights=None, dinov2_weights=None, coarse_res: Union[int,tuple[int,int]] = 560, upsample_res: Union[int,tuple[int,int]] = 864, amp_dtype: torch.dtype = torch.float16):
+ if isinstance(coarse_res, int):
+ coarse_res = (coarse_res, coarse_res)
+ if isinstance(upsample_res, int):
+ upsample_res = (upsample_res, upsample_res)
+
+ assert coarse_res[0] % 14 == 0, "Needs to be multiple of 14 for backbone"
+ assert coarse_res[1] % 14 == 0, "Needs to be multiple of 14 for backbone"
+
+ if weights is None:
+ weights = torch.hub.load_state_dict_from_url(weight_urls["romatch"]["indoor"],
+ map_location=device)
+ if dinov2_weights is None:
+ dinov2_weights = torch.hub.load_state_dict_from_url(weight_urls["dinov2"],
+ map_location=device)
+ model = roma_model(resolution=coarse_res, upsample_preds=True,
+ weights=weights,dinov2_weights = dinov2_weights,device=device, amp_dtype=amp_dtype)
+ model.upsample_res = upsample_res
+ print(f"Using coarse resolution {coarse_res}, and upsample res {model.upsample_res}")
+ return model
diff --git a/submodules/RoMa/romatch/models/model_zoo/roma_models.py b/submodules/RoMa/romatch/models/model_zoo/roma_models.py
new file mode 100644
index 0000000000000000000000000000000000000000..4f8a08fc26d760a09f048bdd98bf8a8fffc8202c
--- /dev/null
+++ b/submodules/RoMa/romatch/models/model_zoo/roma_models.py
@@ -0,0 +1,170 @@
+import warnings
+import torch.nn as nn
+import torch
+from romatch.models.matcher import *
+from romatch.models.transformer import Block, TransformerDecoder, MemEffAttention
+from romatch.models.encoders import *
+from romatch.models.tiny import TinyRoMa
+
+def tiny_roma_v1_model(weights = None, freeze_xfeat=False, exact_softmax=False, xfeat = None):
+ model = TinyRoMa(
+ xfeat = xfeat,
+ freeze_xfeat=freeze_xfeat,
+ exact_softmax=exact_softmax)
+ if weights is not None:
+ model.load_state_dict(weights)
+ return model
+
+def roma_model(resolution, upsample_preds, device = None, weights=None, dinov2_weights=None, amp_dtype: torch.dtype=torch.float16, **kwargs):
+ # romatch weights and dinov2 weights are loaded seperately, as dinov2 weights are not parameters
+ #torch.backends.cuda.matmul.allow_tf32 = True # allow tf32 on matmul TODO: these probably ruin stuff, should be careful
+ #torch.backends.cudnn.allow_tf32 = True # allow tf32 on cudnn
+ warnings.filterwarnings('ignore', category=UserWarning, message='TypedStorage is deprecated')
+ gp_dim = 512
+ feat_dim = 512
+ decoder_dim = gp_dim + feat_dim
+ cls_to_coord_res = 64
+ coordinate_decoder = TransformerDecoder(
+ nn.Sequential(*[Block(decoder_dim, 8, attn_class=MemEffAttention) for _ in range(5)]),
+ decoder_dim,
+ cls_to_coord_res**2 + 1,
+ is_classifier=True,
+ amp = True,
+ pos_enc = False,)
+ dw = True
+ hidden_blocks = 8
+ kernel_size = 5
+ displacement_emb = "linear"
+ disable_local_corr_grad = True
+
+ conv_refiner = nn.ModuleDict(
+ {
+ "16": ConvRefiner(
+ 2 * 512+128+(2*7+1)**2,
+ 2 * 512+128+(2*7+1)**2,
+ 2 + 1,
+ kernel_size=kernel_size,
+ dw=dw,
+ hidden_blocks=hidden_blocks,
+ displacement_emb=displacement_emb,
+ displacement_emb_dim=128,
+ local_corr_radius = 7,
+ corr_in_other = True,
+ amp = True,
+ disable_local_corr_grad = disable_local_corr_grad,
+ bn_momentum = 0.01,
+ ),
+ "8": ConvRefiner(
+ 2 * 512+64+(2*3+1)**2,
+ 2 * 512+64+(2*3+1)**2,
+ 2 + 1,
+ kernel_size=kernel_size,
+ dw=dw,
+ hidden_blocks=hidden_blocks,
+ displacement_emb=displacement_emb,
+ displacement_emb_dim=64,
+ local_corr_radius = 3,
+ corr_in_other = True,
+ amp = True,
+ disable_local_corr_grad = disable_local_corr_grad,
+ bn_momentum = 0.01,
+ ),
+ "4": ConvRefiner(
+ 2 * 256+32+(2*2+1)**2,
+ 2 * 256+32+(2*2+1)**2,
+ 2 + 1,
+ kernel_size=kernel_size,
+ dw=dw,
+ hidden_blocks=hidden_blocks,
+ displacement_emb=displacement_emb,
+ displacement_emb_dim=32,
+ local_corr_radius = 2,
+ corr_in_other = True,
+ amp = True,
+ disable_local_corr_grad = disable_local_corr_grad,
+ bn_momentum = 0.01,
+ ),
+ "2": ConvRefiner(
+ 2 * 64+16,
+ 128+16,
+ 2 + 1,
+ kernel_size=kernel_size,
+ dw=dw,
+ hidden_blocks=hidden_blocks,
+ displacement_emb=displacement_emb,
+ displacement_emb_dim=16,
+ amp = True,
+ disable_local_corr_grad = disable_local_corr_grad,
+ bn_momentum = 0.01,
+ ),
+ "1": ConvRefiner(
+ 2 * 9 + 6,
+ 24,
+ 2 + 1,
+ kernel_size=kernel_size,
+ dw=dw,
+ hidden_blocks = hidden_blocks,
+ displacement_emb = displacement_emb,
+ displacement_emb_dim = 6,
+ amp = True,
+ disable_local_corr_grad = disable_local_corr_grad,
+ bn_momentum = 0.01,
+ ),
+ }
+ )
+ kernel_temperature = 0.2
+ learn_temperature = False
+ no_cov = True
+ kernel = CosKernel
+ only_attention = False
+ basis = "fourier"
+ gp16 = GP(
+ kernel,
+ T=kernel_temperature,
+ learn_temperature=learn_temperature,
+ only_attention=only_attention,
+ gp_dim=gp_dim,
+ basis=basis,
+ no_cov=no_cov,
+ )
+ gps = nn.ModuleDict({"16": gp16})
+ proj16 = nn.Sequential(nn.Conv2d(1024, 512, 1, 1), nn.BatchNorm2d(512))
+ proj8 = nn.Sequential(nn.Conv2d(512, 512, 1, 1), nn.BatchNorm2d(512))
+ proj4 = nn.Sequential(nn.Conv2d(256, 256, 1, 1), nn.BatchNorm2d(256))
+ proj2 = nn.Sequential(nn.Conv2d(128, 64, 1, 1), nn.BatchNorm2d(64))
+ proj1 = nn.Sequential(nn.Conv2d(64, 9, 1, 1), nn.BatchNorm2d(9))
+ proj = nn.ModuleDict({
+ "16": proj16,
+ "8": proj8,
+ "4": proj4,
+ "2": proj2,
+ "1": proj1,
+ })
+ displacement_dropout_p = 0.0
+ gm_warp_dropout_p = 0.0
+ decoder = Decoder(coordinate_decoder,
+ gps,
+ proj,
+ conv_refiner,
+ detach=True,
+ scales=["16", "8", "4", "2", "1"],
+ displacement_dropout_p = displacement_dropout_p,
+ gm_warp_dropout_p = gm_warp_dropout_p)
+
+ encoder = CNNandDinov2(
+ cnn_kwargs = dict(
+ pretrained=False,
+ amp = True),
+ amp = True,
+ use_vgg = True,
+ dinov2_weights = dinov2_weights,
+ amp_dtype=amp_dtype,
+ )
+ h,w = resolution
+ symmetric = True
+ attenuate_cert = True
+ sample_mode = "threshold_balanced"
+ matcher = RegressionMatcher(encoder, decoder, h=h, w=w, upsample_preds=upsample_preds,
+ symmetric = symmetric, attenuate_cert = attenuate_cert, sample_mode = sample_mode, **kwargs).to(device)
+ matcher.load_state_dict(weights)
+ return matcher
diff --git a/submodules/RoMa/romatch/models/tiny.py b/submodules/RoMa/romatch/models/tiny.py
new file mode 100644
index 0000000000000000000000000000000000000000..88f44af6c9a6255831734d096167724f89d040ce
--- /dev/null
+++ b/submodules/RoMa/romatch/models/tiny.py
@@ -0,0 +1,304 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import os
+import torch
+from pathlib import Path
+import math
+import numpy as np
+
+from torch import nn
+from PIL import Image
+from torchvision.transforms import ToTensor
+from romatch.utils.kde import kde
+
+class BasicLayer(nn.Module):
+ """
+ Basic Convolutional Layer: Conv2d -> BatchNorm -> ReLU
+ """
+ def __init__(self, in_channels, out_channels, kernel_size=3, stride=1, padding=1, dilation=1, bias=False, relu = True):
+ super().__init__()
+ self.layer = nn.Sequential(
+ nn.Conv2d( in_channels, out_channels, kernel_size, padding = padding, stride=stride, dilation=dilation, bias = bias),
+ nn.BatchNorm2d(out_channels, affine=False),
+ nn.ReLU(inplace = True) if relu else nn.Identity()
+ )
+
+ def forward(self, x):
+ return self.layer(x)
+
+class TinyRoMa(nn.Module):
+ """
+ Implementation of architecture described in
+ "XFeat: Accelerated Features for Lightweight Image Matching, CVPR 2024."
+ """
+
+ def __init__(self, xfeat = None,
+ freeze_xfeat = True,
+ sample_mode = "threshold_balanced",
+ symmetric = False,
+ exact_softmax = False):
+ super().__init__()
+ del xfeat.heatmap_head, xfeat.keypoint_head, xfeat.fine_matcher
+ if freeze_xfeat:
+ xfeat.train(False)
+ self.xfeat = [xfeat]# hide params from ddp
+ else:
+ self.xfeat = nn.ModuleList([xfeat])
+ self.freeze_xfeat = freeze_xfeat
+ match_dim = 256
+ self.coarse_matcher = nn.Sequential(
+ BasicLayer(64+64+2, match_dim,),
+ BasicLayer(match_dim, match_dim,),
+ BasicLayer(match_dim, match_dim,),
+ BasicLayer(match_dim, match_dim,),
+ nn.Conv2d(match_dim, 3, kernel_size=1, bias=True, padding=0))
+ fine_match_dim = 64
+ self.fine_matcher = nn.Sequential(
+ BasicLayer(24+24+2, fine_match_dim,),
+ BasicLayer(fine_match_dim, fine_match_dim,),
+ BasicLayer(fine_match_dim, fine_match_dim,),
+ BasicLayer(fine_match_dim, fine_match_dim,),
+ nn.Conv2d(fine_match_dim, 3, kernel_size=1, bias=True, padding=0),)
+ self.sample_mode = sample_mode
+ self.sample_thresh = 0.05
+ self.symmetric = symmetric
+ self.exact_softmax = exact_softmax
+
+ @property
+ def device(self):
+ return self.fine_matcher[-1].weight.device
+
+ def preprocess_tensor(self, x):
+ """ Guarantee that image is divisible by 32 to avoid aliasing artifacts. """
+ H, W = x.shape[-2:]
+ _H, _W = (H//32) * 32, (W//32) * 32
+ rh, rw = H/_H, W/_W
+
+ x = F.interpolate(x, (_H, _W), mode='bilinear', align_corners=False)
+ return x, rh, rw
+
+ def forward_single(self, x):
+ with torch.inference_mode(self.freeze_xfeat or not self.training):
+ xfeat = self.xfeat[0]
+ with torch.no_grad():
+ x = x.mean(dim=1, keepdim = True)
+ x = xfeat.norm(x)
+
+ #main backbone
+ x1 = xfeat.block1(x)
+ x2 = xfeat.block2(x1 + xfeat.skip1(x))
+ x3 = xfeat.block3(x2)
+ x4 = xfeat.block4(x3)
+ x5 = xfeat.block5(x4)
+ x4 = F.interpolate(x4, (x3.shape[-2], x3.shape[-1]), mode='bilinear')
+ x5 = F.interpolate(x5, (x3.shape[-2], x3.shape[-1]), mode='bilinear')
+ feats = xfeat.block_fusion( x3 + x4 + x5 )
+ if self.freeze_xfeat:
+ return x2.clone(), feats.clone()
+ return x2, feats
+
+ def to_pixel_coordinates(self, coords, H_A, W_A, H_B = None, W_B = None):
+ if coords.shape[-1] == 2:
+ return self._to_pixel_coordinates(coords, H_A, W_A)
+
+ if isinstance(coords, (list, tuple)):
+ kpts_A, kpts_B = coords[0], coords[1]
+ else:
+ kpts_A, kpts_B = coords[...,:2], coords[...,2:]
+ return self._to_pixel_coordinates(kpts_A, H_A, W_A), self._to_pixel_coordinates(kpts_B, H_B, W_B)
+
+ def _to_pixel_coordinates(self, coords, H, W):
+ kpts = torch.stack((W/2 * (coords[...,0]+1), H/2 * (coords[...,1]+1)),axis=-1)
+ return kpts
+
+ def pos_embed(self, corr_volume: torch.Tensor):
+ B, H1, W1, H0, W0 = corr_volume.shape
+ grid = torch.stack(
+ torch.meshgrid(
+ torch.linspace(-1+1/W1,1-1/W1, W1),
+ torch.linspace(-1+1/H1,1-1/H1, H1),
+ indexing = "xy"),
+ dim = -1).float().to(corr_volume).reshape(H1*W1, 2)
+ down = 4
+ if not self.training and not self.exact_softmax:
+ grid_lr = torch.stack(
+ torch.meshgrid(
+ torch.linspace(-1+down/W1,1-down/W1, W1//down),
+ torch.linspace(-1+down/H1,1-down/H1, H1//down),
+ indexing = "xy"),
+ dim = -1).float().to(corr_volume).reshape(H1*W1 //down**2, 2)
+ cv = corr_volume
+ best_match = cv.reshape(B,H1*W1,H0,W0).argmax(dim=1) # B, HW, H, W
+ P_lowres = torch.cat((cv[:,::down,::down].reshape(B,H1*W1 // down**2,H0,W0), best_match[:,None]),dim=1).softmax(dim=1)
+ pos_embeddings = torch.einsum('bchw,cd->bdhw', P_lowres[:,:-1], grid_lr)
+ pos_embeddings += P_lowres[:,-1] * grid[best_match].permute(0,3,1,2)
+ #print("hej")
+ else:
+ P = corr_volume.reshape(B,H1*W1,H0,W0).softmax(dim=1) # B, HW, H, W
+ pos_embeddings = torch.einsum('bchw,cd->bdhw', P, grid)
+ return pos_embeddings
+
+ def visualize_warp(self, warp, certainty, im_A = None, im_B = None,
+ im_A_path = None, im_B_path = None, symmetric = True, save_path = None, unnormalize = False):
+ device = warp.device
+ H,W2,_ = warp.shape
+ W = W2//2 if symmetric else W2
+ if im_A is None:
+ from PIL import Image
+ im_A, im_B = Image.open(im_A_path).convert("RGB"), Image.open(im_B_path).convert("RGB")
+ if not isinstance(im_A, torch.Tensor):
+ im_A = im_A.resize((W,H))
+ im_B = im_B.resize((W,H))
+ x_B = (torch.tensor(np.array(im_B)) / 255).to(device).permute(2, 0, 1)
+ if symmetric:
+ x_A = (torch.tensor(np.array(im_A)) / 255).to(device).permute(2, 0, 1)
+ else:
+ if symmetric:
+ x_A = im_A
+ x_B = im_B
+ im_A_transfer_rgb = F.grid_sample(
+ x_B[None], warp[:,:W, 2:][None], mode="bilinear", align_corners=False
+ )[0]
+ if symmetric:
+ im_B_transfer_rgb = F.grid_sample(
+ x_A[None], warp[:, W:, :2][None], mode="bilinear", align_corners=False
+ )[0]
+ warp_im = torch.cat((im_A_transfer_rgb,im_B_transfer_rgb),dim=2)
+ white_im = torch.ones((H,2*W),device=device)
+ else:
+ warp_im = im_A_transfer_rgb
+ white_im = torch.ones((H, W), device = device)
+ vis_im = certainty * warp_im + (1 - certainty) * white_im
+ if save_path is not None:
+ from romatch.utils import tensor_to_pil
+ tensor_to_pil(vis_im, unnormalize=unnormalize).save(save_path)
+ return vis_im
+
+ def corr_volume(self, feat0, feat1):
+ """
+ input:
+ feat0 -> torch.Tensor(B, C, H, W)
+ feat1 -> torch.Tensor(B, C, H, W)
+ return:
+ corr_volume -> torch.Tensor(B, H, W, H, W)
+ """
+ B, C, H0, W0 = feat0.shape
+ B, C, H1, W1 = feat1.shape
+ feat0 = feat0.view(B, C, H0*W0)
+ feat1 = feat1.view(B, C, H1*W1)
+ corr_volume = torch.einsum('bci,bcj->bji', feat0, feat1).reshape(B, H1, W1, H0 , W0)/math.sqrt(C) #16*16*16
+ return corr_volume
+
+ @torch.inference_mode()
+ def match_from_path(self, im0_path, im1_path):
+ device = self.device
+ im0 = ToTensor()(Image.open(im0_path))[None].to(device)
+ im1 = ToTensor()(Image.open(im1_path))[None].to(device)
+ return self.match(im0, im1, batched = False)
+
+ @torch.inference_mode()
+ def match(self, im0, im1, *args, batched = True):
+ # stupid
+ if isinstance(im0, (str, Path)):
+ return self.match_from_path(im0, im1)
+ elif isinstance(im0, Image.Image):
+ batched = False
+ device = self.device
+ im0 = ToTensor()(im0)[None].to(device)
+ im1 = ToTensor()(im1)[None].to(device)
+
+ B,C,H0,W0 = im0.shape
+ B,C,H1,W1 = im1.shape
+ self.train(False)
+ corresps = self.forward({"im_A":im0, "im_B":im1})
+ #return 1,1
+ flow = F.interpolate(
+ corresps[4]["flow"],
+ size = (H0, W0),
+ mode = "bilinear", align_corners = False).permute(0,2,3,1).reshape(B,H0,W0,2)
+ grid = torch.stack(
+ torch.meshgrid(
+ torch.linspace(-1+1/W0,1-1/W0, W0),
+ torch.linspace(-1+1/H0,1-1/H0, H0),
+ indexing = "xy"),
+ dim = -1).float().to(flow.device).expand(B, H0, W0, 2)
+
+ certainty = F.interpolate(corresps[4]["certainty"], size = (H0,W0), mode = "bilinear", align_corners = False)
+ warp, cert = torch.cat((grid, flow), dim = -1), certainty[:,0].sigmoid()
+ if batched:
+ return warp, cert
+ else:
+ return warp[0], cert[0]
+
+ def sample(
+ self,
+ matches,
+ certainty,
+ num=5_000,
+ ):
+ H,W,_ = matches.shape
+ if "threshold" in self.sample_mode:
+ upper_thresh = self.sample_thresh
+ certainty = certainty.clone()
+ certainty[certainty > upper_thresh] = 1
+ matches, certainty = (
+ matches.reshape(-1, 4),
+ certainty.reshape(-1),
+ )
+ expansion_factor = 4 if "balanced" in self.sample_mode else 1
+ good_samples = torch.multinomial(certainty,
+ num_samples = min(expansion_factor*num, len(certainty)),
+ replacement=False)
+ good_matches, good_certainty = matches[good_samples], certainty[good_samples]
+ if "balanced" not in self.sample_mode:
+ return good_matches, good_certainty
+ use_half = True if matches.device.type == "cuda" else False
+ down = 1 if matches.device.type == "cuda" else 8
+ density = kde(good_matches, std=0.1, half = use_half, down = down)
+ p = 1 / (density+1)
+ p[density < 10] = 1e-7 # Basically should have at least 10 perfect neighbours, or around 100 ok ones
+ balanced_samples = torch.multinomial(p,
+ num_samples = min(num,len(good_certainty)),
+ replacement=False)
+ return good_matches[balanced_samples], good_certainty[balanced_samples]
+
+
+ def forward(self, batch):
+ """
+ input:
+ x -> torch.Tensor(B, C, H, W) grayscale or rgb images
+ return:
+
+ """
+ im0 = batch["im_A"]
+ im1 = batch["im_B"]
+ corresps = {}
+ im0, rh0, rw0 = self.preprocess_tensor(im0)
+ im1, rh1, rw1 = self.preprocess_tensor(im1)
+ B, C, H0, W0 = im0.shape
+ B, C, H1, W1 = im1.shape
+ to_normalized = torch.tensor((2/W1, 2/H1, 1)).to(im0.device)[None,:,None,None]
+
+ if im0.shape[-2:] == im1.shape[-2:]:
+ x = torch.cat([im0, im1], dim=0)
+ x = self.forward_single(x)
+ feats_x0_c, feats_x1_c = x[1].chunk(2)
+ feats_x0_f, feats_x1_f = x[0].chunk(2)
+ else:
+ feats_x0_f, feats_x0_c = self.forward_single(im0)
+ feats_x1_f, feats_x1_c = self.forward_single(im1)
+ corr_volume = self.corr_volume(feats_x0_c, feats_x1_c)
+ coarse_warp = self.pos_embed(corr_volume)
+ coarse_matches = torch.cat((coarse_warp, torch.zeros_like(coarse_warp[:,-1:])), dim=1)
+ feats_x1_c_warped = F.grid_sample(feats_x1_c, coarse_matches.permute(0, 2, 3, 1)[...,:2], mode = 'bilinear', align_corners = False)
+ coarse_matches_delta = self.coarse_matcher(torch.cat((feats_x0_c, feats_x1_c_warped, coarse_warp), dim=1))
+ coarse_matches = coarse_matches + coarse_matches_delta * to_normalized
+ corresps[8] = {"flow": coarse_matches[:,:2], "certainty": coarse_matches[:,2:]}
+ coarse_matches_up = F.interpolate(coarse_matches, size = feats_x0_f.shape[-2:], mode = "bilinear", align_corners = False)
+ coarse_matches_up_detach = coarse_matches_up.detach()#note the detach
+ feats_x1_f_warped = F.grid_sample(feats_x1_f, coarse_matches_up_detach.permute(0, 2, 3, 1)[...,:2], mode = 'bilinear', align_corners = False)
+ fine_matches_delta = self.fine_matcher(torch.cat((feats_x0_f, feats_x1_f_warped, coarse_matches_up_detach[:,:2]), dim=1))
+ fine_matches = coarse_matches_up_detach+fine_matches_delta * to_normalized
+ corresps[4] = {"flow": fine_matches[:,:2], "certainty": fine_matches[:,2:]}
+ return corresps
\ No newline at end of file
diff --git a/submodules/RoMa/romatch/models/transformer/__init__.py b/submodules/RoMa/romatch/models/transformer/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..983f03ccc51cdbcef6166a160fe50652a81418d7
--- /dev/null
+++ b/submodules/RoMa/romatch/models/transformer/__init__.py
@@ -0,0 +1,48 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from romatch.utils.utils import get_grid, get_autocast_params
+from .layers.block import Block
+from .layers.attention import MemEffAttention
+from .dinov2 import vit_large
+
+class TransformerDecoder(nn.Module):
+ def __init__(self, blocks, hidden_dim, out_dim, is_classifier = False, *args,
+ amp = False, pos_enc = True, learned_embeddings = False, embedding_dim = None, amp_dtype = torch.float16, **kwargs) -> None:
+ super().__init__(*args, **kwargs)
+ self.blocks = blocks
+ self.to_out = nn.Linear(hidden_dim, out_dim)
+ self.hidden_dim = hidden_dim
+ self.out_dim = out_dim
+ self._scales = [16]
+ self.is_classifier = is_classifier
+ self.amp = amp
+ self.amp_dtype = amp_dtype
+ self.pos_enc = pos_enc
+ self.learned_embeddings = learned_embeddings
+ if self.learned_embeddings:
+ self.learned_pos_embeddings = nn.Parameter(nn.init.kaiming_normal_(torch.empty((1, hidden_dim, embedding_dim, embedding_dim))))
+
+ def scales(self):
+ return self._scales.copy()
+
+ def forward(self, gp_posterior, features, old_stuff, new_scale):
+ autocast_device, autocast_enabled, autocast_dtype = get_autocast_params(gp_posterior.device, enabled=self.amp, dtype=self.amp_dtype)
+ with torch.autocast(autocast_device, enabled=autocast_enabled, dtype = autocast_dtype):
+ B,C,H,W = gp_posterior.shape
+ x = torch.cat((gp_posterior, features), dim = 1)
+ B,C,H,W = x.shape
+ grid = get_grid(B, H, W, x.device).reshape(B,H*W,2)
+ if self.learned_embeddings:
+ pos_enc = F.interpolate(self.learned_pos_embeddings, size = (H,W), mode = 'bilinear', align_corners = False).permute(0,2,3,1).reshape(1,H*W,C)
+ else:
+ pos_enc = 0
+ tokens = x.reshape(B,C,H*W).permute(0,2,1) + pos_enc
+ z = self.blocks(tokens)
+ out = self.to_out(z)
+ out = out.permute(0,2,1).reshape(B, self.out_dim, H, W)
+ warp, certainty = out[:, :-1], out[:, -1:]
+ return warp, certainty, None
+
+
diff --git a/submodules/RoMa/romatch/models/transformer/dinov2.py b/submodules/RoMa/romatch/models/transformer/dinov2.py
new file mode 100644
index 0000000000000000000000000000000000000000..b556c63096d17239c8603d5fe626c331963099fd
--- /dev/null
+++ b/submodules/RoMa/romatch/models/transformer/dinov2.py
@@ -0,0 +1,359 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+# References:
+# https://github.com/facebookresearch/dino/blob/main/vision_transformer.py
+# https://github.com/rwightman/pytorch-image-models/tree/master/timm/models/vision_transformer.py
+
+from functools import partial
+import math
+import logging
+from typing import Sequence, Tuple, Union, Callable
+
+import torch
+import torch.nn as nn
+import torch.utils.checkpoint
+from torch.nn.init import trunc_normal_
+
+from .layers import Mlp, PatchEmbed, SwiGLUFFNFused, MemEffAttention, NestedTensorBlock as Block
+
+
+
+def named_apply(fn: Callable, module: nn.Module, name="", depth_first=True, include_root=False) -> nn.Module:
+ if not depth_first and include_root:
+ fn(module=module, name=name)
+ for child_name, child_module in module.named_children():
+ child_name = ".".join((name, child_name)) if name else child_name
+ named_apply(fn=fn, module=child_module, name=child_name, depth_first=depth_first, include_root=True)
+ if depth_first and include_root:
+ fn(module=module, name=name)
+ return module
+
+
+class BlockChunk(nn.ModuleList):
+ def forward(self, x):
+ for b in self:
+ x = b(x)
+ return x
+
+
+class DinoVisionTransformer(nn.Module):
+ def __init__(
+ self,
+ img_size=224,
+ patch_size=16,
+ in_chans=3,
+ embed_dim=768,
+ depth=12,
+ num_heads=12,
+ mlp_ratio=4.0,
+ qkv_bias=True,
+ ffn_bias=True,
+ proj_bias=True,
+ drop_path_rate=0.0,
+ drop_path_uniform=False,
+ init_values=None, # for layerscale: None or 0 => no layerscale
+ embed_layer=PatchEmbed,
+ act_layer=nn.GELU,
+ block_fn=Block,
+ ffn_layer="mlp",
+ block_chunks=1,
+ ):
+ """
+ Args:
+ img_size (int, tuple): input image size
+ patch_size (int, tuple): patch size
+ in_chans (int): number of input channels
+ embed_dim (int): embedding dimension
+ depth (int): depth of transformer
+ num_heads (int): number of attention heads
+ mlp_ratio (int): ratio of mlp hidden dim to embedding dim
+ qkv_bias (bool): enable bias for qkv if True
+ proj_bias (bool): enable bias for proj in attn if True
+ ffn_bias (bool): enable bias for ffn if True
+ drop_path_rate (float): stochastic depth rate
+ drop_path_uniform (bool): apply uniform drop rate across blocks
+ weight_init (str): weight init scheme
+ init_values (float): layer-scale init values
+ embed_layer (nn.Module): patch embedding layer
+ act_layer (nn.Module): MLP activation layer
+ block_fn (nn.Module): transformer block class
+ ffn_layer (str): "mlp", "swiglu", "swiglufused" or "identity"
+ block_chunks: (int) split block sequence into block_chunks units for FSDP wrap
+ """
+ super().__init__()
+ norm_layer = partial(nn.LayerNorm, eps=1e-6)
+
+ self.num_features = self.embed_dim = embed_dim # num_features for consistency with other models
+ self.num_tokens = 1
+ self.n_blocks = depth
+ self.num_heads = num_heads
+ self.patch_size = patch_size
+
+ self.patch_embed = embed_layer(img_size=img_size, patch_size=patch_size, in_chans=in_chans, embed_dim=embed_dim)
+ num_patches = self.patch_embed.num_patches
+
+ self.cls_token = nn.Parameter(torch.zeros(1, 1, embed_dim))
+ self.pos_embed = nn.Parameter(torch.zeros(1, num_patches + self.num_tokens, embed_dim))
+
+ if drop_path_uniform is True:
+ dpr = [drop_path_rate] * depth
+ else:
+ dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)] # stochastic depth decay rule
+
+ if ffn_layer == "mlp":
+ ffn_layer = Mlp
+ elif ffn_layer == "swiglufused" or ffn_layer == "swiglu":
+ ffn_layer = SwiGLUFFNFused
+ elif ffn_layer == "identity":
+
+ def f(*args, **kwargs):
+ return nn.Identity()
+
+ ffn_layer = f
+ else:
+ raise NotImplementedError
+
+ blocks_list = [
+ block_fn(
+ dim=embed_dim,
+ num_heads=num_heads,
+ mlp_ratio=mlp_ratio,
+ qkv_bias=qkv_bias,
+ proj_bias=proj_bias,
+ ffn_bias=ffn_bias,
+ drop_path=dpr[i],
+ norm_layer=norm_layer,
+ act_layer=act_layer,
+ ffn_layer=ffn_layer,
+ init_values=init_values,
+ )
+ for i in range(depth)
+ ]
+ if block_chunks > 0:
+ self.chunked_blocks = True
+ chunked_blocks = []
+ chunksize = depth // block_chunks
+ for i in range(0, depth, chunksize):
+ # this is to keep the block index consistent if we chunk the block list
+ chunked_blocks.append([nn.Identity()] * i + blocks_list[i : i + chunksize])
+ self.blocks = nn.ModuleList([BlockChunk(p) for p in chunked_blocks])
+ else:
+ self.chunked_blocks = False
+ self.blocks = nn.ModuleList(blocks_list)
+
+ self.norm = norm_layer(embed_dim)
+ self.head = nn.Identity()
+
+ self.mask_token = nn.Parameter(torch.zeros(1, embed_dim))
+
+ self.init_weights()
+ for param in self.parameters():
+ param.requires_grad = False
+
+ @property
+ def device(self):
+ return self.cls_token.device
+
+ def init_weights(self):
+ trunc_normal_(self.pos_embed, std=0.02)
+ nn.init.normal_(self.cls_token, std=1e-6)
+ named_apply(init_weights_vit_timm, self)
+
+ def interpolate_pos_encoding(self, x, w, h):
+ previous_dtype = x.dtype
+ npatch = x.shape[1] - 1
+ N = self.pos_embed.shape[1] - 1
+ if npatch == N and w == h:
+ return self.pos_embed
+ pos_embed = self.pos_embed.float()
+ class_pos_embed = pos_embed[:, 0]
+ patch_pos_embed = pos_embed[:, 1:]
+ dim = x.shape[-1]
+ w0 = w // self.patch_size
+ h0 = h // self.patch_size
+ # we add a small number to avoid floating point error in the interpolation
+ # see discussion at https://github.com/facebookresearch/dino/issues/8
+ w0, h0 = w0 + 0.1, h0 + 0.1
+
+ patch_pos_embed = nn.functional.interpolate(
+ patch_pos_embed.reshape(1, int(math.sqrt(N)), int(math.sqrt(N)), dim).permute(0, 3, 1, 2),
+ scale_factor=(w0 / math.sqrt(N), h0 / math.sqrt(N)),
+ mode="bicubic",
+ )
+
+ assert int(w0) == patch_pos_embed.shape[-2] and int(h0) == patch_pos_embed.shape[-1]
+ patch_pos_embed = patch_pos_embed.permute(0, 2, 3, 1).view(1, -1, dim)
+ return torch.cat((class_pos_embed.unsqueeze(0), patch_pos_embed), dim=1).to(previous_dtype)
+
+ def prepare_tokens_with_masks(self, x, masks=None):
+ B, nc, w, h = x.shape
+ x = self.patch_embed(x)
+ if masks is not None:
+ x = torch.where(masks.unsqueeze(-1), self.mask_token.to(x.dtype).unsqueeze(0), x)
+
+ x = torch.cat((self.cls_token.expand(x.shape[0], -1, -1), x), dim=1)
+ x = x + self.interpolate_pos_encoding(x, w, h)
+
+ return x
+
+ def forward_features_list(self, x_list, masks_list):
+ x = [self.prepare_tokens_with_masks(x, masks) for x, masks in zip(x_list, masks_list)]
+ for blk in self.blocks:
+ x = blk(x)
+
+ all_x = x
+ output = []
+ for x, masks in zip(all_x, masks_list):
+ x_norm = self.norm(x)
+ output.append(
+ {
+ "x_norm_clstoken": x_norm[:, 0],
+ "x_norm_patchtokens": x_norm[:, 1:],
+ "x_prenorm": x,
+ "masks": masks,
+ }
+ )
+ return output
+
+ def forward_features(self, x, masks=None):
+ if isinstance(x, list):
+ return self.forward_features_list(x, masks)
+
+ x = self.prepare_tokens_with_masks(x, masks)
+
+ for blk in self.blocks:
+ x = blk(x)
+
+ x_norm = self.norm(x)
+ return {
+ "x_norm_clstoken": x_norm[:, 0],
+ "x_norm_patchtokens": x_norm[:, 1:],
+ "x_prenorm": x,
+ "masks": masks,
+ }
+
+ def _get_intermediate_layers_not_chunked(self, x, n=1):
+ x = self.prepare_tokens_with_masks(x)
+ # If n is an int, take the n last blocks. If it's a list, take them
+ output, total_block_len = [], len(self.blocks)
+ blocks_to_take = range(total_block_len - n, total_block_len) if isinstance(n, int) else n
+ for i, blk in enumerate(self.blocks):
+ x = blk(x)
+ if i in blocks_to_take:
+ output.append(x)
+ assert len(output) == len(blocks_to_take), f"only {len(output)} / {len(blocks_to_take)} blocks found"
+ return output
+
+ def _get_intermediate_layers_chunked(self, x, n=1):
+ x = self.prepare_tokens_with_masks(x)
+ output, i, total_block_len = [], 0, len(self.blocks[-1])
+ # If n is an int, take the n last blocks. If it's a list, take them
+ blocks_to_take = range(total_block_len - n, total_block_len) if isinstance(n, int) else n
+ for block_chunk in self.blocks:
+ for blk in block_chunk[i:]: # Passing the nn.Identity()
+ x = blk(x)
+ if i in blocks_to_take:
+ output.append(x)
+ i += 1
+ assert len(output) == len(blocks_to_take), f"only {len(output)} / {len(blocks_to_take)} blocks found"
+ return output
+
+ def get_intermediate_layers(
+ self,
+ x: torch.Tensor,
+ n: Union[int, Sequence] = 1, # Layers or n last layers to take
+ reshape: bool = False,
+ return_class_token: bool = False,
+ norm=True,
+ ) -> Tuple[Union[torch.Tensor, Tuple[torch.Tensor]]]:
+ if self.chunked_blocks:
+ outputs = self._get_intermediate_layers_chunked(x, n)
+ else:
+ outputs = self._get_intermediate_layers_not_chunked(x, n)
+ if norm:
+ outputs = [self.norm(out) for out in outputs]
+ class_tokens = [out[:, 0] for out in outputs]
+ outputs = [out[:, 1:] for out in outputs]
+ if reshape:
+ B, _, w, h = x.shape
+ outputs = [
+ out.reshape(B, w // self.patch_size, h // self.patch_size, -1).permute(0, 3, 1, 2).contiguous()
+ for out in outputs
+ ]
+ if return_class_token:
+ return tuple(zip(outputs, class_tokens))
+ return tuple(outputs)
+
+ def forward(self, *args, is_training=False, **kwargs):
+ ret = self.forward_features(*args, **kwargs)
+ if is_training:
+ return ret
+ else:
+ return self.head(ret["x_norm_clstoken"])
+
+
+def init_weights_vit_timm(module: nn.Module, name: str = ""):
+ """ViT weight initialization, original timm impl (for reproducibility)"""
+ if isinstance(module, nn.Linear):
+ trunc_normal_(module.weight, std=0.02)
+ if module.bias is not None:
+ nn.init.zeros_(module.bias)
+
+
+def vit_small(patch_size=16, **kwargs):
+ model = DinoVisionTransformer(
+ patch_size=patch_size,
+ embed_dim=384,
+ depth=12,
+ num_heads=6,
+ mlp_ratio=4,
+ block_fn=partial(Block, attn_class=MemEffAttention),
+ **kwargs,
+ )
+ return model
+
+
+def vit_base(patch_size=16, **kwargs):
+ model = DinoVisionTransformer(
+ patch_size=patch_size,
+ embed_dim=768,
+ depth=12,
+ num_heads=12,
+ mlp_ratio=4,
+ block_fn=partial(Block, attn_class=MemEffAttention),
+ **kwargs,
+ )
+ return model
+
+
+def vit_large(patch_size=16, **kwargs):
+ model = DinoVisionTransformer(
+ patch_size=patch_size,
+ embed_dim=1024,
+ depth=24,
+ num_heads=16,
+ mlp_ratio=4,
+ block_fn=partial(Block, attn_class=MemEffAttention),
+ **kwargs,
+ )
+ return model
+
+
+def vit_giant2(patch_size=16, **kwargs):
+ """
+ Close to ViT-giant, with embed-dim 1536 and 24 heads => embed-dim per head 64
+ """
+ model = DinoVisionTransformer(
+ patch_size=patch_size,
+ embed_dim=1536,
+ depth=40,
+ num_heads=24,
+ mlp_ratio=4,
+ block_fn=partial(Block, attn_class=MemEffAttention),
+ **kwargs,
+ )
+ return model
\ No newline at end of file
diff --git a/submodules/RoMa/romatch/models/transformer/layers/__init__.py b/submodules/RoMa/romatch/models/transformer/layers/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..31f196aacac5be8a7c537a3dfa8f97084671b466
--- /dev/null
+++ b/submodules/RoMa/romatch/models/transformer/layers/__init__.py
@@ -0,0 +1,12 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+from .dino_head import DINOHead
+from .mlp import Mlp
+from .patch_embed import PatchEmbed
+from .swiglu_ffn import SwiGLUFFN, SwiGLUFFNFused
+from .block import NestedTensorBlock
+from .attention import MemEffAttention
diff --git a/submodules/RoMa/romatch/models/transformer/layers/attention.py b/submodules/RoMa/romatch/models/transformer/layers/attention.py
new file mode 100644
index 0000000000000000000000000000000000000000..1f9b0c94b40967dfdff4f261c127cbd21328c905
--- /dev/null
+++ b/submodules/RoMa/romatch/models/transformer/layers/attention.py
@@ -0,0 +1,81 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+# References:
+# https://github.com/facebookresearch/dino/blob/master/vision_transformer.py
+# https://github.com/rwightman/pytorch-image-models/tree/master/timm/models/vision_transformer.py
+
+import logging
+
+from torch import Tensor
+from torch import nn
+
+
+logger = logging.getLogger("dinov2")
+
+
+try:
+ from xformers.ops import memory_efficient_attention, unbind, fmha
+
+ XFORMERS_AVAILABLE = True
+except ImportError:
+ logger.warning("xFormers not available")
+ XFORMERS_AVAILABLE = False
+
+
+class Attention(nn.Module):
+ def __init__(
+ self,
+ dim: int,
+ num_heads: int = 8,
+ qkv_bias: bool = False,
+ proj_bias: bool = True,
+ attn_drop: float = 0.0,
+ proj_drop: float = 0.0,
+ ) -> None:
+ super().__init__()
+ self.num_heads = num_heads
+ head_dim = dim // num_heads
+ self.scale = head_dim**-0.5
+
+ self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)
+ self.attn_drop = nn.Dropout(attn_drop)
+ self.proj = nn.Linear(dim, dim, bias=proj_bias)
+ self.proj_drop = nn.Dropout(proj_drop)
+
+ def forward(self, x: Tensor) -> Tensor:
+ B, N, C = x.shape
+ qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4)
+
+ q, k, v = qkv[0] * self.scale, qkv[1], qkv[2]
+ attn = q @ k.transpose(-2, -1)
+
+ attn = attn.softmax(dim=-1)
+ attn = self.attn_drop(attn)
+
+ x = (attn @ v).transpose(1, 2).reshape(B, N, C)
+ x = self.proj(x)
+ x = self.proj_drop(x)
+ return x
+
+
+class MemEffAttention(Attention):
+ def forward(self, x: Tensor, attn_bias=None) -> Tensor:
+ if not XFORMERS_AVAILABLE:
+ assert attn_bias is None, "xFormers is required for nested tensors usage"
+ return super().forward(x)
+
+ B, N, C = x.shape
+ qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads)
+
+ q, k, v = unbind(qkv, 2)
+
+ x = memory_efficient_attention(q, k, v, attn_bias=attn_bias)
+ x = x.reshape([B, N, C])
+
+ x = self.proj(x)
+ x = self.proj_drop(x)
+ return x
diff --git a/submodules/RoMa/romatch/models/transformer/layers/block.py b/submodules/RoMa/romatch/models/transformer/layers/block.py
new file mode 100644
index 0000000000000000000000000000000000000000..a711a1f2ee00c8a6b5e79504f41f13145450af79
--- /dev/null
+++ b/submodules/RoMa/romatch/models/transformer/layers/block.py
@@ -0,0 +1,252 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+# References:
+# https://github.com/facebookresearch/dino/blob/master/vision_transformer.py
+# https://github.com/rwightman/pytorch-image-models/tree/master/timm/layers/patch_embed.py
+
+import logging
+from typing import Callable, List, Any, Tuple, Dict
+
+import torch
+from torch import nn, Tensor
+
+from .attention import Attention, MemEffAttention
+from .drop_path import DropPath
+from .layer_scale import LayerScale
+from .mlp import Mlp
+
+
+logger = logging.getLogger("dinov2")
+
+
+try:
+ from xformers.ops import fmha
+ from xformers.ops import scaled_index_add, index_select_cat
+
+ XFORMERS_AVAILABLE = True
+except ImportError:
+ # logger.warning("xFormers not available")
+ XFORMERS_AVAILABLE = False
+
+
+class Block(nn.Module):
+ def __init__(
+ self,
+ dim: int,
+ num_heads: int,
+ mlp_ratio: float = 4.0,
+ qkv_bias: bool = False,
+ proj_bias: bool = True,
+ ffn_bias: bool = True,
+ drop: float = 0.0,
+ attn_drop: float = 0.0,
+ init_values=None,
+ drop_path: float = 0.0,
+ act_layer: Callable[..., nn.Module] = nn.GELU,
+ norm_layer: Callable[..., nn.Module] = nn.LayerNorm,
+ attn_class: Callable[..., nn.Module] = Attention,
+ ffn_layer: Callable[..., nn.Module] = Mlp,
+ ) -> None:
+ super().__init__()
+ # print(f"biases: qkv: {qkv_bias}, proj: {proj_bias}, ffn: {ffn_bias}")
+ self.norm1 = norm_layer(dim)
+ self.attn = attn_class(
+ dim,
+ num_heads=num_heads,
+ qkv_bias=qkv_bias,
+ proj_bias=proj_bias,
+ attn_drop=attn_drop,
+ proj_drop=drop,
+ )
+ self.ls1 = LayerScale(dim, init_values=init_values) if init_values else nn.Identity()
+ self.drop_path1 = DropPath(drop_path) if drop_path > 0.0 else nn.Identity()
+
+ self.norm2 = norm_layer(dim)
+ mlp_hidden_dim = int(dim * mlp_ratio)
+ self.mlp = ffn_layer(
+ in_features=dim,
+ hidden_features=mlp_hidden_dim,
+ act_layer=act_layer,
+ drop=drop,
+ bias=ffn_bias,
+ )
+ self.ls2 = LayerScale(dim, init_values=init_values) if init_values else nn.Identity()
+ self.drop_path2 = DropPath(drop_path) if drop_path > 0.0 else nn.Identity()
+
+ self.sample_drop_ratio = drop_path
+
+ def forward(self, x: Tensor) -> Tensor:
+ def attn_residual_func(x: Tensor) -> Tensor:
+ return self.ls1(self.attn(self.norm1(x)))
+
+ def ffn_residual_func(x: Tensor) -> Tensor:
+ return self.ls2(self.mlp(self.norm2(x)))
+
+ if self.training and self.sample_drop_ratio > 0.1:
+ # the overhead is compensated only for a drop path rate larger than 0.1
+ x = drop_add_residual_stochastic_depth(
+ x,
+ residual_func=attn_residual_func,
+ sample_drop_ratio=self.sample_drop_ratio,
+ )
+ x = drop_add_residual_stochastic_depth(
+ x,
+ residual_func=ffn_residual_func,
+ sample_drop_ratio=self.sample_drop_ratio,
+ )
+ elif self.training and self.sample_drop_ratio > 0.0:
+ x = x + self.drop_path1(attn_residual_func(x))
+ x = x + self.drop_path1(ffn_residual_func(x)) # FIXME: drop_path2
+ else:
+ x = x + attn_residual_func(x)
+ x = x + ffn_residual_func(x)
+ return x
+
+
+def drop_add_residual_stochastic_depth(
+ x: Tensor,
+ residual_func: Callable[[Tensor], Tensor],
+ sample_drop_ratio: float = 0.0,
+) -> Tensor:
+ # 1) extract subset using permutation
+ b, n, d = x.shape
+ sample_subset_size = max(int(b * (1 - sample_drop_ratio)), 1)
+ brange = (torch.randperm(b, device=x.device))[:sample_subset_size]
+ x_subset = x[brange]
+
+ # 2) apply residual_func to get residual
+ residual = residual_func(x_subset)
+
+ x_flat = x.flatten(1)
+ residual = residual.flatten(1)
+
+ residual_scale_factor = b / sample_subset_size
+
+ # 3) add the residual
+ x_plus_residual = torch.index_add(x_flat, 0, brange, residual.to(dtype=x.dtype), alpha=residual_scale_factor)
+ return x_plus_residual.view_as(x)
+
+
+def get_branges_scales(x, sample_drop_ratio=0.0):
+ b, n, d = x.shape
+ sample_subset_size = max(int(b * (1 - sample_drop_ratio)), 1)
+ brange = (torch.randperm(b, device=x.device))[:sample_subset_size]
+ residual_scale_factor = b / sample_subset_size
+ return brange, residual_scale_factor
+
+
+def add_residual(x, brange, residual, residual_scale_factor, scaling_vector=None):
+ if scaling_vector is None:
+ x_flat = x.flatten(1)
+ residual = residual.flatten(1)
+ x_plus_residual = torch.index_add(x_flat, 0, brange, residual.to(dtype=x.dtype), alpha=residual_scale_factor)
+ else:
+ x_plus_residual = scaled_index_add(
+ x, brange, residual.to(dtype=x.dtype), scaling=scaling_vector, alpha=residual_scale_factor
+ )
+ return x_plus_residual
+
+
+attn_bias_cache: Dict[Tuple, Any] = {}
+
+
+def get_attn_bias_and_cat(x_list, branges=None):
+ """
+ this will perform the index select, cat the tensors, and provide the attn_bias from cache
+ """
+ batch_sizes = [b.shape[0] for b in branges] if branges is not None else [x.shape[0] for x in x_list]
+ all_shapes = tuple((b, x.shape[1]) for b, x in zip(batch_sizes, x_list))
+ if all_shapes not in attn_bias_cache.keys():
+ seqlens = []
+ for b, x in zip(batch_sizes, x_list):
+ for _ in range(b):
+ seqlens.append(x.shape[1])
+ attn_bias = fmha.BlockDiagonalMask.from_seqlens(seqlens)
+ attn_bias._batch_sizes = batch_sizes
+ attn_bias_cache[all_shapes] = attn_bias
+
+ if branges is not None:
+ cat_tensors = index_select_cat([x.flatten(1) for x in x_list], branges).view(1, -1, x_list[0].shape[-1])
+ else:
+ tensors_bs1 = tuple(x.reshape([1, -1, *x.shape[2:]]) for x in x_list)
+ cat_tensors = torch.cat(tensors_bs1, dim=1)
+
+ return attn_bias_cache[all_shapes], cat_tensors
+
+
+def drop_add_residual_stochastic_depth_list(
+ x_list: List[Tensor],
+ residual_func: Callable[[Tensor, Any], Tensor],
+ sample_drop_ratio: float = 0.0,
+ scaling_vector=None,
+) -> Tensor:
+ # 1) generate random set of indices for dropping samples in the batch
+ branges_scales = [get_branges_scales(x, sample_drop_ratio=sample_drop_ratio) for x in x_list]
+ branges = [s[0] for s in branges_scales]
+ residual_scale_factors = [s[1] for s in branges_scales]
+
+ # 2) get attention bias and index+concat the tensors
+ attn_bias, x_cat = get_attn_bias_and_cat(x_list, branges)
+
+ # 3) apply residual_func to get residual, and split the result
+ residual_list = attn_bias.split(residual_func(x_cat, attn_bias=attn_bias)) # type: ignore
+
+ outputs = []
+ for x, brange, residual, residual_scale_factor in zip(x_list, branges, residual_list, residual_scale_factors):
+ outputs.append(add_residual(x, brange, residual, residual_scale_factor, scaling_vector).view_as(x))
+ return outputs
+
+
+class NestedTensorBlock(Block):
+ def forward_nested(self, x_list: List[Tensor]) -> List[Tensor]:
+ """
+ x_list contains a list of tensors to nest together and run
+ """
+ assert isinstance(self.attn, MemEffAttention)
+
+ if self.training and self.sample_drop_ratio > 0.0:
+
+ def attn_residual_func(x: Tensor, attn_bias=None) -> Tensor:
+ return self.attn(self.norm1(x), attn_bias=attn_bias)
+
+ def ffn_residual_func(x: Tensor, attn_bias=None) -> Tensor:
+ return self.mlp(self.norm2(x))
+
+ x_list = drop_add_residual_stochastic_depth_list(
+ x_list,
+ residual_func=attn_residual_func,
+ sample_drop_ratio=self.sample_drop_ratio,
+ scaling_vector=self.ls1.gamma if isinstance(self.ls1, LayerScale) else None,
+ )
+ x_list = drop_add_residual_stochastic_depth_list(
+ x_list,
+ residual_func=ffn_residual_func,
+ sample_drop_ratio=self.sample_drop_ratio,
+ scaling_vector=self.ls2.gamma if isinstance(self.ls1, LayerScale) else None,
+ )
+ return x_list
+ else:
+
+ def attn_residual_func(x: Tensor, attn_bias=None) -> Tensor:
+ return self.ls1(self.attn(self.norm1(x), attn_bias=attn_bias))
+
+ def ffn_residual_func(x: Tensor, attn_bias=None) -> Tensor:
+ return self.ls2(self.mlp(self.norm2(x)))
+
+ attn_bias, x = get_attn_bias_and_cat(x_list)
+ x = x + attn_residual_func(x, attn_bias=attn_bias)
+ x = x + ffn_residual_func(x)
+ return attn_bias.split(x)
+
+ def forward(self, x_or_x_list):
+ if isinstance(x_or_x_list, Tensor):
+ return super().forward(x_or_x_list)
+ elif isinstance(x_or_x_list, list):
+ assert XFORMERS_AVAILABLE, "Please install xFormers for nested tensors usage"
+ return self.forward_nested(x_or_x_list)
+ else:
+ raise AssertionError
diff --git a/submodules/RoMa/romatch/models/transformer/layers/dino_head.py b/submodules/RoMa/romatch/models/transformer/layers/dino_head.py
new file mode 100644
index 0000000000000000000000000000000000000000..7212db92a4fd8d4c7230e284e551a0234e9d8623
--- /dev/null
+++ b/submodules/RoMa/romatch/models/transformer/layers/dino_head.py
@@ -0,0 +1,59 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+import torch
+import torch.nn as nn
+from torch.nn.init import trunc_normal_
+from torch.nn.utils import weight_norm
+
+
+class DINOHead(nn.Module):
+ def __init__(
+ self,
+ in_dim,
+ out_dim,
+ use_bn=False,
+ nlayers=3,
+ hidden_dim=2048,
+ bottleneck_dim=256,
+ mlp_bias=True,
+ ):
+ super().__init__()
+ nlayers = max(nlayers, 1)
+ self.mlp = _build_mlp(nlayers, in_dim, bottleneck_dim, hidden_dim=hidden_dim, use_bn=use_bn, bias=mlp_bias)
+ self.apply(self._init_weights)
+ self.last_layer = weight_norm(nn.Linear(bottleneck_dim, out_dim, bias=False))
+ self.last_layer.weight_g.data.fill_(1)
+
+ def _init_weights(self, m):
+ if isinstance(m, nn.Linear):
+ trunc_normal_(m.weight, std=0.02)
+ if isinstance(m, nn.Linear) and m.bias is not None:
+ nn.init.constant_(m.bias, 0)
+
+ def forward(self, x):
+ x = self.mlp(x)
+ eps = 1e-6 if x.dtype == torch.float16 else 1e-12
+ x = nn.functional.normalize(x, dim=-1, p=2, eps=eps)
+ x = self.last_layer(x)
+ return x
+
+
+def _build_mlp(nlayers, in_dim, bottleneck_dim, hidden_dim=None, use_bn=False, bias=True):
+ if nlayers == 1:
+ return nn.Linear(in_dim, bottleneck_dim, bias=bias)
+ else:
+ layers = [nn.Linear(in_dim, hidden_dim, bias=bias)]
+ if use_bn:
+ layers.append(nn.BatchNorm1d(hidden_dim))
+ layers.append(nn.GELU())
+ for _ in range(nlayers - 2):
+ layers.append(nn.Linear(hidden_dim, hidden_dim, bias=bias))
+ if use_bn:
+ layers.append(nn.BatchNorm1d(hidden_dim))
+ layers.append(nn.GELU())
+ layers.append(nn.Linear(hidden_dim, bottleneck_dim, bias=bias))
+ return nn.Sequential(*layers)
diff --git a/submodules/RoMa/romatch/models/transformer/layers/drop_path.py b/submodules/RoMa/romatch/models/transformer/layers/drop_path.py
new file mode 100644
index 0000000000000000000000000000000000000000..af05625984dd14682cc96a63bf0c97bab1f123b1
--- /dev/null
+++ b/submodules/RoMa/romatch/models/transformer/layers/drop_path.py
@@ -0,0 +1,35 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+# References:
+# https://github.com/facebookresearch/dino/blob/master/vision_transformer.py
+# https://github.com/rwightman/pytorch-image-models/tree/master/timm/layers/drop.py
+
+
+from torch import nn
+
+
+def drop_path(x, drop_prob: float = 0.0, training: bool = False):
+ if drop_prob == 0.0 or not training:
+ return x
+ keep_prob = 1 - drop_prob
+ shape = (x.shape[0],) + (1,) * (x.ndim - 1) # work with diff dim tensors, not just 2D ConvNets
+ random_tensor = x.new_empty(shape).bernoulli_(keep_prob)
+ if keep_prob > 0.0:
+ random_tensor.div_(keep_prob)
+ output = x * random_tensor
+ return output
+
+
+class DropPath(nn.Module):
+ """Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks)."""
+
+ def __init__(self, drop_prob=None):
+ super(DropPath, self).__init__()
+ self.drop_prob = drop_prob
+
+ def forward(self, x):
+ return drop_path(x, self.drop_prob, self.training)
diff --git a/submodules/RoMa/romatch/models/transformer/layers/layer_scale.py b/submodules/RoMa/romatch/models/transformer/layers/layer_scale.py
new file mode 100644
index 0000000000000000000000000000000000000000..ca5daa52bd81d3581adeb2198ea5b7dba2a3aea1
--- /dev/null
+++ b/submodules/RoMa/romatch/models/transformer/layers/layer_scale.py
@@ -0,0 +1,28 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+# Modified from: https://github.com/huggingface/pytorch-image-models/blob/main/timm/models/vision_transformer.py#L103-L110
+
+from typing import Union
+
+import torch
+from torch import Tensor
+from torch import nn
+
+
+class LayerScale(nn.Module):
+ def __init__(
+ self,
+ dim: int,
+ init_values: Union[float, Tensor] = 1e-5,
+ inplace: bool = False,
+ ) -> None:
+ super().__init__()
+ self.inplace = inplace
+ self.gamma = nn.Parameter(init_values * torch.ones(dim))
+
+ def forward(self, x: Tensor) -> Tensor:
+ return x.mul_(self.gamma) if self.inplace else x * self.gamma
diff --git a/submodules/RoMa/romatch/models/transformer/layers/mlp.py b/submodules/RoMa/romatch/models/transformer/layers/mlp.py
new file mode 100644
index 0000000000000000000000000000000000000000..5e4b315f972f9a9f54aef1e4ef4e81b52976f018
--- /dev/null
+++ b/submodules/RoMa/romatch/models/transformer/layers/mlp.py
@@ -0,0 +1,41 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+# References:
+# https://github.com/facebookresearch/dino/blob/master/vision_transformer.py
+# https://github.com/rwightman/pytorch-image-models/tree/master/timm/layers/mlp.py
+
+
+from typing import Callable, Optional
+
+from torch import Tensor, nn
+
+
+class Mlp(nn.Module):
+ def __init__(
+ self,
+ in_features: int,
+ hidden_features: Optional[int] = None,
+ out_features: Optional[int] = None,
+ act_layer: Callable[..., nn.Module] = nn.GELU,
+ drop: float = 0.0,
+ bias: bool = True,
+ ) -> None:
+ super().__init__()
+ out_features = out_features or in_features
+ hidden_features = hidden_features or in_features
+ self.fc1 = nn.Linear(in_features, hidden_features, bias=bias)
+ self.act = act_layer()
+ self.fc2 = nn.Linear(hidden_features, out_features, bias=bias)
+ self.drop = nn.Dropout(drop)
+
+ def forward(self, x: Tensor) -> Tensor:
+ x = self.fc1(x)
+ x = self.act(x)
+ x = self.drop(x)
+ x = self.fc2(x)
+ x = self.drop(x)
+ return x
diff --git a/submodules/RoMa/romatch/models/transformer/layers/patch_embed.py b/submodules/RoMa/romatch/models/transformer/layers/patch_embed.py
new file mode 100644
index 0000000000000000000000000000000000000000..574abe41175568d700a389b8b96d1ba554914779
--- /dev/null
+++ b/submodules/RoMa/romatch/models/transformer/layers/patch_embed.py
@@ -0,0 +1,89 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+# References:
+# https://github.com/facebookresearch/dino/blob/master/vision_transformer.py
+# https://github.com/rwightman/pytorch-image-models/tree/master/timm/layers/patch_embed.py
+
+from typing import Callable, Optional, Tuple, Union
+
+from torch import Tensor
+import torch.nn as nn
+
+
+def make_2tuple(x):
+ if isinstance(x, tuple):
+ assert len(x) == 2
+ return x
+
+ assert isinstance(x, int)
+ return (x, x)
+
+
+class PatchEmbed(nn.Module):
+ """
+ 2D image to patch embedding: (B,C,H,W) -> (B,N,D)
+
+ Args:
+ img_size: Image size.
+ patch_size: Patch token size.
+ in_chans: Number of input image channels.
+ embed_dim: Number of linear projection output channels.
+ norm_layer: Normalization layer.
+ """
+
+ def __init__(
+ self,
+ img_size: Union[int, Tuple[int, int]] = 224,
+ patch_size: Union[int, Tuple[int, int]] = 16,
+ in_chans: int = 3,
+ embed_dim: int = 768,
+ norm_layer: Optional[Callable] = None,
+ flatten_embedding: bool = True,
+ ) -> None:
+ super().__init__()
+
+ image_HW = make_2tuple(img_size)
+ patch_HW = make_2tuple(patch_size)
+ patch_grid_size = (
+ image_HW[0] // patch_HW[0],
+ image_HW[1] // patch_HW[1],
+ )
+
+ self.img_size = image_HW
+ self.patch_size = patch_HW
+ self.patches_resolution = patch_grid_size
+ self.num_patches = patch_grid_size[0] * patch_grid_size[1]
+
+ self.in_chans = in_chans
+ self.embed_dim = embed_dim
+
+ self.flatten_embedding = flatten_embedding
+
+ self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_HW, stride=patch_HW)
+ self.norm = norm_layer(embed_dim) if norm_layer else nn.Identity()
+
+ def forward(self, x: Tensor) -> Tensor:
+ _, _, H, W = x.shape
+ patch_H, patch_W = self.patch_size
+
+ assert H % patch_H == 0, f"Input image height {H} is not a multiple of patch height {patch_H}"
+ assert W % patch_W == 0, f"Input image width {W} is not a multiple of patch width: {patch_W}"
+
+ x = self.proj(x) # B C H W
+ H, W = x.size(2), x.size(3)
+ x = x.flatten(2).transpose(1, 2) # B HW C
+ x = self.norm(x)
+ if not self.flatten_embedding:
+ x = x.reshape(-1, H, W, self.embed_dim) # B H W C
+ return x
+
+ def flops(self) -> float:
+ Ho, Wo = self.patches_resolution
+ flops = Ho * Wo * self.embed_dim * self.in_chans * (self.patch_size[0] * self.patch_size[1])
+ if self.norm is not None:
+ flops += Ho * Wo * self.embed_dim
+ return flops
diff --git a/submodules/RoMa/romatch/models/transformer/layers/swiglu_ffn.py b/submodules/RoMa/romatch/models/transformer/layers/swiglu_ffn.py
new file mode 100644
index 0000000000000000000000000000000000000000..b3324b266fb0a50ccf8c3a0ede2ae10ac4dfa03e
--- /dev/null
+++ b/submodules/RoMa/romatch/models/transformer/layers/swiglu_ffn.py
@@ -0,0 +1,63 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+from typing import Callable, Optional
+
+from torch import Tensor, nn
+import torch.nn.functional as F
+
+
+class SwiGLUFFN(nn.Module):
+ def __init__(
+ self,
+ in_features: int,
+ hidden_features: Optional[int] = None,
+ out_features: Optional[int] = None,
+ act_layer: Callable[..., nn.Module] = None,
+ drop: float = 0.0,
+ bias: bool = True,
+ ) -> None:
+ super().__init__()
+ out_features = out_features or in_features
+ hidden_features = hidden_features or in_features
+ self.w12 = nn.Linear(in_features, 2 * hidden_features, bias=bias)
+ self.w3 = nn.Linear(hidden_features, out_features, bias=bias)
+
+ def forward(self, x: Tensor) -> Tensor:
+ x12 = self.w12(x)
+ x1, x2 = x12.chunk(2, dim=-1)
+ hidden = F.silu(x1) * x2
+ return self.w3(hidden)
+
+
+try:
+ from xformers.ops import SwiGLU
+
+ XFORMERS_AVAILABLE = True
+except ImportError:
+ SwiGLU = SwiGLUFFN
+ XFORMERS_AVAILABLE = False
+
+
+class SwiGLUFFNFused(SwiGLU):
+ def __init__(
+ self,
+ in_features: int,
+ hidden_features: Optional[int] = None,
+ out_features: Optional[int] = None,
+ act_layer: Callable[..., nn.Module] = None,
+ drop: float = 0.0,
+ bias: bool = True,
+ ) -> None:
+ out_features = out_features or in_features
+ hidden_features = hidden_features or in_features
+ hidden_features = (int(hidden_features * 2 / 3) + 7) // 8 * 8
+ super().__init__(
+ in_features=in_features,
+ hidden_features=hidden_features,
+ out_features=out_features,
+ bias=bias,
+ )
diff --git a/submodules/RoMa/romatch/train/__init__.py b/submodules/RoMa/romatch/train/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..90269dc0f345a575e0ba21f5afa34202c7e6b433
--- /dev/null
+++ b/submodules/RoMa/romatch/train/__init__.py
@@ -0,0 +1 @@
+from .train import train_k_epochs
diff --git a/submodules/RoMa/romatch/train/train.py b/submodules/RoMa/romatch/train/train.py
new file mode 100644
index 0000000000000000000000000000000000000000..7cb02ed1e816fd39f174f76ec15bce49ae2a3da8
--- /dev/null
+++ b/submodules/RoMa/romatch/train/train.py
@@ -0,0 +1,102 @@
+from tqdm import tqdm
+from romatch.utils.utils import to_cuda
+import romatch
+import torch
+import wandb
+
+def log_param_statistics(named_parameters, norm_type = 2):
+ named_parameters = list(named_parameters)
+ grads = [p.grad for n, p in named_parameters if p.grad is not None]
+ weight_norms = [p.norm(p=norm_type) for n, p in named_parameters if p.grad is not None]
+ names = [n for n,p in named_parameters if p.grad is not None]
+ param_norm = torch.stack(weight_norms).norm(p=norm_type)
+ device = grads[0].device
+ grad_norms = torch.stack([torch.norm(g.detach(), norm_type).to(device) for g in grads])
+ nans_or_infs = torch.isinf(grad_norms) | torch.isnan(grad_norms)
+ nan_inf_names = [name for name, naninf in zip(names, nans_or_infs) if naninf]
+ total_grad_norm = torch.norm(grad_norms, norm_type)
+ if torch.any(nans_or_infs):
+ print(f"These params have nan or inf grads: {nan_inf_names}")
+ wandb.log({"grad_norm": total_grad_norm.item()}, step = romatch.GLOBAL_STEP)
+ wandb.log({"param_norm": param_norm.item()}, step = romatch.GLOBAL_STEP)
+
+def train_step(train_batch, model, objective, optimizer, grad_scaler, grad_clip_norm = 1.,**kwargs):
+ optimizer.zero_grad()
+ out = model(train_batch)
+ l = objective(out, train_batch)
+ grad_scaler.scale(l).backward()
+ grad_scaler.unscale_(optimizer)
+ log_param_statistics(model.named_parameters())
+ torch.nn.utils.clip_grad_norm_(model.parameters(), grad_clip_norm) # what should max norm be?
+ grad_scaler.step(optimizer)
+ grad_scaler.update()
+ wandb.log({"grad_scale": grad_scaler._scale.item()}, step = romatch.GLOBAL_STEP)
+ if grad_scaler._scale < 1.:
+ grad_scaler._scale = torch.tensor(1.).to(grad_scaler._scale)
+ romatch.GLOBAL_STEP = romatch.GLOBAL_STEP + romatch.STEP_SIZE # increment global step
+ return {"train_out": out, "train_loss": l.item()}
+
+
+def train_k_steps(
+ n_0, k, dataloader, model, objective, optimizer, lr_scheduler, grad_scaler, progress_bar=True, grad_clip_norm = 1., warmup = None, ema_model = None, pbar_n_seconds = 1,
+):
+ for n in tqdm(range(n_0, n_0 + k), disable=(not progress_bar) or romatch.RANK > 0, mininterval=pbar_n_seconds):
+ batch = next(dataloader)
+ model.train(True)
+ batch = to_cuda(batch)
+ train_step(
+ train_batch=batch,
+ model=model,
+ objective=objective,
+ optimizer=optimizer,
+ lr_scheduler=lr_scheduler,
+ grad_scaler=grad_scaler,
+ n=n,
+ grad_clip_norm = grad_clip_norm,
+ )
+ if ema_model is not None:
+ ema_model.update()
+ if warmup is not None:
+ with warmup.dampening():
+ lr_scheduler.step()
+ else:
+ lr_scheduler.step()
+ [wandb.log({f"lr_group_{grp}": lr}) for grp, lr in enumerate(lr_scheduler.get_last_lr())]
+
+
+def train_epoch(
+ dataloader=None,
+ model=None,
+ objective=None,
+ optimizer=None,
+ lr_scheduler=None,
+ epoch=None,
+):
+ model.train(True)
+ print(f"At epoch {epoch}")
+ for batch in tqdm(dataloader, mininterval=5.0):
+ batch = to_cuda(batch)
+ train_step(
+ train_batch=batch, model=model, objective=objective, optimizer=optimizer
+ )
+ lr_scheduler.step()
+ return {
+ "model": model,
+ "optimizer": optimizer,
+ "lr_scheduler": lr_scheduler,
+ "epoch": epoch,
+ }
+
+
+def train_k_epochs(
+ start_epoch, end_epoch, dataloader, model, objective, optimizer, lr_scheduler
+):
+ for epoch in range(start_epoch, end_epoch + 1):
+ train_epoch(
+ dataloader=dataloader,
+ model=model,
+ objective=objective,
+ optimizer=optimizer,
+ lr_scheduler=lr_scheduler,
+ epoch=epoch,
+ )
diff --git a/submodules/RoMa/romatch/utils/__init__.py b/submodules/RoMa/romatch/utils/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..2709f5e586150289085a4e2cbd458bc443fab7f3
--- /dev/null
+++ b/submodules/RoMa/romatch/utils/__init__.py
@@ -0,0 +1,16 @@
+from .utils import (
+ pose_auc,
+ get_pose,
+ compute_relative_pose,
+ compute_pose_error,
+ estimate_pose,
+ estimate_pose_uncalibrated,
+ rotate_intrinsic,
+ get_tuple_transform_ops,
+ get_depth_tuple_transform_ops,
+ warp_kpts,
+ numpy_to_pil,
+ tensor_to_pil,
+ recover_pose,
+ signed_left_to_right_epipolar_distance,
+)
diff --git a/submodules/RoMa/romatch/utils/kde.py b/submodules/RoMa/romatch/utils/kde.py
new file mode 100644
index 0000000000000000000000000000000000000000..46ed2e5e106bbca93e703f39f3ad3af350666e34
--- /dev/null
+++ b/submodules/RoMa/romatch/utils/kde.py
@@ -0,0 +1,13 @@
+import torch
+
+
+def kde(x, std = 0.1, half = True, down = None):
+ # use a gaussian kernel to estimate density
+ if half:
+ x = x.half() # Do it in half precision TODO: remove hardcoding
+ if down is not None:
+ scores = (-torch.cdist(x,x[::down])**2/(2*std**2)).exp()
+ else:
+ scores = (-torch.cdist(x,x)**2/(2*std**2)).exp()
+ density = scores.sum(dim=-1)
+ return density
\ No newline at end of file
diff --git a/submodules/RoMa/romatch/utils/local_correlation.py b/submodules/RoMa/romatch/utils/local_correlation.py
new file mode 100644
index 0000000000000000000000000000000000000000..fe1322a20bf82d0331159f958241cb87f75f4e21
--- /dev/null
+++ b/submodules/RoMa/romatch/utils/local_correlation.py
@@ -0,0 +1,48 @@
+import torch
+import torch.nn.functional as F
+
+def local_correlation(
+ feature0,
+ feature1,
+ local_radius,
+ padding_mode="zeros",
+ flow = None,
+ sample_mode = "bilinear",
+):
+ r = local_radius
+ K = (2*r+1)**2
+ B, c, h, w = feature0.size()
+ corr = torch.empty((B,K,h,w), device = feature0.device, dtype=feature0.dtype)
+ if flow is None:
+ # If flow is None, assume feature0 and feature1 are aligned
+ coords = torch.meshgrid(
+ (
+ torch.linspace(-1 + 1 / h, 1 - 1 / h, h, device=feature0.device),
+ torch.linspace(-1 + 1 / w, 1 - 1 / w, w, device=feature0.device),
+ ),
+ indexing = 'ij'
+ )
+ coords = torch.stack((coords[1], coords[0]), dim=-1)[
+ None
+ ].expand(B, h, w, 2)
+ else:
+ coords = flow.permute(0,2,3,1) # If using flow, sample around flow target.
+ local_window = torch.meshgrid(
+ (
+ torch.linspace(-2*local_radius/h, 2*local_radius/h, 2*r+1, device=feature0.device),
+ torch.linspace(-2*local_radius/w, 2*local_radius/w, 2*r+1, device=feature0.device),
+ ),
+ indexing = 'ij'
+ )
+ local_window = torch.stack((local_window[1], local_window[0]), dim=-1)[
+ None
+ ].expand(1, 2*r+1, 2*r+1, 2).reshape(1, (2*r+1)**2, 2)
+ for _ in range(B):
+ with torch.no_grad():
+ local_window_coords = (coords[_,:,:,None]+local_window[:,None,None]).reshape(1,h,w*(2*r+1)**2,2)
+ window_feature = F.grid_sample(
+ feature1[_:_+1], local_window_coords, padding_mode=padding_mode, align_corners=False, mode = sample_mode, #
+ )
+ window_feature = window_feature.reshape(c,h,w,(2*r+1)**2)
+ corr[_] = (feature0[_,...,None]/(c**.5)*window_feature).sum(dim=0).permute(2,0,1)
+ return corr
diff --git a/submodules/RoMa/romatch/utils/transforms.py b/submodules/RoMa/romatch/utils/transforms.py
new file mode 100644
index 0000000000000000000000000000000000000000..ea6476bd816a31df36f7d1b5417853637b65474b
--- /dev/null
+++ b/submodules/RoMa/romatch/utils/transforms.py
@@ -0,0 +1,118 @@
+from typing import Dict
+import numpy as np
+import torch
+import kornia.augmentation as K
+from kornia.geometry.transform import warp_perspective
+
+# Adapted from Kornia
+class GeometricSequential:
+ def __init__(self, *transforms, align_corners=True) -> None:
+ self.transforms = transforms
+ self.align_corners = align_corners
+
+ def __call__(self, x, mode="bilinear"):
+ b, c, h, w = x.shape
+ M = torch.eye(3, device=x.device)[None].expand(b, 3, 3)
+ for t in self.transforms:
+ if np.random.rand() < t.p:
+ M = M.matmul(
+ t.compute_transformation(x, t.generate_parameters((b, c, h, w)), None)
+ )
+ return (
+ warp_perspective(
+ x, M, dsize=(h, w), mode=mode, align_corners=self.align_corners
+ ),
+ M,
+ )
+
+ def apply_transform(self, x, M, mode="bilinear"):
+ b, c, h, w = x.shape
+ return warp_perspective(
+ x, M, dsize=(h, w), align_corners=self.align_corners, mode=mode
+ )
+
+
+class RandomPerspective(K.RandomPerspective):
+ def generate_parameters(self, batch_shape: torch.Size) -> Dict[str, torch.Tensor]:
+ distortion_scale = torch.as_tensor(
+ self.distortion_scale, device=self._device, dtype=self._dtype
+ )
+ return self.random_perspective_generator(
+ batch_shape[0],
+ batch_shape[-2],
+ batch_shape[-1],
+ distortion_scale,
+ self.same_on_batch,
+ self.device,
+ self.dtype,
+ )
+
+ def random_perspective_generator(
+ self,
+ batch_size: int,
+ height: int,
+ width: int,
+ distortion_scale: torch.Tensor,
+ same_on_batch: bool = False,
+ device: torch.device = torch.device("cpu"),
+ dtype: torch.dtype = torch.float32,
+ ) -> Dict[str, torch.Tensor]:
+ r"""Get parameters for ``perspective`` for a random perspective transform.
+
+ Args:
+ batch_size (int): the tensor batch size.
+ height (int) : height of the image.
+ width (int): width of the image.
+ distortion_scale (torch.Tensor): it controls the degree of distortion and ranges from 0 to 1.
+ same_on_batch (bool): apply the same transformation across the batch. Default: False.
+ device (torch.device): the device on which the random numbers will be generated. Default: cpu.
+ dtype (torch.dtype): the data type of the generated random numbers. Default: float32.
+
+ Returns:
+ params Dict[str, torch.Tensor]: parameters to be passed for transformation.
+ - start_points (torch.Tensor): element-wise perspective source areas with a shape of (B, 4, 2).
+ - end_points (torch.Tensor): element-wise perspective target areas with a shape of (B, 4, 2).
+
+ Note:
+ The generated random numbers are not reproducible across different devices and dtypes.
+ """
+ if not (distortion_scale.dim() == 0 and 0 <= distortion_scale <= 1):
+ raise AssertionError(
+ f"'distortion_scale' must be a scalar within [0, 1]. Got {distortion_scale}."
+ )
+ if not (
+ type(height) is int and height > 0 and type(width) is int and width > 0
+ ):
+ raise AssertionError(
+ f"'height' and 'width' must be integers. Got {height}, {width}."
+ )
+
+ start_points: torch.Tensor = torch.tensor(
+ [[[0.0, 0], [width - 1, 0], [width - 1, height - 1], [0, height - 1]]],
+ device=distortion_scale.device,
+ dtype=distortion_scale.dtype,
+ ).expand(batch_size, -1, -1)
+
+ # generate random offset not larger than half of the image
+ fx = distortion_scale * width / 2
+ fy = distortion_scale * height / 2
+
+ factor = torch.stack([fx, fy], dim=0).view(-1, 1, 2)
+ offset = (torch.rand_like(start_points) - 0.5) * 2
+ end_points = start_points + factor * offset
+
+ return dict(start_points=start_points, end_points=end_points)
+
+
+
+class RandomErasing:
+ def __init__(self, p = 0., scale = 0.) -> None:
+ self.p = p
+ self.scale = scale
+ self.random_eraser = K.RandomErasing(scale = (0.02, scale), p = p)
+ def __call__(self, image, depth):
+ if self.p > 0:
+ image = self.random_eraser(image)
+ depth = self.random_eraser(depth, params=self.random_eraser._params)
+ return image, depth
+
\ No newline at end of file
diff --git a/submodules/RoMa/romatch/utils/utils.py b/submodules/RoMa/romatch/utils/utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..1c522b16b7020779a0ea58b28973f2f609145838
--- /dev/null
+++ b/submodules/RoMa/romatch/utils/utils.py
@@ -0,0 +1,654 @@
+import warnings
+import numpy as np
+import cv2
+import math
+import torch
+from torchvision import transforms
+from torchvision.transforms.functional import InterpolationMode
+import torch.nn.functional as F
+from PIL import Image
+import kornia
+
+def recover_pose(E, kpts0, kpts1, K0, K1, mask):
+ best_num_inliers = 0
+ K0inv = np.linalg.inv(K0[:2,:2])
+ K1inv = np.linalg.inv(K1[:2,:2])
+
+ kpts0_n = (K0inv @ (kpts0-K0[None,:2,2]).T).T
+ kpts1_n = (K1inv @ (kpts1-K1[None,:2,2]).T).T
+
+ for _E in np.split(E, len(E) / 3):
+ n, R, t, _ = cv2.recoverPose(_E, kpts0_n, kpts1_n, np.eye(3), 1e9, mask=mask)
+ if n > best_num_inliers:
+ best_num_inliers = n
+ ret = (R, t, mask.ravel() > 0)
+ return ret
+
+
+
+# Code taken from https://github.com/PruneTruong/DenseMatching/blob/40c29a6b5c35e86b9509e65ab0cd12553d998e5f/validation/utils_pose_estimation.py
+# --- GEOMETRY ---
+def estimate_pose(kpts0, kpts1, K0, K1, norm_thresh, conf=0.99999):
+ if len(kpts0) < 5:
+ return None
+ K0inv = np.linalg.inv(K0[:2,:2])
+ K1inv = np.linalg.inv(K1[:2,:2])
+
+ kpts0 = (K0inv @ (kpts0-K0[None,:2,2]).T).T
+ kpts1 = (K1inv @ (kpts1-K1[None,:2,2]).T).T
+ E, mask = cv2.findEssentialMat(
+ kpts0, kpts1, np.eye(3), threshold=norm_thresh, prob=conf
+ )
+
+ ret = None
+ if E is not None:
+ best_num_inliers = 0
+
+ for _E in np.split(E, len(E) / 3):
+ n, R, t, _ = cv2.recoverPose(_E, kpts0, kpts1, np.eye(3), 1e9, mask=mask)
+ if n > best_num_inliers:
+ best_num_inliers = n
+ ret = (R, t, mask.ravel() > 0)
+ return ret
+
+def estimate_pose_uncalibrated(kpts0, kpts1, K0, K1, norm_thresh, conf=0.99999):
+ if len(kpts0) < 5:
+ return None
+ method = cv2.USAC_ACCURATE
+ F, mask = cv2.findFundamentalMat(
+ kpts0, kpts1, ransacReprojThreshold=norm_thresh, confidence=conf, method=method, maxIters=10000
+ )
+ E = K1.T@F@K0
+ ret = None
+ if E is not None:
+ best_num_inliers = 0
+ K0inv = np.linalg.inv(K0[:2,:2])
+ K1inv = np.linalg.inv(K1[:2,:2])
+
+ kpts0_n = (K0inv @ (kpts0-K0[None,:2,2]).T).T
+ kpts1_n = (K1inv @ (kpts1-K1[None,:2,2]).T).T
+
+ for _E in np.split(E, len(E) / 3):
+ n, R, t, _ = cv2.recoverPose(_E, kpts0_n, kpts1_n, np.eye(3), 1e9, mask=mask)
+ if n > best_num_inliers:
+ best_num_inliers = n
+ ret = (R, t, mask.ravel() > 0)
+ return ret
+
+def unnormalize_coords(x_n,h,w):
+ x = torch.stack(
+ (w * (x_n[..., 0] + 1) / 2, h * (x_n[..., 1] + 1) / 2), dim=-1
+ ) # [-1+1/h, 1-1/h] -> [0.5, h-0.5]
+ return x
+
+
+def rotate_intrinsic(K, n):
+ base_rot = np.array([[0, 1, 0], [-1, 0, 0], [0, 0, 1]])
+ rot = np.linalg.matrix_power(base_rot, n)
+ return rot @ K
+
+
+def rotate_pose_inplane(i_T_w, rot):
+ rotation_matrices = [
+ np.array(
+ [
+ [np.cos(r), -np.sin(r), 0.0, 0.0],
+ [np.sin(r), np.cos(r), 0.0, 0.0],
+ [0.0, 0.0, 1.0, 0.0],
+ [0.0, 0.0, 0.0, 1.0],
+ ],
+ dtype=np.float32,
+ )
+ for r in [np.deg2rad(d) for d in (0, 270, 180, 90)]
+ ]
+ return np.dot(rotation_matrices[rot], i_T_w)
+
+
+def scale_intrinsics(K, scales):
+ scales = np.diag([1.0 / scales[0], 1.0 / scales[1], 1.0])
+ return np.dot(scales, K)
+
+
+def to_homogeneous(points):
+ return np.concatenate([points, np.ones_like(points[:, :1])], axis=-1)
+
+
+def angle_error_mat(R1, R2):
+ cos = (np.trace(np.dot(R1.T, R2)) - 1) / 2
+ cos = np.clip(cos, -1.0, 1.0) # numercial errors can make it out of bounds
+ return np.rad2deg(np.abs(np.arccos(cos)))
+
+
+def angle_error_vec(v1, v2):
+ n = np.linalg.norm(v1) * np.linalg.norm(v2)
+ return np.rad2deg(np.arccos(np.clip(np.dot(v1, v2) / n, -1.0, 1.0)))
+
+
+def compute_pose_error(T_0to1, R, t):
+ R_gt = T_0to1[:3, :3]
+ t_gt = T_0to1[:3, 3]
+ error_t = angle_error_vec(t.squeeze(), t_gt)
+ error_t = np.minimum(error_t, 180 - error_t) # ambiguity of E estimation
+ error_R = angle_error_mat(R, R_gt)
+ return error_t, error_R
+
+
+def pose_auc(errors, thresholds):
+ sort_idx = np.argsort(errors)
+ errors = np.array(errors.copy())[sort_idx]
+ recall = (np.arange(len(errors)) + 1) / len(errors)
+ errors = np.r_[0.0, errors]
+ recall = np.r_[0.0, recall]
+ aucs = []
+ for t in thresholds:
+ last_index = np.searchsorted(errors, t)
+ r = np.r_[recall[:last_index], recall[last_index - 1]]
+ e = np.r_[errors[:last_index], t]
+ aucs.append(np.trapz(r, x=e) / t)
+ return aucs
+
+
+# From Patch2Pix https://github.com/GrumpyZhou/patch2pix
+def get_depth_tuple_transform_ops_nearest_exact(resize=None):
+ ops = []
+ if resize:
+ ops.append(TupleResizeNearestExact(resize))
+ return TupleCompose(ops)
+
+def get_depth_tuple_transform_ops(resize=None, normalize=True, unscale=False):
+ ops = []
+ if resize:
+ ops.append(TupleResize(resize, mode=InterpolationMode.BILINEAR))
+ return TupleCompose(ops)
+
+
+def get_tuple_transform_ops(resize=None, normalize=True, unscale=False, clahe = False, colorjiggle_params = None):
+ ops = []
+ if resize:
+ ops.append(TupleResize(resize))
+ ops.append(TupleToTensorScaled())
+ if normalize:
+ ops.append(
+ TupleNormalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
+ ) # Imagenet mean/std
+ return TupleCompose(ops)
+
+class ToTensorScaled(object):
+ """Convert a RGB PIL Image to a CHW ordered Tensor, scale the range to [0, 1]"""
+
+ def __call__(self, im):
+ if not isinstance(im, torch.Tensor):
+ im = np.array(im, dtype=np.float32).transpose((2, 0, 1))
+ im /= 255.0
+ return torch.from_numpy(im)
+ else:
+ return im
+
+ def __repr__(self):
+ return "ToTensorScaled(./255)"
+
+
+class TupleToTensorScaled(object):
+ def __init__(self):
+ self.to_tensor = ToTensorScaled()
+
+ def __call__(self, im_tuple):
+ return [self.to_tensor(im) for im in im_tuple]
+
+ def __repr__(self):
+ return "TupleToTensorScaled(./255)"
+
+
+class ToTensorUnscaled(object):
+ """Convert a RGB PIL Image to a CHW ordered Tensor"""
+
+ def __call__(self, im):
+ return torch.from_numpy(np.array(im, dtype=np.float32).transpose((2, 0, 1)))
+
+ def __repr__(self):
+ return "ToTensorUnscaled()"
+
+
+class TupleToTensorUnscaled(object):
+ """Convert a RGB PIL Image to a CHW ordered Tensor"""
+
+ def __init__(self):
+ self.to_tensor = ToTensorUnscaled()
+
+ def __call__(self, im_tuple):
+ return [self.to_tensor(im) for im in im_tuple]
+
+ def __repr__(self):
+ return "TupleToTensorUnscaled()"
+
+class TupleResizeNearestExact:
+ def __init__(self, size):
+ self.size = size
+ def __call__(self, im_tuple):
+ return [F.interpolate(im, size = self.size, mode = 'nearest-exact') for im in im_tuple]
+
+ def __repr__(self):
+ return "TupleResizeNearestExact(size={})".format(self.size)
+
+
+class TupleResize(object):
+ def __init__(self, size, mode=InterpolationMode.BICUBIC):
+ self.size = size
+ self.resize = transforms.Resize(size, mode)
+ def __call__(self, im_tuple):
+ return [self.resize(im) for im in im_tuple]
+
+ def __repr__(self):
+ return "TupleResize(size={})".format(self.size)
+
+class Normalize:
+ def __call__(self,im):
+ mean = im.mean(dim=(1,2), keepdims=True)
+ std = im.std(dim=(1,2), keepdims=True)
+ return (im-mean)/std
+
+
+class TupleNormalize(object):
+ def __init__(self, mean, std):
+ self.mean = mean
+ self.std = std
+ self.normalize = transforms.Normalize(mean=mean, std=std)
+
+ def __call__(self, im_tuple):
+ c,h,w = im_tuple[0].shape
+ if c > 3:
+ warnings.warn(f"Number of channels c={c} > 3, assuming first 3 are rgb")
+ return [self.normalize(im[:3]) for im in im_tuple]
+
+ def __repr__(self):
+ return "TupleNormalize(mean={}, std={})".format(self.mean, self.std)
+
+
+class TupleCompose(object):
+ def __init__(self, transforms):
+ self.transforms = transforms
+
+ def __call__(self, im_tuple):
+ for t in self.transforms:
+ im_tuple = t(im_tuple)
+ return im_tuple
+
+ def __repr__(self):
+ format_string = self.__class__.__name__ + "("
+ for t in self.transforms:
+ format_string += "\n"
+ format_string += " {0}".format(t)
+ format_string += "\n)"
+ return format_string
+
+@torch.no_grad()
+def cls_to_flow(cls, deterministic_sampling = True):
+ B,C,H,W = cls.shape
+ device = cls.device
+ res = round(math.sqrt(C))
+ G = torch.meshgrid(
+ *[torch.linspace(-1+1/res, 1-1/res, steps = res, device = device) for _ in range(2)],
+ indexing = 'ij'
+ )
+ G = torch.stack([G[1],G[0]],dim=-1).reshape(C,2)
+ if deterministic_sampling:
+ sampled_cls = cls.max(dim=1).indices
+ else:
+ sampled_cls = torch.multinomial(cls.permute(0,2,3,1).reshape(B*H*W,C).softmax(dim=-1), 1).reshape(B,H,W)
+ flow = G[sampled_cls]
+ return flow
+
+@torch.no_grad()
+def cls_to_flow_refine(cls):
+ B,C,H,W = cls.shape
+ device = cls.device
+ res = round(math.sqrt(C))
+ G = torch.meshgrid(
+ *[torch.linspace(-1+1/res, 1-1/res, steps = res, device = device) for _ in range(2)],
+ indexing = 'ij'
+ )
+ G = torch.stack([G[1],G[0]],dim=-1).reshape(C,2)
+ # FIXME: below softmax line causes mps to bug, don't know why.
+ if device.type == 'mps':
+ cls = cls.log_softmax(dim=1).exp()
+ else:
+ cls = cls.softmax(dim=1)
+ mode = cls.max(dim=1).indices
+
+ index = torch.stack((mode-1, mode, mode+1, mode - res, mode + res), dim = 1).clamp(0,C - 1).long()
+ neighbours = torch.gather(cls, dim = 1, index = index)[...,None]
+ flow = neighbours[:,0] * G[index[:,0]] + neighbours[:,1] * G[index[:,1]] + neighbours[:,2] * G[index[:,2]] + neighbours[:,3] * G[index[:,3]] + neighbours[:,4] * G[index[:,4]]
+ tot_prob = neighbours.sum(dim=1)
+ flow = flow / tot_prob
+ return flow
+
+
+def get_gt_warp(depth1, depth2, T_1to2, K1, K2, depth_interpolation_mode = 'bilinear', relative_depth_error_threshold = 0.05, H = None, W = None):
+
+ if H is None:
+ B,H,W = depth1.shape
+ else:
+ B = depth1.shape[0]
+ with torch.no_grad():
+ x1_n = torch.meshgrid(
+ *[
+ torch.linspace(
+ -1 + 1 / n, 1 - 1 / n, n, device=depth1.device
+ )
+ for n in (B, H, W)
+ ],
+ indexing = 'ij'
+ )
+ x1_n = torch.stack((x1_n[2], x1_n[1]), dim=-1).reshape(B, H * W, 2)
+ mask, x2 = warp_kpts(
+ x1_n.double(),
+ depth1.double(),
+ depth2.double(),
+ T_1to2.double(),
+ K1.double(),
+ K2.double(),
+ depth_interpolation_mode = depth_interpolation_mode,
+ relative_depth_error_threshold = relative_depth_error_threshold,
+ )
+ prob = mask.float().reshape(B, H, W)
+ x2 = x2.reshape(B, H, W, 2)
+ return x2, prob
+
+@torch.no_grad()
+def warp_kpts(kpts0, depth0, depth1, T_0to1, K0, K1, smooth_mask = False, return_relative_depth_error = False, depth_interpolation_mode = "bilinear", relative_depth_error_threshold = 0.05):
+ """Warp kpts0 from I0 to I1 with depth, K and Rt
+ Also check covisibility and depth consistency.
+ Depth is consistent if relative error < 0.2 (hard-coded).
+ # https://github.com/zju3dv/LoFTR/blob/94e98b695be18acb43d5d3250f52226a8e36f839/src/loftr/utils/geometry.py adapted from here
+ Args:
+ kpts0 (torch.Tensor): [N, L, 2] - , should be normalized in (-1,1)
+ depth0 (torch.Tensor): [N, H, W],
+ depth1 (torch.Tensor): [N, H, W],
+ T_0to1 (torch.Tensor): [N, 3, 4],
+ K0 (torch.Tensor): [N, 3, 3],
+ K1 (torch.Tensor): [N, 3, 3],
+ Returns:
+ calculable_mask (torch.Tensor): [N, L]
+ warped_keypoints0 (torch.Tensor): [N, L, 2]
+ """
+ (
+ n,
+ h,
+ w,
+ ) = depth0.shape
+ if depth_interpolation_mode == "combined":
+ # Inspired by approach in inloc, try to fill holes from bilinear interpolation by nearest neighbour interpolation
+ if smooth_mask:
+ raise NotImplementedError("Combined bilinear and NN warp not implemented")
+ valid_bilinear, warp_bilinear = warp_kpts(kpts0, depth0, depth1, T_0to1, K0, K1,
+ smooth_mask = smooth_mask,
+ return_relative_depth_error = return_relative_depth_error,
+ depth_interpolation_mode = "bilinear",
+ relative_depth_error_threshold = relative_depth_error_threshold)
+ valid_nearest, warp_nearest = warp_kpts(kpts0, depth0, depth1, T_0to1, K0, K1,
+ smooth_mask = smooth_mask,
+ return_relative_depth_error = return_relative_depth_error,
+ depth_interpolation_mode = "nearest-exact",
+ relative_depth_error_threshold = relative_depth_error_threshold)
+ nearest_valid_bilinear_invalid = (~valid_bilinear).logical_and(valid_nearest)
+ warp = warp_bilinear.clone()
+ warp[nearest_valid_bilinear_invalid] = warp_nearest[nearest_valid_bilinear_invalid]
+ valid = valid_bilinear | valid_nearest
+ return valid, warp
+
+
+ kpts0_depth = F.grid_sample(depth0[:, None], kpts0[:, :, None], mode = depth_interpolation_mode, align_corners=False)[
+ :, 0, :, 0
+ ]
+ kpts0 = torch.stack(
+ (w * (kpts0[..., 0] + 1) / 2, h * (kpts0[..., 1] + 1) / 2), dim=-1
+ ) # [-1+1/h, 1-1/h] -> [0.5, h-0.5]
+ # Sample depth, get calculable_mask on depth != 0
+ nonzero_mask = kpts0_depth != 0
+
+ # Unproject
+ kpts0_h = (
+ torch.cat([kpts0, torch.ones_like(kpts0[:, :, [0]])], dim=-1)
+ * kpts0_depth[..., None]
+ ) # (N, L, 3)
+ kpts0_n = K0.inverse() @ kpts0_h.transpose(2, 1) # (N, 3, L)
+ kpts0_cam = kpts0_n
+
+ # Rigid Transform
+ w_kpts0_cam = T_0to1[:, :3, :3] @ kpts0_cam + T_0to1[:, :3, [3]] # (N, 3, L)
+ w_kpts0_depth_computed = w_kpts0_cam[:, 2, :]
+
+ # Project
+ w_kpts0_h = (K1 @ w_kpts0_cam).transpose(2, 1) # (N, L, 3)
+ w_kpts0 = w_kpts0_h[:, :, :2] / (
+ w_kpts0_h[:, :, [2]] + 1e-4
+ ) # (N, L, 2), +1e-4 to avoid zero depth
+
+ # Covisible Check
+ h, w = depth1.shape[1:3]
+ covisible_mask = (
+ (w_kpts0[:, :, 0] > 0)
+ * (w_kpts0[:, :, 0] < w - 1)
+ * (w_kpts0[:, :, 1] > 0)
+ * (w_kpts0[:, :, 1] < h - 1)
+ )
+ w_kpts0 = torch.stack(
+ (2 * w_kpts0[..., 0] / w - 1, 2 * w_kpts0[..., 1] / h - 1), dim=-1
+ ) # from [0.5,h-0.5] -> [-1+1/h, 1-1/h]
+ # w_kpts0[~covisible_mask, :] = -5 # xd
+
+ w_kpts0_depth = F.grid_sample(
+ depth1[:, None], w_kpts0[:, :, None], mode=depth_interpolation_mode, align_corners=False
+ )[:, 0, :, 0]
+
+ relative_depth_error = (
+ (w_kpts0_depth - w_kpts0_depth_computed) / w_kpts0_depth
+ ).abs()
+ if not smooth_mask:
+ consistent_mask = relative_depth_error < relative_depth_error_threshold
+ else:
+ consistent_mask = (-relative_depth_error/smooth_mask).exp()
+ valid_mask = nonzero_mask * covisible_mask * consistent_mask
+ if return_relative_depth_error:
+ return relative_depth_error, w_kpts0
+ else:
+ return valid_mask, w_kpts0
+
+imagenet_mean = torch.tensor([0.485, 0.456, 0.406])
+imagenet_std = torch.tensor([0.229, 0.224, 0.225])
+
+
+def numpy_to_pil(x: np.ndarray):
+ """
+ Args:
+ x: Assumed to be of shape (h,w,c)
+ """
+ if isinstance(x, torch.Tensor):
+ x = x.detach().cpu().numpy()
+ if x.max() <= 1.01:
+ x *= 255
+ x = x.astype(np.uint8)
+ return Image.fromarray(x)
+
+
+def tensor_to_pil(x, unnormalize=False):
+ if unnormalize:
+ x = x * (imagenet_std[:, None, None].to(x.device)) + (imagenet_mean[:, None, None].to(x.device))
+ x = x.detach().permute(1, 2, 0).cpu().numpy()
+ x = np.clip(x, 0.0, 1.0)
+ return numpy_to_pil(x)
+
+
+def to_cuda(batch):
+ for key, value in batch.items():
+ if isinstance(value, torch.Tensor):
+ batch[key] = value.cuda()
+ return batch
+
+
+def to_cpu(batch):
+ for key, value in batch.items():
+ if isinstance(value, torch.Tensor):
+ batch[key] = value.cpu()
+ return batch
+
+
+def get_pose(calib):
+ w, h = np.array(calib["imsize"])[0]
+ return np.array(calib["K"]), np.array(calib["R"]), np.array(calib["T"]).T, h, w
+
+
+def compute_relative_pose(R1, t1, R2, t2):
+ rots = R2 @ (R1.T)
+ trans = -rots @ t1 + t2
+ return rots, trans
+
+@torch.no_grad()
+def reset_opt(opt):
+ for group in opt.param_groups:
+ for p in group['params']:
+ if p.requires_grad:
+ state = opt.state[p]
+ # State initialization
+
+ # Exponential moving average of gradient values
+ state['exp_avg'] = torch.zeros_like(p)
+ # Exponential moving average of squared gradient values
+ state['exp_avg_sq'] = torch.zeros_like(p)
+ # Exponential moving average of gradient difference
+ state['exp_avg_diff'] = torch.zeros_like(p)
+
+
+def flow_to_pixel_coords(flow, h1, w1):
+ flow = (
+ torch.stack(
+ (
+ w1 * (flow[..., 0] + 1) / 2,
+ h1 * (flow[..., 1] + 1) / 2,
+ ),
+ axis=-1,
+ )
+ )
+ return flow
+
+to_pixel_coords = flow_to_pixel_coords # just an alias
+
+def flow_to_normalized_coords(flow, h1, w1):
+ flow = (
+ torch.stack(
+ (
+ 2 * (flow[..., 0]) / w1 - 1,
+ 2 * (flow[..., 1]) / h1 - 1,
+ ),
+ axis=-1,
+ )
+ )
+ return flow
+
+to_normalized_coords = flow_to_normalized_coords # just an alias
+
+def warp_to_pixel_coords(warp, h1, w1, h2, w2):
+ warp1 = warp[..., :2]
+ warp1 = (
+ torch.stack(
+ (
+ w1 * (warp1[..., 0] + 1) / 2,
+ h1 * (warp1[..., 1] + 1) / 2,
+ ),
+ axis=-1,
+ )
+ )
+ warp2 = warp[..., 2:]
+ warp2 = (
+ torch.stack(
+ (
+ w2 * (warp2[..., 0] + 1) / 2,
+ h2 * (warp2[..., 1] + 1) / 2,
+ ),
+ axis=-1,
+ )
+ )
+ return torch.cat((warp1,warp2), dim=-1)
+
+
+
+def signed_point_line_distance(point, line, eps: float = 1e-9):
+ r"""Return the distance from points to lines.
+
+ Args:
+ point: (possibly homogeneous) points :math:`(*, N, 2 or 3)`.
+ line: lines coefficients :math:`(a, b, c)` with shape :math:`(*, N, 3)`, where :math:`ax + by + c = 0`.
+ eps: Small constant for safe sqrt.
+
+ Returns:
+ the computed distance with shape :math:`(*, N)`.
+ """
+
+ if not point.shape[-1] in (2, 3):
+ raise ValueError(f"pts must be a (*, 2 or 3) tensor. Got {point.shape}")
+
+ if not line.shape[-1] == 3:
+ raise ValueError(f"lines must be a (*, 3) tensor. Got {line.shape}")
+
+ numerator = (line[..., 0] * point[..., 0] + line[..., 1] * point[..., 1] + line[..., 2])
+ denominator = line[..., :2].norm(dim=-1)
+
+ return numerator / (denominator + eps)
+
+
+def signed_left_to_right_epipolar_distance(pts1, pts2, Fm):
+ r"""Return one-sided epipolar distance for correspondences given the fundamental matrix.
+
+ This method measures the distance from points in the right images to the epilines
+ of the corresponding points in the left images as they reflect in the right images.
+
+ Args:
+ pts1: correspondences from the left images with shape
+ :math:`(*, N, 2 or 3)`. If they are not homogeneous, converted automatically.
+ pts2: correspondences from the right images with shape
+ :math:`(*, N, 2 or 3)`. If they are not homogeneous, converted automatically.
+ Fm: Fundamental matrices with shape :math:`(*, 3, 3)`. Called Fm to
+ avoid ambiguity with torch.nn.functional.
+
+ Returns:
+ the computed Symmetrical distance with shape :math:`(*, N)`.
+ """
+ import kornia
+ if (len(Fm.shape) < 3) or not Fm.shape[-2:] == (3, 3):
+ raise ValueError(f"Fm must be a (*, 3, 3) tensor. Got {Fm.shape}")
+
+ if pts1.shape[-1] == 2:
+ pts1 = kornia.geometry.convert_points_to_homogeneous(pts1)
+
+ F_t = Fm.transpose(dim0=-2, dim1=-1)
+ line1_in_2 = pts1 @ F_t
+
+ return signed_point_line_distance(pts2, line1_in_2)
+
+def get_grid(b, h, w, device):
+ grid = torch.meshgrid(
+ *[
+ torch.linspace(-1 + 1 / n, 1 - 1 / n, n, device=device)
+ for n in (b, h, w)
+ ],
+ indexing = 'ij'
+ )
+ grid = torch.stack((grid[2], grid[1]), dim=-1).reshape(b, h, w, 2)
+ return grid
+
+
+def get_autocast_params(device=None, enabled=False, dtype=None):
+ if device is None:
+ autocast_device = "cuda" if torch.cuda.is_available() else "cpu"
+ else:
+ #strip :X from device
+ autocast_device = str(device).split(":")[0]
+ if 'cuda' in str(device):
+ out_dtype = dtype
+ enabled = True
+ else:
+ out_dtype = torch.bfloat16
+ enabled = False
+ # mps is not supported
+ autocast_device = "cpu"
+ return autocast_device, enabled, out_dtype
\ No newline at end of file
diff --git a/submodules/RoMa/setup.py b/submodules/RoMa/setup.py
new file mode 100644
index 0000000000000000000000000000000000000000..7ec18f3bbb71b85d943fdfeed3ed5c47033aebbc
--- /dev/null
+++ b/submodules/RoMa/setup.py
@@ -0,0 +1,9 @@
+from setuptools import setup, find_packages
+
+setup(
+ name="romatch",
+ packages=find_packages(include=("romatch*",)),
+ version="0.0.1",
+ author="Johan Edstedt",
+ install_requires=open("requirements.txt", "r").read().split("\n"),
+)
diff --git a/submodules/gaussian-splatting/.gitignore b/submodules/gaussian-splatting/.gitignore
new file mode 100644
index 0000000000000000000000000000000000000000..114376106621fc03ff8f923c03536e9dafd0d3f5
--- /dev/null
+++ b/submodules/gaussian-splatting/.gitignore
@@ -0,0 +1,8 @@
+*.pyc
+.vscode
+output
+build
+diff_rasterization/diff_rast.egg-info
+diff_rasterization/dist
+tensorboard_3d
+screenshots
\ No newline at end of file
diff --git a/submodules/gaussian-splatting/.gitmodules b/submodules/gaussian-splatting/.gitmodules
new file mode 100644
index 0000000000000000000000000000000000000000..cafb9acf7dd40829fa53e8991088d7cef617f2d5
--- /dev/null
+++ b/submodules/gaussian-splatting/.gitmodules
@@ -0,0 +1,13 @@
+[submodule "submodules/simple-knn"]
+ path = submodules/simple-knn
+ url = https://gitlab.inria.fr/bkerbl/simple-knn.git
+[submodule "submodules/diff-gaussian-rasterization"]
+ path = submodules/diff-gaussian-rasterization
+ url = https://github.com/graphdeco-inria/diff-gaussian-rasterization.git
+ branch = dr_aa
+[submodule "SIBR_viewers"]
+ path = SIBR_viewers
+ url = https://gitlab.inria.fr/sibr/sibr_core.git
+[submodule "submodules/fused-ssim"]
+ path = submodules/fused-ssim
+ url = https://github.com/rahul-goel/fused-ssim.git
diff --git a/submodules/gaussian-splatting/LICENSE.md b/submodules/gaussian-splatting/LICENSE.md
new file mode 100644
index 0000000000000000000000000000000000000000..18445c6d34aedbf1ab9d282223f8f10ce38cd79a
--- /dev/null
+++ b/submodules/gaussian-splatting/LICENSE.md
@@ -0,0 +1,91 @@
+Gaussian-Splatting License
+===========================
+
+**Inria** and **the Max Planck Institut for Informatik (MPII)** hold all the ownership rights on the *Software* named **gaussian-splatting**.
+The *Software* is in the process of being registered with the Agence pour la Protection des
+Programmes (APP).
+
+The *Software* is still being developed by the *Licensor*.
+
+*Licensor*'s goal is to allow the research community to use, test and evaluate
+the *Software*.
+
+## 1. Definitions
+
+*Licensee* means any person or entity that uses the *Software* and distributes
+its *Work*.
+
+*Licensor* means the owners of the *Software*, i.e Inria and MPII
+
+*Software* means the original work of authorship made available under this
+License ie gaussian-splatting.
+
+*Work* means the *Software* and any additions to or derivative works of the
+*Software* that are made available under this License.
+
+
+## 2. Purpose
+This license is intended to define the rights granted to the *Licensee* by
+Licensors under the *Software*.
+
+## 3. Rights granted
+
+For the above reasons Licensors have decided to distribute the *Software*.
+Licensors grant non-exclusive rights to use the *Software* for research purposes
+to research users (both academic and industrial), free of charge, without right
+to sublicense.. The *Software* may be used "non-commercially", i.e., for research
+and/or evaluation purposes only.
+
+Subject to the terms and conditions of this License, you are granted a
+non-exclusive, royalty-free, license to reproduce, prepare derivative works of,
+publicly display, publicly perform and distribute its *Work* and any resulting
+derivative works in any form.
+
+## 4. Limitations
+
+**4.1 Redistribution.** You may reproduce or distribute the *Work* only if (a) you do
+so under this License, (b) you include a complete copy of this License with
+your distribution, and (c) you retain without modification any copyright,
+patent, trademark, or attribution notices that are present in the *Work*.
+
+**4.2 Derivative Works.** You may specify that additional or different terms apply
+to the use, reproduction, and distribution of your derivative works of the *Work*
+("Your Terms") only if (a) Your Terms provide that the use limitation in
+Section 2 applies to your derivative works, and (b) you identify the specific
+derivative works that are subject to Your Terms. Notwithstanding Your Terms,
+this License (including the redistribution requirements in Section 3.1) will
+continue to apply to the *Work* itself.
+
+**4.3** Any other use without of prior consent of Licensors is prohibited. Research
+users explicitly acknowledge having received from Licensors all information
+allowing to appreciate the adequacy between of the *Software* and their needs and
+to undertake all necessary precautions for its execution and use.
+
+**4.4** The *Software* is provided both as a compiled library file and as source
+code. In case of using the *Software* for a publication or other results obtained
+through the use of the *Software*, users are strongly encouraged to cite the
+corresponding publications as explained in the documentation of the *Software*.
+
+## 5. Disclaimer
+
+THE USER CANNOT USE, EXPLOIT OR DISTRIBUTE THE *SOFTWARE* FOR COMMERCIAL PURPOSES
+WITHOUT PRIOR AND EXPLICIT CONSENT OF LICENSORS. YOU MUST CONTACT INRIA FOR ANY
+UNAUTHORIZED USE: stip-sophia.transfert@inria.fr . ANY SUCH ACTION WILL
+CONSTITUTE A FORGERY. THIS *SOFTWARE* IS PROVIDED "AS IS" WITHOUT ANY WARRANTIES
+OF ANY NATURE AND ANY EXPRESS OR IMPLIED WARRANTIES, WITH REGARDS TO COMMERCIAL
+USE, PROFESSIONNAL USE, LEGAL OR NOT, OR OTHER, OR COMMERCIALISATION OR
+ADAPTATION. UNLESS EXPLICITLY PROVIDED BY LAW, IN NO EVENT, SHALL INRIA OR THE
+AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
+GOODS OR SERVICES, LOSS OF USE, DATA, OR PROFITS OR BUSINESS INTERRUPTION)
+HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING FROM, OUT OF OR
+IN CONNECTION WITH THE *SOFTWARE* OR THE USE OR OTHER DEALINGS IN THE *SOFTWARE*.
+
+## 6. Files subject to permissive licenses
+The contents of the file ```utils/loss_utils.py``` are based on publicly available code authored by Evan Su, which falls under the permissive MIT license.
+
+Title: pytorch-ssim\
+Project code: https://github.com/Po-Hsun-Su/pytorch-ssim\
+Copyright Evan Su, 2017\
+License: https://github.com/Po-Hsun-Su/pytorch-ssim/blob/master/LICENSE.txt (MIT)
\ No newline at end of file
diff --git a/submodules/gaussian-splatting/README.md b/submodules/gaussian-splatting/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..eb2cf6c09d61425e22bf492f36e0922db9d8311a
--- /dev/null
+++ b/submodules/gaussian-splatting/README.md
@@ -0,0 +1,615 @@
+# 3D Gaussian Splatting for Real-Time Radiance Field Rendering
+Bernhard Kerbl*, Georgios Kopanas*, Thomas Leimkühler, George Drettakis (* indicates equal contribution)
+| [Webpage](https://repo-sam.inria.fr/fungraph/3d-gaussian-splatting/) | [Full Paper](https://repo-sam.inria.fr/fungraph/3d-gaussian-splatting/3d_gaussian_splatting_high.pdf) | [Video](https://youtu.be/T_kXY43VZnk) | [Other GRAPHDECO Publications](http://www-sop.inria.fr/reves/publis/gdindex.php) | [FUNGRAPH project page](https://fungraph.inria.fr) |
+| [T&T+DB COLMAP (650MB)](https://repo-sam.inria.fr/fungraph/3d-gaussian-splatting/datasets/input/tandt_db.zip) | [Pre-trained Models (14 GB)](https://repo-sam.inria.fr/fungraph/3d-gaussian-splatting/datasets/pretrained/models.zip) | [Viewers for Windows (60MB)](https://repo-sam.inria.fr/fungraph/3d-gaussian-splatting/binaries/viewers.zip) | [Evaluation Images (7 GB)](https://repo-sam.inria.fr/fungraph/3d-gaussian-splatting/evaluation/images.zip) |
+
+
+This repository contains the official authors implementation associated with the paper "3D Gaussian Splatting for Real-Time Radiance Field Rendering", which can be found [here](https://repo-sam.inria.fr/fungraph/3d-gaussian-splatting/). We further provide the reference images used to create the error metrics reported in the paper, as well as recently created, pre-trained models.
+
+
+
+
+
+
+Abstract: *Radiance Field methods have recently revolutionized novel-view synthesis of scenes captured with multiple photos or videos. However, achieving high visual quality still requires neural networks that are costly to train and render, while recent faster methods inevitably trade off speed for quality. For unbounded and complete scenes (rather than isolated objects) and 1080p resolution rendering, no current method can achieve real-time display rates. We introduce three key elements that allow us to achieve state-of-the-art visual quality while maintaining competitive training times and importantly allow high-quality real-time (≥ 30 fps) novel-view synthesis at 1080p resolution. First, starting from sparse points produced during camera calibration, we represent the scene with 3D Gaussians that preserve desirable properties of continuous volumetric radiance fields for scene optimization while avoiding unnecessary computation in empty space; Second, we perform interleaved optimization/density control of the 3D Gaussians, notably optimizing anisotropic covariance to achieve an accurate representation of the scene; Third, we develop a fast visibility-aware rendering algorithm that supports anisotropic splatting and both accelerates training and allows realtime rendering. We demonstrate state-of-the-art visual quality and real-time rendering on several established datasets.*
+
+
+
+
BibTeX
+
@Article{kerbl3Dgaussians,
+ author = {Kerbl, Bernhard and Kopanas, Georgios and Leimk{\"u}hler, Thomas and Drettakis, George},
+ title = {3D Gaussian Splatting for Real-Time Radiance Field Rendering},
+ journal = {ACM Transactions on Graphics},
+ number = {4},
+ volume = {42},
+ month = {July},
+ year = {2023},
+ url = {https://repo-sam.inria.fr/fungraph/3d-gaussian-splatting/}
+}
+
+
+
+
+
+## Funding and Acknowledgments
+
+This research was funded by the ERC Advanced grant FUNGRAPH No 788065. The authors are grateful to Adobe for generous donations, the OPAL infrastructure from Université Côte d’Azur and for the HPC resources from GENCI–IDRIS (Grant 2022-AD011013409). The authors thank the anonymous reviewers for their valuable feedback, P. Hedman and A. Tewari for proofreading earlier drafts also T. Müller, A. Yu and S. Fridovich-Keil for helping with the comparisons.
+
+## NEW FEATURES !
+
+We have limited resources for maintaining and updating the code. However, we have added a few new features since the original release that are inspired by some of the excellent work many other researchers have been doing on 3DGS. We will be adding other features within the ability of our resources.
+
+**Update of October 2024**: We integrated [training speed acceleration](#training-speed-acceleration) and made it compatible with [depth regularization](#depth-regularization), [anti-aliasing](#anti-aliasing) and [exposure compensation](#exposure-compensation). We have enhanced the SIBR real time viewer by correcting bugs and adding features in the [Top View](#sibr-top-view) that allows visualization of input and user cameras.
+
+**Update of Spring 2024**:
+Orange Labs has kindly added [OpenXR support](#openxr-support) for VR viewing.
+
+## Step-by-step Tutorial
+
+Jonathan Stephens made a fantastic step-by-step tutorial for setting up Gaussian Splatting on your machine, along with instructions for creating usable datasets from videos. If the instructions below are too dry for you, go ahead and check it out [here](https://www.youtube.com/watch?v=UXtuigy_wYc).
+
+## Colab
+
+User [camenduru](https://github.com/camenduru) was kind enough to provide a Colab template that uses this repo's source (status: August 2023!) for quick and easy access to the method. Please check it out [here](https://github.com/camenduru/gaussian-splatting-colab).
+
+## Cloning the Repository
+
+The repository contains submodules, thus please check it out with
+```shell
+# SSH
+git clone git@github.com:graphdeco-inria/gaussian-splatting.git --recursive
+```
+or
+```shell
+# HTTPS
+git clone https://github.com/graphdeco-inria/gaussian-splatting --recursive
+```
+
+## Overview
+
+The codebase has 4 main components:
+- A PyTorch-based optimizer to produce a 3D Gaussian model from SfM inputs
+- A network viewer that allows to connect to and visualize the optimization process
+- An OpenGL-based real-time viewer to render trained models in real-time.
+- A script to help you turn your own images into optimization-ready SfM data sets
+
+The components have different requirements w.r.t. both hardware and software. They have been tested on Windows 10 and Ubuntu Linux 22.04. Instructions for setting up and running each of them are found in the sections below.
+
+
+
+
+## Optimizer
+
+The optimizer uses PyTorch and CUDA extensions in a Python environment to produce trained models.
+
+### Hardware Requirements
+
+- CUDA-ready GPU with Compute Capability 7.0+
+- 24 GB VRAM (to train to paper evaluation quality)
+- Please see FAQ for smaller VRAM configurations
+
+### Software Requirements
+- Conda (recommended for easy setup)
+- C++ Compiler for PyTorch extensions (we used Visual Studio 2019 for Windows)
+- CUDA SDK 11 for PyTorch extensions, install *after* Visual Studio (we used 11.8, **known issues with 11.6**)
+- C++ Compiler and CUDA SDK must be compatible
+
+### Setup
+
+#### Local Setup
+
+Our default, provided install method is based on Conda package and environment management:
+```shell
+SET DISTUTILS_USE_SDK=1 # Windows only
+conda env create --file environment.yml
+conda activate gaussian_splatting
+```
+Please note that this process assumes that you have CUDA SDK **11** installed, not **12**. For modifications, see below.
+
+Tip: Downloading packages and creating a new environment with Conda can require a significant amount of disk space. By default, Conda will use the main system hard drive. You can avoid this by specifying a different package download location and an environment on a different drive:
+
+```shell
+conda config --add pkgs_dirs /
+conda env create --file environment.yml --prefix //gaussian_splatting
+conda activate //gaussian_splatting
+```
+
+#### Modifications
+
+If you can afford the disk space, we recommend using our environment files for setting up a training environment identical to ours. If you want to make modifications, please note that major version changes might affect the results of our method. However, our (limited) experiments suggest that the codebase works just fine inside a more up-to-date environment (Python 3.8, PyTorch 2.0.0, CUDA 12). Make sure to create an environment where PyTorch and its CUDA runtime version match and the installed CUDA SDK has no major version difference with PyTorch's CUDA version.
+
+#### Known Issues
+
+Some users experience problems building the submodules on Windows (```cl.exe: File not found``` or similar). Please consider the workaround for this problem from the FAQ.
+
+### Running
+
+To run the optimizer, simply use
+
+```shell
+python train.py -s
+```
+
+
+Command Line Arguments for train.py
+
+ #### --source_path / -s
+ Path to the source directory containing a COLMAP or Synthetic NeRF data set.
+ #### --model_path / -m
+ Path where the trained model should be stored (```output/``` by default).
+ #### --images / -i
+ Alternative subdirectory for COLMAP images (```images``` by default).
+ #### --eval
+ Add this flag to use a MipNeRF360-style training/test split for evaluation.
+ #### --resolution / -r
+ Specifies resolution of the loaded images before training. If provided ```1, 2, 4``` or ```8```, uses original, 1/2, 1/4 or 1/8 resolution, respectively. For all other values, rescales the width to the given number while maintaining image aspect. **If not set and input image width exceeds 1.6K pixels, inputs are automatically rescaled to this target.**
+ #### --data_device
+ Specifies where to put the source image data, ```cuda``` by default, recommended to use ```cpu``` if training on large/high-resolution dataset, will reduce VRAM consumption, but slightly slow down training. Thanks to [HrsPythonix](https://github.com/HrsPythonix).
+ #### --white_background / -w
+ Add this flag to use white background instead of black (default), e.g., for evaluation of NeRF Synthetic dataset.
+ #### --sh_degree
+ Order of spherical harmonics to be used (no larger than 3). ```3``` by default.
+ #### --convert_SHs_python
+ Flag to make pipeline compute forward and backward of SHs with PyTorch instead of ours.
+ #### --convert_cov3D_python
+ Flag to make pipeline compute forward and backward of the 3D covariance with PyTorch instead of ours.
+ #### --debug
+ Enables debug mode if you experience erros. If the rasterizer fails, a ```dump``` file is created that you may forward to us in an issue so we can take a look.
+ #### --debug_from
+ Debugging is **slow**. You may specify an iteration (starting from 0) after which the above debugging becomes active.
+ #### --iterations
+ Number of total iterations to train for, ```30_000``` by default.
+ #### --ip
+ IP to start GUI server on, ```127.0.0.1``` by default.
+ #### --port
+ Port to use for GUI server, ```6009``` by default.
+ #### --test_iterations
+ Space-separated iterations at which the training script computes L1 and PSNR over test set, ```7000 30000``` by default.
+ #### --save_iterations
+ Space-separated iterations at which the training script saves the Gaussian model, ```7000 30000 ``` by default.
+ #### --checkpoint_iterations
+ Space-separated iterations at which to store a checkpoint for continuing later, saved in the model directory.
+ #### --start_checkpoint
+ Path to a saved checkpoint to continue training from.
+ #### --quiet
+ Flag to omit any text written to standard out pipe.
+ #### --feature_lr
+ Spherical harmonics features learning rate, ```0.0025``` by default.
+ #### --opacity_lr
+ Opacity learning rate, ```0.05``` by default.
+ #### --scaling_lr
+ Scaling learning rate, ```0.005``` by default.
+ #### --rotation_lr
+ Rotation learning rate, ```0.001``` by default.
+ #### --position_lr_max_steps
+ Number of steps (from 0) where position learning rate goes from ```initial``` to ```final```. ```30_000``` by default.
+ #### --position_lr_init
+ Initial 3D position learning rate, ```0.00016``` by default.
+ #### --position_lr_final
+ Final 3D position learning rate, ```0.0000016``` by default.
+ #### --position_lr_delay_mult
+ Position learning rate multiplier (cf. Plenoxels), ```0.01``` by default.
+ #### --densify_from_iter
+ Iteration where densification starts, ```500``` by default.
+ #### --densify_until_iter
+ Iteration where densification stops, ```15_000``` by default.
+ #### --densify_grad_threshold
+ Limit that decides if points should be densified based on 2D position gradient, ```0.0002``` by default.
+ #### --densification_interval
+ How frequently to densify, ```100``` (every 100 iterations) by default.
+ #### --opacity_reset_interval
+ How frequently to reset opacity, ```3_000``` by default.
+ #### --lambda_dssim
+ Influence of SSIM on total loss from 0 to 1, ```0.2``` by default.
+ #### --percent_dense
+ Percentage of scene extent (0--1) a point must exceed to be forcibly densified, ```0.01``` by default.
+
+
+
+
+Note that similar to MipNeRF360, we target images at resolutions in the 1-1.6K pixel range. For convenience, arbitrary-size inputs can be passed and will be automatically resized if their width exceeds 1600 pixels. We recommend to keep this behavior, but you may force training to use your higher-resolution images by setting ```-r 1```.
+
+The MipNeRF360 scenes are hosted by the paper authors [here](https://jonbarron.info/mipnerf360/). You can find our SfM data sets for Tanks&Temples and Deep Blending [here](https://repo-sam.inria.fr/fungraph/3d-gaussian-splatting/datasets/input/tandt_db.zip). If you do not provide an output model directory (```-m```), trained models are written to folders with randomized unique names inside the ```output``` directory. At this point, the trained models may be viewed with the real-time viewer (see further below).
+
+### Evaluation
+By default, the trained models use all available images in the dataset. To train them while withholding a test set for evaluation, use the ```--eval``` flag. This way, you can render training/test sets and produce error metrics as follows:
+```shell
+python train.py -s --eval # Train with train/test split
+python render.py -m # Generate renderings
+python metrics.py -m # Compute error metrics on renderings
+```
+
+If you want to evaluate our [pre-trained models](https://repo-sam.inria.fr/fungraph/3d-gaussian-splatting/datasets/pretrained/models.zip), you will have to download the corresponding source data sets and indicate their location to ```render.py``` with an additional ```--source_path/-s``` flag. Note: The pre-trained models were created with the release codebase. This code base has been cleaned up and includes bugfixes, hence the metrics you get from evaluating them will differ from those in the paper.
+```shell
+python render.py -m -s
+python metrics.py -m
+```
+
+
+Command Line Arguments for render.py
+
+ #### --model_path / -m
+ Path to the trained model directory you want to create renderings for.
+ #### --skip_train
+ Flag to skip rendering the training set.
+ #### --skip_test
+ Flag to skip rendering the test set.
+ #### --quiet
+ Flag to omit any text written to standard out pipe.
+
+ **The below parameters will be read automatically from the model path, based on what was used for training. However, you may override them by providing them explicitly on the command line.**
+
+ #### --source_path / -s
+ Path to the source directory containing a COLMAP or Synthetic NeRF data set.
+ #### --images / -i
+ Alternative subdirectory for COLMAP images (```images``` by default).
+ #### --eval
+ Add this flag to use a MipNeRF360-style training/test split for evaluation.
+ #### --resolution / -r
+ Changes the resolution of the loaded images before training. If provided ```1, 2, 4``` or ```8```, uses original, 1/2, 1/4 or 1/8 resolution, respectively. For all other values, rescales the width to the given number while maintaining image aspect. ```1``` by default.
+ #### --white_background / -w
+ Add this flag to use white background instead of black (default), e.g., for evaluation of NeRF Synthetic dataset.
+ #### --convert_SHs_python
+ Flag to make pipeline render with computed SHs from PyTorch instead of ours.
+ #### --convert_cov3D_python
+ Flag to make pipeline render with computed 3D covariance from PyTorch instead of ours.
+
+
+
+
+Command Line Arguments for metrics.py
+
+ #### --model_paths / -m
+ Space-separated list of model paths for which metrics should be computed.
+
+
+
+We further provide the ```full_eval.py``` script. This script specifies the routine used in our evaluation and demonstrates the use of some additional parameters, e.g., ```--images (-i)``` to define alternative image directories within COLMAP data sets. If you have downloaded and extracted all the training data, you can run it like this:
+```shell
+python full_eval.py -m360 -tat -db
+```
+In the current version, this process takes about 7h on our reference machine containing an A6000. If you want to do the full evaluation on our pre-trained models, you can specify their download location and skip training.
+```shell
+python full_eval.py -o --skip_training -m360 -tat -db
+```
+
+If you want to compute the metrics on our paper's [evaluation images](https://repo-sam.inria.fr/fungraph/3d-gaussian-splatting/evaluation/images.zip), you can also skip rendering. In this case it is not necessary to provide the source datasets. You can compute metrics for multiple image sets at a time.
+```shell
+python full_eval.py -m /garden ... --skip_training --skip_rendering
+```
+
+
+Command Line Arguments for full_eval.py
+
+ #### --skip_training
+ Flag to skip training stage.
+ #### --skip_rendering
+ Flag to skip rendering stage.
+ #### --skip_metrics
+ Flag to skip metrics calculation stage.
+ #### --output_path
+ Directory to put renderings and results in, ```./eval``` by default, set to pre-trained model location if evaluating them.
+ #### --mipnerf360 / -m360
+ Path to MipNeRF360 source datasets, required if training or rendering.
+ #### --tanksandtemples / -tat
+ Path to Tanks&Temples source datasets, required if training or rendering.
+ #### --deepblending / -db
+ Path to Deep Blending source datasets, required if training or rendering.
+
+
+
+## Interactive Viewers
+We provide two interactive viewers for our method: remote and real-time. Our viewing solutions are based on the [SIBR](https://sibr.gitlabpages.inria.fr/) framework, developed by the GRAPHDECO group for several novel-view synthesis projects.
+
+### Hardware Requirements
+- OpenGL 4.5-ready GPU and drivers (or latest MESA software)
+- 4 GB VRAM recommended
+- CUDA-ready GPU with Compute Capability 7.0+ (only for Real-Time Viewer)
+
+### Software Requirements
+- Visual Studio or g++, **not Clang** (we used Visual Studio 2019 for Windows)
+- CUDA SDK 11, install *after* Visual Studio (we used 11.8)
+- CMake (recent version, we used 3.24)
+- 7zip (only on Windows)
+
+### Pre-built Windows Binaries
+We provide pre-built binaries for Windows [here](https://repo-sam.inria.fr/fungraph/3d-gaussian-splatting/binaries/viewers.zip). We recommend using them on Windows for an efficient setup, since the building of SIBR involves several external dependencies that must be downloaded and compiled on-the-fly.
+
+### Installation from Source
+If you cloned with submodules (e.g., using ```--recursive```), the source code for the viewers is found in ```SIBR_viewers```. The network viewer runs within the SIBR framework for Image-based Rendering applications.
+
+#### Windows
+CMake should take care of your dependencies.
+```shell
+cd SIBR_viewers
+cmake -Bbuild .
+cmake --build build --target install --config RelWithDebInfo
+```
+You may specify a different configuration, e.g. ```Debug``` if you need more control during development.
+
+#### Ubuntu 22.04
+You will need to install a few dependencies before running the project setup.
+```shell
+# Dependencies
+sudo apt install -y libglew-dev libassimp-dev libboost-all-dev libgtk-3-dev libopencv-dev libglfw3-dev libavdevice-dev libavcodec-dev libeigen3-dev libxxf86vm-dev libembree-dev
+# Project setup
+cd SIBR_viewers
+cmake -Bbuild . -DCMAKE_BUILD_TYPE=Release # add -G Ninja to build faster
+cmake --build build -j24 --target install
+```
+
+#### Ubuntu 20.04
+Backwards compatibility with Focal Fossa is not fully tested, but building SIBR with CMake should still work after invoking
+```shell
+git checkout fossa_compatibility
+```
+
+### Navigation in SIBR Viewers
+The SIBR interface provides several methods of navigating the scene. By default, you will be started with an FPS navigator, which you can control with ```W, A, S, D, Q, E``` for camera translation and ```I, K, J, L, U, O``` for rotation. Alternatively, you may want to use a Trackball-style navigator (select from the floating menu). You can also snap to a camera from the data set with the ```Snap to``` button or find the closest camera with ```Snap to closest```. The floating menues also allow you to change the navigation speed. You can use the ```Scaling Modifier``` to control the size of the displayed Gaussians, or show the initial point cloud.
+
+### Running the Network Viewer
+
+
+
+https://github.com/graphdeco-inria/gaussian-splatting/assets/40643808/90a2e4d3-cf2e-4633-b35f-bfe284e28ff7
+
+
+
+After extracting or installing the viewers, you may run the compiled ```SIBR_remoteGaussian_app[_config]``` app in ```/bin```, e.g.:
+```shell
+.//bin/SIBR_remoteGaussian_app
+```
+The network viewer allows you to connect to a running training process on the same or a different machine. If you are training on the same machine and OS, no command line parameters should be required: the optimizer communicates the location of the training data to the network viewer. By default, optimizer and network viewer will try to establish a connection on **localhost** on port **6009**. You can change this behavior by providing matching ```--ip``` and ```--port``` parameters to both the optimizer and the network viewer. If for some reason the path used by the optimizer to find the training data is not reachable by the network viewer (e.g., due to them running on different (virtual) machines), you may specify an override location to the viewer by using ```-s ```.
+
+
+Primary Command Line Arguments for Network Viewer
+
+ #### --path / -s
+ Argument to override model's path to source dataset.
+ #### --ip
+ IP to use for connection to a running training script.
+ #### --port
+ Port to use for connection to a running training script.
+ #### --rendering-size
+ Takes two space separated numbers to define the resolution at which network rendering occurs, ```1200``` width by default.
+ Note that to enforce an aspect that differs from the input images, you need ```--force-aspect-ratio``` too.
+ #### --load_images
+ Flag to load source dataset images to be displayed in the top view for each camera.
+
+
+
+### Running the Real-Time Viewer
+
+
+
+
+https://github.com/graphdeco-inria/gaussian-splatting/assets/40643808/0940547f-1d82-4c2f-a616-44eabbf0f816
+
+
+
+
+After extracting or installing the viewers, you may run the compiled ```SIBR_gaussianViewer_app[_config]``` app in ```/bin```, e.g.:
+```shell
+.//bin/SIBR_gaussianViewer_app -m
+```
+
+It should suffice to provide the ```-m``` parameter pointing to a trained model directory. Alternatively, you can specify an override location for training input data using ```-s```. To use a specific resolution other than the auto-chosen one, specify ```--rendering-size ```. Combine it with ```--force-aspect-ratio``` if you want the exact resolution and don't mind image distortion.
+
+**To unlock the full frame rate, please disable V-Sync on your machine and also in the application (Menu → Display). In a multi-GPU system (e.g., laptop) your OpenGL/Display GPU should be the same as your CUDA GPU (e.g., by setting the application's GPU preference on Windows, see below) for maximum performance.**
+
+
+
+In addition to the initial point cloud and the splats, you also have the option to visualize the Gaussians by rendering them as ellipsoids from the floating menu.
+SIBR has many other functionalities, please see the [documentation](https://sibr.gitlabpages.inria.fr/) for more details on the viewer, navigation options etc. There is also a Top View (available from the menu) that shows the placement of the input cameras and the original SfM point cloud; please note that Top View slows rendering when enabled. The real-time viewer also uses slightly more aggressive, fast culling, which can be toggled in the floating menu. If you ever encounter an issue that can be solved by turning fast culling off, please let us know.
+
+
+Primary Command Line Arguments for Real-Time Viewer
+
+ #### --model-path / -m
+ Path to trained model.
+ #### --iteration
+ Specifies which of state to load if multiple are available. Defaults to latest available iteration.
+ #### --path / -s
+ Argument to override model's path to source dataset.
+ #### --rendering-size
+ Takes two space separated numbers to define the resolution at which real-time rendering occurs, ```1200``` width by default. Note that to enforce an aspect that differs from the input images, you need ```--force-aspect-ratio``` too.
+ #### --load_images
+ Flag to load source dataset images to be displayed in the top view for each camera.
+ #### --device
+ Index of CUDA device to use for rasterization if multiple are available, ```0``` by default.
+ #### --no_interop
+ Disables CUDA/GL interop forcibly. Use on systems that may not behave according to spec (e.g., WSL2 with MESA GL 4.5 software rendering).
+
+
+
+## Processing your own Scenes
+
+Our COLMAP loaders expect the following dataset structure in the source path location:
+
+```
+
+|---images
+| |---
+| |---
+| |---...
+|---sparse
+ |---0
+ |---cameras.bin
+ |---images.bin
+ |---points3D.bin
+```
+
+For rasterization, the camera models must be either a SIMPLE_PINHOLE or PINHOLE camera. We provide a converter script ```convert.py```, to extract undistorted images and SfM information from input images. Optionally, you can use ImageMagick to resize the undistorted images. This rescaling is similar to MipNeRF360, i.e., it creates images with 1/2, 1/4 and 1/8 the original resolution in corresponding folders. To use them, please first install a recent version of COLMAP (ideally CUDA-powered) and ImageMagick. Put the images you want to use in a directory ```/input```.
+```
+
+|---input
+ |---
+ |---
+ |---...
+```
+ If you have COLMAP and ImageMagick on your system path, you can simply run
+```shell
+python convert.py -s [--resize] #If not resizing, ImageMagick is not needed
+```
+Alternatively, you can use the optional parameters ```--colmap_executable``` and ```--magick_executable``` to point to the respective paths. Please note that on Windows, the executable should point to the COLMAP ```.bat``` file that takes care of setting the execution environment. Once done, `````` will contain the expected COLMAP data set structure with undistorted, resized input images, in addition to your original images and some temporary (distorted) data in the directory ```distorted```.
+
+If you have your own COLMAP dataset without undistortion (e.g., using ```OPENCV``` camera), you can try to just run the last part of the script: Put the images in ```input``` and the COLMAP info in a subdirectory ```distorted```:
+```
+
+|---input
+| |---
+| |---
+| |---...
+|---distorted
+ |---database.db
+ |---sparse
+ |---0
+ |---...
+```
+Then run
+```shell
+python convert.py -s --skip_matching [--resize] #If not resizing, ImageMagick is not needed
+```
+
+
+Command Line Arguments for convert.py
+
+ #### --no_gpu
+ Flag to avoid using GPU in COLMAP.
+ #### --skip_matching
+ Flag to indicate that COLMAP info is available for images.
+ #### --source_path / -s
+ Location of the inputs.
+ #### --camera
+ Which camera model to use for the early matching steps, ```OPENCV``` by default.
+ #### --resize
+ Flag for creating resized versions of input images.
+ #### --colmap_executable
+ Path to the COLMAP executable (```.bat``` on Windows).
+ #### --magick_executable
+ Path to the ImageMagick executable.
+
+
+
+### Training speed acceleration
+
+We integrated the drop-in replacements from [Taming-3dgs](https://humansensinglab.github.io/taming-3dgs/)1 with [fused ssim](https://github.com/rahul-goel/fused-ssim/tree/main) into the original codebase to speed up training times. Once installed, the accelerated rasterizer delivers a **$\times$ 1.6 training time speedup** using `--optimizer_type default` and a **$\times$ 2.7 training time speedup** using `--optimizer_type sparse_adam`.
+
+To get faster training times you must first install the accelerated rasterizer to your environment:
+
+```bash
+pip uninstall diff-gaussian-rasterization -y
+cd submodules/diff-gaussian-rasterization
+rm -r build
+git checkout 3dgs_accel
+pip install .
+```
+
+Then you can add the following parameter to use the sparse adam optimizer when running `train.py`:
+
+```bash
+--optimizer_type sparse_adam
+```
+
+*Note that this custom rasterizer has a different behaviour than the original version, for more details on training times please see [stats for training times](results.md/#training-times-comparisons)*.
+
+*1. Mallick and Goel, et al. ‘Taming 3DGS: High-Quality Radiance Fields with Limited Resources’. SIGGRAPH Asia 2024 Conference Papers, 2024, https://doi.org/10.1145/3680528.3687694, [github](https://github.com/humansensinglab/taming-3dgs)*
+
+
+### Depth regularization
+
+To have better reconstructed scenes we use depth maps as priors during optimization with each input images. It works best on untextured parts ex: roads and can remove floaters. Several papers have used similar ideas to improve various aspects of 3DGS; (e.g. [DepthRegularizedGS](https://robot0321.github.io/DepthRegGS/index.html), [SparseGS](https://formycat.github.io/SparseGS-Real-Time-360-Sparse-View-Synthesis-using-Gaussian-Splatting/), [DNGaussian](https://fictionarry.github.io/DNGaussian/)). The depth regularization we integrated is that used in our [Hierarchical 3DGS](https://repo-sam.inria.fr/fungraph/hierarchical-3d-gaussians/) paper, but applied to the original 3DGS; for some scenes (e.g., the DeepBlending scenes) it improves quality significantly; for others it either makes a small difference or can even be worse. For example results showing the potential benefit and statistics on quality please see here: [Stats for depth regularization](results.md).
+
+When training on a synthetic dataset, depth maps can be produced and they do not require further processing to be used in our method.
+
+For real world datasets depth maps should be generated for each input images, to generate them please do the following:
+1. Clone [Depth Anything v2](https://github.com/DepthAnything/Depth-Anything-V2?tab=readme-ov-file#usage):
+ ```
+ git clone https://github.com/DepthAnything/Depth-Anything-V2.git
+ ```
+2. Download weights from [Depth-Anything-V2-Large](https://huggingface.co/depth-anything/Depth-Anything-V2-Large/resolve/main/depth_anything_v2_vitl.pth?download=true) and place it under `Depth-Anything-V2/checkpoints/`
+3. Generate depth maps:
+ ```
+ python Depth-Anything-V2/run.py --encoder vitl --pred-only --grayscale --img-path --outdir
+ ```
+5. Generate a `depth_params.json` file using:
+ ```
+ python utils/make_depth_scale.py --base_dir --depths_dir
+ ```
+
+A new parameter should be set when training if you want to use depth regularization `-d `.
+
+### Exposure compensation
+To compensate for exposure changes in the different input images we optimize an affine transformation for each image just as in [Hierarchical 3dgs](https://repo-sam.inria.fr/fungraph/hierarchical-3d-gaussians/).
+
+This can greatly improve reconstruction results for "in the wild" captures, e.g., with a smartphone when the exposure setting of the camera is not fixed. For example results showing the potential benefit and statistics on quality please see here: [Stats for exposure compensation](results.md).
+
+Add the following parameters to enable it:
+```
+--exposure_lr_init 0.001 --exposure_lr_final 0.0001 --exposure_lr_delay_steps 5000 --exposure_lr_delay_mult 0.001 --train_test_exp
+```
+Again, other excellent papers have used similar ideas e.g. [NeRF-W](https://nerf-w.github.io/), [URF](https://urban-radiance-fields.github.io/).
+
+### Anti-aliasing
+We added the EWA Filter from [Mip Splatting](https://niujinshuchong.github.io/mip-splatting/) in our codebase to remove aliasing. It is disabled by default but you can enable it by adding `--antialiasing` when training on a scene using `train.py` or rendering using `render.py`. Antialiasing can be toggled in the SIBR viewer, it is disabled by default but you should enable it when viewing a scene trained using `--antialiasing`.
+
+*this scene was trained using `--antialiasing`*.
+
+### SIBR: Top view
+> `Views > Top view`
+
+The `Top view` renders the SfM point cloud in another view with the corresponding input cameras and the `Point view` user camera. This allows visualization of how far the viewer is from the input cameras for example.
+
+It is a 3D view so the user can navigate through it just as in the `Point view` (modes available: FPS, trackball, orbit).
+
+
+
+
+Options are available to customize this view, meshes can be disabled/enabled and their scales can be modified.
+
+
+
+A useful additional functionality is to move to the position of an input image, and progressively fade out to the SfM point view in that position (e.g., to verify camera alignment). Views from input cameras can be displayed in the `Top view` (*note that `--images-path` must be set in the command line*). One can snap the `Top view` camera to the closest input camera from the user camera in the `Point view` by clicking `Top view settings > Cameras > Snap to closest`.
+
+
+
+
+### OpenXR support
+
+OpenXR is supported in the branch `gaussian_code_release_openxr`
+Within that branch, you can find documentation for VR support [here](https://gitlab.inria.fr/sibr/sibr_core/-/tree/gaussian_code_release_openxr?ref_type=heads).
+
+
+## FAQ
+- *Where do I get data sets, e.g., those referenced in ```full_eval.py```?* The MipNeRF360 data set is provided by the authors of the original paper on the project site. Note that two of the data sets cannot be openly shared and require you to consult the authors directly. For Tanks&Temples and Deep Blending, please use the download links provided at the top of the page. Alternatively, you may access the cloned data (status: August 2023!) from [HuggingFace](https://huggingface.co/camenduru/gaussian-splatting)
+
+
+- *How can I use this for a much larger dataset, like a city district?* The current method was not designed for these, but given enough memory, it should work out. However, the approach can struggle in multi-scale detail scenes (extreme close-ups, mixed with far-away shots). This is usually the case in, e.g., driving data sets (cars close up, buildings far away). For such scenes, you can lower the ```--position_lr_init```, ```--position_lr_final``` and ```--scaling_lr``` (x0.3, x0.1, ...). The more extensive the scene, the lower these values should be. Below, we use default learning rates (left) and ```--position_lr_init 0.000016 --scaling_lr 0.001"``` (right).
+
+|  |  |
+| --- | --- |
+
+- *I'm on Windows and I can't manage to build the submodules, what do I do?* Consider following the steps in the excellent video tutorial [here](https://www.youtube.com/watch?v=UXtuigy_wYc), hopefully they should help. The order in which the steps are done is important! Alternatively, consider using the linked Colab template.
+
+- *It still doesn't work. It says something about ```cl.exe```. What do I do?* User Henry Pearce found a workaround. You can you try adding the visual studio path to your environment variables (your version number might differ);
+```C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\bin\Hostx64\x64```
+Then make sure you start a new conda prompt and cd to your repo location and try this;
+```
+conda activate gaussian_splatting
+cd /gaussian-splatting
+pip install submodules\diff-gaussian-rasterization
+pip install submodules\simple-knn
+```
+
+- *I'm on macOS/Puppy Linux/Greenhat and I can't manage to build, what do I do?* Sorry, we can't provide support for platforms outside of the ones we list in this README. Consider using the linked Colab template.
+
+- *I don't have 24 GB of VRAM for training, what do I do?* The VRAM consumption is determined by the number of points that are being optimized, which increases over time. If you only want to train to 7k iterations, you will need significantly less. To do the full training routine and avoid running out of memory, you can increase the ```--densify_grad_threshold```, ```--densification_interval``` or reduce the value of ```--densify_until_iter```. Note however that this will affect the quality of the result. Also try setting ```--test_iterations``` to ```-1``` to avoid memory spikes during testing. If ```--densify_grad_threshold``` is very high, no densification should occur and training should complete if the scene itself loads successfully.
+
+- *24 GB of VRAM for reference quality training is still a lot! Can't we do it with less?* Yes, most likely. By our calculations it should be possible with **way** less memory (~8GB). If we can find the time we will try to achieve this. If some PyTorch veteran out there wants to tackle this, we look forward to your pull request!
+
+
+- *How can I use the differentiable Gaussian rasterizer for my own project?* Easy, it is included in this repo as a submodule ```diff-gaussian-rasterization```. Feel free to check out and install the package. It's not really documented, but using it from the Python side is very straightforward (cf. ```gaussian_renderer/__init__.py```).
+
+- *Wait, but `````` isn't optimized and could be much better?* There are several parts we didn't even have time to think about improving (yet). The performance you get with this prototype is probably a rather slow baseline for what is physically possible.
+
+- *Something is broken, how did this happen?* We tried hard to provide a solid and comprehensible basis to make use of the paper's method. We have refactored the code quite a bit, but we have limited capacity to test all possible usage scenarios. Thus, if part of the website, the code or the performance is lacking, please create an issue. If we find the time, we will do our best to address it.
diff --git a/submodules/gaussian-splatting/SIBR_viewers/.gitignore b/submodules/gaussian-splatting/SIBR_viewers/.gitignore
new file mode 100644
index 0000000000000000000000000000000000000000..3ffaa95079940db3a5c852f8c8fadd41efe8e722
--- /dev/null
+++ b/submodules/gaussian-splatting/SIBR_viewers/.gitignore
@@ -0,0 +1,45 @@
+extlibs/
+build/
+install/
+src/projects/*
+cmake-gui.exe.stackdump
+__pycache__/
+
+# emacs garbage
+\#*
+.\#*
+
+# vim garbage
+*.swp
+*.swo
+*.idea/
+*.log
+*.sh
+*.tmp
+
+hs_err_*
+
+# re include common public projects
+!src/projects/ulr/
+!src/projects/dataset_tools/
+
+# more vim garbage
+# Swap
+[._]*.s[a-v][a-z]
+!*.svg # comment out if you don't need vector files
+[._]*.sw[a-p]
+[._]s[a-rt-v][a-z]
+[._]ss[a-gi-z]
+[._]sw[a-p]
+
+# Session
+Session.vim
+Sessionx.vim
+
+# Temporary
+.netrwhist
+*~
+# Auto-generated tag files
+tags
+# Persistent undo
+[._]*.un~
\ No newline at end of file
diff --git a/submodules/gaussian-splatting/SIBR_viewers/CMakeLists.txt b/submodules/gaussian-splatting/SIBR_viewers/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..21a3fc85ddfd30e4b10d57a4bd7cb349baf0f612
--- /dev/null
+++ b/submodules/gaussian-splatting/SIBR_viewers/CMakeLists.txt
@@ -0,0 +1,213 @@
+# Copyright (C) 2020, Inria
+# GRAPHDECO research group, https://team.inria.fr/graphdeco
+# All rights reserved.
+#
+# This software is free for non-commercial, research and evaluation use
+# under the terms of the LICENSE.md file.
+#
+# For inquiries contact sibr@inria.fr and/or George.Drettakis@inria.fr
+
+CMAKE_MINIMUM_REQUIRED(VERSION 3.22)
+
+set (CMAKE_SYSTEM_VERSION 10.0.15063.0 CACHE INTERNAL "Cmake system version" FORCE)
+PROJECT(sibr_projects)
+
+set(REQUIRED_VERSION "3.22.0")
+set(CHECKED_VERSION "3.27.0")
+
+if (CMAKE_VERSION VERSION_LESS REQUIRED_VERSION)
+ message(WARNING "Deprecated version of cmake. Please update to at least ${REQUIRED_VERSION} (${CHECKED_VERSION} recommended).")
+elseif (CMAKE_VERSION VERSION_GREATER CHECKED_VERSION)
+ message(WARNING "Untested version of cmake. If you checked everything is working properly, please update ${CHECKED_VERSION} in the main CmakeLists.txt with the version you tested.")
+endif()
+
+## Include cmake stuff (functions/macros) : Modules files
+if(WIN32)
+list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake/windows)
+list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake/windows/Modules)
+else()
+list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake/linux)
+list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake/linux/Modules)
+endif()
+set_property(GLOBAL PROPERTY USE_FOLDERS ON)
+
+## To maintain cmake versions compatibilities
+include(cmake_policies)
+setPolicies()
+
+include(git_describe)
+git_describe(GIT_BRANCH SIBR_CORE_BRANCH GIT_COMMIT_HASH SIBR_CORE_COMMIT_HASH GIT_TAG SIBR_CORE_TAG GIT_VERSION SIBR_CORE_VERSION)
+
+message(STATUS "SIBR version :\n BRANCH ${SIBR_CORE_BRANCH}\n COMMIT_HASH ${SIBR_CORE_COMMIT_HASH}\n TAG ${SIBR_CORE_TAG}\n VERSION ${SIBR_CORE_VERSION}")
+
+if(NOT WIN32)
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+endif()
+
+
+if (WIN32)
+ ## Allow C++11 + other flags
+ include(CheckCXXCompilerFlag)
+ get_filename_component(currentBuildTool ${CMAKE_BUILD_TOOL} NAME_WE) # tool that can launch the native build system. returned value may be the full path
+ if(${currentBuildTool} MATCHES "(msdev|devenv|nmake|MSBuild)")
+
+ add_compile_options("$<$:/W3;/DNOMINMAX;/MP;-D_USE_MATH_DEFINES>")
+ #add_definitions(/W3 /DNOMINMAX /MP -D_USE_MATH_DEFINES)# /D_ITERATOR_DEBUG_LEVEL=1 because you need all external DLl to compile with this flag too
+ set(CMAKE_CONFIGURATION_TYPES "RelWithDebInfo;Release;Debug" CACHE STRING "" FORCE)
+ set(CMAKE_CXX_STANDARD 14)
+ set(CMAKE_CXX_STANDARD_REQUIRED ON)
+ set(CMAKE_CXX_EXTENSIONS OFF)
+ elseif(${currentBuildTool} MATCHES "(make|gmake)")
+ add_definitions("-Wall -Wno-unknown-pragmas -Wno-sign-compare -g -std=c++14 -D__forceinline=\"inline\ __attribute__((always_inline))\"")
+ # CHECK_CXX_COMPILER_FLAG("-std=gnu++11" COMPILER_SUPPORTS_CXX11)
+ # CHECK_CXX_COMPILER_FLAG("-std=gnu++0x" COMPILER_SUPPORTS_CXX0X)
+ # if(COMPILER_SUPPORTS_CXX11)
+ # add_definitions(-std=gnu++11)
+ # elseif(COMPILER_SUPPORTS_CXX0X)
+ # add_definitions(-std=gnu++0x)
+ # else()
+ # message(SEND_ERROR "The compiler ${CMAKE_CXX_COMPILER} has no C++14 support. Please use a different C++ compiler.")
+ # endif()
+ elseif(APPLE) ## \todo TODO: do a better test and send error on unsupported c++14 compiler
+ add_definitions(-std=c++14 -stdlib=libc++)
+ endif()
+else()
+ ## Allow C++11 + other flags
+ include(CheckCXXCompilerFlag)
+ get_filename_component(currentBuildTool ${CMAKE_BUILD_TOOL} NAME_WE) # tool that can launch the native build system. returned value may be the full path
+ if(${currentBuildTool} MATCHES "(msdev|devenv|nmake|MSBuild)")
+
+ add_compile_options("$<$:/W3;/DNOMINMAX;/MP;-D_USE_MATH_DEFINES>")
+ #add_definitions(/W3 /DNOMINMAX /MP -D_USE_MATH_DEFINES)# /D_ITERATOR_DEBUG_LEVEL=1 because you need all external DLl to compile with this flag too
+ set(CMAKE_CONFIGURATION_TYPES "RelWithDebInfo;Release;Debug" CACHE STRING "" FORCE)
+ set(CMAKE_CXX_STANDARD 14)
+ set(CMAKE_CXX_STANDARD_REQUIRED ON)
+ set(CMAKE_CXX_EXTENSIONS OFF)
+ elseif(${currentBuildTool} MATCHES "(make|gmake|ninja)")
+ add_definitions("-fpermissive -fPIC -Wall -Wno-unknown-pragmas -Wno-sign-compare -g -std=c++17 -D__forceinline=\"inline\ __attribute__((always_inline))\"")
+ elseif(APPLE) ## \todo TODO: do a better test and send error on unsupported c++14 compiler
+ add_definitions(-std=c++17 -stdlib=libc++)
+ endif()
+endif()
+
+set(INSTALL_STANDALONE ON)
+
+## Set default build output binaries (used also in sub CMakeLists.txt) :
+set(BIN_BUILT_DIR "bin")
+if(CMAKE_SIZEOF_VOID_P EQUAL 8)
+ set(ARCHI_BUILT_DIR "x64")
+ set(LIB_BUILT_DIR "lib64")
+else()
+ set(ARCHI_BUILT_DIR "x86")
+ set(LIB_BUILT_DIR "lib")
+endif()
+
+option(SEPARATE_CONFIGURATIONS "Clearer separation between configurations" OFF)
+SET(CMAKE_INSTALL_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/install)
+SET(CMAKE_INSTALL_PREFIX ${CMAKE_INSTALL_ROOT})
+
+if(DEFINED CMAKE_BUILD_TYPE) ## for mono config type (make/nmake/ninja based)
+ if(${CMAKE_BUILD_TYPE} MATCHES "Debug")
+ set(CMAKE_DEBUG_POSTFIX "_d")
+ elseif(${CMAKE_BUILD_TYPE} MATCHES "RelWithDebInfo")
+ set(CMAKE_RELWITHDEBINFO_POSTFIX "_rwdi")
+ elseif(${CMAKE_BUILD_TYPE} MATCHES "MinSizeRel")
+ set(CMAKE_MINSIZEREL_POSTFIX "_msr")
+ elseif(${CMAKE_BUILD_TYPE} MATCHES "Release")
+ set(CMAKE_RELEASE_POSTFIX "")
+ endif()
+
+ if(SEPARATE_CONFIGURATIONS)
+ SET(CMAKE_INSTALL_PREFIX_${CMAKE_BUILD_TYPE} ${CMAKE_INSTALL_ROOT}/${CMAKE_BUILD_TYPE})
+ else()
+ SET(CMAKE_INSTALL_PREFIX_${CMAKE_BUILD_TYPE} ${CMAKE_INSTALL_ROOT})
+ endif()
+
+ MESSAGE(STATUS "Install path set to ${CMAKE_INSTALL_PREFIX}.")
+ SET(CMAKE_OUTPUT_LIB_${CMAKE_BUILD_TYPE} ${CMAKE_INSTALL_PREFIX_${CMAKE_BUILD_TYPE}}/lib)
+ SET(CMAKE_OUTPUT_BIN_${CMAKE_BUILD_TYPE} ${CMAKE_INSTALL_PREFIX_${CMAKE_BUILD_TYPE}}/bin)
+
+ set(CMAKE_LIBRARY_OUTPUT_DIRECTORY_${CMAKE_BUILD_TYPE} ${CMAKE_OUTPUT_LIB_${CMAKE_BUILD_TYPE}})
+ set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY_${CMAKE_BUILD_TYPE} ${CMAKE_OUTPUT_LIB_${CMAKE_BUILD_TYPE}})
+ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_${CMAKE_BUILD_TYPE} ${CMAKE_OUTPUT_BIN_${CMAKE_BUILD_TYPE}})
+ set(CMAKE_PDB_OUTPUT_DIRECTORY_${CMAKE_BUILD_TYPE} ${CMAKE_OUTPUT_BIN_${CMAKE_BUILD_TYPE}})
+endif()
+foreach(CONFIG_TYPES ${CMAKE_CONFIGURATION_TYPES}) ## for multi config types (MSVC based)
+ string(TOUPPER ${CONFIG_TYPES} CONFIG_TYPES_UC)
+ if(${CONFIG_TYPES} MATCHES "Debug")
+ set(CMAKE_DEBUG_POSTFIX "_d")
+ elseif(${CONFIG_TYPES} MATCHES "RelWithDebInfo")
+ set(CMAKE_RELWITHDEBINFO_POSTFIX "_rwdi")
+ elseif(${CONFIG_TYPES} MATCHES "MinSizeRel")
+ set(CMAKE_MINSIZEREL_POSTFIX "_msr")
+ elseif(${CMAKE_BUILD_TYPE} MATCHES "Release")
+ set(CMAKE_RELEASE_POSTFIX "")
+ endif()
+
+ if(SEPARATE_CONFIGURATIONS)
+ SET(CMAKE_INSTALL_PREFIX_${CONFIG_TYPES_UC} ${CMAKE_INSTALL_ROOT}/${CONFIG_TYPES})
+ else()
+ SET(CMAKE_INSTALL_PREFIX_${CONFIG_TYPES_UC} ${CMAKE_INSTALL_ROOT})
+ endif()
+
+ MESSAGE(STATUS "Install path for ${CONFIG_TYPES} set to ${CMAKE_INSTALL_PREFIX_${CONFIG_TYPES_UC}}.")
+ SET(CMAKE_OUTPUT_LIB_${CONFIG_TYPES_UC} ${CMAKE_INSTALL_PREFIX_${CONFIG_TYPES_UC}}/lib)
+ SET(CMAKE_OUTPUT_BIN_${CONFIG_TYPES_UC} ${CMAKE_INSTALL_PREFIX_${CONFIG_TYPES_UC}}/bin)
+
+ set(CMAKE_LIBRARY_OUTPUT_DIRECTORY_${CONFIG_TYPES_UC} ${CMAKE_OUTPUT_LIB_${CONFIG_TYPES_UC}})
+ set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY_${CONFIG_TYPES_UC} ${CMAKE_OUTPUT_LIB_${CONFIG_TYPES_UC}})
+ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_${CONFIG_TYPES_UC} ${CMAKE_OUTPUT_BIN_${CONFIG_TYPES_UC}})
+ set(CMAKE_PDB_OUTPUT_DIRECTORY_${CONFIG_TYPES_UC} ${CMAKE_OUTPUT_BIN_${CONFIG_TYPES_UC}})
+endforeach()
+
+
+# Settings for RPATH
+if (NOT WIN32)
+ # Default config of Fedora at INRIA has no LD_LIBRARY_PATH (for security reasons I guess)
+ # So at least I had "./" in RPATH and found link paths
+ #set(CMAKE_SKIP_RPATH TRUE)
+ #SET(CMAKE_SKIP_BUILD_RPATH FALSE)
+ SET(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
+
+ SET(CMAKE_INSTALL_RPATH "$ORIGIN")
+ #SET(CMAKE_INSTALL_RPATH "./")
+ #SET(CMAKE_INSTALL_RPATH "./:/usr/lib64/:/usr/lib/:/usr/local/lib64/:/usr/local/lib/") # This one causes be a problem -> a "default" version of libGL (swrast) is located in /usr/lib64 and was selected instead of nvidia one (in /usr/lib64/nividia)
+
+ SET(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
+endif()
+
+
+set(SIBR_PROGRAMARGS "" CACHE STRING "Default program arguments used in Visual Studio target properties")
+if ("${SIBR_PROGRAMARGS}" STREQUAL "")
+ if (DEFINED ENV{SIBR_PROGRAMARGS})
+ set(SIBR_PROGRAMARGS "$ENV{SIBR_PROGRAMARGS}" CACHE STRING "Default program arguments used in Visual Studio target properties" FORCE)
+ message( STATUS "Using program options found in environment variable 'SIBR_PROGRAMARGS' => '${SIBR_PROGRAMARGS}'")
+ else()
+ message(
+ "Note you can provide default program options for Visual Studio target properties by either setting"
+ " a value for the cmake cached variable 'SIBR_PROGRAMARGS' or by setting a new environment "
+ "variable 'SIBR_PROGRAMARGS'")
+ endif()
+endif()
+
+add_custom_target(PREBUILD ALL)
+
+## Include all projects
+set(SIBR_PROJECTS_SAMPLES_SUBPAGE_REF "")
+set(SIBR_PROJECTS_OURS_SUBPAGE_REF "")
+set(SIBR_PROJECTS_TOOLBOX_SUBPAGE_REF "")
+set(SIBR_PROJECTS_OTHERS_SUBPAGE_REF "")
+set(SIBR_PROJECTS_SAMPLES_REF_REF "")
+set(SIBR_PROJECTS_OURS_REF_REF "")
+set(SIBR_PROJECTS_TOOLBOX_REF_REF "")
+set(SIBR_PROJECTS_OTHERS_REF_REF "")
+set(DOXY_APP_SPECIFIC_IMG_PATH "")
+set(DOXY_DOC_EXCLUDE_PATTERNS_DIRS "")
+ADD_SUBDIRECTORY(src)
+
+
+## handle documentation
+if (WIN32)
+ADD_SUBDIRECTORY(docs)
+endif()
diff --git a/submodules/gaussian-splatting/SIBR_viewers/LICENSE.md b/submodules/gaussian-splatting/SIBR_viewers/LICENSE.md
new file mode 100644
index 0000000000000000000000000000000000000000..0ae59dae71d0f61caa31d7465d2472c0b3186df4
--- /dev/null
+++ b/submodules/gaussian-splatting/SIBR_viewers/LICENSE.md
@@ -0,0 +1,204 @@
+SIBR License
+============
+
+The sibr system is licensed under the Apache 2.0 license, except for some projects in specific branches (in src/projects) that have separate licenses in those directories.
+
+Please verify the LICENSE.md file for those directories.
+
+--------------
+
+
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+
+ Copyright 2024 Inria / Universite Cote d'Azur
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+
diff --git a/submodules/gaussian-splatting/SIBR_viewers/README.md b/submodules/gaussian-splatting/SIBR_viewers/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..381d6b036a7e2990df7e6bd4ae7a04a64a431b2b
--- /dev/null
+++ b/submodules/gaussian-splatting/SIBR_viewers/README.md
@@ -0,0 +1,142 @@
+# SIBR Core
+
+**SIBR** is a System for Image-Based Rendering.
+It is built around the *sibr-core* in this repo and several *Projects* implementing published research papers.
+For more complete documentation, see here: [SIBR Documentation](https://sibr.gitlabpages.inria.fr)
+
+This **SIBR core** repository provides :
+- a basic Image-Based Renderer
+- a per-pixel implementation of Unstructured Lumigraph (ULR)
+- several dataset tools & pipelines do process input images
+
+Details on how to run in the documentation and in the section below.
+If you use this code in a publication, please cite the system as follows:
+
+```
+@misc{sibr2020,
+ author = "Bonopera, Sebastien and Esnault, Jerome and Prakash, Siddhant and Rodriguez, Simon and Thonat, Theo and Benadel, Mehdi and Chaurasia, Gaurav and Philip, Julien and Drettakis, George",
+ title = "sibr: A System for Image Based Rendering",
+ year = "2020",
+ url = "https://gitlab.inria.fr/sibr/sibr_core"
+}
+```
+
+## Setup
+
+**Note**: The current release is for *Windows 10* only. We are planning a Linux release soon.
+
+#### Binary distribution
+
+The easiest way to use SIBR is to download the binary distribution. All steps described below, including all preprocessing for your datasets will work using this code.
+
+Download the distribution from the page: https://sibr.gitlabpages.inria.fr/download.html (Core, 57Mb); unzip the file and rename the directory "install".
+
+#### Install requirements
+
+- [**Visual Studio 2019**](https://visualstudio.microsoft.com/fr/downloads/)
+- [**Cmake 3.16+**](https://cmake.org/download)
+- [**7zip**](https://www.7-zip.org)
+- [**Python 3.8+**](https://www.python.org/downloads/) for shaders installation scripts and dataset preprocess scripts
+- [**Doxygen 1.8.17+**](https://www.doxygen.nl/download.html#srcbin) for documentation
+- [**CUDA 10.1+**](https://developer.nvidia.com/cuda-downloads) and [**CUDnn**](https://developer.nvidia.com/cudnn) if projects requires it
+
+Make sure Python, CUDA and Doxygen are in the PATH
+
+If you have Chocolatey, you can grab most of these with this command:
+
+```sh
+choco install cmake 7zip python3 doxygen.install cuda
+
+## Visual Studio is available on Chocolatey,
+## though we do advise to set it from Visual Studio Installer and to choose your licensing accordingly
+choco install visualstudio2019community
+```
+
+#### Generation of the solution
+
+- Checkout this repository's master branch:
+
+ ```sh
+ ## through HTTPS
+ git clone https://gitlab.inria.fr/sibr/sibr_core.git -b master
+ ## through SSH
+ git clone git@gitlab.inria.fr:sibr/sibr_core.git -b master
+ ```
+- Run Cmake-gui once, select the repo root as a source directory, `build/` as the build directory. Configure, select the Visual Studio C++ Win64 compiler
+- Select the projects you want to generate among the BUILD elements in the list (you can group Cmake flags by categories to access those faster)
+- Generate
+
+#### Compilation
+
+- Open the generated Visual Studio solution (`build/sibr_projects.sln`)
+- Build the `ALL_BUILD` target, and then the `INSTALL` target
+- The compiled executables will be put in `install/bin`
+- TODO: are the DLLs properly installed?
+
+#### Compilation of the documentation
+
+- Open the generated Visual Studio solution (`build/sibr_projects.sln`)
+- Build the `DOCUMENTATION` target
+- Run `install/docs/index.html` in a browser
+
+
+## Scripts
+
+Some scripts will require you to install `PIL`, and `convert` from `ImageMagick`.
+
+```sh
+## To install pillow
+python -m pip install pillow
+
+## If you have Chocolatey, you can install imagemagick from this command
+choco install imagemagick
+```
+
+## Troubleshooting
+
+#### Bugs and Issues
+
+We will track bugs and issues through the Issues interface on gitlab. Inria gitlab does not allow creation of external accounts, so if you have an issue/bug please email sibr@inria.fr
and we will either create a guest account or create the issue on our side.
+
+#### Cmake complaining about the version
+
+if you are the first to use a very recent Cmake version, you will have to update `CHECKED_VERSION` in the root `CmakeLists.txt`.
+
+#### Weird OpenCV error
+
+you probably selected the 32-bits compiler in Cmake-gui.
+
+#### `Cmd.exe failed with error 009` or similar
+
+make sure Python is installed and in the path.
+
+#### `BUILD_ALL` or `INSTALL` fail because of a project you don't really need
+
+build and install each project separately by selecting the proper targets.
+
+#### Error in CUDA headers under Visual Studio 2019
+
+make sure CUDA >= 10.1 (first version to support VS2019) is installed.
+
+## To run an example
+
+For more details, please see the documentation: http://sibr.gitlabpages.inria.fr
+
+Download a dataset from: https://repo-sam.inria.fr/fungraph/sibr-datasets/
+
+e.g., the *sibr-museum-front* dataset in the *DATASETS_PATH* directory.
+
+```
+wget https://repo-sam.inria.fr/fungraph/sibr-datasets/museum_front27_ulr.zip
+```
+
+Once you have built the system or downloaded the binaries (see above), go to *install/bin* and you can run:
+```
+ sibr_ulrv2_app.exe --path DATASETS_PATH/sibr-museum-front
+```
+
+You will have an interactive viewer and you can navigate freely in the captured scene.
+Our default interactive viewer has a main view running the algorithm and a top view to visualize the position of the calibrated cameras. By default you are in WASD mode, and can toggle to trackball using the "y" key. Please see the page [Interface](https://sibr.gitlabpages.inria.fr/docs/nightly/howto_sibr_useful_objects.html) for more details on the interface.
+
+Please see the documentation on how to create a dataset from your own scene, and the various other IBR algorithms available.
+
diff --git a/submodules/gaussian-splatting/SIBR_viewers/cmake/linux/MSVCsetUserCommand.cmake b/submodules/gaussian-splatting/SIBR_viewers/cmake/linux/MSVCsetUserCommand.cmake
new file mode 100644
index 0000000000000000000000000000000000000000..bc49770d644ca2803a9d52f9186d952b40fafdf8
--- /dev/null
+++ b/submodules/gaussian-splatting/SIBR_viewers/cmake/linux/MSVCsetUserCommand.cmake
@@ -0,0 +1,149 @@
+# Copyright (C) 2020, Inria
+# GRAPHDECO research group, https://team.inria.fr/graphdeco
+# All rights reserved.
+#
+# This software is free for non-commercial, research and evaluation use
+# under the terms of the LICENSE.md file.
+#
+# For inquiries contact sibr@inria.fr and/or George.Drettakis@inria.fr
+
+
+if(__MSVCsetUserCommand_cmake_INCLUDED__)
+ return()
+else()
+ set(__MSVCsetUserCommand_cmake_INCLUDED__ ON)
+endif()
+
+## Allow to configure the Debugger settings of visual studio
+## Note: Using this command under linux doesn't affect anything
+## On run Debug Windows local : visual will try to load a specific COMMAND with ARGS in the provided WORKING_DIR
+##
+## usage:
+## MSVCsetUserCommand(
+## [COMMAND | [ PATH [FILE ] ] ]
+## ARGS
+## WORKING_DIR
+## )
+##
+## Warning 1 : All arugments () must be passed under quotes
+## Warning 2 : WORKING_DIR path arg have to finish with remain slah '/'
+## Warning 3 : use COMMAND for external app OR PATH (optionaly with FILE) option(s) to set your built/installed/moved target
+##
+## Example 1:
+## include(MSVCsetUserCommand)
+## MSVCsetUserCommand( UnityRenderingPlugin
+## COMMAND "C:/Program Files (x86)/Unity/Editor/Unity.exe"
+## ARGS "-force-opengl -projectPath \"${CMAKE_HOME_DIRECTORY}/UnityPlugins/RenderingPluginExample/UnityProject\""
+## WORKING_DIR "${CMAKE_HOME_DIRECTORY}/UnityPlugins/RenderingPluginExample/UnityProject"
+## VERBOSE
+## )
+##
+## Example 2:
+## include(MSVCsetUserCommand)
+## MSVCsetUserCommand( ibrApp
+## PATH "C:/Program Files (x86)/workspace/IBR/install"
+## FILE "ibrApp${CMAKE_EXECUTABLE_SUFFIX}" ## this option line is optional since the target name didn't change between build and install step
+## ARGS "-path \"${CMAKE_HOME_DIRECTORY}/dataset\""
+## WORKING_DIR "${CMAKE_HOME_DIRECTORY}"
+## VERBOSE
+## )
+##
+function(MSVCsetUserCommand targetName)
+ cmake_parse_arguments(MSVCsuc "VERBOSE" "PATH;FILE;COMMAND;ARGS;WORKING_DIR" "" ${ARGN} )
+
+ ## If no arguments are given, do not create an unecessary .vcxproj.user file
+ set(MSVCsuc_DEFAULT OFF)
+
+ if(MSVCsuc_PATH AND MSVCsuc_DEFAULT)
+ set(MSVCsuc_DEFAULT OFF)
+ endif()
+
+ if(MSVCsuc_FILE AND MSVCsuc_DEFAULT)
+ set(MSVCsuc_DEFAULT OFF)
+ endif()
+
+ if(NOT MSVCsuc_COMMAND)
+ if(MSVCsuc_PATH AND MSVCsuc_FILE)
+ set(MSVCsuc_COMMAND "${MSVCsuc_PATH}\\${MSVCsuc_FILE}")
+ elseif(MSVCsuc_PATH)
+ set(MSVCsuc_COMMAND "${MSVCsuc_PATH}\\$(TargetFileName)")
+ else()
+ set(MSVCsuc_COMMAND "$(TargetPath)") ## => $(TargetDir)\$(TargetName)$(TargetExt)
+ endif()
+ elseif(MSVCsuc_DEFAULT)
+ set(MSVCsuc_DEFAULT OFF)
+ endif()
+
+ # NOTE: there was a typo here. there is an else if written after else statement
+ # changing the order of the else if statement
+ if(MSVCsuc_WORKING_DIR)
+ file(TO_NATIVE_PATH ${MSVCsuc_WORKING_DIR} MSVCsuc_WORKING_DIR)
+ elseif(MSVCsuc_DEFAULT)
+ set(MSVCsuc_DEFAULT OFF)
+ else()
+ set(MSVCsuc_WORKING_DIR "$(ProjectDir)")
+ endif()
+
+ if(NOT MSVCsuc_ARGS)
+ set(MSVCsuc_ARGS "")
+ elseif(MSVCsuc_DEFAULT)
+ set(MSVCsuc_DEFAULT OFF)
+ endif()
+
+ if(MSVC10 OR (MSVC AND MSVC_VERSION GREATER 1600)) # 2010 or newer
+
+ if(CMAKE_SIZEOF_VOID_P EQUAL 8)
+ set(PLATEFORM_BITS x64)
+ else()
+ set(PLATEFORM_BITS Win32)
+ endif()
+
+ if(NOT MSVCsuc_DEFAULT AND PLATEFORM_BITS)
+
+ file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/${targetName}.vcxproj.user"
+ "
+
+
+ ${MSVCsuc_COMMAND}
+ ${MSVCsuc_ARGS}
+ WindowsLocalDebugger
+ ${MSVCsuc_WORKING_DIR}
+
+
+ ${MSVCsuc_COMMAND}
+ ${MSVCsuc_ARGS}
+ WindowsLocalDebugger
+ ${MSVCsuc_WORKING_DIR}
+
+
+ ${MSVCsuc_COMMAND}
+ ${MSVCsuc_ARGS}
+ WindowsLocalDebugger
+ ${MSVCsuc_WORKING_DIR}
+
+
+ ${MSVCsuc_COMMAND}
+ ${MSVCsuc_ARGS}
+ WindowsLocalDebugger
+ ${MSVCsuc_WORKING_DIR}
+
+ "
+ )
+ if(MSVCsuc_VERBOSE)
+ message(STATUS "[MSVCsetUserCommand] Write ${CMAKE_CURRENT_BINARY_DIR}/${targetName}.vcxproj.user file")
+ message(STATUS " to execute ${MSVCsuc_COMMAND} ${MSVCsuc_ARGS}")
+ message(STATUS " from derectory ${MSVCsuc_WORKING_DIR}")
+ message(STATUS " on visual studio run debugger button")
+ endif()
+
+ else()
+ message(WARNING "PLATEFORM_BITS is undefined...")
+ endif()
+
+ else()
+ if(MSVCsuc_VERBOSE)
+ message(WARNING "MSVCsetUserCommand is disable because too old MSVC is used (need MSVC10 2010 or newer)")
+ endif()
+ endif()
+
+endfunction()
diff --git a/submodules/gaussian-splatting/SIBR_viewers/cmake/linux/Modules/FindASSIMP.cmake b/submodules/gaussian-splatting/SIBR_viewers/cmake/linux/Modules/FindASSIMP.cmake
new file mode 100644
index 0000000000000000000000000000000000000000..edfbb33b78e3cd57bc0fc2c1c6e2a349c32a5bb7
--- /dev/null
+++ b/submodules/gaussian-splatting/SIBR_viewers/cmake/linux/Modules/FindASSIMP.cmake
@@ -0,0 +1,114 @@
+# Copyright (C) 2020, Inria
+# GRAPHDECO research group, https://team.inria.fr/graphdeco
+# All rights reserved.
+#
+# This software is free for non-commercial, research and evaluation use
+# under the terms of the LICENSE.md file.
+#
+# For inquiries contact sibr@inria.fr and/or George.Drettakis@inria.fr
+
+
+## Try to find the ASSIMP library
+## Once done this will define
+##
+## ASSIMP_FOUND - system has ASSIMP
+## ASSIMP_INCLUDE_DIR - The ASSIMP include directory
+## ASSIMP_LIBRARIES - The libraries needed to use ASSIMP
+## ASSIMP_CMD - the full path of ASSIMP executable
+## ASSIMP_DYNAMIC_LIB - the Assimp dynamic lib (available only on windows as .dll file for the moment)
+##
+## Edited for using a bugfixed version of Assimp
+
+if(NOT ASSIMP_DIR)
+ set(ASSIMP_DIR "$ENV{ASSIMP_DIR}" CACHE PATH "ASSIMP root directory")
+ message("NO ASSIMP DIR " ASSIMP_DIR )
+ file(TO_CMAKE_PATH "/data/graphdeco/share/usr/local" ASSIMP_DIR)
+ set(ASSIMP_DIR "/data/graphdeco/share/usr/local" )
+ message("SETTING ASSIMP DIR " ASSIMP_DIR )
+endif()
+if(ASSIMP_DIR)
+ file(TO_CMAKE_PATH ${ASSIMP_DIR} ASSIMP_DIR)
+ file(TO_CMAKE_PATH "/data/graphdeco/share/usr/local" ASSIMP_DIR)
+ message("ASSIMP DIR " ASSIMP_DIR )
+endif()
+
+
+## set the LIB POSTFIX to find in a right directory according to what kind of compiler we use (32/64bits)
+if(CMAKE_SIZEOF_VOID_P EQUAL 8)
+ set(ASSIMP_SEARCH_LIB "lib64")
+ set(ASSIMP_SEARCH_BIN "bin64")
+ set(ASSIMP_SEARCH_LIB_PATHSUFFIXE "x64")
+else()
+ set(ASSIMP_SEARCH_LIB "lib32")
+ set(ASSIMP_SEARCH_BIN "bin32")
+ set(ASSIMP_SEARCH_LIB_PATHSUFFIXE "x86")
+endif()
+
+set(PROGRAMFILESx86 "PROGRAMFILES(x86)")
+
+
+FIND_PATH(ASSIMP_INCLUDE_DIR
+ NAMES assimp/config.h
+ PATHS
+ ${ASSIMP_DIR}
+ ## linux
+ /usr
+ /usr/include
+ /usr/local
+ /opt/local
+ ## windows
+ "$ENV{PROGRAMFILES}/Assimp"
+ "$ENV{${PROGRAMFILESx86}}/Assimp"
+ "$ENV{ProgramW6432}/Assimp"
+ PATH_SUFFIXES include
+)
+
+
+FIND_LIBRARY(ASSIMP_LIBRARY
+ NAMES assimp-vc140-mt assimp
+ PATHS
+ ${ASSIMP_DIR}/${ASSIMP_SEARCH_LIB}
+ ${ASSIMP_DIR}/lib
+ ${ASSIMP_DIR}/lib64
+ ## linux
+ /usr/${ASSIMP_SEARCH_LIB}
+ /usr/local/${ASSIMP_SEARCH_LIB}
+ /opt/local/${ASSIMP_SEARCH_LIB}
+ /usr/lib
+ /usr/lib64
+ /usr/local/lib
+ /opt/local/lib
+ ## windows
+ "$ENV{PROGRAMFILES}/Assimp/${ASSIMP_SEARCH_LIB}"
+ "$ENV{${PROGRAMFILESx86}}/Assimp/${ASSIMP_SEARCH_LIB}"
+ "$ENV{ProgramW6432}/Assimp/${ASSIMP_SEARCH_LIB}"
+ "$ENV{PROGRAMFILES}/Assimp/lib"
+ "$ENV{${PROGRAMFILESx86}}/Assimp/lib"
+ "$ENV{ProgramW6432}/Assimp/lib"
+ PATH_SUFFIXES ${ASSIMP_SEARCH_LIB_PATHSUFFIXE}
+)
+set(ASSIMP_LIBRARIES ${ASSIMP_LIBRARY})
+
+
+if(ASSIMP_LIBRARY)
+ get_filename_component(ASSIMP_LIBRARY_DIR ${ASSIMP_LIBRARY} PATH)
+ if(WIN32)
+ file(GLOB ASSIMP_DYNAMIC_LIB "${ASSIMP_LIBRARY_DIR}/assimp*.dll")
+ if(NOT ASSIMP_DYNAMIC_LIB)
+ message("ASSIMP_DYNAMIC_LIB is missing... at ${ASSIMP_LIBRARY_DIR}")
+ endif()
+ endif()
+ set(ASSIMP_DYNAMIC_LIB ${ASSIMP_DYNAMIC_LIB} CACHE PATH "Windows dll location")
+endif()
+
+MARK_AS_ADVANCED(ASSIMP_DYNAMIC_LIB ASSIMP_INCLUDE_DIR ASSIMP_LIBRARIES)
+
+INCLUDE(FindPackageHandleStandardArgs)
+FIND_PACKAGE_HANDLE_STANDARD_ARGS(ASSIMP
+ REQUIRED_VARS ASSIMP_INCLUDE_DIR ASSIMP_LIBRARIES
+ FAIL_MESSAGE "ASSIMP wasn't found correctly. Set ASSIMP_DIR to the root SDK installation directory."
+)
+
+if(NOT ASSIMP_FOUND)
+ set(ASSIMP_DIR "" CACHE STRING "Path to ASSIMP install directory")
+endif()
diff --git a/submodules/gaussian-splatting/SIBR_viewers/cmake/linux/Modules/FindEGL.cmake b/submodules/gaussian-splatting/SIBR_viewers/cmake/linux/Modules/FindEGL.cmake
new file mode 100644
index 0000000000000000000000000000000000000000..41d45cb08d09903887d17f564c6ab50e34723e28
--- /dev/null
+++ b/submodules/gaussian-splatting/SIBR_viewers/cmake/linux/Modules/FindEGL.cmake
@@ -0,0 +1,161 @@
+#.rst:
+# FindEGL
+# -------
+#
+# Try to find EGL.
+#
+# This will define the following variables:
+#
+# ``EGL_FOUND``
+# True if (the requested version of) EGL is available
+# ``EGL_VERSION``
+# The version of EGL; note that this is the API version defined in the
+# headers, rather than the version of the implementation (eg: Mesa)
+# ``EGL_LIBRARIES``
+# This can be passed to target_link_libraries() instead of the ``EGL::EGL``
+# target
+# ``EGL_INCLUDE_DIRS``
+# This should be passed to target_include_directories() if the target is not
+# used for linking
+# ``EGL_DEFINITIONS``
+# This should be passed to target_compile_options() if the target is not
+# used for linking
+#
+# If ``EGL_FOUND`` is TRUE, it will also define the following imported target:
+#
+# ``EGL::EGL``
+# The EGL library
+#
+# In general we recommend using the imported target, as it is easier to use.
+# Bear in mind, however, that if the target is in the link interface of an
+# exported library, it must be made available by the package config file.
+#
+# Since pre-1.0.0.
+
+#=============================================================================
+# Copyright 2014 Alex Merry
+# Copyright 2014 Martin Gräßlin
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# 1. Redistributions of source code must retain the copyright
+# notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+# 3. The name of the author may not be used to endorse or promote products
+# derived from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#=============================================================================
+
+include(CheckCXXSourceCompiles)
+include(CMakePushCheckState)
+
+# Use pkg-config to get the directories and then use these values
+# in the FIND_PATH() and FIND_LIBRARY() calls
+find_package(PkgConfig)
+pkg_check_modules(PKG_EGL QUIET egl)
+
+set(EGL_DEFINITIONS ${PKG_EGL_CFLAGS_OTHER})
+
+find_path(EGL_INCLUDE_DIR
+ NAMES
+ EGL/egl.h
+ HINTS
+ ${PKG_EGL_INCLUDE_DIRS}
+)
+find_library(EGL_LIBRARY
+ NAMES
+ EGL
+ HINTS
+ ${PKG_EGL_LIBRARY_DIRS}
+)
+
+# NB: We do *not* use the version information from pkg-config, as that
+# is the implementation version (eg: the Mesa version)
+if(EGL_INCLUDE_DIR)
+ # egl.h has defines of the form EGL_VERSION_x_y for each supported
+ # version; so the header for EGL 1.1 will define EGL_VERSION_1_0 and
+ # EGL_VERSION_1_1. Finding the highest supported version involves
+ # finding all these defines and selecting the highest numbered.
+ file(READ "${EGL_INCLUDE_DIR}/EGL/egl.h" _EGL_header_contents)
+ string(REGEX MATCHALL
+ "[ \t]EGL_VERSION_[0-9_]+"
+ _EGL_version_lines
+ "${_EGL_header_contents}"
+ )
+ unset(_EGL_header_contents)
+ foreach(_EGL_version_line ${_EGL_version_lines})
+ string(REGEX REPLACE
+ "[ \t]EGL_VERSION_([0-9_]+)"
+ "\\1"
+ _version_candidate
+ "${_EGL_version_line}"
+ )
+ string(REPLACE "_" "." _version_candidate "${_version_candidate}")
+ if(NOT DEFINED EGL_VERSION OR EGL_VERSION VERSION_LESS _version_candidate)
+ set(EGL_VERSION "${_version_candidate}")
+ endif()
+ endforeach()
+ unset(_EGL_version_lines)
+endif()
+
+cmake_push_check_state(RESET)
+list(APPEND CMAKE_REQUIRED_LIBRARIES "${EGL_LIBRARY}")
+list(APPEND CMAKE_REQUIRED_INCLUDES "${EGL_INCLUDE_DIR}")
+
+check_cxx_source_compiles("
+#include
+
+int main(int argc, char *argv[]) {
+ EGLint x = 0; EGLDisplay dpy = 0; EGLContext ctx = 0;
+ eglDestroyContext(dpy, ctx);
+}" HAVE_EGL)
+
+cmake_pop_check_state()
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(EGL
+ FOUND_VAR
+ EGL_FOUND
+ REQUIRED_VARS
+ EGL_LIBRARY
+ EGL_INCLUDE_DIR
+ HAVE_EGL
+ VERSION_VAR
+ EGL_VERSION
+)
+
+if(EGL_FOUND AND NOT TARGET EGL::EGL)
+ add_library(EGL::EGL UNKNOWN IMPORTED)
+ set_target_properties(EGL::EGL PROPERTIES
+ IMPORTED_LOCATION "${EGL_LIBRARY}"
+ INTERFACE_COMPILE_OPTIONS "${EGL_DEFINITIONS}"
+ INTERFACE_INCLUDE_DIRECTORIES "${EGL_INCLUDE_DIR}"
+ )
+endif()
+
+mark_as_advanced(EGL_LIBRARY EGL_INCLUDE_DIR HAVE_EGL)
+
+# compatibility variables
+set(EGL_LIBRARIES ${EGL_LIBRARY})
+set(EGL_INCLUDE_DIRS ${EGL_INCLUDE_DIR})
+set(EGL_VERSION_STRING ${EGL_VERSION})
+
+include(FeatureSummary)
+set_package_properties(EGL PROPERTIES
+ URL "https://www.khronos.org/egl/"
+ DESCRIPTION "A platform-agnostic mechanism for creating rendering surfaces for use with other graphics libraries, such as OpenGL|ES and OpenVG."
+)
diff --git a/submodules/gaussian-splatting/SIBR_viewers/cmake/linux/Modules/FindEmbree.cmake b/submodules/gaussian-splatting/SIBR_viewers/cmake/linux/Modules/FindEmbree.cmake
new file mode 100644
index 0000000000000000000000000000000000000000..0d07237f6670f3b27592dbdaecf883a531b8c346
--- /dev/null
+++ b/submodules/gaussian-splatting/SIBR_viewers/cmake/linux/Modules/FindEmbree.cmake
@@ -0,0 +1,94 @@
+# Copyright (C) 2020, Inria
+# GRAPHDECO research group, https://team.inria.fr/graphdeco
+# All rights reserved.
+#
+# This software is free for non-commercial, research and evaluation use
+# under the terms of the LICENSE.md file.
+#
+# For inquiries contact sibr@inria.fr and/or George.Drettakis@inria.fr
+
+## Important Note:
+## This is not an official Find*cmake. It has been written for searching through
+## a custom path (EMBREE_DIR) before checking elsewhere.
+##
+## FindEMBREE.cmake
+## Find EMBREE's includes and library
+##
+## This module defines :
+## [in] EMBREE_DIR, The base directory to search for EMBREE (as cmake var or env var)
+## [out] EMBREE_INCLUDE_DIR where to find EMBREE.h
+## [out] EMBREE_LIBRARIES, EMBREE_LIBRARY, libraries to link against to use EMBREE
+## [out] EMBREE_FOUND, If false, do not try to use EMBREE.
+##
+
+
+if(NOT EMBREE_DIR)
+ set(EMBREE_DIR "$ENV{EMBREE_DIR}" CACHE PATH "EMBREE root directory")
+endif()
+if(EMBREE_DIR)
+ file(TO_CMAKE_PATH ${EMBREE_DIR} EMBREE_DIR)
+endif()
+
+
+## set the LIB POSTFIX to find in a right directory according to what kind of compiler we use (32/64bits)
+if(CMAKE_SIZEOF_VOID_P EQUAL 8)
+ set(EMBREE_SEARCH_LIB "lib64")
+ set(EMBREE_SEARCH_BIN "bin64")
+ set(EMBREE_SEARCH_LIB_PATHSUFFIXE "x64")
+else()
+ set(EMBREE_SEARCH_LIB "lib32")
+ set(EMBREE_SEARCH_BIN "bin32")
+ set(EMBREE_SEARCH_LIB_PATHSUFFIXE "x86")
+endif()
+
+set(PROGRAMFILESx86 "PROGRAMFILES(x86)")
+
+FIND_PATH(EMBREE_INCLUDE_DIR
+ NAMES embree3/rtcore_geometry.h
+ PATHS
+ ${EMBREE_DIR}
+ ## linux
+ /usr
+ /usr/local
+ /opt/local
+ ## windows
+ "$ENV{PROGRAMFILES}/EMBREE"
+ "$ENV{${PROGRAMFILESx86}}/EMBREE"
+ "$ENV{ProgramW6432}/EMBREE"
+ PATH_SUFFIXES include
+)
+
+FIND_LIBRARY(EMBREE_LIBRARY
+ NAMES embree3
+ PATHS
+ ${EMBREE_DIR}/${EMBREE_SEARCH_LIB}
+ ${EMBREE_DIR}/lib
+ ## linux
+ /usr/${EMBREE_SEARCH_LIB}
+ /usr/local/${EMBREE_SEARCH_LIB}
+ /opt/local/${EMBREE_SEARCH_LIB}
+ /usr/lib
+ /usr/local/lib
+ /opt/local/lib
+ ## windows
+ "$ENV{PROGRAMFILES}/EMBREE/${EMBREE_SEARCH_LIB}"
+ "$ENV{${PROGRAMFILESx86}}/EMBREE/${EMBREE_SEARCH_LIB}"
+ "$ENV{ProgramW6432}/EMBREE/${EMBREE_SEARCH_LIB}"
+ "$ENV{PROGRAMFILES}/EMBREE/lib"
+ "$ENV{${PROGRAMFILESx86}}/EMBREE/lib"
+ "$ENV{ProgramW6432}/EMBREE/lib"
+ PATH_SUFFIXES ${EMBREE_SEARCH_LIB_PATHSUFFIXE}
+)
+set(EMBREE_LIBRARIES ${EMBREE_LIBRARY})
+
+MARK_AS_ADVANCED(EMBREE_INCLUDE_DIR EMBREE_LIBRARIES)
+
+INCLUDE(FindPackageHandleStandardArgs)
+FIND_PACKAGE_HANDLE_STANDARD_ARGS(EMBREE
+ REQUIRED_VARS EMBREE_INCLUDE_DIR EMBREE_LIBRARIES
+ FAIL_MESSAGE "EMBREE wasn't found correctly. Set EMBREE_DIR to the root SDK installation directory."
+)
+
+if(NOT EMBREE_FOUND)
+ set(EMBREE_DIR "" CACHE STRING "Path to EMBREE install directory")
+endif()
diff --git a/submodules/gaussian-splatting/SIBR_viewers/cmake/linux/Modules/FindFFMPEG.cmake b/submodules/gaussian-splatting/SIBR_viewers/cmake/linux/Modules/FindFFMPEG.cmake
new file mode 100644
index 0000000000000000000000000000000000000000..e60cee8d812f574226c71ff38ec461872f7e63d1
--- /dev/null
+++ b/submodules/gaussian-splatting/SIBR_viewers/cmake/linux/Modules/FindFFMPEG.cmake
@@ -0,0 +1,110 @@
+# Copyright (C) 2020, Inria
+# GRAPHDECO research group, https://team.inria.fr/graphdeco
+# All rights reserved.
+#
+# This software is free for non-commercial, research and evaluation use
+# under the terms of the LICENSE.md file.
+#
+# For inquiries contact sibr@inria.fr and/or George.Drettakis@inria.fr
+
+
+## Try to find the FFMPEG library
+## Once done this will define
+##
+## FFMPEG_FOUND - system has FFmpeg
+## FFMPEG_INCLUDE_DIR - The FFmpeg include directory
+## FFMPEG_LIBRARIES - The libraries needed to use FFmpeg
+## FFMPEG_DYNAMIC_LIBS - DLLs for windows
+
+
+if(NOT FFMPEG_DIR)
+ set(FFMPEG_DIR "$ENV{FFMPEG_DIR}" CACHE PATH "FFMPEG_DIR root directory")
+endif()
+
+if(FFMPEG_DIR)
+ file(TO_CMAKE_PATH ${FFMPEG_DIR} FFMPEG_DIR)
+endif()
+
+MACRO(FFMPEG_FIND varname shortname headername)
+
+ # Path to include dirs
+ FIND_PATH(FFMPEG_${varname}_INCLUDE_DIRS
+ NAMES "lib${shortname}/${headername}"
+ PATHS
+ "${FFMPEG_DIR}/include" # modify this to adapt according to OS/compiler
+ "/usr/include"
+ "/usr/include/ffmpeg"
+ )
+
+ #Add libraries
+ IF(${FFMPEG_${varname}_INCLUDE_DIRS} STREQUAL "FFMPEG_${varname}_INCLUDE_DIR-NOTFOUND")
+ MESSAGE(STATUS "Can't find includes for ${shortname}...")
+ ELSE()
+ FIND_LIBRARY(FFMPEG_${varname}_LIBRARIES
+ NAMES ${shortname}
+ PATHS
+ ${FFMPEG_DIR}/lib
+ "/usr/lib"
+ "/usr/lib64"
+ "/usr/local/lib"
+ "/usr/local/lib64"
+ )
+
+ # set libraries and other variables
+ SET(FFMPEG_${varname}_FOUND 1)
+ SET(FFMPEG_${varname}_INCLUDE_DIRS ${FFMPEG_${varname}_INCLUDE_DIR})
+ SET(FFMPEG_${varname}_LIBS ${FFMPEG_${varname}_LIBRARIES})
+ ENDIF()
+ ENDMACRO(FFMPEG_FIND)
+
+#Calls to ffmpeg_find to get librarires ------------------------------
+FFMPEG_FIND(LIBAVFORMAT avformat avformat.h)
+FFMPEG_FIND(LIBAVDEVICE avdevice avdevice.h)
+FFMPEG_FIND(LIBAVCODEC avcodec avcodec.h)
+FFMPEG_FIND(LIBAVUTIL avutil avutil.h)
+FFMPEG_FIND(LIBSWSCALE swscale swscale.h)
+
+# check if libs are found and set FFMPEG related variables
+#SET(FFMPEG_FOUND "NO")
+IF(FFMPEG_LIBAVFORMAT_FOUND
+ AND FFMPEG_LIBAVDEVICE_FOUND
+ AND FFMPEG_LIBAVCODEC_FOUND
+ AND FFMPEG_LIBAVUTIL_FOUND
+ AND FFMPEG_LIBSWSCALE_FOUND)
+
+ # All ffmpeg libs are here
+ SET(FFMPEG_FOUND "YES")
+ SET(FFMPEG_INCLUDE_DIR ${FFMPEG_LIBAVFORMAT_INCLUDE_DIRS})
+ SET(FFMPEG_LIBRARY_DIRS ${FFMPEG_LIBAVFORMAT_LIBRARY_DIRS})
+ SET(FFMPEG_LIBRARIES
+ ${FFMPEG_LIBAVFORMAT_LIBS}
+ ${FFMPEG_LIBAVDEVICE_LIBS}
+ ${FFMPEG_LIBAVCODEC_LIBS}
+ ${FFMPEG_LIBAVUTIL_LIBS}
+ ${FFMPEG_LIBSWSCALE_LIBS} )
+
+ # add dynamic libraries
+ if(WIN32)
+ file(GLOB FFMPEG_DYNAMIC_LIBS "${FFMPEG_DIR}/bin/*.dll")
+ if(NOT FFMPEG_DYNAMIC_LIBS)
+ message("FFMPEG_DYNAMIC_LIBS is missing...")
+ endif()
+ set(FFMPEG_DYNAMIC_LIBS ${FFMPEG_DYNAMIC_LIBS} CACHE PATH "Windows dll location")
+endif()
+
+ mark_as_advanced(FFMPEG_INCLUDE_DIR FFMPEG_LIBRARY_DIRS FFMPEG_LIBRARIES FFMPEG_DYNAMIC_LIBS)
+ELSE ()
+ MESSAGE(STATUS "Could not find FFMPEG")
+ENDIF()
+
+
+INCLUDE(FindPackageHandleStandardArgs)
+FIND_PACKAGE_HANDLE_STANDARD_ARGS(FFMPEG
+ REQUIRED_VARS FFMPEG_INCLUDE_DIR FFMPEG_LIBRARIES
+ FAIL_MESSAGE "FFmpeg wasn't found correctly. Set FFMPEG_DIR to the root SDK installation directory."
+)
+
+if(NOT FFMPEG_FOUND)
+ set(FFMPEG_DIR "" CACHE STRING "Path to FFmpeg install directory")
+endif()
+
diff --git a/submodules/gaussian-splatting/SIBR_viewers/cmake/linux/Modules/FindGLFW.cmake b/submodules/gaussian-splatting/SIBR_viewers/cmake/linux/Modules/FindGLFW.cmake
new file mode 100644
index 0000000000000000000000000000000000000000..14263de4365a6ac72b55d874373a589e3d604933
--- /dev/null
+++ b/submodules/gaussian-splatting/SIBR_viewers/cmake/linux/Modules/FindGLFW.cmake
@@ -0,0 +1,109 @@
+##=============================================================================
+##
+## Copyright (c) Kitware, Inc.
+## All rights reserved.
+## See LICENSE.txt for details.
+##
+## This software is distributed WITHOUT ANY WARRANTY; without even
+## the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+## PURPOSE. See the above copyright notice for more information.
+##
+## Copyright 2016 Sandia Corporation.
+## Copyright 2016 UT-Battelle, LLC.
+## Copyright 2016 Los Alamos National Security.
+##
+## Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
+## the U.S. Government retains certain rights in this software.
+## Under the terms of Contract DE-AC52-06NA25396 with Los Alamos National
+## Laboratory (LANL), the U.S. Government retains certain rights in
+## this software.
+##
+##=============================================================================
+# Try to find EGL library and include dir.
+# Once done this will define
+#
+# GLFW_FOUND
+# GLFW_INCLUDE_DIR
+# GLFW_LIBRARY
+#
+
+include(FindPackageHandleStandardArgs)
+
+if (WIN32)
+ find_path( GLFW_INCLUDE_DIR
+ NAMES
+ GLFW/glfw3.h
+ PATHS
+ ${PROJECT_SOURCE_DIR}/shared_external/glfw/include
+ ${PROJECT_SOURCE_DIR}/../shared_external/glfw/include
+ ${GLFW_LOCATION}/include
+ $ENV{GLFW_LOCATION}/include
+ $ENV{PROGRAMFILES}/GLFW/include
+ ${GLFW_LOCATION}
+ $ENV{GLFW_LOCATION}
+ DOC "The directory where GLFW/glfw3.h resides" )
+ if(ARCH STREQUAL "x86")
+ find_library( GLFW_LIBRARY
+ NAMES
+ glfw3
+ PATHS
+ ${GLFW_LOCATION}/lib
+ $ENV{GLFW_LOCATION}/lib
+ $ENV{PROGRAMFILES}/GLFW/lib
+ DOC "The GLFW library")
+ else()
+ find_library( GLFW_LIBRARY
+ NAMES
+ glfw3
+ PATHS
+ ${GLFW_LOCATION}/lib
+ $ENV{GLFW_LOCATION}/lib
+ $ENV{PROGRAMFILES}/GLFW/lib
+ DOC "The GLFW library")
+ endif()
+endif ()
+
+if (${CMAKE_HOST_UNIX})
+ message("GFLW LOCATION " $ENV{GLFW_LOCATION} )
+ find_path( GLFW_INCLUDE_DIR
+ NAMES
+ GLFW/glfw3.h
+ PATHS
+# ${GLFW_LOCATION}/include
+ $ENV{GLFW_LOCATION}/include
+# /usr/include
+# /usr/local/include
+# /sw/include
+# /opt/local/include
+# NO_DEFAULT_PATH
+ DOC "The directory where GLFW/glfw3.h resides"
+ )
+ find_library( GLFW_LIBRARY
+ NAMES
+ glfw3 glfw
+ PATHS
+# ${GLFW_LOCATION}/lib
+ $ENV{GLFW_LOCATION}/lib
+ $ENV{GLFW_LOCATION}/lib64
+# /usr/lib64
+# /usr/lib
+# /usr/local/lib64
+# /usr/local/lib
+# /sw/lib
+# /opt/local/lib
+# /usr/lib/x86_64-linux-gnu
+# NO_DEFAULT_PATH
+ DOC "The GLFW library")
+
+ set( GLFW_INCLUDE_DIR $ENV{GLFW_LOCATION}/include )
+ set( GLFW_LIBRARY $ENV{GLFW_LOCATION}/lib64/libglfw3.a )
+ message("*************==========> FindGLFW .cmake " ${GLFW_INCLUDE_DIR} " LIB " ${GLFW_LIBRARY} )
+endif ()
+
+find_package_handle_standard_args(GLFW DEFAULT_MSG
+ GLFW_INCLUDE_DIR
+ GLFW_LIBRARY
+)
+
+mark_as_advanced( GLFW_FOUND )
+
diff --git a/submodules/gaussian-splatting/SIBR_viewers/cmake/linux/Win3rdParty.cmake b/submodules/gaussian-splatting/SIBR_viewers/cmake/linux/Win3rdParty.cmake
new file mode 100644
index 0000000000000000000000000000000000000000..7e42fbb9f4353c2208ed3d6f44cf7acc3fccedc2
--- /dev/null
+++ b/submodules/gaussian-splatting/SIBR_viewers/cmake/linux/Win3rdParty.cmake
@@ -0,0 +1,337 @@
+# Copyright (C) 2020, Inria
+# GRAPHDECO research group, https://team.inria.fr/graphdeco
+# All rights reserved.
+#
+# This software is free for non-commercial, research and evaluation use
+# under the terms of the LICENSE.md file.
+#
+# For inquiries contact sibr@inria.fr and/or George.Drettakis@inria.fr
+
+
+## This file should be include and use only on WIN32 OS and once
+## It allow to auto check/download and use a preconfigured 3rdParty binaries for cmake usage
+## It use the downloadAndExtractZipFile cmake module to work.
+##
+if(__Win3rdParty_cmake_INCLUDED__)
+ return()
+else()
+ set(__Win3rdParty_cmake_INCLUDED__ ON)
+endif()
+
+
+##
+## To be sure to reset an empty cached variable but keep any other kind of variables
+##
+## Usage:
+## check_cached_var( [FORCE])
+##
+## is the cached cmake variable you need to reset
+## is the new default value of the reseted cached cmake variable
+## is the kind of GUI cache input can be : FILEPATH; PATH; STRING or BOOL
+## is the associated GUI cache input documentation display in the GUI
+## FORCE option could be use to reset a cached variable even if it is not empty.
+##
+macro(check_cached_var var resetedCachedValue cacheType cacheDoc)
+ # message(STATUS "inside check_cached_var macro. argn=${ARGN}")
+ cmake_parse_arguments(ccv "FORCE" "" "" ${ARGN})
+
+ if(ccv_FORCE)
+ set(FORCE FORCE)
+ else()
+ set(FORCE )
+ endif()
+
+ if(NOT ${var} OR ccv_FORCE)
+ unset(${var} CACHE)
+ # message(STATUS "setting new cache value. var ${var} = ${resetedCachedValue}")
+ set(${var} "${resetedCachedValue}" CACHE ${cacheType} "${cacheDoc}" ${FORCE})
+ endif()
+endmacro()
+
+
+##
+## Win3rdParty function allow to specify a directory which contain all necessary windows dependenties.
+## By uploading 3rdParty directory (which contain dependencies, *.lib, *.dll... for a specific version of compiler) onto Gforge file tab,
+## you get back an URL of download you can give to this function with a directory name. So you can provide multiple 3rdParty version of same dependencies (MSVC11, MSVC12...).
+## By providing a prefix to this function, you allow to use different kind of 3rdParty which can be handled by CMAKE OPTIONS depending on what your framework need for example.
+##
+## Usage 1:
+## Win3rdParty( MSVC
+## [MSVC ] [...]
+## [VCID] [DEFAULT_USE] [VERBOSE] )
+##
+## * allow to identify which 3rdParty you process (prefix name)
+## * MSVC flag could be MSVC11 or MSVC12 (any element of the MSVC_VERSIONS_LIST) and refer to a 3rdParty compiler with :
+## * which will be the local pathName of the downloaded 3rdParty : relative to CMAKE_BINARY_DIR
+## * which is the link location of the 3rdParty zip
+## * VCID flag will make available a cache variable ${prefix}_WIN3RDPARTY_VCID
+## * DEFAULT_USE flag [ON|OFF] may be used to set default value of cmake cached variable : _WIN3RDPARTY_USE [default to ON]
+##
+## WARNING:
+## This function define CACHE variables you can use after :
+## * ${prefix}_WIN3RDPARTY_USE : allow to check/downloaded win3rdParty dir (it will force the cached variables for this dependency folder generally _DIR>)
+## * ${prefix}_WIN3RDPARTY_DIR : where is your local win3rdParty dir (the PATH)
+## * ${prefix}_WIN3RDPARTY_VCID : [if VCID flag is used] the MSVC id (commonly used to prefix/suffix library name, see boost or CGAL)
+##
+## If you want to add a win3rdParty version, please:
+## 1- build dependencies on your local side with the compiler you want
+## 2- build your own zip with your built dependencies
+## 3- upload it (onto the forge where the project is stored) and copy the link location in order to use it for this function
+## 4- if you just introduced a new MSVC version, add it to the MSVC_VERSIONS_LIST bellow
+##
+## In a second pass, you can also use this function to set necessary cmake cached variables in order to let cmake find packages of these 3rdParty.
+##
+## Usage 2:
+## win3rdParty( [VERBOSE] MULTI_SET|SET
+## CHECK_CACHED_VAR [LIST] [DOC ]
+## [ CHECK_CACHED_VAR [LIST] [DOC ] ] [...]
+##
+## * MULTI_SET or SET flags are used to tell cmake that all next arguments will use repeated flags with differents entries (SET mean we will provide only one set of arguments, without repetition)
+## * CHECK_CACHED_VAR are the repeated flag which contain differents entries
+## * is the cmake variable you want to be cached for the project
+## * is the kind of cmake variable (couble be: FILEPATH; PATH; STRING or BOOL) => see check_cached_var.
+## * LIST optional flag could be used with CHECK_CACHED_VAR when = STRING. It allow to handle multiple STRINGS value list.
+## * is the value of the variable (if FILEPATH, PATH or STRING: use quotes, if BOOL : use ON/OFF)
+## * DOC optional flag is used to have a tooltips info about this new cmake variable entry into the GUI (use quotes).
+##
+## Full example 1 :
+## win3rdParty(COMMON MSVC11 "win3rdParty-MSVC11" "https://path.to/an.archive.7z"
+## SET CHECK_CACHED_VAR SuiteSparse_DIR PATH "SuiteSparse-4.2.1" DOC "default empty doc"
+## )
+##
+## WARNING:
+## For the 2nd usage (with MULTI_SET), if you planned to set some CACHED_VAR using/composed by ${prefix}_WIN3RDPARTY_* just set in this macro (usage 1),
+## then (due to the not yet existing var) you will need to call this function 2 times :
+## One for the 1st usage (downloading of the current compiler 3rdParty).
+## One for the MLUTI_SET flag which will use existsing ${prefix}_WIN3RDPARTY_* cached var.
+##
+## Full example 2 :
+## win3rdParty(COMMON MSVC11 "win3rdParty-MSVC11" "https://path.to/an.archive.7z")
+## win3rdParty(COMMON MULTI_SET
+## CHECK_CACHED_VAR CGAL_INCLUDE_DIR PATH "CGAL-4.3/include" DOC "default empty doc"
+## CHECK_CACHED_VAR CGAL_LIBRARIES STRING LIST "debug;CGAL-4.3/lib${LIB_POSTFIX}/CGAL-${WIN3RDPARTY_COMMON_VCID}-mt-gd-4.3.lib;optimized;CGAL-4.3/lib${LIB_POSTFIX}/CGAL-${WIN3RDPARTY_COMMON_VCID}-mt-4.3.lib"
+##
+##
+## WARNING: This function use internaly :
+## * downloadAndExtractZipFile.cmake
+## * parse_arguments_multi.cmake
+## * check_cached_var macro
+##
+function(win3rdParty prefix )
+
+ # ARGV: list of all arguments given to the macro/function
+ # ARGN: list of remaining arguments
+
+ if(NOT WIN32)
+ return()
+ endif()
+
+ ## set the handled version of MSVC
+ ## if you plan to add a win3rdParty dir to download with a new MSVC version: build the win3rdParty dir and add the MSCV entry here.
+ set(MSVC_VERSIONS_LIST "MSVC17;MSVC11;MSVC12;MSVC14")
+
+ #include(CMakeParseArguments) # CMakeParseArguments is obsolete since cmake 3.5
+ # cmake_parse_arguments ( args)
+ # : options (flags) pass to the macro
+ # : options that neeed a value
+ # : options that neeed more than one value
+ cmake_parse_arguments(w3p "VCID" "VERBOSE;TIMEOUT;DEFAULT_USE" "${MSVC_VERSIONS_LIST};MULTI_SET;SET" ${ARGN})
+
+ # message(STATUS "value of w3p_VCID = ${w3p_VCID}")
+ # message(STATUS "value of w3p_VERBOSE = ${w3p_VERBOSE}")
+ # message(STATUS "value of w3p_TIMEOUT = ${w3p_TIMEOUT}")
+ # message(STATUS "value of w3p_DEFAULT_USE = ${w3p_DEFAULT_USE}")
+
+ # foreach (loop_var ${MSVC_VERSIONS_LIST})
+ # message(STATUS "value of w3p_${loop_var} = ${w3p_${loop_var}}")
+ # endforeach(loop_var)
+
+ # message(STATUS "value of w3p_MULTI_SET = ${w3p_MULTI_SET}")
+ # message(STATUS "value of w3p_SET = ${w3p_SET}")
+
+ # message("values for MSVC = ${w3p_MSVC14}")
+
+ if(NOT w3p_TIMEOUT)
+ set(w3p_TIMEOUT 300)
+ endif()
+
+ if(NOT DEFINED w3p_DEFAULT_USE)
+ set(w3p_DEFAULT_USE ON)
+ endif()
+
+
+ ## 1st use (check/update|download) :
+ set(${prefix}_WIN3RDPARTY_USE ${w3p_DEFAULT_USE} CACHE BOOL "Use required 3rdParty binaries from ${prefix}_WIN3RDPARTY_DIR or download it if not exist")
+
+
+ ## We want to test if each version of MSVC was filled by the function (see associated parameters)
+ ## As CMake is running only for one version of MSVC, if that MSVC version was filled, we get back associated parameters,
+ ## otherwise we can't use the downloadAndExtractZipFile with win3rdParty.
+ set(enableWin3rdParty OFF)
+
+ foreach(MSVC_VER ${MSVC_VERSIONS_LIST})
+ if(${MSVC_VER} AND w3p_${MSVC_VER} OR ${MSVC_TOOLSET_VERSION} EQUAL 143 AND ${MSVC_VER} STREQUAL "MSVC17")
+ list(LENGTH w3p_${MSVC_VER} count)
+ if("${count}" LESS "2")
+ #message(WARNING "You are using ${MSVC_VER} with ${prefix}_WIN3RDPARTY_USE=${${prefix}_WIN3RDPARTY_USE}, but win3rdParty function isn't filled for ${MSVC_VER}!")
+ else()
+ list(GET w3p_${MSVC_VER} 0 Win3rdPartyName)
+ list(GET w3p_${MSVC_VER} 1 Win3rdPartyUrl)
+ if(w3p_VCID)
+ ## try to get the VcId of MSVC. See also MSVC_VERSION cmake var in the doc.
+ string(REGEX REPLACE "MS([A-Za-z_0-9-]+)" "\\1" vcId ${MSVC_VER})
+ string(TOLOWER ${vcId} vcId)
+ set(${prefix}_WIN3RDPARTY_VCID "${vcId}0" CACHE STRING "the MSVC id (commonly used to prefix/suffix library name, see boost or CGAL)")
+ mark_as_advanced(${prefix}_WIN3RDPARTY_VCID)
+ endif()
+ set(enableWin3rdParty ON)
+ set(suffixCompilerID ${MSVC_VER})
+ break()
+ endif()
+ endif()
+ endforeach()
+ ## If previous step succeed to get MSVC dirname and URL of the current MSVC version, use it to auto download/update the win3rdParty dir
+ if(enableWin3rdParty AND ${prefix}_WIN3RDPARTY_USE)
+
+ if(IS_ABSOLUTE "${Win3rdPartyName}")
+ else()
+ set(Win3rdPartyName "${CMAKE_BINARY_DIR}/${Win3rdPartyName}")
+ endif()
+
+ if(NOT EXISTS "${Win3rdPartyName}")
+ file(MAKE_DIRECTORY ${Win3rdPartyName})
+ endif()
+
+ include(downloadAndExtractZipFile)
+ downloadAndExtractZipFile( "${Win3rdPartyUrl}" ## URL link location
+ "Win3rdParty-${prefix}-${suffixCompilerID}.7z" ## where download it: relative path, so default to CMAKE_BINARY_DIR
+ "${Win3rdPartyName}" ## where extract it : fullPath (default relative to CMAKE_BINARY_DIR)
+ CHECK_DIRTY_URL "${Win3rdPartyName}/Win3rdPartyUrl" ## last downloaded url file : fullPath (default relative to CMAKE_BINARY_DIR)
+ TIMEOUT ${w3p_TIMEOUT}
+ VERBOSE ${w3p_VERBOSE}
+ )
+ file(GLOB checkDl "${Win3rdPartyName}/*")
+ list(LENGTH checkDl checkDlCount)
+ if("${checkDlCount}" GREATER "1")
+ else()
+ message("The downloadAndExtractZipFile didn't work...?")
+ set(enableWin3rdParty OFF)
+ endif()
+ endif()
+
+ ## Try to auto set ${prefix}_WIN3RDPARTY_DIR or let user set it manually
+ set(${prefix}_WIN3RDPARTY_DIR "" CACHE PATH "windows ${Win3rdPartyName} dir to ${prefix} dependencies of the project")
+
+ if(NOT ${prefix}_WIN3RDPARTY_DIR AND ${prefix}_WIN3RDPARTY_USE)
+ if(EXISTS "${Win3rdPartyName}")
+ unset(${prefix}_WIN3RDPARTY_DIR CACHE)
+ set(${prefix}_WIN3RDPARTY_DIR "${Win3rdPartyName}" CACHE PATH "dir to ${prefix} dependencies of the project")
+ endif()
+ endif()
+
+ if(EXISTS ${${prefix}_WIN3RDPARTY_DIR})
+ message(STATUS "Found a 3rdParty ${prefix} dir : ${${prefix}_WIN3RDPARTY_DIR}.")
+ set(enableWin3rdParty ON)
+ elseif(${prefix}_WIN3RDPARTY_USE)
+ message(WARNING "${prefix}_WIN3RDPARTY_USE=${${prefix}_WIN3RDPARTY_USE} but ${prefix}_WIN3RDPARTY_DIR=${${prefix}_WIN3RDPARTY_DIR}.")
+ set(enableWin3rdParty OFF)
+ endif()
+
+ ## Final check
+ if(NOT enableWin3rdParty)
+ message("Disable ${prefix}_WIN3RDPARTY_USE (cmake cached var will be not set), due to a win3rdParty problem.")
+ message("You still can set ${prefix}_WIN3RDPARTY_DIR to an already downloaded Win3rdParty directory location.")
+ set(${prefix}_WIN3RDPARTY_USE OFF CACHE BOOL "Use required 3rdParty binaries from ${prefix}_WIN3RDPARTY_DIR or download it if not exist" FORCE)
+ endif()
+
+ ## 2nd use : handle multi values args to set cached cmake variables in order to ease the next find_package call
+ if(${prefix}_WIN3RDPARTY_USE AND ${prefix}_WIN3RDPARTY_DIR)
+ if(w3p_VERBOSE)
+ message(STATUS "Try to set cmake cached variables for ${prefix} required libraries directly from : ${${prefix}_WIN3RDPARTY_DIR}.")
+ endif()
+
+ include(parse_arguments_multi)
+ # message (STATUS "before defining an override of parse_arguments_multi_function")
+ function(parse_arguments_multi_function ) ## overloaded function to handle all CHECK_CACHED_VAR values list (see: parse_arguments_multi)
+ # message(STATUS "inside overloaded parse_arguments_multi_function defined in Win3rdParty.cmake")
+ # message(STATUS ${ARGN})
+ ## we know the function take 3 args : var cacheType resetedCachedValue (see check_cached_var)
+ cmake_parse_arguments(pamf "" "DOC" "LIST" ${ARGN})
+
+ ## var and cacheType are mandatory (with the resetedCachedValue)
+ set(var ${ARGV0})
+ set(cacheType ${ARGV1})
+ # message(STATUS "var=${var} and cacheType=${cacheType} list=${pamf_LIST}")
+ if(pamf_DOC)
+ set(cacheDoc ${pamf_DOC})
+ else()
+ set(cacheDoc "")
+ endif()
+ if(pamf_LIST)
+ set(value ${pamf_LIST})
+ else()
+ # message("USING ARGV2 with value ${ARGV2}")
+ set(value ${ARGV2})
+ endif()
+ # message("inside override function in Win3rdparty.cmake value+ ${value}")
+ if("${cacheType}" MATCHES "PATH" AND EXISTS "${${prefix}_WIN3RDPARTY_DIR}/${value}")
+ # message("math with path")
+ set(resetedCachedValue "${${prefix}_WIN3RDPARTY_DIR}/${value}") ## path relative to ${prefix}_WIN3RDPARTY_DIR
+ elseif ("${cacheType}" MATCHES "PATH" AND EXISTS "${${prefix}_WIN3RDPARTY_DIR}")
+ set(resetedCachedValue "${${prefix}_WIN3RDPARTY_DIR}") ## path relative to ${prefix}_WIN3RDPARTY_DIR
+ elseif("${cacheType}" MATCHES "STRING")
+ foreach(var IN LISTS value)
+ if(EXISTS "${${prefix}_WIN3RDPARTY_DIR}/${var}")
+ list(APPEND resetedCachedValue "${${prefix}_WIN3RDPARTY_DIR}/${var}") ## string item of the string list is a path => make relative to ${prefix}_WIN3RDPARTY_DIR
+ else()
+ list(APPEND resetedCachedValue ${var}) ## string item of the string list is not an existing path => simply use the item
+ endif()
+ endforeach()
+ else()
+ set(resetedCachedValue "${value}") ## could be a BOOL or a STRING
+ endif()
+
+ ## call our macro to reset cmake cache variable if empty
+ check_cached_var(${var} "${resetedCachedValue}" ${cacheType} "${cacheDoc}" FORCE)
+
+ endfunction()
+ # message (STATUS "after defining an override of parse_arguments_multi_function")
+
+ if(w3p_MULTI_SET)
+ parse_arguments_multi(CHECK_CACHED_VAR w3p_MULTI_SET ${w3p_MULTI_SET}) ## internaly will call our overloaded parse_arguments_multi_function
+ elseif(w3p_SET)
+ # message("calling set version of parse_arguments_multi with w3p_set = ${w3p_SET}")
+ parse_arguments_multi(CHECK_CACHED_VAR w3p_SET ${w3p_SET})
+ endif()
+
+ endif()
+
+endfunction()
+
+## cmake variables introspection to globally activate/deactivate ${prefix}_WIN3RDPARTY_USE
+## This "one shot" call (only one for the next cmake configure) will automatically then reset the global variable WIN3RDPARTY_USE to UserDefined (do nothing).
+## use (call it) before and after the call of all your win3rdParty functions
+function(Win3rdPartyGlobalCacheAction )
+ set(WIN3RDPARTY_USE "UserDefined" CACHE STRING "Choose how to handle all cmake cached *_WIN3RDPARTY_USE for the next configure.\nCould be:\nUserDefined [default]\nActivateAll\nDesactivateAll" )
+ set_property(CACHE WIN3RDPARTY_USE PROPERTY STRINGS "UserDefined;ActivateAll;DesactivateAll" )
+ if(${WIN3RDPARTY_USE} MATCHES "UserDefined")
+ else()
+ if(${WIN3RDPARTY_USE} MATCHES "ActivateAll")
+ set(win3rdPvalue ON)
+ elseif(${WIN3RDPARTY_USE} MATCHES "DesactivateAll")
+ set(win3rdPvalue OFF)
+ endif()
+ get_cmake_property(_variableNames CACHE_VARIABLES)
+ foreach (_variableName ${_variableNames})
+ string(REGEX MATCH "[A-Za-z_0-9-]+_WIN3RDPARTY_USE" win3rdpartyUseCacheVar ${_variableName})
+ if(win3rdpartyUseCacheVar)
+ string(REGEX REPLACE "([A-Za-z_0-9-]+_WIN3RDPARTY_USE)" "\\1" win3rdpartyUseCacheVar ${_variableName})
+ set(${win3rdpartyUseCacheVar} ${win3rdPvalue} CACHE BOOL "Use required 3rdParty binaries from ${prefix}_WIN3RDPARTY_DIR or download it if not exist" FORCE)
+ message(STATUS "${win3rdpartyUseCacheVar} cached variable set to ${win3rdPvalue}.")
+ endif()
+ endforeach()
+ set(WIN3RDPARTY_USE "UserDefined" CACHE STRING "Choose how to handle all cmake cached *_WIN3RDPARTY_USE for the next configure.\nCould be:\nUserDefined [default]\nActivateAll\nDesactivateAll" FORCE)
+ message(STATUS "reset WIN3RDPARTY_USE to UserDefined.")
+ endif()
+ mark_as_advanced(WIN3RDPARTY_USE)
+endfunction()
diff --git a/submodules/gaussian-splatting/SIBR_viewers/cmake/linux/cmake_policies.cmake b/submodules/gaussian-splatting/SIBR_viewers/cmake/linux/cmake_policies.cmake
new file mode 100644
index 0000000000000000000000000000000000000000..679fd8427d4c503ac420fc566a2ec4ae5c2825fc
--- /dev/null
+++ b/submodules/gaussian-splatting/SIBR_viewers/cmake/linux/cmake_policies.cmake
@@ -0,0 +1,19 @@
+# Copyright (C) 2020, Inria
+# GRAPHDECO research group, https://team.inria.fr/graphdeco
+# All rights reserved.
+#
+# This software is free for non-commercial, research and evaluation use
+# under the terms of the LICENSE.md file.
+#
+# For inquiries contact sibr@inria.fr and/or George.Drettakis@inria.fr
+
+
+if(__set_policies_INCLUDED__)
+ return()
+else()
+ set(__set_policies_INCLUDED__ ON)
+endif()
+
+macro(setPolicies)
+ # No more policies to enforce
+endmacro()
diff --git a/submodules/gaussian-splatting/SIBR_viewers/cmake/linux/dependencies.cmake b/submodules/gaussian-splatting/SIBR_viewers/cmake/linux/dependencies.cmake
new file mode 100644
index 0000000000000000000000000000000000000000..28eb3ba31d43e80efad6f574fab7d072b1f95909
--- /dev/null
+++ b/submodules/gaussian-splatting/SIBR_viewers/cmake/linux/dependencies.cmake
@@ -0,0 +1,324 @@
+# Copyright (C) 2020, Inria
+# GRAPHDECO research group, https://team.inria.fr/graphdeco
+# All rights reserved.
+#
+# This software is free for non-commercial, research and evaluation use
+# under the terms of the LICENSE.md file.
+#
+# For inquiries contact sibr@inria.fr and/or George.Drettakis@inria.fr
+
+
+## Included once for all sub project.
+## It contain the whole cmake instructions to find necessary common dependencies.
+## 3rdParty (provided by sibr_addlibrary win3rdParty or from external packages) are then available in cmake sub projects.
+##
+## Do not include this file more than once but you can modify it to fit to your own project.
+## So please, read it carefully because you can use on of these dependencies for your project or appen new one.
+##
+## As it is included after camke options, you can use conditional if()/endif() to encapsulate your 3rdParty.
+##
+
+## win3rdParty function allowing to auto check/download/update binaries dependencies for current windows compiler
+## Please open this file in order to get more documentation and usage examples.
+include(Win3rdParty)
+
+include(sibr_library)
+
+Win3rdPartyGlobalCacheAction()
+
+find_package(OpenGL REQUIRED)
+
+set(OpenGL_GL_PREFERENCE "GLVND")
+
+############
+## Find GLEW
+############
+##for headless rendering
+find_package(EGL QUIET)
+
+if(EGL_FOUND)
+ add_definitions(-DGLEW_EGL)
+ message("Activating EGL support for headless GLFW/GLEW")
+else()
+ message("EGL not found : EGL support for headless GLFW/GLEW is disabled")
+endif()
+
+if (MSVC11 OR MSVC12)
+ set(glew_multiset_arguments
+ CHECK_CACHED_VAR GLEW_INCLUDE_DIR PATH "glew-1.10.0/include" DOC "default empty doc"
+ CHECK_CACHED_VAR GLEW_LIBRARIES STRING LIST "debug;glew-1.10.0/${LIB_BUILT_DIR}/glew32d.lib;optimized;glew-1.10.0/${LIB_BUILT_DIR}/glew32.lib" DOC "default empty doc"
+ )
+elseif (MSVC14 OR MSVC17)
+ set(glew_multiset_arguments
+ CHECK_CACHED_VAR GLEW_INCLUDE_DIR PATH "glew-2.0.0/include" DOC "default empty doc"
+ CHECK_CACHED_VAR GLEW_SHARED_LIBRARY_RELEASE PATH "glew-2.0.0/${LIB_BUILT_DIR}/glew32.lib"
+ CHECK_CACHED_VAR GLEW_STATIC_LIBRARY_RELEASE PATH "glew-2.0.0/${LIB_BUILT_DIR}/glew32s.lib"
+ CHECK_CACHED_VAR GLEW_SHARED_LIBRARY_DEBUG PATH "glew-2.0.0/${LIB_BUILT_DIR}/glew32d.lib"
+ CHECK_CACHED_VAR GLEW_STATIC_LIBRARY_DEBUG PATH "glew-2.0.0/${LIB_BUILT_DIR}/glew32sd.lib"
+ )
+else ()
+ message("There is no provided GLEW library for your compiler, relying on find_package to find it")
+endif()
+sibr_addlibrary(NAME GLEW #VERBOSE ON
+ MSVC11 "https://repo-sam.inria.fr/fungraph/dependencies/ibr-common/win3rdParty-MSVC11-splitted%20version/glew-1.10.0.7z"
+ MSVC12 "https://repo-sam.inria.fr/fungraph/dependencies/ibr-common/win3rdParty-MSVC11-splitted%20version/glew-1.10.0.7z"
+ MSVC14 "https://repo-sam.inria.fr/fungraph/dependencies/ibr-common/win3rdParty-MSVC15-splitted%20version/glew-2.0.0.7z" # using recompiled version of glew
+ MULTI_SET ${glew_multiset_arguments}
+)
+set(GLEW_VERBOSE ON)
+FIND_PACKAGE(GLEW REQUIRED)
+IF(GLEW_FOUND)
+ INCLUDE_DIRECTORIES(${GLEW_INCLUDE_DIR})
+ELSE(GLEW_FOUND)
+ MESSAGE("GLEW not found. Set GLEW_DIR to base directory of GLEW.")
+ENDIF(GLEW_FOUND)
+
+
+##############
+## Find ASSIMP
+##############
+if (MSVC11 OR MSVC12)
+ set(assimp_set_arguments
+ CHECK_CACHED_VAR ASSIMP_DIR PATH "Assimp_3.1_fix"
+ )
+elseif (MSVC14 OR MSVC17)
+ set(assimp_set_arguments
+ CHECK_CACHED_VAR ASSIMP_DIR PATH "Assimp-4.1.0"
+ )
+else ()
+ message("There is no provided ASSIMP library for your compiler, relying on find_package to find it")
+endif()
+
+sibr_addlibrary(NAME ASSIMP #VERBOSE ON
+ MSVC11 "https://repo-sam.inria.fr/fungraph/dependencies/ibr-common/win3rdParty-MSVC11-splitted%20version/Assimp_3.1_fix.7z"
+ MSVC12 "https://repo-sam.inria.fr/fungraph/dependencies/ibr-common/win3rdParty-MSVC11-splitted%20version/Assimp_3.1_fix.7z"
+ MSVC14 "https://repo-sam.inria.fr/fungraph/dependencies/ibr-common/win3rdParty-MSVC15-splitted%20version/Assimp-4.1.0.7z"
+ MULTI_SET
+ ${assimp_set_arguments}
+)
+
+find_package(ASSIMP REQUIRED)
+include_directories(${ASSIMP_INCLUDE_DIR})
+
+################
+## Find FFMPEG
+################
+sibr_addlibrary(NAME FFMPEG
+ MSVC11 "https://repo-sam.inria.fr/fungraph/dependencies/ibr-common/win3rdParty-MSVC11-splitted%20version/ffmpeg.zip"
+ MSVC12 "https://repo-sam.inria.fr/fungraph/dependencies/ibr-common/win3rdParty-MSVC11-splitted%20version/ffmpeg.zip"
+ MSVC14 "https://repo-sam.inria.fr/fungraph/dependencies/ibr-common/win3rdParty-MSVC15-splitted%20version/ffmpeg-4.0.2-win64-win3rdParty.7z"
+ SET CHECK_CACHED_VAR FFMPEG_DIR PATH ${FFMPEG_WIN3RDPARTY_DIR}
+)
+find_package(FFMPEG)
+include_directories(${FFMPEG_INCLUDE_DIR})
+## COMMENT OUT ALL FFMPEG FOR CLUSTER
+
+###################
+## Find embree3
+###################
+sibr_addlibrary(
+ NAME embree3
+ MSVC11 "https://repo-sam.inria.fr/fungraph/dependencies/ibr-common/win3rdParty-MSVC11-splitted%20version/embree2.7.0.x64.windows.7z"
+ MSVC14 "https://repo-sam.inria.fr/fungraph/dependencies/ibr-common/win3rdParty-MSVC15-splitted%20version/embree-3.6.1.x64.vc14.windows.7z" # TODO SV: provide a valid version if required
+)
+
+# CLUSTER
+#find_package(embree 3.0 REQUIRED PATHS "/data/graphdeco/share/embree/usr/local/lib64/cmake/" )
+find_package(embree 3.0 )
+
+###################
+## Find eigen3
+###################
+sibr_addlibrary(
+ NAME eigen3
+ #MSVC11 "https://repo-sam.inria.fr/fungraph/dependencies/ibr-common/win3rdParty-MSVC11-splitted%20version/eigen-eigen-dc6cfdf9bcec.7z"
+ #MSVC14 "https://repo-sam.inria.fr/fungraph/dependencies/ibr-common/win3rdParty-MSVC11-splitted%20version/eigen-eigen-dc6cfdf9bcec.7z" # TODO SV: provide a valid version if required
+ MSVC11 "https://repo-sam.inria.fr/fungraph/dependencies/ibr-common/win3rdParty-MSVC15-splitted%20version/eigen3.7z"
+ MSVC14 "https://repo-sam.inria.fr/fungraph/dependencies/ibr-common/win3rdParty-MSVC15-splitted%20version/eigen3.7z"
+ SET CHECK_CACHED_VAR eigen3_DIR PATH "eigen/share/eigen3/cmake"
+)
+include_directories(/usr/include/eigen3)
+add_definitions(-DEIGEN_INITIALIZE_MATRICES_BY_ZERO)
+
+#############
+## Find Boost
+#############
+set(Boost_REQUIRED_COMPONENTS "system;chrono;filesystem;date_time" CACHE INTERNAL "Boost Required Components")
+
+if (WIN32)
+ # boost multiset arguments
+ if (MSVC11 OR MSVC12)
+ set(boost_multiset_arguments
+ CHECK_CACHED_VAR BOOST_ROOT PATH "boost_1_55_0"
+ CHECK_CACHED_VAR BOOST_INCLUDEDIR PATH "boost_1_55_0"
+ CHECK_CACHED_VAR BOOST_LIBRARYDIR PATH "boost_1_55_0/${LIB_BUILT_DIR}"
+ #CHECK_CACHED_VAR Boost_COMPILER STRING "-${Boost_WIN3RDPARTY_VCID}" DOC "vcid (eg: -vc110 for MSVC11)"
+ CHECK_CACHED_VAR Boost_COMPILER STRING "-vc110" DOC "vcid (eg: -vc110 for MSVC11)" # NOTE: if it doesnt work, uncomment this option and set the right value for VisualC id
+ )
+ elseif (MSVC14 OR MSVC17)
+ set(boost_multiset_arguments
+ CHECK_CACHED_VAR BOOST_ROOT PATH "boost-1.71"
+ CHECK_CACHED_VAR BOOST_INCLUDEDIR PATH "boost-1.71"
+ CHECK_CACHED_VAR BOOST_LIBRARYDIR PATH "boost-1.71/${LIB_BUILT_DIR}"
+ CHECK_CACHED_VAR Boost_COMPILER STRING "-vc141" DOC "vcid (eg: -vc110 for MSVC11)" # NOTE: if it doesnt work, uncomment this option and set the right value for VisualC id
+ )
+
+ option(BOOST_MINIMAL_VERSION "Only get minimal Boost dependencies" ON)
+
+ if(${BOOST_MINIMAL_VERSION})
+ set(BOOST_MSVC14_ZIP "boost-1.71-ibr-minimal.7z")
+ else()
+ set(BOOST_MSVC14_ZIP "boost-1.71.7z")
+ endif()
+ else ()
+ message("There is no provided Boost library for your compiler, relying on find_package to find it")
+ endif()
+
+ sibr_addlibrary(NAME Boost VCID TIMEOUT 600 #VERBOSE ON
+ MSVC11 "https://repo-sam.inria.fr/fungraph/dependencies/ibr-common/win3rdParty-MSVC11-splitted%20version/boost_1_55_0.7z"
+ MSVC12 "https://repo-sam.inria.fr/fungraph/dependencies/ibr-common/win3rdParty-MSVC11-splitted%20version/boost_1_55_0.7z"
+ MSVC14 "https://repo-sam.inria.fr/fungraph/dependencies/ibr-common/win3rdParty-MSVC15-splitted%20version/${BOOST_MSVC14_ZIP}" # boost compatible with msvc14
+ MULTI_SET ${boost_multiset_arguments}
+ CHECK_CACHED_VAR Boost_NO_SYSTEM_PATHS BOOL ON DOC "Set to ON to disable searching in locations not specified by these boost cached hint variables"
+ CHECK_CACHED_VAR Boost_NO_BOOST_CMAKE BOOL ON DOC "Set to ON to disable the search for boost-cmake (package cmake config file if boost was built with cmake)"
+ )
+ if(NOT Boost_COMPILER AND Boost_WIN3RDPARTY_USE)
+ message(WARNING "Boost_COMPILER is not set and it's needed.")
+ endif()
+endif()
+
+find_package(Boost 1.65.0 REQUIRED COMPONENTS ${Boost_REQUIRED_COMPONENTS})
+# for CLUSTER
+##find_package(Boost 1.58.0 REQUIRED COMPONENTS ${Boost_REQUIRED_COMPONENTS})
+
+
+if(WIN32)
+ add_compile_options("$<$:/EHsc>")
+ #add_definitions(/EHsc)
+endif()
+
+if(Boost_LIB_DIAGNOSTIC_DEFINITIONS)
+ add_definitions(${Boost_LIB_DIAGNOSTIC_DEFINITIONS})
+endif()
+
+#if(WIN32)
+ add_definitions(-DBOOST_ALL_DYN_LINK -DBOOST_ALL_NO_LIB)
+#endif()
+
+include_directories(${BOOST_INCLUDEDIR} ${Boost_INCLUDE_DIRS})
+link_directories(${BOOST_LIBRARYDIR} ${Boost_LIBRARY_DIRS})
+
+
+##############
+## Find OpenMP
+##############
+find_package(OpenMP)
+
+##############
+## Find OpenCV
+##############
+if (WIN32)
+ if (${MSVC_TOOLSET_VERSION} EQUAL 143)
+ MESSAGE("SPECIAL OPENCV HANDLING")
+ set(opencv_set_arguments
+ CHECK_CACHED_VAR OpenCV_DIR PATH "install" ## see OpenCVConfig.cmake
+ )
+ elseif (MSVC11 OR MSVC12)
+ set(opencv_set_arguments
+ CHECK_CACHED_VAR OpenCV_DIR PATH "opencv/build" ## see OpenCVConfig.cmake
+ )
+ elseif (MSVC14)
+ set(opencv_set_arguments
+ CHECK_CACHED_VAR OpenCV_DIR PATH "opencv-4.5.0/build" ## see OpenCVConfig.cmake
+ )
+ else ()
+ message("There is no provided OpenCV library for your compiler, relying on find_package to find it")
+ endif()
+else()
+ message("There is no provided OpenCV library for your compiler, relying on find_package to find it")
+endif()
+
+sibr_addlibrary(NAME OpenCV #VERBOSE ON
+ MSVC11 "https://repo-sam.inria.fr/fungraph/dependencies/sibr/~0.9/opencv.7z"
+ MSVC12 "https://repo-sam.inria.fr/fungraph/dependencies/sibr/~0.9/opencv.7z"
+ MSVC14 "https://repo-sam.inria.fr/fungraph/dependencies/sibr/~0.9/opencv-4.5.0.7z" # opencv compatible with msvc14 and with contribs
+ MSVC17 "https://repo-sam.inria.fr/fungraph/dependencies/sibr/~0.9/opencv4-8.7z"
+ SET ${opencv_set_arguments}
+ )
+find_package(OpenCV 4.5 REQUIRED) ## Use directly the OpenCVConfig.cmake provided
+## FOR CLUSTER
+###find_package(OpenCV 4.5 REQUIRED PATHS "/data/graphdeco/share/opencv/usr/local/lib64/cmake/opencv4/" ) ## Use directly the OpenCVConfig.cmake provided
+
+ ##https://stackoverflow.com/questions/24262081/cmake-relwithdebinfo-links-to-debug-libs
+set_target_properties(${OpenCV_LIBS} PROPERTIES MAP_IMPORTED_CONFIG_RELWITHDEBINFO RELEASE)
+
+add_definitions(-DOPENCV_TRAITS_ENABLE_DEPRECATED)
+
+if(OpenCV_INCLUDE_DIRS)
+ foreach(inc ${OpenCV_INCLUDE_DIRS})
+ if(NOT EXISTS ${inc})
+ set(OpenCV_INCLUDE_DIR "" CACHE PATH "additional custom include DIR (in case of trouble to find it (fedora 17 opencv package))")
+ endif()
+ endforeach()
+ if(OpenCV_INCLUDE_DIR)
+ list(APPEND OpenCV_INCLUDE_DIRS ${OpenCV_INCLUDE_DIR})
+ include_directories(${OpenCV_INCLUDE_DIRS})
+ endif()
+endif()
+
+###################
+## Find GLFW
+###################
+sibr_addlibrary(
+ NAME glfw3
+ MSVC11 "https://repo-sam.inria.fr/fungraph/dependencies/ibr-common/win3rdParty-MSVC15-splitted%20version/glfw-3.2.1.7z"
+ MSVC14 "https://repo-sam.inria.fr/fungraph/dependencies/ibr-common/win3rdParty-MSVC15-splitted%20version/glfw-3.2.1.7z" # TODO SV: provide a valid version if required
+ SET CHECK_CACHED_VAR glfw3_DIR PATH "glfw-3.2.1"
+)
+
+### FOR CLUSTER COMMENT OUT lines above, uncomment lines below
+##find_package(GLFW REQUIRED 3.3 )
+##message("***********=============> GLFW IS " ${GLFW_LIBRARY})
+##message("***********=============> GLFW IS " ${GLFW_LIBRARIES})
+
+find_package(glfw3 REQUIRED)
+
+sibr_gitlibrary(TARGET imgui
+ GIT_REPOSITORY "https://gitlab.inria.fr/sibr/libs/imgui.git"
+ GIT_TAG "741fb3ab6c7e1f7cef23ad0501a06b7c2b354944"
+)
+
+## FOR CLUSTER COMMENT OUT nativefiledialog
+sibr_gitlibrary(TARGET nativefiledialog
+ GIT_REPOSITORY "https://gitlab.inria.fr/sibr/libs/nativefiledialog.git"
+ GIT_TAG "ae2fab73cf44bebdc08d997e307c8df30bb9acec"
+)
+
+
+sibr_gitlibrary(TARGET mrf
+ GIT_REPOSITORY "https://gitlab.inria.fr/sibr/libs/mrf.git"
+ GIT_TAG "30c3c9494a00b6346d72a9e37761824c6f2b7207"
+)
+
+sibr_gitlibrary(TARGET nanoflann
+ GIT_REPOSITORY "https://gitlab.inria.fr/sibr/libs/nanoflann.git"
+ GIT_TAG "7a20a9ac0a1d34850fc3a9e398fc4a7618e8a69a"
+)
+
+sibr_gitlibrary(TARGET picojson
+ GIT_REPOSITORY "https://gitlab.inria.fr/sibr/libs/picojson.git"
+ GIT_TAG "7cf8feee93c8383dddbcb6b64cf40b04e007c49f"
+)
+
+sibr_gitlibrary(TARGET rapidxml
+ GIT_REPOSITORY "https://gitlab.inria.fr/sibr/libs/rapidxml.git"
+ GIT_TAG "069e87f5ec5ce1745253bd64d89644d6b894e516"
+)
+
+sibr_gitlibrary(TARGET xatlas
+ GIT_REPOSITORY "https://gitlab.inria.fr/sibr/libs/xatlas.git"
+ GIT_TAG "0fbe06a5368da13fcdc3ee48d4bdb2919ed2a249"
+ INCLUDE_DIRS "source/xatlas"
+)
+
+Win3rdPartyGlobalCacheAction()
diff --git a/submodules/gaussian-splatting/SIBR_viewers/cmake/linux/downloadAndExtractZipFile.cmake b/submodules/gaussian-splatting/SIBR_viewers/cmake/linux/downloadAndExtractZipFile.cmake
new file mode 100644
index 0000000000000000000000000000000000000000..7f5fc2bb080fc158840125e69c3d36d169b69235
--- /dev/null
+++ b/submodules/gaussian-splatting/SIBR_viewers/cmake/linux/downloadAndExtractZipFile.cmake
@@ -0,0 +1,243 @@
+# Copyright (C) 2020, Inria
+# GRAPHDECO research group, https://team.inria.fr/graphdeco
+# All rights reserved.
+#
+# This software is free for non-commercial, research and evaluation use
+# under the terms of the LICENSE.md file.
+#
+# For inquiries contact sibr@inria.fr and/or George.Drettakis@inria.fr
+
+
+## downloadAndExtractZipFile cmake function
+## Provide a way to download zip file from public internet ZIP_URL host
+## and to extract it in a specific EXCTRATED_ZIP_PATH destination.
+## This function use 7-Zip external tool to maximize the compatibles formats.
+## This will be not download again if the EXCTRATED_ZIP_PATH already exist and DL_FORCE is set to OFF.
+## This will try to unzip file if already exist in the ZIP_DL_PATH.
+##
+## If EXCTRATED_ZIP_PATH and/or ZIP_DL_PATH are not full path,
+## it will be interpreted relative to CMAKE_BINARY_DIR
+##
+## Usage example :
+## include(downloadAndExtractZipFile)
+## downloadAndExtractZipFile(
+## http://www.cs.cornell.edu/~snavely/bundler/distr/bundler-v0.4-source.zip
+## ${CMAKE_BINARY_DIR}/Bundler/bundler-v0.4-source.zip
+## ${CMAKE_BINARY_DIR}/Bundler
+## [DL_FORCE ON|OFF]
+## [TIMEOUT]
+## [CHECK_DIRTY_URL]
+## )
+##
+## option DL_FORCE will redownload the zip file [deafult to OFF]
+## option TIMEOUT will end the unzip process after this period of time [default to 600s]
+## option CHECK_DIRTY_URL will write into the given file the downloaded URL and then,
+## next time, if the URL was updated, it detect it with this file
+## and will download the last version. This prevent to alway set manually DL_FORCE to ON...
+##
+if(__downloadAndExtractZipFile_cmake_INCLUDED__)
+ return()
+else()
+ set(__downloadAndExtractZipFile_cmake_INCLUDED__ ON)
+endif()
+
+function(downloadAndExtractZipFile ZIP_URL ZIP_DL_PATH EXCTRATED_ZIP_PATH)
+
+ # message(STATUS "zipUrl=${ZIP_URL} zipDlPath=${ZIP_DL_PATH} extractedZipPath=${EXCTRATED_ZIP_PATH}")
+ cmake_parse_arguments(dwnlezf "" "VERBOSE;DL_FORCE;TIMEOUT;CHECK_DIRTY_URL" "" ${ARGN})
+
+ set(PROGRAMFILESx86 "PROGRAMFILES(x86)")
+
+ ## Check entries mandatory args
+ if(IS_ABSOLUTE "${ZIP_DL_PATH}")
+ else()
+ set(ZIP_DL_PATH "${CMAKE_BINARY_DIR}/${ZIP_DL_PATH}")
+ endif()
+ if(IS_ABSOLUTE "${EXCTRATED_ZIP_PATH}")
+ else()
+ set(EXCTRATED_ZIP_PATH "${CMAKE_BINARY_DIR}/${EXCTRATED_ZIP_PATH}")
+ endif()
+ if(NOT EXISTS "${EXCTRATED_ZIP_PATH}")
+ file(MAKE_DIRECTORY ${EXCTRATED_ZIP_PATH})
+ endif()
+
+ # SB: Once, one of downloaded zip was corrupted by an error message coming from the server.
+ if(EXISTS "${ZIP_DL_PATH}")
+ # So I check for removing such corrupted files
+ message("Removing previous ${ZIP_DL_PATH} (might be corrupted)")
+ file(REMOVE "${ZIP_DL_PATH}")
+ if(EXISTS "${dwnlezf_CHECK_DIRTY_URL}")
+ # and remove the previous (corrupted) made 'Win3rdPartyUrl' file
+ file(REMOVE "${dwnlezf_CHECK_DIRTY_URL}")
+ endif()
+ endif()
+
+ ## Check entries optional args
+ macro(readDirtyUrl )
+ if(dwnlezf_CHECK_DIRTY_URL)
+ if(IS_ABSOLUTE "${dwnlezf_CHECK_DIRTY_URL}")
+ else()
+ set(dwnlezf_CHECK_DIRTY_URL "${CMAKE_BINARY_DIR}/${dwnlezf_CHECK_DIRTY_URL}")
+ endif()
+ get_filename_component(unzipDir ${EXCTRATED_ZIP_PATH} NAME)
+ get_filename_component(unzipPath ${EXCTRATED_ZIP_PATH} PATH)
+ message(STATUS "Checking ${unzipDir} [from ${unzipPath}]...")
+ if(EXISTS "${dwnlezf_CHECK_DIRTY_URL}")
+ get_filename_component(CHECK_DIRTY_URL_FILENAME ${dwnlezf_CHECK_DIRTY_URL} NAME)
+ file(STRINGS "${dwnlezf_CHECK_DIRTY_URL}" contents)
+ list(GET contents 0 downloadURL)
+ list(REMOVE_AT contents 0)
+ if("${downloadURL}" MATCHES "${ZIP_URL}")
+ if(dwnlezf_VERBOSE)
+ message(STATUS "Your downloaded version (URL) seems to be up to date. Let me check if nothing is missing... (see ${dwnlezf_CHECK_DIRTY_URL}).")
+ endif()
+ file(GLOB PATHNAME_PATTERN_LIST "${EXCTRATED_ZIP_PATH}/*") ## is there something inside the downloaded destination ?
+ unset(NAME_PATTERN_LIST)
+ foreach(realPathPattern ${PATHNAME_PATTERN_LIST})
+ get_filename_component(itemName ${realPathPattern} NAME)
+ list(APPEND NAME_PATTERN_LIST ${itemName})
+ endforeach()
+ if(NAME_PATTERN_LIST)
+ foreach(item ${contents})
+ list(FIND NAME_PATTERN_LIST ${item} id)
+ if(${id} MATCHES "-1")
+ message(STATUS "${item} is missing, your downloaded version content changed, need to redownload it.")
+ set(ZIP_DL_FORCE ON)
+ break()
+ else()
+ list(REMOVE_AT NAME_PATTERN_LIST ${id})
+ set(ZIP_DL_FORCE OFF)
+ endif()
+ endforeach()
+ if(NOT ZIP_DL_FORCE AND NAME_PATTERN_LIST)
+ message("Yours seems to be up to date (regarding to ${CHECK_DIRTY_URL_FILENAME})!\nBut there are additional files/folders into your downloaded destination (feel free to clean it if you want).")
+ foreach(item ${NAME_PATTERN_LIST})
+ if(item)
+ message("${item}")
+ endif()
+ endforeach()
+ endif()
+ endif()
+ else()
+ set(ZIP_DL_FORCE ON)
+ message(STATUS "Your downloaded version is dirty (too old).")
+ endif()
+ else()
+ file(GLOB PATHNAME_PATTERN_LIST "${EXCTRATED_ZIP_PATH}/*") ## is there something inside the downloaded destination ?
+ if(NOT PATHNAME_PATTERN_LIST)
+ message("We found nothing into ${EXCTRATED_ZIP_PATH}, we will try to download it for you now.")
+ endif()
+ set(ZIP_DL_FORCE ON)
+ endif()
+ endif()
+ endmacro()
+ readDirtyUrl()
+ if(NOT ZIP_DL_FORCE)
+ return() ## do not need to further (as we are up to date, just exit the function
+ endif()
+
+ macro(writeDirtyUrl )
+ if(dwnlezf_CHECK_DIRTY_URL)
+ file(WRITE "${dwnlezf_CHECK_DIRTY_URL}" "${ZIP_URL}\n")
+ file(GLOB PATHNAME_PATTERN_LIST "${EXCTRATED_ZIP_PATH}/*") ## is there something inside the downloaded destination ?
+ unset(NAME_PATTERN_LIST)
+ foreach(realPathPattern ${PATHNAME_PATTERN_LIST})
+ get_filename_component(itemName ${realPathPattern} NAME)
+ list(APPEND NAME_PATTERN_LIST ${itemName})
+ endforeach()
+ if(NAME_PATTERN_LIST)
+ foreach(item ${NAME_PATTERN_LIST})
+ file(APPEND "${dwnlezf_CHECK_DIRTY_URL}" "${item}\n")
+ endforeach()
+ endif()
+ endif()
+ endmacro()
+
+ if(dwnlezf_DL_FORCE)
+ set(ZIP_DL_FORCE ON)
+ endif()
+
+ if(NOT dwnlezf_TIMEOUT)
+ set(dwnlezf_TIMEOUT 600)
+ endif()
+ math(EXPR dwnlezf_TIMEOUT_MIN "${dwnlezf_TIMEOUT}/60")
+
+ macro(unzip whichZipFile)
+ if(NOT SEVEN_ZIP_CMD)
+ find_program(SEVEN_ZIP_CMD NAMES 7z 7za p7zip DOC "7-zip executable" PATHS "$ENV{PROGRAMFILES}/7-Zip" "$ENV{${PROGRAMFILESx86}}/7-Zip" "$ENV{ProgramW6432}/7-Zip")
+ endif()
+ if(SEVEN_ZIP_CMD)
+ if(dwnlezf_VERBOSE)
+ message(STATUS "UNZIP: please, WAIT UNTIL ${SEVEN_ZIP_CMD} finished...\n(no more than ${dwnlezf_TIMEOUT_MIN} min)")
+ else()
+ message(STATUS "UNZIP...wait...")
+ endif()
+ execute_process( COMMAND ${SEVEN_ZIP_CMD} x ${whichZipFile} -y
+ WORKING_DIRECTORY ${EXCTRATED_ZIP_PATH} TIMEOUT ${dwnlezf_TIMEOUT}
+ RESULT_VARIABLE resVar OUTPUT_VARIABLE outVar ERROR_VARIABLE errVar
+ )
+ if(${resVar} MATCHES "0")
+ if(dwnlezf_VERBOSE)
+ message(STATUS "SUCESS to unzip in ${EXCTRATED_ZIP_PATH}. Now we can remove the downloaded zip file.")
+ endif()
+ execute_process(COMMAND ${CMAKE_COMMAND} -E remove ${whichZipFile})
+ mark_as_advanced(SEVEN_ZIP_CMD)
+ else()
+ message(WARNING "something wrong in ${EXCTRATED_ZIP_PATH}\n with \"${SEVEN_ZIP_CMD} x ${whichZipFile} -y\", redo or try to unzip by yourself...")
+ message("unzip: resVar=${resVar}")
+ message("unzip: outVar=${outVar}")
+ message("unzip: errVar=${errVar}")
+ message("unzip: failed or canceled or timeout")
+ endif()
+ else()
+ message(WARNING "You need 7zip (http://www.7-zip.org/download.html) to unzip the downloaded dir.")
+ set(SEVEN_ZIP_CMD "" CACHE FILEPATH "7-zip executable")
+ mark_as_advanced(CLEAR SEVEN_ZIP_CMD)
+ endif()
+ endmacro()
+
+ if(dwnlezf_VERBOSE)
+ message(STATUS "Trying to look ${ZIP_DL_PATH} if a zip file exist...")
+ endif()
+ if(EXISTS "${ZIP_DL_PATH}")
+
+ ## already downloaded, so just unzip it
+ unzip(${ZIP_DL_PATH})
+ writeDirtyUrl()
+
+ elseif(ZIP_DL_FORCE)
+
+ ## the download part (+ unzip)
+ message(STATUS "Let me try to download package for you : ${ZIP_URL}")
+ if(dwnlezf_VERBOSE)
+ message(STATUS "Downloading...\n SRC=${ZIP_URL}\n DEST=${ZIP_DL_PATH}.tmp\n INACTIVITY_TIMEOUT=180s")
+ endif()
+ file(DOWNLOAD ${ZIP_URL} ${ZIP_DL_PATH}.tmp INACTIVITY_TIMEOUT 360 STATUS status SHOW_PROGRESS)
+
+ list(GET status 0 numResult)
+ if(${numResult} MATCHES "0")
+
+ if(dwnlezf_VERBOSE)
+ message(STATUS "Download succeed, so let me rename the tmp file to unzip it")
+ endif()
+ execute_process(COMMAND ${CMAKE_COMMAND} -E rename ${ZIP_DL_PATH}.tmp ${ZIP_DL_PATH})
+ unzip(${ZIP_DL_PATH})
+ writeDirtyUrl()
+
+ else()
+
+ list(GET status 1 errMsg)
+ message(WARNING "DOWNLOAD ${ZIP_URL} to ${ZIP_DL_PATH} failed\n:${errMsg}")
+ message(WARNING "OK, you need to download the ${ZIP_URL} manually and put it into ${ZIP_DL_PATH}")
+ message("Take a look at the project website page to check available URL.")
+
+ endif()
+
+ endif()
+
+ ## clean up the tmp downloaded file
+ if(EXISTS "${ZIP_DL_PATH}.tmp")
+ execute_process(COMMAND ${CMAKE_COMMAND} -E remove ${ZIP_DL_PATH}.tmp)
+ endif()
+
+endfunction()
diff --git a/submodules/gaussian-splatting/SIBR_viewers/cmake/linux/git_describe.cmake b/submodules/gaussian-splatting/SIBR_viewers/cmake/linux/git_describe.cmake
new file mode 100644
index 0000000000000000000000000000000000000000..638d70bd9440fe80ff1844500d3acf8f05e669b8
--- /dev/null
+++ b/submodules/gaussian-splatting/SIBR_viewers/cmake/linux/git_describe.cmake
@@ -0,0 +1,114 @@
+# Copyright (C) 2020, Inria
+# GRAPHDECO research group, https://team.inria.fr/graphdeco
+# All rights reserved.
+#
+# This software is free for non-commercial, research and evaluation use
+# under the terms of the LICENSE.md file.
+#
+# For inquiries contact sibr@inria.fr and/or George.Drettakis@inria.fr
+
+
+if(__git_describe_INCLUDED__)
+ return()
+else()
+ set(__git_describe_INCLUDED__ ON)
+endif()
+
+find_package(Git)
+if(Git_FOUND)
+ message(STATUS "Git found: ${GIT_EXECUTABLE}")
+else()
+ message(FATAL_ERROR "Git not found. Aborting")
+endif()
+
+macro(git_describe)
+ cmake_parse_arguments(GIT_DESCRIBE "" "GIT_URL;GIT_BRANCH;GIT_COMMIT_HASH;GIT_TAG;GIT_VERSION;PATH" "" ${ARGN})
+
+ if(NOT GIT_DESCRIBE_PATH)
+ set(GIT_DESCRIBE_PATH ${CMAKE_SOURCE_DIR})
+ endif()
+
+ if(GIT_DESCRIBE_GIT_URL)
+ # Get the current remote
+ execute_process(
+ COMMAND git remote
+ WORKING_DIRECTORY ${GIT_DESCRIBE_PATH}
+ OUTPUT_VARIABLE GIT_DESCRIBE_GIT_REMOTE
+ OUTPUT_STRIP_TRAILING_WHITESPACE
+ ERROR_QUIET
+ )
+
+ # Get the current remote
+ execute_process(
+ COMMAND git remote get-url ${GIT_DESCRIBE_GIT_REMOTE}
+ WORKING_DIRECTORY ${GIT_DESCRIBE_PATH}
+ OUTPUT_VARIABLE ${GIT_DESCRIBE_GIT_URL}
+ OUTPUT_STRIP_TRAILING_WHITESPACE
+ ERROR_QUIET
+ )
+ endif()
+
+ if(GIT_DESCRIBE_GIT_BRANCH)
+ # Get the current working branch
+ execute_process(
+ COMMAND git rev-parse --abbrev-ref HEAD
+ WORKING_DIRECTORY ${GIT_DESCRIBE_PATH}
+ OUTPUT_VARIABLE ${GIT_DESCRIBE_GIT_BRANCH}
+ OUTPUT_STRIP_TRAILING_WHITESPACE
+ ERROR_QUIET
+ )
+ endif()
+
+ if(GIT_DESCRIBE_GIT_COMMIT_HASH)
+ # Get the latest abbreviated commit hash of the working branch
+ execute_process(
+ COMMAND git rev-parse HEAD
+ WORKING_DIRECTORY ${GIT_DESCRIBE_PATH}
+ OUTPUT_VARIABLE ${GIT_DESCRIBE_GIT_COMMIT_HASH}
+ OUTPUT_STRIP_TRAILING_WHITESPACE
+ ERROR_QUIET
+ )
+ endif()
+
+ if(GIT_DESCRIBE_GIT_TAG)
+ # Get the tag
+ execute_process(
+ COMMAND git describe --tags --exact-match
+ WORKING_DIRECTORY ${GIT_DESCRIBE_PATH}
+ OUTPUT_VARIABLE ${GIT_DESCRIBE_GIT_TAG}
+ OUTPUT_STRIP_TRAILING_WHITESPACE
+ ERROR_QUIET
+ )
+ endif()
+
+ if(GIT_DESCRIBE_GIT_VERSION)
+ # Get the version from git describe
+ execute_process(
+ COMMAND git describe
+ WORKING_DIRECTORY ${GIT_DESCRIBE_PATH}
+ OUTPUT_VARIABLE ${GIT_DESCRIBE_GIT_VERSION}
+ OUTPUT_STRIP_TRAILING_WHITESPACE
+ ERROR_QUIET
+ )
+
+ if(${GIT_DESCRIBE_GIT_VERSION} STREQUAL "")
+ execute_process(
+ COMMAND git rev-parse --abbrev-ref HEAD
+ WORKING_DIRECTORY ${GIT_DESCRIBE_PATH}
+ OUTPUT_VARIABLE GIT_DESCRIBE_GIT_VERSION_BRANCH
+ OUTPUT_STRIP_TRAILING_WHITESPACE
+ ERROR_QUIET
+ )
+ execute_process(
+ COMMAND git log -1 --format=%h
+ WORKING_DIRECTORY ${GIT_DESCRIBE_PATH}
+ OUTPUT_VARIABLE GIT_DESCRIBE_GIT_VERSION_COMMIT
+ OUTPUT_STRIP_TRAILING_WHITESPACE
+ ERROR_QUIET
+ )
+
+ set(${GIT_DESCRIBE_GIT_VERSION} "${GIT_DESCRIBE_GIT_VERSION_BRANCH}-${GIT_DESCRIBE_GIT_VERSION_COMMIT}")
+ endif()
+ endif()
+
+endmacro()
\ No newline at end of file
diff --git a/submodules/gaussian-splatting/SIBR_viewers/cmake/linux/include_once.cmake b/submodules/gaussian-splatting/SIBR_viewers/cmake/linux/include_once.cmake
new file mode 100644
index 0000000000000000000000000000000000000000..d28b39cfeb18e2369ccee1d9f038cfdd71958296
--- /dev/null
+++ b/submodules/gaussian-splatting/SIBR_viewers/cmake/linux/include_once.cmake
@@ -0,0 +1,22 @@
+# Copyright (C) 2020, Inria
+# GRAPHDECO research group, https://team.inria.fr/graphdeco
+# All rights reserved.
+#
+# This software is free for non-commercial, research and evaluation use
+# under the terms of the LICENSE.md file.
+#
+# For inquiries contact sibr@inria.fr and/or George.Drettakis@inria.fr
+
+
+macro(include_once file)
+ get_filename_component(INCLUDE_ONCE_FILEPATH ${file} REALPATH)
+ string(REGEX REPLACE "(\\.|\\/+|\\:|\\\\+)" "_" INCLUDE_ONCE_FILEPATH ${INCLUDE_ONCE_FILEPATH})
+ get_property(INCLUDED_${INCLUDE_ONCE_FILEPATH}_LOCAL GLOBAL PROPERTY INCLUDED_${INCLUDE_ONCE_FILEPATH})
+ if (INCLUDED_${INCLUDE_ONCE_FILEPATH}_LOCAL)
+ return()
+ else()
+ set_property(GLOBAL PROPERTY INCLUDED_${INCLUDE_ONCE_FILEPATH} true)
+
+ include(${file})
+ endif()
+endmacro()
\ No newline at end of file
diff --git a/submodules/gaussian-splatting/SIBR_viewers/cmake/linux/install_runtime.cmake b/submodules/gaussian-splatting/SIBR_viewers/cmake/linux/install_runtime.cmake
new file mode 100644
index 0000000000000000000000000000000000000000..695c4b330955d0df7027b7a56a425b30a84acd53
--- /dev/null
+++ b/submodules/gaussian-splatting/SIBR_viewers/cmake/linux/install_runtime.cmake
@@ -0,0 +1,887 @@
+# Copyright (C) 2020, Inria
+# GRAPHDECO research group, https://team.inria.fr/graphdeco
+# All rights reserved.
+#
+# This software is free for non-commercial, research and evaluation use
+# under the terms of the LICENSE.md file.
+#
+# For inquiries contact sibr@inria.fr and/or George.Drettakis@inria.fr
+
+
+## This file is mainly used to allow runtime installation
+## There are some utilities cmake functions to ease the generic deployement (abstract common usage of cmake)...
+##
+## You cannot run your programm automaticaly from your CNAKE_BINARY_DIR when you build
+## as it will miss all dependencies and ressources files...
+## You have to run install target in order to test your programm.
+##
+## The only one function/macros you may use inside your sub-CMakeLists.txt (sub-project) is :
+## ******************
+## ibr_install_target macro => see documentation at the end of this file
+## ******************
+## It use these utilities cmake functions to abstract the installation in an uniform way for all sub-projects.
+##
+if(__install_runtime_cmake_INCLUDED__)
+ return()
+else()
+ set(__install_runtime_cmake_INCLUDED__ ON)
+endif()
+
+
+##
+## Allow to write a resource config file which contain additional ressource paths
+## (used by IBR_Common Resource system to load shaders and potentialy images, plugins and so on)
+##
+## ADD option list all the paths to add in the file (relative paths are interpreted relative to working dir of the executable)
+## INSTALL option to specify where we want to install this file
+##
+## Example usage:
+## resourceFile(ADD "shaders" "${PROJECT_NAME}_rsc" INSTALL bin)
+##
+macro(resourceFile)
+ cmake_parse_arguments(rsc "" "INSTALL;FILE_PATH;CONFIG_TYPE" "ADD" ${ARGN}) ## both args are directory path
+
+ if(rsc_ADD)
+ unset(IBR_RSC_FILE_CONTENT_LIST)
+ if(EXISTS "${rsc_FILE_PATH}")
+ file(READ "${rsc_FILE_PATH}" IBR_RSC_FILE_CONTENT)
+ string(REGEX REPLACE "\n" ";" IBR_RSC_FILE_CONTENT_LIST "${IBR_RSC_FILE_CONTENT}")
+ endif()
+ list(APPEND IBR_RSC_FILE_CONTENT_LIST "${rsc_ADD}")
+ list(REMOVE_DUPLICATES IBR_RSC_FILE_CONTENT_LIST)
+ file(WRITE "${rsc_FILE_PATH}" "")
+ foreach(rscDir ${IBR_RSC_FILE_CONTENT_LIST})
+ file(APPEND "${rsc_FILE_PATH}" "${rscDir}\n")
+ endforeach()
+ unset(rsc_ADD)
+ endif()
+
+ if(rsc_INSTALL)
+ install(FILES ${rsc_FILE_PATH} CONFIGURATIONS ${rsc_CONFIG_TYPE} DESTINATION ${rsc_INSTALL})
+ unset(rsc_INSTALL)
+ endif()
+endmacro()
+
+
+##
+## Install *.pdb generated file for the current cmake project
+## assuming the output target name is the cmake project name.
+## This macro is useful for crossplateform multi config mode.
+##
+## Usage Example:
+##
+## if(DEFINED CMAKE_BUILD_TYPE) ## for make/nmake based
+## installPDB(${PROJECT_NAME} ${CMAKE_BUILD_TYPE} RUNTIME_DEST bin ARCHIVE_DEST lib LIBRARY_DEST lib)
+## endif()
+## foreach(CONFIG_TYPES ${CMAKE_CONFIGURATION_TYPES}) ## for multi config types (MSVC based)
+## installPDB(${PROJECT_NAME} ${CONFIG_TYPES} RUNTIME_DEST bin ARCHIVE_DEST lib LIBRARY_DEST lib)
+## endforeach()
+##
+macro(installPDB targetName configType)
+ cmake_parse_arguments(instpdb "" "COMPONENT" "ARCHIVE_DEST;LIBRARY_DEST;RUNTIME_DEST" ${ARGN}) ## both args are directory path
+
+ if(NOT MSVC)
+ return()
+ endif()
+
+ ## Check if DESTINATION are provided according to the TYPE of the given target (see install command doc to see correspodances)
+ get_target_property(type ${targetName} TYPE)
+ if(${type} MATCHES "EXECUTABLE" AND instpdb_RUNTIME_DEST)
+ set(pdb_DESTINATION ${instpdb_RUNTIME_DEST})
+ elseif(${type} MATCHES "STATIC_LIBRARY" AND instpdb_ARCHIVE_DEST)
+ set(pdb_DESTINATION ${instpdb_ARCHIVE_DEST})
+ elseif(${type} MATCHES "MODULE_LIBRARY" AND instpdb_LIBRARY_DEST)
+ set(pdb_DESTINATION ${instpdb_LIBRARY_DEST})
+ elseif(${type} MATCHES "SHARED_LIBRARY")
+ if(WIN32 AND instpdb_RUNTIME_DEST)
+ set(pdb_DESTINATION ${instpdb_RUNTIME_DEST})
+ else()
+ set(pdb_DESTINATION ${instpdb_LIBRARY_DEST})
+ endif()
+ endif()
+
+ if(NOT pdb_DESTINATION)
+ set(pdb_DESTINATION bin) ## default destination of the pdb file
+ endif()
+
+ if(NOT instpdb_COMPONENT)
+ set(instpdb_COMPONENT )
+ else()
+ set(instpdb_COMPONENT COMPONENT ${instpdb_COMPONENT})
+ endif()
+
+ string(TOUPPER ${configType} CONFIG_TYPES_UC)
+ get_target_property(PDB_PATH ${targetName} PDB_OUTPUT_DIRECTORY_${CONFIG_TYPES_UC})
+
+ get_target_property(confModePostfix ${targetName} ${CONFIG_TYPES_UC}_POSTFIX)
+ if(NOT confModePostfix)
+ set(confModePostfix "")
+ endif()
+ set_target_properties(${targetName} PROPERTIES PDB_NAME_${CONFIG_TYPES_UC} ${targetName}${confModePostfix})
+ get_target_property(PDB_NAME ${targetName} PDB_NAME_${CONFIG_TYPES_UC})# if not set, this is empty
+
+ if(EXISTS "${PDB_PATH}/${PDB_NAME}.pdb")
+ install(FILES "${PDB_PATH}/${PDB_NAME}.pdb" CONFIGURATIONS ${configType} DESTINATION ${pdb_DESTINATION} ${instpdb_COMPONENT} OPTIONAL)
+ endif()
+endmacro()
+
+
+##
+## Add additional target to install a project independently and based on its component
+## configMode is used to prevent default Release installation (we want also to install in other build/config type)
+##
+macro(installTargetProject targetOfProject targetOfInstallProject)
+ if(DEFINED CMAKE_BUILD_TYPE) ## for make/nmake based
+ set(configMode ${CMAKE_BUILD_TYPE})
+ elseif(MSVC)
+ ## $(Configuration) will be one of the following : Debug, Release, MinSizeRel, RelWithDebInfo
+ set(configMode $(Configuration))
+ endif()
+ if(configMode)
+ get_target_property(srcFiles ${targetOfProject} SOURCES)
+ add_custom_target( ${targetOfInstallProject} #ALL
+ ${CMAKE_COMMAND} -DBUILD_TYPE=${configMode} -DCOMPONENT=${targetOfInstallProject} -P ${CMAKE_BINARY_DIR}/cmake_install.cmake
+ DEPENDS ${srcFiles}
+ COMMENT "run the installation only for ${targetOfProject}" VERBATIM
+ )
+ add_dependencies(${targetOfInstallProject} ${targetOfProject})
+
+ get_target_property(INSTALL_BUILD_FOLDER ${targetOfProject} FOLDER)
+ set_target_properties(${targetOfInstallProject} PROPERTIES FOLDER ${INSTALL_BUILD_FOLDER})
+ endif()
+endmacro()
+
+# Collect all currently added targets in all subdirectories
+#
+# Parameters:
+# - _result the list containing all found targets
+# - _dir root directory to start looking from
+function(get_all_targets _result _dir)
+ get_property(_subdirs DIRECTORY "${_dir}" PROPERTY SUBDIRECTORIES)
+ foreach(_subdir IN LISTS _subdirs)
+ get_all_targets(${_result} "${_subdir}")
+ endforeach()
+
+ get_directory_property(_sub_targets DIRECTORY "${_dir}" BUILDSYSTEM_TARGETS)
+ set(${_result} ${${_result}} ${_sub_targets} PARENT_SCOPE)
+endfunction()
+
+##
+## Add targets for building and installing subdirectories
+macro(subdirectory_target target directory build_folder)
+ add_custom_target(${target}
+ COMMENT "run build for all projects in this directory" VERBATIM
+ )
+ get_all_targets(ALL_TARGETS ${directory})
+ add_dependencies(${target} ${ALL_TARGETS})
+ add_custom_target(${target}_install
+ ${CMAKE_COMMAND} -DBUILD_TYPE=$ -DCOMPONENT=${target}_install -P ${CMAKE_BINARY_DIR}/cmake_install.cmake
+ COMMENT "run install for all projects in this directory" VERBATIM
+ )
+ add_dependencies(${target}_install ${target})
+
+ set_target_properties(${target} PROPERTIES FOLDER ${build_folder})
+ set_target_properties(${target}_install PROPERTIES FOLDER ${build_folder})
+endmacro()
+
+
+## CMAKE install all required dependencies for an application (included system OS files like msvc*.dll for example)
+##
+## install_runtime(
+## [TARGET name]
+## [PLUGINS name [nameN ...] [PLUGIN_PATH_NAME currentPathName [FROM_REL_PATH matchDirFromCurrentPathName] [PLUGIN_PATH_DEST installDir] ]
+## [PLUGINS ...]
+## [DIRS path [pathN ...] ]
+## [TARGET_LIBRARIES filePath [filePathN ...] ]
+## [TARGET_PACKAGES packageName [packageNameN ...] ]
+## [COMPONENT installComponentName]
+## [PLAUSIBLES_POSTFIX Debug_postfix [MinSizeRel_postfix relWithDebInfo_postfix ...] ]
+## [VERBOSE]
+## )
+##
+## installedFilePathTargetAppToResolve : the final installed targetApp absolute full file path name you want to resolve
+##
+## TARGET : The target app we want to install. If given, it's used to look for link libraries paths (best choice to use, strongly advised to use it)
+##
+## PLUGINS : Some application built use/load some plugins which can't be detect inside its binary,
+## so, here you can specify which plugins the application use/load in order to install them
+## and resolve also there dependencies.
+## With PLUGINS multi FLAGS :
+## PLUGIN_PATH_NAME : The current plugin full file path we want to install
+## FROM_REL_PATH : [optional: default only the file is kept] From which matching dir of the plugin path we want to install (keep the directories structure)
+## PLUGIN_PATH_DEST : [optional: default relative to executable directory] Where (full path to the install directory) we will install the plugin file (or file path)
+##
+## DIRS : A list of directories to looking for dependencies
+## TARGET_LIBRARIES : DEPRECATED (use TARGET flag instead) : The cmake content variables used for the target_link_libraries( ...)
+## TARGET_PACKAGES : DEPRECATED (use TARGET flag instead) : The cmake package names used for the findPackage(...) for your targetApp
+## ADVICE: This flag add entries in cache (like: _DIR), it could be useful to fill these variable!
+## COMPONENT : (default to runtime) Is the component name associated to the installation
+## It is used when you want to install separatly some part of your projets (see install cmake doc)
+## VERBOSE : For debug or to get more informations in the output console
+##
+## Usage:
+## install_runtime(${CMAKE_INSTALL_PREFIX}/${EXECUTABLE_NAME}${CMAKE_EXECUTABLE_SUFFIX}
+## VERBOSE
+## TARGET ${PROJECT_NAME}
+## PLAUSIBLES_POSTFIX _d
+## PLUGINS
+## PLUGIN_PATH_NAME ${PLUGIN_PATH_NAME}${CMAKE_SHARED_MODULE_SUFFIX} ## will be installed (default exec path if no PLUGINS_DEST) and then will be resolved
+## FROM_REL_PATH plugins ## optional, used especially for keeping qt plugins tree structure
+## PLUGIN_PATH_DEST ${CMAKE_INSTALL_PREFIX}/plugins ## (or relative path 'plugins' will be interpreted relative to installed executable)
+## DIRS ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_BINARY_DIR}
+## TARGET_LIBRARIES ${OPENGL_LIBRARIES} ## DEPRECATED (use TARGET flag instead)
+## ${GLEW_LIBRARIES}
+## ${GLUT_LIBRARIES}
+## ${Boost_LIBRARIES}
+## ${SuiteSparse_LIBRARIES}
+## ${CGAL_LIBRARIES}
+## TARGET_PACKAGES OPENGL ## DEPRECATED (use TARGET flag instead)
+## GLEW
+## GLUT
+## CGAL
+## Boost
+## SuiteSparse
+## )
+##
+## For plugins part, it use our internal parse_arguments_multi.cmake
+##
+function(install_runtime installedFilePathTargetAppToResolve)
+ set(optionsArgs "VERBOSE")
+ set(oneValueArgs "COMPONENT;INSTALL_FOLDER;CONFIG_TYPE")
+ set(multiValueArgs "DIRS;PLUGINS;TARGET_LIBRARIES;TARGET_PACKAGES;TARGET;PLAUSIBLES_POSTFIX")
+ cmake_parse_arguments(inst_run "${optionsArgs}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN} )
+
+ if(IS_ABSOLUTE ${installedFilePathTargetAppToResolve})
+ else()
+ set(installedFilePathTargetAppToResolve ${inst_run_INSTALL_FOLDER}/${installedFilePathTargetAppToResolve})
+ endif()
+
+ get_filename_component(EXEC_NAME ${installedFilePathTargetAppToResolve} NAME_WE)
+ get_filename_component(EXEC_PATH ${installedFilePathTargetAppToResolve} PATH)
+
+ if(NOT inst_run_COMPONENT)
+ set(inst_run_COMPONENT runtime)
+ endif()
+
+
+ ## Try to append as more possible as possible paths to find dependencies (deprecated since we can use target_properties to get back paths)
+ set(libPaths )
+ foreach(libraryFileName ${inst_run_TARGET_LIBRARIES})
+ if(IS_DIRECTORY "${libraryFileName}")
+ list(APPEND libPaths "${libraryFileName}")
+ else()
+ get_filename_component(libpath "${libraryFileName}" PATH)
+ if(EXISTS "${libpath}")
+ list(APPEND libPaths "${libpath}")
+ endif()
+ endif()
+ endforeach()
+
+ ## This macro is used internaly here to recursilvely get path of LINK_LIBRARIES of each non imported target
+ ## Typically if you have 2 internal dependencies between cmake targets, we want cmake to be able to get back path where are these dependencies
+ macro(recurseDepList target)
+ get_target_property(linkLibs ${target} LINK_LIBRARIES)
+ foreach(lib ${linkLibs})
+ string(FIND ${lib} ">" strId) ## cmake is using generator-expression?
+ if(TARGET ${lib})
+ ## Skipping interface libraries as they're system ones
+ get_target_property(type ${lib} TYPE)
+ get_target_property(imported ${lib} IMPORTED)
+ if(type STREQUAL "INTERFACE_LIBRARY")
+ get_target_property(imp_loc ${lib} INTERFACE_IMPORTED_LOCATION)
+ if(imp_loc)
+ get_filename_component(imp_loc ${imp_loc} PATH)
+ list(APPEND targetLibPath ${imp_loc})
+ endif()
+ get_target_property(loc ${lib} INTERFACE_LOCATION)
+ if(loc)
+ get_filename_component(loc ${loc} PATH)
+ list(APPEND targetLibPath ${loc})
+ endif()
+ ## it's not a path but a single target name
+ ## for build-target which are part of the current cmake configuration : nothing to do as cmake already know the output path
+ ## for imported target, we need to look for theire imported location
+ elseif(imported)
+ get_target_property(imp_loc ${lib} IMPORTED_LOCATION)
+ if(imp_loc)
+ get_filename_component(imp_loc ${imp_loc} PATH)
+ list(APPEND targetLibPath ${imp_loc})
+ endif()
+ get_target_property(loc ${lib} LOCATION)
+ if(loc)
+ get_filename_component(loc ${loc} PATH)
+ list(APPEND targetLibPath ${loc})
+ endif()
+ else()
+ recurseDepList(${lib})
+ endif()
+ elseif(NOT ${strId} MATCHES -1) ## mean cmake use generator-expression (CMAKE VERSION > 3.0)
+ string(REGEX MATCH ">:[@A-Za-z_:/.0-9-]+" targetLibPath ${lib})
+ string(REGEX REPLACE ">:([@A-Za-z_:/.0-9-]+)" "\\1" targetLibPath ${targetLibPath})
+ get_filename_component(targetLibPath ${targetLibPath} PATH)
+ elseif(EXISTS ${lib})
+ set(targetLibPath ${lib})
+ get_filename_component(targetLibPath ${targetLibPath} PATH)
+ else()
+ #message(STATUS "[install_runtime] skip link library : ${lib} , of target ${target}")
+ endif()
+ if(targetLibPath)
+ list(APPEND targetLinkLibsPathList ${targetLibPath})
+ endif()
+ endforeach()
+ if(targetLinkLibsPathList)
+ list(REMOVE_DUPLICATES targetLinkLibsPathList)
+ endif()
+ endmacro()
+ if(inst_run_TARGET)
+ recurseDepList(${inst_run_TARGET})
+ if(targetLinkLibsPathList)
+ list(APPEND libPaths ${targetLinkLibsPathList})
+ endif()
+ endif()
+
+ if(libPaths)
+ list(REMOVE_DUPLICATES libPaths)
+ foreach(libPath ${libPaths})
+ get_filename_component(path ${libPath} PATH)
+ list(APPEND libPaths ${path})
+ endforeach()
+ endif()
+
+
+ ## possible speciale dir(s) according to the build system and OS
+ if(CMAKE_SIZEOF_VOID_P EQUAL 8)
+ set(BUILD_TYPES_FOR_DLL "x64")
+ if(WIN32)
+ list(APPEND BUILD_TYPES_FOR_DLL "Win64")
+ endif()
+ else()
+ set(BUILD_TYPES_FOR_DLL "x86")
+ if(WIN32)
+ list(APPEND BUILD_TYPES_FOR_DLL "Win32")
+ endif()
+ endif()
+
+
+ ## Try to append as more as possible paths to find dependencies (here, mainly for *.dll)
+ foreach(dir ${inst_run_DIRS} ${libPaths})
+ if(EXISTS "${dir}/bin")
+ list(APPEND inst_run_DIRS "${dir}/bin")
+ elseif(EXISTS "${dir}")
+ list(APPEND inst_run_DIRS "${dir}")
+ endif()
+ endforeach()
+ list(REMOVE_DUPLICATES inst_run_DIRS)
+ foreach(dir ${inst_run_DIRS})
+ if(EXISTS "${dir}")
+ list(APPEND argDirs ${dir})
+ foreach(BUILD_TYPE_FOR_DLL ${BUILD_TYPES_FOR_DLL})
+ if(EXISTS "${dir}/${BUILD_TYPE_FOR_DLL}")
+ list(APPEND argDirs "${dir}/${BUILD_TYPE_FOR_DLL}")
+ endif()
+ foreach(OUTPUTCONFIG ${CMAKE_CONFIGURATION_TYPES}) ## for windows multi-generator (MSVC)
+ if(EXISTS "${dir}/${BUILD_TYPE_FOR_DLL}/${OUTPUTCONFIG}")
+ list(APPEND argDirs "${dir}/${BUILD_TYPE_FOR_DLL}/${OUTPUTCONFIG}")
+ endif()
+ endforeach()
+ if(CMAKE_BUILD_TYPE) ## for single generator (makefiles)
+ if(EXISTS "${dir}/${BUILD_TYPE_FOR_DLL}/${CMAKE_BUILD_TYPE}")
+ list(APPEND argDirs "${dir}/${BUILD_TYPE_FOR_DLL}/${CMAKE_BUILD_TYPE}")
+ endif()
+ endif()
+ endforeach()
+ foreach(OUTPUTCONFIG ${CMAKE_CONFIGURATION_TYPES}) ## for windows multi-generator (MSVC)
+ if(EXISTS "${dir}/${OUTPUTCONFIG}")
+ list(APPEND argDirs "${dir}/${OUTPUTCONFIG}")
+ endif()
+ foreach(BUILD_TYPE_FOR_DLL ${BUILD_TYPES_FOR_DLL})
+ if(EXISTS "${dir}/${OUTPUTCONFIG}/${BUILD_TYPE_FOR_DLL}")
+ list(APPEND argDirs "${dir}/${OUTPUTCONFIG}/${BUILD_TYPE_FOR_DLL}")
+ endif()
+ endforeach()
+ endforeach()
+ if(CMAKE_BUILD_TYPE) ## for single generator (makefiles)
+ if(EXISTS "${dir}/${CMAKE_BUILD_TYPE}")
+ list(APPEND argDirs "${dir}/${CMAKE_BUILD_TYPE}")
+ endif()
+ foreach(BUILD_TYPE_FOR_DLL ${BUILD_TYPES_FOR_DLL})
+ if(EXISTS "${dir}/${CMAKE_BUILD_TYPE}/${BUILD_TYPE_FOR_DLL}")
+ list(APPEND argDirs "${dir}/${CMAKE_BUILD_TYPE}/${BUILD_TYPE_FOR_DLL}")
+ endif()
+ endforeach()
+ endif()
+ endif()
+ endforeach()
+ if(argDirs)
+ list(REMOVE_DUPLICATES argDirs)
+ endif()
+
+
+ ## Try to append as more possible paths to find dependencies (here, mainly for *.dll)
+ foreach(packageName ${inst_run_TARGET_PACKAGES})
+ if(EXISTS "${${packageName}_DIR}")
+ list(APPEND packageDirs ${${packageName}_DIR})
+ list(APPEND packageDirs ${${packageName}_DIR}/bin)
+ foreach(BUILD_TYPE_FOR_DLL ${BUILD_TYPES_FOR_DLL})
+ if(EXISTS "${${packageName}_DIR}/bin/${BUILD_TYPE_FOR_DLL}")
+ list(APPEND packageDirs "${${packageName}_DIR}/bin/${BUILD_TYPE_FOR_DLL}")
+ endif()
+ foreach(OUTPUTCONFIG ${CMAKE_CONFIGURATION_TYPES}) ## for windows multi-generator (MSVC)
+ if(EXISTS "${${packageName}_DIR}/bin/${BUILD_TYPE_FOR_DLL}/${OUTPUTCONFIG}")
+ list(APPEND packageDirs "${${packageName}_DIR}/bin/${BUILD_TYPE_FOR_DLL}/${OUTPUTCONFIG}")
+ endif()
+ endforeach()
+ if(CMAKE_BUILD_TYPE) ## for single generator (makefiles)
+ if(EXISTS "${${packageName}_DIR}/bin/${BUILD_TYPE_FOR_DLL}/${CMAKE_BUILD_TYPE}")
+ list(APPEND packageDirs "${${packageName}_DIR}/bin/${BUILD_TYPE_FOR_DLL}/${CMAKE_BUILD_TYPE}")
+ endif()
+ endif()
+ endforeach()
+ foreach(OUTPUTCONFIG ${CMAKE_CONFIGURATION_TYPES}) ## for windows multi-generator (MSVC)
+ if(EXISTS "${${packageName}_DIR}/bin/${OUTPUTCONFIG}")
+ list(APPEND packageDirs "${${packageName}_DIR}/bin/${OUTPUTCONFIG}")
+ endif()
+ foreach(BUILD_TYPE_FOR_DLL ${BUILD_TYPES_FOR_DLL})
+ if(EXISTS "${${packageName}_DIR}/bin/${OUTPUTCONFIG}/${BUILD_TYPE_FOR_DLL}")
+ list(APPEND packageDirs "${${packageName}_DIR}/bin/${OUTPUTCONFIG}/${BUILD_TYPE_FOR_DLL}")
+ endif()
+ endforeach()
+ endforeach()
+ if(CMAKE_BUILD_TYPE) ## for single generator (makefiles)
+ if(EXISTS "${${packageName}_DIR}/bin/${CMAKE_BUILD_TYPE}")
+ list(APPEND packageDirs "${${packageName}_DIR}/bin/${CMAKE_BUILD_TYPE}")
+ endif()
+ foreach(BUILD_TYPE_FOR_DLL ${BUILD_TYPES_FOR_DLL})
+ if(EXISTS "${${packageName}_DIR}/bin/${CMAKE_BUILD_TYPE}/${BUILD_TYPE_FOR_DLL}")
+ list(APPEND packageDirs "${${packageName}_DIR}/bin/${CMAKE_BUILD_TYPE}/${BUILD_TYPE_FOR_DLL}")
+ endif()
+ endforeach()
+ endif()
+ else()
+ set(${packageName}_DIR "$ENV{${packageName}_DIR}" CACHE PATH "${packageName}_DIR root directory for looking for dirs containning *.dll")
+ endif()
+ endforeach()
+ if(packageDirs)
+ list(REMOVE_DUPLICATES packageDirs)
+ endif()
+
+
+ set(dirsToLookFor "${EXEC_PATH}")
+ if(packageDirs)
+ list(APPEND dirsToLookFor ${packageDirs})
+ endif()
+ if(argDirs)
+ list(APPEND dirsToLookFor ${argDirs})
+ endif()
+ get_property(used_LINK_DIRECTORIES DIRECTORY PROPERTY LINK_DIRECTORIES)
+ if (used_LINK_DIRECTORIES)
+ list(APPEND dirsToLookFor ${used_LINK_DIRECTORIES})
+ list(REMOVE_DUPLICATES dirsToLookFor)
+ endif()
+
+
+ ## handle plugins
+ set(pluginsList "")
+ include(parse_arguments_multi) ## this function will process recursively items of the sub-list [default print messages]
+ function(parse_arguments_multi_function results)
+ cmake_parse_arguments(pamf "VERBOSE" "PLUGIN_PATH_DEST;FROM_REL_PATH;EXEC_PATH;COMPONENT" "" ${ARGN}) ## EXEC_PATH and COMPONENT are for exclusive internal use
+ list(REMOVE_DUPLICATES pamf_UNPARSED_ARGUMENTS)
+ foreach(PLUGIN_PATH_NAME ${pamf_UNPARSED_ARGUMENTS})
+ if(EXISTS ${PLUGIN_PATH_NAME})
+ if(IS_DIRECTORY ${PLUGIN_PATH_NAME})
+ if(pamf_VERBOSE)
+ message(WARNING "${PLUGIN_PATH_NAME} IS_DIRECTORY, cannot installed a directory, please give a path filename")
+ endif()
+ else()
+ if(NOT pamf_PLUGIN_PATH_DEST)
+ set(PLUGIN_PATH_DEST ${pamf_EXEC_PATH}) ## the default dest value
+ else()
+ set(PLUGIN_PATH_DEST ${pamf_PLUGIN_PATH_DEST})
+ endif()
+
+ if(pamf_FROM_REL_PATH)
+ file(TO_CMAKE_PATH ${PLUGIN_PATH_NAME} PLUGIN_PATH_NAME)
+ get_filename_component(PLUGIN_PATH ${PLUGIN_PATH_NAME} PATH)
+ unset(PLUGIN_PATH_LIST)
+ unset(PLUGIN_PATH_LIST_COUNT)
+ unset(PLUGIN_REL_PATH_LIST)
+ unset(PLUGIN_REL_PATH)
+ string(REPLACE "/" ";" PLUGIN_PATH_LIST ${PLUGIN_PATH}) ## create a list of dir
+ list(FIND PLUGIN_PATH_LIST ${pamf_FROM_REL_PATH} id)
+ list(LENGTH PLUGIN_PATH_LIST PLUGIN_PATH_LIST_COUNT)
+ if(${id} GREATER 0)
+ math(EXPR id "${id}+1") ## matches relative path not include
+ math(EXPR PLUGIN_PATH_LIST_COUNT "${PLUGIN_PATH_LIST_COUNT}-1") ## the end of the list
+ foreach(i RANGE ${id} ${PLUGIN_PATH_LIST_COUNT})
+ list(GET PLUGIN_PATH_LIST ${i} out)
+ list(APPEND PLUGIN_REL_PATH_LIST ${out})
+ endforeach()
+ foreach(dir ${PLUGIN_REL_PATH_LIST})
+ set(PLUGIN_REL_PATH "${PLUGIN_REL_PATH}/${dir}")
+ endforeach()
+ endif()
+ set(PLUGIN_PATH_DEST ${PLUGIN_PATH_DEST}${PLUGIN_REL_PATH})
+ endif()
+
+ install(FILES ${PLUGIN_PATH_NAME} CONFIGURATIONS ${inst_run_CONFIG_TYPE} DESTINATION ${PLUGIN_PATH_DEST} COMPONENT ${pamf_COMPONENT})
+ get_filename_component(pluginName ${PLUGIN_PATH_NAME} NAME)
+ if(IS_ABSOLUTE ${PLUGIN_PATH_DEST})
+ else()
+ set(PLUGIN_PATH_DEST ${inst_run_INSTALL_FOLDER}/${PLUGIN_PATH_DEST})
+ endif()
+ list(APPEND pluginsList ${PLUGIN_PATH_DEST}/${pluginName})
+ endif()
+ else()
+ message(WARNING "You need to provide a valid PLUGIN_PATH_NAME")
+ set(pluginsList )
+ endif()
+ endforeach()
+ set(${results} ${pluginsList} PARENT_SCOPE)
+ endfunction()
+
+ if(inst_run_VERBOSE)
+ list(APPEND extra_flags_to_add VERBOSE)
+ endif()
+ list(APPEND extra_flags_to_add EXEC_PATH ${EXEC_PATH} COMPONENT ${inst_run_COMPONENT}) ## for internal use inside overloaded function
+ list(LENGTH inst_run_PLUGINS inst_run_PLUGINS_count)
+ if(${inst_run_PLUGINS_count} GREATER 0)
+ parse_arguments_multi(PLUGIN_PATH_NAME inst_run_PLUGINS ${inst_run_PLUGINS} ## see internal overload parse_arguments_multi_function for processing each sub-list
+ NEED_RESULTS ${inst_run_PLUGINS_count} ## this is used to check when we are in the first loop (in order to reset parse_arguments_multi_results)
+ EXTRAS_FLAGS ${extra_flags_to_add} ## this is used to allow catching additional internal flags of our overloaded function
+ )
+ endif()
+
+ #message(parse_arguments_multi_results = ${parse_arguments_multi_results})
+ list(APPEND pluginsList ${parse_arguments_multi_results})
+
+
+
+ ## Install rules for required system runtimes such as MSVCRxx.dll
+ set(CMAKE_INSTALL_SYSTEM_RUNTIME_LIBS_SKIP ON)
+ include(InstallRequiredSystemLibraries)
+ if(CMAKE_INSTALL_SYSTEM_RUNTIME_LIBS)
+ install(FILES ${CMAKE_INSTALL_SYSTEM_RUNTIME_LIBS}
+ CONFIGURATIONS ${inst_run_CONFIG_TYPE}
+ DESTINATION ${EXEC_PATH}
+ COMPONENT ${inst_run_COMPONENT}
+ )
+ endif()
+
+ ## print what we are doing to do
+ if(inst_run_VERBOSE)
+ message(STATUS "[install_runtime] On install target call, cmake will try to resolve dependencies for given app:\n ${installedFilePathTargetAppToResolve} (with plausible postfix: ${inst_run_PLAUSIBLES_POSTFIX})")
+ if(pluginsList)
+ message(STATUS " and also for plugins :")
+ foreach(plugin ${pluginsList})
+ message(STATUS " ${plugin}")
+ endforeach()
+ endif()
+ message(STATUS " Looking for dependencies into:")
+ foreach(dir ${dirsToLookFor})
+ message(STATUS " ${dir}")
+ endforeach()
+ endif()
+
+ ## Install rules for required dependencies libs/plugins for the target app
+ ## will resolve all installed target files with config modes postfixes
+ string(TOUPPER ${inst_run_CONFIG_TYPE} inst_run_CONFIG_TYPE_UC)
+ get_target_property(postfix ${inst_run_TARGET} "${inst_run_CONFIG_TYPE_UC}_POSTFIX")
+ install(CODE "set(target \"${inst_run_TARGET}\")" COMPONENT ${inst_run_COMPONENT} CONFIGURATIONS ${CONFIG_TYPE})
+ install(CODE "set(inst_run_CONFIG_TYPE \"${inst_run_CONFIG_TYPE}\")" COMPONENT ${inst_run_COMPONENT} CONFIGURATIONS ${CONFIG_TYPE})
+ install(CODE "set(inst_run_INSTALL_FOLDER \"${inst_run_INSTALL_FOLDER}\")" COMPONENT ${inst_run_COMPONENT} CONFIGURATIONS ${CONFIG_TYPE})
+ install(CODE "set(app \"${EXEC_PATH}/${EXEC_NAME}${postfix}${CMAKE_EXECUTABLE_SUFFIX}\")" COMPONENT ${inst_run_COMPONENT} CONFIGURATIONS ${CONFIG_TYPE})
+ install(CODE "set(dirsToLookFor \"${dirsToLookFor}\")" COMPONENT ${inst_run_COMPONENT} CONFIGURATIONS ${CONFIG_TYPE})
+ install(CODE
+ [[
+ if("${CMAKE_INSTALL_CONFIG_NAME}" STREQUAL "${inst_run_CONFIG_TYPE}")
+ message(STATUS "Installing ${target} dependencies...")
+
+ file(GET_RUNTIME_DEPENDENCIES
+ EXECUTABLES ${app}
+ RESOLVED_DEPENDENCIES_VAR _r_deps
+ UNRESOLVED_DEPENDENCIES_VAR _u_deps
+ CONFLICTING_DEPENDENCIES_PREFIX _c_deps
+ DIRECTORIES ${dirsToLookFor}
+ PRE_EXCLUDE_REGEXES "api-ms-*"
+ POST_EXCLUDE_REGEXES ".*system32/.*\\.dll" ".*SysWOW64/.*\\.dll"
+ )
+
+ if(_u_deps)
+ message(WARNING "There were unresolved dependencies for executable ${EXEC_FILE}: \"${_u_deps}\"!")
+ endif()
+ if(_c_deps_FILENAMES)
+ message(WARNING "There were conflicting dependencies for executable ${EXEC_FILE}: \"${_c_deps_FILENAMES}\"!")
+ endif()
+
+ foreach(_file ${_r_deps})
+ file(INSTALL
+ DESTINATION "${inst_run_INSTALL_FOLDER}/bin"
+ TYPE SHARED_LIBRARY
+ FOLLOW_SYMLINK_CHAIN
+ FILES "${_file}"
+ )
+ endforeach()
+ endif()
+ ]]
+ COMPONENT ${inst_run_COMPONENT} CONFIGURATIONS ${CONFIG_TYPE}
+ )
+
+endfunction()
+
+## High level macro to install resources in the correct folder
+##
+## EXECUTABLE: [opt] option to copy files as programs
+## RELATIVE : [opt] copy files relatively to current folder
+## TYPE : [opt] type and folder where to store the files
+## FOLDER : [opt] subfolder to use
+## FILES : [opt] contains a list of resources files to copy to install folder
+macro(ibr_install_rsc target)
+ cmake_parse_arguments(install_rsc_${target} "EXECUTABLE;RELATIVE" "TYPE;FOLDER" "FILES" ${ARGN})
+ set(rsc_target "${target}_${install_rsc_${target}_TYPE}")
+
+ if(install_rsc_${target}_FOLDER)
+ set(rsc_folder "${install_rsc_${target}_TYPE}/${install_rsc_${target}_FOLDER}")
+ else()
+ set(rsc_folder "${install_rsc_${target}_TYPE}")
+ endif()
+
+ add_custom_target(${rsc_target}
+ COMMENT "run the ${install_rsc_${target}_TYPE} installation only for ${target} (component ${rsc_target})"
+ VERBATIM)
+ foreach(scriptFile ${install_rsc_${target}_FILES})
+ if(install_rsc_${target}_RELATIVE)
+ file(RELATIVE_PATH relativeFilename ${CMAKE_CURRENT_SOURCE_DIR} ${scriptFile})
+ else()
+ get_filename_component(relativeFilename ${scriptFile} NAME)
+ endif()
+
+ if(DEFINED CMAKE_BUILD_TYPE) ## for make/nmake based
+ add_custom_command(TARGET ${rsc_target} POST_BUILD
+ COMMAND ${CMAKE_COMMAND} -E
+ copy_if_different ${scriptFile} ${CMAKE_INSTALL_PREFIX_${CMAKE_BUILD_TYPE}}/${rsc_folder}/${relativeFilename})
+ endif()
+ foreach(CONFIG_TYPES ${CMAKE_CONFIGURATION_TYPES}) ## for multi config types (MSVC based)
+ string(TOUPPER ${CONFIG_TYPES} CONFIG_TYPES_UC)
+ add_custom_command(TARGET ${rsc_target} POST_BUILD
+ COMMAND ${CMAKE_COMMAND} -E
+ copy_if_different ${scriptFile} ${CMAKE_INSTALL_PREFIX_${CONFIG_TYPES_UC}}/${rsc_folder}/${relativeFilename})
+ endforeach()
+ endforeach()
+
+ get_target_property(INSTALL_RSC_BUILD_FOLDER ${target} FOLDER)
+ set_target_properties(${rsc_target} PROPERTIES FOLDER ${INSTALL_RSC_BUILD_FOLDER})
+
+ add_dependencies(${target} ${rsc_target})
+ add_dependencies(PREBUILD ${rsc_target})
+
+ if(DEFINED CMAKE_BUILD_TYPE) ## for make/nmake based
+ resourceFile(ADD ${rsc_folder} CONFIG_TYPE ${CMAKE_BUILD_TYPE} FILE_PATH "${CMAKE_INSTALL_PREFIX_${CMAKE_BUILD_TYPE}}/ibr_resources.ini")
+
+ if(install_rsc_${target}_EXECUTABLE)
+ install(
+ PROGRAMS ${install_rsc_${target}_FILES}
+ CONFIGURATIONS ${CMAKE_BUILD_TYPE}
+ DESTINATION "${CMAKE_INSTALL_PREFIX_${CMAKE_BUILD_TYPE}}/${rsc_folder}"
+ )
+ else()
+ install(
+ FILES ${install_rsc_${target}_FILES}
+ CONFIGURATIONS ${CMAKE_BUILD_TYPE}
+ DESTINATION "${CMAKE_INSTALL_PREFIX_${CMAKE_BUILD_TYPE}}/${rsc_folder}"
+ )
+ endif()
+ endif()
+ foreach(CONFIG_TYPES ${CMAKE_CONFIGURATION_TYPES}) ## for multi config types (MSVC based)
+ string(TOUPPER ${CONFIG_TYPES} CONFIG_TYPES_UC)
+ resourceFile(ADD ${rsc_folder} CONFIG_TYPE ${CONFIG_TYPES} FILE_PATH "${CMAKE_INSTALL_PREFIX_${CONFIG_TYPES_UC}}/ibr_resources.ini")
+
+ if(install_rsc_${target}_EXECUTABLE)
+ install(
+ PROGRAMS ${install_rsc_${target}_FILES}
+ CONFIGURATIONS ${CONFIG_TYPES}
+ DESTINATION "${CMAKE_INSTALL_PREFIX_${CONFIG_TYPES_UC}}/${rsc_folder}"
+ )
+ else()
+ install(
+ FILES ${install_rsc_${target}_FILES}
+ CONFIGURATIONS ${CONFIG_TYPES}
+ DESTINATION "${CMAKE_INSTALL_PREFIX_${CONFIG_TYPES_UC}}/${rsc_folder}"
+ )
+ endif()
+ endforeach()
+endmacro()
+
+
+## High level macro to install in an homogen way all our ibr targets (it use some functions inside this file)
+##
+## RSC_FILE_ADD : [opt] is used to auto write/append relative paths of target resources into a common file
+## INSTALL_PDB : [opt] is used to auto install PDB file (when using MSVC according to the target type)
+## STANDALONE : [opt] bool ON/OFF var to call install_runtime or not (for bundle resolution)
+## DIRS : [opt] used if STANDALONE set to ON, see install_runtime doc
+## PLUGINS: [opt] used if STANDALONE set to ON, see install_runtime doc
+## MSVC_CMD : [opt] used to specify an absolute filePathName application to launch with the MSVC IDE Debugger associated to this target (project file)
+## MSVC_ARGS : [opt] load the MSVC debugger with correct settings (app path, args, working dir)
+##
+macro(ibr_install_target target)
+ cmake_parse_arguments(ibrInst${target} "VERBOSE;INSTALL_PDB" "COMPONENT;MSVC_ARGS;STANDALONE;RSC_FOLDER" "SHADERS;RESOURCES;SCRIPTS;DIRS;PLUGINS" ${ARGN})
+
+ if(ibrInst${target}_RSC_FOLDER)
+ set(rsc_folder "${ibrInst${target}_RSC_FOLDER}")
+ else()
+ set(rsc_folder "${target}")
+ endif()
+
+ if(ibrInst${target}_SHADERS)
+ ibr_install_rsc(${target} EXECUTABLE TYPE "shaders" FOLDER ${rsc_folder} FILES "${ibrInst${target}_SHADERS}")
+ endif()
+
+ if(ibrInst${target}_RESOURCES)
+ ibr_install_rsc(${target} TYPE "resources" FOLDER ${rsc_folder} FILES "${ibrInst${target}_RESOURCES}")
+ endif()
+
+ if(ibrInst${target}_SCRIPTS)
+ ibr_install_rsc(${target} EXECUTABLE TYPE "scripts" FOLDER ${rsc_folder} FILES "${ibrInst${target}_SCRIPTS}")
+ endif()
+
+ if(ibrInst${target}_COMPONENT)
+ set(installCompArg COMPONENT ${ibrInst${target}_COMPONENT})
+ ## Create a custom install target based on COMPONENT
+ installTargetProject(${target} ${ibrInst${target}_COMPONENT})
+ endif()
+
+ if(DEFINED CMAKE_BUILD_TYPE) ## for make/nmake based
+ set_target_properties(${target} PROPERTIES ${CMAKE_BUILD_TYPE}_POSTFIX "${CMAKE_${CMAKE_BUILD_TYPE}_POSTFIX}")
+ get_target_property(CURRENT_TARGET_BUILD_TYPE_POSTFIX ${target} ${CMAKE_BUILD_TYPE}_POSTFIX)
+ endif()
+ foreach(CONFIG_TYPES ${CMAKE_CONFIGURATION_TYPES}) ## for multi config types (MSVC based)
+ string(TOUPPER ${CONFIG_TYPES} CONFIG_TYPES_UC)
+ set_target_properties(${target} PROPERTIES ${CONFIG_TYPES_UC}_POSTFIX "${CMAKE_${CONFIG_TYPES_UC}_POSTFIX}")
+ get_target_property(CURRENT_TARGET_BUILD_TYPE_POSTFIX ${target} ${CONFIG_TYPES_UC}_POSTFIX)
+ endforeach()
+
+ ## Specify default installation rules
+ if(DEFINED CMAKE_BUILD_TYPE) ## for make/nmake based
+ install(TARGETS ${target}
+ CONFIGURATIONS ${CMAKE_BUILD_TYPE}
+ LIBRARY DESTINATION ${CMAKE_LIBRARY_OUTPUT_DIRECTORY_${CMAKE_BUILD_TYPE}} ${installCompArg}
+ ARCHIVE DESTINATION ${CMAKE_ARCHIVE_OUTPUT_DIRECTORY_${CMAKE_BUILD_TYPE}} ${installCompArg}
+ RUNTIME DESTINATION ${CMAKE_RUNTIME_OUTPUT_DIRECTORY_${CMAKE_BUILD_TYPE}} ${installCompArg}
+ )
+ install(TARGETS ${target}
+ CONFIGURATIONS ${CMAKE_BUILD_TYPE}
+ LIBRARY DESTINATION ${CMAKE_RUNTIME_OUTPUT_DIRECTORY_${CMAKE_BUILD_TYPE}} ${installCompArg}
+ ARCHIVE DESTINATION ${CMAKE_RUNTIME_OUTPUT_DIRECTORY_${CMAKE_BUILD_TYPE}} ${installCompArg}
+ )
+ endif()
+ foreach(CONFIG_TYPES ${CMAKE_CONFIGURATION_TYPES}) ## for multi config types (MSVC based)
+ string(TOUPPER ${CONFIG_TYPES} CONFIG_TYPES_UC)
+ install(TARGETS ${target}
+ CONFIGURATIONS ${CONFIG_TYPES}
+ LIBRARY DESTINATION ${CMAKE_LIBRARY_OUTPUT_DIRECTORY_${CONFIG_TYPES_UC}} ${installCompArg}
+ ARCHIVE DESTINATION ${CMAKE_ARCHIVE_OUTPUT_DIRECTORY_${CONFIG_TYPES_UC}} ${installCompArg}
+ RUNTIME DESTINATION ${CMAKE_RUNTIME_OUTPUT_DIRECTORY_${CONFIG_TYPES_UC}} ${installCompArg}
+ )
+ install(TARGETS ${target}
+ CONFIGURATIONS ${CONFIG_TYPES}
+ LIBRARY DESTINATION ${CMAKE_RUNTIME_OUTPUT_DIRECTORY_${CONFIG_TYPES_UC}} ${installCompArg}
+ ARCHIVE DESTINATION ${CMAKE_RUNTIME_OUTPUT_DIRECTORY_${CONFIG_TYPES_UC}} ${installCompArg}
+ )
+ endforeach()
+
+ if(ibrInst${target}_INSTALL_PDB)
+ if(DEFINED CMAKE_BUILD_TYPE) ## for make/nmake based
+ installPDB(${target} ${CMAKE_BUILD_TYPE}
+ LIBRARY_DEST ${CMAKE_LIBRARY_OUTPUT_DIRECTORY_${CMAKE_BUILD_TYPE}}
+ ARCHIVE_DEST ${CMAKE_ARCHIVE_OUTPUT_DIRECTORY_${CMAKE_BUILD_TYPE}}
+ RUNTIME_DEST ${CMAKE_RUNTIME_OUTPUT_DIRECTORY_${CMAKE_BUILD_TYPE}}
+ )
+ endif()
+ foreach(CONFIG_TYPES ${CMAKE_CONFIGURATION_TYPES}) ## for multi config types (MSVC based)
+ string(TOUPPER ${CONFIG_TYPES} CONFIG_TYPES_UC)
+ installPDB(${target} ${CONFIG_TYPES}
+ LIBRARY_DEST ${CMAKE_LIBRARY_OUTPUT_DIRECTORY_${CONFIG_TYPES_UC}}
+ ARCHIVE_DEST ${CMAKE_ARCHIVE_OUTPUT_DIRECTORY_${CONFIG_TYPES_UC}}
+ RUNTIME_DEST ${CMAKE_RUNTIME_OUTPUT_DIRECTORY_${CONFIG_TYPES_UC}}
+ )
+ endforeach()
+ endif()
+
+ ## install dynamic necessary dependencies
+ if(ibrInst${target}_STANDALONE)
+ get_target_property(type ${target} TYPE)
+ if(${type} MATCHES "EXECUTABLE")
+
+ if(ibrInst${target}_VERBOSE)
+ set(VERBOSE VERBOSE)
+ else()
+ set(VERBOSE )
+ endif()
+
+ if(DEFINED CMAKE_BUILD_TYPE) ## for make/nmake based
+ install_runtime(bin/${target}${CMAKE_EXECUTABLE_SUFFIX} ## default relative to CMAKE_INSTALL_PREFIX
+ INSTALL_FOLDER "${CMAKE_INSTALL_PREFIX_${CMAKE_BUILD_TYPE}}"
+ CONFIG_TYPE ${CMAKE_BUILD_TYPE}
+ ${VERBOSE}
+ TARGET ${target}
+ ${installCompArg}
+ PLUGINS ## will be installed
+ ${ibrInst${target}_PLUGINS}
+ DIRS ${CMAKE_RUNTIME_OUTPUT_DIRECTORY_${CMAKE_BUILD_TYPE}}
+ ${ibrInst${target}_DIRS}
+ )
+ endif()
+ foreach(CONFIG_TYPES ${CMAKE_CONFIGURATION_TYPES}) ## for multi config types (MSVC based)
+ string(TOUPPER ${CONFIG_TYPES} CONFIG_TYPES_UC)
+ install_runtime(bin/${target}${CMAKE_EXECUTABLE_SUFFIX} ## default relative to CMAKE_INSTALL_PREFIX
+ INSTALL_FOLDER "${CMAKE_INSTALL_PREFIX_${CONFIG_TYPES_UC}}"
+ CONFIG_TYPE ${CONFIG_TYPES}
+ ${VERBOSE}
+ TARGET ${target}
+ ${installCompArg}
+ PLUGINS ## will be installed
+ ${ibrInst${target}_PLUGINS}
+ DIRS ${CMAKE_RUNTIME_OUTPUT_DIRECTORY_${CONFIG_TYPES_UC}}
+ ${ibrInst${target}_DIRS}
+ )
+ endforeach()
+ else()
+ message(WARNING "STANDALONE option is only compatible with EXECUTABLES target type. Skip the STANDALONE installation process.")
+ endif()
+ endif()
+
+ ## Provide a way to directly load the MSVC debugger with correct settings
+ if(MSVC)
+ if(ibrInst${target}_MSVC_CMD) ## command absolute filePathName is optional as the default is to use the installed target file application
+ set(msvcCmdArg COMMAND ${ibrInst${target}_MSVC_CMD}) ## flag following by the value (both to pass to the MSVCsetUserCommand function)
+ endif()
+ if(ibrInst${target}_MSVC_ARGS) ## args (between quotes) are optional
+ set(msvcArgsArg ARGS ${ibrInst${target}_MSVC_ARGS}) ## flag following by the value (both to pass to the MSVCsetUserCommand function)
+ endif()
+ get_target_property(type ${target} TYPE)
+ if( (ibrInst${target}_MSVC_CMD OR ibrInst${target}_MSVC_ARGS) OR (${type} MATCHES "EXECUTABLE") )
+ include(MSVCsetUserCommand)
+ if(DEFINED CMAKE_BUILD_TYPE) ## for make/nmake based
+ MSVCsetUserCommand( ${target}
+ PATH ${CMAKE_OUTPUT_BIN_${CMAKE_BUILD_TYPE}} ##FILE option not necessary since it deduced from targetName
+ ARGS "${SIBR_PROGRAMARGS}"
+ ${msvcCmdArg}
+ #${msvcArgsArg}
+ WORKING_DIR ${CMAKE_OUTPUT_BIN_${CMAKE_BUILD_TYPE}}
+ )
+ endif()
+ foreach(CONFIG_TYPES ${CMAKE_CONFIGURATION_TYPES}) ## for multi config types (MSVC based)
+ string(TOUPPER ${CONFIG_TYPES} CONFIG_TYPES_UC)
+ MSVCsetUserCommand( ${target}
+ PATH ${CMAKE_OUTPUT_BIN_${CONFIG_TYPES_UC}} ##FILE option not necessary since it deduced from targetName
+ ARGS "${SIBR_PROGRAMARGS}"
+ ${msvcCmdArg}
+ #${msvcArgsArg}
+ WORKING_DIR ${CMAKE_OUTPUT_BIN_${CONFIG_TYPES_UC}}
+ )
+ endforeach()
+ elseif(NOT ${type} MATCHES "EXECUTABLE")
+ #message("Cannot set MSVCsetUserCommand with target ${target} without COMMAND parameter as it is not an executable (skip it)")
+ endif()
+ endif()
+
+endmacro()
diff --git a/submodules/gaussian-splatting/SIBR_viewers/cmake/linux/parse_arguments_multi.cmake b/submodules/gaussian-splatting/SIBR_viewers/cmake/linux/parse_arguments_multi.cmake
new file mode 100644
index 0000000000000000000000000000000000000000..4f19e414fd3dc12a6af222f2471eec7410f617ee
--- /dev/null
+++ b/submodules/gaussian-splatting/SIBR_viewers/cmake/linux/parse_arguments_multi.cmake
@@ -0,0 +1,304 @@
+# Copyright (C) 2020, Inria
+# GRAPHDECO research group, https://team.inria.fr/graphdeco
+# All rights reserved.
+#
+# This software is free for non-commercial, research and evaluation use
+# under the terms of the LICENSE.md file.
+#
+# For inquiries contact sibr@inria.fr and/or George.Drettakis@inria.fr
+
+
+if(NOT WIN32 OR __parse_arguments_multi_cmake_INCLUDED__)
+ return()
+else()
+ set(__parse_arguments_multi_cmake_INCLUDED__ ON)
+endif()
+
+## This macro allow to process repeating multi value args from a given function which use cmake_parse_arguments module.
+##
+## cmake_parse_arguments multi args standard behavior:
+## function(foo)
+## cmake_parse_arguments(arg "" "" "MULTI" ${ARGN})
+## foreach(item IN LISTS arg_MULTI)
+## message(STATUS "${item}")
+## endforeach()
+## endfunction()
+## foo(MULTI x y MULTI z w)
+## The above code outputs 'z' and 'w'. It originally expected it to output all of 'x' 'y' 'z' 'w'.
+##
+## Using this macro inside a function which want to handle repeating multi args values
+## will recursively iterate onto the multi tags list to process each sub list.
+## It take as 1st argument the subTag flag to separate sub list from the main multi list.
+## It take as 2nd argument the nameList of the main multi list (the multiValuesArgs from cmake_parse_arguments: here it is MULTI in the example)
+## and that's why it is important that it should be a macro and not a function (to get access to external variable).
+## Then you give the content of this list allowing to be processed by the macro.
+##
+## parse_arguments_multi macro call a parse_arguments_multi_function which do actually the process from the given sub-list.
+## By default this function only print infos about what variables you are trying to pass/process (only verbose messages),
+## but, by overloading this cmake function, you will be able to externalize the process of your multi argument list.
+##
+## Usage (into a function) :
+## parse_arguments_multi(
+## [NEED_RESULTS ] [EXTRAS_FLAGS <...> <...> ...]
+## )
+##
+## Simple usage example [user point of view]:
+## foo(MULTI
+## SUB_MULTI x y
+## SUB_MULTI z w
+## )
+##
+## Simple usage example [inside a function]:
+## function(foo)
+## cmake_parse_arguments(arg "" "" "MULTI" ${ARGN})
+## include(parse_arguments_multi)
+## function(parse_arguments_multi_function )
+## #message("I'm an overloaded cmake function used by parse_arguments_multi")
+## #message("I'm processing first part of my sub list: ${ARGN}")
+## message("ARGV0=${ARGV0}")
+## message("ARGV1=${ARGV1}")
+## endfunction()
+## parse_arguments_multi(SUB_MULTI arg_MULTI ${arg_MULTI}) ## this function will process recusively items of the sub-list [default print messages]
+## endfunction()
+##
+## Will print:
+## ARGV0=z
+## ARGV1=w
+## ARGV0=x
+## ARGV1=y
+##
+## WARNING : DO NEVER ADD EXTRA THINGS TO parse_arguments_multi MACRO :
+## parse_arguments_multi(SUB_MULTI arg_MULTI ${arg_MULTI} EXTRAS foo bar SOMTHING) => will failed !!
+## use EXTRAS_FLAGS instead !!
+##
+## Advanced usage example [user point of view]:
+## bar(C:/prout/test.exe VERBOSE
+## PLUGINS
+## PLUGIN_PATH_NAME x PLUGIN_PATH_DEST w
+## PLUGIN_PATH_NAME a b PLUGIN_PATH_DEST y
+## PLUGIN_PATH_NAME c
+## )
+##
+## Advanced usage example [inside a function]:
+## function(bar execFilePathName)
+## cmake_parse_arguments(arg "VERBOSE" "" "PLUGINS" ${ARGN})
+##
+## include(parse_arguments_multi)
+## function(parse_arguments_multi_function results)
+## cmake_parse_arguments(pamf "VERBOSE" "PLUGIN_PATH_DEST;EXEC_PATH" "" ${ARGN}) ## EXEC_PATH is for internal use
+## message("")
+## message("I'm an overloaded cmake function used by parse_arguments_multi from install_runtime function")
+## message("I'm processing first part of my sub list: ${ARGN}")
+## message("PLUGIN_PATH_NAME = ${pamf_UNPARSED_ARGUMENTS}")
+## message(pamf_VERBOSE = ${pamf_VERBOSE})
+## message("pamf_PLUGIN_PATH_DEST = ${pamf_PLUGIN_PATH_DEST}")
+## message(pamf_EXEC_PATH = ${pamf_EXEC_PATH})
+## if(NOT ${pamf_PLUGIN_PATH_DEST})
+## set(pamf_PLUGIN_PATH_DEST ${pamf_EXEC_PATH})
+## endif()
+## foreach(plugin ${pamf_UNPARSED_ARGUMENTS})
+## get_filename_component(pluginName ${plugin} NAME)
+## list(APPEND pluginsList ${pamf_PLUGIN_PATH_DEST}/${pluginName})
+## endforeach()
+## set(${results} ${pluginsList} PARENT_SCOPE)
+## endfunction()
+##
+## if(arg_VERBOSE)
+## list(APPEND extra_flags_to_add VERBOSE) ## here we transmit the VERNOSE flag
+## endif()
+## get_filename_component(EXEC_PATH ${execFilePathName} PATH) ## will be the default value if PLUGIN_PATH_DEST option is not provided
+## list(APPEND extra_flags_to_add EXEC_PATH ${EXEC_PATH})
+## list(LENGTH arg_PLUGINS arg_PLUGINS_count)
+## parse_arguments_multi(PLUGIN_PATH_NAME arg_PLUGINS ${arg_PLUGINS}
+## NEED_RESULTS ${arg_PLUGINS_count} ## this is used to check when we are in the first loop (in order to reset parse_arguments_multi_results)
+## EXTRAS_FLAGS ${extra_flags_to_add} ## this is used to allow catching VERBOSE and PLUGIN_PATH_DEST flags of our overloaded function
+## )
+## endfunction()
+## message(parse_arguments_multi_results = ${parse_arguments_multi_results}) ## list of the whole pluginsList
+## #Will print w/x;a/y;b/y;C:/prout/c
+##
+## NOTE that here, since our overloaded function need to provide a result list, we use the other parse_arguments_multi_function signature (the which one with a results arg)
+##
+
+function(parse_arguments_multi_function_default) ## used in case of you want to reset the default behavior of this function process
+ message("[default function] parse_arguments_multi_function(ARGC=${ARGC} ARGV=${ARGV} ARGN=${ARGN})")
+ message("This function is used by parse_arguments_multi and have to be overloaded to process sub list of multi values args")
+endfunction()
+
+function(parse_arguments_multi_function ) ## => the function to overload
+ parse_arguments_multi_function_default(${ARGN})
+endfunction()
+
+## first default signature above
+##------------------------------
+## second results signature behind
+
+function(parse_arguments_multi_function_default result) ## used in case of you want to reset the default behavior of this function process
+ message("[default function] parse_arguments_multi_function(ARGC=${ARGC} ARGV=${ARGV} ARGN=${ARGN})")
+ message("This function is used by parse_arguments_multi and have to be overloaded to process sub list of muluti values args")
+endfunction()
+
+function(parse_arguments_multi_function result) ## => the function to overload
+ parse_arguments_multi_function_default(result ${ARGN})
+endfunction()
+
+## => the macro to use inside your function which use cmake_parse_arguments
+# NOTE: entry point of parse_arguments_multi, which is called from win3rdPart)
+macro(parse_arguments_multi multiArgsSubTag multiArgsList #<${multiArgsList}> the content of the list
+)
+ # message (STATUS "")
+ # message(STATUS "calling parse_arguemnts_multi defined in parse_arguments_multi.cmake:141")
+ # message(STATUS "multiArgsSubTag = ${multiArgsSubTag}") # CHECK_CACHED_VAR
+ # message(STATUS "multiArgsList = ${multiArgsList}") # it contains the name of the variable which is holding the list i.e w3p_MULTI_SET
+ # message(STATUS "value of ${multiArgsList} = ${${multiArgsList}}") # a semicolon separated list of values passed to SET or MULTISET keyword in win3rdParty
+ # message(STATUS "actual values ARGN = ${ARGN}") # the same as ${${multiArgsList}}
+
+ ## INFO
+ ## starting from CMake 3.5 cmake_parse_arguments is not a module anymore and now is a native CMake command.
+ ## the behaviour is different though
+ ## In CMake 3.4, if you pass multiple times a multi_value_keyword, CMake returns the values of the LAST match
+ ## In CMake 3.5 and above, CMake returns the whole list of values that were following that multi_value_keyword
+ ## example:
+ ## cmake_parse_arguments(
+ ##
+ ## "" # options
+ ## "" # one value keywords
+ ## "MY_MULTI_VALUE_TAG"
+ ## MY_MULTI_VALUE_TAG value1 value2
+ ## MY_MULTI_VALUE_TAG value3 value4
+ ## MY_MULTI_VALUE_TAG value5 value6
+ ## )
+ ## result in CMake 3.4
+ ## _MY_MULTI_VALUE_TAG = "value5;value6"
+ ##
+ ## result in CMake 3.8
+ ## _MY_MULTI_VALUE_TAG = "value5;value6"
+
+ #include(CMakeParseArguments) #module CMakeParseArguments is obsolete since cmake 3.5
+ # cmake_parse_arguments ( args)
+ # : options (flags) pass to the macro
+ # : options that neeed a value
+ # : options that neeed more than one value
+ cmake_parse_arguments(_pam "" "NEED_RESULTS" "${multiArgsSubTag};EXTRAS_FLAGS" ${ARGN})
+
+ ## multiArgsList is the name of the list used by the multiValuesOption flag from the cmake_parse_arguments of the user function
+ ## that's why we absolutly need to use MACRO here (and also for passing parse_arguments_multi_results when NEED_RESULTS flag is set)
+
+ ## for debugging
+ #message("")
+ #message("[parse_arguments_multi] => ARGN = ${ARGN}")
+ #message("_pam_NEED_RESULTS=${_pam_NEED_RESULTS}")
+ #message("_pam_EXTRAS_FLAGS=${_pam_EXTRAS_FLAGS}")
+ # foreach(var ${_pam_${multiArgsSubTag}})
+ # message("arg=${var}")
+ # endforeach()
+
+ if (${CMAKE_VERSION} VERSION_GREATER "3.5")
+ # lets make ${_pam_${multiArgsSubTag}} behave as it is in version 3.4
+ # that means, cmake_parse_arguments should have only the last values of a multi set for a given keyword
+
+ # message("")
+ # message("values in multiArgsList")
+ # foreach(val ${${multiArgsList}})
+ # message(STATUS ${val})
+ # endforeach()
+ # message("end values in multiArgsList")
+
+
+ set(lastIndexFound OFF)
+ list(LENGTH ${multiArgsList} argnLength)
+ # message(${argnLength})
+ math(EXPR argnLength "${argnLength}-1") # make last index a valid one
+ set(recordIndex 0)
+ set(records "") # clear records list
+ set(record0 "") # clear first record list
+ foreach(iter RANGE ${argnLength})
+ list(GET ${multiArgsList} ${iter} value)
+ # message(STATUS "index=${iter} value=${value}")
+ if (${value} STREQUAL ${multiArgsSubTag})
+ if (lastIndexFound)
+ list(APPEND records ${recordIndex}) # records store the list NAMES
+ math(EXPR recordIndex "${recordIndex}+1")
+ set(record${recordIndex} "") # clear record list
+ else ()
+ set(lastIndexFound ON)
+ endif()
+
+ set(lastIndex ${iter})
+ else ()
+ if (lastIndexFound)
+ # message(${value})
+ list(APPEND record${recordIndex} ${value})
+ endif()
+ endif()
+ endforeach()
+
+ # save the last list of values
+ if (lastIndexFound)
+ list(APPEND records ${recordIndex}) # records store the list NAMES
+ endif()
+
+ # set multiArgsList to make it behave like CMake 3.4
+ # message("")
+ # message("using my records")
+ foreach(recordName ${records})
+ # message(${recordName})
+ # foreach(value ${record${recordName}})
+ # message(${value})
+ # endforeach()
+ # message("")
+ set(_pam_${multiArgsSubTag} ${record${recordName}})
+ endforeach()
+ # message(${_pam_${multiArgsSubTag}})
+
+ # message("")
+ # message("using argn")
+ # foreach(value ${ARGN})
+ # message(${value})
+ # endforeach()
+ endif() # end if cmake > 3.5
+
+ # message("values with pam ${_pam_${multiArgsSubTag}}")
+
+ ## check and init
+ list(LENGTH ${multiArgsList} globalListCount) # GLUT_TRACE: globalListCound=16 in CMake3.4 and CMake3.8
+ # message(STATUS "nr items in multiArgsList: ${globalListCount}")
+ math(EXPR globalListCount "${globalListCount}-1") ## because it will contain [multiArgsSubTag + ${multiArgsList}]
+ if(_pam_NEED_RESULTS)
+ if(${globalListCount} EQUAL ${_pam_NEED_RESULTS})
+ ## first time we enter into this macro (because we call it recursively)
+ unset(parse_arguments_multi_results)
+ endif()
+ endif()
+
+ ## process the part of the multi agrs list
+ ## ${ARGN} shouldn't be passed to the function in order to avoid missmatch size list ${multiArgsList} and _pam_${multiArgsSubTag}
+ ## if you want to pass extra internal flags from your function to this callback, use EXTRAS_FLAGS
+ if(_pam_NEED_RESULTS)
+ parse_arguments_multi_function(parse_arguments_multi_function_result ${_pam_${multiArgsSubTag}} ${_pam_EXTRAS_FLAGS})
+ list(APPEND parse_arguments_multi_results ${parse_arguments_multi_function_result})
+ else()
+ # message(STATUS "about to call parse_arguments_multi_function in parse_arguments_multi.cmake:177 ${_pam_${multiArgsSubTag}} and extra flags ${_pam_EXTRAS_FLAGS}")
+ parse_arguments_multi_function(${_pam_${multiArgsSubTag}} ${_pam_EXTRAS_FLAGS})
+ endif()
+
+ ## remove just processed items from the main list to process (multiArgsList)
+ list(REVERSE ${multiArgsList})
+ list(LENGTH _pam_${multiArgsSubTag} subTagListCount)
+ unset(ids)
+ foreach(id RANGE ${subTagListCount})
+ list(APPEND ids ${id})
+ endforeach()
+ list(REMOVE_AT ${multiArgsList} ${ids})
+ list(REVERSE ${multiArgsList})
+
+ ## test if remain sub multi list to process (recursive call) or finish the process
+ list(LENGTH ${multiArgsList} mainTagListCount)
+ if(${mainTagListCount} GREATER 1)
+ ## do not pass ${ARGN} just because it will re pass the initial 2 inputs args and we wont as they was consumed (in order to avoir conflicts)
+ # message(STATUS "about to call a parse_arguments_multi but without knowing where the definition is going to be taken from")
+ parse_arguments_multi(${multiArgsSubTag} ${multiArgsList} ${${multiArgsList}}
+ NEED_RESULTS ${_pam_NEED_RESULTS} EXTRAS_FLAGS ${_pam_EXTRAS_FLAGS}
+ )
+ endif()
+endmacro()
diff --git a/submodules/gaussian-splatting/SIBR_viewers/cmake/linux/sibr_library.cmake b/submodules/gaussian-splatting/SIBR_viewers/cmake/linux/sibr_library.cmake
new file mode 100644
index 0000000000000000000000000000000000000000..08a30ad940dbd53f14640b53c5d410264d910cad
--- /dev/null
+++ b/submodules/gaussian-splatting/SIBR_viewers/cmake/linux/sibr_library.cmake
@@ -0,0 +1,174 @@
+# Copyright (C) 2020, Inria
+# GRAPHDECO research group, https://team.inria.fr/graphdeco
+# All rights reserved.
+#
+# This software is free for non-commercial, research and evaluation use
+# under the terms of the LICENSE.md file.
+#
+# For inquiries contact sibr@inria.fr and/or George.Drettakis@inria.fr
+
+
+# NOTE
+# This feature is used to easily download, store and link external dependencies. This
+# requires to prepare pre-compiled libraries (to download). For now, packages have
+# only be prepare for Windows 64-bit with Visual Studio 2012. (You should re-build
+# everything if you want to use another version of Visual Studio/ another compiler).
+
+# NOTE ABOUT UNIX SYSTEMS
+# There is no need for "searching mechanism". This function is discard and your
+# libraries should be installed is the standard folders that are:
+#
+# /usr/include/
+# /usr/lib/
+# /usr/lib64/
+# for packages downloaded using apt-get/yum
+#
+# /usr/local/include/
+# /usr/local/lib/
+# /usr/local/lib64/
+# for packages manually installed ("make install")
+#
+# if you encounter problems when linking (e.g. lib not found even if it is installed),
+# please check these folders are in your search PATH environment variables.
+
+set(EXTLIBS_PACKAGE_FOLDER "${CMAKE_SOURCE_DIR}/extlibs")
+
+function(sibr_addlibrary)
+ if(NOT WIN32)
+ return()
+ endif()
+
+ file(MAKE_DIRECTORY ${EXTLIBS_PACKAGE_FOLDER})
+ cmake_parse_arguments(args "VCID" "VERBOSE;TIMEOUT;DEFAULT_USE;NAME;VERSION;MSVC11;MSVC12;MSVC14;MSVC17" "MULTI_SET;SET" ${ARGN})
+
+
+ if (NOT "${args_VERSION}" MATCHES "")
+ message(WARNING "VERSION is not implemented yet")
+ endif()
+
+ set(lcname "")
+ set(ucname "")
+ string(TOLOWER "${args_NAME}" lcname)
+ string(TOUPPER "${args_NAME}" ucname)
+
+ set(LIB_PACKAGE_FOLDER "${EXTLIBS_PACKAGE_FOLDER}/${lcname}")
+ win3rdParty(${ucname}
+ $
+ VERBOSE ${args_VERBOSE}
+ TIMEOUT ${args_TIMEOUT}
+ DEFAULT_USE ${args_DEFAULT_USE}
+ MSVC11 "${LIB_PACKAGE_FOLDER}" "${args_MSVC11}"
+ MSVC12 "${LIB_PACKAGE_FOLDER}" "${args_MSVC12}"
+ MSVC14 "${LIB_PACKAGE_FOLDER}" "${args_MSVC14}" # TODO SV: make sure to build this library if required
+ MSVC17 "${LIB_PACKAGE_FOLDER}" "${args_MSVC17}"
+ SET ${args_SET}
+ MULTI_SET ${args_MULTI_SET}
+ )
+
+ # Add include/ directory
+ # and lib/ directories
+
+ # TODO SV: paths not matching with current hierarchy. example: libraw/libraw-0.17.1/include
+ # SR: The link directories will also be used to lookup for dependency DLLs to copy in the install directory.
+ # Some libraries put the DLLs in the bin/ directory, so we include those.
+ file(GLOB subdirs RELATIVE ${LIB_PACKAGE_FOLDER} ${LIB_PACKAGE_FOLDER}/*)
+ set(dirlist "")
+ foreach(dir ${subdirs})
+ if(IS_DIRECTORY ${LIB_PACKAGE_FOLDER}/${dir})
+ # message("adding ${LIB_PACKAGE_FOLDER}/${dir}/include/ to the include directories")
+ include_directories("${LIB_PACKAGE_FOLDER}/${dir}/include/")
+ # message("adding ${LIB_PACKAGE_FOLDER}/${dir}/lib[64] to the link directories")
+ link_directories("${LIB_PACKAGE_FOLDER}/${dir}/")
+ link_directories("${LIB_PACKAGE_FOLDER}/${dir}/lib/")
+ link_directories("${LIB_PACKAGE_FOLDER}/${dir}/lib64/")
+ link_directories("${LIB_PACKAGE_FOLDER}/${dir}/bin/")
+ endif()
+ endforeach()
+
+endfunction()
+
+include(FetchContent)
+include(git_describe)
+include(install_runtime)
+
+function(sibr_gitlibrary)
+ cmake_parse_arguments(args "" "TARGET;GIT_REPOSITORY;GIT_TAG;ROOT_DIR;SOURCE_DIR" "INCLUDE_DIRS" ${ARGN})
+ if(NOT args_TARGET)
+ message(FATAL "Error on sibr_gitlibrary : please define your target name.")
+ return()
+ endif()
+
+ if(NOT args_ROOT_DIR)
+ set(args_ROOT_DIR ${args_TARGET})
+ endif()
+
+ if(NOT args_SOURCE_DIR)
+ set(args_SOURCE_DIR ${args_TARGET})
+ endif()
+
+ if(args_GIT_REPOSITORY AND args_GIT_TAG)
+ if(EXISTS ${CMAKE_SOURCE_DIR}/extlibs/${args_ROOT_DIR}/${args_SOURCE_DIR}/.git)
+ git_describe(
+ PATH ${CMAKE_SOURCE_DIR}/extlibs/${args_ROOT_DIR}/${args_SOURCE_DIR}
+ GIT_URL SIBR_GITLIBRARY_URL
+ GIT_BRANCH SIBR_GITLIBRARY_BRANCH
+ GIT_COMMIT_HASH SIBR_GITLIBRARY_COMMIT_HASH
+ GIT_TAG SIBR_GITLIBRARY_TAG
+ )
+
+ if((SIBR_GITLIBRARY_URL STREQUAL args_GIT_REPOSITORY) AND
+ ((SIBR_GITLIBRARY_BRANCH STREQUAL args_GIT_TAG) OR
+ (SIBR_GITLIBRARY_TAG STREQUAL args_GIT_TAG) OR
+ (SIBR_GITLIBRARY_COMMIT_HASH STREQUAL args_GIT_TAG)))
+ message(STATUS "Library ${args_TARGET} already available, skipping.")
+ set(SIBR_GITLIBRARY_DECLARED ON)
+ else()
+ message(STATUS "Adding library ${args_TARGET} from git...")
+ endif()
+ endif()
+
+ FetchContent_Declare(${args_TARGET}
+ GIT_REPOSITORY ${args_GIT_REPOSITORY}
+ GIT_TAG ${args_GIT_TAG}
+ GIT_SHALLOW ON
+ SOURCE_DIR ${CMAKE_SOURCE_DIR}/extlibs/${args_ROOT_DIR}/${args_SOURCE_DIR}
+ SUBBUILD_DIR ${CMAKE_SOURCE_DIR}/extlibs/${args_ROOT_DIR}/subbuild
+ BINARY_DIR ${CMAKE_SOURCE_DIR}/extlibs/${args_ROOT_DIR}/build
+ )
+ FetchContent_GetProperties(${args_TARGET})
+ string(TOLOWER "" lcTargetName)
+
+ if((NOT SIBR_GITLIBRARY_DECLARED) AND (NOT ${lcTargetName}_POPULATED))
+ message(STATUS "Populating library ${args_TARGET}...")
+ FetchContent_Populate(${args_TARGET} QUIET
+ GIT_REPOSITORY ${args_GIT_REPOSITORY}
+ GIT_TAG ${args_GIT_TAG}
+ SOURCE_DIR ${CMAKE_SOURCE_DIR}/extlibs/${args_ROOT_DIR}/${args_SOURCE_DIR}
+ SUBBUILD_DIR ${CMAKE_SOURCE_DIR}/extlibs/${args_ROOT_DIR}/subbuild
+ BINARY_DIR ${CMAKE_SOURCE_DIR}/extlibs/${args_ROOT_DIR}/build
+ )
+ endif()
+
+ add_subdirectory(${CMAKE_SOURCE_DIR}/extlibs/${args_ROOT_DIR}/${args_SOURCE_DIR} ${CMAKE_SOURCE_DIR}/extlibs/${args_ROOT_DIR}/build)
+
+ get_target_property(type ${args_TARGET} TYPE)
+ if(NOT (type STREQUAL "INTERFACE_LIBRARY"))
+ set_target_properties(${args_TARGET} PROPERTIES FOLDER "extlibs")
+
+ ibr_install_target(${args_TARGET}
+ COMPONENT ${args_TARGET}_install ## will create custom target to install only this project
+ )
+ endif()
+
+ list(APPEND ${args_TARGET}_INCLUDE_DIRS ${EXTLIBS_PACKAGE_FOLDER}/${args_ROOT_DIR})
+ list(APPEND ${args_TARGET}_INCLUDE_DIRS ${EXTLIBS_PACKAGE_FOLDER}/${args_ROOT_DIR}/${args_SOURCE_DIR})
+
+ foreach(args_INCLUDE_DIR ${args_INCLUDE_DIRS})
+ list(APPEND ${args_TARGET}_INCLUDE_DIRS ${EXTLIBS_PACKAGE_FOLDER}/${args_ROOT_DIR}/${args_SOURCE_DIR}/${args_INCLUDE_DIR})
+ endforeach()
+
+ include_directories(${${args_TARGET}_INCLUDE_DIRS})
+ else()
+ message(FATAL "Error on sibr_gitlibrary for target ${args_TARGET}: missing git tag or git url.")
+ endif()
+endfunction()
\ No newline at end of file
diff --git a/submodules/gaussian-splatting/SIBR_viewers/cmake/windows/MSVCsetUserCommand.cmake b/submodules/gaussian-splatting/SIBR_viewers/cmake/windows/MSVCsetUserCommand.cmake
new file mode 100644
index 0000000000000000000000000000000000000000..bc49770d644ca2803a9d52f9186d952b40fafdf8
--- /dev/null
+++ b/submodules/gaussian-splatting/SIBR_viewers/cmake/windows/MSVCsetUserCommand.cmake
@@ -0,0 +1,149 @@
+# Copyright (C) 2020, Inria
+# GRAPHDECO research group, https://team.inria.fr/graphdeco
+# All rights reserved.
+#
+# This software is free for non-commercial, research and evaluation use
+# under the terms of the LICENSE.md file.
+#
+# For inquiries contact sibr@inria.fr and/or George.Drettakis@inria.fr
+
+
+if(__MSVCsetUserCommand_cmake_INCLUDED__)
+ return()
+else()
+ set(__MSVCsetUserCommand_cmake_INCLUDED__ ON)
+endif()
+
+## Allow to configure the Debugger settings of visual studio
+## Note: Using this command under linux doesn't affect anything
+## On run Debug Windows local : visual will try to load a specific COMMAND with ARGS in the provided WORKING_DIR
+##
+## usage:
+## MSVCsetUserCommand(
+## [COMMAND | [ PATH [FILE ] ] ]
+## ARGS
+## WORKING_DIR
+## )
+##
+## Warning 1 : All arugments () must be passed under quotes
+## Warning 2 : WORKING_DIR path arg have to finish with remain slah '/'
+## Warning 3 : use COMMAND for external app OR PATH (optionaly with FILE) option(s) to set your built/installed/moved target
+##
+## Example 1:
+## include(MSVCsetUserCommand)
+## MSVCsetUserCommand( UnityRenderingPlugin
+## COMMAND "C:/Program Files (x86)/Unity/Editor/Unity.exe"
+## ARGS "-force-opengl -projectPath \"${CMAKE_HOME_DIRECTORY}/UnityPlugins/RenderingPluginExample/UnityProject\""
+## WORKING_DIR "${CMAKE_HOME_DIRECTORY}/UnityPlugins/RenderingPluginExample/UnityProject"
+## VERBOSE
+## )
+##
+## Example 2:
+## include(MSVCsetUserCommand)
+## MSVCsetUserCommand( ibrApp
+## PATH "C:/Program Files (x86)/workspace/IBR/install"
+## FILE "ibrApp${CMAKE_EXECUTABLE_SUFFIX}" ## this option line is optional since the target name didn't change between build and install step
+## ARGS "-path \"${CMAKE_HOME_DIRECTORY}/dataset\""
+## WORKING_DIR "${CMAKE_HOME_DIRECTORY}"
+## VERBOSE
+## )
+##
+function(MSVCsetUserCommand targetName)
+ cmake_parse_arguments(MSVCsuc "VERBOSE" "PATH;FILE;COMMAND;ARGS;WORKING_DIR" "" ${ARGN} )
+
+ ## If no arguments are given, do not create an unecessary .vcxproj.user file
+ set(MSVCsuc_DEFAULT OFF)
+
+ if(MSVCsuc_PATH AND MSVCsuc_DEFAULT)
+ set(MSVCsuc_DEFAULT OFF)
+ endif()
+
+ if(MSVCsuc_FILE AND MSVCsuc_DEFAULT)
+ set(MSVCsuc_DEFAULT OFF)
+ endif()
+
+ if(NOT MSVCsuc_COMMAND)
+ if(MSVCsuc_PATH AND MSVCsuc_FILE)
+ set(MSVCsuc_COMMAND "${MSVCsuc_PATH}\\${MSVCsuc_FILE}")
+ elseif(MSVCsuc_PATH)
+ set(MSVCsuc_COMMAND "${MSVCsuc_PATH}\\$(TargetFileName)")
+ else()
+ set(MSVCsuc_COMMAND "$(TargetPath)") ## => $(TargetDir)\$(TargetName)$(TargetExt)
+ endif()
+ elseif(MSVCsuc_DEFAULT)
+ set(MSVCsuc_DEFAULT OFF)
+ endif()
+
+ # NOTE: there was a typo here. there is an else if written after else statement
+ # changing the order of the else if statement
+ if(MSVCsuc_WORKING_DIR)
+ file(TO_NATIVE_PATH ${MSVCsuc_WORKING_DIR} MSVCsuc_WORKING_DIR)
+ elseif(MSVCsuc_DEFAULT)
+ set(MSVCsuc_DEFAULT OFF)
+ else()
+ set(MSVCsuc_WORKING_DIR "$(ProjectDir)")
+ endif()
+
+ if(NOT MSVCsuc_ARGS)
+ set(MSVCsuc_ARGS "")
+ elseif(MSVCsuc_DEFAULT)
+ set(MSVCsuc_DEFAULT OFF)
+ endif()
+
+ if(MSVC10 OR (MSVC AND MSVC_VERSION GREATER 1600)) # 2010 or newer
+
+ if(CMAKE_SIZEOF_VOID_P EQUAL 8)
+ set(PLATEFORM_BITS x64)
+ else()
+ set(PLATEFORM_BITS Win32)
+ endif()
+
+ if(NOT MSVCsuc_DEFAULT AND PLATEFORM_BITS)
+
+ file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/${targetName}.vcxproj.user"
+ "
+
+
+ ${MSVCsuc_COMMAND}
+ ${MSVCsuc_ARGS}
+ WindowsLocalDebugger
+ ${MSVCsuc_WORKING_DIR}
+
+
+ ${MSVCsuc_COMMAND}
+ ${MSVCsuc_ARGS}
+ WindowsLocalDebugger
+ ${MSVCsuc_WORKING_DIR}
+
+
+ ${MSVCsuc_COMMAND}
+ ${MSVCsuc_ARGS}
+ WindowsLocalDebugger
+ ${MSVCsuc_WORKING_DIR}
+
+
+ ${MSVCsuc_COMMAND}
+ ${MSVCsuc_ARGS}
+ WindowsLocalDebugger
+ ${MSVCsuc_WORKING_DIR}
+
+ "
+ )
+ if(MSVCsuc_VERBOSE)
+ message(STATUS "[MSVCsetUserCommand] Write ${CMAKE_CURRENT_BINARY_DIR}/${targetName}.vcxproj.user file")
+ message(STATUS " to execute ${MSVCsuc_COMMAND} ${MSVCsuc_ARGS}")
+ message(STATUS " from derectory ${MSVCsuc_WORKING_DIR}")
+ message(STATUS " on visual studio run debugger button")
+ endif()
+
+ else()
+ message(WARNING "PLATEFORM_BITS is undefined...")
+ endif()
+
+ else()
+ if(MSVCsuc_VERBOSE)
+ message(WARNING "MSVCsetUserCommand is disable because too old MSVC is used (need MSVC10 2010 or newer)")
+ endif()
+ endif()
+
+endfunction()
diff --git a/submodules/gaussian-splatting/SIBR_viewers/cmake/windows/Modules/FindASSIMP.cmake b/submodules/gaussian-splatting/SIBR_viewers/cmake/windows/Modules/FindASSIMP.cmake
new file mode 100644
index 0000000000000000000000000000000000000000..f92c8c003c3b109037ae39f316b17a360b44093a
--- /dev/null
+++ b/submodules/gaussian-splatting/SIBR_viewers/cmake/windows/Modules/FindASSIMP.cmake
@@ -0,0 +1,104 @@
+# Copyright (C) 2020, Inria
+# GRAPHDECO research group, https://team.inria.fr/graphdeco
+# All rights reserved.
+#
+# This software is free for non-commercial, research and evaluation use
+# under the terms of the LICENSE.md file.
+#
+# For inquiries contact sibr@inria.fr and/or George.Drettakis@inria.fr
+
+
+## Try to find the ASSIMP library
+## Once done this will define
+##
+## ASSIMP_FOUND - system has ASSIMP
+## ASSIMP_INCLUDE_DIR - The ASSIMP include directory
+## ASSIMP_LIBRARIES - The libraries needed to use ASSIMP
+## ASSIMP_CMD - the full path of ASSIMP executable
+## ASSIMP_DYNAMIC_LIB - the Assimp dynamic lib (available only on windows as .dll file for the moment)
+##
+## Edited for using a bugfixed version of Assimp
+
+if(NOT ASSIMP_DIR)
+ set(ASSIMP_DIR "$ENV{ASSIMP_DIR}" CACHE PATH "ASSIMP root directory")
+endif()
+if(ASSIMP_DIR)
+ file(TO_CMAKE_PATH ${ASSIMP_DIR} ASSIMP_DIR)
+endif()
+
+
+## set the LIB POSTFIX to find in a right directory according to what kind of compiler we use (32/64bits)
+if(CMAKE_SIZEOF_VOID_P EQUAL 8)
+ set(ASSIMP_SEARCH_LIB "lib64")
+ set(ASSIMP_SEARCH_BIN "bin64")
+ set(ASSIMP_SEARCH_LIB_PATHSUFFIXE "x64")
+else()
+ set(ASSIMP_SEARCH_LIB "lib32")
+ set(ASSIMP_SEARCH_BIN "bin32")
+ set(ASSIMP_SEARCH_LIB_PATHSUFFIXE "x86")
+endif()
+
+set(PROGRAMFILESx86 "PROGRAMFILES(x86)")
+
+
+FIND_PATH(ASSIMP_INCLUDE_DIR
+ NAMES assimp/config.h
+ PATHS
+ ${ASSIMP_DIR}
+ ## linux
+ /usr
+ /usr/local
+ /opt/local
+ ## windows
+ "$ENV{PROGRAMFILES}/Assimp"
+ "$ENV{${PROGRAMFILESx86}}/Assimp"
+ "$ENV{ProgramW6432}/Assimp"
+ PATH_SUFFIXES include
+)
+
+
+FIND_LIBRARY(ASSIMP_LIBRARY
+ NAMES assimp-vc140-mt
+ PATHS
+ ${ASSIMP_DIR}/${ASSIMP_SEARCH_LIB}
+ ${ASSIMP_DIR}/lib
+ ${ASSIMP_DIR}/lib64
+ ## linux
+ /usr/${ASSIMP_SEARCH_LIB}
+ /usr/local/${ASSIMP_SEARCH_LIB}
+ /opt/local/${ASSIMP_SEARCH_LIB}
+ /usr/lib
+ /usr/local/lib
+ /opt/local/lib
+ ## windows
+ "$ENV{PROGRAMFILES}/Assimp/${ASSIMP_SEARCH_LIB}"
+ "$ENV{${PROGRAMFILESx86}}/Assimp/${ASSIMP_SEARCH_LIB}"
+ "$ENV{ProgramW6432}/Assimp/${ASSIMP_SEARCH_LIB}"
+ "$ENV{PROGRAMFILES}/Assimp/lib"
+ "$ENV{${PROGRAMFILESx86}}/Assimp/lib"
+ "$ENV{ProgramW6432}/Assimp/lib"
+ PATH_SUFFIXES ${ASSIMP_SEARCH_LIB_PATHSUFFIXE}
+)
+set(ASSIMP_LIBRARIES ${ASSIMP_LIBRARY})
+
+
+if(ASSIMP_LIBRARY)
+ get_filename_component(ASSIMP_LIBRARY_DIR ${ASSIMP_LIBRARY} PATH)
+ file(GLOB ASSIMP_DYNAMIC_LIB "${ASSIMP_LIBRARY_DIR}/assimp*.dll")
+ if(NOT ASSIMP_DYNAMIC_LIB)
+ message("ASSIMP_DYNAMIC_LIB is missing... at ${ASSIMP_LIBRARY_DIR}")
+ endif()
+ set(ASSIMP_DYNAMIC_LIB ${ASSIMP_DYNAMIC_LIB} CACHE PATH "Windows dll location")
+endif()
+
+MARK_AS_ADVANCED(ASSIMP_DYNAMIC_LIB ASSIMP_INCLUDE_DIR ASSIMP_LIBRARIES)
+
+INCLUDE(FindPackageHandleStandardArgs)
+FIND_PACKAGE_HANDLE_STANDARD_ARGS(ASSIMP
+ REQUIRED_VARS ASSIMP_INCLUDE_DIR ASSIMP_LIBRARIES
+ FAIL_MESSAGE "ASSIMP wasn't found correctly. Set ASSIMP_DIR to the root SDK installation directory."
+)
+
+if(NOT ASSIMP_FOUND)
+ set(ASSIMP_DIR "" CACHE STRING "Path to ASSIMP install directory")
+endif()
diff --git a/submodules/gaussian-splatting/SIBR_viewers/cmake/windows/Modules/FindEmbree.cmake b/submodules/gaussian-splatting/SIBR_viewers/cmake/windows/Modules/FindEmbree.cmake
new file mode 100644
index 0000000000000000000000000000000000000000..27908b58240afe19eee4e7466eda9c8557f8f0ae
--- /dev/null
+++ b/submodules/gaussian-splatting/SIBR_viewers/cmake/windows/Modules/FindEmbree.cmake
@@ -0,0 +1,95 @@
+# Copyright (C) 2020, Inria
+# GRAPHDECO research group, https://team.inria.fr/graphdeco
+# All rights reserved.
+#
+# This software is free for non-commercial, research and evaluation use
+# under the terms of the LICENSE.md file.
+#
+# For inquiries contact sibr@inria.fr and/or George.Drettakis@inria.fr
+
+
+## Important Note:
+## This is not an official Find*cmake. It has been written for searching through
+## a custom path (EMBREE_DIR) before checking elsewhere.
+##
+## FindEMBREE.cmake
+## Find EMBREE's includes and library
+##
+## This module defines :
+## [in] EMBREE_DIR, The base directory to search for EMBREE (as cmake var or env var)
+## [out] EMBREE_INCLUDE_DIR where to find EMBREE.h
+## [out] EMBREE_LIBRARIES, EMBREE_LIBRARY, libraries to link against to use EMBREE
+## [out] EMBREE_FOUND, If false, do not try to use EMBREE.
+##
+
+
+if(NOT EMBREE_DIR)
+ set(EMBREE_DIR "$ENV{EMBREE_DIR}" CACHE PATH "EMBREE root directory")
+endif()
+if(EMBREE_DIR)
+ file(TO_CMAKE_PATH ${EMBREE_DIR} EMBREE_DIR)
+endif()
+
+
+## set the LIB POSTFIX to find in a right directory according to what kind of compiler we use (32/64bits)
+if(CMAKE_SIZEOF_VOID_P EQUAL 8)
+ set(EMBREE_SEARCH_LIB "lib64")
+ set(EMBREE_SEARCH_BIN "bin64")
+ set(EMBREE_SEARCH_LIB_PATHSUFFIXE "x64")
+else()
+ set(EMBREE_SEARCH_LIB "lib32")
+ set(EMBREE_SEARCH_BIN "bin32")
+ set(EMBREE_SEARCH_LIB_PATHSUFFIXE "x86")
+endif()
+
+set(PROGRAMFILESx86 "PROGRAMFILES(x86)")
+
+FIND_PATH(EMBREE_INCLUDE_DIR
+ NAMES embree3/rtcore_geometry.h
+ PATHS
+ ${EMBREE_DIR}
+ ## linux
+ /usr
+ /usr/local
+ /opt/local
+ ## windows
+ "$ENV{PROGRAMFILES}/EMBREE"
+ "$ENV{${PROGRAMFILESx86}}/EMBREE"
+ "$ENV{ProgramW6432}/EMBREE"
+ PATH_SUFFIXES include
+)
+
+FIND_LIBRARY(EMBREE_LIBRARY
+ NAMES embree3
+ PATHS
+ ${EMBREE_DIR}/${EMBREE_SEARCH_LIB}
+ ${EMBREE_DIR}/lib
+ ## linux
+ /usr/${EMBREE_SEARCH_LIB}
+ /usr/local/${EMBREE_SEARCH_LIB}
+ /opt/local/${EMBREE_SEARCH_LIB}
+ /usr/lib
+ /usr/local/lib
+ /opt/local/lib
+ ## windows
+ "$ENV{PROGRAMFILES}/EMBREE/${EMBREE_SEARCH_LIB}"
+ "$ENV{${PROGRAMFILESx86}}/EMBREE/${EMBREE_SEARCH_LIB}"
+ "$ENV{ProgramW6432}/EMBREE/${EMBREE_SEARCH_LIB}"
+ "$ENV{PROGRAMFILES}/EMBREE/lib"
+ "$ENV{${PROGRAMFILESx86}}/EMBREE/lib"
+ "$ENV{ProgramW6432}/EMBREE/lib"
+ PATH_SUFFIXES ${EMBREE_SEARCH_LIB_PATHSUFFIXE}
+)
+set(EMBREE_LIBRARIES ${EMBREE_LIBRARY})
+
+MARK_AS_ADVANCED(EMBREE_INCLUDE_DIR EMBREE_LIBRARIES)
+
+INCLUDE(FindPackageHandleStandardArgs)
+FIND_PACKAGE_HANDLE_STANDARD_ARGS(EMBREE
+ REQUIRED_VARS EMBREE_INCLUDE_DIR EMBREE_LIBRARIES
+ FAIL_MESSAGE "EMBREE wasn't found correctly. Set EMBREE_DIR to the root SDK installation directory."
+)
+
+if(NOT EMBREE_FOUND)
+ set(EMBREE_DIR "" CACHE STRING "Path to EMBREE install directory")
+endif()
diff --git a/submodules/gaussian-splatting/SIBR_viewers/cmake/windows/Modules/FindFFmpeg.cmake b/submodules/gaussian-splatting/SIBR_viewers/cmake/windows/Modules/FindFFmpeg.cmake
new file mode 100644
index 0000000000000000000000000000000000000000..5b208b64e117989fd38a39d0e561f1cd041a0f42
--- /dev/null
+++ b/submodules/gaussian-splatting/SIBR_viewers/cmake/windows/Modules/FindFFmpeg.cmake
@@ -0,0 +1,104 @@
+# Copyright (C) 2020, Inria
+# GRAPHDECO research group, https://team.inria.fr/graphdeco
+# All rights reserved.
+#
+# This software is free for non-commercial, research and evaluation use
+# under the terms of the LICENSE.md file.
+#
+# For inquiries contact sibr@inria.fr and/or George.Drettakis@inria.fr
+
+
+## Try to find the FFMPEG library
+## Once done this will define
+##
+## FFMPEG_FOUND - system has FFmpeg
+## FFMPEG_INCLUDE_DIR - The FFmpeg include directory
+## FFMPEG_LIBRARIES - The libraries needed to use FFmpeg
+## FFMPEG_DYNAMIC_LIBS - DLLs for windows
+
+
+if(NOT FFMPEG_DIR)
+ set(FFMPEG_DIR "$ENV{FFMPEG_DIR}" CACHE PATH "FFMPEG_DIR root directory")
+endif()
+
+if(FFMPEG_DIR)
+ file(TO_CMAKE_PATH ${FFMPEG_DIR} FFMPEG_DIR)
+endif()
+
+MACRO(FFMPEG_FIND varname shortname headername)
+
+ # Path to include dirs
+ FIND_PATH(FFMPEG_${varname}_INCLUDE_DIRS
+ NAMES "lib${shortname}/${headername}"
+ PATHS
+ "${FFMPEG_DIR}/include" # modify this to adapt according to OS/compiler
+ )
+
+ #Add libraries
+ IF(${FFMPEG_${varname}_INCLUDE_DIRS} STREQUAL "FFMPEG_${varname}_INCLUDE_DIR-NOTFOUND")
+ MESSAGE(STATUS "Can't find includes for ${shortname}...")
+ ELSE()
+ FIND_LIBRARY(FFMPEG_${varname}_LIBRARIES
+ NAMES ${shortname}
+ PATHS
+ ${FFMPEG_DIR}/lib
+ )
+
+ # set libraries and other variables
+ SET(FFMPEG_${varname}_FOUND 1)
+ SET(FFMPEG_${varname}_INCLUDE_DIRS ${FFMPEG_${varname}_INCLUDE_DIR})
+ SET(FFMPEG_${varname}_LIBS ${FFMPEG_${varname}_LIBRARIES})
+ ENDIF()
+ ENDMACRO(FFMPEG_FIND)
+
+#Calls to ffmpeg_find to get librarires ------------------------------
+FFMPEG_FIND(LIBAVFORMAT avformat avformat.h)
+FFMPEG_FIND(LIBAVDEVICE avdevice avdevice.h)
+FFMPEG_FIND(LIBAVCODEC avcodec avcodec.h)
+FFMPEG_FIND(LIBAVUTIL avutil avutil.h)
+FFMPEG_FIND(LIBSWSCALE swscale swscale.h)
+
+# check if libs are found and set FFMPEG related variables
+#SET(FFMPEG_FOUND "NO")
+IF(FFMPEG_LIBAVFORMAT_FOUND
+ AND FFMPEG_LIBAVDEVICE_FOUND
+ AND FFMPEG_LIBAVCODEC_FOUND
+ AND FFMPEG_LIBAVUTIL_FOUND
+ AND FFMPEG_LIBSWSCALE_FOUND)
+
+ # All ffmpeg libs are here
+ SET(FFMPEG_FOUND "YES")
+ SET(FFMPEG_INCLUDE_DIR ${FFMPEG_LIBAVFORMAT_INCLUDE_DIRS})
+ SET(FFMPEG_LIBRARY_DIRS ${FFMPEG_LIBAVFORMAT_LIBRARY_DIRS})
+ SET(FFMPEG_LIBRARIES
+ ${FFMPEG_LIBAVFORMAT_LIBS}
+ ${FFMPEG_LIBAVDEVICE_LIBS}
+ ${FFMPEG_LIBAVCODEC_LIBS}
+ ${FFMPEG_LIBAVUTIL_LIBS}
+ ${FFMPEG_LIBSWSCALE_LIBS} )
+
+ # add dynamic libraries
+ if(WIN32)
+ file(GLOB FFMPEG_DYNAMIC_LIBS "${FFMPEG_DIR}/bin/*.dll")
+ if(NOT FFMPEG_DYNAMIC_LIBS)
+ message("FFMPEG_DYNAMIC_LIBS is missing...")
+ endif()
+ set(FFMPEG_DYNAMIC_LIBS ${FFMPEG_DYNAMIC_LIBS} CACHE PATH "Windows dll location")
+endif()
+
+ mark_as_advanced(FFMPEG_INCLUDE_DIR FFMPEG_LIBRARY_DIRS FFMPEG_LIBRARIES FFMPEG_DYNAMIC_LIBS)
+ELSE ()
+ MESSAGE(STATUS "Could not find FFMPEG")
+ENDIF()
+
+
+INCLUDE(FindPackageHandleStandardArgs)
+FIND_PACKAGE_HANDLE_STANDARD_ARGS(FFMPEG
+ REQUIRED_VARS FFMPEG_INCLUDE_DIR FFMPEG_LIBRARIES
+ FAIL_MESSAGE "FFmpeg wasn't found correctly. Set FFMPEG_DIR to the root SDK installation directory."
+)
+
+if(NOT FFMPEG_FOUND)
+ set(FFMPEG_DIR "" CACHE STRING "Path to FFmpeg install directory")
+endif()
+
diff --git a/submodules/gaussian-splatting/SIBR_viewers/cmake/windows/Win3rdParty.cmake b/submodules/gaussian-splatting/SIBR_viewers/cmake/windows/Win3rdParty.cmake
new file mode 100644
index 0000000000000000000000000000000000000000..7e42fbb9f4353c2208ed3d6f44cf7acc3fccedc2
--- /dev/null
+++ b/submodules/gaussian-splatting/SIBR_viewers/cmake/windows/Win3rdParty.cmake
@@ -0,0 +1,337 @@
+# Copyright (C) 2020, Inria
+# GRAPHDECO research group, https://team.inria.fr/graphdeco
+# All rights reserved.
+#
+# This software is free for non-commercial, research and evaluation use
+# under the terms of the LICENSE.md file.
+#
+# For inquiries contact sibr@inria.fr and/or George.Drettakis@inria.fr
+
+
+## This file should be include and use only on WIN32 OS and once
+## It allow to auto check/download and use a preconfigured 3rdParty binaries for cmake usage
+## It use the downloadAndExtractZipFile cmake module to work.
+##
+if(__Win3rdParty_cmake_INCLUDED__)
+ return()
+else()
+ set(__Win3rdParty_cmake_INCLUDED__ ON)
+endif()
+
+
+##
+## To be sure to reset an empty cached variable but keep any other kind of variables
+##
+## Usage:
+## check_cached_var( [FORCE])
+##
+## is the cached cmake variable you need to reset
+## is the new default value of the reseted cached cmake variable
+## is the kind of GUI cache input can be : FILEPATH; PATH; STRING or BOOL
+## is the associated GUI cache input documentation display in the GUI
+## FORCE option could be use to reset a cached variable even if it is not empty.
+##
+macro(check_cached_var var resetedCachedValue cacheType cacheDoc)
+ # message(STATUS "inside check_cached_var macro. argn=${ARGN}")
+ cmake_parse_arguments(ccv "FORCE" "" "" ${ARGN})
+
+ if(ccv_FORCE)
+ set(FORCE FORCE)
+ else()
+ set(FORCE )
+ endif()
+
+ if(NOT ${var} OR ccv_FORCE)
+ unset(${var} CACHE)
+ # message(STATUS "setting new cache value. var ${var} = ${resetedCachedValue}")
+ set(${var} "${resetedCachedValue}" CACHE ${cacheType} "${cacheDoc}" ${FORCE})
+ endif()
+endmacro()
+
+
+##
+## Win3rdParty function allow to specify a directory which contain all necessary windows dependenties.
+## By uploading 3rdParty directory (which contain dependencies, *.lib, *.dll... for a specific version of compiler) onto Gforge file tab,
+## you get back an URL of download you can give to this function with a directory name. So you can provide multiple 3rdParty version of same dependencies (MSVC11, MSVC12...).
+## By providing a prefix to this function, you allow to use different kind of 3rdParty which can be handled by CMAKE OPTIONS depending on what your framework need for example.
+##
+## Usage 1:
+## Win3rdParty( MSVC
+## [MSVC ] [...]
+## [VCID] [DEFAULT_USE] [VERBOSE] )
+##
+## * allow to identify which 3rdParty you process (prefix name)
+## * MSVC flag could be MSVC11 or MSVC12 (any element of the MSVC_VERSIONS_LIST) and refer to a 3rdParty compiler with :
+## * which will be the local pathName of the downloaded 3rdParty : relative to CMAKE_BINARY_DIR
+## * which is the link location of the 3rdParty zip
+## * VCID flag will make available a cache variable ${prefix}_WIN3RDPARTY_VCID
+## * DEFAULT_USE flag [ON|OFF] may be used to set default value of cmake cached variable : _WIN3RDPARTY_USE [default to ON]
+##
+## WARNING:
+## This function define CACHE variables you can use after :
+## * ${prefix}_WIN3RDPARTY_USE : allow to check/downloaded win3rdParty dir (it will force the cached variables for this dependency folder generally _DIR>)
+## * ${prefix}_WIN3RDPARTY_DIR : where is your local win3rdParty dir (the PATH)
+## * ${prefix}_WIN3RDPARTY_VCID : [if VCID flag is used] the MSVC id (commonly used to prefix/suffix library name, see boost or CGAL)
+##
+## If you want to add a win3rdParty version, please:
+## 1- build dependencies on your local side with the compiler you want
+## 2- build your own zip with your built dependencies
+## 3- upload it (onto the forge where the project is stored) and copy the link location in order to use it for this function
+## 4- if you just introduced a new MSVC version, add it to the MSVC_VERSIONS_LIST bellow
+##
+## In a second pass, you can also use this function to set necessary cmake cached variables in order to let cmake find packages of these 3rdParty.
+##
+## Usage 2:
+## win3rdParty( [VERBOSE] MULTI_SET|SET
+## CHECK_CACHED_VAR [LIST] [DOC ]
+## [ CHECK_CACHED_VAR [LIST] [DOC ] ] [...]
+##
+## * MULTI_SET or SET flags are used to tell cmake that all next arguments will use repeated flags with differents entries (SET mean we will provide only one set of arguments, without repetition)
+## * CHECK_CACHED_VAR are the repeated flag which contain differents entries
+## * is the cmake variable you want to be cached for the project
+## * is the kind of cmake variable (couble be: FILEPATH; PATH; STRING or BOOL) => see check_cached_var.
+## * LIST optional flag could be used with CHECK_CACHED_VAR when = STRING. It allow to handle multiple STRINGS value list.
+## * is the value of the variable (if FILEPATH, PATH or STRING: use quotes, if BOOL : use ON/OFF)
+## * DOC optional flag is used to have a tooltips info about this new cmake variable entry into the GUI (use quotes).
+##
+## Full example 1 :
+## win3rdParty(COMMON MSVC11 "win3rdParty-MSVC11" "https://path.to/an.archive.7z"
+## SET CHECK_CACHED_VAR SuiteSparse_DIR PATH "SuiteSparse-4.2.1" DOC "default empty doc"
+## )
+##
+## WARNING:
+## For the 2nd usage (with MULTI_SET), if you planned to set some CACHED_VAR using/composed by ${prefix}_WIN3RDPARTY_* just set in this macro (usage 1),
+## then (due to the not yet existing var) you will need to call this function 2 times :
+## One for the 1st usage (downloading of the current compiler 3rdParty).
+## One for the MLUTI_SET flag which will use existsing ${prefix}_WIN3RDPARTY_* cached var.
+##
+## Full example 2 :
+## win3rdParty(COMMON MSVC11 "win3rdParty-MSVC11" "https://path.to/an.archive.7z")
+## win3rdParty(COMMON MULTI_SET
+## CHECK_CACHED_VAR CGAL_INCLUDE_DIR PATH "CGAL-4.3/include" DOC "default empty doc"
+## CHECK_CACHED_VAR CGAL_LIBRARIES STRING LIST "debug;CGAL-4.3/lib${LIB_POSTFIX}/CGAL-${WIN3RDPARTY_COMMON_VCID}-mt-gd-4.3.lib;optimized;CGAL-4.3/lib${LIB_POSTFIX}/CGAL-${WIN3RDPARTY_COMMON_VCID}-mt-4.3.lib"
+##
+##
+## WARNING: This function use internaly :
+## * downloadAndExtractZipFile.cmake
+## * parse_arguments_multi.cmake
+## * check_cached_var macro
+##
+function(win3rdParty prefix )
+
+ # ARGV: list of all arguments given to the macro/function
+ # ARGN: list of remaining arguments
+
+ if(NOT WIN32)
+ return()
+ endif()
+
+ ## set the handled version of MSVC
+ ## if you plan to add a win3rdParty dir to download with a new MSVC version: build the win3rdParty dir and add the MSCV entry here.
+ set(MSVC_VERSIONS_LIST "MSVC17;MSVC11;MSVC12;MSVC14")
+
+ #include(CMakeParseArguments) # CMakeParseArguments is obsolete since cmake 3.5
+ # cmake_parse_arguments ( args)
+ # : options (flags) pass to the macro
+ # : options that neeed a value
+ # : options that neeed more than one value
+ cmake_parse_arguments(w3p "VCID" "VERBOSE;TIMEOUT;DEFAULT_USE" "${MSVC_VERSIONS_LIST};MULTI_SET;SET" ${ARGN})
+
+ # message(STATUS "value of w3p_VCID = ${w3p_VCID}")
+ # message(STATUS "value of w3p_VERBOSE = ${w3p_VERBOSE}")
+ # message(STATUS "value of w3p_TIMEOUT = ${w3p_TIMEOUT}")
+ # message(STATUS "value of w3p_DEFAULT_USE = ${w3p_DEFAULT_USE}")
+
+ # foreach (loop_var ${MSVC_VERSIONS_LIST})
+ # message(STATUS "value of w3p_${loop_var} = ${w3p_${loop_var}}")
+ # endforeach(loop_var)
+
+ # message(STATUS "value of w3p_MULTI_SET = ${w3p_MULTI_SET}")
+ # message(STATUS "value of w3p_SET = ${w3p_SET}")
+
+ # message("values for MSVC = ${w3p_MSVC14}")
+
+ if(NOT w3p_TIMEOUT)
+ set(w3p_TIMEOUT 300)
+ endif()
+
+ if(NOT DEFINED w3p_DEFAULT_USE)
+ set(w3p_DEFAULT_USE ON)
+ endif()
+
+
+ ## 1st use (check/update|download) :
+ set(${prefix}_WIN3RDPARTY_USE ${w3p_DEFAULT_USE} CACHE BOOL "Use required 3rdParty binaries from ${prefix}_WIN3RDPARTY_DIR or download it if not exist")
+
+
+ ## We want to test if each version of MSVC was filled by the function (see associated parameters)
+ ## As CMake is running only for one version of MSVC, if that MSVC version was filled, we get back associated parameters,
+ ## otherwise we can't use the downloadAndExtractZipFile with win3rdParty.
+ set(enableWin3rdParty OFF)
+
+ foreach(MSVC_VER ${MSVC_VERSIONS_LIST})
+ if(${MSVC_VER} AND w3p_${MSVC_VER} OR ${MSVC_TOOLSET_VERSION} EQUAL 143 AND ${MSVC_VER} STREQUAL "MSVC17")
+ list(LENGTH w3p_${MSVC_VER} count)
+ if("${count}" LESS "2")
+ #message(WARNING "You are using ${MSVC_VER} with ${prefix}_WIN3RDPARTY_USE=${${prefix}_WIN3RDPARTY_USE}, but win3rdParty function isn't filled for ${MSVC_VER}!")
+ else()
+ list(GET w3p_${MSVC_VER} 0 Win3rdPartyName)
+ list(GET w3p_${MSVC_VER} 1 Win3rdPartyUrl)
+ if(w3p_VCID)
+ ## try to get the VcId of MSVC. See also MSVC_VERSION cmake var in the doc.
+ string(REGEX REPLACE "MS([A-Za-z_0-9-]+)" "\\1" vcId ${MSVC_VER})
+ string(TOLOWER ${vcId} vcId)
+ set(${prefix}_WIN3RDPARTY_VCID "${vcId}0" CACHE STRING "the MSVC id (commonly used to prefix/suffix library name, see boost or CGAL)")
+ mark_as_advanced(${prefix}_WIN3RDPARTY_VCID)
+ endif()
+ set(enableWin3rdParty ON)
+ set(suffixCompilerID ${MSVC_VER})
+ break()
+ endif()
+ endif()
+ endforeach()
+ ## If previous step succeed to get MSVC dirname and URL of the current MSVC version, use it to auto download/update the win3rdParty dir
+ if(enableWin3rdParty AND ${prefix}_WIN3RDPARTY_USE)
+
+ if(IS_ABSOLUTE "${Win3rdPartyName}")
+ else()
+ set(Win3rdPartyName "${CMAKE_BINARY_DIR}/${Win3rdPartyName}")
+ endif()
+
+ if(NOT EXISTS "${Win3rdPartyName}")
+ file(MAKE_DIRECTORY ${Win3rdPartyName})
+ endif()
+
+ include(downloadAndExtractZipFile)
+ downloadAndExtractZipFile( "${Win3rdPartyUrl}" ## URL link location
+ "Win3rdParty-${prefix}-${suffixCompilerID}.7z" ## where download it: relative path, so default to CMAKE_BINARY_DIR
+ "${Win3rdPartyName}" ## where extract it : fullPath (default relative to CMAKE_BINARY_DIR)
+ CHECK_DIRTY_URL "${Win3rdPartyName}/Win3rdPartyUrl" ## last downloaded url file : fullPath (default relative to CMAKE_BINARY_DIR)
+ TIMEOUT ${w3p_TIMEOUT}
+ VERBOSE ${w3p_VERBOSE}
+ )
+ file(GLOB checkDl "${Win3rdPartyName}/*")
+ list(LENGTH checkDl checkDlCount)
+ if("${checkDlCount}" GREATER "1")
+ else()
+ message("The downloadAndExtractZipFile didn't work...?")
+ set(enableWin3rdParty OFF)
+ endif()
+ endif()
+
+ ## Try to auto set ${prefix}_WIN3RDPARTY_DIR or let user set it manually
+ set(${prefix}_WIN3RDPARTY_DIR "" CACHE PATH "windows ${Win3rdPartyName} dir to ${prefix} dependencies of the project")
+
+ if(NOT ${prefix}_WIN3RDPARTY_DIR AND ${prefix}_WIN3RDPARTY_USE)
+ if(EXISTS "${Win3rdPartyName}")
+ unset(${prefix}_WIN3RDPARTY_DIR CACHE)
+ set(${prefix}_WIN3RDPARTY_DIR "${Win3rdPartyName}" CACHE PATH "dir to ${prefix} dependencies of the project")
+ endif()
+ endif()
+
+ if(EXISTS ${${prefix}_WIN3RDPARTY_DIR})
+ message(STATUS "Found a 3rdParty ${prefix} dir : ${${prefix}_WIN3RDPARTY_DIR}.")
+ set(enableWin3rdParty ON)
+ elseif(${prefix}_WIN3RDPARTY_USE)
+ message(WARNING "${prefix}_WIN3RDPARTY_USE=${${prefix}_WIN3RDPARTY_USE} but ${prefix}_WIN3RDPARTY_DIR=${${prefix}_WIN3RDPARTY_DIR}.")
+ set(enableWin3rdParty OFF)
+ endif()
+
+ ## Final check
+ if(NOT enableWin3rdParty)
+ message("Disable ${prefix}_WIN3RDPARTY_USE (cmake cached var will be not set), due to a win3rdParty problem.")
+ message("You still can set ${prefix}_WIN3RDPARTY_DIR to an already downloaded Win3rdParty directory location.")
+ set(${prefix}_WIN3RDPARTY_USE OFF CACHE BOOL "Use required 3rdParty binaries from ${prefix}_WIN3RDPARTY_DIR or download it if not exist" FORCE)
+ endif()
+
+ ## 2nd use : handle multi values args to set cached cmake variables in order to ease the next find_package call
+ if(${prefix}_WIN3RDPARTY_USE AND ${prefix}_WIN3RDPARTY_DIR)
+ if(w3p_VERBOSE)
+ message(STATUS "Try to set cmake cached variables for ${prefix} required libraries directly from : ${${prefix}_WIN3RDPARTY_DIR}.")
+ endif()
+
+ include(parse_arguments_multi)
+ # message (STATUS "before defining an override of parse_arguments_multi_function")
+ function(parse_arguments_multi_function ) ## overloaded function to handle all CHECK_CACHED_VAR values list (see: parse_arguments_multi)
+ # message(STATUS "inside overloaded parse_arguments_multi_function defined in Win3rdParty.cmake")
+ # message(STATUS ${ARGN})
+ ## we know the function take 3 args : var cacheType resetedCachedValue (see check_cached_var)
+ cmake_parse_arguments(pamf "" "DOC" "LIST" ${ARGN})
+
+ ## var and cacheType are mandatory (with the resetedCachedValue)
+ set(var ${ARGV0})
+ set(cacheType ${ARGV1})
+ # message(STATUS "var=${var} and cacheType=${cacheType} list=${pamf_LIST}")
+ if(pamf_DOC)
+ set(cacheDoc ${pamf_DOC})
+ else()
+ set(cacheDoc "")
+ endif()
+ if(pamf_LIST)
+ set(value ${pamf_LIST})
+ else()
+ # message("USING ARGV2 with value ${ARGV2}")
+ set(value ${ARGV2})
+ endif()
+ # message("inside override function in Win3rdparty.cmake value+ ${value}")
+ if("${cacheType}" MATCHES "PATH" AND EXISTS "${${prefix}_WIN3RDPARTY_DIR}/${value}")
+ # message("math with path")
+ set(resetedCachedValue "${${prefix}_WIN3RDPARTY_DIR}/${value}") ## path relative to ${prefix}_WIN3RDPARTY_DIR
+ elseif ("${cacheType}" MATCHES "PATH" AND EXISTS "${${prefix}_WIN3RDPARTY_DIR}")
+ set(resetedCachedValue "${${prefix}_WIN3RDPARTY_DIR}") ## path relative to ${prefix}_WIN3RDPARTY_DIR
+ elseif("${cacheType}" MATCHES "STRING")
+ foreach(var IN LISTS value)
+ if(EXISTS "${${prefix}_WIN3RDPARTY_DIR}/${var}")
+ list(APPEND resetedCachedValue "${${prefix}_WIN3RDPARTY_DIR}/${var}") ## string item of the string list is a path => make relative to ${prefix}_WIN3RDPARTY_DIR
+ else()
+ list(APPEND resetedCachedValue ${var}) ## string item of the string list is not an existing path => simply use the item
+ endif()
+ endforeach()
+ else()
+ set(resetedCachedValue "${value}") ## could be a BOOL or a STRING
+ endif()
+
+ ## call our macro to reset cmake cache variable if empty
+ check_cached_var(${var} "${resetedCachedValue}" ${cacheType} "${cacheDoc}" FORCE)
+
+ endfunction()
+ # message (STATUS "after defining an override of parse_arguments_multi_function")
+
+ if(w3p_MULTI_SET)
+ parse_arguments_multi(CHECK_CACHED_VAR w3p_MULTI_SET ${w3p_MULTI_SET}) ## internaly will call our overloaded parse_arguments_multi_function
+ elseif(w3p_SET)
+ # message("calling set version of parse_arguments_multi with w3p_set = ${w3p_SET}")
+ parse_arguments_multi(CHECK_CACHED_VAR w3p_SET ${w3p_SET})
+ endif()
+
+ endif()
+
+endfunction()
+
+## cmake variables introspection to globally activate/deactivate ${prefix}_WIN3RDPARTY_USE
+## This "one shot" call (only one for the next cmake configure) will automatically then reset the global variable WIN3RDPARTY_USE to UserDefined (do nothing).
+## use (call it) before and after the call of all your win3rdParty functions
+function(Win3rdPartyGlobalCacheAction )
+ set(WIN3RDPARTY_USE "UserDefined" CACHE STRING "Choose how to handle all cmake cached *_WIN3RDPARTY_USE for the next configure.\nCould be:\nUserDefined [default]\nActivateAll\nDesactivateAll" )
+ set_property(CACHE WIN3RDPARTY_USE PROPERTY STRINGS "UserDefined;ActivateAll;DesactivateAll" )
+ if(${WIN3RDPARTY_USE} MATCHES "UserDefined")
+ else()
+ if(${WIN3RDPARTY_USE} MATCHES "ActivateAll")
+ set(win3rdPvalue ON)
+ elseif(${WIN3RDPARTY_USE} MATCHES "DesactivateAll")
+ set(win3rdPvalue OFF)
+ endif()
+ get_cmake_property(_variableNames CACHE_VARIABLES)
+ foreach (_variableName ${_variableNames})
+ string(REGEX MATCH "[A-Za-z_0-9-]+_WIN3RDPARTY_USE" win3rdpartyUseCacheVar ${_variableName})
+ if(win3rdpartyUseCacheVar)
+ string(REGEX REPLACE "([A-Za-z_0-9-]+_WIN3RDPARTY_USE)" "\\1" win3rdpartyUseCacheVar ${_variableName})
+ set(${win3rdpartyUseCacheVar} ${win3rdPvalue} CACHE BOOL "Use required 3rdParty binaries from ${prefix}_WIN3RDPARTY_DIR or download it if not exist" FORCE)
+ message(STATUS "${win3rdpartyUseCacheVar} cached variable set to ${win3rdPvalue}.")
+ endif()
+ endforeach()
+ set(WIN3RDPARTY_USE "UserDefined" CACHE STRING "Choose how to handle all cmake cached *_WIN3RDPARTY_USE for the next configure.\nCould be:\nUserDefined [default]\nActivateAll\nDesactivateAll" FORCE)
+ message(STATUS "reset WIN3RDPARTY_USE to UserDefined.")
+ endif()
+ mark_as_advanced(WIN3RDPARTY_USE)
+endfunction()
diff --git a/submodules/gaussian-splatting/SIBR_viewers/cmake/windows/cmake_policies.cmake b/submodules/gaussian-splatting/SIBR_viewers/cmake/windows/cmake_policies.cmake
new file mode 100644
index 0000000000000000000000000000000000000000..679fd8427d4c503ac420fc566a2ec4ae5c2825fc
--- /dev/null
+++ b/submodules/gaussian-splatting/SIBR_viewers/cmake/windows/cmake_policies.cmake
@@ -0,0 +1,19 @@
+# Copyright (C) 2020, Inria
+# GRAPHDECO research group, https://team.inria.fr/graphdeco
+# All rights reserved.
+#
+# This software is free for non-commercial, research and evaluation use
+# under the terms of the LICENSE.md file.
+#
+# For inquiries contact sibr@inria.fr and/or George.Drettakis@inria.fr
+
+
+if(__set_policies_INCLUDED__)
+ return()
+else()
+ set(__set_policies_INCLUDED__ ON)
+endif()
+
+macro(setPolicies)
+ # No more policies to enforce
+endmacro()
diff --git a/submodules/gaussian-splatting/SIBR_viewers/cmake/windows/dependencies.cmake b/submodules/gaussian-splatting/SIBR_viewers/cmake/windows/dependencies.cmake
new file mode 100644
index 0000000000000000000000000000000000000000..947f88fb75a35ab86e65f8085e1302eea05695d6
--- /dev/null
+++ b/submodules/gaussian-splatting/SIBR_viewers/cmake/windows/dependencies.cmake
@@ -0,0 +1,292 @@
+# Copyright (C) 2020, Inria
+# GRAPHDECO research group, https://team.inria.fr/graphdeco
+# All rights reserved.
+#
+# This software is free for non-commercial, research and evaluation use
+# under the terms of the LICENSE.md file.
+#
+# For inquiries contact sibr@inria.fr and/or George.Drettakis@inria.fr
+
+
+## Included once for all sub project.
+## It contain the whole cmake instructions to find necessary common dependencies.
+## 3rdParty (provided by sibr_addlibrary win3rdParty or from external packages) are then available in cmake sub projects.
+##
+## Do not include this file more than once but you can modify it to fit to your own project.
+## So please, read it carefully because you can use on of these dependencies for your project or appen new one.
+##
+## As it is included after camke options, you can use conditional if()/endif() to encapsulate your 3rdParty.
+##
+
+## win3rdParty function allowing to auto check/download/update binaries dependencies for current windows compiler
+## Please open this file in order to get more documentation and usage examples.
+include(Win3rdParty)
+
+include(sibr_library)
+
+Win3rdPartyGlobalCacheAction()
+
+find_package(OpenGL REQUIRED)
+
+############
+## Find GLEW
+############
+if (MSVC11 OR MSVC12)
+ set(glew_multiset_arguments
+ CHECK_CACHED_VAR GLEW_INCLUDE_DIR PATH "glew-1.10.0/include" DOC "default empty doc"
+ CHECK_CACHED_VAR GLEW_LIBRARIES STRING LIST "debug;glew-1.10.0/${LIB_BUILT_DIR}/glew32d.lib;optimized;glew-1.10.0/${LIB_BUILT_DIR}/glew32.lib" DOC "default empty doc"
+ )
+elseif (MSVC14 OR MSVC17)
+ set(glew_multiset_arguments
+ CHECK_CACHED_VAR GLEW_INCLUDE_DIR PATH "glew-2.0.0/include" DOC "default empty doc"
+ CHECK_CACHED_VAR GLEW_SHARED_LIBRARY_RELEASE PATH "glew-2.0.0/${LIB_BUILT_DIR}/glew32.lib"
+ CHECK_CACHED_VAR GLEW_STATIC_LIBRARY_RELEASE PATH "glew-2.0.0/${LIB_BUILT_DIR}/glew32s.lib"
+ CHECK_CACHED_VAR GLEW_SHARED_LIBRARY_DEBUG PATH "glew-2.0.0/${LIB_BUILT_DIR}/glew32d.lib"
+ CHECK_CACHED_VAR GLEW_STATIC_LIBRARY_DEBUG PATH "glew-2.0.0/${LIB_BUILT_DIR}/glew32sd.lib"
+ )
+else ()
+ message("There is no provided GLEW library for your version of MSVC")
+endif()
+sibr_addlibrary(NAME GLEW #VERBOSE ON
+ MSVC11 "https://repo-sam.inria.fr/fungraph/dependencies/ibr-common/win3rdParty-MSVC11-splitted%20version/glew-1.10.0.7z"
+ MSVC12 "https://repo-sam.inria.fr/fungraph/dependencies/ibr-common/win3rdParty-MSVC11-splitted%20version/glew-1.10.0.7z"
+ MSVC14 "https://repo-sam.inria.fr/fungraph/dependencies/ibr-common/win3rdParty-MSVC15-splitted%20version/glew-2.0.0.7z" # using recompiled version of glew
+ MULTI_SET ${glew_multiset_arguments}
+)
+set(GLEW_VERBOSE ON)
+FIND_PACKAGE(GLEW REQUIRED)
+IF(GLEW_FOUND)
+ INCLUDE_DIRECTORIES(${GLEW_INCLUDE_DIR})
+ELSE(GLEW_FOUND)
+ MESSAGE("GLEW not found. Set GLEW_DIR to base directory of GLEW.")
+ENDIF(GLEW_FOUND)
+
+
+##############
+## Find ASSIMP
+##############
+if (MSVC11 OR MSVC12)
+ set(assimp_set_arguments
+ CHECK_CACHED_VAR ASSIMP_DIR PATH "Assimp_3.1_fix"
+ )
+elseif (MSVC14 OR MSVC17)
+ set(assimp_set_arguments
+ CHECK_CACHED_VAR ASSIMP_DIR PATH "Assimp-4.1.0"
+ )
+else ()
+ message("There is no provided ASSIMP library for your version of MSVC")
+endif()
+
+sibr_addlibrary(NAME ASSIMP #VERBOSE ON
+ MSVC11 "https://repo-sam.inria.fr/fungraph/dependencies/ibr-common/win3rdParty-MSVC11-splitted%20version/Assimp_3.1_fix.7z"
+ MSVC12 "https://repo-sam.inria.fr/fungraph/dependencies/ibr-common/win3rdParty-MSVC11-splitted%20version/Assimp_3.1_fix.7z"
+ MSVC14 "https://repo-sam.inria.fr/fungraph/dependencies/ibr-common/win3rdParty-MSVC15-splitted%20version/Assimp-4.1.0.7z"
+ MULTI_SET
+ ${assimp_set_arguments}
+)
+
+find_package(ASSIMP REQUIRED)
+include_directories(${ASSIMP_INCLUDE_DIR})
+
+################
+## Find FFMPEG
+################
+sibr_addlibrary(NAME FFMPEG
+ MSVC11 "https://repo-sam.inria.fr/fungraph/dependencies/ibr-common/win3rdParty-MSVC11-splitted%20version/ffmpeg.zip"
+ MSVC12 "https://repo-sam.inria.fr/fungraph/dependencies/ibr-common/win3rdParty-MSVC11-splitted%20version/ffmpeg.zip"
+ MSVC14 "https://repo-sam.inria.fr/fungraph/dependencies/ibr-common/win3rdParty-MSVC15-splitted%20version/ffmpeg-4.0.2-win64-win3rdParty.7z"
+ SET CHECK_CACHED_VAR FFMPEG_DIR PATH ${FFMPEG_WIN3RDPARTY_DIR}
+)
+find_package(FFMPEG QUIET)
+include_directories(${FFMPEG_INCLUDE_DIR})
+
+###################
+## Find embree3
+###################
+sibr_addlibrary(
+ NAME embree3
+ MSVC11 "https://repo-sam.inria.fr/fungraph/dependencies/ibr-common/win3rdParty-MSVC11-splitted%20version/embree2.7.0.x64.windows.7z"
+ MSVC14 "https://repo-sam.inria.fr/fungraph/dependencies/ibr-common/win3rdParty-MSVC15-splitted%20version/embree-3.6.1.x64.vc14.windows.7z" # TODO SV: provide a valid version if required
+)
+
+###################
+## Find eigen3
+###################
+sibr_addlibrary(
+ NAME eigen3
+ #MSVC11 "https://repo-sam.inria.fr/fungraph/dependencies/ibr-common/win3rdParty-MSVC11-splitted%20version/eigen-eigen-dc6cfdf9bcec.7z"
+ #MSVC14 "https://repo-sam.inria.fr/fungraph/dependencies/ibr-common/win3rdParty-MSVC11-splitted%20version/eigen-eigen-dc6cfdf9bcec.7z" # TODO SV: provide a valid version if required
+ MSVC11 "https://repo-sam.inria.fr/fungraph/dependencies/ibr-common/win3rdParty-MSVC15-splitted%20version/eigen3.7z"
+ MSVC14 "https://repo-sam.inria.fr/fungraph/dependencies/ibr-common/win3rdParty-MSVC15-splitted%20version/eigen3.7z"
+ SET CHECK_CACHED_VAR eigen3_DIR PATH "eigen/share/eigen3/cmake"
+)
+include_directories(/usr/include/eigen3)
+add_definitions(-DEIGEN_INITIALIZE_MATRICES_BY_ZERO)
+
+#############
+## Find Boost
+#############
+set(Boost_REQUIRED_COMPONENTS "system;chrono;filesystem;date_time" CACHE INTERNAL "Boost Required Components")
+
+if (WIN32)
+ # boost multiset arguments
+ if (MSVC11 OR MSVC12)
+ set(boost_multiset_arguments
+ CHECK_CACHED_VAR BOOST_ROOT PATH "boost_1_55_0"
+ CHECK_CACHED_VAR BOOST_INCLUDEDIR PATH "boost_1_55_0"
+ CHECK_CACHED_VAR BOOST_LIBRARYDIR PATH "boost_1_55_0/${LIB_BUILT_DIR}"
+ #CHECK_CACHED_VAR Boost_COMPILER STRING "-${Boost_WIN3RDPARTY_VCID}" DOC "vcid (eg: -vc110 for MSVC11)"
+ CHECK_CACHED_VAR Boost_COMPILER STRING "-vc110" DOC "vcid (eg: -vc110 for MSVC11)" # NOTE: if it doesnt work, uncomment this option and set the right value for VisualC id
+ )
+ elseif (MSVC14 OR MSVC17)
+ set(boost_multiset_arguments
+ CHECK_CACHED_VAR BOOST_ROOT PATH "boost-1.71"
+ CHECK_CACHED_VAR BOOST_INCLUDEDIR PATH "boost-1.71"
+ CHECK_CACHED_VAR BOOST_LIBRARYDIR PATH "boost-1.71/${LIB_BUILT_DIR}"
+ CHECK_CACHED_VAR Boost_COMPILER STRING "-vc141" DOC "vcid (eg: -vc110 for MSVC11)" # NOTE: if it doesnt work, uncomment this option and set the right value for VisualC id
+ )
+
+ option(BOOST_MINIMAL_VERSION "Only get minimal Boost dependencies" ON)
+
+ if(${BOOST_MINIMAL_VERSION})
+ set(BOOST_MSVC14_ZIP "boost-1.71-ibr-minimal.7z")
+ else()
+ set(BOOST_MSVC14_ZIP "boost-1.71.7z")
+ endif()
+ else ()
+ message("There is no provided Boost library for your version of MSVC")
+ endif()
+
+ sibr_addlibrary(NAME Boost VCID TIMEOUT 600 #VERBOSE ON
+ MSVC11 "https://repo-sam.inria.fr/fungraph/dependencies/ibr-common/win3rdParty-MSVC11-splitted%20version/boost_1_55_0.7z"
+ MSVC12 "https://repo-sam.inria.fr/fungraph/dependencies/ibr-common/win3rdParty-MSVC11-splitted%20version/boost_1_55_0.7z"
+ MSVC14 "https://repo-sam.inria.fr/fungraph/dependencies/ibr-common/win3rdParty-MSVC15-splitted%20version/${BOOST_MSVC14_ZIP}" # boost compatible with msvc14
+ MULTI_SET ${boost_multiset_arguments}
+ CHECK_CACHED_VAR Boost_NO_SYSTEM_PATHS BOOL ON DOC "Set to ON to disable searching in locations not specified by these boost cached hint variables"
+ CHECK_CACHED_VAR Boost_NO_BOOST_CMAKE BOOL ON DOC "Set to ON to disable the search for boost-cmake (package cmake config file if boost was built with cmake)"
+ )
+ if(NOT Boost_COMPILER AND Boost_WIN3RDPARTY_USE)
+ message(WARNING "Boost_COMPILER is not set and it's needed.")
+ endif()
+endif()
+
+find_package(Boost 1.71.0 REQUIRED COMPONENTS ${Boost_REQUIRED_COMPONENTS})
+
+if(WIN32)
+ add_compile_options("$<$:/EHsc>")
+ #add_definitions(/EHsc)
+endif()
+
+if(Boost_LIB_DIAGNOSTIC_DEFINITIONS)
+ add_definitions(${Boost_LIB_DIAGNOSTIC_DEFINITIONS})
+endif()
+
+#if(WIN32)
+ add_definitions(-DBOOST_ALL_DYN_LINK -DBOOST_ALL_NO_LIB)
+#endif()
+
+include_directories(${BOOST_INCLUDEDIR} ${Boost_INCLUDE_DIRS})
+link_directories(${BOOST_LIBRARYDIR} ${Boost_LIBRARY_DIRS})
+
+
+##############
+## Find OpenMP
+##############
+find_package(OpenMP)
+
+sibr_addlibrary(
+ NAME NativeFileDialog
+ MSVC14 "https://repo-sam.inria.fr/fungraph/dependencies/sibr/~0.9/nfd.7z"
+)
+
+##############
+## Find OpenCV
+##############
+if (WIN32)
+ if (${MSVC_TOOLSET_VERSION} EQUAL 143)
+ MESSAGE("SPECIAL OPENCV HANDLING")
+ set(opencv_set_arguments
+ CHECK_CACHED_VAR OpenCV_DIR PATH "install" ## see OpenCVConfig.cmake
+ )
+ elseif (MSVC11 OR MSVC12)
+ set(opencv_set_arguments
+ CHECK_CACHED_VAR OpenCV_DIR PATH "opencv/build" ## see OpenCVConfig.cmake
+ )
+ elseif (MSVC14)
+ set(opencv_set_arguments
+ CHECK_CACHED_VAR OpenCV_DIR PATH "opencv-4.5.0/build" ## see OpenCVConfig.cmake
+ )
+ else ()
+ message("There is no provided OpenCV library for your compiler, relying on find_package to find it")
+ endif()
+else()
+ message("There is no provided OpenCV library for your compiler, relying on find_package to find it")
+endif()
+
+sibr_addlibrary(NAME OpenCV #VERBOSE ON
+ MSVC11 "https://repo-sam.inria.fr/fungraph/dependencies/sibr/~0.9/opencv.7z"
+ MSVC12 "https://repo-sam.inria.fr/fungraph/dependencies/sibr/~0.9/opencv.7z"
+ MSVC14 "https://repo-sam.inria.fr/fungraph/dependencies/sibr/~0.9/opencv-4.5.0.7z" # opencv compatible with msvc14 and with contribs
+ MSVC17 "https://repo-sam.inria.fr/fungraph/dependencies/sibr/~0.9/opencv4-8.7z"
+ SET ${opencv_set_arguments}
+ )
+find_package(OpenCV REQUIRED) ## Use directly the OpenCVConfig.cmake provided
+
+ ##https://stackoverflow.com/questions/24262081/cmake-relwithdebinfo-links-to-debug-libs
+set_target_properties(${OpenCV_LIBS} PROPERTIES MAP_IMPORTED_CONFIG_RELWITHDEBINFO RELEASE)
+
+add_definitions(-DOPENCV_TRAITS_ENABLE_DEPRECATED)
+
+if(OpenCV_INCLUDE_DIRS)
+ foreach(inc ${OpenCV_INCLUDE_DIRS})
+ if(NOT EXISTS ${inc})
+ set(OpenCV_INCLUDE_DIR "" CACHE PATH "additional custom include DIR (in case of trouble to find it (fedora 17 opencv package))")
+ endif()
+ endforeach()
+ if(OpenCV_INCLUDE_DIR)
+ list(APPEND OpenCV_INCLUDE_DIRS ${OpenCV_INCLUDE_DIR})
+ include_directories(${OpenCV_INCLUDE_DIRS})
+ endif()
+endif()
+
+###################
+## Find GLFW
+###################
+sibr_addlibrary(
+ NAME GLFW
+ MSVC11 "https://repo-sam.inria.fr/fungraph/dependencies/ibr-common/win3rdParty-MSVC15-splitted%20version/glfw-3.2.1.7z"
+ MSVC14 "https://repo-sam.inria.fr/fungraph/dependencies/ibr-common/win3rdParty-MSVC15-splitted%20version/glfw-3.2.1.7z" # TODO SV: provide a valid version if required
+)
+
+sibr_gitlibrary(TARGET imgui
+ GIT_REPOSITORY "https://gitlab.inria.fr/sibr/libs/imgui.git"
+ GIT_TAG "e7f0fa31b9fa3ee4ecd2620b9951f131b4e377c6"
+)
+
+sibr_gitlibrary(TARGET mrf
+ GIT_REPOSITORY "https://gitlab.inria.fr/sibr/libs/mrf.git"
+ GIT_TAG "564e5e0b395c788d2f8b2cf4f879fed2493faea7"
+)
+
+sibr_gitlibrary(TARGET nanoflann
+ GIT_REPOSITORY "https://gitlab.inria.fr/sibr/libs/nanoflann.git"
+ GIT_TAG "7a20a9ac0a1d34850fc3a9e398fc4a7618e8a69a"
+)
+
+sibr_gitlibrary(TARGET picojson
+ GIT_REPOSITORY "https://gitlab.inria.fr/sibr/libs/picojson.git"
+ GIT_TAG "7cf8feee93c8383dddbcb6b64cf40b04e007c49f"
+)
+
+sibr_gitlibrary(TARGET rapidxml
+ GIT_REPOSITORY "https://gitlab.inria.fr/sibr/libs/rapidxml.git"
+ GIT_TAG "069e87f5ec5ce1745253bd64d89644d6b894e516"
+)
+
+sibr_gitlibrary(TARGET xatlas
+ GIT_REPOSITORY "https://gitlab.inria.fr/sibr/libs/xatlas.git"
+ GIT_TAG "0fbe06a5368da13fcdc3ee48d4bdb2919ed2a249"
+ INCLUDE_DIRS "source/xatlas"
+)
+
+Win3rdPartyGlobalCacheAction()
diff --git a/submodules/gaussian-splatting/SIBR_viewers/cmake/windows/downloadAndExtractZipFile.cmake b/submodules/gaussian-splatting/SIBR_viewers/cmake/windows/downloadAndExtractZipFile.cmake
new file mode 100644
index 0000000000000000000000000000000000000000..7f5fc2bb080fc158840125e69c3d36d169b69235
--- /dev/null
+++ b/submodules/gaussian-splatting/SIBR_viewers/cmake/windows/downloadAndExtractZipFile.cmake
@@ -0,0 +1,243 @@
+# Copyright (C) 2020, Inria
+# GRAPHDECO research group, https://team.inria.fr/graphdeco
+# All rights reserved.
+#
+# This software is free for non-commercial, research and evaluation use
+# under the terms of the LICENSE.md file.
+#
+# For inquiries contact sibr@inria.fr and/or George.Drettakis@inria.fr
+
+
+## downloadAndExtractZipFile cmake function
+## Provide a way to download zip file from public internet ZIP_URL host
+## and to extract it in a specific EXCTRATED_ZIP_PATH destination.
+## This function use 7-Zip external tool to maximize the compatibles formats.
+## This will be not download again if the EXCTRATED_ZIP_PATH already exist and DL_FORCE is set to OFF.
+## This will try to unzip file if already exist in the ZIP_DL_PATH.
+##
+## If EXCTRATED_ZIP_PATH and/or ZIP_DL_PATH are not full path,
+## it will be interpreted relative to CMAKE_BINARY_DIR
+##
+## Usage example :
+## include(downloadAndExtractZipFile)
+## downloadAndExtractZipFile(
+## http://www.cs.cornell.edu/~snavely/bundler/distr/bundler-v0.4-source.zip
+## ${CMAKE_BINARY_DIR}/Bundler/bundler-v0.4-source.zip
+## ${CMAKE_BINARY_DIR}/Bundler
+## [DL_FORCE ON|OFF]
+## [TIMEOUT]
+## [CHECK_DIRTY_URL]
+## )
+##
+## option DL_FORCE will redownload the zip file [deafult to OFF]
+## option TIMEOUT will end the unzip process after this period of time [default to 600s]
+## option CHECK_DIRTY_URL will write into the given file the downloaded URL and then,
+## next time, if the URL was updated, it detect it with this file
+## and will download the last version. This prevent to alway set manually DL_FORCE to ON...
+##
+if(__downloadAndExtractZipFile_cmake_INCLUDED__)
+ return()
+else()
+ set(__downloadAndExtractZipFile_cmake_INCLUDED__ ON)
+endif()
+
+function(downloadAndExtractZipFile ZIP_URL ZIP_DL_PATH EXCTRATED_ZIP_PATH)
+
+ # message(STATUS "zipUrl=${ZIP_URL} zipDlPath=${ZIP_DL_PATH} extractedZipPath=${EXCTRATED_ZIP_PATH}")
+ cmake_parse_arguments(dwnlezf "" "VERBOSE;DL_FORCE;TIMEOUT;CHECK_DIRTY_URL" "" ${ARGN})
+
+ set(PROGRAMFILESx86 "PROGRAMFILES(x86)")
+
+ ## Check entries mandatory args
+ if(IS_ABSOLUTE "${ZIP_DL_PATH}")
+ else()
+ set(ZIP_DL_PATH "${CMAKE_BINARY_DIR}/${ZIP_DL_PATH}")
+ endif()
+ if(IS_ABSOLUTE "${EXCTRATED_ZIP_PATH}")
+ else()
+ set(EXCTRATED_ZIP_PATH "${CMAKE_BINARY_DIR}/${EXCTRATED_ZIP_PATH}")
+ endif()
+ if(NOT EXISTS "${EXCTRATED_ZIP_PATH}")
+ file(MAKE_DIRECTORY ${EXCTRATED_ZIP_PATH})
+ endif()
+
+ # SB: Once, one of downloaded zip was corrupted by an error message coming from the server.
+ if(EXISTS "${ZIP_DL_PATH}")
+ # So I check for removing such corrupted files
+ message("Removing previous ${ZIP_DL_PATH} (might be corrupted)")
+ file(REMOVE "${ZIP_DL_PATH}")
+ if(EXISTS "${dwnlezf_CHECK_DIRTY_URL}")
+ # and remove the previous (corrupted) made 'Win3rdPartyUrl' file
+ file(REMOVE "${dwnlezf_CHECK_DIRTY_URL}")
+ endif()
+ endif()
+
+ ## Check entries optional args
+ macro(readDirtyUrl )
+ if(dwnlezf_CHECK_DIRTY_URL)
+ if(IS_ABSOLUTE "${dwnlezf_CHECK_DIRTY_URL}")
+ else()
+ set(dwnlezf_CHECK_DIRTY_URL "${CMAKE_BINARY_DIR}/${dwnlezf_CHECK_DIRTY_URL}")
+ endif()
+ get_filename_component(unzipDir ${EXCTRATED_ZIP_PATH} NAME)
+ get_filename_component(unzipPath ${EXCTRATED_ZIP_PATH} PATH)
+ message(STATUS "Checking ${unzipDir} [from ${unzipPath}]...")
+ if(EXISTS "${dwnlezf_CHECK_DIRTY_URL}")
+ get_filename_component(CHECK_DIRTY_URL_FILENAME ${dwnlezf_CHECK_DIRTY_URL} NAME)
+ file(STRINGS "${dwnlezf_CHECK_DIRTY_URL}" contents)
+ list(GET contents 0 downloadURL)
+ list(REMOVE_AT contents 0)
+ if("${downloadURL}" MATCHES "${ZIP_URL}")
+ if(dwnlezf_VERBOSE)
+ message(STATUS "Your downloaded version (URL) seems to be up to date. Let me check if nothing is missing... (see ${dwnlezf_CHECK_DIRTY_URL}).")
+ endif()
+ file(GLOB PATHNAME_PATTERN_LIST "${EXCTRATED_ZIP_PATH}/*") ## is there something inside the downloaded destination ?
+ unset(NAME_PATTERN_LIST)
+ foreach(realPathPattern ${PATHNAME_PATTERN_LIST})
+ get_filename_component(itemName ${realPathPattern} NAME)
+ list(APPEND NAME_PATTERN_LIST ${itemName})
+ endforeach()
+ if(NAME_PATTERN_LIST)
+ foreach(item ${contents})
+ list(FIND NAME_PATTERN_LIST ${item} id)
+ if(${id} MATCHES "-1")
+ message(STATUS "${item} is missing, your downloaded version content changed, need to redownload it.")
+ set(ZIP_DL_FORCE ON)
+ break()
+ else()
+ list(REMOVE_AT NAME_PATTERN_LIST ${id})
+ set(ZIP_DL_FORCE OFF)
+ endif()
+ endforeach()
+ if(NOT ZIP_DL_FORCE AND NAME_PATTERN_LIST)
+ message("Yours seems to be up to date (regarding to ${CHECK_DIRTY_URL_FILENAME})!\nBut there are additional files/folders into your downloaded destination (feel free to clean it if you want).")
+ foreach(item ${NAME_PATTERN_LIST})
+ if(item)
+ message("${item}")
+ endif()
+ endforeach()
+ endif()
+ endif()
+ else()
+ set(ZIP_DL_FORCE ON)
+ message(STATUS "Your downloaded version is dirty (too old).")
+ endif()
+ else()
+ file(GLOB PATHNAME_PATTERN_LIST "${EXCTRATED_ZIP_PATH}/*") ## is there something inside the downloaded destination ?
+ if(NOT PATHNAME_PATTERN_LIST)
+ message("We found nothing into ${EXCTRATED_ZIP_PATH}, we will try to download it for you now.")
+ endif()
+ set(ZIP_DL_FORCE ON)
+ endif()
+ endif()
+ endmacro()
+ readDirtyUrl()
+ if(NOT ZIP_DL_FORCE)
+ return() ## do not need to further (as we are up to date, just exit the function
+ endif()
+
+ macro(writeDirtyUrl )
+ if(dwnlezf_CHECK_DIRTY_URL)
+ file(WRITE "${dwnlezf_CHECK_DIRTY_URL}" "${ZIP_URL}\n")
+ file(GLOB PATHNAME_PATTERN_LIST "${EXCTRATED_ZIP_PATH}/*") ## is there something inside the downloaded destination ?
+ unset(NAME_PATTERN_LIST)
+ foreach(realPathPattern ${PATHNAME_PATTERN_LIST})
+ get_filename_component(itemName ${realPathPattern} NAME)
+ list(APPEND NAME_PATTERN_LIST ${itemName})
+ endforeach()
+ if(NAME_PATTERN_LIST)
+ foreach(item ${NAME_PATTERN_LIST})
+ file(APPEND "${dwnlezf_CHECK_DIRTY_URL}" "${item}\n")
+ endforeach()
+ endif()
+ endif()
+ endmacro()
+
+ if(dwnlezf_DL_FORCE)
+ set(ZIP_DL_FORCE ON)
+ endif()
+
+ if(NOT dwnlezf_TIMEOUT)
+ set(dwnlezf_TIMEOUT 600)
+ endif()
+ math(EXPR dwnlezf_TIMEOUT_MIN "${dwnlezf_TIMEOUT}/60")
+
+ macro(unzip whichZipFile)
+ if(NOT SEVEN_ZIP_CMD)
+ find_program(SEVEN_ZIP_CMD NAMES 7z 7za p7zip DOC "7-zip executable" PATHS "$ENV{PROGRAMFILES}/7-Zip" "$ENV{${PROGRAMFILESx86}}/7-Zip" "$ENV{ProgramW6432}/7-Zip")
+ endif()
+ if(SEVEN_ZIP_CMD)
+ if(dwnlezf_VERBOSE)
+ message(STATUS "UNZIP: please, WAIT UNTIL ${SEVEN_ZIP_CMD} finished...\n(no more than ${dwnlezf_TIMEOUT_MIN} min)")
+ else()
+ message(STATUS "UNZIP...wait...")
+ endif()
+ execute_process( COMMAND ${SEVEN_ZIP_CMD} x ${whichZipFile} -y
+ WORKING_DIRECTORY ${EXCTRATED_ZIP_PATH} TIMEOUT ${dwnlezf_TIMEOUT}
+ RESULT_VARIABLE resVar OUTPUT_VARIABLE outVar ERROR_VARIABLE errVar
+ )
+ if(${resVar} MATCHES "0")
+ if(dwnlezf_VERBOSE)
+ message(STATUS "SUCESS to unzip in ${EXCTRATED_ZIP_PATH}. Now we can remove the downloaded zip file.")
+ endif()
+ execute_process(COMMAND ${CMAKE_COMMAND} -E remove ${whichZipFile})
+ mark_as_advanced(SEVEN_ZIP_CMD)
+ else()
+ message(WARNING "something wrong in ${EXCTRATED_ZIP_PATH}\n with \"${SEVEN_ZIP_CMD} x ${whichZipFile} -y\", redo or try to unzip by yourself...")
+ message("unzip: resVar=${resVar}")
+ message("unzip: outVar=${outVar}")
+ message("unzip: errVar=${errVar}")
+ message("unzip: failed or canceled or timeout")
+ endif()
+ else()
+ message(WARNING "You need 7zip (http://www.7-zip.org/download.html) to unzip the downloaded dir.")
+ set(SEVEN_ZIP_CMD "" CACHE FILEPATH "7-zip executable")
+ mark_as_advanced(CLEAR SEVEN_ZIP_CMD)
+ endif()
+ endmacro()
+
+ if(dwnlezf_VERBOSE)
+ message(STATUS "Trying to look ${ZIP_DL_PATH} if a zip file exist...")
+ endif()
+ if(EXISTS "${ZIP_DL_PATH}")
+
+ ## already downloaded, so just unzip it
+ unzip(${ZIP_DL_PATH})
+ writeDirtyUrl()
+
+ elseif(ZIP_DL_FORCE)
+
+ ## the download part (+ unzip)
+ message(STATUS "Let me try to download package for you : ${ZIP_URL}")
+ if(dwnlezf_VERBOSE)
+ message(STATUS "Downloading...\n SRC=${ZIP_URL}\n DEST=${ZIP_DL_PATH}.tmp\n INACTIVITY_TIMEOUT=180s")
+ endif()
+ file(DOWNLOAD ${ZIP_URL} ${ZIP_DL_PATH}.tmp INACTIVITY_TIMEOUT 360 STATUS status SHOW_PROGRESS)
+
+ list(GET status 0 numResult)
+ if(${numResult} MATCHES "0")
+
+ if(dwnlezf_VERBOSE)
+ message(STATUS "Download succeed, so let me rename the tmp file to unzip it")
+ endif()
+ execute_process(COMMAND ${CMAKE_COMMAND} -E rename ${ZIP_DL_PATH}.tmp ${ZIP_DL_PATH})
+ unzip(${ZIP_DL_PATH})
+ writeDirtyUrl()
+
+ else()
+
+ list(GET status 1 errMsg)
+ message(WARNING "DOWNLOAD ${ZIP_URL} to ${ZIP_DL_PATH} failed\n:${errMsg}")
+ message(WARNING "OK, you need to download the ${ZIP_URL} manually and put it into ${ZIP_DL_PATH}")
+ message("Take a look at the project website page to check available URL.")
+
+ endif()
+
+ endif()
+
+ ## clean up the tmp downloaded file
+ if(EXISTS "${ZIP_DL_PATH}.tmp")
+ execute_process(COMMAND ${CMAKE_COMMAND} -E remove ${ZIP_DL_PATH}.tmp)
+ endif()
+
+endfunction()
diff --git a/submodules/gaussian-splatting/SIBR_viewers/cmake/windows/git_describe.cmake b/submodules/gaussian-splatting/SIBR_viewers/cmake/windows/git_describe.cmake
new file mode 100644
index 0000000000000000000000000000000000000000..638d70bd9440fe80ff1844500d3acf8f05e669b8
--- /dev/null
+++ b/submodules/gaussian-splatting/SIBR_viewers/cmake/windows/git_describe.cmake
@@ -0,0 +1,114 @@
+# Copyright (C) 2020, Inria
+# GRAPHDECO research group, https://team.inria.fr/graphdeco
+# All rights reserved.
+#
+# This software is free for non-commercial, research and evaluation use
+# under the terms of the LICENSE.md file.
+#
+# For inquiries contact sibr@inria.fr and/or George.Drettakis@inria.fr
+
+
+if(__git_describe_INCLUDED__)
+ return()
+else()
+ set(__git_describe_INCLUDED__ ON)
+endif()
+
+find_package(Git)
+if(Git_FOUND)
+ message(STATUS "Git found: ${GIT_EXECUTABLE}")
+else()
+ message(FATAL_ERROR "Git not found. Aborting")
+endif()
+
+macro(git_describe)
+ cmake_parse_arguments(GIT_DESCRIBE "" "GIT_URL;GIT_BRANCH;GIT_COMMIT_HASH;GIT_TAG;GIT_VERSION;PATH" "" ${ARGN})
+
+ if(NOT GIT_DESCRIBE_PATH)
+ set(GIT_DESCRIBE_PATH ${CMAKE_SOURCE_DIR})
+ endif()
+
+ if(GIT_DESCRIBE_GIT_URL)
+ # Get the current remote
+ execute_process(
+ COMMAND git remote
+ WORKING_DIRECTORY ${GIT_DESCRIBE_PATH}
+ OUTPUT_VARIABLE GIT_DESCRIBE_GIT_REMOTE
+ OUTPUT_STRIP_TRAILING_WHITESPACE
+ ERROR_QUIET
+ )
+
+ # Get the current remote
+ execute_process(
+ COMMAND git remote get-url ${GIT_DESCRIBE_GIT_REMOTE}
+ WORKING_DIRECTORY ${GIT_DESCRIBE_PATH}
+ OUTPUT_VARIABLE ${GIT_DESCRIBE_GIT_URL}
+ OUTPUT_STRIP_TRAILING_WHITESPACE
+ ERROR_QUIET
+ )
+ endif()
+
+ if(GIT_DESCRIBE_GIT_BRANCH)
+ # Get the current working branch
+ execute_process(
+ COMMAND git rev-parse --abbrev-ref HEAD
+ WORKING_DIRECTORY ${GIT_DESCRIBE_PATH}
+ OUTPUT_VARIABLE ${GIT_DESCRIBE_GIT_BRANCH}
+ OUTPUT_STRIP_TRAILING_WHITESPACE
+ ERROR_QUIET
+ )
+ endif()
+
+ if(GIT_DESCRIBE_GIT_COMMIT_HASH)
+ # Get the latest abbreviated commit hash of the working branch
+ execute_process(
+ COMMAND git rev-parse HEAD
+ WORKING_DIRECTORY ${GIT_DESCRIBE_PATH}
+ OUTPUT_VARIABLE ${GIT_DESCRIBE_GIT_COMMIT_HASH}
+ OUTPUT_STRIP_TRAILING_WHITESPACE
+ ERROR_QUIET
+ )
+ endif()
+
+ if(GIT_DESCRIBE_GIT_TAG)
+ # Get the tag
+ execute_process(
+ COMMAND git describe --tags --exact-match
+ WORKING_DIRECTORY ${GIT_DESCRIBE_PATH}
+ OUTPUT_VARIABLE ${GIT_DESCRIBE_GIT_TAG}
+ OUTPUT_STRIP_TRAILING_WHITESPACE
+ ERROR_QUIET
+ )
+ endif()
+
+ if(GIT_DESCRIBE_GIT_VERSION)
+ # Get the version from git describe
+ execute_process(
+ COMMAND git describe
+ WORKING_DIRECTORY ${GIT_DESCRIBE_PATH}
+ OUTPUT_VARIABLE ${GIT_DESCRIBE_GIT_VERSION}
+ OUTPUT_STRIP_TRAILING_WHITESPACE
+ ERROR_QUIET
+ )
+
+ if(${GIT_DESCRIBE_GIT_VERSION} STREQUAL "")
+ execute_process(
+ COMMAND git rev-parse --abbrev-ref HEAD
+ WORKING_DIRECTORY ${GIT_DESCRIBE_PATH}
+ OUTPUT_VARIABLE GIT_DESCRIBE_GIT_VERSION_BRANCH
+ OUTPUT_STRIP_TRAILING_WHITESPACE
+ ERROR_QUIET
+ )
+ execute_process(
+ COMMAND git log -1 --format=%h
+ WORKING_DIRECTORY ${GIT_DESCRIBE_PATH}
+ OUTPUT_VARIABLE GIT_DESCRIBE_GIT_VERSION_COMMIT
+ OUTPUT_STRIP_TRAILING_WHITESPACE
+ ERROR_QUIET
+ )
+
+ set(${GIT_DESCRIBE_GIT_VERSION} "${GIT_DESCRIBE_GIT_VERSION_BRANCH}-${GIT_DESCRIBE_GIT_VERSION_COMMIT}")
+ endif()
+ endif()
+
+endmacro()
\ No newline at end of file
diff --git a/submodules/gaussian-splatting/SIBR_viewers/cmake/windows/include_once.cmake b/submodules/gaussian-splatting/SIBR_viewers/cmake/windows/include_once.cmake
new file mode 100644
index 0000000000000000000000000000000000000000..d28b39cfeb18e2369ccee1d9f038cfdd71958296
--- /dev/null
+++ b/submodules/gaussian-splatting/SIBR_viewers/cmake/windows/include_once.cmake
@@ -0,0 +1,22 @@
+# Copyright (C) 2020, Inria
+# GRAPHDECO research group, https://team.inria.fr/graphdeco
+# All rights reserved.
+#
+# This software is free for non-commercial, research and evaluation use
+# under the terms of the LICENSE.md file.
+#
+# For inquiries contact sibr@inria.fr and/or George.Drettakis@inria.fr
+
+
+macro(include_once file)
+ get_filename_component(INCLUDE_ONCE_FILEPATH ${file} REALPATH)
+ string(REGEX REPLACE "(\\.|\\/+|\\:|\\\\+)" "_" INCLUDE_ONCE_FILEPATH ${INCLUDE_ONCE_FILEPATH})
+ get_property(INCLUDED_${INCLUDE_ONCE_FILEPATH}_LOCAL GLOBAL PROPERTY INCLUDED_${INCLUDE_ONCE_FILEPATH})
+ if (INCLUDED_${INCLUDE_ONCE_FILEPATH}_LOCAL)
+ return()
+ else()
+ set_property(GLOBAL PROPERTY INCLUDED_${INCLUDE_ONCE_FILEPATH} true)
+
+ include(${file})
+ endif()
+endmacro()
\ No newline at end of file
diff --git a/submodules/gaussian-splatting/SIBR_viewers/cmake/windows/install_runtime.cmake b/submodules/gaussian-splatting/SIBR_viewers/cmake/windows/install_runtime.cmake
new file mode 100644
index 0000000000000000000000000000000000000000..3d4b74e6953c97975d9a7378643dbe8e877055b3
--- /dev/null
+++ b/submodules/gaussian-splatting/SIBR_viewers/cmake/windows/install_runtime.cmake
@@ -0,0 +1,880 @@
+# Copyright (C) 2020, Inria
+# GRAPHDECO research group, https://team.inria.fr/graphdeco
+# All rights reserved.
+#
+# This software is free for non-commercial, research and evaluation use
+# under the terms of the LICENSE.md file.
+#
+# For inquiries contact sibr@inria.fr and/or George.Drettakis@inria.fr
+
+
+## This file is mainly used to allow runtime installation
+## There are some utilities cmake functions to ease the generic deployement (abstract common usage of cmake)...
+##
+## You cannot run your programm automaticaly from your CNAKE_BINARY_DIR when you build
+## as it will miss all dependencies and ressources files...
+## You have to run install target in order to test your programm.
+##
+## The only one function/macros you may use inside your sub-CMakeLists.txt (sub-project) is :
+## ******************
+## ibr_install_target macro => see documentation at the end of this file
+## ******************
+## It use these utilities cmake functions to abstract the installation in an uniform way for all sub-projects.
+##
+if(__install_runtime_cmake_INCLUDED__)
+ return()
+else()
+ set(__install_runtime_cmake_INCLUDED__ ON)
+endif()
+
+
+##
+## Allow to write a resource config file which contain additional ressource paths
+## (used by IBR_Common Resource system to load shaders and potentialy images, plugins and so on)
+##
+## ADD option list all the paths to add in the file (relative paths are interpreted relative to working dir of the executable)
+## INSTALL option to specify where we want to install this file
+##
+## Example usage:
+## resourceFile(ADD "shaders" "${PROJECT_NAME}_rsc" INSTALL bin)
+##
+macro(resourceFile)
+ cmake_parse_arguments(rsc "" "INSTALL;FILE_PATH;CONFIG_TYPE" "ADD" ${ARGN}) ## both args are directory path
+
+ if(rsc_ADD)
+ unset(IBR_RSC_FILE_CONTENT_LIST)
+ if(EXISTS "${rsc_FILE_PATH}")
+ file(READ "${rsc_FILE_PATH}" IBR_RSC_FILE_CONTENT)
+ string(REGEX REPLACE "\n" ";" IBR_RSC_FILE_CONTENT_LIST "${IBR_RSC_FILE_CONTENT}")
+ endif()
+ list(APPEND IBR_RSC_FILE_CONTENT_LIST "${rsc_ADD}")
+ list(REMOVE_DUPLICATES IBR_RSC_FILE_CONTENT_LIST)
+ file(WRITE "${rsc_FILE_PATH}" "")
+ foreach(rscDir ${IBR_RSC_FILE_CONTENT_LIST})
+ file(APPEND "${rsc_FILE_PATH}" "${rscDir}\n")
+ endforeach()
+ unset(rsc_ADD)
+ endif()
+
+ if(rsc_INSTALL)
+ install(FILES ${rsc_FILE_PATH} CONFIGURATIONS ${rsc_CONFIG_TYPE} DESTINATION ${rsc_INSTALL})
+ unset(rsc_INSTALL)
+ endif()
+endmacro()
+
+
+##
+## Install *.pdb generated file for the current cmake project
+## assuming the output target name is the cmake project name.
+## This macro is useful for crossplateform multi config mode.
+##
+## Usage Example:
+##
+## if(DEFINED CMAKE_BUILD_TYPE) ## for make/nmake based
+## installPDB(${PROJECT_NAME} ${CMAKE_BUILD_TYPE} RUNTIME_DEST bin ARCHIVE_DEST lib LIBRARY_DEST lib)
+## endif()
+## foreach(CONFIG_TYPES ${CMAKE_CONFIGURATION_TYPES}) ## for multi config types (MSVC based)
+## installPDB(${PROJECT_NAME} ${CONFIG_TYPES} RUNTIME_DEST bin ARCHIVE_DEST lib LIBRARY_DEST lib)
+## endforeach()
+##
+macro(installPDB targetName configType)
+ cmake_parse_arguments(instpdb "" "COMPONENT" "ARCHIVE_DEST;LIBRARY_DEST;RUNTIME_DEST" ${ARGN}) ## both args are directory path
+
+ if(NOT MSVC)
+ return()
+ endif()
+
+ ## Check if DESTINATION are provided according to the TYPE of the given target (see install command doc to see correspodances)
+ get_target_property(type ${targetName} TYPE)
+ if(${type} MATCHES "EXECUTABLE" AND instpdb_RUNTIME_DEST)
+ set(pdb_DESTINATION ${instpdb_RUNTIME_DEST})
+ elseif(${type} MATCHES "STATIC_LIBRARY" AND instpdb_ARCHIVE_DEST)
+ set(pdb_DESTINATION ${instpdb_ARCHIVE_DEST})
+ elseif(${type} MATCHES "MODULE_LIBRARY" AND instpdb_LIBRARY_DEST)
+ set(pdb_DESTINATION ${instpdb_LIBRARY_DEST})
+ elseif(${type} MATCHES "SHARED_LIBRARY")
+ if(WIN32 AND instpdb_RUNTIME_DEST)
+ set(pdb_DESTINATION ${instpdb_RUNTIME_DEST})
+ else()
+ set(pdb_DESTINATION ${instpdb_LIBRARY_DEST})
+ endif()
+ endif()
+
+ if(NOT pdb_DESTINATION)
+ set(pdb_DESTINATION bin) ## default destination of the pdb file
+ endif()
+
+ if(NOT instpdb_COMPONENT)
+ set(instpdb_COMPONENT )
+ else()
+ set(instpdb_COMPONENT COMPONENT ${instpdb_COMPONENT})
+ endif()
+
+ string(TOUPPER ${configType} CONFIG_TYPES_UC)
+ get_target_property(PDB_PATH ${targetName} PDB_OUTPUT_DIRECTORY_${CONFIG_TYPES_UC})
+
+ get_target_property(confModePostfix ${targetName} ${CONFIG_TYPES_UC}_POSTFIX)
+ if(NOT confModePostfix)
+ set(confModePostfix "")
+ endif()
+ set_target_properties(${targetName} PROPERTIES PDB_NAME_${CONFIG_TYPES_UC} ${targetName}${confModePostfix})
+ get_target_property(PDB_NAME ${targetName} PDB_NAME_${CONFIG_TYPES_UC})# if not set, this is empty
+
+ if(EXISTS "${PDB_PATH}/${PDB_NAME}.pdb")
+ install(FILES "${PDB_PATH}/${PDB_NAME}.pdb" CONFIGURATIONS ${configType} DESTINATION ${pdb_DESTINATION} ${instpdb_COMPONENT} OPTIONAL)
+ endif()
+endmacro()
+
+
+##
+## Add additional target to install a project independently and based on its component
+## configMode is used to prevent default Release installation (we want also to install in other build/config type)
+##
+macro(installTargetProject targetOfProject targetOfInstallProject)
+ if(DEFINED CMAKE_BUILD_TYPE) ## for make/nmake based
+ set(configMode ${CMAKE_BUILD_TYPE})
+ elseif(MSVC)
+ ## $(Configuration) will be one of the following : Debug, Release, MinSizeRel, RelWithDebInfo
+ set(configMode $(Configuration))
+ endif()
+ if(configMode)
+ get_target_property(srcFiles ${targetOfProject} SOURCES)
+ add_custom_target( ${targetOfInstallProject} #ALL
+ ${CMAKE_COMMAND} -DBUILD_TYPE=${configMode} -DCOMPONENT=${targetOfInstallProject} -P ${CMAKE_BINARY_DIR}/cmake_install.cmake
+ DEPENDS ${srcFiles}
+ COMMENT "run the installation only for ${targetOfProject}" VERBATIM
+ )
+ add_dependencies(${targetOfInstallProject} ${targetOfProject})
+
+ get_target_property(INSTALL_BUILD_FOLDER ${targetOfProject} FOLDER)
+ set_target_properties(${targetOfInstallProject} PROPERTIES FOLDER ${INSTALL_BUILD_FOLDER})
+ endif()
+endmacro()
+
+# Collect all currently added targets in all subdirectories
+#
+# Parameters:
+# - _result the list containing all found targets
+# - _dir root directory to start looking from
+function(get_all_targets _result _dir)
+ get_property(_subdirs DIRECTORY "${_dir}" PROPERTY SUBDIRECTORIES)
+ foreach(_subdir IN LISTS _subdirs)
+ get_all_targets(${_result} "${_subdir}")
+ endforeach()
+
+ get_directory_property(_sub_targets DIRECTORY "${_dir}" BUILDSYSTEM_TARGETS)
+ set(${_result} ${${_result}} ${_sub_targets} PARENT_SCOPE)
+endfunction()
+
+##
+## Add targets for building and installing subdirectories
+macro(subdirectory_target target directory build_folder)
+ add_custom_target(${target}
+ COMMENT "run build for all projects in this directory" VERBATIM
+ )
+ get_all_targets(ALL_TARGETS ${directory})
+ add_dependencies(${target} ${ALL_TARGETS})
+ add_custom_target(${target}_install
+ ${CMAKE_COMMAND} -DBUILD_TYPE=$ -DCOMPONENT=${target}_install -P ${CMAKE_BINARY_DIR}/cmake_install.cmake
+ COMMENT "run install for all projects in this directory" VERBATIM
+ )
+ add_dependencies(${target}_install ${target})
+
+ set_target_properties(${target} PROPERTIES FOLDER ${build_folder})
+ set_target_properties(${target}_install PROPERTIES FOLDER ${build_folder})
+endmacro()
+
+
+## CMAKE install all required dependencies for an application (included system OS files like msvc*.dll for example)
+##
+## install_runtime(
+## [TARGET name]
+## [PLUGINS name [nameN ...] [PLUGIN_PATH_NAME currentPathName [FROM_REL_PATH matchDirFromCurrentPathName] [PLUGIN_PATH_DEST installDir] ]
+## [PLUGINS ...]
+## [DIRS path [pathN ...] ]
+## [TARGET_LIBRARIES filePath [filePathN ...] ]
+## [TARGET_PACKAGES packageName [packageNameN ...] ]
+## [COMPONENT installComponentName]
+## [PLAUSIBLES_POSTFIX Debug_postfix [MinSizeRel_postfix relWithDebInfo_postfix ...] ]
+## [VERBOSE]
+## )
+##
+## installedFilePathTargetAppToResolve : the final installed targetApp absolute full file path name you want to resolve
+##
+## TARGET : The target app we want to install. If given, it's used to look for link libraries paths (best choice to use, strongly advised to use it)
+##
+## PLUGINS : Some application built use/load some plugins which can't be detect inside its binary,
+## so, here you can specify which plugins the application use/load in order to install them
+## and resolve also there dependencies.
+## With PLUGINS multi FLAGS :
+## PLUGIN_PATH_NAME : The current plugin full file path we want to install
+## FROM_REL_PATH : [optional: default only the file is kept] From which matching dir of the plugin path we want to install (keep the directories structure)
+## PLUGIN_PATH_DEST : [optional: default relative to executable directory] Where (full path to the install directory) we will install the plugin file (or file path)
+##
+## DIRS : A list of directories to looking for dependencies
+## TARGET_LIBRARIES : DEPRECATED (use TARGET flag instead) : The cmake content variables used for the target_link_libraries( ...)
+## TARGET_PACKAGES : DEPRECATED (use TARGET flag instead) : The cmake package names used for the findPackage(...) for your targetApp
+## ADVICE: This flag add entries in cache (like: _DIR), it could be useful to fill these variable!
+## COMPONENT : (default to runtime) Is the component name associated to the installation
+## It is used when you want to install separatly some part of your projets (see install cmake doc)
+## VERBOSE : For debug or to get more informations in the output console
+##
+## Usage:
+## install_runtime(${CMAKE_INSTALL_PREFIX}/${EXECUTABLE_NAME}${CMAKE_EXECUTABLE_SUFFIX}
+## VERBOSE
+## TARGET ${PROJECT_NAME}
+## PLAUSIBLES_POSTFIX _d
+## PLUGINS
+## PLUGIN_PATH_NAME ${PLUGIN_PATH_NAME}${CMAKE_SHARED_MODULE_SUFFIX} ## will be installed (default exec path if no PLUGINS_DEST) and then will be resolved
+## FROM_REL_PATH plugins ## optional, used especially for keeping qt plugins tree structure
+## PLUGIN_PATH_DEST ${CMAKE_INSTALL_PREFIX}/plugins ## (or relative path 'plugins' will be interpreted relative to installed executable)
+## DIRS ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_BINARY_DIR}
+## TARGET_LIBRARIES ${OPENGL_LIBRARIES} ## DEPRECATED (use TARGET flag instead)
+## ${GLEW_LIBRARIES}
+## ${GLUT_LIBRARIES}
+## ${Boost_LIBRARIES}
+## ${SuiteSparse_LIBRARIES}
+## ${CGAL_LIBRARIES}
+## TARGET_PACKAGES OPENGL ## DEPRECATED (use TARGET flag instead)
+## GLEW
+## GLUT
+## CGAL
+## Boost
+## SuiteSparse
+## )
+##
+## For plugins part, it use our internal parse_arguments_multi.cmake
+##
+function(install_runtime installedFilePathTargetAppToResolve)
+ set(optionsArgs "VERBOSE")
+ set(oneValueArgs "COMPONENT;INSTALL_FOLDER;CONFIG_TYPE")
+ set(multiValueArgs "DIRS;PLUGINS;TARGET_LIBRARIES;TARGET_PACKAGES;TARGET;PLAUSIBLES_POSTFIX")
+ cmake_parse_arguments(inst_run "${optionsArgs}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN} )
+
+ if(IS_ABSOLUTE ${installedFilePathTargetAppToResolve})
+ else()
+ set(installedFilePathTargetAppToResolve ${inst_run_INSTALL_FOLDER}/${installedFilePathTargetAppToResolve})
+ endif()
+
+ get_filename_component(EXEC_NAME ${installedFilePathTargetAppToResolve} NAME_WE)
+ get_filename_component(EXEC_PATH ${installedFilePathTargetAppToResolve} PATH)
+
+ if(NOT inst_run_COMPONENT)
+ set(inst_run_COMPONENT runtime)
+ endif()
+
+
+ ## Try to append as more possible as possible paths to find dependencies (deprecated since we can use target_properties to get back paths)
+ set(libPaths )
+ foreach(libraryFileName ${inst_run_TARGET_LIBRARIES})
+ if(IS_DIRECTORY "${libraryFileName}")
+ list(APPEND libPaths "${libraryFileName}")
+ else()
+ get_filename_component(libpath "${libraryFileName}" PATH)
+ if(EXISTS "${libpath}")
+ list(APPEND libPaths "${libpath}")
+ endif()
+ endif()
+ endforeach()
+
+ ## This macro is used internaly here to recursilvely get path of LINK_LIBRARIES of each non imported target
+ ## Typically if you have 2 internal dependencies between cmake targets, we want cmake to be able to get back path where are these dependencies
+ macro(recurseDepList target)
+ get_target_property(linkLibs ${target} LINK_LIBRARIES)
+ foreach(lib ${linkLibs})
+ string(FIND ${lib} ">" strId) ## cmake is using generator-expression?
+ if(TARGET ${lib})
+ ## Skipping interface libraries as they're system ones
+ get_target_property(type ${lib} TYPE)
+ get_target_property(imported ${lib} IMPORTED)
+ if(type STREQUAL "INTERFACE_LIBRARY")
+ get_target_property(imp_loc ${lib} INTERFACE_IMPORTED_LOCATION)
+ if(imp_loc)
+ get_filename_component(imp_loc ${imp_loc} PATH)
+ list(APPEND targetLibPath ${imp_loc})
+ endif()
+ get_target_property(loc ${lib} INTERFACE_LOCATION)
+ if(loc)
+ get_filename_component(loc ${loc} PATH)
+ list(APPEND targetLibPath ${loc})
+ endif()
+ ## it's not a path but a single target name
+ ## for build-target which are part of the current cmake configuration : nothing to do as cmake already know the output path
+ ## for imported target, we need to look for theire imported location
+ elseif(imported)
+ get_target_property(imp_loc ${lib} IMPORTED_LOCATION)
+ if(imp_loc)
+ get_filename_component(imp_loc ${imp_loc} PATH)
+ list(APPEND targetLibPath ${imp_loc})
+ endif()
+ get_target_property(loc ${lib} LOCATION)
+ if(loc)
+ get_filename_component(loc ${loc} PATH)
+ list(APPEND targetLibPath ${loc})
+ endif()
+ else()
+ recurseDepList(${lib})
+ endif()
+ elseif(NOT ${strId} MATCHES -1) ## mean cmake use generator-expression (CMAKE VERSION > 3.0)
+ string(REGEX MATCH ">:[@A-Za-z_:/.0-9-]+" targetLibPath ${lib})
+ string(REGEX REPLACE ">:([@A-Za-z_:/.0-9-]+)" "\\1" targetLibPath ${targetLibPath})
+ get_filename_component(targetLibPath ${targetLibPath} PATH)
+ elseif(EXISTS ${lib})
+ set(targetLibPath ${lib})
+ get_filename_component(targetLibPath ${targetLibPath} PATH)
+ else()
+ #message(STATUS "[install_runtime] skip link library : ${lib} , of target ${target}")
+ endif()
+ if(targetLibPath)
+ list(APPEND targetLinkLibsPathList ${targetLibPath})
+ endif()
+ endforeach()
+ if(targetLinkLibsPathList)
+ list(REMOVE_DUPLICATES targetLinkLibsPathList)
+ endif()
+ endmacro()
+ if(inst_run_TARGET)
+ recurseDepList(${inst_run_TARGET})
+ if(targetLinkLibsPathList)
+ list(APPEND libPaths ${targetLinkLibsPathList})
+ endif()
+ endif()
+
+ if(libPaths)
+ list(REMOVE_DUPLICATES libPaths)
+ foreach(libPath ${libPaths})
+ get_filename_component(path ${libPath} PATH)
+ list(APPEND libPaths ${path})
+ endforeach()
+ endif()
+
+
+ ## possible speciale dir(s) according to the build system and OS
+ if(CMAKE_SIZEOF_VOID_P EQUAL 8)
+ set(BUILD_TYPES_FOR_DLL "x64")
+ if(WIN32)
+ list(APPEND BUILD_TYPES_FOR_DLL "Win64")
+ endif()
+ else()
+ set(BUILD_TYPES_FOR_DLL "x86")
+ if(WIN32)
+ list(APPEND BUILD_TYPES_FOR_DLL "Win32")
+ endif()
+ endif()
+
+
+ ## Try to append as more as possible paths to find dependencies (here, mainly for *.dll)
+ foreach(dir ${inst_run_DIRS} ${libPaths})
+ if(EXISTS "${dir}/bin")
+ list(APPEND inst_run_DIRS "${dir}/bin")
+ elseif(EXISTS "${dir}")
+ list(APPEND inst_run_DIRS "${dir}")
+ endif()
+ endforeach()
+ list(REMOVE_DUPLICATES inst_run_DIRS)
+ foreach(dir ${inst_run_DIRS})
+ if(EXISTS "${dir}")
+ list(APPEND argDirs ${dir})
+ foreach(BUILD_TYPE_FOR_DLL ${BUILD_TYPES_FOR_DLL})
+ if(EXISTS "${dir}/${BUILD_TYPE_FOR_DLL}")
+ list(APPEND argDirs "${dir}/${BUILD_TYPE_FOR_DLL}")
+ endif()
+ foreach(OUTPUTCONFIG ${CMAKE_CONFIGURATION_TYPES}) ## for windows multi-generator (MSVC)
+ if(EXISTS "${dir}/${BUILD_TYPE_FOR_DLL}/${OUTPUTCONFIG}")
+ list(APPEND argDirs "${dir}/${BUILD_TYPE_FOR_DLL}/${OUTPUTCONFIG}")
+ endif()
+ endforeach()
+ if(CMAKE_BUILD_TYPE) ## for single generator (makefiles)
+ if(EXISTS "${dir}/${BUILD_TYPE_FOR_DLL}/${CMAKE_BUILD_TYPE}")
+ list(APPEND argDirs "${dir}/${BUILD_TYPE_FOR_DLL}/${CMAKE_BUILD_TYPE}")
+ endif()
+ endif()
+ endforeach()
+ foreach(OUTPUTCONFIG ${CMAKE_CONFIGURATION_TYPES}) ## for windows multi-generator (MSVC)
+ if(EXISTS "${dir}/${OUTPUTCONFIG}")
+ list(APPEND argDirs "${dir}/${OUTPUTCONFIG}")
+ endif()
+ foreach(BUILD_TYPE_FOR_DLL ${BUILD_TYPES_FOR_DLL})
+ if(EXISTS "${dir}/${OUTPUTCONFIG}/${BUILD_TYPE_FOR_DLL}")
+ list(APPEND argDirs "${dir}/${OUTPUTCONFIG}/${BUILD_TYPE_FOR_DLL}")
+ endif()
+ endforeach()
+ endforeach()
+ if(CMAKE_BUILD_TYPE) ## for single generator (makefiles)
+ if(EXISTS "${dir}/${CMAKE_BUILD_TYPE}")
+ list(APPEND argDirs "${dir}/${CMAKE_BUILD_TYPE}")
+ endif()
+ foreach(BUILD_TYPE_FOR_DLL ${BUILD_TYPES_FOR_DLL})
+ if(EXISTS "${dir}/${CMAKE_BUILD_TYPE}/${BUILD_TYPE_FOR_DLL}")
+ list(APPEND argDirs "${dir}/${CMAKE_BUILD_TYPE}/${BUILD_TYPE_FOR_DLL}")
+ endif()
+ endforeach()
+ endif()
+ endif()
+ endforeach()
+ if(argDirs)
+ list(REMOVE_DUPLICATES argDirs)
+ endif()
+
+
+ ## Try to append as more possible paths to find dependencies (here, mainly for *.dll)
+ foreach(packageName ${inst_run_TARGET_PACKAGES})
+ if(EXISTS "${${packageName}_DIR}")
+ list(APPEND packageDirs ${${packageName}_DIR})
+ list(APPEND packageDirs ${${packageName}_DIR}/bin)
+ foreach(BUILD_TYPE_FOR_DLL ${BUILD_TYPES_FOR_DLL})
+ if(EXISTS "${${packageName}_DIR}/bin/${BUILD_TYPE_FOR_DLL}")
+ list(APPEND packageDirs "${${packageName}_DIR}/bin/${BUILD_TYPE_FOR_DLL}")
+ endif()
+ foreach(OUTPUTCONFIG ${CMAKE_CONFIGURATION_TYPES}) ## for windows multi-generator (MSVC)
+ if(EXISTS "${${packageName}_DIR}/bin/${BUILD_TYPE_FOR_DLL}/${OUTPUTCONFIG}")
+ list(APPEND packageDirs "${${packageName}_DIR}/bin/${BUILD_TYPE_FOR_DLL}/${OUTPUTCONFIG}")
+ endif()
+ endforeach()
+ if(CMAKE_BUILD_TYPE) ## for single generator (makefiles)
+ if(EXISTS "${${packageName}_DIR}/bin/${BUILD_TYPE_FOR_DLL}/${CMAKE_BUILD_TYPE}")
+ list(APPEND packageDirs "${${packageName}_DIR}/bin/${BUILD_TYPE_FOR_DLL}/${CMAKE_BUILD_TYPE}")
+ endif()
+ endif()
+ endforeach()
+ foreach(OUTPUTCONFIG ${CMAKE_CONFIGURATION_TYPES}) ## for windows multi-generator (MSVC)
+ if(EXISTS "${${packageName}_DIR}/bin/${OUTPUTCONFIG}")
+ list(APPEND packageDirs "${${packageName}_DIR}/bin/${OUTPUTCONFIG}")
+ endif()
+ foreach(BUILD_TYPE_FOR_DLL ${BUILD_TYPES_FOR_DLL})
+ if(EXISTS "${${packageName}_DIR}/bin/${OUTPUTCONFIG}/${BUILD_TYPE_FOR_DLL}")
+ list(APPEND packageDirs "${${packageName}_DIR}/bin/${OUTPUTCONFIG}/${BUILD_TYPE_FOR_DLL}")
+ endif()
+ endforeach()
+ endforeach()
+ if(CMAKE_BUILD_TYPE) ## for single generator (makefiles)
+ if(EXISTS "${${packageName}_DIR}/bin/${CMAKE_BUILD_TYPE}")
+ list(APPEND packageDirs "${${packageName}_DIR}/bin/${CMAKE_BUILD_TYPE}")
+ endif()
+ foreach(BUILD_TYPE_FOR_DLL ${BUILD_TYPES_FOR_DLL})
+ if(EXISTS "${${packageName}_DIR}/bin/${CMAKE_BUILD_TYPE}/${BUILD_TYPE_FOR_DLL}")
+ list(APPEND packageDirs "${${packageName}_DIR}/bin/${CMAKE_BUILD_TYPE}/${BUILD_TYPE_FOR_DLL}")
+ endif()
+ endforeach()
+ endif()
+ else()
+ set(${packageName}_DIR "$ENV{${packageName}_DIR}" CACHE PATH "${packageName}_DIR root directory for looking for dirs containning *.dll")
+ endif()
+ endforeach()
+ if(packageDirs)
+ list(REMOVE_DUPLICATES packageDirs)
+ endif()
+
+
+ set(dirsToLookFor "${EXEC_PATH}")
+ if(packageDirs)
+ list(APPEND dirsToLookFor ${packageDirs})
+ endif()
+ if(argDirs)
+ list(APPEND dirsToLookFor ${argDirs})
+ endif()
+ get_property(used_LINK_DIRECTORIES DIRECTORY PROPERTY LINK_DIRECTORIES)
+ if (used_LINK_DIRECTORIES)
+ list(APPEND dirsToLookFor ${used_LINK_DIRECTORIES})
+ list(REMOVE_DUPLICATES dirsToLookFor)
+ endif()
+
+
+ ## handle plugins
+ set(pluginsList "")
+ include(parse_arguments_multi) ## this function will process recursively items of the sub-list [default print messages]
+ function(parse_arguments_multi_function results)
+ cmake_parse_arguments(pamf "VERBOSE" "PLUGIN_PATH_DEST;FROM_REL_PATH;EXEC_PATH;COMPONENT" "" ${ARGN}) ## EXEC_PATH and COMPONENT are for exclusive internal use
+ list(REMOVE_DUPLICATES pamf_UNPARSED_ARGUMENTS)
+ foreach(PLUGIN_PATH_NAME ${pamf_UNPARSED_ARGUMENTS})
+ if(EXISTS ${PLUGIN_PATH_NAME})
+ if(IS_DIRECTORY ${PLUGIN_PATH_NAME})
+ if(pamf_VERBOSE)
+ message(WARNING "${PLUGIN_PATH_NAME} IS_DIRECTORY, cannot installed a directory, please give a path filename")
+ endif()
+ else()
+ if(NOT pamf_PLUGIN_PATH_DEST)
+ set(PLUGIN_PATH_DEST ${pamf_EXEC_PATH}) ## the default dest value
+ else()
+ set(PLUGIN_PATH_DEST ${pamf_PLUGIN_PATH_DEST})
+ endif()
+
+ if(pamf_FROM_REL_PATH)
+ file(TO_CMAKE_PATH ${PLUGIN_PATH_NAME} PLUGIN_PATH_NAME)
+ get_filename_component(PLUGIN_PATH ${PLUGIN_PATH_NAME} PATH)
+ unset(PLUGIN_PATH_LIST)
+ unset(PLUGIN_PATH_LIST_COUNT)
+ unset(PLUGIN_REL_PATH_LIST)
+ unset(PLUGIN_REL_PATH)
+ string(REPLACE "/" ";" PLUGIN_PATH_LIST ${PLUGIN_PATH}) ## create a list of dir
+ list(FIND PLUGIN_PATH_LIST ${pamf_FROM_REL_PATH} id)
+ list(LENGTH PLUGIN_PATH_LIST PLUGIN_PATH_LIST_COUNT)
+ if(${id} GREATER 0)
+ math(EXPR id "${id}+1") ## matches relative path not include
+ math(EXPR PLUGIN_PATH_LIST_COUNT "${PLUGIN_PATH_LIST_COUNT}-1") ## the end of the list
+ foreach(i RANGE ${id} ${PLUGIN_PATH_LIST_COUNT})
+ list(GET PLUGIN_PATH_LIST ${i} out)
+ list(APPEND PLUGIN_REL_PATH_LIST ${out})
+ endforeach()
+ foreach(dir ${PLUGIN_REL_PATH_LIST})
+ set(PLUGIN_REL_PATH "${PLUGIN_REL_PATH}/${dir}")
+ endforeach()
+ endif()
+ set(PLUGIN_PATH_DEST ${PLUGIN_PATH_DEST}${PLUGIN_REL_PATH})
+ endif()
+
+ install(FILES ${PLUGIN_PATH_NAME} CONFIGURATIONS ${inst_run_CONFIG_TYPE} DESTINATION ${PLUGIN_PATH_DEST} COMPONENT ${pamf_COMPONENT})
+ get_filename_component(pluginName ${PLUGIN_PATH_NAME} NAME)
+ if(IS_ABSOLUTE ${PLUGIN_PATH_DEST})
+ else()
+ set(PLUGIN_PATH_DEST ${inst_run_INSTALL_FOLDER}/${PLUGIN_PATH_DEST})
+ endif()
+ list(APPEND pluginsList ${PLUGIN_PATH_DEST}/${pluginName})
+ endif()
+ else()
+ message(WARNING "You need to provide a valid PLUGIN_PATH_NAME")
+ set(pluginsList )
+ endif()
+ endforeach()
+ set(${results} ${pluginsList} PARENT_SCOPE)
+ endfunction()
+
+ if(inst_run_VERBOSE)
+ list(APPEND extra_flags_to_add VERBOSE)
+ endif()
+ list(APPEND extra_flags_to_add EXEC_PATH ${EXEC_PATH} COMPONENT ${inst_run_COMPONENT}) ## for internal use inside overloaded function
+ list(LENGTH inst_run_PLUGINS inst_run_PLUGINS_count)
+ if(${inst_run_PLUGINS_count} GREATER 0)
+ parse_arguments_multi(PLUGIN_PATH_NAME inst_run_PLUGINS ${inst_run_PLUGINS} ## see internal overload parse_arguments_multi_function for processing each sub-list
+ NEED_RESULTS ${inst_run_PLUGINS_count} ## this is used to check when we are in the first loop (in order to reset parse_arguments_multi_results)
+ EXTRAS_FLAGS ${extra_flags_to_add} ## this is used to allow catching additional internal flags of our overloaded function
+ )
+ endif()
+
+ #message(parse_arguments_multi_results = ${parse_arguments_multi_results})
+ list(APPEND pluginsList ${parse_arguments_multi_results})
+
+
+
+ ## Install rules for required system runtimes such as MSVCRxx.dll
+ set(CMAKE_INSTALL_SYSTEM_RUNTIME_LIBS_SKIP ON)
+ include(InstallRequiredSystemLibraries)
+ if(CMAKE_INSTALL_SYSTEM_RUNTIME_LIBS)
+ install(FILES ${CMAKE_INSTALL_SYSTEM_RUNTIME_LIBS}
+ CONFIGURATIONS ${inst_run_CONFIG_TYPE}
+ DESTINATION ${EXEC_PATH}
+ COMPONENT ${inst_run_COMPONENT}
+ )
+ endif()
+
+ ## print what we are doing to do
+ if(inst_run_VERBOSE)
+ message(STATUS "[install_runtime] On install target call, cmake will try to resolve dependencies for given app:\n ${installedFilePathTargetAppToResolve} (with plausible postfix: ${inst_run_PLAUSIBLES_POSTFIX})")
+ if(pluginsList)
+ message(STATUS " and also for plugins :")
+ foreach(plugin ${pluginsList})
+ message(STATUS " ${plugin}")
+ endforeach()
+ endif()
+ message(STATUS " Looking for dependencies into:")
+ foreach(dir ${dirsToLookFor})
+ message(STATUS " ${dir}")
+ endforeach()
+ endif()
+
+ ## Install rules for required dependencies libs/plugins for the target app
+ ## will resolve all installed target files with config modes postfixes
+ string(TOUPPER ${inst_run_CONFIG_TYPE} inst_run_CONFIG_TYPE_UC)
+ get_target_property(postfix ${inst_run_TARGET} "${inst_run_CONFIG_TYPE_UC}_POSTFIX")
+ install(CODE "set(target \"${inst_run_TARGET}\")" COMPONENT ${inst_run_COMPONENT} CONFIGURATIONS ${CONFIG_TYPE})
+ install(CODE "set(inst_run_CONFIG_TYPE \"${inst_run_CONFIG_TYPE}\")" COMPONENT ${inst_run_COMPONENT} CONFIGURATIONS ${CONFIG_TYPE})
+ install(CODE "set(inst_run_INSTALL_FOLDER \"${inst_run_INSTALL_FOLDER}\")" COMPONENT ${inst_run_COMPONENT} CONFIGURATIONS ${CONFIG_TYPE})
+ install(CODE "set(app \"${EXEC_PATH}/${EXEC_NAME}${postfix}${CMAKE_EXECUTABLE_SUFFIX}\")" COMPONENT ${inst_run_COMPONENT} CONFIGURATIONS ${CONFIG_TYPE})
+ install(CODE "set(dirsToLookFor \"${dirsToLookFor}\")" COMPONENT ${inst_run_COMPONENT} CONFIGURATIONS ${CONFIG_TYPE})
+ install(CODE
+ [[
+ if("${CMAKE_INSTALL_CONFIG_NAME}" STREQUAL "${inst_run_CONFIG_TYPE}")
+ message(STATUS "Installing ${target} dependencies...")
+
+ file(GET_RUNTIME_DEPENDENCIES
+ EXECUTABLES ${app}
+ RESOLVED_DEPENDENCIES_VAR _r_deps
+ UNRESOLVED_DEPENDENCIES_VAR _u_deps
+ CONFLICTING_DEPENDENCIES_PREFIX _c_deps
+ DIRECTORIES ${dirsToLookFor}
+ PRE_EXCLUDE_REGEXES "api-ms-*"
+ POST_EXCLUDE_REGEXES ".*system32/.*\\.dll" ".*SysWOW64/.*\\.dll"
+ )
+
+ if(_u_deps)
+ message(WARNING "There were unresolved dependencies for executable ${EXEC_FILE}: \"${_u_deps}\"!")
+ endif()
+ if(_c_deps_FILENAMES)
+ message(WARNING "There were conflicting dependencies for executable ${EXEC_FILE}: \"${_c_deps_FILENAMES}\"!")
+ endif()
+
+ foreach(_file ${_r_deps})
+ file(INSTALL
+ DESTINATION "${inst_run_INSTALL_FOLDER}/bin"
+ TYPE SHARED_LIBRARY
+ FOLLOW_SYMLINK_CHAIN
+ FILES "${_file}"
+ )
+ endforeach()
+ endif()
+ ]]
+ COMPONENT ${inst_run_COMPONENT} CONFIGURATIONS ${CONFIG_TYPE}
+ )
+
+
+endfunction()
+
+## High level macro to install resources in the correct folder
+##
+## EXECUTABLE: [opt] option to copy files as programs
+## RELATIVE : [opt] copy files relatively to current folder
+## TYPE : [opt] type and folder where to store the files
+## FOLDER : [opt] subfolder to use
+## FILES : [opt] contains a list of resources files to copy to install folder
+macro(ibr_install_rsc target)
+ cmake_parse_arguments(install_rsc_${target} "EXECUTABLE;RELATIVE" "TYPE;FOLDER" "FILES" ${ARGN})
+ set(rsc_target "${target}_${install_rsc_${target}_TYPE}")
+
+ if(install_rsc_${target}_FOLDER)
+ set(rsc_folder "${install_rsc_${target}_TYPE}/${install_rsc_${target}_FOLDER}")
+ else()
+ set(rsc_folder "${install_rsc_${target}_TYPE}")
+ endif()
+
+ add_custom_target(${rsc_target}
+ COMMENT "run the ${install_rsc_${target}_TYPE} installation only for ${target} (component ${rsc_target})"
+ VERBATIM)
+ foreach(scriptFile ${install_rsc_${target}_FILES})
+ if(install_rsc_${target}_RELATIVE)
+ file(RELATIVE_PATH relativeFilename ${CMAKE_CURRENT_SOURCE_DIR} ${scriptFile})
+ else()
+ get_filename_component(relativeFilename ${scriptFile} NAME)
+ endif()
+
+ if(DEFINED CMAKE_BUILD_TYPE) ## for make/nmake based
+ add_custom_command(TARGET ${rsc_target} POST_BUILD
+ COMMAND ${CMAKE_COMMAND} -E
+ copy_if_different ${scriptFile} ${CMAKE_INSTALL_PREFIX_${CMAKE_BUILD_TYPE}}/${rsc_folder}/${relativeFilename})
+ endif()
+ foreach(CONFIG_TYPES ${CMAKE_CONFIGURATION_TYPES}) ## for multi config types (MSVC based)
+ string(TOUPPER ${CONFIG_TYPES} CONFIG_TYPES_UC)
+ add_custom_command(TARGET ${rsc_target} POST_BUILD
+ COMMAND ${CMAKE_COMMAND} -E
+ copy_if_different ${scriptFile} ${CMAKE_INSTALL_PREFIX_${CONFIG_TYPES_UC}}/${rsc_folder}/${relativeFilename})
+ endforeach()
+ endforeach()
+
+ get_target_property(INSTALL_RSC_BUILD_FOLDER ${target} FOLDER)
+ set_target_properties(${rsc_target} PROPERTIES FOLDER ${INSTALL_RSC_BUILD_FOLDER})
+
+ add_dependencies(${target} ${rsc_target})
+ add_dependencies(PREBUILD ${rsc_target})
+
+ if(DEFINED CMAKE_BUILD_TYPE) ## for make/nmake based
+ resourceFile(ADD ${rsc_folder} CONFIG_TYPE ${CMAKE_BUILD_TYPE} FILE_PATH "${CMAKE_INSTALL_PREFIX_${CMAKE_BUILD_TYPE}}/ibr_resources.ini")
+
+ if(install_rsc_${target}_EXECUTABLE)
+ install(
+ PROGRAMS ${install_rsc_${target}_FILES}
+ CONFIGURATIONS ${CMAKE_BUILD_TYPE}
+ DESTINATION "${CMAKE_INSTALL_PREFIX_${CMAKE_BUILD_TYPE}}/${rsc_folder}"
+ )
+ else()
+ install(
+ FILES ${install_rsc_${target}_FILES}
+ CONFIGURATIONS ${CMAKE_BUILD_TYPE}
+ DESTINATION "${CMAKE_INSTALL_PREFIX_${CMAKE_BUILD_TYPE}}/${rsc_folder}"
+ )
+ endif()
+ endif()
+ foreach(CONFIG_TYPES ${CMAKE_CONFIGURATION_TYPES}) ## for multi config types (MSVC based)
+ string(TOUPPER ${CONFIG_TYPES} CONFIG_TYPES_UC)
+ resourceFile(ADD ${rsc_folder} CONFIG_TYPE ${CONFIG_TYPES} FILE_PATH "${CMAKE_INSTALL_PREFIX_${CONFIG_TYPES_UC}}/ibr_resources.ini")
+
+ if(install_rsc_${target}_EXECUTABLE)
+ install(
+ PROGRAMS ${install_rsc_${target}_FILES}
+ CONFIGURATIONS ${CONFIG_TYPES}
+ DESTINATION "${CMAKE_INSTALL_PREFIX_${CONFIG_TYPES_UC}}/${rsc_folder}"
+ )
+ else()
+ install(
+ FILES ${install_rsc_${target}_FILES}
+ CONFIGURATIONS ${CONFIG_TYPES}
+ DESTINATION "${CMAKE_INSTALL_PREFIX_${CONFIG_TYPES_UC}}/${rsc_folder}"
+ )
+ endif()
+ endforeach()
+endmacro()
+
+
+## High level macro to install in an homogen way all our ibr targets (it use some functions inside this file)
+##
+## RSC_FILE_ADD : [opt] is used to auto write/append relative paths of target resources into a common file
+## INSTALL_PDB : [opt] is used to auto install PDB file (when using MSVC according to the target type)
+## STANDALONE : [opt] bool ON/OFF var to call install_runtime or not (for bundle resolution)
+## DIRS : [opt] used if STANDALONE set to ON, see install_runtime doc
+## PLUGINS: [opt] used if STANDALONE set to ON, see install_runtime doc
+## MSVC_CMD : [opt] used to specify an absolute filePathName application to launch with the MSVC IDE Debugger associated to this target (project file)
+## MSVC_ARGS : [opt] load the MSVC debugger with correct settings (app path, args, working dir)
+##
+macro(ibr_install_target target)
+ cmake_parse_arguments(ibrInst${target} "VERBOSE;INSTALL_PDB" "COMPONENT;MSVC_ARGS;STANDALONE;RSC_FOLDER" "SHADERS;RESOURCES;SCRIPTS;DIRS;PLUGINS" ${ARGN})
+
+ if(ibrInst${target}_RSC_FOLDER)
+ set(rsc_folder "${ibrInst${target}_RSC_FOLDER}")
+ else()
+ set(rsc_folder "${target}")
+ endif()
+
+ if(ibrInst${target}_SHADERS)
+ ibr_install_rsc(${target} EXECUTABLE TYPE "shaders" FOLDER ${rsc_folder} FILES "${ibrInst${target}_SHADERS}")
+ endif()
+
+ if(ibrInst${target}_RESOURCES)
+ ibr_install_rsc(${target} TYPE "resources" FOLDER ${rsc_folder} FILES "${ibrInst${target}_RESOURCES}")
+ endif()
+
+ if(ibrInst${target}_SCRIPTS)
+ ibr_install_rsc(${target} EXECUTABLE TYPE "scripts" FOLDER ${rsc_folder} FILES "${ibrInst${target}_SCRIPTS}")
+ endif()
+
+ if(ibrInst${target}_COMPONENT)
+ set(installCompArg COMPONENT ${ibrInst${target}_COMPONENT})
+ ## Create a custom install target based on COMPONENT
+ installTargetProject(${target} ${ibrInst${target}_COMPONENT})
+ endif()
+
+ if(DEFINED CMAKE_BUILD_TYPE) ## for make/nmake based
+ string(TOUPPER ${CMAKE_BUILD_TYPE} CONFIG_TYPES_UC)
+ set_target_properties(${target} PROPERTIES ${CONFIG_TYPES_UC}_POSTFIX "${CMAKE_${CONFIG_TYPES_UC}_POSTFIX}")
+ get_target_property(CURRENT_TARGET_BUILD_TYPE_POSTFIX ${target} ${CONFIG_TYPES_UC}_POSTFIX)
+ endif()
+ foreach(CONFIG_TYPES ${CMAKE_CONFIGURATION_TYPES}) ## for multi config types (MSVC based)
+ string(TOUPPER ${CONFIG_TYPES} CONFIG_TYPES_UC)
+ set_target_properties(${target} PROPERTIES ${CONFIG_TYPES_UC}_POSTFIX "${CMAKE_${CONFIG_TYPES_UC}_POSTFIX}")
+ get_target_property(CURRENT_TARGET_BUILD_TYPE_POSTFIX ${target} ${CONFIG_TYPES_UC}_POSTFIX)
+ endforeach()
+
+ ## Specify default installation rules
+ if(DEFINED CMAKE_BUILD_TYPE) ## for make/nmake based
+ string(TOUPPER ${CMAKE_BUILD_TYPE} CONFIG_TYPES_UC)
+ install(TARGETS ${target}
+ CONFIGURATIONS ${CMAKE_BUILD_TYPE}
+ LIBRARY DESTINATION ${CMAKE_LIBRARY_OUTPUT_DIRECTORY_${CMAKE_BUILD_TYPE}} ${installCompArg}
+ ARCHIVE DESTINATION ${CMAKE_ARCHIVE_OUTPUT_DIRECTORY_${CMAKE_BUILD_TYPE}} ${installCompArg}
+ RUNTIME DESTINATION ${CMAKE_RUNTIME_OUTPUT_DIRECTORY_${CMAKE_BUILD_TYPE}} ${installCompArg}
+ )
+ endif()
+ foreach(CONFIG_TYPES ${CMAKE_CONFIGURATION_TYPES}) ## for multi config types (MSVC based)
+ string(TOUPPER ${CONFIG_TYPES} CONFIG_TYPES_UC)
+ install(TARGETS ${target}
+ CONFIGURATIONS ${CONFIG_TYPES}
+ LIBRARY DESTINATION ${CMAKE_LIBRARY_OUTPUT_DIRECTORY_${CONFIG_TYPES_UC}} ${installCompArg}
+ ARCHIVE DESTINATION ${CMAKE_ARCHIVE_OUTPUT_DIRECTORY_${CONFIG_TYPES_UC}} ${installCompArg}
+ RUNTIME DESTINATION ${CMAKE_RUNTIME_OUTPUT_DIRECTORY_${CONFIG_TYPES_UC}} ${installCompArg}
+ )
+ endforeach()
+
+ if(ibrInst${target}_INSTALL_PDB)
+ if(DEFINED CMAKE_BUILD_TYPE) ## for make/nmake based
+ installPDB(${target} ${CMAKE_BUILD_TYPE}
+ LIBRARY_DEST ${CMAKE_LIBRARY_OUTPUT_DIRECTORY_${CMAKE_BUILD_TYPE}}
+ ARCHIVE_DEST ${CMAKE_ARCHIVE_OUTPUT_DIRECTORY_${CMAKE_BUILD_TYPE}}
+ RUNTIME_DEST ${CMAKE_RUNTIME_OUTPUT_DIRECTORY_${CMAKE_BUILD_TYPE}}
+ )
+ endif()
+ foreach(CONFIG_TYPES ${CMAKE_CONFIGURATION_TYPES}) ## for multi config types (MSVC based)
+ string(TOUPPER ${CONFIG_TYPES} CONFIG_TYPES_UC)
+ installPDB(${target} ${CONFIG_TYPES}
+ LIBRARY_DEST ${CMAKE_LIBRARY_OUTPUT_DIRECTORY_${CONFIG_TYPES_UC}}
+ ARCHIVE_DEST ${CMAKE_ARCHIVE_OUTPUT_DIRECTORY_${CONFIG_TYPES_UC}}
+ RUNTIME_DEST ${CMAKE_RUNTIME_OUTPUT_DIRECTORY_${CONFIG_TYPES_UC}}
+ )
+ endforeach()
+ endif()
+
+ ## install dynamic necessary dependencies
+ if(ibrInst${target}_STANDALONE)
+ get_target_property(type ${target} TYPE)
+ if(${type} MATCHES "EXECUTABLE")
+
+ if(ibrInst${target}_VERBOSE)
+ set(VERBOSE VERBOSE)
+ else()
+ set(VERBOSE )
+ endif()
+
+ if(DEFINED CMAKE_BUILD_TYPE) ## for make/nmake based
+ install_runtime(bin/${target}${CMAKE_EXECUTABLE_SUFFIX} ## default relative to CMAKE_INSTALL_PREFIX
+ INSTALL_FOLDER "${CMAKE_INSTALL_PREFIX_${CMAKE_BUILD_TYPE}}"
+ CONFIG_TYPE ${CMAKE_BUILD_TYPE}
+ ${VERBOSE}
+ TARGET ${target}
+ ${installCompArg}
+ PLUGINS ## will be installed
+ ${ibrInst${target}_PLUGINS}
+ DIRS ${CMAKE_RUNTIME_OUTPUT_DIRECTORY_${CONFIG_TYPES_UC}}
+ ${ibrInst${target}_DIRS}
+ )
+ endif()
+ foreach(CONFIG_TYPES ${CMAKE_CONFIGURATION_TYPES}) ## for multi config types (MSVC based)
+ string(TOUPPER ${CONFIG_TYPES} CONFIG_TYPES_UC)
+ install_runtime(bin/${target}${CMAKE_EXECUTABLE_SUFFIX} ## default relative to CMAKE_INSTALL_PREFIX
+ INSTALL_FOLDER "${CMAKE_INSTALL_PREFIX_${CONFIG_TYPES_UC}}"
+ CONFIG_TYPE ${CONFIG_TYPES}
+ ${VERBOSE}
+ TARGET ${target}
+ ${installCompArg}
+ PLUGINS ## will be installed
+ ${ibrInst${target}_PLUGINS}
+ DIRS ${CMAKE_RUNTIME_OUTPUT_DIRECTORY_${CONFIG_TYPES_UC}}
+ ${ibrInst${target}_DIRS}
+ )
+ endforeach()
+ else()
+ message(WARNING "STANDALONE option is only compatible with EXECUTABLES target type. Skip the STANDALONE installation process.")
+ endif()
+ endif()
+
+ ## Provide a way to directly load the MSVC debugger with correct settings
+ if(MSVC)
+ if(ibrInst${target}_MSVC_CMD) ## command absolute filePathName is optional as the default is to use the installed target file application
+ set(msvcCmdArg COMMAND ${ibrInst${target}_MSVC_CMD}) ## flag following by the value (both to pass to the MSVCsetUserCommand function)
+ endif()
+ if(ibrInst${target}_MSVC_ARGS) ## args (between quotes) are optional
+ set(msvcArgsArg ARGS ${ibrInst${target}_MSVC_ARGS}) ## flag following by the value (both to pass to the MSVCsetUserCommand function)
+ endif()
+ get_target_property(type ${target} TYPE)
+ if( (ibrInst${target}_MSVC_CMD OR ibrInst${target}_MSVC_ARGS) OR (${type} MATCHES "EXECUTABLE") )
+ include(MSVCsetUserCommand)
+ if(DEFINED CMAKE_BUILD_TYPE) ## for make/nmake based
+ MSVCsetUserCommand( ${target}
+ PATH ${CMAKE_OUTPUT_BIN_${CMAKE_BUILD_TYPE}} ##FILE option not necessary since it deduced from targetName
+ ARGS "${SIBR_PROGRAMARGS}"
+ ${msvcCmdArg}
+ #${msvcArgsArg}
+ WORKING_DIR ${CMAKE_OUTPUT_BIN_${CMAKE_BUILD_TYPE}}
+ )
+ endif()
+ foreach(CONFIG_TYPES ${CMAKE_CONFIGURATION_TYPES}) ## for multi config types (MSVC based)
+ string(TOUPPER ${CONFIG_TYPES} CONFIG_TYPES_UC)
+ MSVCsetUserCommand( ${target}
+ PATH ${CMAKE_OUTPUT_BIN_${CONFIG_TYPES_UC}} ##FILE option not necessary since it deduced from targetName
+ ARGS "${SIBR_PROGRAMARGS}"
+ ${msvcCmdArg}
+ #${msvcArgsArg}
+ WORKING_DIR ${CMAKE_OUTPUT_BIN_${CONFIG_TYPES_UC}}
+ )
+ endforeach()
+ elseif(NOT ${type} MATCHES "EXECUTABLE")
+ #message("Cannot set MSVCsetUserCommand with target ${target} without COMMAND parameter as it is not an executable (skip it)")
+ endif()
+ endif()
+
+endmacro()
diff --git a/submodules/gaussian-splatting/SIBR_viewers/cmake/windows/parse_arguments_multi.cmake b/submodules/gaussian-splatting/SIBR_viewers/cmake/windows/parse_arguments_multi.cmake
new file mode 100644
index 0000000000000000000000000000000000000000..4f19e414fd3dc12a6af222f2471eec7410f617ee
--- /dev/null
+++ b/submodules/gaussian-splatting/SIBR_viewers/cmake/windows/parse_arguments_multi.cmake
@@ -0,0 +1,304 @@
+# Copyright (C) 2020, Inria
+# GRAPHDECO research group, https://team.inria.fr/graphdeco
+# All rights reserved.
+#
+# This software is free for non-commercial, research and evaluation use
+# under the terms of the LICENSE.md file.
+#
+# For inquiries contact sibr@inria.fr and/or George.Drettakis@inria.fr
+
+
+if(NOT WIN32 OR __parse_arguments_multi_cmake_INCLUDED__)
+ return()
+else()
+ set(__parse_arguments_multi_cmake_INCLUDED__ ON)
+endif()
+
+## This macro allow to process repeating multi value args from a given function which use cmake_parse_arguments module.
+##
+## cmake_parse_arguments multi args standard behavior:
+## function(foo)
+## cmake_parse_arguments(arg "" "" "MULTI" ${ARGN})
+## foreach(item IN LISTS arg_MULTI)
+## message(STATUS "${item}")
+## endforeach()
+## endfunction()
+## foo(MULTI x y MULTI z w)
+## The above code outputs 'z' and 'w'. It originally expected it to output all of 'x' 'y' 'z' 'w'.
+##
+## Using this macro inside a function which want to handle repeating multi args values
+## will recursively iterate onto the multi tags list to process each sub list.
+## It take as 1st argument the subTag flag to separate sub list from the main multi list.
+## It take as 2nd argument the nameList of the main multi list (the multiValuesArgs from cmake_parse_arguments: here it is MULTI in the example)
+## and that's why it is important that it should be a macro and not a function (to get access to external variable).
+## Then you give the content of this list allowing to be processed by the macro.
+##
+## parse_arguments_multi macro call a parse_arguments_multi_function which do actually the process from the given sub-list.
+## By default this function only print infos about what variables you are trying to pass/process (only verbose messages),
+## but, by overloading this cmake function, you will be able to externalize the process of your multi argument list.
+##
+## Usage (into a function) :
+## parse_arguments_multi(
+## [NEED_RESULTS ] [EXTRAS_FLAGS <...> <...> ...]
+## )
+##
+## Simple usage example [user point of view]:
+## foo(MULTI
+## SUB_MULTI x y
+## SUB_MULTI z w
+## )
+##
+## Simple usage example [inside a function]:
+## function(foo)
+## cmake_parse_arguments(arg "" "" "MULTI" ${ARGN})
+## include(parse_arguments_multi)
+## function(parse_arguments_multi_function )
+## #message("I'm an overloaded cmake function used by parse_arguments_multi")
+## #message("I'm processing first part of my sub list: ${ARGN}")
+## message("ARGV0=${ARGV0}")
+## message("ARGV1=${ARGV1}")
+## endfunction()
+## parse_arguments_multi(SUB_MULTI arg_MULTI ${arg_MULTI}) ## this function will process recusively items of the sub-list [default print messages]
+## endfunction()
+##
+## Will print:
+## ARGV0=z
+## ARGV1=w
+## ARGV0=x
+## ARGV1=y
+##
+## WARNING : DO NEVER ADD EXTRA THINGS TO parse_arguments_multi MACRO :
+## parse_arguments_multi(SUB_MULTI arg_MULTI ${arg_MULTI} EXTRAS foo bar SOMTHING) => will failed !!
+## use EXTRAS_FLAGS instead !!
+##
+## Advanced usage example [user point of view]:
+## bar(C:/prout/test.exe VERBOSE
+## PLUGINS
+## PLUGIN_PATH_NAME x PLUGIN_PATH_DEST w
+## PLUGIN_PATH_NAME a b PLUGIN_PATH_DEST y
+## PLUGIN_PATH_NAME c
+## )
+##
+## Advanced usage example [inside a function]:
+## function(bar execFilePathName)
+## cmake_parse_arguments(arg "VERBOSE" "" "PLUGINS" ${ARGN})
+##
+## include(parse_arguments_multi)
+## function(parse_arguments_multi_function results)
+## cmake_parse_arguments(pamf "VERBOSE" "PLUGIN_PATH_DEST;EXEC_PATH" "" ${ARGN}) ## EXEC_PATH is for internal use
+## message("")
+## message("I'm an overloaded cmake function used by parse_arguments_multi from install_runtime function")
+## message("I'm processing first part of my sub list: ${ARGN}")
+## message("PLUGIN_PATH_NAME = ${pamf_UNPARSED_ARGUMENTS}")
+## message(pamf_VERBOSE = ${pamf_VERBOSE})
+## message("pamf_PLUGIN_PATH_DEST = ${pamf_PLUGIN_PATH_DEST}")
+## message(pamf_EXEC_PATH = ${pamf_EXEC_PATH})
+## if(NOT ${pamf_PLUGIN_PATH_DEST})
+## set(pamf_PLUGIN_PATH_DEST ${pamf_EXEC_PATH})
+## endif()
+## foreach(plugin ${pamf_UNPARSED_ARGUMENTS})
+## get_filename_component(pluginName ${plugin} NAME)
+## list(APPEND pluginsList ${pamf_PLUGIN_PATH_DEST}/${pluginName})
+## endforeach()
+## set(${results} ${pluginsList} PARENT_SCOPE)
+## endfunction()
+##
+## if(arg_VERBOSE)
+## list(APPEND extra_flags_to_add VERBOSE) ## here we transmit the VERNOSE flag
+## endif()
+## get_filename_component(EXEC_PATH ${execFilePathName} PATH) ## will be the default value if PLUGIN_PATH_DEST option is not provided
+## list(APPEND extra_flags_to_add EXEC_PATH ${EXEC_PATH})
+## list(LENGTH arg_PLUGINS arg_PLUGINS_count)
+## parse_arguments_multi(PLUGIN_PATH_NAME arg_PLUGINS ${arg_PLUGINS}
+## NEED_RESULTS ${arg_PLUGINS_count} ## this is used to check when we are in the first loop (in order to reset parse_arguments_multi_results)
+## EXTRAS_FLAGS ${extra_flags_to_add} ## this is used to allow catching VERBOSE and PLUGIN_PATH_DEST flags of our overloaded function
+## )
+## endfunction()
+## message(parse_arguments_multi_results = ${parse_arguments_multi_results}) ## list of the whole pluginsList
+## #Will print w/x;a/y;b/y;C:/prout/c
+##
+## NOTE that here, since our overloaded function need to provide a result list, we use the other parse_arguments_multi_function signature (the which one with a results arg)
+##
+
+function(parse_arguments_multi_function_default) ## used in case of you want to reset the default behavior of this function process
+ message("[default function] parse_arguments_multi_function(ARGC=${ARGC} ARGV=${ARGV} ARGN=${ARGN})")
+ message("This function is used by parse_arguments_multi and have to be overloaded to process sub list of multi values args")
+endfunction()
+
+function(parse_arguments_multi_function ) ## => the function to overload
+ parse_arguments_multi_function_default(${ARGN})
+endfunction()
+
+## first default signature above
+##------------------------------
+## second results signature behind
+
+function(parse_arguments_multi_function_default result) ## used in case of you want to reset the default behavior of this function process
+ message("[default function] parse_arguments_multi_function(ARGC=${ARGC} ARGV=${ARGV} ARGN=${ARGN})")
+ message("This function is used by parse_arguments_multi and have to be overloaded to process sub list of muluti values args")
+endfunction()
+
+function(parse_arguments_multi_function result) ## => the function to overload
+ parse_arguments_multi_function_default(result ${ARGN})
+endfunction()
+
+## => the macro to use inside your function which use cmake_parse_arguments
+# NOTE: entry point of parse_arguments_multi, which is called from win3rdPart)
+macro(parse_arguments_multi multiArgsSubTag multiArgsList #<${multiArgsList}> the content of the list
+)
+ # message (STATUS "")
+ # message(STATUS "calling parse_arguemnts_multi defined in parse_arguments_multi.cmake:141")
+ # message(STATUS "multiArgsSubTag = ${multiArgsSubTag}") # CHECK_CACHED_VAR
+ # message(STATUS "multiArgsList = ${multiArgsList}") # it contains the name of the variable which is holding the list i.e w3p_MULTI_SET
+ # message(STATUS "value of ${multiArgsList} = ${${multiArgsList}}") # a semicolon separated list of values passed to SET or MULTISET keyword in win3rdParty
+ # message(STATUS "actual values ARGN = ${ARGN}") # the same as ${${multiArgsList}}
+
+ ## INFO
+ ## starting from CMake 3.5 cmake_parse_arguments is not a module anymore and now is a native CMake command.
+ ## the behaviour is different though
+ ## In CMake 3.4, if you pass multiple times a multi_value_keyword, CMake returns the values of the LAST match
+ ## In CMake 3.5 and above, CMake returns the whole list of values that were following that multi_value_keyword
+ ## example:
+ ## cmake_parse_arguments(
+ ##
+ ## "" # options
+ ## "" # one value keywords
+ ## "MY_MULTI_VALUE_TAG"
+ ## MY_MULTI_VALUE_TAG value1 value2
+ ## MY_MULTI_VALUE_TAG value3 value4
+ ## MY_MULTI_VALUE_TAG value5 value6
+ ## )
+ ## result in CMake 3.4
+ ## _MY_MULTI_VALUE_TAG = "value5;value6"
+ ##
+ ## result in CMake 3.8
+ ## _MY_MULTI_VALUE_TAG = "value5;value6"
+
+ #include(CMakeParseArguments) #module CMakeParseArguments is obsolete since cmake 3.5
+ # cmake_parse_arguments ( args)
+ # : options (flags) pass to the macro
+ # : options that neeed a value
+ # : options that neeed more than one value
+ cmake_parse_arguments(_pam "" "NEED_RESULTS" "${multiArgsSubTag};EXTRAS_FLAGS" ${ARGN})
+
+ ## multiArgsList is the name of the list used by the multiValuesOption flag from the cmake_parse_arguments of the user function
+ ## that's why we absolutly need to use MACRO here (and also for passing parse_arguments_multi_results when NEED_RESULTS flag is set)
+
+ ## for debugging
+ #message("")
+ #message("[parse_arguments_multi] => ARGN = ${ARGN}")
+ #message("_pam_NEED_RESULTS=${_pam_NEED_RESULTS}")
+ #message("_pam_EXTRAS_FLAGS=${_pam_EXTRAS_FLAGS}")
+ # foreach(var ${_pam_${multiArgsSubTag}})
+ # message("arg=${var}")
+ # endforeach()
+
+ if (${CMAKE_VERSION} VERSION_GREATER "3.5")
+ # lets make ${_pam_${multiArgsSubTag}} behave as it is in version 3.4
+ # that means, cmake_parse_arguments should have only the last values of a multi set for a given keyword
+
+ # message("")
+ # message("values in multiArgsList")
+ # foreach(val ${${multiArgsList}})
+ # message(STATUS ${val})
+ # endforeach()
+ # message("end values in multiArgsList")
+
+
+ set(lastIndexFound OFF)
+ list(LENGTH ${multiArgsList} argnLength)
+ # message(${argnLength})
+ math(EXPR argnLength "${argnLength}-1") # make last index a valid one
+ set(recordIndex 0)
+ set(records "") # clear records list
+ set(record0 "") # clear first record list
+ foreach(iter RANGE ${argnLength})
+ list(GET ${multiArgsList} ${iter} value)
+ # message(STATUS "index=${iter} value=${value}")
+ if (${value} STREQUAL ${multiArgsSubTag})
+ if (lastIndexFound)
+ list(APPEND records ${recordIndex}) # records store the list NAMES
+ math(EXPR recordIndex "${recordIndex}+1")
+ set(record${recordIndex} "") # clear record list
+ else ()
+ set(lastIndexFound ON)
+ endif()
+
+ set(lastIndex ${iter})
+ else ()
+ if (lastIndexFound)
+ # message(${value})
+ list(APPEND record${recordIndex} ${value})
+ endif()
+ endif()
+ endforeach()
+
+ # save the last list of values
+ if (lastIndexFound)
+ list(APPEND records ${recordIndex}) # records store the list NAMES
+ endif()
+
+ # set multiArgsList to make it behave like CMake 3.4
+ # message("")
+ # message("using my records")
+ foreach(recordName ${records})
+ # message(${recordName})
+ # foreach(value ${record${recordName}})
+ # message(${value})
+ # endforeach()
+ # message("")
+ set(_pam_${multiArgsSubTag} ${record${recordName}})
+ endforeach()
+ # message(${_pam_${multiArgsSubTag}})
+
+ # message("")
+ # message("using argn")
+ # foreach(value ${ARGN})
+ # message(${value})
+ # endforeach()
+ endif() # end if cmake > 3.5
+
+ # message("values with pam ${_pam_${multiArgsSubTag}}")
+
+ ## check and init
+ list(LENGTH ${multiArgsList} globalListCount) # GLUT_TRACE: globalListCound=16 in CMake3.4 and CMake3.8
+ # message(STATUS "nr items in multiArgsList: ${globalListCount}")
+ math(EXPR globalListCount "${globalListCount}-1") ## because it will contain [multiArgsSubTag + ${multiArgsList}]
+ if(_pam_NEED_RESULTS)
+ if(${globalListCount} EQUAL ${_pam_NEED_RESULTS})
+ ## first time we enter into this macro (because we call it recursively)
+ unset(parse_arguments_multi_results)
+ endif()
+ endif()
+
+ ## process the part of the multi agrs list
+ ## ${ARGN} shouldn't be passed to the function in order to avoid missmatch size list ${multiArgsList} and _pam_${multiArgsSubTag}
+ ## if you want to pass extra internal flags from your function to this callback, use EXTRAS_FLAGS
+ if(_pam_NEED_RESULTS)
+ parse_arguments_multi_function(parse_arguments_multi_function_result ${_pam_${multiArgsSubTag}} ${_pam_EXTRAS_FLAGS})
+ list(APPEND parse_arguments_multi_results ${parse_arguments_multi_function_result})
+ else()
+ # message(STATUS "about to call parse_arguments_multi_function in parse_arguments_multi.cmake:177 ${_pam_${multiArgsSubTag}} and extra flags ${_pam_EXTRAS_FLAGS}")
+ parse_arguments_multi_function(${_pam_${multiArgsSubTag}} ${_pam_EXTRAS_FLAGS})
+ endif()
+
+ ## remove just processed items from the main list to process (multiArgsList)
+ list(REVERSE ${multiArgsList})
+ list(LENGTH _pam_${multiArgsSubTag} subTagListCount)
+ unset(ids)
+ foreach(id RANGE ${subTagListCount})
+ list(APPEND ids ${id})
+ endforeach()
+ list(REMOVE_AT ${multiArgsList} ${ids})
+ list(REVERSE ${multiArgsList})
+
+ ## test if remain sub multi list to process (recursive call) or finish the process
+ list(LENGTH ${multiArgsList} mainTagListCount)
+ if(${mainTagListCount} GREATER 1)
+ ## do not pass ${ARGN} just because it will re pass the initial 2 inputs args and we wont as they was consumed (in order to avoir conflicts)
+ # message(STATUS "about to call a parse_arguments_multi but without knowing where the definition is going to be taken from")
+ parse_arguments_multi(${multiArgsSubTag} ${multiArgsList} ${${multiArgsList}}
+ NEED_RESULTS ${_pam_NEED_RESULTS} EXTRAS_FLAGS ${_pam_EXTRAS_FLAGS}
+ )
+ endif()
+endmacro()
diff --git a/submodules/gaussian-splatting/SIBR_viewers/cmake/windows/sibr_library.cmake b/submodules/gaussian-splatting/SIBR_viewers/cmake/windows/sibr_library.cmake
new file mode 100644
index 0000000000000000000000000000000000000000..61f4219241ca249645e47604640fb2bb99c50668
--- /dev/null
+++ b/submodules/gaussian-splatting/SIBR_viewers/cmake/windows/sibr_library.cmake
@@ -0,0 +1,169 @@
+# Copyright (C) 2020, Inria
+# GRAPHDECO research group, https://team.inria.fr/graphdeco
+# All rights reserved.
+#
+# This software is free for non-commercial, research and evaluation use
+# under the terms of the LICENSE.md file.
+#
+# For inquiries contact sibr@inria.fr and/or George.Drettakis@inria.fr
+
+
+# NOTE
+# This feature is used to easily download, store and link external dependencies. This
+# requires to prepare pre-compiled libraries (to download). For now, packages have
+# only be prepare for Windows 64-bit with Visual Studio 2012. (You should re-build
+# everything if you want to use another version of Visual Studio/ another compiler).
+
+# NOTE ABOUT UNIX SYSTEMS
+# There is no need for "searching mechanism". This function is discard and your
+# libraries should be installed is the standard folders that are:
+#
+# /usr/include/
+# /usr/lib/
+# /usr/lib64/
+# for packages downloaded using apt-get/yum
+#
+# /usr/local/include/
+# /usr/local/lib/
+# /usr/local/lib64/
+# for packages manually installed ("make install")
+#
+# if you encounter problems when linking (e.g. lib not found even if it is installed),
+# please check these folders are in your search PATH environment variables.
+
+set(EXTLIBS_PACKAGE_FOLDER "${CMAKE_SOURCE_DIR}/extlibs")
+
+function(sibr_addlibrary)
+ if(NOT WIN32)
+ return()
+ endif()
+
+ file(MAKE_DIRECTORY ${EXTLIBS_PACKAGE_FOLDER})
+ cmake_parse_arguments(args "VCID" "VERBOSE;TIMEOUT;DEFAULT_USE;NAME;VERSION;MSVC11;MSVC12;MSVC14;MSVC17" "MULTI_SET;SET" ${ARGN})
+
+
+ if (NOT "${args_VERSION}" MATCHES "")
+ message(WARNING "VERSION is not implemented yet")
+ endif()
+
+ set(lcname "")
+ set(ucname "")
+ string(TOLOWER "${args_NAME}" lcname)
+ string(TOUPPER "${args_NAME}" ucname)
+
+ set(LIB_PACKAGE_FOLDER "${EXTLIBS_PACKAGE_FOLDER}/${lcname}")
+ win3rdParty(${ucname}
+ $
+ VERBOSE ${args_VERBOSE}
+ TIMEOUT ${args_TIMEOUT}
+ DEFAULT_USE ${args_DEFAULT_USE}
+ MSVC11 "${LIB_PACKAGE_FOLDER}" "${args_MSVC11}"
+ MSVC12 "${LIB_PACKAGE_FOLDER}" "${args_MSVC12}"
+ MSVC14 "${LIB_PACKAGE_FOLDER}" "${args_MSVC14}" # TODO SV: make sure to build this library if required
+ MSVC17 "${LIB_PACKAGE_FOLDER}" "${args_MSVC17}"
+ SET ${args_SET}
+ MULTI_SET ${args_MULTI_SET}
+ )
+
+ # Add include/ directory
+ # and lib/ directories
+
+ # TODO SV: paths not matching with current hierarchy. example: libraw/libraw-0.17.1/include
+ # SR: The link directories will also be used to lookup for dependency DLLs to copy in the install directory.
+ # Some libraries put the DLLs in the bin/ directory, so we include those.
+ file(GLOB subdirs RELATIVE ${LIB_PACKAGE_FOLDER} ${LIB_PACKAGE_FOLDER}/*)
+ set(dirlist "")
+ foreach(dir ${subdirs})
+ if(IS_DIRECTORY ${LIB_PACKAGE_FOLDER}/${dir})
+ # message("adding ${LIB_PACKAGE_FOLDER}/${dir}/include/ to the include directories")
+ include_directories("${LIB_PACKAGE_FOLDER}/${dir}/include/")
+ # message("adding ${LIB_PACKAGE_FOLDER}/${dir}/lib[64] to the link directories")
+ link_directories("${LIB_PACKAGE_FOLDER}/${dir}/")
+ link_directories("${LIB_PACKAGE_FOLDER}/${dir}/lib/")
+ link_directories("${LIB_PACKAGE_FOLDER}/${dir}/lib64/")
+ link_directories("${LIB_PACKAGE_FOLDER}/${dir}/bin/")
+ endif()
+ endforeach()
+
+endfunction()
+
+include(FetchContent)
+include(git_describe)
+
+function(sibr_gitlibrary)
+ cmake_parse_arguments(args "" "TARGET;GIT_REPOSITORY;GIT_TAG;ROOT_DIR;SOURCE_DIR" "INCLUDE_DIRS" ${ARGN})
+ if(NOT args_TARGET)
+ message(FATAL "Error on sibr_gitlibrary : please define your target name.")
+ return()
+ endif()
+
+ if(NOT args_ROOT_DIR)
+ set(args_ROOT_DIR ${args_TARGET})
+ endif()
+
+ if(NOT args_SOURCE_DIR)
+ set(args_SOURCE_DIR ${args_TARGET})
+ endif()
+
+ if(args_GIT_REPOSITORY AND args_GIT_TAG)
+ if(EXISTS ${CMAKE_SOURCE_DIR}/extlibs/${args_ROOT_DIR}/${args_SOURCE_DIR}/.git)
+ git_describe(
+ PATH ${CMAKE_SOURCE_DIR}/extlibs/${args_ROOT_DIR}/${args_SOURCE_DIR}
+ GIT_URL SIBR_GITLIBRARY_URL
+ GIT_BRANCH SIBR_GITLIBRARY_BRANCH
+ GIT_COMMIT_HASH SIBR_GITLIBRARY_COMMIT_HASH
+ GIT_TAG SIBR_GITLIBRARY_TAG
+ )
+
+ if((SIBR_GITLIBRARY_URL STREQUAL args_GIT_REPOSITORY) AND
+ ((SIBR_GITLIBRARY_BRANCH STREQUAL args_GIT_TAG) OR
+ (SIBR_GITLIBRARY_TAG STREQUAL args_GIT_TAG) OR
+ (SIBR_GITLIBRARY_COMMIT_HASH STREQUAL args_GIT_TAG)))
+ message(STATUS "Library ${args_TARGET} already available, skipping.")
+ set(SIBR_GITLIBRARY_DECLARED ON)
+ else()
+ message(STATUS "Adding library ${args_TARGET} from git...")
+ endif()
+ endif()
+
+ FetchContent_Declare(${args_TARGET}
+ GIT_REPOSITORY ${args_GIT_REPOSITORY}
+ GIT_TAG ${args_GIT_TAG}
+ GIT_SHALLOW ON
+ SOURCE_DIR ${CMAKE_SOURCE_DIR}/extlibs/${args_ROOT_DIR}/${args_SOURCE_DIR}
+ SUBBUILD_DIR ${CMAKE_SOURCE_DIR}/extlibs/${args_ROOT_DIR}/subbuild
+ BINARY_DIR ${CMAKE_SOURCE_DIR}/extlibs/${args_ROOT_DIR}/build
+ )
+ FetchContent_GetProperties(${args_TARGET})
+ string(TOLOWER "" lcTargetName)
+
+ if((NOT SIBR_GITLIBRARY_DECLARED) AND (NOT ${lcTargetName}_POPULATED))
+ message(STATUS "Populating library ${args_TARGET}...")
+ FetchContent_Populate(${args_TARGET} QUIET
+ GIT_REPOSITORY ${args_GIT_REPOSITORY}
+ GIT_TAG ${args_GIT_TAG}
+ SOURCE_DIR ${CMAKE_SOURCE_DIR}/extlibs/${args_ROOT_DIR}/${args_SOURCE_DIR}
+ SUBBUILD_DIR ${CMAKE_SOURCE_DIR}/extlibs/${args_ROOT_DIR}/subbuild
+ BINARY_DIR ${CMAKE_SOURCE_DIR}/extlibs/${args_ROOT_DIR}/build
+ )
+ endif()
+
+ add_subdirectory(${CMAKE_SOURCE_DIR}/extlibs/${args_ROOT_DIR}/${args_SOURCE_DIR} ${CMAKE_SOURCE_DIR}/extlibs/${args_ROOT_DIR}/build)
+
+ get_target_property(type ${args_TARGET} TYPE)
+ if(NOT (type STREQUAL "INTERFACE_LIBRARY"))
+ set_target_properties(${args_TARGET} PROPERTIES FOLDER "extlibs")
+ endif()
+
+ list(APPEND ${args_TARGET}_INCLUDE_DIRS ${EXTLIBS_PACKAGE_FOLDER}/${args_ROOT_DIR})
+ list(APPEND ${args_TARGET}_INCLUDE_DIRS ${EXTLIBS_PACKAGE_FOLDER}/${args_ROOT_DIR}/${args_SOURCE_DIR})
+
+ foreach(args_INCLUDE_DIR ${args_INCLUDE_DIRS})
+ list(APPEND ${args_TARGET}_INCLUDE_DIRS ${EXTLIBS_PACKAGE_FOLDER}/${args_ROOT_DIR}/${args_SOURCE_DIR}/${args_INCLUDE_DIR})
+ endforeach()
+
+ include_directories(${${args_TARGET}_INCLUDE_DIRS})
+ else()
+ message(FATAL "Error on sibr_gitlibrary for target ${args_TARGET}: missing git tag or git url.")
+ endif()
+endfunction()
\ No newline at end of file
diff --git a/submodules/gaussian-splatting/SIBR_viewers/src/CMakeLists.txt b/submodules/gaussian-splatting/SIBR_viewers/src/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..eab58728e0e9f0c3311e845bf11ed3d56bf9ea32
--- /dev/null
+++ b/submodules/gaussian-splatting/SIBR_viewers/src/CMakeLists.txt
@@ -0,0 +1,176 @@
+# Copyright (C) 2020, Inria
+# GRAPHDECO research group, https://team.inria.fr/graphdeco
+# All rights reserved.
+#
+# This software is free for non-commercial, research and evaluation use
+# under the terms of the LICENSE.md file.
+#
+# For inquiries contact sibr@inria.fr and/or George.Drettakis@inria.fr
+
+
+################################################################################
+# This CMakeLists.txt manages which projects should be built and add their #
+# dependencies. #
+################################################################################
+set(SIBR_FOLDER "core")
+set_property(GLOBAL PROPERTY PREDEFINED_TARGETS_FOLDER "")
+
+option(BUILD_SIBR "Build core libs of SIBR (sibr_system, sibr_graphics, sibr_view, sibr_assets, ...)" ON)
+
+#https://stackoverflow.com/questions/7787823/cmake-how-to-get-the-name-of-all-subdirectories-of-a-directory
+MACRO(SUBDIRLIST result curdir)
+ FILE(GLOB children RELATIVE ${curdir} ${curdir}/*)
+ SET(dirlist "")
+ foreach(child ${children})
+ IF(IS_DIRECTORY ${curdir}/${child})
+ LIST(APPEND dirlist ${child})
+ ENDIF()
+ endforeach()
+ SET(${result} ${dirlist})
+ENDMACRO()
+
+set(SIBR_PROJECTS_FOLDER "${CMAKE_CURRENT_SOURCE_DIR}/projects")
+SUBDIRLIST(SUBDIRS ${SIBR_PROJECTS_FOLDER})
+
+list(APPEND PROJECT_SUBFOLDERS "apps" "preprocess" "renderer" "scripts" "library")
+
+# Moving ulr to the top of the list
+list(PREPEND SUBDIRS "dataset_tools" "ulr" "basic")
+list(REMOVE_DUPLICATES SUBDIRS)
+
+## DEPS ##
+include(include_once)
+
+message(STATUS "\n\n****************** Handling core dependencies ******************")
+
+include_once(dependencies) ## Map/bind 3rdParty/external dependencies packages to cmake
+
+message(STATUS "****************************************************************\n\n")
+
+foreach(subdir ${SUBDIRS})
+ set(${subdir}_ROOT_DIR "${SIBR_PROJECTS_FOLDER}/${subdir}")
+ set(PROJECT_NAME "BUILD_IBR_${subdir}")
+ string(TOUPPER ${PROJECT_NAME} PROJECT_NAME)
+ if(${${PROJECT_NAME}})
+ foreach(PROJECT_SUBFOLDER ${PROJECT_SUBFOLDERS})
+ if(EXISTS "${${subdir}_ROOT_DIR}/${PROJECT_SUBFOLDER}/cmake/Modules")
+ list(APPEND CMAKE_MODULE_PATH ${${subdir}_ROOT_DIR}/${PROJECT_SUBFOLDER}/cmake/Modules)
+ endif()
+
+ if(EXISTS "${${subdir}_ROOT_DIR}/${PROJECT_SUBFOLDER}/cmake/dependencies.cmake")
+ message(STATUS "********* Handling ${subdir} ${PROJECT_SUBFOLDER} dependencies *********")
+ include_once("${${subdir}_ROOT_DIR}/${PROJECT_SUBFOLDER}/cmake/dependencies.cmake")
+
+ message(STATUS "****************************************************************\n\n")
+ endif()
+ endforeach()
+ endif()
+endforeach()
+
+Win3rdPartyGlobalCacheAction()
+
+include_directories(.)
+
+if (BUILD_SIBR)
+ add_subdirectory(core/system)
+ add_subdirectory(core/graphics)
+ add_subdirectory(core/renderer)
+ add_subdirectory(core/raycaster)
+ add_subdirectory(core/view)
+ add_subdirectory(core/scene)
+ add_subdirectory(core/assets)
+ add_subdirectory(core/imgproc)
+ add_subdirectory(core/video)
+endif()
+
+set(PROJECTS_ON_AT_FIRST_BUILD "basic" "gaussianviewer" "remote")
+
+foreach(subdir ${SUBDIRS})
+ message(STATUS "Adding ${subdir} project")
+ set(PROJECT_NAME "BUILD_IBR_${subdir}")
+ string(TOUPPER ${PROJECT_NAME} PROJECT_NAME)
+
+ if(NOT (DEFINED ${PROJECT_NAME}))
+ foreach(PROJECT_SUBFOLDER ${PROJECT_SUBFOLDERS})
+ if(EXISTS "${${subdir}_ROOT_DIR}/${PROJECT_SUBFOLDER}/CMakeLists.txt")
+ if(subdir IN_LIST PROJECTS_ON_AT_FIRST_BUILD)
+ option(${PROJECT_NAME} "Build project \"${subdir}\"" ON)
+ else()
+ option(${PROJECT_NAME} "Build project \"${subdir}\"" OFF)
+ endif()
+ break()
+ endif()
+ endforeach()
+ endif()
+
+ message(STATUS "${PROJECT_NAME} is ${${PROJECT_NAME}}")
+
+ if(${${PROJECT_NAME}})
+ if(EXISTS "${${subdir}_ROOT_DIR}/CMakeLists.txt")
+ add_subdirectory("${${subdir}_ROOT_DIR}")
+ else()
+ foreach(PROJECT_SUBFOLDER ${PROJECT_SUBFOLDERS})
+ if(EXISTS "${${subdir}_ROOT_DIR}/${PROJECT_SUBFOLDER}/CMakeLists.txt")
+ add_subdirectory("${${subdir}_ROOT_DIR}/${PROJECT_SUBFOLDER}")
+ endif()
+ endforeach()
+ endif()
+
+ if(EXISTS "${${subdir}_ROOT_DIR}/documentation/" AND BUILD_DOCUMENTATION)
+ unset(PROJECT_PAGE)
+ unset(PROJECT_LINK)
+ unset(PROJECT_DESCRIPTION)
+ unset(PROJECT_TYPE)
+ include("${${subdir}_ROOT_DIR}/documentation/${subdir}_doc.cmake" OPTIONAL)
+
+ if(NOT DEFINED PROJECT_PAGE)
+ set(PROJECT_PAGE "${subdir}Page")
+ endif()
+
+ if(NOT DEFINED PROJECT_TYPE)
+ set(PROJECT_TYPE "OTHERS")
+ endif()
+
+ set(PROJECT_SUBPAGE_REF " - @subpage ${PROJECT_PAGE}")
+ set(PROJECT_REF_REF " - @ref ${PROJECT_PAGE}")
+
+ if(DEFINED PROJECT_LINK)
+ string(APPEND PROJECT_SUBPAGE_REF " (${PROJECT_LINK})")
+ string(APPEND PROJECT_REF_REF " (${PROJECT_LINK})")
+ endif()
+
+ if(DEFINED PROJECT_DESCRIPTION)
+ string(APPEND PROJECT_SUBPAGE_REF ": ${PROJECT_DESCRIPTION}")
+ string(APPEND PROJECT_REF_REF " (${PROJECT_DESCRIPTION})")
+ endif()
+
+ string(APPEND SIBR_PROJECTS_${PROJECT_TYPE}_SUBPAGE_REF_LOCAL "${PROJECT_SUBPAGE_REF}\n")
+ string(APPEND SIBR_PROJECTS_${PROJECT_TYPE}_REF_REF_LOCAL "${PROJECT_REF_REF}\n")
+
+ if(EXISTS "${${subdir}_ROOT_DIR}/documentation/img")
+ set(DOXY_APP_SPECIFIC_IMG_PATH_LOCAL "${DOXY_APP_SPECIFIC_IMG_PATH_LOCAL} ${${subdir}_ROOT_DIR}/documentation/img")
+ endif()
+
+ if(EXISTS "${${subdir}_ROOT_DIR}/LICENSE.md")
+ set(DOXY_DOC_EXCLUDE_PATTERNS_DIRS_LOCAL "${DOXY_DOC_EXCLUDE_PATTERNS_DIRS_LOCAL} ${${subdir}_ROOT_DIR}/LICENSE.md")
+ endif()
+ endif()
+ else()
+ set(DOXY_DOC_EXCLUDE_PATTERNS_DIRS_LOCAL "${DOXY_DOC_EXCLUDE_PATTERNS_DIRS_LOCAL} ${${subdir}_ROOT_DIR}")
+ endif()
+endforeach()
+
+set(SIBR_PROJECTS_SAMPLES_SUBPAGE_REF "${SIBR_PROJECTS_SAMPLES_SUBPAGE_REF_LOCAL}" PARENT_SCOPE)
+set(SIBR_PROJECTS_OURS_SUBPAGE_REF "${SIBR_PROJECTS_OURS_SUBPAGE_REF_LOCAL}" PARENT_SCOPE)
+set(SIBR_PROJECTS_TOOLBOX_SUBPAGE_REF "${SIBR_PROJECTS_TOOLBOX_SUBPAGE_REF_LOCAL}" PARENT_SCOPE)
+set(SIBR_PROJECTS_OTHERS_SUBPAGE_REF "${SIBR_PROJECTS_OTHERS_SUBPAGE_REF_LOCAL}" PARENT_SCOPE)
+set(SIBR_PROJECTS_SAMPLES_REF_REF "${SIBR_PROJECTS_SAMPLES_REF_REF_LOCAL}" PARENT_SCOPE)
+set(SIBR_PROJECTS_OURS_REF_REF "${SIBR_PROJECTS_OURS_REF_REF_LOCAL}" PARENT_SCOPE)
+set(SIBR_PROJECTS_TOOLBOX_REF_REF "${SIBR_PROJECTS_TOOLBOX_REF_REF_LOCAL}" PARENT_SCOPE)
+set(SIBR_PROJECTS_OTHERS_REF_REF "${SIBR_PROJECTS_OTHERS_REF_REF_LOCAL}" PARENT_SCOPE)
+set(DOXY_APP_SPECIFIC_IMG_PATH "${DOXY_APP_SPECIFIC_IMG_PATH_LOCAL}" PARENT_SCOPE)
+set(DOXY_DOC_EXCLUDE_PATTERNS_DIRS "${DOXY_DOC_EXCLUDE_PATTERNS_DIRS_LOCAL}" PARENT_SCOPE)
+
+if (BUILD_IBR_TFGL_INTEROP)
+ add_subdirectory(projects/tfgl_interop/renderer/custom_ops)
+endif()
diff --git a/submodules/gaussian-splatting/SIBR_viewers/src/core/assets/ActiveImageFile.cpp b/submodules/gaussian-splatting/SIBR_viewers/src/core/assets/ActiveImageFile.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..620bbbe278aa4f1c0e4590ea39ce8ca29ab3c6d1
--- /dev/null
+++ b/submodules/gaussian-splatting/SIBR_viewers/src/core/assets/ActiveImageFile.cpp
@@ -0,0 +1,62 @@
+/*
+ * Copyright (C) 2020, Inria
+ * GRAPHDECO research group, https://team.inria.fr/graphdeco
+ * All rights reserved.
+ *
+ * This software is free for non-commercial, research and evaluation use
+ * under the terms of the LICENSE.md file.
+ *
+ * For inquiries contact sibr@inria.fr and/or George.Drettakis@inria.fr
+ */
+
+
+# include
+# include
+# include "core/assets/ActiveImageFile.hpp"
+
+namespace sibr
+{
+ bool ActiveImageFile::load( const std::string& filename, bool verbose )
+ {
+
+ std::fstream file(filename, std::ios::in);
+ if(_numImages == 0 )
+ SIBR_WRG << "No Images Loaded while loading '"<> imageId;
+ _active[imageId] = true;
+ }
+
+ if( verbose )
+ SIBR_FLOG << "'"<< filename <<"' successfully loaded." << std::endl;
+ else
+ std::cerr<< "." ;
+ return true;
+ }
+ else {
+ for(int i=0; i < _numImages; i++)
+ _active[i]=true;
+ if( verbose )
+ SIBR_WRG << "file not found: '"<& active( void ) const { return _active; }
+
+
+ private:
+ std::vector _active; ///< Flags denoting which images are active.
+ int _numImages = 0; ///< Number of images.
+
+ };
+
+
+} // namespace sibr
diff --git a/submodules/gaussian-splatting/SIBR_viewers/src/core/assets/CMakeLists.txt b/submodules/gaussian-splatting/SIBR_viewers/src/core/assets/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..e68317caab93bc0366ce0a25702a764c05cb3d91
--- /dev/null
+++ b/submodules/gaussian-splatting/SIBR_viewers/src/core/assets/CMakeLists.txt
@@ -0,0 +1,45 @@
+# Copyright (C) 2020, Inria
+# GRAPHDECO research group, https://team.inria.fr/graphdeco
+# All rights reserved.
+#
+# This software is free for non-commercial, research and evaluation use
+# under the terms of the LICENSE.md file.
+#
+# For inquiries contact sibr@inria.fr and/or George.Drettakis@inria.fr
+
+
+project(sibr_assets)
+
+file(GLOB SOURCES "*.cpp" "*.h" "*.hpp" )
+source_group("Source Files" FILES ${SOURCES})
+
+file(GLOB SOURCES "*.cpp" "*.h" "*.hpp")
+
+
+## Specify target rules
+add_library(${PROJECT_NAME} SHARED ${SOURCES})
+
+include_directories(
+ ${Boost_INCLUDE_DIRS}
+ ${xatlas_INCLUDE_DIRS}
+)
+target_link_libraries(${PROJECT_NAME}
+ ${Boost_LIBRARIES}
+ ${ASSIMP_LIBRARIES}
+ ${GLEW_LIBRARIES}
+ ${OPENGL_LIBRARIES}
+ ${OpenCV_LIBRARIES}
+ OpenMP::OpenMP_CXX
+ sibr_graphics
+ xatlas
+)
+
+add_definitions(-DSIBR_ASSETS_EXPORTS -DBOOST_ALL_DYN_LINK)
+
+set_target_properties(${PROJECT_NAME} PROPERTIES FOLDER ${SIBR_FOLDER})
+
+## High level macro to install in an homogen way all our ibr targets
+include(install_runtime)
+ibr_install_target(${PROJECT_NAME}
+ INSTALL_PDB ## mean install also MSVC IDE *.pdb file (DEST according to target type)
+)
diff --git a/submodules/gaussian-splatting/SIBR_viewers/src/core/assets/CameraRecorder.cpp b/submodules/gaussian-splatting/SIBR_viewers/src/core/assets/CameraRecorder.cpp
new file mode 100755
index 0000000000000000000000000000000000000000..b821210d7f7d216ef66e63e299edd76f0e56f191
--- /dev/null
+++ b/submodules/gaussian-splatting/SIBR_viewers/src/core/assets/CameraRecorder.cpp
@@ -0,0 +1,515 @@
+/*
+ * Copyright (C) 2020, Inria
+ * GRAPHDECO research group, https://team.inria.fr/graphdeco
+ * All rights reserved.
+ *
+ * This software is free for non-commercial, research and evaluation use
+ * under the terms of the LICENSE.md file.
+ *
+ * For inquiries contact sibr@inria.fr and/or George.Drettakis@inria.fr
+ */
+
+
+#include
+#include "core/assets/CameraRecorder.hpp"
+#include "core/assets/InputCamera.hpp"
+#include
+
+namespace sibr
+{
+ void CameraRecorder::use(Camera& cam)
+ {
+ if (_recording) {
+ _cameras.push_back(cam);
+ }
+ else if (_playing && _pos < _cameras.size() ) {
+ const float znear = cam.znear();
+ const float zfar = cam.zfar();
+
+ if( !_playNoInterp ) {
+ //std::cout << _playing << std::endl;
+ // If we reach the last frame of the interpolation b/w two cameras, skip to next camera.
+ if (_interp > (1.0f - _speed))
+ {
+ _interp = 0.0f;
+ _pos++;
+ }
+ // Interpolate between the two closest cameras.
+ const float k = std::min(std::max(_interp, 0.0f), 1.0f);
+
+ sibr::Camera & camStart = _cameras[std::min(int(_pos), int(_cameras.size()) - 1)];
+ sibr::Camera & camNext = _cameras[std::min(int(_pos) + 1, int(_cameras.size())-1)];
+
+ cam = sibr::Camera::interpolate(camStart, camNext, k);
+
+ _interp += _speed;
+ }
+ else {
+ cam = _cameras[_pos];
+ _pos++;
+ if (_pos == _cameras.size())
+ _playNoInterp = false;
+
+ }
+
+ // Preserve the znear and zfar.
+ cam.znear(znear);
+ cam.zfar(zfar);
+
+ if (_saving) {
+ std::ostringstream ssZeroPad;
+ ssZeroPad << std::setw(8) << std::setfill('0') << (_pos);
+ cam.setSavePath(_savingPath + "/" + ssZeroPad.str() + ".png");
+ //std::cout << "Saving frame as: " << cam.savePath() << std::endl;
+ }
+ if (_savingVideo) {
+ cam.setDebugVideo(true);
+ }
+ if (_pos >= _cameras.size())
+ {
+ stop();
+ SIBR_LOG << "[CameraRecorder] - Playback Finished" << std::endl;
+ }
+ }
+ else {
+ //std::cout << _playing << std::endl;
+ cam.setSavePath("");
+ cam.setDebugVideo(false);
+ }
+ }
+
+ void CameraRecorder::playback(void)
+ {
+ stop();
+ _playing = true;
+ SIBR_LOG << "[CameraRecorder] - Playing" << std::endl;
+ }
+
+ void CameraRecorder::record(void)
+ {
+ stop();
+ _recording = true;
+ SIBR_LOG << "[CameraRecorder] - Recording" << std::endl;
+ }
+
+ void sibr::CameraRecorder::saving(std::string savePath)
+ {
+ _saving = true;
+ _savingPath = savePath;
+ SIBR_LOG << "[CameraRecorder] - Recording" << std::endl;
+ }
+
+ void CameraRecorder::savingVideo(bool saveVideo)
+ {
+ _savingVideo = saveVideo;
+ }
+
+ void sibr::CameraRecorder::stopSaving(void)
+ {
+ _saving = false;
+ _savingPath = "";
+ }
+
+ void CameraRecorder::stop(void)
+ {
+ _recording = _playing = false;
+ _pos = 0;
+ _interp = 0.0f;
+ }
+
+ void CameraRecorder::reset(void)
+ {
+ stop();
+ _cameras.clear();
+ }
+
+ bool CameraRecorder::load(const std::string& filename)
+ {
+ reset();
+
+ sibr::ByteStream stream;
+
+ if (stream.load(filename) == false)
+ return false;
+
+ int32 num = 0;
+
+ std::cout << " CameraRecorder::load " << num << std::endl;
+
+ stream >> num;
+ while (num > 0)
+ {
+ Camera cam;
+ stream >> cam;
+ _cameras.emplace_back(std::move(cam));
+ --num;
+ }
+ SIBR_LOG << "[CameraRecorder] - Loaded from " << filename << std::endl;
+ return stream;
+ }
+
+ void CameraRecorder::save(const std::string& filename)
+ {
+ sibr::ByteStream stream;
+ int32 num = (int32)_cameras.size();
+ stream << num;
+ for (const Camera& cam : _cameras)
+ stream << cam;
+
+ stream.saveToFile(filename);
+ SIBR_LOG << "[CameraRecorder] - Saved " << num << " cameras to " << filename << std::endl;
+ }
+
+ bool CameraRecorder::safeLoad(const std::string & filename, int w, int h)
+ {
+ Path path = Path(filename);
+
+ if (path.extension().string() == ".out") {
+ loadBundle(filename, w, h);
+ return true;
+ } else if (path.extension().string() == ".path") {
+ return load(filename);
+ }
+ SIBR_WRG << "Unable to load camera path" << std::endl;
+ return false;
+ }
+
+ void CameraRecorder::loadBundle(const std::string & filePath, int w, int h)
+ {
+ const std::string bundlerFile = filePath;
+ SIBR_LOG << "Loading bundle path." << std::endl;
+
+ // check bundler file
+ std::ifstream bundle_file(bundlerFile);
+ SIBR_ASSERT(bundle_file.is_open());
+
+ // read number of images
+ std::string line;
+ getline(bundle_file, line); // ignore first line - contains version
+ int numImages = 0;
+ bundle_file >> numImages; // read first value (number of images)
+ getline(bundle_file, line); // ignore the rest of the line
+
+ std::vector cameras(numImages);
+ // Parse bundle.out file for camera calibration parameters
+ for (int i = 0; i < numImages; i++) {
+
+ Matrix4f m;
+ bundle_file >> m(0) >> m(1) >> m(2) >> m(3) >> m(4);
+ bundle_file >> m(5) >> m(6) >> m(7) >> m(8) >> m(9);
+ bundle_file >> m(10) >> m(11) >> m(12) >> m(13) >> m(14);
+
+ cameras[i] = InputCamera::Ptr(new InputCamera(i, w, h, m, true));
+
+ cameras[i]->znear(0.2f); cameras[i]->zfar(250.f);
+
+ }
+
+ for (const InputCamera::Ptr cam : cameras)
+ {
+ _cameras.push_back(*cam);
+ }
+
+ }
+
+ void CameraRecorder::loadColmap(const std::string &filePath, int w, int h)
+ {
+ SIBR_LOG << "Loading colmap path." << std::endl;
+ boost::filesystem::path colmapDir ( filePath );
+
+ SIBR_LOG << "DEBUG: colmap path dir " << colmapDir.parent_path().string() << std::endl;
+
+ std::vector path = InputCamera::loadColmap(colmapDir.parent_path().string(), float(0.01), 1000, false );
+ for (const InputCamera::Ptr cam : path)
+ {
+ _cameras.push_back(*cam);
+ }
+ }
+
+ void CameraRecorder::loadLookat(const std::string &filePath, int w, int h)
+ {
+ SIBR_LOG << "Loading lookat path." << std::endl;
+ std::vector path = InputCamera::loadLookat(filePath, std::vector{Vector2u(w, h)});
+ for (const InputCamera::Ptr cam : path)
+ {
+ _cameras.push_back(*cam);
+ }
+ }
+
+ void CameraRecorder::saveAsBundle(const std::string & filePath, const int height, const int step)
+ {
+
+ std::ofstream out(filePath.c_str(), std::ios::binary);
+ if (!out.good()) {
+ SIBR_LOG << "ERROR: cannot write to the file '" << filePath << "'" << std::endl;
+ return;
+ }
+
+ if(_cameras.empty()) {
+ return;
+ }
+
+ const int size = static_cast(_cameras.size() / step);
+
+ out << "# Bundle file v0.3\n";
+ out << size << " " << 0 << "\n";
+
+ for (int i = 0; i < _cameras.size(); i += step) {
+
+ const sibr::Camera cam = _cameras[i];
+ sibr::Quaternionf q = cam.rotation();
+ sibr::Matrix3f m1 = q.toRotationMatrix();
+ sibr::Vector3f pos = -m1.transpose() * cam.position();
+
+ float m[15];
+ m[0] = 0.5f*height / tan(cam.fovy() / 2.f); m[1] = 0.0f; m[2] = 0.0f;
+ m[3] = m1(0, 0), m[4] = m1(1, 0), m[5] = m1(2, 0);
+ m[6] = m1(0, 1), m[7] = m1(1, 1), m[8] = m1(2, 1);
+ m[9] = m1(0, 2), m[10] = m1(1, 2), m[11] = m1(2, 2);
+ m[12] = pos.x(), m[13] = pos.y(), m[14] = pos.z();
+
+ out << m[0] << " " << m[1] << " " << m[2] << std::endl;
+ out << m[3] << " " << m[4] << " " << m[5] << std::endl;
+ out << m[6] << " " << m[7] << " " << m[8] << std::endl;
+ out << m[9] << " " << m[10] << " " << m[11] << std::endl;
+ out << m[12] << " " << m[13] << " " << m[14] << std::endl;
+ }
+ out << std::endl;
+ out.close();
+
+ SIBR_LOG << "[CameraRecorder] - Saved " << _cameras.size() << " cameras to " << filePath << " (using fovy " <<_cameras[0].fovy() << ")." << std::endl;
+
+ }
+
+ void CameraRecorder::saveAsColmap(const std::string& filePath, const int height, const int width)
+ {
+
+ std::string basepath = parentDirectory(filePath);
+ std::cout << basepath << std::endl;
+ std::string images_filepath = basepath + "/images.txt";
+ std::string cameras_filepath = basepath + "/cameras.txt";
+
+ std::ofstream out_images(images_filepath.c_str(), std::ios::binary);
+ std::ofstream out_cameras(cameras_filepath.c_str(), std::ios::binary);
+
+ if (!out_images.good()) {
+ SIBR_LOG << "ERROR: cannot write to the file '" << filePath << "'" << std::endl;
+ return;
+ }
+ if (!out_cameras.good()) {
+ SIBR_LOG << "ERROR: cannot write to the file '" << filePath << "'" << std::endl;
+ return;
+ }
+
+ if (_cameras.empty()) {
+ return;
+ }
+
+ const int size = static_cast(_cameras.size());
+
+ sibr::Matrix3f converter;
+ converter << 1, 0, 0,
+ 0, -1, 0,
+ 0, 0, -1;
+
+
+
+ out_images << "# Image list with two lines of data per image:" << std::endl;
+ out_images << "# IMAGE_ID, QW, QX, QY, QZ, TX, TY, TZ, CAMERA_ID, NAME" << std::endl;
+ out_images << "# POINTS2D[] as (X, Y, POINT3D_ID)" << std::endl;
+ for (int i = 0; i < _cameras.size(); i++) {
+ sibr::Matrix3f tmp = _cameras[i].rotation().toRotationMatrix() * converter;
+ sibr::Matrix3f Qinv = tmp.transpose();
+ sibr::Quaternionf q = quatFromMatrix(Qinv);
+ sibr::Vector3f t = converter * _cameras[i].rotation().toRotationMatrix().transpose() * (-_cameras[i].position());
+
+
+ out_images << i << " " << -_cameras[i].rotation().x() << " " << -_cameras[i].rotation().w() << " " << -_cameras[i].rotation().z() << " " << _cameras[i].rotation().y() << " " <<
+ _cameras[i].view()(0, 3) << " " << -_cameras[i].view()(1, 3) << " " << -_cameras[i].view()(2, 3) << " " << i << " " << "00000000.png" << std::endl << std::endl;
+
+ float focal = 0.5f * height / tan(_cameras[i].fovy() / 2.f);
+ //float focal = 1.0f / (tan(0.5f * cam.fovy()) * 2.0f / float(height));
+ out_cameras << i << " " << "PINHOLE" << " " << width << " " << height << " " << focal << " " << focal << " " << width / 2.0 << " " << height / 2.0 << std::endl;
+ }
+
+ out_images << std::endl;
+ out_images.close();
+ out_cameras << std::endl;
+ out_cameras.close();
+ SIBR_LOG << "[CameraRecorder] - Saved " << _cameras.size() << " cameras to " << filePath << " (using fovy " << _cameras[0].fovy() << ")." << std::endl;
+
+ }
+
+
+ void CameraRecorder::saveAsFRIBRBundle(const std::string & dirPath, const int width, const int height)
+ {
+ const std::string bundlepath = dirPath + "/path.rd.out";
+ const std::string listpath = dirPath + "/list.txt";
+ const std::string imagesDir = dirPath + "/visualize/";
+ sibr::makeDirectory(dirPath);
+ sibr::makeDirectory(imagesDir);
+ std::ofstream out(bundlepath);
+ std::ofstream outList(listpath);
+ if (out.good() && outList.good()) {
+ const int size = static_cast(_cameras.size() / 1);
+ out << "# Bundle file v0.3\n";
+ out << size << " " << 0 << "\n";
+ sibr::Matrix3f converter;
+ converter << 1, 0, 0,
+ 0, -1, 0,
+ 0, 0, -1;
+ sibr::Matrix3f from_cv;
+ from_cv << 1, 0, 0,
+ 0, -1, 0,
+ 0, 0, -1;
+ for (int i = 0; i < _cameras.size(); ++i) {
+
+ const sibr::Camera cam = _cameras[i];
+
+ sibr::Matrix3f orientation = cam.rotation().toRotationMatrix();
+ sibr::Vector3f position = cam.position();
+ sibr::Matrix3f rotation_cv = converter.transpose() * orientation.transpose() * converter;
+ sibr::Matrix3f rotation_bundler = from_cv * rotation_cv;
+ sibr::Vector3f position_cv = converter.transpose() * position;
+ sibr::Vector3f translation_cv = -(rotation_cv * position_cv);
+ sibr::Vector3f pos = from_cv * translation_cv;
+
+ sibr::Matrix3f m1 = rotation_bundler.transpose();
+ float m[15];
+ m[0] = 0.5f*height / tan(cam.fovy() / 2.f); m[1] = 0.0f; m[2] = 0.0f;
+ m[3] = m1(0, 0), m[4] = m1(1, 0), m[5] = m1(2, 0);
+ m[6] = m1(0, 1), m[7] = m1(1, 1), m[8] = m1(2, 1);
+ m[9] = m1(0, 2), m[10] = m1(1, 2), m[11] = m1(2, 2);
+ m[12] = pos.x(), m[13] = pos.y(), m[14] = pos.z();
+ out << m[0] << " " << m[1] << " " << m[2] << std::endl;
+ out << m[3] << " " << m[4] << " " << m[5] << std::endl;
+ out << m[6] << " " << m[7] << " " << m[8] << std::endl;
+ out << m[9] << " " << m[10] << " " << m[11] << std::endl;
+ out << m[12] << " " << m[13] << " " << m[14] << std::endl;
+
+ const std::string imageName = sibr::intToString<8>(i) + ".jpg";
+ outList << "visualize/" + imageName << " 0 " << m[0] << std::endl;
+
+ cv::Mat3b dummy(height, width);
+ cv::imwrite(imagesDir + imageName, dummy);
+ }
+ out << std::endl;
+ out.close();
+ outList.close();
+
+ SIBR_LOG << "[CameraRecorder] - Saved " << _cameras.size() << " cameras to " << dirPath << "." << std::endl;
+ }
+ }
+
+ void CameraRecorder::saveAsLookAt(const std::string & filePath) const
+ {
+ InputCamera::saveAsLookat(_cameras, filePath);
+ }
+
+ // offline path rendering
+ bool CameraRecorder::loadPath(const std::string& pathFileName, int w, int h) {
+ _savingPath = parentDirectory(pathFileName);
+ if (!fileExists(pathFileName)) {
+ SIBR_WRG << "Camera path " << pathFileName << " doesnt exist. " << std::endl;
+ return false;
+ }
+ _ow = w, _oh = h;
+ if (boost::filesystem::extension(pathFileName) == ".out")
+ loadBundle(pathFileName, w, h);
+ else if (boost::filesystem::extension(pathFileName) == ".lookat")
+ loadLookat(pathFileName, w, h);
+ else if (boost::filesystem::extension(pathFileName) == ".txt")
+ loadColmap(pathFileName, w, h);
+ else
+ load(pathFileName);
+ return true;
+ }
+
+ void CameraRecorder::recordOfflinePath(const std::string& outPathDir, ViewBase::Ptr view, const std::string& prefix) {
+ sibr::ImageRGBA32F::Ptr outImage;
+ outImage.reset(new ImageRGBA32F(_ow, _oh));
+ std::string outpathd = outPathDir;
+
+ sibr::RenderTargetRGBA32F::Ptr outFrame;
+ outFrame.reset(new RenderTargetRGBA32F(_ow, _oh));
+ std::string outFileName;
+
+ boost::filesystem::path dstFolder;
+
+ outpathd = outPathDir;
+
+ if (outPathDir == "pathOutput" && _savingPath != "") { // default to path parent, saved by loadPath
+ outpathd = _savingPath + "/" + "pathOutput";
+ dstFolder = outpathd;
+ if (!directoryExists(outpathd) && !boost::filesystem::create_directories(dstFolder))
+ SIBR_ERR << "Error creating directory " << dstFolder << std::endl;
+ }
+
+ if( prefix != "" )
+ dstFolder = outpathd = outpathd + "/" + prefix;
+ else
+ dstFolder = outpathd ;
+
+ if (!directoryExists(outpathd) && !boost::filesystem::create_directories(dstFolder))
+ SIBR_ERR << "Error creating directory " << dstFolder << std::endl;
+
+ std::cout << "Rendering path with " << _cameras.size() << " cameras to " << outpathd << std::endl;
+
+ for (int i = 0; i < _cameras.size(); ++i) {
+ outFrame->clear();
+ std::ostringstream ssZeroPad;
+ ssZeroPad << std::setw(8) << std::setfill('0') << i;
+ outFileName = outpathd + "/" + ssZeroPad.str() + ".png";
+ std::cout << outFileName << " " << std::endl;
+ view->onRenderIBR(*outFrame, _cameras[i]);
+ outFrame->readBack(*outImage);
+ outImage->save(outFileName, false);
+ }
+ std::cout << std::endl;
+
+ std::cout << "Done rendering path. " << std::endl;
+
+ }
+
+ void CameraRecorder::saveImage(const std::string& outPathDir, const Camera& cam, int w, int h) {
+ sibr::ImageRGBA32F::Ptr outImage;
+ _ow = w, _oh = h;
+ outImage.reset(new ImageRGBA32F(_ow, _oh));
+ std::string outpathd = outPathDir;
+
+ sibr::RenderTargetRGBA32F::Ptr outFrame;
+ outFrame.reset(new RenderTargetRGBA32F(_ow, _oh));
+ std::string outFileName;
+
+ boost::filesystem::path dstFolder;
+
+ outpathd = outPathDir;
+
+ if (outPathDir == "") { // default to path parent, saved by loadPath
+ outpathd = _dsPath + "/" + "pathOutput";
+ dstFolder = outpathd;
+ if (!directoryExists(outpathd) && !boost::filesystem::create_directories(dstFolder))
+ SIBR_ERR << "Error creating directory " << dstFolder << std::endl;
+ }
+
+ dstFolder = outpathd;
+
+ if (!directoryExists(outpathd) && !boost::filesystem::create_directories(dstFolder))
+ SIBR_ERR << "Error creating directory " << dstFolder << std::endl;
+
+ std::cout << "Saving current camera to " << outpathd << std::endl;
+
+ outFrame->clear();
+ std::ostringstream ssZeroPad;
+ static int i = 0;
+ ssZeroPad << std::setw(8) << std::setfill('0') << i++;
+ outFileName = outpathd + "/" + ssZeroPad.str() + ".png";
+ std::cout << outFileName << " " << std::endl;
+ _view->onRenderIBR(*outFrame, cam);
+ outFrame->readBack(*outImage);
+ outImage->save(outFileName, false);
+ std::cout << std::endl;
+ std::cout << "Done saving image. " << std::endl;
+ }
+
+} // namespace sibr
diff --git a/submodules/gaussian-splatting/SIBR_viewers/src/core/assets/CameraRecorder.hpp b/submodules/gaussian-splatting/SIBR_viewers/src/core/assets/CameraRecorder.hpp
new file mode 100755
index 0000000000000000000000000000000000000000..44dd414495ce45eeca7b82845926c023846a8bcf
--- /dev/null
+++ b/submodules/gaussian-splatting/SIBR_viewers/src/core/assets/CameraRecorder.hpp
@@ -0,0 +1,236 @@
+/*
+ * Copyright (C) 2020, Inria
+ * GRAPHDECO research group, https://team.inria.fr/graphdeco
+ * All rights reserved.
+ *
+ * This software is free for non-commercial, research and evaluation use
+ * under the terms of the LICENSE.md file.
+ *
+ * For inquiries contact sibr@inria.fr and/or George.Drettakis@inria.fr
+ */
+
+
+#pragma once
+
+# include
+# include "core/assets/Config.hpp"
+# include "core/graphics/Camera.hpp"
+# include "core/view/ViewBase.hpp"
+
+
+# define SIBR_CAMERARECORDER_DEFAULTFILE "camera-record.bytes"
+
+namespace sibr
+{
+ /** This class handles the recording and replay of a stream of cameras.
+ \ingroup sibr_assets
+ */
+ class SIBR_ASSETS_EXPORT CameraRecorder
+ {
+ public:
+
+ /**
+ Default constructor.
+ */
+ CameraRecorder(void) :
+ _pos(0), _recording(false), _playing(false), _saving(false), _savingPath(""), _savingVideo(false), _savingVideoPath(""), _speed(1.0f), _interp(0.0f), _playNoInterp(false) {
+ //load();
+ }
+ /**
+ Default destructor.
+ */
+ ~CameraRecorder( void ) {
+ }
+
+ /**
+ This method supports two modes: if the recorder is currently recording,
+ the camera argument will be saved into the recording; else if the recorder
+ is playing, the camera argument will be set to the current replaid camera.
+ \param cam A reference to the camera to record/update.
+ */
+ void use( Camera& cam );
+
+ /**
+ Start playing the recorded camera stream from the beginning, at a rate of one step for each "use" call.
+ */
+ void playback( void );
+
+ /**
+ Start recording a new camera stream, at a rate of one camera saved for each "use" call.
+ */
+ void record( void );
+
+ /**
+ Start asking the renderer to save the frames, at a rate of one image saved for each "use" call.
+ */
+ void saving(std::string savePath);
+
+ /**
+ Toggle the save flag for video frames when replaying.
+ \param saveVideo the new flag status
+ */
+ void savingVideo(bool saveVideo);
+
+ /**
+ Stop saving.
+ */
+ void stopSaving(void);
+
+ /**
+ Stop playing/recording.
+ */
+ void stop( void );
+
+ /**
+ Clear the current recording.
+ */
+ void reset( void );
+
+ /**
+ Set value of play no interpolation
+ */
+ void playNoInterpolation( bool val ) { _playNoInterp = val; }
+
+ /**
+ Load a recorded camera stream from a given file.
+ \param filename Optional path to the file to load from. By default will try to
+ load SIBR_CAMERARECORDER_DEFAULTFILE from the current directory.
+ \return a boolean denoting the loading success/failure.
+ */
+ bool load( const std::string& filename=SIBR_CAMERARECORDER_DEFAULTFILE );
+
+ /**
+ Save the current recording stream to a given file.
+ \param filename Optional path to the file to write to. By default will try to
+ write to SIBR_CAMERARECORDER_DEFAULTFILE in the current directory.
+ */
+ void save( const std::string& filename=SIBR_CAMERARECORDER_DEFAULTFILE );
+
+ /** Load recorded path based on file extension.
+ *\param filename the file to load
+ *\param w resoltuion width
+ *\param h resolution height
+ *\return a success boolean
+ *\note w and h are needed when loading a Bundle.
+ */
+ bool safeLoad(const std::string& filename, int w = 1920, int h = 1080);
+
+ /**
+ Load a recording stream saved as a bundle file (useful for path from FRIBR).
+ \param filePath Path to the bundle file to write to.
+ \param w the image width to use for Fov computation
+ \param h the image height
+ */
+ void loadBundle(const std::string & filePath, int w = 1920, int h = 1080);
+
+ /**
+ *Load a camera path generated by the camera editor / bledner plugin
+ *\param filePath Path to the .lookat file.
+ *\param w Width of the cameras to create.
+ *\param h Height of the cameras to create.
+ */
+ void loadLookat(const std::string &filePath, int w = 1920, int h = 1080);
+
+ /**
+ *Load a camera path generated by colmap requires filename images.txt for now TODO GD; fix this
+ *\param filePath Path to the images.txt file; assumes that a cameras.txt is "next to" it;
+ *\param w Width of the cameras to create.
+ *\param h Height of the cameras to create.
+ */
+ void loadColmap(const std::string &filePath, int w = 1920, int h = 1080);
+
+
+ /**
+ Save the current recording stream as a bundle file.
+ \param filePath Path to the bundle file to write to.
+ \param height the height in pixels of the camera. Used to compute focal length in mm as expected by bundle.
+ \param step set to a value greater than 1 to save every "step" camera in the recording stream.
+ */
+ void saveAsBundle(const std::string & filePath, const int height, const int step = 1);
+ void saveAsColmap(const std::string& filePath, const int height, const int width);
+
+ /**
+ Save the current recording stream as a bundle file and a series of empty images for FRIBR compatibility.
+ \param dirPath Path to the directory to export to.
+ \param width the width in pixels of the camera.
+ \param height the height in pixels of the camera.
+ */
+ void saveAsFRIBRBundle(const std::string & dirPath, const int width, const int height);
+
+ /**
+ Save the current recording stream as a lookat file.
+ \param filePath Path to the lookat file to write to.
+ */
+ void saveAsLookAt(const std::string& filePath) const;
+
+ /**
+ \return a boolean denoting if the recorder is currently playing.
+ */
+ bool isPlaying() const { return _playing; }
+
+ /**
+ \return a boolean denoting if the recorder is currently recording.
+ */
+ bool isRecording() const { return _recording; }
+
+ /**
+ \return a boolean denoting if the recorder is currently asking frames to be saved.
+ */
+ bool isSaving() const { return _saving; }
+
+ /**
+ \return A reference to the current stream of recorded cameras.
+ */
+ std::vector& cams() { return _cameras; }
+
+ /**
+ Updates the cameras from a vector, usefull to play already loaded path.
+ */
+ void cams(std::vector& cams) { _cameras = cams; }
+
+
+ /**
+ Load a path, based on file extension name. Cameras loaded into _cameras
+ */
+ bool loadPath(const std::string& pathFileName, int w, int h);
+
+ /**
+ Play path for offline rendering using abstract View interface
+ */
+ void recordOfflinePath(const std::string& outPathDir, ViewBase::Ptr view, const std::string& prefix);
+
+ /**
+ Save an image
+ */
+ void setViewPath(ViewBase::Ptr view, const std::string& dataset_path) { _view = view; _dsPath = dataset_path; };
+
+ void saveImage(const std::string& outPathDir, const Camera& cam, int w, int h);
+
+ /**
+ * \return the interpolation speed
+ */
+ float & speed() { return _speed; }
+
+ private:
+ std::string _dsPath; // path to dataset
+ ViewBase::Ptr _view; // view to save images
+ uint _pos; ///< Current camera ID for replay.
+ std::vector _cameras; ///< List of recorded cameras.
+ bool _recording; ///< Are we currently recording.
+ bool _playing; ///< Are we currently playing.
+ bool _saving; ///< Are we saving the path as images.
+ std::string _savingPath; ///< Destination base path for saved images.
+ bool _savingVideo; ///< Are we saving the path as video.
+ std::string _savingVideoPath; ///< Destination base path for saved video.
+ float _speed; ///< Playback speed.
+ float _interp; ///< Current interpoaltion factor.
+ bool _playNoInterp; ///< Just play the cameras, make sure focal is preserved
+ int _ow, _oh; ///< offline output path resolution
+ };
+
+ ///// DEFINITIONS /////
+
+
+
+} // namespace sibr
+
diff --git a/submodules/gaussian-splatting/SIBR_viewers/src/core/assets/Config.hpp b/submodules/gaussian-splatting/SIBR_viewers/src/core/assets/Config.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..3386bc132505fb74fc7d335ba8c175f795ae5812
--- /dev/null
+++ b/submodules/gaussian-splatting/SIBR_viewers/src/core/assets/Config.hpp
@@ -0,0 +1,99 @@
+/*
+ * Copyright (C) 2020, Inria
+ * GRAPHDECO research group, https://team.inria.fr/graphdeco
+ * All rights reserved.
+ *
+ * This software is free for non-commercial, research and evaluation use
+ * under the terms of the LICENSE.md file.
+ *
+ * For inquiries contact sibr@inria.fr and/or George.Drettakis@inria.fr
+ */
+
+
+#pragma once
+
+# include "core/graphics/Config.hpp"
+# include
+
+
+#ifdef SIBR_OS_WINDOWS
+//// Export Macro (used for creating DLLs) ////
+# ifdef SIBR_STATIC_DEFINE
+# define SIBR_EXPORT
+# define SIBR_NO_EXPORT
+# else
+# ifndef SIBR_ASSETS_EXPORT
+# ifdef SIBR_ASSETS_EXPORTS
+ /* We are building this library */
+# define SIBR_ASSETS_EXPORT __declspec(dllexport)
+# else
+ /* We are using this library */
+# define SIBR_ASSETS_EXPORT __declspec(dllimport)
+# endif
+# endif
+# ifndef SIBR_NO_EXPORT
+# define SIBR_NO_EXPORT
+# endif
+# endif
+# else
+# define SIBR_ASSETS_EXPORT
+# endif
+
+namespace sibr
+{
+ /**
+ * Utility that converts an integer id to a string using
+ * the "most used" format.
+ * \param id the id to convert (fi 7)
+ * \return the corresponding string (fi "0000007")
+ * \ingroup sibr_assets
+ */
+ inline std::string imageIdToString( int id ) {
+ std::ostringstream oss;
+ oss << std::setfill('0') << std::setw(8) << id;
+ return oss.str();
+ }
+
+ /** Generate a string representation of an integer, padded with zeros.
+ * \param id the integer
+ * \return the padded string
+ * \note The template int value determines the padding count.
+ * \ingroup sibr_assets
+ * */
+ template std::string intToString(int id) {
+ std::ostringstream oss;
+ oss << std::setfill('0') << std::setw(N) << id;
+ return oss.str();
+ }
+
+ /**
+ * Get the default path and filename used for the proxy
+ * mesh.
+ * \param datasetPath the base path
+ * \return the mesh path
+ * \ingroup sibr_assets
+ */
+ inline std::string getProxyFilename( const std::string& datasetPath ) {
+ return datasetPath + "/pmvs/models/pmvs_recon.ply";
+ }
+
+ /**
+ * Loading status for streaming.
+ * \todo Rename the following status into: NotLoaded, CPULoading, CPUReady, GPUReady, Failure.
+ * \ingroup sibr_assets
+ */
+ namespace LoadingStatus
+ {
+ enum Enum
+ {
+ NotLoaded = 0,
+ InProgress,
+ CPUReady,
+ Successful,
+ Failure,
+
+ Count
+ };
+ } // namespace LoadingStatus
+
+} // namespace sibr
diff --git a/submodules/gaussian-splatting/SIBR_viewers/src/core/assets/IFileLoader.hpp b/submodules/gaussian-splatting/SIBR_viewers/src/core/assets/IFileLoader.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..9d24dc248526f58b2ec7486a589f36502ba0f05c
--- /dev/null
+++ b/submodules/gaussian-splatting/SIBR_viewers/src/core/assets/IFileLoader.hpp
@@ -0,0 +1,37 @@
+/*
+ * Copyright (C) 2020, Inria
+ * GRAPHDECO research group, https://team.inria.fr/graphdeco
+ * All rights reserved.
+ *
+ * This software is free for non-commercial, research and evaluation use
+ * under the terms of the LICENSE.md file.
+ *
+ * For inquiries contact sibr@inria.fr and/or George.Drettakis@inria.fr
+ */
+
+
+#pragma once
+
+# include "core/assets/Config.hpp"
+
+namespace sibr
+{
+ /** General file loading interface.
+ \ingroup sibr_assets
+ */
+ class SIBR_ASSETS_EXPORT IFileLoader
+ {
+ public:
+
+ /** Destructor. */
+ virtual ~IFileLoader( void ) { }
+
+ /** Load the file content from disk.
+ \param filename path to the file
+ \param verbose display information
+ \return a boolean denoting success
+ */
+ virtual bool load( const std::string& filename, bool verbose = true ) = 0;
+ };
+
+} // namespace sibr
diff --git a/submodules/gaussian-splatting/SIBR_viewers/src/core/assets/ImageListFile.cpp b/submodules/gaussian-splatting/SIBR_viewers/src/core/assets/ImageListFile.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..c354d7557147dac2f5c69a8a143e1380afc0f49b
--- /dev/null
+++ b/submodules/gaussian-splatting/SIBR_viewers/src/core/assets/ImageListFile.cpp
@@ -0,0 +1,52 @@
+/*
+ * Copyright (C) 2020, Inria
+ * GRAPHDECO research group, https://team.inria.fr/graphdeco
+ * All rights reserved.
+ *
+ * This software is free for non-commercial, research and evaluation use
+ * under the terms of the LICENSE.md file.
+ *
+ * For inquiries contact sibr@inria.fr and/or George.Drettakis@inria.fr
+ */
+
+
+
+# include
+# include
+# include "core/assets/ImageListFile.hpp"
+
+namespace sibr
+{
+ bool ImageListFile::load( const std::string& filename, bool verbose )
+ {
+
+ std::fstream file(filename, std::ios::in);
+
+ _infos.clear();
+ if (file)
+ {
+ while (file.eof() == false)
+ {
+ Infos i;
+ file >> i.filename >> i.width >> i.height;
+ if (i.filename.size())
+ _infos.emplace_back(std::move(i));
+ }
+
+ // store basename
+ boost::filesystem::path path(filename);
+ _basename = path.parent_path().string();
+
+ if( verbose )
+ SIBR_FLOG << "'"<< filename <<"' successfully loaded." << std::endl;
+
+ return true;
+ }
+ else
+ SIBR_WRG << "file not found: '"<& infos( void ) const { return _infos; }
+
+ /** Image absename.
+ *\return the basename
+ **/
+ const std::string& basename( void ) const { return _basename; }
+
+ /** Load images.
+ \return the loaded images
+ */
+ template
+ std::vector loadImages( void ) const;
+
+ /** Load images, applying an active images file filter.
+ \param ac the active list file
+ \return the loaded images
+ \note Non-active images are present but empty.
+ */
+ template
+ std::vector loadImages( const ActiveImageFile& ac) const;
+
+
+ private:
+ std::vector _infos; ///< Image infos.
+ std::string _basename; ///< Root name.
+
+ };
+
+ ///// DEFINITIONS /////
+
+
+ template
+ std::vector ImageListFile::loadImages( const ActiveImageFile& ac ) const {
+ std::vector out;
+
+ SIBR_LOG << "[ImageListFile] loading images";
+ out.resize(_infos.size());
+ if (_infos.empty() == false)
+ {
+ #pragma omp parallel for
+ for (int i = 0; i < _infos.size(); ++i)
+ if( ac.active()[i] )
+ out[i].load(_basename + "/" + _infos.at(i).filename, false);
+ }
+ else
+ SIBR_WRG << "cannot load images (ImageListFile is empty. Did you use ImageListFile::load(...) before ?";
+
+ std::cout << std::endl;
+ return out;
+ }
+
+ template
+ std::vector ImageListFile::loadImages( void ) const {
+ std::vector out;
+
+ std::cerr << "[ImageListFile] loading images";
+ out.resize(_infos.size());
+ if (_infos.empty() == false)
+ {
+ #pragma omp parallel for
+ for (int i = 0; i < _infos.size(); ++i)
+ out[i].load(_basename + "/" + _infos.at(i).filename, false);
+ }
+ else
+ SIBR_WRG << "cannot load images (ImageListFile is empty. Did you use ImageListFile::load(...) before ?";
+
+ return out;
+ }
+
+} // namespace sibr
diff --git a/submodules/gaussian-splatting/SIBR_viewers/src/core/assets/InputCamera.cpp b/submodules/gaussian-splatting/SIBR_viewers/src/core/assets/InputCamera.cpp
new file mode 100755
index 0000000000000000000000000000000000000000..97f07547f0de68a6688658078bed0a754e6126e6
--- /dev/null
+++ b/submodules/gaussian-splatting/SIBR_viewers/src/core/assets/InputCamera.cpp
@@ -0,0 +1,1577 @@
+/*
+ * Copyright (C) 2020, Inria
+ * GRAPHDECO research group, https://team.inria.fr/graphdeco
+ * All rights reserved.
+ *
+ * This software is free for non-commercial, research and evaluation use
+ * under the terms of the LICENSE.md file.
+ *
+ * For inquiries contact sibr@inria.fr and/or George.Drettakis@inria.fr
+ */
+
+
+#include "core/assets/ActiveImageFile.hpp"
+#include "core/assets/InputCamera.hpp"
+#include
+#include
+#include "core/system/String.hpp"
+#include "picojson/picojson.hpp"
+
+
+// Colmap binary stuff
+#include "colmapheader.h"
+typedef uint32_t image_t;
+typedef uint32_t camera_t;
+typedef uint64_t point3D_t;
+typedef uint32_t point2D_t;
+
+#define SIBR_INPUTCAMERA_BINARYFILE_VERSION 10
+#define IBRVIEW_TOPVIEW_SAVEVERSION "version002"
+#define FOCAL_X_UNDEFINED -1
+
+namespace sibr
+{
+ InputCamera::InputCamera(float f, float k1, float k2, int w, int h, int id) :
+ _focal(f), _k1(k1), _k2(k2), _w(w), _h(h), _id(id), _active(true), _name(""), _focalx(FOCAL_X_UNDEFINED)
+ {
+ // Update fov and aspect ratio.
+ float fov = 2.0f * atan(0.5f * h / f);
+ float aspect = float(w) / float(h);
+
+ Camera::aspect(aspect);
+ Camera::fovy(fov);
+
+ _id = id;
+ }
+
+ InputCamera::InputCamera(float fy, float fx, float k1, float k2, int w, int h, int id) :
+ _focal(fy), _k1(k1), _k2(k2), _w(w), _h(h), _id(id), _active(true), _name(""), _focalx(fx)
+ {
+ // Update fov and aspect ratio.
+ float fovY = 2.0f * atan(0.5f * h / fy);
+ float fovX = 2.0f * atan(0.5f * w / fx);
+
+ Camera::aspect(tan(fovX / 2) / tan(fovY / 2));
+ Camera::fovy(fovY);
+
+ _id = id;
+ }
+
+
+ InputCamera::InputCamera(int id, int w, int h, sibr::Matrix4f m, bool active) :
+ _active(active)
+ {
+ Vector3f t;
+ float r[9];
+
+ for (int i = 0; i < 9; i++) r[i] = m(3 + i);
+ for (int i = 0; i < 3; i++) t[i] = m(12 + i);
+
+ _w = w;
+ _h = h;
+
+ _focal = m(0);
+ _focalx = FOCAL_X_UNDEFINED;
+ _k1 = m(1);
+ _k2 = m(2);
+
+ float fov = 2.0f * atan(0.5f * h / m(0));
+ float aspect = float(w) / float(h);
+
+ sibr::Matrix3f matRotation;
+ matRotation <<
+ r[0], r[1], r[2],
+ r[3], r[4], r[5],
+ r[6], r[7], r[8]
+ ;
+
+ Camera::aspect(aspect);
+ Camera::fovy(fov);
+
+ // http://www.cs.cornell.edu/~snavely/bundler/bundler-v0.4-manual.html#S6
+ // Do pos = -R' * t
+ const sibr::Matrix3f orientation = matRotation.transpose();
+ sibr::Vector3f position = -orientation * t;
+ Camera::position(position);
+ Camera::rotation(Quaternionf(orientation));
+
+
+ Camera::principalPoint({ 0.5f, 0.5f });
+
+
+ _id = id;
+ _name = "";
+ }
+
+
+
+ InputCamera::InputCamera(int id, int w, int h, sibr::Vector3f& position, sibr::Matrix3f& orientation, float focal, float k1, float k2, bool active) :
+ _active(active)
+ {
+
+
+ _w = w;
+ _h = h;
+
+ _focal = focal;
+ _focalx = FOCAL_X_UNDEFINED;
+ _k1 = k1;
+ _k2 = k2;
+
+ float fov = 2.0f * atan(0.5f * h / _focal);
+ float aspect = float(w) / float(h);
+
+
+
+ Camera::aspect(aspect);
+ Camera::fovy(fov);
+
+ // http://www.cs.cornell.edu/~snavely/bundler/bundler-v0.4-manual.html#S6
+ // Do pos = -R' * t
+
+ Camera::position(position);
+ Camera::rotation(Quaternionf(orientation));
+
+ _id = id;
+ _name = "";
+ }
+
+ InputCamera::InputCamera(const Camera& c, int w, int h) : Camera(c) {
+ _focal = 1.0f / (tan(0.5f * fovy()) * 2.0f / float(h));
+ _focalx = FOCAL_X_UNDEFINED;
+ _k1 = _k2 = 0;
+ _w = w;
+ _h = h;
+ _id = 0;
+ _name = "";
+ _active = true;
+ aspect(float(_w) / float(_h));
+ }
+
+ // ------------------------------------------------------------------------
+
+ void InputCamera::size(uint w, uint h) { _w = w; _h = h; }
+ uint InputCamera::w(void) const { return _w; }
+ uint InputCamera::h(void) const { return _h; }
+ bool InputCamera::isActive(void) const { return _active; }
+
+ /* compatibility for preprocess (depth) */
+
+
+ Vector3f InputCamera::projectScreen(const Vector3f& pt) const {
+ Vector3f proj_pt = project(pt);
+ Vector3f screen_pt((proj_pt[0] + 1.f) * _w / 2.0f, (1.f - proj_pt[1]) * _h / 2.0f, proj_pt[2] * 0.5f + 0.5f);
+
+ return screen_pt;
+ }
+
+ float InputCamera::focal() const { return _focal; };
+ float InputCamera::focalx() const { return _focalx; };
+ float InputCamera::k1() const { return _k1; };
+ float InputCamera::k2() const { return _k2; };
+
+ InputCamera InputCamera::resizedH(int h) const {
+
+ int w = int(_aspect * h);
+
+ float sibr_focal = h * _focal / _h;
+ float k1 = _k1;
+ float k2 = _k2;
+ int id = _id;
+
+ sibr::Matrix4f m;
+
+ sibr::InputCamera cam(sibr_focal, k1, k2, w, h, id);
+
+ cam.rotation(rotation());
+ cam.position(position());
+
+ cam.znear(znear());
+ cam.zfar(zfar());
+ cam.name(name());
+
+ return cam;
+ }
+
+ InputCamera InputCamera::resizedW(int w) const {
+
+ int h = int(float(w) / _aspect);
+
+ float sibr_focal = h * _focal / _h;
+ float k1 = _k1;
+ float k2 = _k2;
+ int id = _id;
+
+ sibr::Matrix4f m;
+
+ sibr::InputCamera cam(sibr_focal, k1, k2, w, h, id);
+
+ cam.rotation(rotation());
+ cam.position(position());
+
+ cam.znear(znear());
+ cam.zfar(zfar());
+ cam.name(name());
+
+ return cam;
+ }
+
+
+
+ std::vector InputCamera::load(const std::string& datasetPath, float zNear, float zFar, const std::string& bundleName, const std::string& listName)
+ {
+ const std::string bundlerFile = datasetPath + "/cameras/" + bundleName;
+ const std::string listFile = datasetPath + "/images/" + listName;
+ const std::string clipFile = datasetPath + "/clipping_planes.txt";
+
+ // Loading clipping planes if they are available.
+ SIBR_LOG << "Loading clipping planes from " << clipFile << std::endl;
+
+ struct Z {
+ Z() {}
+ Z(float f, float n) : far(f), near(n) {}
+ float far;
+ float near;
+ };
+ std::vector nearsFars;
+
+ { // Load znear & zfar for unprojecting depth samples
+
+ float z;
+ std::ifstream zfile(clipFile);
+ // During preprocessing clipping planes are not yet defined
+ // the preprocess utility "depth" defines this
+ // SIBR_ASSERT(zfile.is_open());
+ if (zfile.is_open()) {
+ int num_z_values = 0;
+ while (zfile >> z) {
+ if (num_z_values % 2 == 0) {
+ nearsFars.push_back(Z());
+ nearsFars[nearsFars.size() - 1].near = z;
+ }
+ else {
+ nearsFars[nearsFars.size() - 1].far = z;
+ }
+ ++num_z_values;
+ }
+
+ if (num_z_values > 0 && num_z_values % 2 != 0) {
+ nearsFars.resize(nearsFars.size() - 1);
+ }
+
+ if (nearsFars.size() == 0) {
+ SIBR_WRG << " Could not extract at leat 2 clipping planes from '" << clipFile << "' ." << std::endl;
+ }
+ }
+ else {
+ SIBR_WRG << "Cannot open '" << clipFile << "' (not clipping plane loaded)." << std::endl;
+ }
+
+ }
+
+ // Load cameras and images infos.
+ SIBR_LOG << "Loading input cameras." << std::endl;
+ auto cameras = InputCamera::loadBundle(bundlerFile, zNear, zFar, listFile);
+
+ if (!nearsFars.empty()) {
+ for (int cid = 0; cid < cameras.size(); ++cid) {
+ const int zid = std::min(cid, int(nearsFars.size()) - 1);
+ cameras[cid]->znear(nearsFars[zid].near);
+ cameras[cid]->zfar(nearsFars[zid].far);
+ }
+ }
+
+ // Load active images
+ ActiveImageFile activeImageFile;
+ activeImageFile.setNumImages((int)cameras.size());
+ // load active image file and set (in)active
+ if (activeImageFile.load(datasetPath + "/active_images.txt", false)) {
+ for (int i = 0; i < (int)cameras.size(); i++) {
+ if (!activeImageFile.active()[i])
+ cameras[i]->setActive(false);
+ }
+ }
+
+ // Load excluded images
+ ActiveImageFile excludeImageFile;
+ excludeImageFile.setNumImages((int)cameras.size());
+ // load exclude image file and set *in*active
+ if (excludeImageFile.load(datasetPath + "/exclude_images.txt", false)) {
+ for (int i = 0; i < (int)cameras.size(); i++) {
+ // Attn (GD): invert the meaning of active for exclude:
+ // only file numbers explicitly in exclude_images are set
+ // to active, and these are the only ones we set to *inactive*
+ // should really create a separate class or have a flag "invert"
+ if (excludeImageFile.active()[i])
+ cameras[i]->setActive(false);
+ }
+ }
+ return cameras;
+ }
+
+ std::vector InputCamera::loadNVM(const std::string& nvmPath, float zNear, float zFar, std::vector wh)
+ {
+ std::ifstream in(nvmPath);
+ std::vector cameras;
+
+ if (in.is_open())
+ {
+ int rotation_parameter_num = 4;
+ bool format_r9t = false;
+ std::string token;
+ if (in.peek() == 'N')
+ {
+ in >> token; //file header
+ if (strstr(token.c_str(), "R9T"))
+ {
+ rotation_parameter_num = 9; //rotation as 3x3 matrix
+ format_r9t = true;
+ }
+ }
+
+ int ncam = 0, npoint = 0, nproj = 0;
+ // read # of cameras
+ in >> ncam; if (ncam <= 1) return std::vector();
+
+ //read the camera parameters
+
+ std::function