Large Scale Datasets¶
In this tutorial we will build up a CLI tool to create a large dataset from many blender scenes. Specifically, for each scene we will uniformly sample multiple sub-trajectories (making sure none overlap) and, for each, we’ll render the ground truth RGB, depth maps, etc. The final product will look something like this:
Note
For brevity, there’s a few things that have been omitted in this tutorial, for the full source, see scripts/mkdataset.py. Notably, extra dependencies may be needed.
Here, we assume each scene is setup correctly and has an animation range of [1-600]. The scenes used for this example can be found here, and the final dataset can be is available here.
For clarity, we’ll refer to a single blend-file as a scene, and a sequence will refer to a rendered portion of a scene. So if we use sequences_per_scene=10 and we’re rendering from 20 scenes, we will have 200 sequences which will be saved roughly like so:
DATASETS-DIR
└── renders
├── SCENE-NAME
│ ├── SEQUENCE-ID
│ │ ├── frames/
│ │ ├── depths/
│ │ ├── normals/
│ │ ├── segmentations/
│ │ ├── flows/
│ │ └── transforms.json
│ ├── SEQUENCE-ID/...
│ └── ...
└── SCENE-NAME/...
See also
For more about the dataset schema see Data Format and Loading.
TODO
This example is currently missing interpolation or sensor emulation, and will be extended soon.
To enable easy configuration and CLI parsing, we re-use the render configuration class used in the render-animation CLI which stores all important parameters such as render device, dimensions, and types of ground truth to use:
@dataclass
class RenderConfig:
executable: Path | None = None
"""Path to blender executable"""
height: int | None = None
"""Height of rendered frames"""
width: int | None = None
"""Width of rendered frames"""
include_composites: bool = False
"""If true, enable composited outputs"""
composites: CompositesConfig = field(default_factory=CompositesConfig)
"""Composited frames configuration options"""
include_frames: bool = True
"""If true, enable ground truth frame outputs"""
frames: FramesConfig = field(default_factory=FramesConfig)
"""Ground truth frames configuration options"""
include_depths: bool = False
"""If true, enable depth map outputs"""
depths: DepthsConfig = field(default_factory=DepthsConfig)
"""Depth maps configuration options"""
include_normals: bool = False
"""If true, enable normal map outputs"""
normals: NormalsConfig = field(default_factory=NormalsConfig)
"""Normal maps configuration options"""
include_flows: bool = False
"""If true, enable optical flow outputs"""
flows: FlowsConfig = field(default_factory=FlowsConfig)
"""Optical flow configuration options"""
include_segmentations: bool = False
"""If true, enable segmentation map outputs"""
segmentations: SegmentationsConfig = field(default_factory=SegmentationsConfig)
"""Segmentation maps configuration options"""
include_materials: bool = False
"""If true, enable material map outputs"""
materials: MaterialsConfig = field(default_factory=MaterialsConfig)
"""Material maps configuration options"""
include_diffuse_pass: bool = False
"""If true, enable diffuse light pass outputs"""
diffuse_pass: DiffusePassConfig = field(default_factory=DiffusePassConfig)
"""Diffuse light passes configuration options"""
include_specular_pass: bool = False
"""If true, enable specular light pass outputs"""
specular_pass: SpecularPassConfig = field(default_factory=SpecularPassConfig)
"""Specular light passes configuration options"""
include_points: bool = False
"""If true, enable world-space point map outputs"""
points: PointsConfig = field(default_factory=PointsConfig)
"""Point maps configuration options"""
include_all: bool = False
"""If true, enable all ground truth outputs"""
previews: bool = True
"""If false, disable all preview visualizations of auxiliary outputs"""
keyframe_multiplier: float = 1.0
"""Stretch keyframes by this amount, eg: 2.0 will slow down time"""
timeout: int = -1
"""Maximum allowed time in seconds to wait to connect to render instance"""
autoexec: bool = True
"""If true, allow python execution of embedded scripts (warning: potentially dangerous)"""
device_type: Literal["cpu", "cuda", "optix", "metal"] = "optix"
"""Name of device to use, one of "cpu", "cuda", "optix", "metal", etc"""
adaptive_threshold: float = 0.05
"""Noise threshold of rendered images, for higher quality frames make this threshold smaller.
The default value is intentionally a little high to speed up renders"""
max_samples: int = 256
"""Maximum number of samples per pixel to take"""
use_denoising: bool = True
"""If enabled, a denoising pass will be used"""
log_dir: Path = Path("logs/")
"""Directory to use for logging"""
allow_skips: bool = True
"""If true, skip rendering a frame if it already exists"""
unbind_camera: bool = False
"""Free the camera from it's parents, any constraints and animations it may have.
Ensures it uses the world's coordinate frame and the provided camera trajectory"""
use_animations: bool = True
"""Allow any animations to play out, if false, scene will be static"""
use_motion_blur: bool | None = None
"""Enable realistic motion blur. cannot be used if also rendering optical flow"""
addons: list[str] | None = None
"""List of extra addons to enable"""
jobs: int = 1
"""Number of concurrent render jobs"""
autoscale: bool = False
"""Set number of jobs automatically based on available VRAM and `max_job_vram` when enabled"""
max_job_vram: MemSize | None = None
"""Maximum allowable VRAM per job in bytes (limit is not enforced, simply used for `autoscale`)"""
def __post_init__(self):
# Note: Using post init with tyro is not best practice, as it will be called multiple
# times. However here we are just propagating values of aliases, so it should be ok.
# See: https://brentyi.github.io/tyro/examples/overriding_configs/#dataclasses-defaults
if self.include_all:
self.include_composites = True
self.include_frames = True
self.include_depths = True
self.include_normals = True
self.include_flows = True
self.include_segmentations = True
self.include_materials = True
self.include_diffuse_pass = True
self.include_specular_pass = True
self.include_points = True
self.depths.preview &= self.previews
self.normals.preview &= self.previews
self.flows.preview &= self.previews
self.segmentations.preview &= self.previews
self.materials.preview &= self.previews
self.points.preview &= self.previews
We can then reuse the same render-job as render-animation, except we’ll use it in conjunction with a BlenderClients.pool in order to render multiple scenes at once (as opposed to a single scene being rendered with multiple jobs). Putting it all together we have:
@app.command
def create_datasets(
scenes_dir: str | os.PathLike,
datasets_dir: str | os.PathLike,
render_config: RenderConfig,
sequences_per_scene: int = 1,
num_frames: int | None = None,
allow_skips: bool = False,
dry_run: bool = False,
):
"""Create datasets by rendering out sequences from many blend-files.
Args:
scenes_dir (str | os.PathLike): Directory to search for blend files in (includes sub-directories 1-level deep).
Every scene is assumed to be animated between frames 1-600.
datasets_dir (str | os.PathLike): Dataset output folder, ground truth renders will be saved in
`<datasets_dir>/renders/<scene_name>/<sequence_id>` where `scene_name` is the stem of the blender file (filename
without extension), and `sequence_id` is defined as `<keyframe_multiplier>-<frame_start>-<frame_start+num_frames>`.
render_config (RenderConfig): Render configuration.
sequences_per_scene (int, optional): Number of sequences per scene to render. The start of each sequence is sampled
uniformly from the animation range [1, 600].
num_frames (int | None, optional): Number of frames to render per sequence. If None, render everything.
allow_skips (bool, optional): If true, allow skipping over whole sequences if their corresponding root directory exists.
dry_run (bool, optional): if true, nothing will be rendered at all.
"""
# Sample sequences and validate args.
frame_starts, num_frames = sample_scenes(render_config, sequences_per_scene, num_frames)
# Find all sequences, i.e: pairs of blend-files and frame ranges (start, start+num-frames)
scenes = find_blends(scenes_dir)
sequences = list(itertools.product(scenes, frame_starts))
log.info(f"Generating {len(sequences)} sequences from {len(scenes)} scenes...")
# Note: Shuffling the sequences helps with throughput as different scenes are rendered at once
random.seed(123456789)
random.shuffle(sequences)
# Define helper to map each sequence to a unique path
def get_sequence_dir(scene_name, frame_start):
Path(datasets_dir).mkdir(parents=True, exist_ok=True)
sequence_id = f"{int(render_config.keyframe_multiplier):03}-{frame_start:05}-{frame_start + num_frames:05}"
sequence_dir = Path(datasets_dir) / "renders" / scene_name / sequence_id
return sequence_dir.resolve()
# Queue up and run all render tasks
with (
BlenderClients.pool(
jobs=render_config.jobs,
log=Path(render_config.log_dir),
timeout=render_config.timeout,
executable=render_config.executable,
autoexec=render_config.autoexec,
) as pool,
PoolProgress() as progress,
):
for blend_file, frame_start in sequences:
sequence_dir = get_sequence_dir(blend_file.stem, frame_start)
if not allow_skips or not (sequence_dir / "transforms.json").exists():
# Note: The client will be automagically passed to `render` here.
tick = progress.add_task(f"{blend_file.stem} ({frame_start}-{frame_start + num_frames})")
pool.apply_async(
render_job,
args=(blend_file, sequence_dir),
kwds=dict(
frame_start=frame_start,
frame_end=frame_start + num_frames,
config=render_config,
dry_run=dry_run,
update_fn=tick,
),
)
else:
log.info(f"Skipping: {sequence_dir}")
progress.wait()
pool.close()
pool.join()
# Gather some metadata about every sequence and save it to a "info.json" file.
with multiprocess.Pool(render_config.jobs) as pool:
info_fn = partial(sequence_info, keyframe_multiplier=render_config.keyframe_multiplier)
sequence_dirs = [get_sequence_dir(blend_file.stem, frame_start) for blend_file, frame_start in sequences]
list(pool.imap(info_fn, track(sequence_dirs, description="Gathering Metadata...")))
This CLI can be used, for instance, like so:
CUDA_VISIBLE_DEVICES=0 python scripts/mkdataset.py create-datasets \
--scenes-dir=scenes/ --datasets-dir=datasets/ --sequences-per-scene=1 \
--render-config.width=800 --render-config.height=800 \
--render-config.depths --render-config.normals \
--render-config.flows --render-config.segmentations \
--render-config.keyframe-multiplier=2.0 --render-config.jobs=5
Which will render the dataset show above at a framerate of 100fps (original fps of 50 time 2x keyframe multiplier).