Render Model+States to Video#

This guide continues the content from Render Model+State to Image by rendering multiple images as a frame sequence that is then written to a video file. This can be extremely useful for linking model motion to data (you can composite the render with plots) and presentations.

Setup Example Data#

If you want to run the Python code in this guide, you must first set up the Example Data.

Setup PyAV#

In addition to the Core Software, this guide additionally requires installing PyAV, which enables encoding image data into common video file formats. You can install it from the command line similarly to how you installed the Core Software:

pip install av

Load Model + Resampled States#

For video encoding, it’s important to choose (and stick to) a particular framerate. The most common framerates are 24, 30, and 60 frames per second (FPS).

In most cases, data will not be available at those rates, so you’ll need to resample it. Here is an example of resampling an .mot file using pandas’ interpolation support (related: Load Data into DataFrames):

import numpy as np
import opynsim as opyn
import pandas as pd

# Read model
model = opyn.read_osim("pragmatic_resources/gait2354/subject01.osim").compile()

# Read, normalize (radians), and resample data frame @ 60 Hz
df = opyn.read_mot("pragmatic_resources/gait2354/OutputReference/subject01_walk1_ik.mot")
df = model.convert_data_frame_to_radians(df)
df = df.to_pandas()
df = df.set_index("time")
t_60hz = np.arange(df.index.min(), df.index.max(), 1/60.0)
df = (
    df.reindex(df.index.union(t_60hz))
        .interpolate(method="index")
        .loc[t_60hz]
)

# Read data frame into a `ModelStates` @ 60 Hz.
states = model.states_from_data_frame(opyn.DataFrame(df), realized_to=opyn.STAGE_REPORT)

Render Video Frames#

Once you have prepared the state(s) you would like to render, you can then set up a rendering loop where each iteration generates one frame of the video. This is a general software pattern that can be used to produce any animation - the only actual requirement is that you can render one frame for each time point.

Basic Example: Directly Rendering to Video#

This example renders each model state directly into to a video file (.mp4) with no compositing of extra information:

import av
import numpy as np
import opynsim as opyn
import opynsim.graphics
import pandas as pd

# Load and prepare model + states.
fps = 60
model = opyn.read_osim("pragmatic_resources/gait2354/subject01.osim").compile()
df = opyn.read_mot("pragmatic_resources/gait2354/OutputReference/subject01_walk1_ik.mot")
df = model.convert_data_frame_to_radians(df)
df = df.to_pandas()
df = df.set_index("time")
ts = np.arange(df.index.min(), df.index.max(), 1/fps)
df = df.reindex(df.index.union(ts)).interpolate(method="index").loc[ts]
states = model.states_from_data_frame(opyn.DataFrame(df), realized_to=opyn.STAGE_REPORT)

# Make output video container + stream.
w, h = (1920, 1080)
container = av.open("animation.mp4", mode="w")
stream = container.add_stream("h264", rate=fps)
stream.width = w
stream.height = h
stream.pix_fmt = "yuv420p"
stream.options = {"crf": "23"}  # quality/bitrate

# Render each state as a frame in the video.
camera = opyn.graphics.Camera()
camera.direction = np.array([-1, 0, 0])
camera.position = np.array([1.7, 1, 0])
options = {
    "dimensions":       (w, h),
    "camera":           camera,
    "background_color": opyn.graphics.Color.white,
    "scene_cache":      opyn.graphics.SceneCache(),  # performance
}
for state in states:
    texture = opyn.graphics.render_model_in_state(model, state, **options)
    frame = av.VideoFrame.from_ndarray(texture.pixels_rgb24(), format="rgb24")
    for packet in stream.encode(frame):
        container.mux(packet)

# Finalize video file
for packet in stream.encode():
    container.mux(packet)
container.close()

Advanced Example#

TODO: this needs to be updated from something I was playing around with.

#!/usr/bin/env python3

# dependencies (install these): imageio opynsim pandas pyarrow matplotlib

import opynsim as opyn            # musculoskeletal modelling
import opynsim.graphics           # musculoskeletal modelling (3D visualization)
import imageio                    # video encoding
import numpy as np                # numerics (unit conversion, pixel combining)
import pandas as pd               # data manipulation
import matplotlib.pyplot as plt   # 2D plotting/visualization

print("--- loading model + motion ---")
model = opyn.read_osim("InnoTreat_SSM.osim").compile()
mot = opyn.read_mot("Healthy_mean_head_right_rep_1.mot")
mot = model.convert_data_frame_to_radians(mot)  # Pandas doesn't keep track of MOT files' "inDegrees"

print("--- resampling motion with pandas at 60FPS for video rendering ---")
df = mot.to_pandas()
df = df.set_index(pd.to_timedelta(df.pop("time"), unit="s"))
df = df.resample(pd.Timedelta(seconds=1/60)).mean().interpolate()
df.index = df.index.total_seconds()
#df = df.loc[:1.0]  # take the first second <-------- USE THIS WHEN DEVELOPING (faster)

print("--- converting motion data into a ModelState series ---")
states = model.states_from_data_frame(opyn.DataFrame(df), realized_to=opyn.STAGE_REPORT)

print("--- collecting relevant output data from states ---")
ts = []
ld_forces = []
ds_forces = []
se_angle = []
for state in states:
    ts.append(state.time)
    ld_forces.append(model.get_output_value(state, "/forceset/LatissimusDorsi_S[fiber_force]"))
    ds_forces.append(model.get_output_value(state, "/forceset/DeltoideusSpinae_med[fiber_force]"))
    val_rad = model.get_output_value(state, "/jointset/GlenoHumeral/shoulder_elv[value]")
    se_angle.append(np.degrees(val_rad))

print("--- creating matplotlib plots of the data ---")
fig, (ax1, ax2, ax3) = plt.subplots(3, 1, figsize=(4.5, 4.8), dpi=100, sharex=True)

ax1.plot(ts, ld_forces, color="royalblue", linewidth=2, label="LD Force")
ax1.set_ylabel("Force (N)")
ax1.set_title("Latissimus Dorsi Fiber Force", fontsize=10)
ax1.grid(True, linestyle="--", alpha=0.6)

ax2.plot(ts, ds_forces, color="crimson", linewidth=2, label="DS Force")
ax2.set_ylabel("Force (N)")
ax2.set_title("Deltoideus Spinae Fiber Force", fontsize=10)
ax2.grid(True, linestyle="--", alpha=0.6)

ax3.plot(ts, se_angle, color="forestgreen", linewidth=2, label="Elevation")
ax3.set_xlabel("Time (s)")  # Bottom plot has the X (rest are aligned by sharex=True)
ax3.set_ylabel("Angle (°)")
ax3.set_title("Shoulder Elevation Angle", fontsize=10)
ax3.grid(True, linestyle="--", alpha=0.6)

# Create dynamic vertical "time cursor" lines for all 3 axes
cursor1 = ax1.axvline(x=ts[0], color='black', linestyle='--', linewidth=1.5)
cursor2 = ax2.axvline(x=ts[0], color='black', linestyle='--', linewidth=1.5)
cursor3 = ax3.axvline(x=ts[0], color='black', linestyle='--', linewidth=1.5)

# Tighten spacing layout adjustments (slightly tucked the margins to fit 3 titles comfortably)
plt.subplots_adjust(left=0.18, right=0.95, top=0.94, bottom=0.10, hspace=0.45)

print("--- rendering video frames ---")

# Figure out initial camera polar coordinates (as an example).
initial_pos = np.array([0.05, -0.1, 0.8])
camera_target = np.array([0.0, 0.0, 0.0])
radius = np.sqrt(initial_pos[0]**2 + initial_pos[2]**2)
initial_theta = np.arctan2(initial_pos[2], initial_pos[0])
angle_per_frame = (2. * np.pi) / len(states)

# Initialize a scene camera (updated every frame)
camera = opyn.graphics.Camera()
rendering_params = {
    "scene_cache": opyn.graphics.SceneCache(),
    "background_color": opyn.graphics.Color.white,
    "camera": camera,
    "dimensions": (300, 480),
}
with imageio.get_writer("output.mp4", fps=60) as writer:
    for idx, state in enumerate(states):
        print(f"t={state.time}")

        theta = initial_theta + (idx * angle_per_frame)

        # Update camera
        camera.position = np.array([radius * np.cos(theta), initial_pos[1], radius * np.sin(theta)])
        camera.direction = camera_target - camera.position
        camera.direction = camera.direction / np.linalg.norm(camera.direction)  # normalize
        camera.up = np.array([0.0, 1.0, 0.0])

        # Render 3D scene to pixels.
        frame = opyn.graphics.render_model_in_state(model, state, **rendering_params)
        render_pixels = frame.pixels_rgb24()

        # Update X line on plots, then redraw the plot so the lines are shown.
        cursor1.set_xdata([state.time])
        cursor2.set_xdata([state.time])
        cursor3.set_xdata([state.time])
        fig.canvas.draw()

        # Extract the plot's pixels as RGB
        plot_pixels = np.asarray(fig.canvas.buffer_rgba())[..., :3]

        # Composite the raw pixel data in-memory using numpy
        composite_frame = np.hstack((render_pixels, plot_pixels))

        # Write the composite to the video file
        writer.append_data(composite_frame)

Render Model+States to Video

Contents