LTX Video generation

cd src/maxdiffusion/models/ltx_video/utils
python convert_torch_weights_to_jax.py \
  --ckpt_path /path/to/weights \
  --transformer_config_path ../ltxv-13B.json

conditioning_media_paths: ["/path/to/image.jpg"]
conditioning_start_frames: [0]
conditioning_strengths: [1.0]

python src/maxdiffusion/generate_ltx_video.py \
  src/maxdiffusion/configs/ltx_video.yml \
  output_dir="/path/to/weights" \
  config_path="src/maxdiffusion/models/ltx_video/ltxv-13B.json"

prompt_enhancement_words_threshold: 20

prompt_enhancement_words_threshold: 0

height_padded = ((config.height - 1) // 32 + 1) * 32
width_padded = ((config.width - 1) // 32 + 1) * 32
num_frames_padded = ((config.num_frames - 2) // 8 + 1) * 8 + 1
padding = calculate_padding(config.height, config.width, height_padded, width_padded)

pipeline_type: "multi-scale"

def prepare_conditioning(
    conditioning_media_paths: List[str],
    conditioning_strengths: List[float],
    conditioning_start_frames: List[int],
    height: int,
    width: int,
    padding: tuple[int, int, int, int],
) -> Optional[List[ConditioningItem]]:
  conditioning_items = []
  for path, strength, start_frame in zip(conditioning_media_paths, conditioning_strengths, conditioning_start_frames):
    media_tensor = load_media_file(
        media_path=path,
        height=height,
        width=width,
        max_frames=1,
        padding=padding,
        just_crop=True,
    )
    conditioning_items.append(ConditioningItem(media_tensor, start_frame, strength))
  return conditioning_items

def load_image_to_tensor_with_resize_and_crop(
    image_input: Union[str, Image.Image],
    target_height: int = 512,
    target_width: int = 768,
    just_crop: bool = False,
) -> torch.Tensor:
  # Load image
  if isinstance(image_input, str):
    image = Image.open(image_input).convert("RGB")
  else:
    image = image_input
  
  # Aspect ratio crop
  aspect_ratio_target = target_width / target_height
  aspect_ratio_frame = input_width / input_height
  # ... crop logic ...
  
  # Optional resize
  if not just_crop:
    image = image.resize((target_width, target_height))
  
  # Convert to tensor and apply Gaussian blur
  frame_tensor = TVF.to_tensor(image)
  frame_tensor = TVF.gaussian_blur(frame_tensor, kernel_size=3, sigma=1.0)
  
  # CRF compression simulation
  frame_tensor_hwc = frame_tensor.permute(1, 2, 0)
  frame_tensor_hwc = crf_compressor.compress(frame_tensor_hwc)
  
  # Normalize to [-1, 1]
  frame_tensor = frame_tensor_hwc.permute(2, 0, 1) * 255.0
  frame_tensor = (frame_tensor / 127.5) - 1.0
  
  return frame_tensor.unsqueeze(0).unsqueeze(2)  # (B, C, F, H, W)

images = pipeline(
    height=height_padded,
    width=width_padded,
    num_frames=num_frames_padded,
    is_video=True,
    output_type="pt",
    config=config,
    enhance_prompt=enhance_prompt,
    conditioning_items=conditioning_items,
    seed=config.seed,
)

# Remove padding
(pad_left, pad_right, pad_top, pad_bottom) = padding
images = images[:, :, :config.num_frames, pad_top:pad_bottom, pad_left:pad_right]

# Convert to numpy and save
for i in range(images.shape[0]):
  video_np = images[i].permute(1, 2, 3, 0).detach().float().numpy()
  video_np = (video_np * 255).astype(np.uint8)
  
  if video_np.shape[0] == 1:
    # Save as image
    imageio.imwrite(output_filename, video_np[0])
  else:
    # Save as video
    with imageio.get_writer(output_filename, fps=fps) as video:
      for frame in video_np:
        video.append_data(frame)