ika-rwth-aachen/Cam2BEV

visualization

KimJunHan opened this issue · 3 comments

KimJunHan commented

How to visualization like your github website

like

image

let me know the step by step plz

lreiher commented

Here is a script that we have used to create this and similar visualizations from individual image files. Note that the script depends on ffmpeg. The script should be pretty self-explanatory.

mosaic

Usage

usage: video-mosaic.py [-h] [--prediction PREDICTION] [-c {4,6}] [-r R] [-b B]
                       folder output

Creates a mosaic of camera images, drone images, and network prediction.

positional arguments:
  folder                directory with subfolders 'front', 'homo', ...
  output                output directory for video frames and video

optional arguments:
  -h, --help            show this help message and exit
  --prediction PREDICTION
                        folder name in FOLDER containing the predictions
  -c {4,6}              number of cameras
  -r R                  video fps [Hz]
  -b B                  border strength [px]

Code

#!/usr/bin/env python

import os
import argparse
import tqdm
import subprocess
import numpy as np
import cv2


MAIN_IMG_SHAPE = (256, 512)


# parse command line arguments
parser = argparse.ArgumentParser(description="Creates a mosaic of camera images, drone images, and network prediction.")
parser.add_argument("folder", type=str, help="directory with subfolders 'front', 'homo', ...")
parser.add_argument("output", type=str, help="output directory for video frames and video")
parser.add_argument("--prediction", type=str, help="folder name in FOLDER containing the predictions", default="prediction")
parser.add_argument("-c", type=int, help="number of cameras", choices=(4,6), default=4)
parser.add_argument("-r", type=int, help="video fps [Hz]", default=20)
parser.add_argument("-b", type=int, help="border strength [px]", default=4)
args = parser.parse_args()

base = os.path.abspath(os.path.expanduser(args.folder))
output = os.path.abspath(os.path.expanduser(args.output))
n_cams = args.c
fps = args.r


# define subfolders
folder = {}
if n_cams == 4:
    folder["front"]         = os.path.join(base, "front")
    folder["rear"]          = os.path.join(base, "rear")
    folder["left"]          = os.path.join(base, "left")
    folder["right"]         = os.path.join(base, "right")
    folder["front_cc"]      = os.path.join(base, "front_cc")
    folder["rear_cc"]       = os.path.join(base, "rear_cc")
    folder["left_cc"]       = os.path.join(base, "left_cc")
    folder["right_cc"]      = os.path.join(base, "right_cc")
elif n_cams == 6:
    folder["front"]             = os.path.join(base, "front")
    folder["rear"]              = os.path.join(base, "rear")
    folder["front_left"]        = os.path.join(base, "front_left")
    folder["front_right"]       = os.path.join(base, "front_right")
    folder["rear_left"]         = os.path.join(base, "rear_left")
    folder["rear_right"]        = os.path.join(base, "rear_right")
    folder["front_cc"]          = os.path.join(base, "front_cc")
    folder["rear_cc"]           = os.path.join(base, "rear_cc")
    folder["front_left_cc"]     = os.path.join(base, "front_left_cc")
    folder["front_right_cc"]    = os.path.join(base, "front_right_cc")
    folder["rear_left_cc"]      = os.path.join(base, "rear_left_cc")
    folder["rear_right_cc"]     = os.path.join(base, "rear_right_cc")
folder["homo"]          = os.path.join(base, "homo")
folder["drone"]         = os.path.join(base, "drone")
folder["prediction"]    = os.path.join(base, args.prediction)
if not os.path.exists(output):
    os.makedirs(output)


# get filenames
files =  {}
for key, fol in folder.items():
    files[key] = sorted([os.path.join(fol, f) for f in os.listdir(fol)])
n = len(files["prediction"])


# determine mosaic image shapes
BORDER = args.b
SHAPE1 = np.array(MAIN_IMG_SHAPE, dtype=np.uint32)
SHAPE2 = np.array(((SHAPE1[0] - BORDER) / 2, (SHAPE1[0] - BORDER)), dtype=np.uint32)
SHAPE3 = np.array(((SHAPE1[0] - 3 * BORDER) / 4, (SHAPE1[0] - 3 * BORDER) / 2), dtype=np.uint32)
MOSAIC_SHAPE = np.array((SHAPE1[0] + BORDER + SHAPE2[0], SHAPE1[1] + BORDER + SHAPE2[1]), dtype=np.uint32)
# aliases
b = BORDER
h1, w1 = SHAPE1
h2, w2 = SHAPE2
h3, w3 = SHAPE3


# define helper functions
def load_image(filename):
    img = cv2.imread(filename)
    return img

def resize_image(img, shape, interpolation=cv2.INTER_CUBIC):

    # resize relevant image axis to length of corresponding target axis while preserving aspect ratio
    axis = 0 if float(shape[0]) / float(img.shape[0]) > float(shape[1]) / float(img.shape[1]) else 1
    factor = float(shape[axis]) / float(img.shape[axis])
    img = cv2.resize(img, (0,0), fx=factor, fy=factor, interpolation=interpolation)

    # crop other image axis to match target shape
    center = img.shape[int(not axis)] / 2.0
    step = shape[int(not axis)] / 2.0
    left = int(center-step)
    right = int(center+step)
    if axis == 0:
        img = img[:, left:right]
    else:
        img = img[left:right, :]

    return img


# process all samples
for i in tqdm.tqdm(range(n)):

    # init mosaic
    mosaic = np.zeros((MOSAIC_SHAPE[0], MOSAIC_SHAPE[1], 3))

    # load images

    prediction = load_image(files["prediction"][i])
    if n_cams == 6:
        tmp = np.zeros((SHAPE1[0], SHAPE1[1], 3))
        tmp[int((h1 - prediction.shape[0]) / 2) : int((h1 + prediction.shape[0]) / 2), :] = prediction
        prediction = tmp
    prediction = resize_image(prediction, SHAPE1)

    drone = load_image(files["drone"][i])
    drone = resize_image(drone, SHAPE1)

    blend = 0.5 * prediction + 0.5 * drone
    prediction = resize_image(prediction, SHAPE2)
    drone = resize_image(drone, SHAPE2)

    homo = load_image(files["homo"][i])
    homo = resize_image(homo, SHAPE2)

    if n_cams == 4:

        front = load_image(files["front"][i])
        front = resize_image(front, SHAPE3)

        rear = load_image(files["rear"][i])
        rear = resize_image(rear, SHAPE3)

        left = load_image(files["left"][i])
        left = resize_image(left, SHAPE3)

        right = load_image(files["right"][i])
        right = resize_image(right, SHAPE3)

        front_cc = load_image(files["front_cc"][i])
        front_cc = resize_image(front_cc, SHAPE3)

        rear_cc = load_image(files["rear_cc"][i])
        rear_cc = resize_image(rear_cc, SHAPE3)

        left_cc = load_image(files["left_cc"][i])
        left_cc = resize_image(left_cc, SHAPE3)

        right_cc = load_image(files["right_cc"][i])
        right_cc = resize_image(right_cc, SHAPE3)

    elif n_cams == 6:

        front = load_image(files["front"][i])
        front = resize_image(front, SHAPE3)

        rear = load_image(files["rear"][i])
        rear = resize_image(rear, SHAPE3)

        front_left = load_image(files["front_left"][i])
        front_left = resize_image(front_left, SHAPE3)

        front_right = load_image(files["front_right"][i])
        front_right = resize_image(front_right, SHAPE3)

        rear_left = load_image(files["rear_left"][i])
        rear_left = resize_image(rear_left, SHAPE3)

        rear_right = load_image(files["rear_right"][i])
        rear_right = resize_image(rear_right, SHAPE3)


    # fill mosaic
    mosaic[               : h2         ,                 : w2                 ] = homo
    mosaic[    h2 + b     : 2 * h2 + b ,                 : w2                 ] = drone
    mosaic[2 * h2 + 2 * b :            ,                 : w2                 ] = prediction
    mosaic[    h2 + b     :            , w2 + b          :                    ] = blend
    mosaic[0              : h3         , w2 + b          : w2 + b + w3        ] = front     if n_cams == 4 else 0
    mosaic[0              : h3         , w2 + 2 * b + w3 : w2 + 2 * b + 2 * w3] = rear      if n_cams == 4 else front
    mosaic[h3 + b         : 2 * h3 + b , w2 + b          : w2 + b + w3        ] = left      if n_cams == 4 else rear_left
    mosaic[h3 + b         : 2 * h3 + b , w2 + 2 * b + w3 : w2 + 2 * b + 2 * w3] = right     if n_cams == 4 else front_left
    mosaic[0              : h3         , w2 + 3 * b + 2 * w3 : w2 + 3*b + 3*w3] = front_cc  if n_cams == 4 else rear
    mosaic[0              : h3         , w2 + 4 * b + 3 * w3 : w2 + 4*b + 4*w3] = rear_cc   if n_cams == 4 else 0
    mosaic[h3 + b         : 2 * h3 + b , w2 + 3 * b + 2 * w3 : w2 + 3*b + 3*w3] = left_cc   if n_cams == 4 else front_right
    mosaic[h3 + b         : 2 * h3 + b , w2 + 4 * b + 3 * w3 : w2 + 4*b + 4*w3] = right_cc  if n_cams == 4 else rear_right

    # include filename as caption
    caption = os.path.splitext(os.path.basename(files["prediction"][i]))[0]
    cv2.putText(mosaic, caption, (w2 + b + 5, mosaic.shape[0] - 5), 
        fontFace=cv2.FONT_HERSHEY_SIMPLEX, 
        fontScale=0.25,
        color=(255, 255, 255),
        thickness=1)

    # save mosaic frame
    cv2.imwrite(os.path.join(output, f"{caption}.png"), mosaic)


# run ffmpeg
ffmpeg = subprocess.Popen(["ffmpeg",
    "-framerate", f"{fps}",
    "-pattern_type", "glob",
    "-i", f"\"{os.path.join(output, '*.png')}\"",
    "-c:v", "libx264",
    "-profile:v", "high",
    "-crf", "20",
    "-pix_fmt", "yuv420p",
    os.path.join(output, f"mosaic.mp4")]
)
KimJunHan commented
lreiher commented

Yes, the input folder needs to have subdirectories front, homo, ..., where the images are stored. As I said, the script is self-explanatory, we cannot provide further support beyond this point.