透视扭曲

概述

透视扭曲 示例应用程序接受一个输入视频，并输出一个视频，其中每一帧都应用了不同的透视扭曲。 结果是透视弹跳效果。 可以修改示例应用程序以从相机获取输入并实时应用效果。

说明

命令行参数为

<backend> <input video>

其中

• backend: 可以是 cpu、cuda 或 vic (仅在 Jetson 设备上)。 它定义了将执行处理的后端。
• input video: 应用效果的视频文件；它接受 .mp4、.avi 以及可能取决于 OpenCV 支持的其他格式。

VPI 示例安装程序包含一些可以用作输入的示例视频。 它们位于 /opt/nvidia/vpi3/samples/assets/ 目录中。

这是一个调用示例

• C++

  ./vpi_sample_10_perspwarp cuda ../assets/noisy.mp4

• Python

  python3 main.py cuda ../assets/noisy.mp4

应用程序将处理 noisy.mp4 并创建 perspwarp_cuda.mp4，其中时间变化的透视扭曲应用于输入帧。

结果

输入视频	透视效果

源代码

为了方便起见，这里提供了也安装在示例目录中的代码。

语言 C++ Python

import sys
import vpi
import numpy as np
from math import sin, cos, pi
from argparse import ArgumentParser
import cv2


# ----------------------------
# Parse command line arguments

parser = ArgumentParser()
parser.add_argument('backend', choices=['cpu', 'cuda','vic'],
 help='Backend to be used for processing')

parser.add_argument('input',
 help='Input video to be denoised')

args = parser.parse_args();

if args.backend == 'cuda'
 backend = vpi.Backend.CUDA
elif args.backend == 'cpu'
 backend = vpi.Backend.CPU
else
 assert args.backend == 'vic'
 backend = vpi.Backend.VIC

# -----------------------------
# Open input and output videos

inVideo = cv2.VideoCapture(args.input)

fourcc = cv2.VideoWriter_fourcc(*'MPEG')
inSize = (int(inVideo.get(cv2.CAP_PROP_FRAME_WIDTH)), int(inVideo.get(cv2.CAP_PROP_FRAME_HEIGHT)))
fps = inVideo.get(cv2.CAP_PROP_FPS)

outVideo = cv2.VideoWriter('perspwarp_python'+str(sys.version_info[0])+'_'+args.backend+'.mp4',
 fourcc, fps, inSize)

#--------------------------------------------------------------
# Main processing loop
curFrame = 1
while True
 print("Frame: {}".format(curFrame))
 curFrame+=1

 # Read one input frame
 ret, cvFrame = inVideo.read()
 if not ret
 break

 # Convert it to NV12_ER format to be used by VPI
 with vpi.Backend.CUDA
 frame = vpi.asimage(cvFrame).convert(vpi.Format.NV12_ER)

 # Calculate the transformation to be applied ------------

 # Move image's center to origin of coordinate system
 T1 = np.array([[1, 0, -frame.width/2.0],
 [0, 1, -frame.height/2.0],
 [0, 0, 1]])

 # Apply some time-dependent perspective transform
 v1 = sin(curFrame/30.0*2*pi/2)*0.0005
 v2 = cos(curFrame/30.0*2*pi/3)*0.0005
 P = np.array([[0.66, 0, 0],
 [0, 0.66, 0],
 [v1, v2, 1]])

 # Move image's center back to where it was
 T2 = np.array([[1, 0, frame.width/2.0],
 [0, 1, frame.height/2.0],
 [0, 0, 1]])

 # Do perspective warp using the backend passed in the command line.
 with backend
 frame = frame.perspwarp(np.matmul(T2, np.matmul(P, T1)))

 # Convert it to RGB8 for output using the CUDA backend
 with vpi.Backend.CUDA
 frame = frame.convert(vpi.Format.RGB8)

 # Write the denoised frame to the output video
 with frame.rlock_cpu() as data
 outVideo.write(data)

#include <opencv2/core/version.hpp>
#include <opencv2/imgcodecs.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include <opencv2/videoio.hpp>
#include <vpi/OpenCVInterop.hpp>

#include <vpi/Image.h>
#include <vpi/Status.h>
#include <vpi/Stream.h>
#include <vpi/algo/ConvertImageFormat.h>
#include <vpi/algo/PerspectiveWarp.h>

#include <algorithm>
#include <cstring> // for memset
#include <fstream>
#include <iostream>
#include <random>
#include <sstream>

#define CHECK_STATUS(STMT) \
 do \
 { \
 VPIStatus status = (STMT); \
 if (status != VPI_SUCCESS) \
 { \
 char buffer[VPI_MAX_STATUS_MESSAGE_LENGTH]; \
 vpiGetLastStatusMessage(buffer, sizeof(buffer)); \
 std::ostringstream ss; \
 ss << vpiStatusGetName(status) << ": " << buffer; \
 throw std::runtime_error(ss.str()); \
 } \
 } while (0);

static void MatrixMultiply(VPIPerspectiveTransform &r, const VPIPerspectiveTransform &a,
 const VPIPerspectiveTransform &b)
{
 for (int i = 0; i < 3; ++i)
 {
 for (int j = 0; j < 3; ++j)
 {
 r[i][j] = a[i][0] * b[0][j];
 for (int k = 1; k < 3; ++k)
 {
 r[i][j] += a[i][k] * b[k][j];
 }
 }
 }
}

int main(int argc, char *argv[])
{
 // OpenCV image that will be wrapped by a VPIImage.
 // Define it here so that it's destroyed *after* wrapper is destroyed
 cv::Mat cvFrame;

 // VPI objects that will be used
 VPIStream stream = NULL;
 VPIImage imgInput = NULL, imgOutput = NULL;
 VPIImage frameBGR = NULL;

 int retval = 0;

 try
 {
 // =============================
 // Parse command line parameters

 if (argc != 3)
 {
 throw std::runtime_error(std::string("Usage: ") + argv[0] + " <cpu|vic|cuda> <input_video>");
 }

 std::string strBackend = argv[1];
 std::string strInputVideo = argv[2];

 // Now parse the backend
 VPIBackend backend;

 if (strBackend == "cpu")
 {
 backend = VPI_BACKEND_CPU;
 }
 else if (strBackend == "cuda")
 {
 backend = VPI_BACKEND_CUDA;
 }
 else if (strBackend == "vic")
 {
 backend = VPI_BACKEND_VIC;
 }
 else
 {
 throw std::runtime_error("Backend '" + strBackend +
 "' not recognized, it must be either cpu, cuda or vic.");
 }

 // ===============================
 // Prepare input and output videos

 // Load the input video
 cv::VideoCapture invid;
 if (!invid.open(strInputVideo))
 {
 throw std::runtime_error("Can't open '" + strInputVideo + "'");
 }

 // Open the output video for writing using input's characteristics
 int w = invid.get(cv::CAP_PROP_FRAME_WIDTH);
 int h = invid.get(cv::CAP_PROP_FRAME_HEIGHT);
 int fourcc = cv::VideoWriter::fourcc('M', 'P', 'E', 'G');
 double fps = invid.get(cv::CAP_PROP_FPS);

 cv::VideoWriter outVideo("perspwarp_" + strBackend + ".mp4", fourcc, fps, cv::Size(w, h));
 if (!outVideo.isOpened())
 {
 throw std::runtime_error("Can't create output video");
 }

 // =================================
 // Allocate all VPI resources needed

 // Create the stream for the given backend. We'll be using CUDA for image format conversion.
 CHECK_STATUS(vpiStreamCreate(backend | VPI_BACKEND_CUDA, &stream));

 CHECK_STATUS(vpiImageCreate(w, h, VPI_IMAGE_FORMAT_NV12_ER, 0, &imgInput));
 CHECK_STATUS(vpiImageCreate(w, h, VPI_IMAGE_FORMAT_NV12_ER, 0, &imgOutput));

 VPIPerspectiveTransform xform;
 memset(&xform, 0, sizeof(xform));

 // ====================
 // Main processing loop

 int curFrame = 1;
 while (invid.read(cvFrame))
 {
 printf("Frame: %d\n", curFrame++);

 if (frameBGR == NULL)
 {
 // Ceate a VPIImage that wraps the frame
 CHECK_STATUS(vpiImageCreateWrapperOpenCVMat(cvFrame, 0, &frameBGR));
 }
 else
 {
 // reuse existing VPIImage wrapper to wrap the new frame.
 CHECK_STATUS(vpiImageSetWrappedOpenCVMat(frameBGR, cvFrame));
 }

 // First convert it to NV12 using CUDA
 CHECK_STATUS(vpiSubmitConvertImageFormat(stream, VPI_BACKEND_CUDA, frameBGR, imgInput, NULL));

 // move image's center to origin of coordinate system
 VPIPerspectiveTransform t1 = {{1, 0, -w / 2.0f}, {0, 1, -h / 2.0f}, {0, 0, 1}};

 // Apply some time-dependent perspective transform
 float v1 = sin(curFrame / 30.0 * 2 * M_PI / 2) * 0.0005f;
 float v2 = cos(curFrame / 30.0 * 2 * M_PI / 3) * 0.0005f;
 VPIPerspectiveTransform P = {{0.66, 0, 0}, {0, 0.66, 0}, {v1, v2, 1}};

 // move image's center back to where it was.
 VPIPerspectiveTransform t2 = {{1, 0, w / 2.0f}, {0, 1, h / 2.0f}, {0, 0, 1}};

 // Apply the transforms defined above.
 VPIPerspectiveTransform tmp;
 MatrixMultiply(tmp, P, t1);
 MatrixMultiply(xform, t2, tmp);

 // Do perspective warp using the backend passed in the command line.
 // Passing NULL as grid to make it use a dense grid, for better quality.
 CHECK_STATUS(vpiSubmitPerspectiveWarp(stream, backend, imgInput, xform, imgOutput, NULL, VPI_INTERP_LINEAR,
 VPI_BORDER_ZERO, 0));

 // Convert output back to BGR using CUDA
 CHECK_STATUS(vpiSubmitConvertImageFormat(stream, VPI_BACKEND_CUDA, imgOutput, frameBGR, NULL));
 CHECK_STATUS(vpiStreamSync(stream));

 // Now add it to the output video stream
 VPIImageData imgdata;
 CHECK_STATUS(vpiImageLockData(frameBGR, VPI_LOCK_READ, VPI_IMAGE_BUFFER_HOST_PITCH_LINEAR, &imgdata));

 cv::Mat outFrame;
 CHECK_STATUS(vpiImageDataExportOpenCVMat(imgdata, &outFrame));
 outVideo << outFrame;

 CHECK_STATUS(vpiImageUnlock(frameBGR));
 }
 }
 catch (std::exception &e)
 {
 std::cerr << e.what() << std::endl;
 retval = 1;
 }

 // =========================
 // Destroy all VPI resources

 vpiStreamDestroy(stream);
 vpiImageDestroy(imgInput);
 vpiImageDestroy(imgOutput);
 vpiImageDestroy(frameBGR);

 return retval;
}