\documentclass[aspectratio=169,obeyspaces,spaces,hyphens,dvipsnames]{beamer}
\usepackage[utf8]{inputenc}
\usepackage{lmodern}% http://ctan.org/pkg/lm
\usepackage{minted}
\usepackage{hyperref}
\usepackage{xcolor}
\usepackage{pgfplots}
\usepackage{tikz}
\usepackage{stmaryrd}
\usepackage[normalem]{ulem}

\mode<presentation>
\usetheme{Bootlin}

\def\signed #1{{\leavevmode\unskip\nobreak\hfil\penalty50\hskip2em
  \hbox{}\nobreak\hfil(#1)
  \parfillskip=0pt \finalhyphendemerits=0 \endgraf}}

\newsavebox\mybox
\newenvironment{aquote}[1]
  {\savebox\mybox{#1}\begin{quotation}}
  {\signed{\usebox\mybox}\end{quotation}}

\tikzstyle{block} = [rectangle, draw, fill=white, text width=4em, text centered, minimum height=3em, node distance=10em]
\tikzstyle{blockish} = [draw, fill=white, text width=4em, text centered, node distance=8em]
\tikzstyle{arrow} = [thick,->,>=stealth]

\title{Supporting Hardware-Accelerated Video Encoding with Mainline}
\authors{Paul Kocialkowski}
\email{paul@bootlin.com}
\slidesurl{https://bootlin.com/pub/conferences/}
\institute{Bootlin}
\conference{Embedded Linux Conference Europe}

\begin{document}

\addtocontents{toc}{\protect\setcounter{tocdepth}{-1}}
\section{Supporting Hardware-Accelerated Video Encoding with Mainline}
\addtocontents{toc}{\protect\setcounter{tocdepth}{2}}

\begin{frame}{Paul Kocialkowski}
  \begin{itemize}
  \item Embedded Linux engineer at Bootlin
    \begin{itemize}
    \item Embedded Linux {\bf expertise}
    \item {\bf Development}, consulting and training
    \item Strong open-source focus
    \end{itemize}
  \item Open-source contributor
    \begin{itemize}
    \item Co-maintainer of the \textbf{cedrus} VPU driver in V4L2
    \item Contributor to the \textbf{sun4i-drm} DRM driver
    \item Developed the \textbf{displaying and rendering graphics with Linux} training
    \item Contributed \textbf{Allwinner MIPI CSI-2} support
    \end{itemize}
  \item Living in {\bf Toulouse}, south-west of France
  \end{itemize}
\end{frame}

\subsection{H.264 Encoding}

\begin{frame}{Need for video encoding}
  \begin{itemize}
  \item Representing pictures takes \textbf{significant memory}
  \item Example for a 10-minute \textbf{1920x1080} (32 bpp) video at 30 fps:
    \begin{itemize}
    \item \(1920 \times 1080 \times 4\) = 7.91 MiB/frame
    \item \(1920 \times 1080 \times 4 \times 30\)  = 237.3 MiB/s
    \item \(1920 \times 1080 \times 4 \times 30 \times 10 \times 60\) = \textbf{142.4 GiB}
    \end{itemize}
  \item Significant sizes are an issue for \textbf{storage and network transmission}
  \item Video encoding aims at solving the issue:
    \begin{itemize}
    \item Applying methods to \textbf{reduce the storage/transmission size}
    \item Adding encoding and decoding \textbf{overhead/latency}
    \item Keeping the perceived quality under control: \textbf{size/quality trade-off}
    \end{itemize}
  \item Only the \textbf{currently-active frames} are kept in memory when decoding
  \end{itemize}
\end{frame}

\begin{frame}{Codec, Bitstream and Container}
  \begin{itemize}
  \item Formats in which video is encoded are called \textbf{video codecs}
    \begin{itemize}
    \item e.g. MJPEG, MPEG-2, MPEG-4 Visual (DivX), H.264/AVC, H.265/HEVC
    \item Spanning over 7 generations with enriched features
    \end{itemize}
  \item Video codecs are \textbf{format specifications} for both:
    \begin{itemize}
    \item \textbf{Compressed video data}, that can be decoded into frames
    \item \textbf{Meta-data} that configures the decoder (to match encoder settings)
    \end{itemize}
  \item The video \textbf{bitstream} packs the data continuously (often with formatting)
  \item A \textbf{container} packs the video bitstream with other sources (audio, subtitles)
  \end{itemize}

  \vspace{0.5em}

  \begin{center}
  \includegraphics[width=0.8\linewidth]{images/video-file.pdf}
  \end{center}
\end{frame}

\begin{frame}{H.264 Introduction}
  \begin{itemize}
  \item ITU-T \textbf{H.264}, aka ISO \textbf{MPEG-4 AVC}, aka ISO MPEG-4 Part 10
  \item Probably one of the most popular and used codecs nowadays
  \item Supports both \textbf{progressive} and \textbf{interlaced} (used for TV broadcast)
  \item Specific \textbf{profiles} support a sub-set of compression features, such as:
    \begin{itemize}
    \item \textbf{Baseline}: Simple profile with few features (low resources)
    \item \textbf{High}: More features and flexibility
    \end{itemize}
  \item \textbf{Levels} limit the maximum bitrates and dimensions
  \item Designed for efficient \textbf{hardware implementations}
    \begin{itemize}
    \item Usually limited to specific profiles/levels
    \end{itemize}
  \item Extended with \textbf{annex specifications}:
    \begin{itemize}
    \item \textbf{H.264 SVC}: temporal/spatial/quality scalability
    \item \textbf{H.264 MVC}: multi-view (stereoscopy)
    \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}{H.264 Semantics}
  \begin{itemize}
  \item H.264 specifies the \textbf{semantics and syntax} to store compressed video
  \item Information is split into \textbf{Network Abstraction Layer Units} (NALUs)
  \item Each NALU has a common header and a specific type:
    \begin{itemize}
    \item \textbf{Sequence Parameter Set} (SPS): meta-data for the sequence
    \item \textbf{Picture Parameter Set} (PPS): meta-data for the picture
    \item \textbf{Coded slice data}: slice header and data
    \item More for extra information and specific slice coding
    \end{itemize}
  \item Meta-data is \textbf{bit-aligned} and often \textbf{conditional}
  \item NALUs are prefixed with a byte-aligned start code: \(00 00 00 01\) in \textbf{Annex-B format}
  \item Pictures are divided into blocks of 16x16 pixels called \textbf{macroblocks}
  \item Sets of macroblocks are grouped as \textbf{slices}
  \item Slices have a \textbf{specific type}, depending on the prediction mode:\\
        \textbf{I} slices for intra prediction and \textbf{P/B} slices for inter
  \end{itemize}
\end{frame}

\begin{frame}{H.264 Compression Techniques: Color Sub-sampling}
  \begin{center}
  \includegraphics[height=3.5em]{images/barn.jpg} \raisebox{1.5em}{\(\Rightarrow\)}
  \includegraphics[height=3.5em]{images/barn-yuv.jpg}
  \end{center}

  \begin{itemize}
  \item \textbf{Chroma sub-sampling} is used to reduce the bpp
    \begin{itemize}
    \item The Human visual system is more sensitive to luminance than chrominance
    \item Color-model and color-space conversion, e.g. sRGB to YUV Rec. 709
    \item Spatial sub-sampling is applied to chrominance
    \item YUV 4:2:0 gives 12 bpp, reduces size by 2 without significant quality loss
    \end{itemize}
  \end{itemize}

  \begin{center}
  \includegraphics[width=0.5\linewidth]{images/yuv-sub-sampling.pdf}
  \end{center}
\end{frame}

\begin{frame}{H.264 Compression Techniques: Quantization}
  \begin{itemize}
  \item Macroblocks are converted from spatial to \textbf{frequency domain},\\
        using a \textbf{discrete cosine transform (DCT)} operation
  \item A \textbf{quantization step} (\(Q_{step}\)) parameter divides coefficients before rounding,\\
  \end{itemize}
  \[ X_q = round\left(\dfrac{X}{Q_{step}}\right)\]
  \begin{itemize}
  \item A \textbf{quantization parameter} (\(QP \in \llbracket 0;51 \rrbracket \)) indexes the quantization step
  \item \textbf{Details in the picture} are lost as \(QP\) and \(Q_{step}\) increase
  \item Quantized coefficients are laid out in \textbf{zig-zag order} to group zeros,\\
        easily compressed with entropy coding
  \end{itemize}
\end{frame}

\begin{frame}{H.264 Compression Techniques: Spatial}
  \begin{itemize}
  \item Pictures that can be decoded alone are \textbf{intra-coded (I slices)}
  \item \textbf{Redundancy} often exists within a picture
  \item Pixels can be \textbf{deduced from neighbors} with prediction patterns
  \item H.264 supports many \textbf{intra prediction modes} (with specified directions)
  \end{itemize}
  \vspace{1em}
  \begin{center}
  \includegraphics[height=6em]{images/intra-pred-dir.png}\\
  \textit{Intra prediction directions}
  \end{center}
\end{frame}


\begin{frame}{H.264 Compression Techniques: Temporal}
  \begin{itemize}
  \item In most videos, \textbf{subsequent pictures are mostly the same}
  \item \textbf{Temporal differences} can be represented instead of each full picture
  \item \textbf{Motion vectors} between pictures are estimated at encoding
  \item They are applied to \textbf{reference pictures} for inter-picture prediction
  \item H.264 supports up to \textbf{16 references}
  \item References need to be kept decoded and alive in memory
  \end{itemize}

  \begin{center}
  \includegraphics[width=0.35\linewidth]{images/caminandes-prev.jpg}
  \hspace{4em}
  \includegraphics[width=0.35\linewidth]{images/caminandes-next.jpg}
  \end{center}
\end{frame}

\begin{frame}[fragile]{H.264 Compression Techniques: Temporal}
  \begin{center}
  \includegraphics[width=0.8\linewidth]{images/caminandes-mv.jpg}\\
  \textit{Motion vectors visualized using:}
  \begin{minted}[fontsize=\small]{console}
    ffplay -flags2 +export_mvs input.mp4 -vf codecview=mv=pf+bf+bb
  \end{minted}
  \end{center}
\end{frame}

\begin{frame}{H.264 Compression Techniques: Temporal}
  \begin{itemize}
  \item Types of inter prediction in H.264:
    \begin{itemize}
    \item \textbf{Backwards prediction} (P slices): using previous pictures
    \item \textbf{Bidirectional prediction} (B slices): using previous and following pictures
    \end{itemize}
  \item An \textbf{intra-coded picture} is necessary for inter prediction
  \item Following pictures for B slices need to \textbf{come first in bitstream order} % they are P slices
  \item A \textbf{group of pictures} (GOP) is sequence starting with an intra picture
  \end{itemize}
  \begin{center}
  \includegraphics[height=6em]{images/ippb.pdf}
  \end{center}
  \begin{itemize}
  \item Bidirectional inter prediction introduces \textbf{latency} when encoding and decoding
  \end{itemize}
\end{frame}

\begin{frame}{H.264 Compression Techniques: Entropy}
  \begin{itemize}
  \item A final \textbf{entropic compression pass} is applied to produce the bitstream
  \item Entropy coding assigns \textbf{shorter symbols to frequent occurrences}
  \item \textbf{Lossless compression} method that yields good results for video
  \item Syntax elements (meta-data) numbers are coded as \textbf{Exponential Golomb}
  \item Quantized DCT coefficients are coded using either:
    \begin{itemize}
    \item \textbf{CAVLC}: Context-Adaptive Variable Length Coding (default)
    \item \textbf{CABAC}: Context-Adaptive Binary Arithmetic Coding (advanced)
    \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}{H.264 Encoding Rate Control}
  \begin{itemize}
  \item Encoders apply a trade-off between \textbf{quality and bitstream size}\\
        the process is called \textbf{rate control} in general
  \item Quality is controlled by the \textbf{quantization parameter} \(QP\)
  \item \textbf{Rate control} modes:
    \begin{itemize}
    \item \textbf{CQP}: constant \(QP \in \llbracket 0;51 \rrbracket \) parameter for all frames
    \item \textbf{CRF}: constant rate factor (quality) \(CRF \in \llbracket 0;51 \rrbracket \)
    \item \textbf{CBR}: constant bitrate (\(kb/s\))
    \item \textbf{ABR}: average bitrate (\(kb/s\)) for the whole sequence,\\
          works best with two-passs encoding
    \end{itemize}
  \item The most appropriate mode depends on the \textbf{use-case}
  \item Quality can be evaluated using a \textbf{PSNR} metric
  \end{itemize}
\end{frame}

\subsection{Hantro H1 H.264 Encoder}

\begin{frame}{Hantro H1 Outline}
  \begin{itemize}
  \item The \textbf{Hantro H1} is a common hardware H.264/VP8/JPEG encoder
    \begin{itemize}
    \item Initially developed by Hantro Oy
    \item Acquired by On2 Technologies in 2007
    \item Acquired by Google in 2010
    \item Distributed as WebM Video Encoder Hardware IP
    \item Distributed with H.264 by VeriSilicon since 2015
    \end{itemize}
  \item Found in some \textbf{embedded ARM SoCs}:
    \begin{itemize}
    \item \textbf{Rockchip}: RK3288, RK3328, RK3399, PX30, RK1808
    \item \textbf{NXP}: i.MX8MM
    \end{itemize}
  \item Supports encoding H.264 in \textbf{1080p at 30 or 60 fps}
  \item Supports \textbf{Baseline, Main and High} H.264 Profiles,\\
        also MVC Stereo High
  \end{itemize}
\end{frame}

\begin{frame}{Hantro H1 Block Diagram}
  \begin{center}
  \includegraphics[width=0.7\linewidth]{images/hantro-h1-block.pdf}\\
  \textit{Hantro H1 Block Diagram from the i.MX8MM Manual}
  \end{center}
\end{frame}

\begin{frame}[fragile]{Hantro H1 Operation}
  \begin{itemize}
  \item \textbf{Stateless implementation} (no micro-controller/firmware)
  \item \textbf{Pre-processor} with cropping, rotation, scaling, CSC and stabilization
  \item Produces \textbf{slice NALUs} to memory, as Annex-B or direct NALU
  \item \textbf{Meta-data} (PPS, SPS) is generated in software, with \textbf{parameter constraints}:
    \begin{itemize}
    \item SPS \verb,pic_order_cnt_type = 2,
    \item SPS \verb,log2_max_frame_num_minus4 = 12,
    \item PPS \verb,weighted_bipred_idc = 0,
    \end{itemize}
  \item Only supports \textbf{I and P slices} (no B slices), for embedded recording
  \item \textbf{References} (for P slices) are stored in dedicated \textbf{reconstruction buffers}
  \item \textbf{CABAC tables} (for High-Profile) are also stored in a dedicated buffer
  \end{itemize}
\end{frame}

\begin{frame}{Hantro H1 Internal Rate Control Mechanisms}
  \begin{itemize}
  \item Base \textbf{Quantization} is specified with \(QP, QP_{min}, QP_{max}\)
  \item Advanced \textbf{internal mechanisms} exist for Rate Control in Hantro H1
    \begin{itemize}
    \item Allow \textbf{QP adjustments} during the encoding process
    \item \textbf{No longer used} by reference software nowadays
    \end{itemize}
  \item \textbf{MAD} (mean absolute difference) mechanism:
    \begin{itemize}
    \item Threshold value (\(MAD_{threshold}\)) for QP increase/decrease (\(\Delta_{QP}\))
    \item Single threshold and delta for a picture
    \end{itemize}
  \item \textbf{Checkpoints} mechanism:
    \begin{itemize}
    \item Checkpoints at regular macroblock distance, with up to 10 checkpoints
    \item Targets for cumulative non-zero quantization coefficients
    \item Error between target and actual count is evaluated at checkpoints
    \item A \(\Delta_{QP}\) is applied depending on the error
    \end{itemize}
  \item \textbf{Feedback data} (from registers) is used for \textbf{control loop regulation}
  \end{itemize}
\end{frame}

\begin{frame}{Hantro H1 Feedback Data}
  \begin{itemize}
  \item \textbf{QP sum}: sum of the \(QP\) value for each macroblock:
  \[ QP_{sum} = \sum_{macroblocks} QP_{macroblock} \]
  \item \textbf{RLC count}: number of non-zero quantization coefficients in the picture
  \item \textbf{Checkpoint values}: number of non-zero quantization coefficients at specified macroblock intervals
  \item \textbf{MAD count}: number of macroblocks under a specified \textbf{mean absolute difference} threshold
  \end{itemize}
\end{frame}

\subsection{V4L2 Integration for Stateless Encoding}

\begin{frame}[fragile]{V4L2 stateful encoding support}
  \begin{itemize}
  \item V4L2 already supports \textbf{stateful} H.264 encoders:
    \begin{itemize}
    \item Using the \verb,V4L2_PIX_FMT_H264, pixel format for \textbf{H.264 bitstream}
    \item Producing both \textbf{slice and meta-data} NALUs
    \item Using the \textbf{V4L2 M2M} framework
    \item Drivers: \verb;coda, mtk-vcodec, venus, s5p-mfc, hva;
    \end{itemize}
  \item Various \textbf{generic V4L2 controls} allow configuring the encode run:
    \begin{itemize}
    \item \verb,V4L2_CID_MPEG_VIDEO_H264_PROFILE,, \verb,V4L2_CID_MPEG_VIDEO_H264_LEVEL,
    \item \verb,V4L2_CID_MPEG_VIDEO_H264_8X8_TRANSFORM,, \verb,V4L2_CID_MPEG_VIDEO_H264_ENTROPY_MODE,
    \item \verb,V4L2_CID_MPEG_VIDEO_BITRATE_MODE,, \verb,V4L2_CID_MPEG_VIDEO_BITRATE,
    \end{itemize}
  \item Some drivers have \textbf{specific V4L2 controls} too:
    \begin{itemize}
    \item \verb,V4L2_CID_MPEG_MFC51_VIDEO_FORCE_FRAME_TYPE,
    \end{itemize}
  \item \textbf{Rate-control} is implemented by the encoder firmware
  \item \textbf{State and reference} management is also done by the firmware
  \end{itemize}
\end{frame}

\begin{frame}{V4L2 Stateless Encoding Considerations}
  \begin{itemize}
  \item With \textbf{stateless encoding} on the Hantro H1, many parameters can be set:
    \begin{itemize}
    \item Most of the PPS/SPS/slice header parameters
    \item Some are restricted to specific values
    \end{itemize}
  \item \textbf{State} is tracked by V4L2 and userspace:
    \begin{itemize}
    \item Buffers and parameters are tied (using the \textbf{Media Request API})
    \item Reconstruction buffers need to be kept around
    \item They are provided as references when needed
    \end{itemize}
  \item \textbf{Rate control} is left to the V4L2 driver and/or userspace:
    \begin{itemize}
    \item Feedback data needs to be provided
    \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}[fragile]{V4L2 Stateless Encoding: Existing Hantro H1 Support}
  \begin{itemize}
  \item Chromium OS supports the \textbf{Hantro H1 on Rockchip}
  \item \textbf{Chromium OS kernel} implementation (downstream based on Linux 4.4):\\
        \small\url{https://chromium.googlesource.com/chromiumos/third_party/kernel/+/chromeos-4.4/drivers/media/platform/rockchip-vpu}
  \item \textbf{Chromium OS userspace} \verb,libv4l2plugins, implementation :\\
        \small\url{https://chromium.googlesource.com/chromiumos/third_party/libv4lplugins/}
  \item \textbf{Rockchip's MPP} supports the Hantro H1 (VEPU1/VEPU2):\\
        \small\url{https://github.com/rockchip-linux/mpp}\\
        \small\url{http://opensource.rock-chips.com/wiki_Mpp}
  \end{itemize}
\end{frame}

\begin{frame}[fragile]{V4L2 Stateless Encoding: First Approach}
  \begin{itemize}
  \item The mainline \verb,hantro, driver (in staging) supports \textbf{Hantro G1 decoding}
  \item Bootlin added support for \textbf{H.264 Hantro H1 encoding} to the driver
  \item Inspired by Chromium OS and MPP implementations
  \item Using the \textbf{Media Request API} and V4L2 controls
  \item Rate control (CBR) is done \textbf{fully in userspace} based on feedback
  \item \textbf{Kernel side} (based on Linux 5.4 with backported media patches):\\
        \small\url{https://github.com/bootlin/linux/tree/hantro/h264-encoding}
  \item \textbf{Userspace side}: \small\url{https://github.com/bootlin/v4l2-hantro-h264-encoder}
  \end{itemize}
\end{frame}

\begin{frame}[fragile]{V4L2 Stateless Encoding: First Approach API}
  \begin{minipage}[t]{0.45\textwidth}
  \begin{minted}[fontsize=\footnotesize]{c}
struct v4l2_ctrl_h264_encode_params {
  /* Slice parameters */
  __u8 slice_type;
  __u8 pic_parameter_set_id;
  __u16 frame_num;
  __u16 idr_pic_id;
  __u8 cabac_init_idc;
  __u8 disable_deblocking_filter_idc;
  __s8 slice_alpha_c0_offset_div2;
  __s8 slice_beta_offset_div2;
  __s32 slice_size_mb_rows;
  /* PPS parameters */
  __s8 pic_init_qp_minus26;
  __s8 chroma_qp_index_offset;
  __u32 flags; /* V4L2_H264_ENCODE_FLAG_ */
  /* Reference */
  __u64 reference_ts;
};
  \end{minted}
  \end{minipage}
  \hfill
  \begin{minipage}[t]{0.45\textwidth}
  \begin{minted}[fontsize=\footnotesize]{c}
struct v4l2_ctrl_h264_encode_rc {
  __u32 qp;
  __u32 qp_min;
  __u32 qp_max;
  __s32 mad_qp_delta;
  __u32 mad_threshold;

  __u32 cp_distance_mbs;
  __u32 cp_target[10];
  __s32 cp_target_error[6];
  __s32 cp_qp_delta[7];
};

struct v4l2_ctrl_h264_encode_feedback {
  __u32 qp_sum;
  __u32 cp[10];
  __u32 mad_count;
  __u32 rlc_count;
};
  \end{minted}
  \end{minipage}
\end{frame}

\begin{frame}{V4L2 Stateless Encoding Approaches: Proposals}
  \begin{itemize}
  \item Major downside is the \textbf{lack of genericity} for a single API
  \item Using \textbf{generic V4L2 controls} for encode parameters could work:
    \begin{itemize}
    \item Existing stateful controls (profile/level/features)
    \item Additional controls to indicate \textbf{references}
    \end{itemize}
  \item Generic rate control done in \textbf{userspace}:
    \begin{itemize}
    \item Requires generic controls (QP, slice type/GOP can be enough)
    \item Requires generic feedback data (RLC and QP sum can be)
    \item Cannot support hardware-specific mechanisms
    \item Encourages proprietary implementations
    \end{itemize}
  \item Rate control done in \textbf{kernel drivers}:
    \begin{itemize}
    \item Easier for userspace but no fine control
    \item Can reuse existing stateful V4L2 RC controls
    \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}{V4L2 Stateless Encoding Approaches: Plan}
  \begin{itemize}
  \item Bootlin has interest in starting a discussion
  \item Design decisions are needed to upstream Hantro H1 support
  \item Also concerns other stateless encoders (e.g. on Allwinner)
    \begin{itemize}
    \item Little information is available currently
    \end{itemize}
  \item Feel free to let us know about:
    \begin{itemize}
    \item Interest in the topic
    \item Details of stateless hardware that could be affected
    \item Relevant use-cases to support for hardware encoding
    \end{itemize}
  \end{itemize}
\end{frame}

\questionslide

\end{document}
