\documentclass[aspectratio=169,obeyspaces,spaces,hyphens,dvipsnames]{beamer}
\usepackage[utf8]{inputenc}
\usepackage{minted}
\usepackage{hyperref}

\mode<presentation>
\usetheme{Bootlin}

\def\signed #1{{\leavevmode\unskip\nobreak\hfil\penalty50\hskip2em
  \hbox{}\nobreak\hfil(#1)%
  \parfillskip=0pt \finalhyphendemerits=0 \endgraf}}

\newsavebox\mybox
\newenvironment{aquote}[1]
  {\savebox\mybox{#1}\begin{quotation}}
  {\signed{\usebox\mybox}\end{quotation}}

\title{Supporting Hardware Codecs in a Linux system}
\conference{Embedded Linux Conference Europe 2018}
\authors{Maxime Ripard}
\email{maxime@bootlin.com}
\institute{Bootlin}
\slidesurl{https://bootlin.com/pub/conferences/2018/elce/ripard-v4l2-codec}

\begin{document}

\section{Supporting Hardware Codecs in a Linux system}

\begin{frame}{Maxime Ripard}
  \begin{itemize}
  \item Embedded Linux engineer and trainer at Bootlin
    \begin{itemize}
    \item Embedded Linux {\bf development}: kernel and driver
      development, system integration, boot time and power consumption
      optimization, consulting, etc.
    \item Embedded Linux, Linux driver development, Yocto Project /
      OpenEmbedded and Buildroot {\bf training}, with materials
      freely available under a Creative Commons license.
    \item \url{https://bootlin.com}
    \end{itemize}
  \item Contributions
    \begin{itemize}
    \item {\bf Co-maintainer of the sunXi SoCs} from Allwinner in
      {\bf Linux} and {\bf U-Boot}
    \item {\bf Co-maintainer of drm-misc} in {\bf Linux}
    \item Contributor to a couple of other open-source projects, {\bf
        Buildroot}, {\bf U-Boot}, {\bf Barebox}
    \end{itemize}
  \item Living in {\bf Toulouse}, south west of France
  \end{itemize}
\end{frame}

\subsection{Introduction to Video Decoding}

\begin{frame}{Video Decoding}
  \begin{itemize}
  \item An encoded video is basically two things:
    \begin{itemize}
    \item A container, which will organize the video and audio
      streams, and the metadata
    \item A codec, which will encode a given audio or video stream
    \end{itemize}
  \item We're only interested in the (video) codec here
  \item With a codec, the video stream is encoded into a bitstream
  \end{itemize}
\end{frame}

\begin{frame}{Bitstream}
  \begin{itemize}
  \item {\it A bitstream is a sequence of bits.} -- Wikipedia
  \item In the context of codecs, it refers to the compressed output
    of the encoder, the at-rest data
  \item It's mainly composed of a few things:
    \begin{itemize}
    \item Separators between encoded data
    \item Metadata holding the compression parameters
    \item Slices, buffers holding the compressed data
    \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}{Decoder}
  \begin{center}
    \includegraphics[width=\textwidth]{video-decoder.pdf}
  \end{center}
\end{frame}

\begin{frame}{Stateful Codec}
  \begin{center}
    \includegraphics[width=\textwidth]{stateful-codec.pdf}
  \end{center}
\end{frame}

\subsection{Video Decoding in Linux}

\begin{frame}{V4L2}
  \begin{itemize}
  \item Introduced in 2002, in 2.5.46
  \item Supports a wide range of devices
    \begin{itemize}
    \item Video Capture (camera, tuners)
    \item Memory to memory devices (hardware codecs, scalers, deinterlacers)
    \item Radio receivers and transceivers
    \item SDR
    \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}{V4L2: A Dumb M2M Pipeline}
  \begin{center}
    \includegraphics[height=0.8\textheight]{v4l2-dumb.pdf}
  \end{center}
\end{frame}

\begin{frame}{V4L2: Stateful Codec Support}
  \begin{center}
    \includegraphics[height=0.8\textheight]{v4l2-stateful-codec.pdf}
  \end{center}
\end{frame}

\subsection{And then, everything falls apart...}

\begin{frame}{Stateless Codec}
  \begin{center}
    \includegraphics[width=\textwidth]{stateless-codec.pdf}
  \end{center}
\end{frame}

\begin{frame}{Request API}
  \begin{itemize}
  \item An API that let's you tie controls with buffers
  \item First RFC sent in 2015 by Hans Verkuil
  \item Taken over in 2016 by Laurent Pinchart and Sakari Ailus
  \item Taken over in 2017 by Alexandre Courbot
  \item Taken over in 2018 by Hans Verkuil
  \item Merged in 4.20 (Yay!)
  \end{itemize}
\end{frame}

\begin{frame}{Stateless Codec Stack}
  \begin{center}
    \includegraphics[height=0.8\textheight]{v4l2-stateless-codec.pdf}
  \end{center}
\end{frame}

\subsection{Enters Allwinner}

\begin{frame}{Allwinner}
  \begin{itemize}
  \item Produces multimedia SoCs, targeted at tablets and STBs
  \item Found in a number of cheap SBCs
  \item Just like many multimedia SoCs, it has a hardware
  codec\dots
  \item \dots a stateless codec that is
  \end{itemize}
\end{frame}

\begin{frame}{Allwinner BSP Stack}
  \begin{itemize}
  \item Linux 3.4, or 3.10 these days
  \item Not using v4l2, but a stack split in two parts:
    \begin{itemize}
    \item A kernel driver that manages the resources, and provides
      access to the codec registers
    \item A userspace stack implementing the logic to decode and
      encode videos, that used to be closed source
    \end{itemize}
  \item Obviously not compatible with mainline kernel
  \end{itemize}
\end{frame}

\begin{frame}{Reverse-Engineering Effort}
  \begin{itemize}
  \item Reverse-engineering done by the {\it Cedrus} effort in order
    to have an opensource stack
  \item MPEG2, MPEG4, H264, H265 and VP8 Decoding and H264 Encoding 
  \item Targeted at Allwinner's BSP kernel
  \item Kernel obviously not maintained, and seriously outdated now
  \item But very functional, supports many codecs, decoding and
    encoding, pretty much done
  \item libvdpau implementation to hook into popular media players
  \end{itemize}
\end{frame}

\begin{frame}{First attempt at a mainline driver}
  \begin{itemize}
  \item Summer Internship in 2016, by Florent Revest
  \item Worked on an RFC, to do a mainline-based driver
  \item Quite successful:
    \begin{itemize}
    \item MPEG2 implementation
    \item libva implementation to integrate in the popular media players
    \end{itemize}
  \item But was still a prototype:
    \begin{itemize}
    \item Still had bugs
    \item Slow, and could only play videos at any other resolution
      than the display
    \item Who cares about MPEG2 anymore?
    \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}{Kickstarter Campaign}
  \begin{itemize}
  \item Huge effort to bring it into a production-ready state
  \item Let's fund it through a Kickstarter campaign!
  \item The goals we achieved were to develop the driver for the most
    used SoCs, with H264 and H265 decoding support
  \item Funded a full-time 6-month internship, and a part-time
    engineer
  \item Built on top of the prototype
  \end{itemize}
\end{frame}

\begin{frame}{Cedrus Stack}
  \begin{center}
    \includegraphics[height=0.8\textheight]{v4l2-cedrus.pdf}
  \end{center}
\end{frame}

\subsection{Displaying is hard}

\begin{frame}{In an ideal world...}
  \begin{center}
    \includegraphics[height=0.8\textheight]{kodi.pdf}
  \end{center}
\end{frame}

\begin{frame}{Except that...}
  \begin{itemize}
  \item The decoded frame is in a proprietary format...
    \begin{itemize}
    \item ... that the display engine can understand
    \end{itemize}
  \item The decoded frame is at the video resolution, not the window
    resolution...
    \begin{itemize}
    \item ... and the display engine can upscale or downscale it at
      will
    \end{itemize}
  \item But X11 doesn't let you do that easily
  \end{itemize}
\end{frame}

\begin{frame}{Possible solutions}
  \begin{itemize}
  \item X doesn't have any knowledge about proprietary formats
  \item Let's convert it in software!
    \begin{itemize}
    \item Reaaaaaally slow, even without scaling
    \item Hardware video playback is supposed to be {\bf more} efficient
    \end{itemize}
  \item The X Video Extension is supposed to let you do that
    \begin{itemize}
    \item But it's completely deprecated
    \item Only one video would be playable at the same time (on our platform)
    \item Possible glitches
    \end{itemize}
  \item You could import the decoded frame in the GPU, and do the
    format conversion and scaling there
    \begin{itemize}
    \item But the OpenGL blob has a lot of constraints
    \item And memory bandwidth is a sparse resource
    \end{itemize}
  \item Wayland!
    \begin{itemize}
    \item You have to patch all Wayland compositors to deal with your
      proprietary format
    \item And you leave out all the users that are stuck with X11
    \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}{Current State}
  \begin{itemize}
  \item The Request API is merged in 4.20
    \begin{itemize}
    \item Libva implementation working on top of it, with MPEG2, H264
      and H265 decoding support
    \item Test tool to simulate some video decoding (v4l2-request-test)
    \end{itemize}
  \item Our Cedrus driver is merged in 4.20 as well (but in staging)
    \begin{itemize}
    \item Only MPEG2 for now
    \item H264 and H265 patches have been sent
    \end{itemize}
  \end{itemize}
\end{frame}

\questionslide{}

\end{document}
