\documentclass[aspectratio=169,obeyspaces,spaces,hyphens,dvipsnames]{beamer}
\usepackage[utf8]{inputenc}
\usepackage{lmodern}% http://ctan.org/pkg/lm
\usepackage{minted}
\usepackage{hyperref}
\usepackage{xcolor}
\usepackage{pgfplots}
\usepackage{tikz}
\usepackage{stmaryrd}
\usepackage[normalem]{ulem}

\mode<presentation>
\usetheme{Bootlin}

\def\signed #1{{\leavevmode\unskip\nobreak\hfil\penalty50\hskip2em
  \hbox{}\nobreak\hfil(#1)
  \parfillskip=0pt \finalhyphendemerits=0 \endgraf}}

\newsavebox\mybox
\newenvironment{aquote}[1]
  {\savebox\mybox{#1}\begin{quotation}}
  {\signed{\usebox\mybox}\end{quotation}}

\makeatletter
\newbox\@backgroundblock
\newenvironment{backgroundblock}[2]{%
  \global\setbox\@backgroundblock=\vbox\bgroup%
    \unvbox\@backgroundblock%
    \vbox to0pt\bgroup\vskip#2\hbox to0pt\bgroup\hskip#1\relax%
}{\egroup\egroup\egroup}
\addtobeamertemplate{background}{\box\@backgroundblock}{}
\makeatother

\tikzstyle{block} = [rectangle, draw, fill=white, text width=4em, text centered, minimum height=3em, node distance=10em]
\tikzstyle{blockish} = [draw, fill=white, text width=4em, text centered, node distance=8em]
\tikzstyle{arrow} = [thick,->,>=stealth]

\title{An Overview of the Linux and Userspace Graphics Stack}
\authors{Paul Kocialkowski}
\email{paul@bootlin.com}
\slidesurl{https://bootlin.com/pub/conferences/}
\institute{Bootlin}
\conference{Live Embedded Event}

\begin{document}

\addtocontents{toc}{\protect\setcounter{tocdepth}{-1}}
\section{An Overview of the Linux and Userspace Graphics Stack}
\addtocontents{toc}{\protect\setcounter{tocdepth}{2}}

\begin{frame}{Paul Kocialkowski}
  \begin{itemize}
  \item Embedded Linux engineer at Bootlin
    \begin{itemize}
    \item Embedded Linux {\bf expertise}
    \item {\bf Development}, consulting and training
    \item Strong open-source focus
    \end{itemize}
  \item Open-source contributor
    \begin{itemize}
    \item Co-maintainer of the \textbf{cedrus} VPU driver in V4L2
    \item Contributor to the \textbf{sun4i-drm} DRM driver
    \item Developed the \textbf{displaying and rendering graphics with Linux} training
    \item Contributing \textbf{Allwinner MIPI CSI-2} support in V4L2
    \end{itemize}
  \item Living in {\bf Toulouse}, south-west of France
  \end{itemize}
\end{frame}

\subsection{Introduction}

\begin{frame}{What All the Fuss is About with Graphics}
  \vspace{1em}
  \begin{minipage}{0.7\textwidth}
  \begin{center}
  \textbf{What do we mean by graphics?}
  \end{center}

  \begin{itemize}
  \item Graphics deals with digital representation of \textbf{light}
  \item Taking the form of \textbf{pictures} or \textbf{frames}
  \item Light in the physical world is continuous
  \item Digital pictures are \textbf{discrete} or \textbf{quantized}
  \item Discrete picture elements are \textbf{pixels}
  \item Using a \textbf{color model} and \textbf{color space}
  \end{itemize}
  \end{minipage}%
  \begin{minipage}{0.3\textwidth}
  \centering
  \includegraphics[width=\textwidth]{images/ccc-rocket.jpg}
  \end{minipage}
\end{frame}

\begin{frame}{All About Pictures and Pixels}
  \begin{minipage}{0.2\textwidth}
  \centering
  \includegraphics[width=5.5em]{images/barn.jpg}
  \end{minipage}%
  \begin{minipage}{0.8\textwidth}
  \begin{itemize}
  \item Pictures have \textbf{dimensions} (width and height) in pixels
  \item \textbf{Aspect ratio} is the width:height fraction
  \item \textbf{Resolution} links pixels to length units (px/in)
  \item Specified \textbf{scan order} in memory
  \item Pixels have a specific \textbf{format}:
    \begin{itemize}
    \item Color channels in a color space
    \item Alpha (transparency) channel
    \item Depth and bits per pixel (bpp)
    \item Organization in memory as planes
    \item Sub-sampling
    \end{itemize}
  \end{itemize}
  \end{minipage}
\end{frame}

\begin{frame}{All the Things Dealing with Pixels}
  \textbf{Graphics:}
  \begin{itemize}
  \item \textbf{Displaying}: producing light from a digital picture
    \begin{itemize}
    \item Monitors, panels, projectors
    \end{itemize}
  \item \textbf{Rendering}: generating digital pictures from primitives
    \begin{itemize}
    \item 3D rendering, 2D shape drawing, font rendering and more
    \end{itemize}
  \item \textbf{Processing}: transforming digital pictures
    \begin{itemize}
    \item Filtering, scaling, converting, compositing and more
    \end{itemize}
  \end{itemize}

  \textbf{Media:}
  \begin{itemize}
  \item \textbf{Decoding/encoding}: (un)compressing pictures
    \begin{itemize}
    \item Picture codecs (JPEG, PNG, etc), Video codecs (H.264, VP8, etc)
    \end{itemize}
  \item \textbf{Capturing/outputting}: receiving or sending pictures from/to the outside 
    \begin{itemize}
    \item Cameras, DVB and more
    \end{itemize}
  \end{itemize}
\end{frame}

\subsection{Graphics Hardware Components}

\begin{frame}{Display Hardware (Source)}
  Display output is implemented through many components:
  \begin{itemize}
  \item \textbf{Framebuffer}: pixel memory
  \item \textbf{Display engine}: hardware compositor (from planes)
  \item \textbf{Timings controller}: CRT-compatible timings
  \item \textbf{Display protocol controller}: protocol logic
  \item \textbf{Display interface PHY}: protocol physical layer
  \item \textbf{Connector and cable}: link to the display device (monitor or panel)
  \end{itemize}

  \begin{center}
  \includegraphics[height=5em]{images/ati-hercules-1986.png}
  \includegraphics[height=5em]{images/panel.png}
  \includegraphics[height=5em]{images/crt-color.png}
  \end{center}
\end{frame}

\begin{frame}{Rendering and Processing Hardware}
  A few types of implementations are used for \textbf{rendering} and \textbf{processing}:
  \begin{itemize}
  \item \textbf{GPUs} (Graphics Processing Units):
    \begin{itemize}
    \item Highly specialized architectures and ISAs
    \item Designed for 3D rendering, can also do 2D and processing
    \item Loaded with small programs (shaders)
    \item Configured with a command stream
    \end{itemize}
  \item \textbf{DSPs} (Digital Signal Processors):
    \begin{itemize}
    \item Dedicated processors with a specialized ISA
    \item Run with a dedicated firmware or RTOS
    \end{itemize}
  \item \textbf{Fixed-function ISPs} (Image Signal Processors):
    \begin{itemize}
    \item Hardware implementations for specific tasks
    \end{itemize}
  \item \textbf{CPU-based} implementations
    \begin{itemize}
    \item All done in software (often use SIMD)
    \end{itemize}
  \end{itemize}
\end{frame}

\subsection{Generic Concepts for Software}

\begin{frame}{Display Software Concepts}
  \begin{itemize}
  \item Display access must be \textbf{exclusive} to a single program
  \item A \textbf{display server} manages the display framebuffer(s):
    \begin{itemize}
    \item Provides a protocol for clients
    \item Gathers buffers and updates from clients
    \item Handles input events
    \item Forwards relevant input events to clients
    \item In charge of security and isolation concerns
    \end{itemize}
  \item A \textbf{compositor} produces the final image from client sources
  \item A \textbf{window manager} defines policy between clients
  \end{itemize}
\end{frame}

\begin{frame}{Render Software Concepts}
  \begin{itemize}
  \item Unlike display, GPUs can run different \textbf{jobs in parallel}
  \item GPU rendering is based on \textbf{primitives}:
    \begin{itemize}
    \item Vertices, lines and triangles
    \item Given positions in 3D
    \end{itemize}
  \item \textbf{Textures and maps} can be involved as well
  \item Shaders (programs) define the result:
    \begin{itemize}
    \item \textbf{Vertex shaders} for transformations and lighting
    \item \textbf{Fragment shaders} for color and textures
    \item More advanced shaders also exist
    \item Shaders are \textbf{compiled on-the-fly} from source or intermediate forms (IRs)
    \item Binary shaders are attached to the GPU \textbf{command stream}
    \end{itemize}
  \item Multiple \textbf{rendering passes} can be used
  \end{itemize}
\end{frame}

\subsection{Linux and Userspace Graphics Stack}

\begin{frame}{Displaying Stack: Kernel}
  \begin{itemize}
  \item Historic and legacy subsystem for display: \textbf{fbdev}
    \begin{itemize}
    \item Very limited: static setup, no pipeline, pre-allocated buffers, sync issues
    \item Still used by fbcon for on-display console
    \item Available through \code{/dev/fb0} (please stop using it)
    \item Source at \code{drivers/video/fbdev} in Linux
    \end{itemize}
  \item Current and relevant interface: \textbf{DRM KMS}
    \begin{itemize}
    \item Exposes each display pipeline element for configuration
    \item Generic uAPI with a property-based system
    \item Dynamic framebuffer allocation
    \item Atomic API for synchronizing changes
    \item Legacy compatibility layer with fbdev
    \item Source at \code{drivers/gpu/drm} in Linux
    \end{itemize}
  \item The \textbf{TTY} subsystem is also involved:
    \begin{itemize}
    \item Graphics mode switch to detach fbcon
    \item Virtual Terminal (VT) switching
    \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}{Displaying Stack: Userspace Protocols and Servers}
  \begin{itemize}
  \item \textbf{X} is the historical display server project used with Linux:
    \begin{itemize}
    \item \textbf{X11} (X protocol version 11) is the protocol
    \item Completed with many (many) \textbf{protocol extensions}:\\
          e.g. \code{XrandR}, \code{XSHM}, \code{Xinput2}, \code{Composite}
    \item \textbf{Xorg} is the reference implementation
    \item Using hardware-specific \textbf{drivers} for display and input:\\
          e.g. \code{xf86-input-libinput}, \code{xf86-video-modesetting}, \code{xf86-video-fbdev}
    \item X provides server-side rendering (not used a lot nowadays)
    \item Comes with various \textbf{security issues} and \textbf{limitations}
    \item No longer adapted to modern-day computers, nor embedded
    \item Most of its work is delegated through extensions
    \end{itemize}
  \end{itemize}
  \begin{backgroundblock}{34em}{16em}
  \includegraphics[height=4em]{images/x11-logo.png}
  \end{backgroundblock}
\end{frame}

\begin{frame}{Displaying Stack: Userspace Protocols and Servers}
  \begin{itemize}
  \item \textbf{Wayland} is a modern display server:
    \begin{itemize}
    \item Designed from scratch to avoid common limitations and issues in X
    \item Wayland is a \textbf{protocol specification}, not an implementation\\
          coming as a core protocol and optional extensions (e.g. \code{XDG-Shell})
    \item Server implementations are called \textbf{Wayland compositors}
    \item \textbf{Weston} is the Wayland compositor reference implementation\\
    \item Other implementations: \code{sway} (wlroots), \code{mutter} (GNOME), \code{Kwin} (KDE)
    \item Improved security and isolation between clients
    \item Compatible with X applications via \textbf{XWayland}
    \end{itemize}
  \item Other display servers also exist:
    \begin{itemize}
    \item \textbf{Mir}: Canonical's display server, more or less abandoned
    \item \textbf{SurfaceFlinger}: Android's display server
    \end{itemize}
  \item \textbf{Display managers} are commonly used at startup:
    \begin{itemize}
    \item Serve as login screens at startup
    \item Launch display servers and environments for users
    \end{itemize}
  \end{itemize}
  \begin{backgroundblock}{33.5em}{15.5em}
  \includegraphics[height=5em]{images/wayland-logo.png}
  \end{backgroundblock}
\end{frame}

\begin{frame}{Displaying Stack: Userspace Libraries}
  \begin{minipage}{0.2\textwidth}
  \centering
  \includegraphics[width=4em]{images/gtk-logo.png}\\
  \vspace{1em}
  \includegraphics[width=4em]{images/qt-logo.png}\\
  \vspace{0.5em}
  \includegraphics[width=4em]{images/enlightenment-logo.png}\\
  \vspace{0.5em}
  \includegraphics[width=4em]{images/sdl-logo.png}\\
  \end{minipage}%
  \begin{minipage}{0.8\textwidth}
  \begin{itemize}
  \item Libraries implement low-level display server protocols:
    \begin{itemize}
    \item \textbf{Wayland}: \code{libwayland-client}, \code{libwayland-server}
    \item \textbf{X11}: \code{XCB}, \code{Xlib}
    \end{itemize}
  \item Graphics toolkits abstract display server protocols:
    \begin{itemize}
    \item \textbf{GTK} (C): Widget-based UI toolkit for X and Wayland
    \item \textbf{Qt} (C++): Widget-based UI toolkit for X and Wayland
    \item \textbf{EFL} (C): Lightweight UI and application library
    \item \textbf{SDL} (C): Drawing-oriented graphics library (used in games)
    \end{itemize}
  \item \textbf{Desktop environments} are based on a given toolkit:
    \begin{itemize}
    \item Provide a desktop UI and a set of base applications
    \item Implement a window manager/compositor
    \end{itemize}
  \item Calls to the \textbf{DRM uAPI} are wrapped by \code{libdrm}
    \begin{itemize}
    \item Used by every single program that supports DRM
    \end{itemize}
  \end{itemize}
  \end{minipage}
\end{frame}

\begin{frame}{Rendering Stack for 3D: Kernel}
  \begin{itemize}
  \item The DRM subsystem is also in charge of \textbf{managing GPUs}
  \item Unlike DRM KMS, no generic interface but \textbf{driver-specific uAPIs}\\
        supported with thin helpers in \code{libdrm}
  \item Handles various \textbf{low-level aspects}:
    \begin{itemize}
    \item \textbf{Memory buffers} (BO) management with \code{GEM}
    \item \textbf{Command stream} validation and submission
    \item \textbf{Tasks} scheduling with the DRM scheduler
    \end{itemize}
  \item Most of the heavy lifting is left to userspace (only I/O in kernel)
  \item \textbf{Proprietary implementations} use their custom interfaces\\
        found in downstream kernels or out-of-tree drivers
  \end{itemize}
\end{frame}

\begin{frame}{Rendering Stack for 3D: Userspace APIs}
  Generic APIs are used for programs to leverage the GPU:
  \begin{itemize}
  \item \textbf{OpenGL} for rendering with desktop GPUs:
    \begin{itemize}
    \item Compromise between complexity and control
    \item Stateful and context-based programming model
    \item Using GLSL (GL shading language) for shader sources
    \end{itemize}
  \item \textbf{OpenGL ES} for rendering with embedded GPUs
  \item \textbf{EGL} for interfacing OpenGL with the display stack
    \begin{itemize}
    \item Provides scanout buffers and sync
    \item Supports X11, Wayland, Android and more
    \item Replaces the legacy GLX for X11
    \end{itemize}
  \item \textbf{Vulkan} for advanced GPU usage:
    \begin{itemize}
    \item Low-level API with direct programming and memory management
    \item Uses its own display stack integration: Vulkan WSI
    \item Supports more than rendering (e.g. compute)
    \item Uses a pre-built shader format: SPIR-V
    \end{itemize}
  \end{itemize}
  \begin{backgroundblock}{28em}{4em}
  \includegraphics[height=3.25em]{images/opengl-logo.png}
  \end{backgroundblock}
  \begin{backgroundblock}{28em}{8.75em}
  \includegraphics[height=3.25em]{images/opengl-es-logo.png}
  \end{backgroundblock}
  \begin{backgroundblock}{28.2em}{13.5em}
  \includegraphics[height=3em]{images/egl-logo.png}
  \end{backgroundblock}
  \begin{backgroundblock}{28.2em}{19em}
  \includegraphics[height=2.5em]{images/vulkan-logo.png}
  \end{backgroundblock}
\end{frame}

\begin{frame}{Rendering Stack for 3D: Userspace Implementations}
  \begin{itemize}
  \item \textbf{Mesa 3D} is the reference free software rendering library:
    \begin{itemize}
    \item Supports OpenGL, OpenGL ES and Vulkan APIs (also Direct 3D 9)
    \item Supports GPUs that have a DRM render driver:\\
          \code{radeon}, \code{amdgpu}, \code{nouveau}, \code{etnaviv}, \code{vc4/v3d}, \code{lima}, \code{panfrost}
    \item Implements software rendering fallbacks:\\
          \code{softpipe}, \code{swr}, \code{llvmpipe}, \code{lavapipe}
    \item Implements shader compilation with intermediate representations (IRs)
    \item Also supports GPU video decoding through \code{VDPAU}, \code{VAAPI} or \code{OMX}
    \item Also supports compute via OpenCL (\code{clover} driver)
    \end{itemize}
  \item \textbf{Proprietary} libraries have their own secret implementations
  \end{itemize}
\end{frame}

\begin{frame}{Rendering Stack for 2D: Libraries}
  \begin{itemize}
  \item General \textbf{drawing/rasterization}:
    \begin{itemize}
    \item \textbf{cairo}: widely-used drawing library
    \item \textbf{Skia}: Google's drawing library
    \end{itemize}
  \item \textbf{Font} rendering:
    \begin{itemize}
    \item \textbf{FreeType}: historical vector font rendering library
    \item \textbf{HarfBuzz}: recent vector font rendering library
    \end{itemize}
  \item \textbf{User interface} rendering:
    \begin{itemize}
    \item Full widget toolkits: \textbf{GTK}, \textbf{Qt}, \textbf{EFL} and more
    \item Immediate-mode GUIs: \textbf{Dear ImGui}, \textbf{nuklear}
    \item Animations: \textbf{Clutter}
    \end{itemize}
  \item Mostly \textbf{CPU-based} implementations
  \item Sometimes leverage \textbf{GPU rendering} through 3D APIs and shaders
  \end{itemize}
\end{frame}

\begin{frame}{Processing Stack: Libraries}
  \begin{itemize}
  \item Processing can be implemented:
    \begin{itemize}
    \item Using optimized \textbf{CPU-based} algorithms
    \item Using specific \textbf{SIMD} CPU instructions (NEON, SSE, AVX)
    \item Using {GPU rendering} through 3D APIs and shaders
    \end{itemize}
  \item Various libraries exist:
    \begin{itemize}
    \item FFmpeg's \textbf{libswscale} for pixel format conversion and scaling
    \item \textbf{Pixman} for various pixel operations
    \item ARM's \textbf{Ne10} for NEON-accelerated pixel operations
    \item \textbf{FFTW} for fast Fourier transforms
    \item \textbf{G'MIC} image processing framework
    \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}{Graphics Stack Overview}
  \begin{center}
  \includegraphics[width=0.8\linewidth]{images/wayland-stack.png}
  \end{center}
\end{frame}

\begin{frame}{Advanced Topics: Memory Sharing and Fences}
  \begin{itemize}
  \item Copying buffers between (hardware) components is a \textbf{major bottleneck}
  \item Specific APIs are used to share references (file descriptors) between applications:
    \begin{itemize}
    \item \textbf{Shared memory} (SHMem) for system memory pages
    \item \textbf{DMA-BUF memory} for device-allocated memory
    \end{itemize}
  \item Synchronization between hardware devices is possible with \textbf{fences}:
    \begin{itemize}
    \item A graphics pipeline is configured with fence references (file descriptors)
    \item Fences are \textbf{signaled} when a device is done
    \item The next device in the chain is then \textbf{triggered} by the kernel
    \item No userspace roundtrip is necessary
    \end{itemize}
  \end{itemize}
\end{frame}

\questionslide

\end{document}
