\documentclass[aspectratio=169,obeyspaces,spaces,hyphens,dvipsnames,handout]{beamer}
\usepackage[utf8]{inputenc}
\usepackage{lmodern}% http://ctan.org/pkg/lm
\usepackage{minted}
\usepackage{hyperref}
\usepackage{xcolor}
\usepackage{pgfplots}
\usepackage{tikz}
\usepackage{stmaryrd}
\usepackage[normalem]{ulem}

\mode<presentation>
\usetheme{Bootlin}

\def\signed #1{{\leavevmode\unskip\nobreak\hfil\penalty50\hskip2em
  \hbox{}\nobreak\hfil(#1)
  \parfillskip=0pt \finalhyphendemerits=0 \endgraf}}

\newsavebox\mybox
\newenvironment{aquote}[1]
  {\savebox\mybox{#1}\begin{quotation}}
  {\signed{\usebox\mybox}\end{quotation}}

\makeatletter
\newbox\@backgroundblock
\newenvironment{backgroundblock}[2]{%
  \global\setbox\@backgroundblock=\vbox\bgroup%
    \unvbox\@backgroundblock%
    \vbox to0pt\bgroup\vskip#2\hbox to0pt\bgroup\hskip#1\relax%
}{\egroup\egroup\egroup}
\addtobeamertemplate{background}{\box\@backgroundblock}{}
\makeatother

\tikzstyle{block} = [rectangle, draw=blorange, text width=8em, text centered, rounded corners, minimum height=3em]
\tikzstyle{arrow} = [thick,->,>=stealth,shorten >= 0.2em,shorten <= 0.2em]

\title{Advanced Camera Support on Allwinner SoCs with Mainline Linux}
\authors{Paul Kocialkowski}
\email{paul@bootlin.com}
\slidesurl{https://bootlin.com/pub/conferences/}
\institute{Bootlin}
\conference{EOSS 2023}

\begin{document}

\addtocontents{toc}{\protect\setcounter{tocdepth}{-1}}
\section{A Current Overview of the DRM KMS Driver-Side APIs}
\addtocontents{toc}{\protect\setcounter{tocdepth}{2}}

\begin{frame}{Paul Kocialkowski}
  \begin{itemize}
  \item Embedded Linux engineer at Bootlin
    \begin{itemize}
    \item Embedded Linux {\bf expertise}
    \item {\bf Development}, consulting and training
    \item Strong open-source focus
    \end{itemize}
  \item Open-source contributor
    \begin{itemize}
    \item Contributor to the \textbf{cedrus} Allwinner Video Engine V4L2 driver
    \item Contributor to the \textbf{sun6i-csi} Allwinner video capture V4L2 driver
    \item Author of the \textbf{sun6i-isp} and \textbf{sun6i-mipi-csi2} Allwinner video capture V4L2 drivers
    \item Author of the \textbf{ov5648} and \textbf{ov8865} camera sensor V4L2 drivers
    \item Contributor to the \textbf{sun4i-drm} and \textbf{vc4} DRM drivers
    \item Author of the \textbf{logicvc-drm} DRM driver
    \item Author of the \textbf{displaying and rendering graphics with Linux} training
    \end{itemize}
  \item Living in {\bf Toulouse}, south-west of France
  \end{itemize}
\end{frame}

\subsection{Introduction}

\begin{frame}{Display Hardware Pipeline}

\begin{center}
\begin{tikzpicture}[shorten >=1pt,auto,node distance=8em,semithick, scale=0.5, every node/.style={scale=0.5}]

\node [block] (framebuffer-1) {Framebuffer};
\node [block,dashed] (framebuffer-2) [below of=framebuffer-1, node distance=4em] {Framebuffer};

\node [block] (plane-1) [right of=framebuffer-1, node distance=12em] {Plane};
\node [block,dashed] (plane-2) [below of=plane-1, node distance=4em] {Plane};

\node [block] (crtc) [right of=plane-1, node distance=12em] {CRTC};
\node [block] (encoder) [right of=crtc, node distance=12em] {Encoder};
\node [block,dashed] (bridge) [right of=encoder, node distance=12em] {Bridge};

\node [block] (panel) [right of=bridge, node distance=12em] {Panel / Monitor};

\node (display-engine-left) [left of=plane-1, node distance = 6em] {};
\node (display-engine-right) [right of=encoder, node distance = 6em] {};
\node (display-engine-top-left) [above of=display-engine-left, node distance = 5em] {};
\node (display-engine-top-right) [above of=display-engine-right, node distance = 5em] {};
\node (display-engine-bot-left) [below of=display-engine-left, node distance = 7em] {};
\node (display-engine-bot-right) [below of=display-engine-right, node distance = 7em] {};
\node (display-engine-text) [below of=encoder, node distance=5em, blorange] {Display controller};

\draw [dashed, draw=blorange] (display-engine-top-left) -- (display-engine-top-right) -- (display-engine-bot-right) -- (display-engine-bot-left) -- (display-engine-top-left);

\node (plane-2-right-arrow) [right of=plane-2, node distance=6em] {};
\node (crtc-left-arrow) [left of=crtc, node distance=6em] {};

\draw [arrow] (framebuffer-1) -- (plane-1) node [midway,above=2em] {Memory};
\draw [arrow] (framebuffer-2) -- (plane-2);

\draw [arrow] (plane-1) -- (crtc) node [midway,above=2em] {FIFOs};
\draw [arrow] (crtc) -- (encoder) node [midway,above=2em] {Sync FIFO};
\draw [arrow] (encoder) -- (bridge) node [midway,above=2em] {Display Interface};
\draw [arrow] (bridge) -- (panel) node [midway,above=2em] {Display Interface};

\node (connector-center) [right of=bridge, node distance=6em] {};
\node (connector-top) [above of=connector-center, node distance=2em] {};
\node (connector-bottom) [below of=connector-center, node distance=5em, blorange] {Connector};

\draw [dashed, blorange] (connector-top) -- (connector-bottom);

\draw [arrow,-, shorten >= -0.2pt] (plane-2) -- (plane-2-right-arrow.center) -- (crtc-left-arrow.center);
\end{tikzpicture}
\end{center}

\begin{itemize}
\item \textbf{Framebuffer}: Memory buffer(s) with pixel data
\item \textbf{Plane}: Pixel mixing (rotation, scaling, format and more), layers
\item \textbf{CRTC}: Timings generation and pixel streaming
\item \textbf{Encoder}: Interface physical adaptation (protocol, signals)
\item \textbf{Bridge}: Interface transcoding
\item \textbf{Panel}: Display surface emitting/reflecting light
\item \textbf{Monitor}: Peripheral integrating a panel
\end{itemize}

\end{frame}

\begin{frame}{Display Hardware Overview}
Typical hardware components:
\begin{itemize}
\item Display controller in (PCIe) graphics card {\color{blorange}(x86)} or as unit in SoC {\color{blorange}(embedded)}
\item On-board and internal bridge(s)
\item Display connector(s), always-connected panel(s)
\end{itemize}~\\

Memory cases:
\begin{itemize}
\item Dedicated memory in graphics card (video RAM)
\item Shared memory with the system
\begin{itemize}
\item With IOMMU: general system pages, scatter-gather
\item Without IOMMU: reserved contiguous memory
\end{itemize}
\item CPU access and cache management:
\begin{itemize}
\item Automatic with cache-coherent interconnect
\item Manual cache invalidate/flush or cache disabled otherwise
\end{itemize}
\end{itemize}
\end{frame}

\begin{frame}{Display Support in Linux and Userspace}

Linux userspace APIs for display:
\begin{itemize}
\item \textbf{fbdev} interface: deprecated
\begin{itemize}
\item Simple uAPI with pre-allocated memory
\item Very limited use-cases and configuration
\item Bad performance (no zero-copy)
\item No new drivers, will be removed eventually
\end{itemize}
\item \textbf{DRM KMS}/\textbf{DRM mode} uAPI:
\begin{itemize}
\item Proper pipeline configuration (planes, CRTC, connector)
\item Well-balanced abstraction of hardware complexity
\item DRM Atomic uAPI
\item DRM GEM/TTM memory management
\item DRM Prime for dma-buf zero-copy
\item DRM Sync Object for fences
\item Various drivers for graphics cards and embedded
\end{itemize}
\end{itemize}
\end{frame}

\begin{frame}{Display Support in Linux and Userspace}

Userspace software status:
\begin{itemize}
\item DRM KMS supported and used by most components
\item fbdev still supported as fallback
\item fbdev still used by quick-and-dirty projects (please stop)
\end{itemize}~\\

Userspace software display support:
\begin{itemize}
\item \textbf{Display servers}: weston, sway, mutter, kwin, xorg
\item \textbf{Graphics libraries}: SDL, Qt (eglfs), DirectFB2, LVGL, Mesa (GBM)
\item \textbf{Media libraries}: GStreamer (kmssink), FFmpeg (kmsgrab), libcamera
\item \textbf{Tools}: modetest, igt, kmscube, glmark2
\end{itemize}

\end{frame}

\subsection{DRM KMS Internals}

\begin{frame}[fragile]{DRM KMS Components Overview}
Kernel hardware components:
\begin{itemize}
\item Drivers registered from bus infrastructure:
\begin{itemize}
\item \verb,pci, bus for graphics cards
\item \verb,platform, bus for SoC units
\item \verb,i2c,, \verb,spi,, etc for bridges and panels
\item \verb,platform, or \verb,mipi_dsi, for panels
\end{itemize}
\item Drivers register components to DRM KMS framework:
\begin{itemize}
\item \kstruct{drm_device}, for display controllers
\item \kstruct{drm_bridge}, for bridges (internal or external)
\item \kstruct{drm_panel}, for panels
\end{itemize}
\item Drivers need to identify eachother:
\begin{itemize}
\item Represent pipeline topology
\item Retrieve remote component structures for API use
\item Static description via device-tree graph with \verb,port,/\verb,endpoint, (or ACPI)
\item Node nesting for bus (DSI)
\end{itemize}
\end{itemize}
\end{frame}

\begin{frame}[fragile]{DRM KMS Components Integration (Device-Tree Example)}

\begin{minipage}[t]{0.45\textwidth}
\begin{minted}[fontsize=\scriptsize]{perl}
tcon0: lcd-controller@1c0c000 {
  compatible = "allwinner,sun50i-a64-tcon-lcd",
               "allwinner,sun8i-a83t-tcon-lcd";
  [...]

  ports {
    [...]

    tcon0_out: port@1 {
      reg = <1>;
      [...]

      tcon0_out_dsi: endpoint@1 {
        reg = <1>;
        remote-endpoint = <&dsi_in_tcon0>;
        allwinner,tcon-channel = <1>;
      };
    };
  };
};
\end{minted}
\end{minipage}
\hfill
\begin{minipage}[t]{0.45\textwidth}
\begin{minted}[fontsize=\scriptsize]{perl}
dsi: dsi@1ca0000 {
  compatible = "allwinner,sun50i-a64-mipi-dsi";
  [...]

  port {
    dsi_in_tcon0: endpoint {
      remote-endpoint = <&tcon0_out_dsi>;
    };
  };

  panel@0 {
    compatible = "xingbangda,xbd599";
    reg = <0>;
    [...]
  };
};
\end{minted}
\end{minipage}

\end{frame}

\begin{frame}[fragile]{DRM Display Controller: Base Driver}

\begin{itemize}
\item Driver data static declaration: \kstruct{drm_driver},
\begin{itemize}
\item \verb,driver_features,: bitfield of \ksym{DRIVER_MODESET}, \ksym{DRIVER_ATOMIC}, \ksym{DRIVER_GEM}
\item \verb,fops,: default definitions with \ksym{DEFINE_DRM_GEM_FOPS},
\item \verb,name,, \verb,desc,, \verb,date,/\verb,major,/\verb,minor, for information
\item Various operation callbacks, default definitions with \ksym{DRM_GEM_DMA_DRIVER_OPS}
\end{itemize}
\item Device data: \kstruct{drm_device}
\begin{itemize}
\item Created by the DRM framework
\item Associated with character devices (\verb,card,, \verb,render,) for uAPI
\end{itemize}
\item Setup at \codelink{probe()}:
\begin{itemize}
\item Allocate \kstruct{drm_device} from \kstruct{drm_driver} with \kfunc{drm_dev_alloc}\\
devm-managed with \kfunc{devm_drm_dev_alloc}, can allocate parent structure
\item Check if mode setting is allowed with \kfunc{drm_firmware_drivers_only}\\
honor the \verb,nomodeset, kernel command line parameter
\item Initialize display controller components
\item Register device with \kfunc{drm_dev_register}
\end{itemize}
\end{itemize}

\end{frame}

\begin{frame}{DRM Display Controller: Base Driver}

\begin{itemize}
\item Components registration typical order:
\begin{itemize}
\item Planes with \kstruct{drm_plane}
\item CRTC with \kstruct{drm_crtc}, attach to planes
\item Encoder with \kstruct{drm_encoder}, attach to CRTC
\item Connector with \kstruct{drm_connector}, attach to encoder
\end{itemize}
\item Cleanup at \codelink{remove()}:
\begin{itemize}
\item Unregister with \kfunc{drm_dev_unregister}
\item Call shutdown helper \kfunc{drm_atomic_helper_shutdown} to stop CRTCs
\item Put reference (non-devm) with \kfunc{drm_dev_put}
\end{itemize}
\item Power management at \codelink{suspend()}/\codelink{resume()}:
\begin{itemize}
\item Pipeline configuration saved/disabled at suspend with: \kfunc{drm_mode_config_helper_suspend}
\item Pipeline configuration restored/enabled at resume with: \kfunc{drm_mode_config_helper_resume}
\end{itemize}
\end{itemize}

\end{frame}

\begin{frame}[fragile]{DRM Display Controller: Memory Management}

\begin{itemize}
\item Translation Table Manager (TTM) memory manager:
\begin{itemize}
\item Historic memory manager for DRM, big and complex
\item Supports both shared system memory and dedicated video memory,\\
used by graphics card drivers
\end{itemize}
\item Graphics Execution Manager (GEM) memory manager:
\begin{itemize}
\item New design from Intel, focused on sharing code
\item Only supports shared system memory, used by most embedded drivers
\item Provides \kstruct{drm_driver} \verb,fops, and callbacks with \ksym{DEFINE_DRM_GEM_FOPS} and \ksym{DRM_GEM_DMA_DRIVER_OPS}
\item Supports driver-specific \verb,dumb_create, operation with \ksym{DRM_GEM_DMA_DRIVER_OPS_WITH_DUMB_CREATE}
\item Allocates write-combined DMA buffers with \kfunc{dma_alloc_wc}\\
contiguous or not depending on IOMMU presence, coherent
\item Also supports non-coherent (requires explicit sync)
\item Helper to get framebuffer DMA address: \kfunc{drm_fb_dma_get_gem_addr}
\end{itemize}
\end{itemize}

\end{frame}

\begin{frame}[fragile]{DRM Display Controller: Memory Management (CMA, Device-Tree)}

\begin{itemize}
\item GEM can typically use the Contiguous Memory Allocator (CMA):
\begin{itemize}
\item Meant for large contiguous buffer allocation
\item Uses reclaimable reserved pools of (DRAM) memory
\item Reserved early at boot with static size
\end{itemize}
\item Default system pool available to every device:
\begin{itemize}
\item Size configured with the \kconfig{CONFIG_CMA_SIZE_MBYTES} option\\
or with the \verb,cma, kernel command line parameter
\end{itemize}
\item Dedicated pool attached to specific devices:
\begin{itemize}
\item Declared in device-tree with compatible \kcompat{shared-dma-pool}{shared-dma-pool}\\
under \verb,reserved-memory, node
\item Attached to node in device-tree with \verb,memory-region, property
\item Attached to \kstruct{device} with \kfunc{of_reserved_mem_device_init}
\item Detached from \kstruct{device} with \kfunc{of_reserved_mem_device_release}
\item DMA allocations with \kstruct{device} then automatically use dedicated pool
\item Can be made default pool with \verb:linux,cma-default;: in device-tree
\end{itemize}
\end{itemize}

\end{frame}

\begin{frame}[fragile]{DRM Display Controller: Memory Management (CMA, Device-Tree)}

\begin{minipage}[t]{0.45\textwidth}
\begin{minted}[fontsize=\scriptsize]{perl}
/ {
  [...]

  reserved-memory {
    #address-cells = <1>;
    #size-cells = <1>;
    ranges;

    gfx_memory: framebuffer {
      size = <0x01000000>;
      alignment = <0x01000000>;
      compatible = "shared-dma-pool";
      reusable;
    };
  };
};
\end{minted}
\end{minipage}
\hfill
\begin{minipage}[t]{0.45\textwidth}
\begin{minted}[fontsize=\scriptsize]{perl}
gfx: display@1e6e6000 {
  compatible = "aspeed,ast2600-gfx", "syscon";
  [...]

  memory-region = <&gfx_memory>;
};
\end{minted}
\end{minipage}

\end{frame}

\begin{frame}[fragile]{DRM Display Controller: Mode Config}

\begin{itemize}
\item Mode config data: \kstruct{drm_mode_config} from \kstruct{drm_device}
\begin{itemize}
\item \verb;{min,max}_{width,height};: framebuffer dimension limits
\item \verb,preferred_depth,: default framebuffer pixel depth
\item \verb,funcs,: driver-specific \kstruct{drm_mode_config_funcs}
\end{itemize}
\item Mode config functions: \kstruct{drm_mode_config_funcs} (boilerplate)
\begin{itemize}
\item \verb,fb_create,: framebuffer creation with \kfunc{drm_gem_fb_create}
\item \verb,atomic_check,: atomic commit validation with \kfunc{drm_atomic_helper_check}
\item \verb,atomic_commit,: atomic commit entry point with \kfunc{drm_atomic_helper_commit}
\end{itemize}
\item Setup at \codelink{probe()}:
\begin{itemize}
\item Initialize mode config with \kfunc{drmm_mode_config_init}\\
automatically calls \kfunc{drm_mode_config_cleanup} using \verb,destroy, functions
\item Configure mode config data fields
\item Reset global pipeline state with \kfunc{drm_mode_config_reset} using \verb,reset, functions
\item Register device with \kfunc{drm_dev_register}
\end{itemize}
\end{itemize}

\end{frame}

\begin{frame}{DRM Display Controller: Atomic}

\begin{itemize}
\item Atomic support:
\begin{itemize}
\item Group batches of changes together as atomic commit
\item Provided by userspace as a list of property changes
\item Atomic state managed by the KMS framework
\end{itemize}

\item Atomic state: \kstruct{drm_atomic_state}
\begin{itemize}
\item New/old configuration state for internal components
\item Derived to component-specific structures
\item Used to configure hardware registers
\end{itemize}

\item Non-atomic is considered legacy and not covered here
\begin{itemize}
\item Expected that new KMS drivers are atomic nowadays
\item Converting non-atomic drivers is welcome
\end{itemize}
\end{itemize}

\end{frame}

\begin{frame}[fragile]{DRM Display Controller: Planes}

\begin{itemize}
\item Plane data: \kstruct{drm_plane}
\begin{itemize}
\item \verb,type,: one of \ksym{DRM_PLANE_TYPE_PRIMARY}, \ksym{DRM_PLANE_TYPE_OVERLAY}, \ksym{DRM_PLANE_TYPE_CURSOR}
\item \verb,possible_crtcs,: valid CRTCs with \kfunc{drm_crtc_mask}
\item \verb,formats,: list of supported pixel formats
\item \verb,modifiers,: list of supported pixel format modifiers (tiling, etc)
\end{itemize}
\item Plane functions: \kstruct{drm_plane_funcs} (boilerplate)
\begin{itemize}
\item \verb,reset,: plane state reset with \kfunc{drm_atomic_helper_plane_reset}
\item \verb,destroy,: plane destruction with \kfunc{drm_plane_cleanup}
\item \verb;atomic_{duplicate,destroy}_state;: plane state handling with \kfunc{drm_atomic_helper_plane_duplicate_state}, \kfunc{drm_atomic_helper_plane_destroy_state}
\item \verb;{update,disable}_plane;: plane configuration with \kfunc{drm_atomic_helper_update_plane}, \kfunc{drm_atomic_helper_disable_plane}
\end{itemize}
\end{itemize}

\end{frame}

\begin{frame}[fragile]{DRM Display Controller: Planes}

\begin{itemize}
\item Plane atomic state: \kstruct{drm_plane_state}
\begin{itemize}
\item New/old plane states available from \kstruct{drm_atomic_state}:\\
\kfunc{drm_atomic_get_new_plane_state}, \kfunc{drm_atomic_get_old_plane_state}
\item Associated CRTC: \kstruct{drm_crtc}
\item Associated framebuffer: \kstruct{drm_framebuffer}
\item Various base and optional properties
\end{itemize}
\item Plane helper functions: \kstruct{drm_plane_helper_funcs}
\begin{itemize}
\item \verb,atomic_check,: driver-specific atomic state validation\\
can use \kfunc{drm_atomic_helper_check_plane_state} with scaling/position features
\item \verb,atomic_update,: driver-specific plane configuration\\
register configuration using plane (and crtc) atomic state
\item \verb,atomic_disable,: driver-specific plane disable
\end{itemize}
\item Setup at \codelink{probe()}:
\begin{itemize}
\item Initialize and register \kstruct{drm_plane} with \kfunc{drm_universal_plane_init}
\item Register plane helpers with \kfunc{drm_plane_helper_add}
\item Configure available plane properties
\end{itemize}
\end{itemize}

\end{frame}

\begin{frame}{DRM Display Controller: Plane Properties}

\begin{itemize}
\item Plane properties are exposed to userspace and used in configuration
\item Base properties are registered with \kfunc{drm_universal_plane_init}
\item Optional properties can be registered by driver:
\begin{itemize}
\item Plane-wide alpha: \kfunc{drm_plane_create_alpha_property}
\item Plane stacking order: \kfunc{drm_plane_create_zpos_property}, \kfunc{drm_plane_create_zpos_immutable_property}
\item Rotation: \kfunc{drm_plane_create_rotation_property}
\item Blend mode: \kfunc{drm_plane_create_blend_mode_property}
\item Scaling filter: \kfunc{drm_plane_create_scaling_filter_property}
\item Custom driver-specific properties may also exist
\end{itemize}
\end{itemize}

\end{frame}

\begin{frame}[fragile]{DRM Display Controller: Metadata}

\begin{itemize}
\item Display mode: \kstruct{drm_display_mode}
\begin{itemize}
\item Describes display timings and \textit{some} signal characteristics\\
(sync polarity, composite sync, double/half clock rate)
\item List provided by connector, with preferred indication
\item Chosen by userspace to configure CRTC with property
\end{itemize}

\item Display information: \kstruct{drm_display_info}
\begin{itemize}
\item Describes pixel interface characteristics
\item \verb,bus_formats,: list of \codelink{MEDIA_BUS_FMT_*} formats
\item \verb,bus_flags,: bitfield of \codelink{DRM_BUS_FLAG_*} for signal characteristics
\item \verb;{width,height}_mm;: physical surface dimensions
\end{itemize}

\item Both are retreived either:
\begin{itemize}
\item Dynamically with EDID (\kstruct{edid}) via DDC for monitors
\item Statically with hardcoded definitions for panels
\end{itemize}
\end{itemize}

\end{frame}

\begin{frame}[fragile]{DRM Display Controller: Connector}

\begin{itemize}
\item Connector data: \kstruct{drm_connector}
\begin{itemize}
\item \verb,type,: display interface indication, \codelink{DRM_MODE_CONNECTOR_*}
\item \verb,status,: one of \ksym{connector_status_connected},\\\ksym{connector_status_disconnected}, \ksym{connector_status_unknown}
\item \verb,possible_encoders,: valid encoders with \kfunc{drm_encoder_mask}
\item \verb,modes,: list of available display interface modes
\end{itemize}
\item Connector functions: \kstruct{drm_connector_funcs} (boilerplate)
\begin{itemize}
\item \verb,reset,: connector state reset with \kfunc{drm_atomic_helper_connector_reset}
\item \verb,destroy, connector destruction with \kfunc{drm_connector_cleanup}
\item \verb;atomic_{duplicate,destroy}_state;: connector state handling with \kfunc{drm_atomic_helper_connector_duplicate_state}, \kfunc{drm_atomic_helper_connector_destroy_state}
\item \verb,fill_modes,: get mode list from available sources with \kfunc{drm_helper_probe_single_connector_modes}
\end{itemize}
\end{itemize}

\end{frame}

\begin{frame}[fragile]{DRM Display Controller: Connector}

\begin{itemize}
\item Connector atomic state: \kstruct{drm_connector_state}
\begin{itemize}
\item New/old connector states available from \kstruct{drm_atomic_state}:
\kfunc{drm_atomic_get_new_connector_state}, \kfunc{drm_atomic_get_old_connector_state}
\item Associated CRTC and encoder: \kstruct{drm_crtc}, \kstruct{drm_encoder}
\item Various base and optional properties
\end{itemize}

\item Connector helper functions: \kstruct{drm_connector_helper_funcs}
\begin{itemize}
\item \verb,get_modes,: retrieve list of modes with \kfunc{drm_get_edid} and \kfunc{drm_add_edid_modes} or \kfunc{drm_panel_get_modes}
\item \verb;mode_{valid,fixup};: validate/fixup proposed mode with hardware constraints
\item \verb,detect,: detect connector status
\end{itemize}

\item Setup at \codelink{probe()}:
\begin{itemize}
\item Initialize and register \kstruct{drm_connector} with \kfunc{drm_connector_init}
\item Register connector helpers with \kfunc{drm_connector_helper_add}
\item Attach encoder to connector with \kfunc{drm_connector_attach_encoder}
\end{itemize}
\end{itemize}

\end{frame}

\begin{frame}[fragile]{DRM Display Controller: Connector Hotplug}

\begin{itemize}
\item Connector status detection:
\begin{itemize}
\item Detect connector plug/unplug for monitors
\item Using dedicated pin or status, associated interrupt or not
\item Updates \kstruct{drm_connector} \verb,status, using \verb,detect, helper function
\item Notify userspace via sysfs uevent \verb,HOTPLUG=1, (and \verb,CONNECTOR=*,)
\end{itemize}

\item Interrupt-based detection:
\begin{itemize}
\item Set \kstruct{drm_connector} \verb,polled, field to \ksym{DRM_CONNECTOR_POLL_HPD}
\item Event reported from IRQ handler with \kfunc{drm_connector_helper_hpd_irq_event} or \kfunc{drm_helper_hpd_irq_event} (global)
\end{itemize}

\item Active polling (10 Hz):
\begin{itemize}
\item Set \kstruct{drm_connector} \verb,polled, field to \ksym{DRM_CONNECTOR_POLL_CONNECT} \codelink{|} \ksym{DRM_CONNECTOR_POLL_DISCONNECT}
\item Dedicated worker started with \kfunc{drm_kms_helper_poll_init}
\item Dedicated worker stopped with \kfunc{drm_kms_helper_poll_fini}
\end{itemize}

\end{itemize}

\end{frame}

\begin{frame}[fragile]{DRM Display Controller: CRTC}

\begin{itemize}
\item CRTC data: \kstruct{drm_crtc}
\begin{itemize}
\item \verb,primary,, \verb,cursor,: legacy planes for compatibility (replaced by atomic state)
\item \verb,mode,: legacy mode for compatibility (replaced by atomic state)
\end{itemize}

\item CRTC functions: \kstruct{drm_crtc_funcs} (mostly boilerplate)
\begin{itemize}
\item \verb,reset,: crtc state reset with \kfunc{drm_atomic_helper_crtc_reset}
\item \verb,destroy, crtc destruction with \kfunc{drm_crtc_cleanup}
\item \verb;atomic_{duplicate,destroy}_state;: crtc state handling with \kfunc{drm_atomic_helper_crtc_duplicate_state}, \kfunc{drm_atomic_helper_crtc_destroy_state}
\item \verb,set_config,: crtc configuration with \kfunc{drm_atomic_helper_set_config}
\item \verb,page_flip,: crtc page flip with \kfunc{drm_atomic_helper_page_flip}
\item \verb,enable_vblank,: driver-specific vblank interrupt enable
\item \verb,disable_vblank,: driver-specific vblank interrupt disable
\end{itemize}
\end{itemize}

\end{frame}

\begin{frame}[fragile]{DRM Display Controller: CRTC}

\begin{itemize}
\item CRTC atomic state: \kstruct{drm_crtc_state}
\begin{itemize}
\item New/old crtc states available from \kstruct{drm_atomic_state}:
\kfunc{drm_atomic_get_new_crtc_state}, \kfunc{drm_atomic_get_old_crtc_state}
\item Associated planes, connectors and encoders with \kfunc{drm_plane_mask}, \kfunc{drm_connector_mask}, \kfunc{drm_encoder_mask}
\item Adjusted and requested modes: \verb,adjusted_mode,, \verb,mode,
\item Pending vblank \verb,event, with \kstruct{drm_pending_vblank_event}
\item Various properties (gamma, scaling filter)
\end{itemize}

\item CRTC helper functions: \kstruct{drm_crtc_helper_funcs}
\begin{itemize}
\item \verb;mode_{valid,fixup};: validate/fixup proposed mode with hardware constraints
\item \verb,atomic_check,: driver-specific atomic state validation
\item \verb,atomic_enable,: driver-specific crtc configuration using atomic state,\\
enable vblank with \kfunc{drm_crtc_vblank_on}
\item \verb,atomic_disable,: driver-specific crtc disable,\\
disable vblank with \kfunc{drm_crtc_vblank_off}
\end{itemize}
\end{itemize}

\end{frame}

\begin{frame}[fragile]{DRM Display Controller: CRTC}

\begin{itemize}
\item Setup at \codelink{probe()}:
\begin{itemize}
\item Initialize and register \kstruct{drm_crtc} with \kfunc{drm_crtc_init_with_planes}
\item Register CRTC helpers with \kfunc{drm_crtc_helper_add}
\item Provide \verb,port, \kstruct{device_node} with \kfunc{of_graph_get_port_by_id},\\
used by \kfunc{drm_of_find_possible_crtcs} with multiple device-tree nodes
\end{itemize}

\item Vblank reporting with \kfunc{drm_crtc_handle_vblank} in interrupt handler
\begin{itemize}
\item Wake-up any task waiting for vblank
\end{itemize}
\item Vblank userspace event handling: \kstruct{drm_pending_vblank_event}
\begin{itemize}
\item Locked with \kstruct{drm_device} \verb,event_lock,
\item Grabbed at \verb,atomic_enable, from atomic state \verb,event, with \kfunc{drm_crtc_vblank_get}\\
copied and removed from \kstruct{drm_crtc_state}
\item Returned in interrupt handler with \kfunc{drm_crtc_send_vblank_event} and \kfunc{drm_crtc_vblank_put}
\end{itemize}
\end{itemize}

\end{frame}

\begin{frame}[fragile]{DRM Display Controller: Encoder}

\begin{itemize}
\item Encoder data: \kstruct{drm_encoder}
\begin{itemize}
\item \verb,type,: physical encoding indication, \codelink{DRM_MODE_ENCODER_*}
\item \verb,possible_crtcs,: valid CRTCs with \kfunc{drm_crtc_mask}
\end{itemize}

\item Encoder functions: \kstruct{drm_encoder_funcs} (mostly boilerplate)
\begin{itemize}
\item \verb,reset,: encoder state reset (optional)
\item \verb,destroy, encoder destruction with \kfunc{drm_encoder_cleanup}
\end{itemize}

\end{itemize}

\end{frame}

\begin{frame}[fragile]{DRM Display Controller: Encoder}

\begin{itemize}
\item No encoder atomic state, using crtc and connector state

\item Encoder helper functions: \kstruct{drm_encoder_helper_funcs}
\begin{itemize}
\item \verb;mode_{valid,fixup};: validate/fixup proposed mode with hardware constraints
\item \verb,atomic_check,: driver-specific atomic state validation
\item \verb,atomic_enable,: driver-specific encoder configuration using atomic state
\item \verb,atomic_disable,: driver-specific encoder disable
\end{itemize}

\item Setup at \codelink{probe()}:
\begin{itemize}
\item Initialize and register \kstruct{drm_encoder} with \kfunc{drm_encoder_init} or \kfunc{drm_simple_encoder_init} (boilerplate funcs)
\item Register encoder helpers with \kfunc{drm_encoder_helper_add}
\item Attach encoder to connector with \kfunc{drm_connector_attach_encoder}
\end{itemize}
\end{itemize}

\end{frame}

\begin{frame}[fragile]{DRM Bridge}

\begin{itemize}
\item Bridge data: \kstruct{drm_bridge}
\begin{itemize}
\item \verb,ops,: bitfield of \codelink{DRM_BRIDGE_OP_DETECT}, \codelink{DRM_BRIDGE_OP_EDID}, \codelink{DRM_BRIDGE_OP_HPD}, \codelink{DRM_BRIDGE_OP_MODES}
\item \verb,type,: terminal connector type, \codelink{DRM_MODE_CONNECTOR_*}
\item \verb,timings,: optional \kstruct{drm_bridge_timings} with input bus flags
\item \verb,chain_node,: list of \kstruct{drm_bridge} for chaining bridges
\item \verb,encoder,: \kstruct{drm_encoder} currently attached to the bridge
\item Not tied to a specific \kstruct{drm_device}
\end{itemize}
\item Bridge functions: \kstruct{drm_bridge_funcs} (boilerplate)
\begin{itemize}
\item \verb;atomic_{duplicate,destroy}_state;: bridge state handling with \kfunc{drm_atomic_helper_bridge_duplicate_state}, \kfunc{drm_atomic_helper_bridge_destroy_state,}
\item \verb,atomic_reset,: bridge state reset with \kfunc{drm_atomic_helper_bridge_reset}
\item No \verb,cleanup, callback since \kfunc{drm_mode_config_cleanup} relates to \kstruct{drm_device}
\end{itemize}
\end{itemize}

\end{frame}

\begin{frame}[fragile]{DRM Bridge}

\begin{itemize}
\item Bridge state: \kstruct{drm_bridge_state} (useful for chaining)
\begin{itemize}
\item New/old bridge states available from \kstruct{drm_atomic_state}:
\kfunc{drm_atomic_get_new_bridge_state}, \kfunc{drm_atomic_get_old_bridge_state}
\item Input/output bus configuration: \kstruct{drm_bus_cfg} with\\
\codelink{MEDIA_BUS_FMT_*} if available or \ksym{MEDIA_BUS_FMT_FIXED}
\end{itemize}
\item Bridge functions: \kstruct{drm_bridge_funcs}
\begin{itemize}
\item \verb,attach,/\verb,detach,: create/destroy connector, unless \ksym{DRM_BRIDGE_ATTACH_NO_CONNECTOR} flag is specified\\
or attach/detach next bridge with \kfunc{drm_bridge_attach}
\item \verb;mode_{valid,fixup};: validate/fixup proposed mode with hardware constraints
\item \verb;atomic_get_{input,output}_bus_fmts;: report supported input/output bus formats for negotiation in bridge chains
\item \verb,atomic_enable,: driver-specific bridge configuration using (crtc) atomic state
\item \verb,atomic_disable,: driver-specific bridge disable
\item \verb,atomic_check,: driver-specific atomic state validation
\end{itemize}
\end{itemize}

\end{frame}

\begin{frame}[fragile]{DRM Bridge}

\begin{itemize}
\item Setup at \codelink{probe()} (dedicated driver):
\begin{itemize}
\item Configure \verb,type,, \verb,ops,, \verb,funcs,, \verb,of_node, and possibly \verb,timings,
\item Initialize and register \kstruct{drm_bridge} with \kfunc{drm_bridge_add}\\
devm-managed with \kfunc{devm_drm_bridge_add}
\end{itemize}
\item Cleanup at \codelink{remove()}:
\begin{itemize}
\item Unregister and cleanup with \kfunc{drm_bridge_remove}
\end{itemize}

\item Display controller driver usage:
\begin{itemize}
\item Identified via device-tree with \kfunc{drm_of_find_panel_or_bridge} or \kfunc{devm_drm_of_get_bridge}
\item Attached to encoder with \kfunc{drm_bridge_attach},\\
\item Automatically detached during encoder cleanup
\item Connector is created by (final) bridge directly
\item Bridge functions are called automatically once attached with encoder
\end{itemize}
\end{itemize}

\end{frame}

\begin{frame}[fragile]{DRM Panel}

\begin{itemize}
\item Panel data: \kstruct{drm_panel}
\begin{itemize}
\item \verb,backlight,: \kstruct{backlight_device} for the attached backlight
\item \verb,connector_type,: display interface indication, \codelink{DRM_MODE_CONNECTOR_*}
\item Not tied to a specific \kstruct{drm_device}
\end{itemize}

\item Panel functions: \kstruct{drm_panel_funcs}
\begin{itemize}
\item \verb,prepare,: Setup panel (typically power lines, configuration)
\item \verb,enable,: Turn on panel, start expecting data
\item \verb,disable,: Turn off panel, stop expecting data
\item \verb,unprepare,: Cleanup panel
\item \verb;get_{modes,timings};: return list of supported modes/timings
\end{itemize}
\item No panel atomic state:
\begin{itemize}
\item Timings are known in advance with a single mode (typical)
\item Generally no need to configure timings explicity
\end{itemize}
\end{itemize}

\end{frame}

\begin{frame}[fragile]{DRM Panel}

\begin{itemize}
\item Setup at \codelink{probe()} (dedicated driver):
\begin{itemize}
\item Initialize panel with \kfunc{drm_panel_init} given \verb,funcs, and \verb,type,
\item Attach backlight via device-tree using \kfunc{drm_panel_of_backlight}
\item Register panel with \kfunc{drm_panel_add}
\end{itemize}

\item Cleanup at \codelink{remove()}:
\begin{itemize}
\item Unregister panel with \kfunc{drm_panel_remove}
\end{itemize}

\item Display controller driver usage (deprecated):
\begin{itemize}
\item Identified via device-tree with \kfunc{drm_of_find_panel_or_bridge}
\item Register encoder and connector for panel
\item Return panel modes in connector \verb,get_modes, with \kfunc{drm_panel_get_modes}
\item Enable panel in encoder \verb,atomic_enable, with \kfunc{drm_panel_prepare} and \kfunc{drm_panel_enable} (automatically enables backlight)
\item Disable panel in encoder \verb,atomic_disable, with \kfunc{drm_panel_unprepare} and \kfunc{drm_panel_disable}
\end{itemize}
\end{itemize}

\end{frame}

\begin{frame}[fragile]{DRM Panel Bridge}

\begin{itemize}
\item Differentiated handling for bridges and panels is painful:
\begin{itemize}
\item Both are \textit{in-fine} connected to an encoder
\item Panels require drm device to manage the connector
\item Bridges register their own connector
\item Provided functions are comparable (enable, disable, modes list)
\end{itemize}

\item DRM Panel Bridge API closes the gap:
\begin{itemize}
\item Unified API for both, using \kstruct{drm_bridge}
\item Boilerplate connector registered transparently for panels
\item Calls back panel functions in bridge functions
\end{itemize}

\item Display controller driver usage:
\begin{itemize}
\item Use \kfunc{devm_drm_of_get_bridge} instead of \kfunc{drm_of_find_panel_or_bridge}
\item Use bridge normally, no particular difference
\end{itemize}
\end{itemize}

\end{frame}

\begin{frame}{DRM Bridge and Panel Drivers}

\begin{itemize}
\item Generic drivers provide static data and regulator/gpio integration

\item Generic bridge drivers:
\begin{itemize}
\item \ksym{simple-bridge}: static bridge timings with dedicated device-tree compatibles
\item \ksym{lvds-codec}: generic device-tree compatibles
\item \ksym{display-connector}: generic device-tree compatibles
\end{itemize}

\item Generic panel drivers:
\begin{itemize}
\item \ksym{panel-simple}: static modes list, panel-specific device-tree compatibles
\item \ksym{panel-lvds}: timings from device-tree properties
\item \ksym{panel-edp}: static modes list, panel-specific device-tree compatibles,\\edid fixup tables
\end{itemize}

\item Specific drivers:
\begin{itemize}
\item Generally require specific register configuration
\item Pitfall: panel and LCD controller confusion
\item Device-tree compatible must \textbf{not} be the LCD controller name
\item Having a common driver for a LCD controller is a good idea\\
(with panel-specific compatibles)
\end{itemize}

\end{itemize}

\end{frame}

\begin{frame}[fragile]{DRM Bridge/Panel Generic Drivers: panel-simple}

\begin{minipage}[t]{0.45\textwidth}
\begin{minted}[fontsize=\scriptsize]{c}
static const struct drm_display_mode
lemaker_bl035_rgb_002_mode = {
  .clock = 7000,
  .hdisplay = 320,
  .hsync_start = 320 + 20,
  .hsync_end = 320 + 20 + 30,
  .htotal = 320 + 20 + 30 + 38,
  .vdisplay = 240,
  .vsync_start = 240 + 4,
  .vsync_end = 240 + 4 + 3,
  .vtotal = 240 + 4 + 3 + 15,
};

static const struct of_device_id platform_of_match[] = {
  [...]
  {
    .compatible = "lemaker,bl035-rgb-002",
    .data = &lemaker_bl035_rgb_002,
  },
  [...]
};
\end{minted}
\end{minipage}
\hfill
\begin{minipage}[t]{0.45\textwidth}
\begin{minted}[fontsize=\scriptsize]{c}
static const struct panel_desc
lemaker_bl035_rgb_002 = {
  .modes = &lemaker_bl035_rgb_002_mode,
  .num_modes = 1,
  .size = {
    .width = 70,
    .height = 52,
  },
  .bus_format = MEDIA_BUS_FMT_RGB888_1X24,
  .bus_flags = DRM_BUS_FLAG_DE_LOW,
};
\end{minted}
\end{minipage}

\end{frame}

\begin{frame}[fragile]{DRM Repositories and Lists}
Repositories:
\begin{itemize}
\item DRM (top): \url{https://cgit.freedesktop.org/drm/drm}
\begin{itemize}
\item Branches: \verb,drm-next,, \verb,drm-fixes,
\item Maintainer: Dave Airlie
\end{itemize}
\item DRM Misc: \url{https://cgit.freedesktop.org/drm/drm-misc}
\begin{itemize}
\item Branches: \verb,drm-misc-next,, \verb,drm-misc-fixes,
\item Maintainers: Maarten Lankhorst, Maxime Ripard, and Thomas Zimmermann
\end{itemize}
\item Hardware-specific:
\begin{itemize}
\item DRM Intel: \url{https://cgit.freedesktop.org/drm/drm-intel}
\item DRM AMD: \url{https://gitlab.freedesktop.org/agd5f/linux.git}
\item DRM Nouveau: \url{https://gitlab.freedesktop.org/drm/nouveau}
\end{itemize}
\end{itemize}
\vspace{0.5em}
Patch submission and tracking:
\begin{itemize}
\item Patchwork: \url{https://patchwork.freedesktop.org}
\item Mailing list: \url{dri-devel@lists.freedesktop.org}
\end{itemize}

\end{frame}

\questionslide

\end{document}
