\documentclass[aspectratio=169,obeyspaces,spaces,hyphens,dvipsnames]{beamer}
\usepackage[utf8]{inputenc}
\usepackage{lmodern}% http://ctan.org/pkg/lm
\usepackage{minted}
\usepackage{hyperref}
\usepackage{xcolor}
\usepackage{pgfplots}
\usepackage{tikz}
\usepackage{stmaryrd}
\usepackage[normalem]{ulem}

\mode<presentation>
\usetheme{Bootlin}

\def\signed #1{{\leavevmode\unskip\nobreak\hfil\penalty50\hskip2em
  \hbox{}\nobreak\hfil(#1)
  \parfillskip=0pt \finalhyphendemerits=0 \endgraf}}

\newsavebox\mybox
\newenvironment{aquote}[1]
  {\savebox\mybox{#1}\begin{quotation}}
  {\signed{\usebox\mybox}\end{quotation}}

\makeatletter
\newbox\@backgroundblock
\newenvironment{backgroundblock}[2]{%
  \global\setbox\@backgroundblock=\vbox\bgroup%
    \unvbox\@backgroundblock%
    \vbox to0pt\bgroup\vskip#2\hbox to0pt\bgroup\hskip#1\relax%
}{\egroup\egroup\egroup}
\addtobeamertemplate{background}{\box\@backgroundblock}{}
\makeatother

\tikzstyle{block} = [rectangle, draw, fill=white, text width=4em, text centered, minimum height=3em, node distance=10em]
\tikzstyle{blockish} = [draw, fill=white, text width=4em, text centered, node distance=8em]
\tikzstyle{arrow} = [thick,->,>=stealth]

\title{Advanced Camera Support on Allwinner SoCs with Mainline Linux}
\authors{Paul Kocialkowski}
\email{paul@bootlin.com}
\slidesurl{https://bootlin.com/pub/conferences/}
\institute{Bootlin}
\conference{Embedded Linux Conference 2021}

\begin{document}

\addtocontents{toc}{\protect\setcounter{tocdepth}{-1}}
\section{Advanced Camera Support on Allwinner SoCs with Mainline Linux}
\addtocontents{toc}{\protect\setcounter{tocdepth}{2}}

\begin{frame}{Paul Kocialkowski}
  \begin{itemize}
  \item Embedded Linux engineer at Bootlin
    \begin{itemize}
    \item Embedded Linux {\bf expertise}
    \item {\bf Development}, consulting and training
    \item Strong open-source focus
    \end{itemize}
  \item Open-source contributor
    \begin{itemize}
    \item Co-maintainer of the \textbf{cedrus} VPU driver in V4L2
    \item Contributor to the \textbf{sun4i-drm} DRM driver
    \item Contributing the \textbf{logicvc-drm} DRM driver
    \item Developed the \textbf{displaying and rendering graphics with Linux} training
    \end{itemize}
  \item Living in {\bf Toulouse}, south-west of France
  \end{itemize}
\end{frame}

\subsection{An Introduction to Image Capture Technology}

\begin{frame}{Overview of the Digital Image Capture Chain}

\begin{center}
\includegraphics[width=0.8\linewidth]{images/image-capture-pipeline.png}\\
An image capture chain
\end{center}~

\begin{itemize}
\item \textbf{Optics}: shape light rays
\item \textbf{Sensor}: convert light to digital values
\item \textbf{Interface}: transport values
\item \textbf{Processing}: produce good-looking pictures
\item \textbf{Display/encoding}: show/store pictures \textit{(out of the scope of this talk)}
\end{itemize}
\end{frame}

\begin{frame}{Hardware Interfaces for Capture}
Sensors need to transmit data:
\begin{itemize}
\item Analog interfaces (CVBS, etc) are mostly deprecated
\item \textbf{Parallel} digital interfaces: basic, BT.656\\
\textit{typically used with old and low-end sensors}
\item \textbf{Serial} digital interfaces: MIPI CSI-2, LVDS, SDI, HiSPi\\
\textit{typically used with high-end sensors}
\end{itemize}~

\textbf{Basic parallel} interface:
\begin{itemize}
\item One TTL signal per bit, usually 8/10/12/16/24 bits width
\item Pixel clock and sync signals (hsync, vsync)
\end{itemize}~

\textbf{MIPI CSI-2} serial interface:
\begin{itemize}
\item Differential pairs, using double data rate (DDR)
\item One clock lane (high rates) and 1-4 data lanes
\end{itemize}

\end{frame}

\begin{frame}{Processing RAW Images}
Data coming from a sensor ADC needs processing:
\begin{itemize}
\item Data corresponds to a \textbf{bayer pattern}, not pixels
\item Brightness is linear, not adapted for display
\item Sensors have a non-zero \textbf{dark-level current}
\item Noise is present, color is off, image looks bad
\item Enhancement takes place in \textbf{Image Signal Processors (ISPs)}
\end{itemize}~

Three distinct domains are involved:
\begin{enumerate}
\item \textbf{Bayer domain}, ends with debayering step
\item \textbf{RGB domain}, ends with YUV conversion
\item \textbf{YUV domain}, ends with final picture
\end{enumerate}
\end{frame}

\begin{frame}{Processing RAW Images: Illustration}

\begin{minipage}[b]{0.3\textwidth}
\begin{center}
\includegraphics[width=\linewidth]{images/step-bayer.png}
Bayer step
\end{center}
\end{minipage}
\hfill
\begin{minipage}[b]{0.3\textwidth}
\begin{center}
\includegraphics[width=0.8\linewidth]{images/step-rgb-wb.jpg}
\includegraphics[width=0.8\linewidth]{images/step-rgb-gamma.jpg}
\includegraphics[width=0.8\linewidth]{images/step-rgb.jpg}
RGB step
\end{center}
\end{minipage}
\hfill
\begin{minipage}[b]{0.3\textwidth}
\begin{center}
\includegraphics[width=0.8\linewidth]{images/step-y.jpg}
\includegraphics[width=0.8\linewidth]{images/step-u.jpg}
\includegraphics[width=0.8\linewidth]{images/step-v.jpg}
YUV step
\end{center}
\end{minipage}

\end{frame}

\begin{frame}{Image Enhancements in ISPs}

Various enhancements are usually applied to the image:
\begin{itemize}
\item \textbf{Dead pixel correction}: discard invalid values
\item \textbf{Black level correction}: remove dark level current
\item \textbf{White balance}: adjust R-G-B balance with coefficients/offsets
\item \textbf{Noise filtering}: remove electronic noise
\item \textbf{Color matrix}: adjust colors for fidelity
\item \textbf{Gamma}: adjust brightness curve for non-linearity
\item \textbf{Saturation}: adjust colorfulness
\item \textbf{Brightness}: adjust global luminosity
\item \textbf{Contrast}: adjust bright/dark difference
\end{itemize}
\end{frame}

\begin{frame}{Image Enhancements in ISPs}
More advanced enhancements may also be applied:
\begin{itemize}
\item \textbf{Lens shading}: correct lens irregular brightness
\item \textbf{Lens dewarp}: correct lens geometry distortion effect
\item \textbf{Stabilization}: crop to remove shaking
\item \textbf{Color LUT}: Translate colors with a specific style
\end{itemize}~

Hardware implementations:
\begin{itemize}
\item ISPs embedded in sensors tend to be \textbf{simple}
\begin{itemize}
\item Provide YUV data to the camera interface
\end{itemize}
\item Multimedia Systems on a Chip often have an \textbf{advanced ISP}
\begin{itemize}
\item Require raw bayer data on the camera interface
\item Require specific calibration data for the sensor/lens
\end{itemize}
\end{itemize}
\end{frame}

\begin{frame}{Parameters to Adjust}
Some parameters depend on the situation:
\begin{itemize}
\item \textbf{Focus} depends on the area of interest
\item \textbf{White balance} depends on the light source(s)
\item \textbf{Exposure} depends on the amount of light
\end{itemize}~

Exposure depends on a few parameters:
\begin{itemize}
\item Diaphragm \textbf{aperture} (f-number)
\item \textbf{Exposure time} (shutter speed)
\item \textbf{Amplifier gain} (ISO number equivalent)
\end{itemize}~

Advanced users will set parameters manually, with artistic implications
\end{frame}

\begin{frame}{Automatic Parameters Control with 3A}

In other cases, automatic parameters control is desirable:
\begin{itemize}
\item \textbf{Automatic exposition}: manage exposure time and gain (optionally diaphragm)
\item \textbf{Auto-focus}: detect blurry and sharp areas, adjust with focus coil
\item \textbf{Auto white balance}: detect dominant lighting and adjust
\end{itemize}~

Implemented using 3A algorithms:
\begin{itemize}
\item General algorithms described in \textbf{academic literature}
\item Involve a \textbf{feedback loop} system, using statistics
\item Implementations are usually \textbf{hardware specific} (ISP and sensor),\\
often considered to be the secret sauce!
\end{itemize}
\end{frame}

\subsection{Status of Allwiner Camera Support in\\Mainline Linux}

\begin{frame}{Allwinner Hardware for Camera Support}

\begin{itemize}
\item \textbf{CSI} controller for parallel/BT.656 interfaces
  \begin{itemize}
  \item First generation: A10, A13, A20, R40
  \item Second generation: A31, A23, A33, A83T, H3, H5, V3, A64
  \item Third generation (\textbf{CSIC}): A63, H6, V5, V536, V533, H616, D1
  \end{itemize}
  \begin{center}
  \textbf{Nearly always present}
  \end{center}
\item \textbf{MIPI CSI-2} interface controller
  \begin{itemize}
  \item Specific implementations: A80, A83T
  \item First generation: A31, V3, T7?
  \item Second generation (\textbf{combo} with sub-LVDS, HiSPi): V5, V536, V533
  \end{itemize}
  \begin{center}
  \textbf{Not always present}
  \end{center}
\item \textbf{ISP} processors
  \begin{itemize}
  \item First generation (glued to CSI): A10, A20, R40?
  \item Second generation (separate): A31, A80, (A23), (H3), (H5), A83T, V3
  \item Third generation (ISP500/ISP520/ISP521): V5, V536, H616
  \end{itemize}
  \begin{center}
  \textbf{Usually present when MIPI CSI-2 is present}
  \end{center}
\end{itemize}
\end{frame}

\begin{frame}{Mainline Linux Support and Allwinner Camera Support}
Allwinner platform support in mainline Linux:
\begin{itemize}
\item Long-time effort from the \textbf{sunxi community}, very active\\
  \url{https://linux-sunxi.org/Linux_mainlining_effort}
\item Multimedia areas are often the last missing parts
\item Allwinner started contributing very recently
\end{itemize}~

Camera support in mainline Linux:
\begin{itemize}
\item \code{sun4i-csi} driver for first generation CSI
\item \code{sun6i-csi} driver for second generation CSI
\item Third generation CSI support is missing
\item MIPI CSI-2 and ISP support was entirely missing\\
  \textit{non-free blobs for ISP support and A80 MIPI CSI-2 in SDK}
\end{itemize}
\end{frame}

\begin{frame}{Camera Support in Linux with V4L2}
\begin{center}
\textbf{Video4Linux2} (V4L2) is the subsystem/API for media support in Linux
\end{center}

\begin{itemize}
\item Supports various types of \textbf{pixel-related devices}\\
  \textit{basically anything that is not a display or gpu}
\item Provides userspace with \textbf{video devices} (e.g. \code{/dev/video0})
\item Implements a generic \textbf{userspace API} including:
\begin{itemize}
  \item Format negotiation, implemented in \code{struct v4l2_ioctl_ops}
  \item Memory management (alloc, free, mmap), implemented in \code{struct vb2_mem_ops}
  \item A queue interface for buffers of a given type (output, capture...),\\implemented in \code{struct vb2_ops}
  \item A control interface for configuration
\end{itemize}
\item Good fit for \textbf{all-in-one devices} (e.g. USB UVC cameras)\\
  \textit{assumes that a memory (DMA) interface is available}
\end{itemize}

\end{frame}

\begin{frame}{V4L2 Support for Complex Camera Systems : Subdevs}

Complex systems bring the need for \textbf{more refinement}:
\begin{itemize}
\item Internal blocks with FIFOs
\item External devices with interfaces (e.g. sensors)
\item Possibility to configure each block and the topology
\end{itemize}~

Hence the notion of \textbf{subdevs} was introduced to V4L2:
\begin{itemize}
\item Represent a single block (usually not DMA-capable)
\item Exposed to userspace via dedicated nodes \code{ /dev/v4l-subdev0}
\item Dedicated format configuration, implemented in \code{struct v4l2_subdev_pad_ops}
\item Dedicated stream management, implemented in \code{struct v4l2_subdev_video_ops}
\item Called by video devices with \code{v4l2_subdev_call}
\end{itemize}
\end{frame}

\begin{frame}{V4L2 Support for Complex Camera Systems : Subdevs Integration}
\begin{center}
Subdevs need to be \textbf{parented to a v4l2 device} (controlling entity)
\end{center}

Simple case: the \textbf{all-in-one driver}
\begin{itemize}
\item A single driver may register a parent v4l2 device, a video device and subdev(s)
\item The subdev can be registered directly:\\
\code{v4l2_device_register_subdev(v4l2_dev, subdev);}
\end{itemize}~\\

Complex case: \textbf{multiple drivers} involved
\begin{itemize}
\item The video device driver will typically register a v4l2 device
\item Each subdev driver will register its subdev asynchronously:\\
\code{v4l2_async_register_subdev(subdev);}
\item A driver that needs a subdev needs to identify and wait for it
\end{itemize}

\end{frame}

\begin{frame}[fragile]{V4L2 Support for Complex Camera Systems : Fwnode Graph}

The fwnode graph represents the connection between different blocks:
\begin{itemize}
\item Typically described in device-tree with port/endpoint
\item The meaning of each port is described in the device-tree bindings
\item Endpoints are retrieved by the driver and parsed with a helper:
\code{fwnode_graph_get_endpoint_by_id()}\\
\code{v4l2_fwnode_endpoint_parse()}
\item May contain an indication of the bus type:\\
\code{enum v4l2_mbus_type}, e.g. \code{V4L2_MBUS_CSI2_DPHY}
\item As well as bus-specific information:\\
e.g. \code{struct v4l2_fwnode_bus_mipi_csi2}
\end{itemize}

\end{frame}

\begin{frame}[fragile]{V4L2 Support for Complex Camera Systems : Fwnode Graph}
Device-tree example for camera to MIPI CSI-2 bridge:\\
~\\
\begin{minipage}[t]{0.45\textwidth}
  \begin{minted}[fontsize=\scriptsize]{perl}
imx219: camera@10 {
  compatible = "sony,imx219";
  ...
  port {
    camera_to_bridge: endpoint {
      data-lanes = <1 2>;
      link-frequencies = /bits/ 64 <456000000>;
      remote-endpoint = <&bridge_from_camera>;
    };
  };
};
\end{minted}
\end{minipage}
\hfill
\begin{minipage}[t]{0.45\textwidth}
  \begin{minted}[fontsize=\scriptsize]{perl}
mipi_csi2: csi@1cb1000 {
  compatible = "allwinner,sun8i-v3s-mipi-csi2";
  ...
  ports {
    ...
    port@0 {
      reg = <0>;
      bridge_from_camera: endpoint {
        data-lanes = <1 2>;
        remote-endpoint = <&camera_to_bridge>;
      };
    };
    ...
  };
};
  \end{minted}
\end{minipage}
\end{frame}

\begin{frame}{V4L2 Support for Complex Camera Systems : Async Subdevs}

\textbf{Async registration} allows other drivers to use the subdev:
\begin{itemize}
\item A link between devices is described with \textbf{fwnode graph}
\item An \textbf{async notifier} will match and notify when the subdev is available:\\
\code{v4l2_async_notifier_add_fwnode_remote_subdev}
\item The async notifier can be used by the driver with a v4l2 device:\\
\code{v4l2_async_notifier_register(v4l2_dev, notifier);}
\item Or by a subdev that needs another subdev (e.g. a bridge):\\
\code{v4l2_async_subdev_notifier_register(subdev, notifier);}
\item A callback gives the requesting driver a \code{struct v4l2_subdev}
\end{itemize}

\end{frame}


\begin{frame}{V4L2 Support for Complex Camera Systems : Media Controller}

The \textbf{media controller} API provides coordination between blocks:
\begin{itemize}
\item Each block is an \textbf{entity} with sink/source \textbf{pads}\\
derivated from a video device or a subdev
\item Entities declare a particular function\\
e.g. \code{MEDIA_ENT_F_PROC_VIDEO_PIXEL_FORMATTER}
\item \textbf{Links} between pads of entities are created by drivers,\\
may allow userspace to enable/disable them
\item Grouped in a media device (tied to a v4l2 device)
\item Performs \textbf{runtime validation} for links, implemented in \code{struct media_entity_operations}'s \code{link_validate}
\item Topology is \textbf{exposed to userspace}, usually controlled with \code{media-ctl}:\\
\code{media-ctl -l '"sun6i-csi-bridge":1 -> "sun6i-csi-capture":0[1]'}
\end{itemize}
\end{frame}

\begin{frame}{V4L2 Support for Complex Camera Systems : Media Controller}
\begin{center}
\includegraphics[width=0.55\linewidth]{images/imx-media-topology.png}\\
The i.MX capture driver's media topology
\end{center}
\end{frame}

\begin{frame}{V4L2 Support for Image Signal Processors (ISPs)}
Specific aspects related to ISPs:
\begin{itemize}
\item Usually have an internal pipeline with \textbf{multiple blocks}
\item Parameters are \textbf{highly specific} (not a good fit for V4L2 controls)
\item Provide stats \textbf{information buffers} (3A, histogram)
\item Exposes one or multiple \textbf{capture interfaces}
\end{itemize}~\\

ISPs integration in V4L2:
\begin{itemize}
\item Processor represented by a subdev/media entity:
\code{MEDIA_ENT_F_PROC_VIDEO_ISP}
\item \textbf{Capture video devices} for pixels: queues with type \code{V4L2_BUF_TYPE_VIDEO_CAPTURE}
\item \textbf{Meta output video devices} for parameters: queue with type \code{V4L2_BUF_TYPE_META_OUTPUT} with dedicated (struct) buffer type
\item \textbf{Meta capture video devices} for stats: queue with type \code{V4L2_BUF_TYPE_META_CAPTURE} with dedicated (struct) buffer type
\end{itemize}
\end{frame}

\begin{frame}{V4L2 Support for Image Signal Processors (ISPs): rkisp1}

\begin{minipage}{0.55\textwidth}
Example driver: \textbf{rkisp1}
\begin{itemize}
\item \code{rkisp1_isp} subdev device to coordinate
\item \code{rkisp1_mainpath}, \code{rkisp1_selfpath} giving pixels, with resizers
\item \code{rkisp1_params} taking \code{struct rkisp1_params_cfg}
\item \code{rkisp1_stats} giving \code{struct rkisp1_stat_buffer}
\end{itemize}
\end{minipage}
\begin{minipage}{0.4\textwidth}
\begin{center}
\includegraphics[width=\linewidth]{images/rkisp1-topology.png}\\
The rkisp1 media topology
\end{center}
\end{minipage}
\end{frame}

\subsection{Accomplished Work for Advanced Camera support on Allwinner}

\begin{frame}{Scope of Our Work}

\textbf{First phase}: OV5648 with MIPI CSI-2 on Allwinner V3s
\begin{itemize}
\item Add support for the OV5648 sensor
\item Add support for V3 MIPI CSI-2
\item Capture raw Bayer data
\end{itemize}~\\

\textbf{2020 summer internship}: OV8865 with MIPI CSI-2 on Allwinner A83T
\begin{itemize}
\item Same goal with OV8865 and A83T (different MIPI CSI-2 controller)
\item Using the BananaPi M3 board
\end{itemize}~\\

\textbf{Second phase}: Basic ISP support on Allwinner V3s
\begin{itemize}
\item Debayering (to YUV) with gain/offset
\item 2D noise reduction
\end{itemize}

\end{frame}

\begin{frame}{BananaPi-M3 and OV8865}

\begin{center}
\includegraphics[width=0.5\linewidth]{images/bananapi-m3.jpg}\\
The BananaPi-M3 with OV8865 connected
\end{center}

\end{frame}

\begin{frame}{Support for New Sensors: OV5648 and OV8865}

Writing image sensor drivers is hard!
\begin{itemize}
\item Reference code abundantly uses \textbf{static large arrays} of register/values
\item Tailored for a \textbf{specific input clock frequency} (usually 24 MHz)
\item Documentation is \textbf{not very precise}, often lacks some registers
\end{itemize}~\\

Taking the hard path to write nice drivers:
\begin{itemize}
\item \textbf{Proper definitions} for all (known) registers
\item \textbf{Helper functions} for the different parts
\item \textbf{Descriptive structures} for each supported mode
\item Documenting the \textbf{clock tree}
\end{itemize}
\end{frame}

\begin{frame}[fragile]{Proper Drivers for Image Sensors: OV5648 Clock Tree and Mode}

\begin{minipage}{0.45\textwidth}
\begin{minted}[fontsize=\scriptsize]{c}
struct ov5648_pll1_config {
  unsigned int pll_pre_div;
  unsigned int pll_mul;
  unsigned int sys_div;
  unsigned int root_div;
  unsigned int sclk_div;
  unsigned int mipi_div;
};

struct ov5648_pll2_config {
  unsigned int plls_pre_div;
  unsigned int plls_div_r;
  unsigned int plls_mul;
  unsigned int sys_div;
  unsigned int sel_div;
};
\end{minted}
\end{minipage}
\hfill
\begin{minipage}{0.45\textwidth}
\begin{minted}[fontsize=\tiny]{c}
struct ov5648_mode {
  unsigned int crop_start_x;
  unsigned int offset_x;
  unsigned int output_size_x;
  unsigned int crop_end_x;
  unsigned int hts;

  unsigned int crop_start_y;
  unsigned int offset_y;
  unsigned int output_size_y;
  unsigned int crop_end_y;
  unsigned int vts;

  bool binning_x;
  bool binning_y;

  unsigned int inc_x_odd;
  unsigned int inc_x_even;
  unsigned int inc_y_odd;
  unsigned int inc_y_even;

  /* 8-bit frame interval followed by 10-bit frame interval. */
  struct v4l2_fract frame_interval[2];

  /* 8-bit config followed by 10-bit config. */
  const struct ov5648_pll1_config *pll1_config[2];
  const struct ov5648_pll2_config *pll2_config;

  const struct ov5648_register_value *register_values;
  unsigned int register_values_count;
};
\end{minted}
\end{minipage}
\end{frame}

\begin{frame}[fragile]{Proper Drivers for Image Sensors: Patch Series}
\begin{itemize}
\item First iteration sent out in October 2020
\item Final iteration (v7) accepted in December 2020
\end{itemize}~\\

\begin{minted}[fontsize=\scriptsize]{c}
 Documentation/devicetree/bindings/media/i2c/ovti,ov5648.yaml |  115 ++
 Documentation/devicetree/bindings/media/i2c/ovti,ov8865.yaml |  124 ++
 arch/arm/boot/dts/sun8i-a83t-bananapi-m3.dts                 |  102 ++
 drivers/media/i2c/Kconfig                                    |   26 +
 drivers/media/i2c/Makefile                                   |    2 +
 drivers/media/i2c/ov5648.c                                   | 2624 ++++++++++++++++++++++++++++++++++++++++++
 drivers/media/i2c/ov8865.c                                   | 2972 ++++++++++++++++++++++++++++++++++++++++++++++++
 7 files changed, 5965 insertions(+)

\end{minted}
\end{frame}

\begin{frame}{A31/V3 and A83T MIPI CSI-2 Support}

\begin{itemize}
\item MIPI CSI-2 controllers feed (raw) data to the \textbf{CSI controller}
\item Represented as bridges (subdevs) between CSI and the sensor
\item Requires \textbf{adaptation to the CSI code} to select interface
\item Needs to get sensor \textbf{pixel rate} from dedicated control: \code{V4L2_CID_PIXEL_RATE}
\item Using a D-PHY block with the \textbf{generic Linux PHY API}
\begin{itemize}
\item \code{phy_mipi_dphy_get_default_config} helper not accounting for DDR
\end{itemize}
\end{itemize}~

A83T Support:
\begin{itemize}
\item \textbf{Reference source code} in Allwinner SDK:\\
\code{drivers/media/video/sunxi-vfe/mipi_csi/bsp_mipi_csi.c}
\item Some \textbf{magic values} in registers (undocumented)
\item D-PHY is mixed with controller registers
\begin{itemize}
\item In-driver PHY provider and consumer
\end{itemize}
\end{itemize}~

\end{frame}

\begin{frame}{A31/V3 and A83T MIPI CSI-2 Support}

A31/V3 Support:
\begin{itemize}
\item \textbf{Reference source code} in Allwinner SDK:\\
\code{drivers/media/video/sunxi-vfe/mipi_csi/{protocol,dphy}}
\item \textbf{Documentation} available in A31 user manual
\item Same D-PHY block used for MIPI DSI, in Rx mode instead of Tx
\item Driver already exists for Tx, needs direction selection:
\begin{itemize}
\item Describe with submode? Not a run-time decision...
\item Describe with different compatible? Same hardware block...
\item Describe with optional device-tree property
\end{itemize}
\end{itemize}
\end{frame}


\begin{frame}[fragile]{V3 and A83T MIPI CSI-2 Support: Patch Series}
\begin{itemize}
\item First iteration sent out in October 2020
\item Series later integrated with ISP work
\end{itemize}~

\begin{minted}[fontsize=\scriptsize]{c}
 arch/arm/boot/dts/sun8i-a83t.dtsi                                          |  26 ++
 arch/arm/boot/dts/sun8i-v3s.dtsi                                           |  68 ++++
 drivers/media/platform/sunxi/sun6i-csi/sun6i_csi.c                         | 218 ++++++++----
 drivers/media/platform/sunxi/sun6i-csi/sun6i_csi.h                         |  65 ++--
 drivers/media/platform/sunxi/sun6i-csi/sun6i_video.c                       |  57 +--
 drivers/media/platform/sunxi/sun6i-csi/sun6i_video.h                       |   7 +-
 drivers/media/platform/sunxi/sun6i-mipi-csi2/sun6i_mipi_csi2.c             | 600 +++++++++++++++++++++++++++++++
 drivers/media/platform/sunxi/sun6i-mipi-csi2/sun6i_mipi_csi2.h             | 117 ++++++
 drivers/media/platform/sunxi/sun8i-a83t-mipi-csi2/Kconfig                  |  11 +
 drivers/media/platform/sunxi/sun8i-a83t-mipi-csi2/Makefile                 |   4 +
 drivers/media/platform/sunxi/sun8i-a83t-mipi-csi2/sun8i_a83t_dphy.c        |  92 +++++
 drivers/media/platform/sunxi/sun8i-a83t-mipi-csi2/sun8i_a83t_dphy.h        |  39 ++
 drivers/media/platform/sunxi/sun8i-a83t-mipi-csi2/sun8i_a83t_mipi_csi2.c   | 666 +++++++++++++++++++++++++++++++++++
 drivers/media/platform/sunxi/sun8i-a83t-mipi-csi2/sun8i_a83t_mipi_csi2.h   | 197 +++++++++++
 drivers/phy/allwinner/phy-sun6i-mipi-dphy.c                                | 164 ++++++++-
 25 files changed, 2633 insertions(+), 141 deletions(-)
\end{minted}
\end{frame}

\begin{frame}{ISP Support and Integration}
Input/output aspects:
\begin{itemize}
\item ISP takes (raw) data from one of the \textbf{CSI controller(s)}
\item DRAM input exists in theory but unable to make it work
\item Input/interface part of CSI controller needs to be configured
\item \textbf{Internal mux} routes data to ISP instead of CSI DMA
\begin{itemize}
\item Impossible to switch back to CSI DMA without reboot
\end{itemize}
\item \textbf{Two outputs} available: main-channel and sub-channel
\end{itemize}~

\textbf{Major CSI rework} required:
\begin{itemize}
\item Separate bridge from DMA engine (subdev and video device)
\item Register with ISP's v4l2/media devices for common topology
\item Allow standalone use (both with and without ISP enabled):
\code{sun6i_csi_isp_detect} helper
\end{itemize}

\end{frame}

\begin{frame}{ISP Support and Integration: Topology}

\begin{minipage}{0.45\textwidth}
\begin{center}
\includegraphics[width=0.6\linewidth]{images/sun6i-isp-topology.png}\\
The sun6i-isp/sun6i-csi media topology
\end{center}
\end{minipage}
\hfill
\begin{minipage}{0.45\textwidth}
CSI components:
\begin{itemize}
\item \code{sun6i-csi-bridge}
\item \code{sun6i-csi-capture}
\end{itemize}~

ISP components:
\begin{itemize}
\item \code{sun6i-isp-proc}
\item \code{sun6i-isp-params}
\item \code{sun6i-isp-capture}
\end{itemize}~

MIPI CSI-2 interface:
\begin{itemize}
\item \code{sun6i-mipi-csi2}
\item \code{sun8i-a83t-mipi-csi2}
\end{itemize}
\end{minipage}

\end{frame}

\begin{frame}{ISP Support and Integration: Sync Mechanism}
Unusual sync mechanism:
\begin{itemize}
\item Registers \code{0x0-0x3f} are \textbf{accessed directly}
\item Other registers are prepared in a \textbf{load buffer}
\begin{itemize}
\item Load and save buffers are allocated in DRAM and provided to ISP
\end{itemize}
\item A \textbf{flag} indicates that load buffer was updated
\item At next vsync:
\begin{itemize}
\item ISP reads new register values from load buffer
\item ISP writes old register values to save buffer 
\item New values become active (can be read from MMIO)
\end{itemize}
\end{itemize}

\end{frame}

\begin{frame}{ISP Support and Integration: Features and API}

Parameters configure \textbf{modules of the ISP}:
\begin{itemize}
\item Passed via \code{sun6i-isp-params} video device
\item uAPI structure: \code{struct sun6i_isp_params_config}
\item Applied to next load buffer update
\end{itemize}~

Supported features:
\begin{itemize}
\item \textbf{Bayer coefficients}, with R/GR/GB/B gain/offset:\\
\code{struct sun6i_isp_params_config_bayer}
\item \textbf{2D noise filtering} (BDNF) coefficients for G and R/B:
\code{struct sun6i_isp_params_config_bdnf}
\item Submitted to \textbf{staging} since a stable uAPI needs all features covered
\end{itemize}
\end{frame}

\begin{frame}[fragile]{ISP Driver and Integration: Patch Series}
\begin{itemize}
\item First iteration sent out in September 2021
\end{itemize}~

\begin{minted}[fontsize=\scriptsize]{c}
 drivers/media/platform/sunxi/sun6i-csi/sun6i_csi.c                         | 1051 +++++++++-----------------------
 drivers/media/platform/sunxi/sun6i-csi/sun6i_csi.h                         |  155 ++---
 drivers/media/platform/sunxi/sun6i-csi/sun6i_csi_bridge.c                  |  895 +++++++++++++++++++++++++++
 drivers/media/platform/sunxi/sun6i-csi/sun6i_csi_bridge.h                  |   64 ++
 drivers/media/platform/sunxi/sun6i-csi/sun6i_csi_capture.c                 | 1094 ++++++++++++++++++++++++++++++++++
 drivers/media/platform/sunxi/sun6i-csi/sun6i_csi_capture.h                 |   73 +++
 drivers/media/platform/sunxi/sun6i-csi/sun6i_csi_reg.h                     |  364 ++++++-----
 drivers/staging/media/sunxi/sun6i-isp/sun6i_isp.c                          |  577 ++++++++++++++++++
 drivers/staging/media/sunxi/sun6i-isp/sun6i_isp.h                          |   86 +++
 drivers/staging/media/sunxi/sun6i-isp/sun6i_isp_capture.c                  |  759 +++++++++++++++++++++++
 drivers/staging/media/sunxi/sun6i-isp/sun6i_isp_capture.h                  |   79 +++
 drivers/staging/media/sunxi/sun6i-isp/sun6i_isp_params.c                   |  571 ++++++++++++++++++
 drivers/staging/media/sunxi/sun6i-isp/sun6i_isp_params.h                   |   53 ++
 drivers/staging/media/sunxi/sun6i-isp/sun6i_isp_proc.c                     |  598 +++++++++++++++++++
 drivers/staging/media/sunxi/sun6i-isp/sun6i_isp_proc.h                     |   61 ++
 drivers/staging/media/sunxi/sun6i-isp/sun6i_isp_reg.h                      |  275 +++++++++
 drivers/staging/media/sunxi/sun6i-isp/uapi/sun6i-isp-config.h              |   43 ++
 51 files changed, 8702 insertions(+), 1808 deletions(-)
\end{minted}
\end{frame}

\subsection{Future Work and Improvements}

\begin{frame}{Remaining Features to Implement}

Roadmap for ISP driver completeness:
\begin{itemize}
\item Support \textbf{more platforms} (at least A83T)
\item Declare \textbf{hardware revisions} (modules availability):\\
\code{media_dev->hw_revision}
\item Support for \textbf{stats} (hist/ae/awb/af/afs)
\item Support for \textbf{sub-channel, scaling and rotation}
\item \textbf{Complete uAPI} that describes all modules
\item Support for \textbf{all available modules}
\begin{itemize}
\item Start with black level correction, color matrix and gamma
\end{itemize}
\item Userspace \textbf{3A algorithms} support
\end{itemize}

\end{frame}

\begin{frame}{Integration with libcamera}

\begin{center}
\includegraphics[width=0.3\linewidth]{images/libcamera-banner.png}\\
\end{center}

\begin{itemize}
\item Community-driven project for advanced camera support: \textbf{libcamera}
\item Provides \textbf{abstraction} for applications, GStreamer, Android
\item Implements \textbf{complex pipeline support}
\item Implements \textbf{hardware-specific 3A algorithms}
\item Good fit for Allwinner A31 ISP userspace support
\end{itemize}

\end{frame}

\begin{frame}{Hardware Availability}

\begin{center}
\includegraphics[width=0.55\linewidth]{images/s3-olinuxino.jpg}\\
Olimex announced the S3-OLinuXino, with a RPi-compatible MIPI CSI-2 connector!
\end{center}

\end{frame}

\questionslide

\subsection{Extra Slides}

\begin{frame}{Camera Optical Systems}
Optical systems have multiple elements and purposes:
\begin{itemize}
\item \textbf{Lens} to make light converge towards sensor surface
\begin{itemize}
\item \textbf{Focal length} (f) indicates the amount of convergence
\item Sets the angle of view, results in magnification/zoom effect
\item Optional moving elements to define focus plane
\end{itemize}
\item Optional \textbf{focus coil} to electrically control focus adjustment
\item Optional \textbf{diaphragm} to control aperture
\begin{itemize}
\item \textbf{F-number} (e.g. \code{f/1.8}) indicates how open the diaphragm is
\item Aperture decreases with f-number (diaphragm closes)
\end{itemize}
\end{itemize}
\end{frame}

\begin{frame}{Camera Optical Systems: Illustration}

\begin{center}
\includegraphics[width=0.6\linewidth]{images/optical-system.png}\\
Camera optical system
\end{center}

\begin{center}
\includegraphics[width=0.7\linewidth]{images/lens-diaphragm.jpg}\\
Diaphragm aperture variation \textit{(CC BY-SA 3.0, KoeppiK, Wikimedia Commons)}
\end{center}

\end{frame}

\begin{frame}{Image Sensors}

Components of an image sensor:
\begin{enumerate}
\item Color Filter Array (CFA) following a \textbf{Bayer pattern} (R/G/G/B)
\item Photo-sensitive cells (\textbf{photosites}) in CMOS or CCD technology
\item \textbf{Amplifier and ADC} to produce digital values
\begin{itemize}
\item Generally 8, 10 or 12-bit data
\end{itemize}
\item Configurable \textbf{shutter speed} (exposure time)
\item \textbf{Clocks and timings} for frame rate
\begin{itemize}
\item Capture cycle repeatedly following precise timings
\item External clock reference for internal PLLs
\item Limits exposure time
\end{itemize}
\item \textbf{Processing} (more or less advanced)
\item Control and \textbf{configuration} interface
\begin{itemize}
\item Usually configured via I2C or SPI
\end{itemize}
\item Data \textbf{transmission} interface
\end{enumerate}
\end{frame}

\begin{frame}{Image Sensors: Illustration}

\begin{minipage}{0.55\textwidth}
\begin{center}
\includegraphics[width=\linewidth]{images/ov5648-block.png}\\
OV5648 block diagram \textit{(Omnivision)}
\end{center}
\end{minipage}
\hfill
\begin{minipage}{0.4\textwidth}
\begin{center}
\includegraphics[width=0.9\linewidth]{images/bayer-pattern.png}\\
Bayer pattern \textit{(CC BY-SA 3.0, Cburnett, Wikimedia Commons)}
\end{center}
\end{minipage}

\end{frame}

\begin{frame}{Hardware Interfaces for Capture: Schematics}

\begin{minipage}{0.55\textwidth}
\begin{center}
\includegraphics[width=\linewidth]{images/csi-camera.png}
\end{center}
\end{minipage}
\hfill
\begin{minipage}{0.4\textwidth}
\begin{center}
\includegraphics[width=\linewidth]{images/mipi-csi2-camera.png}
\end{center}
\end{minipage}

\begin{center}
Parallel and MIPI CSI-2 interfaces on the S3-OLinuXino
\end{center}
\end{frame}

\end{document}
