\documentclass[aspectratio=169,obeyspaces,spaces,hyphens,dvipsnames]{beamer} \usepackage[utf8]{inputenc} \usepackage{lmodern}% http://ctan.org/pkg/lm \usepackage{minted} \usepackage{hyperref} \usepackage{xcolor} \usepackage{pgfplots} \usepackage{tikz} \usepackage{stmaryrd} \usepackage[normalem]{ulem} \mode \usetheme{Bootlin} \def\signed #1{{\leavevmode\unskip\nobreak\hfil\penalty50\hskip2em \hbox{}\nobreak\hfil(#1) \parfillskip=0pt \finalhyphendemerits=0 \endgraf}} \newsavebox\mybox \newenvironment{aquote}[1] {\savebox\mybox{#1}\begin{quotation}} {\signed{\usebox\mybox}\end{quotation}} \makeatletter \newbox\@backgroundblock \newenvironment{backgroundblock}[2]{% \global\setbox\@backgroundblock=\vbox\bgroup% \unvbox\@backgroundblock% \vbox to0pt\bgroup\vskip#2\hbox to0pt\bgroup\hskip#1\relax% }{\egroup\egroup\egroup} \addtobeamertemplate{background}{\box\@backgroundblock}{} \makeatother \tikzstyle{block} = [rectangle, draw, fill=white, text width=4em, text centered, minimum height=3em, node distance=10em] \tikzstyle{blockish} = [draw, fill=white, text width=4em, text centered, node distance=8em] \tikzstyle{arrow} = [thick,->,>=stealth] \title{Advanced Camera Support on Allwinner SoCs with Mainline Linux} \authors{Paul Kocialkowski} \email{paul@bootlin.com} \slidesurl{https://bootlin.com/pub/conferences/} \institute{Bootlin} \conference{Embedded Recipes 2022} \begin{document} \addtocontents{toc}{\protect\setcounter{tocdepth}{-1}} \section{Advanced Camera Support on Allwinner SoCs with Mainline Linux} \addtocontents{toc}{\protect\setcounter{tocdepth}{2}} \begin{frame}{Paul Kocialkowski} \begin{itemize} \item Embedded Linux engineer at Bootlin \begin{itemize} \item Embedded Linux {\bf expertise} \item {\bf Development}, consulting and training \item Strong open-source focus \end{itemize} \item Open-source contributor \begin{itemize} \item Co-maintainer of the \textbf{cedrus} VPU driver in V4L2 \item Contributor to the \textbf{sun4i-drm} DRM driver \item Contributing the \textbf{logicvc-drm} DRM driver \item Developed the \textbf{displaying and rendering graphics with Linux} training \end{itemize} \item Living in {\bf Toulouse}, south-west of France \end{itemize} \end{frame} \subsection{An Introduction to Image Capture Technology} \begin{frame}{Overview of the Digital Image Capture Chain} \begin{center} \includegraphics[width=0.8\linewidth]{images/image-capture-pipeline.png}\\ An image capture chain \end{center}~ \begin{itemize} \item \textbf{Optics}: shape light rays \item \textbf{Sensor}: convert light to digital values \item \textbf{Interface}: transport values \item \textbf{Processing}: produce good-looking pictures \item \textbf{Display/encoding}: show/store pictures \textit{(out of the scope of this talk)} \end{itemize} \end{frame} \begin{frame}{Processing RAW Images} Data coming from a sensor ADC needs processing: \begin{itemize} \item Data corresponds to a \textbf{bayer pattern}, not pixels \item Brightness is linear, not adapted for display \item Sensors have a non-zero \textbf{dark-level current} \item Noise is present, color is off, image looks bad \end{itemize} \begin{center} Enhancement takes place in \textbf{Image Signal Processors (ISPs)} \end{center} Three distinct domains are involved: \begin{enumerate} \item \textbf{Bayer domain}, ends with debayering step \item \textbf{RGB domain}, ends with YUV conversion \item \textbf{YUV domain}, ends with final picture \end{enumerate} \end{frame} \begin{frame}{Image Enhancements in ISPs} Various enhancements are usually applied to the image: \begin{itemize} \item \textbf{Dead pixel correction}: discard invalid values \item \textbf{Black level correction}: remove dark level current \item \textbf{White balance}: adjust R-G-B balance with coefficients/offsets \item \textbf{Noise filtering}: remove electronic noise \item \textbf{Color matrix}: adjust colors for fidelity \item \textbf{Gamma}: adjust brightness curve for non-linearity \item \textbf{Saturation}: adjust colorfulness \item \textbf{Brightness}: adjust global luminosity \item \textbf{Contrast}: adjust bright/dark difference \end{itemize} \end{frame} \begin{frame}{Image Enhancements in ISPs} More advanced enhancements may also be applied: \begin{itemize} \item \textbf{Lens shading}: correct lens irregular brightness \item \textbf{Lens dewarp}: correct lens geometry distortion effect \item \textbf{Stabilization}: crop to remove shaking \item \textbf{Color LUT}: Translate colors with a specific style \end{itemize}~ Hardware implementations: \begin{itemize} \item ISPs embedded in sensors tend to be \textbf{simple} \begin{itemize} \item Provide YUV data to the camera interface \end{itemize} \item Multimedia Systems on a Chip often have an \textbf{advanced ISP} \begin{itemize} \item Require raw bayer data on the camera interface \item Require specific calibration data for the sensor/lens \end{itemize} \end{itemize} \end{frame} \begin{frame}{Processing RAW Images: Illustration} \begin{minipage}[b]{0.3\textwidth} \begin{center} \includegraphics[width=\linewidth]{images/step-bayer.png} Bayer step \end{center} \end{minipage} \hfill \begin{minipage}[b]{0.3\textwidth} \begin{center} \includegraphics[width=0.8\linewidth]{images/step-rgb-wb.jpg} \includegraphics[width=0.8\linewidth]{images/step-rgb-gamma.jpg} \includegraphics[width=0.8\linewidth]{images/step-rgb.jpg} RGB step \end{center} \end{minipage} \hfill \begin{minipage}[b]{0.3\textwidth} \begin{center} \includegraphics[width=0.8\linewidth]{images/step-y.jpg} \includegraphics[width=0.8\linewidth]{images/step-u.jpg} \includegraphics[width=0.8\linewidth]{images/step-v.jpg} YUV step \end{center} \end{minipage} \end{frame} \begin{frame}{Parameters to Adjust} Some parameters depend on the situation: \begin{itemize} \item \textbf{Focus} depends on the area of interest \item \textbf{White balance} depends on the light source(s) \item \textbf{Exposure} depends on the amount of light \end{itemize}~ \pause Exposure depends on a few parameters: \begin{itemize} \item Diaphragm \textbf{aperture} (f-number) \item \textbf{Exposure time} (shutter speed) \item \textbf{Amplifier gain} (ISO number equivalent) \end{itemize}~ Advanced users will set parameters manually, with artistic implications \end{frame} \begin{frame}{Automatic Parameters Control with 3A} In other cases, automatic parameters control is desirable: \begin{itemize} \item \textbf{Automatic exposition}: manage exposure time and gain (optionally diaphragm) \item \textbf{Auto-focus}: detect blurry and sharp areas, adjust with focus coil \item \textbf{Auto white balance}: detect dominant lighting and adjust \end{itemize}~ \pause Implemented using 3A algorithms: \begin{itemize} \item General algorithms described in \textbf{academic literature} \item Involve a \textbf{feedback loop} system, using statistics \item Implementations are usually \textbf{hardware specific} (ISP and sensor),\\ often considered to be the secret sauce! \end{itemize} \end{frame} \begin{frame}{Hardware Interfaces for Capture} Sensors need to transmit data: \begin{itemize} \item Analog interfaces (CVBS, etc) are mostly deprecated \item \textbf{Parallel} digital interfaces: basic, BT.656\\ \textit{typically used with old and low-end sensors} \item \textbf{Serial} digital interfaces: MIPI CSI-2, LVDS, SDI, HiSPi\\ \textit{typically used with high-end sensors} \end{itemize}~ \pause \textbf{Basic parallel} interface: \begin{itemize} \item One TTL signal per bit, usually 8/10/12/16/24 bits width \item Pixel clock and sync signals (hsync, vsync) \end{itemize}~ \pause \textbf{MIPI CSI-2} serial interface: \begin{itemize} \item Differential pairs, using double data rate (DDR) \item One clock lane (high rates) and 1-4 data lanes \end{itemize} \end{frame} \subsection{Scope and Use Case} \begin{frame}{Scope and Use Case: Allwinner + MIPI CSI-2 + ISP} \begin{minipage}{0.55\textwidth} \textbf{Allwinner platforms} (V3 and A83T): \begin{itemize} \item Systems on a Chip with ARM CPUs \item MIPI CSI-2 receiver \item Camera interface (CSI) \item Image Signal Processor (ISP) \end{itemize}~ \pause \textbf{Image sensors} (OV8865, OV5648): \begin{itemize} \item I2C control interface \item MIPI CSI-2 transmitter \item Bayer RAW formats (10/12 bits) \item Minimal to inexistent onboard ISP \end{itemize} \end{minipage} \hfill \begin{minipage}{0.4\textwidth} \begin{center} \includegraphics[width=\linewidth]{images/bananapi-m3.jpg}\\ The BananaPi-M3 with OV8865 connected \end{center} \end{minipage} \end{frame} \subsection{Status of Allwiner Camera Support in\\Mainline Linux} \begin{frame}{Mainline Linux Support and Allwinner Camera Support} Allwinner platform support in mainline Linux: \begin{itemize} \item Long-time effort from the \textbf{sunxi community}, very active\\ \url{https://linux-sunxi.org/Linux_mainlining_effort} \item Multimedia areas are often the last missing parts \item Allwinner started contributing (more or less) very recently \end{itemize}~ \pause Camera support in mainline Linux: \begin{itemize} \item \code{sun4i-csi} driver for first generation CSI \item \code{sun6i-csi} driver for second generation CSI \item Third generation CSI support is missing \item MIPI CSI-2 and ISP support was entirely missing\\ \textit{non-free blobs for ISP support and A80 MIPI CSI-2 in SDK} \end{itemize} \end{frame} \begin{frame}{Camera Support in Linux with V4L2} \begin{center} \textbf{Video4Linux2} (V4L2) is the subsystem/API for media support in Linux \end{center} \begin{itemize} \item Supports various types of \textbf{pixel-related devices}\\ \textit{basically anything that is not a display or gpu} \item Provides userspace with \textbf{video devices} (e.g. \code{/dev/video0}) \item Implements a generic \textbf{userspace API} including: \begin{itemize} \item Format negotiation, implemented in \code{struct v4l2_ioctl_ops} \item Memory management (alloc, free, mmap), implemented in \code{struct vb2_mem_ops} \item A queue interface for buffers of a given type (output, capture...),\\implemented in \code{struct vb2_ops} \item A control interface for configuration \end{itemize} \item Good fit for \textbf{all-in-one devices} (e.g. USB UVC cameras)\\ \textit{assumes that a memory (DMA) interface is available} \end{itemize} \end{frame} \begin{frame}{V4L2 Support for Complex Camera Systems : Subdevs} Complex systems bring the need for \textbf{more refinement}: \begin{itemize} \item Internal blocks with FIFOs \item External devices with interfaces (e.g. sensors) \item Possibility to configure each block and the topology \end{itemize}~ \pause Hence the notion of \textbf{subdevs} was introduced to V4L2: \begin{itemize} \item Represent a single block (usually not DMA-capable) \item Exposed to userspace via dedicated nodes \code{ /dev/v4l-subdev0} \item Dedicated format configuration, implemented in \code{struct v4l2_subdev_pad_ops} \item Dedicated stream management, implemented in \code{struct v4l2_subdev_video_ops} \item Called by video devices with \code{v4l2_subdev_call} \end{itemize} \end{frame} \begin{frame}{V4L2 Support for Complex Camera Systems : Subdevs Integration} \begin{center} Subdevs need to be \textbf{parented to a v4l2 device} (controlling entity) \end{center} Simple case: the \textbf{all-in-one driver} \begin{itemize} \item A single driver may register a parent v4l2 device, a video device and subdev(s) \item The subdev can be registered directly:\\ \code{v4l2_device_register_subdev(v4l2_dev, subdev);} \end{itemize}~\\ \pause Complex case: \textbf{multiple drivers} involved \begin{itemize} \item The video device driver will typically register a v4l2 device \item Each subdev driver will register its subdev asynchronously:\\ \code{v4l2_async_register_subdev(subdev);} \item A driver that needs a subdev needs to identify and wait for it \end{itemize} \end{frame} \begin{frame}[fragile]{V4L2 Support for Complex Camera Systems : Fwnode Graph} The fwnode graph represents the connection between different blocks: \begin{itemize} \item Typically described in device-tree with port/endpoint \item The meaning of each port is described in the device-tree bindings \item Endpoints are retrieved by the driver and parsed with a helper: \code{fwnode_graph_get_endpoint_by_id()}\\ \code{v4l2_fwnode_endpoint_parse()} \item May contain an indication of the bus type:\\ \code{enum v4l2_mbus_type}, e.g. \code{V4L2_MBUS_CSI2_DPHY} \item As well as bus-specific information:\\ e.g. \code{struct v4l2_fwnode_bus_mipi_csi2} \end{itemize} \end{frame} \begin{frame}[fragile]{V4L2 Support for Complex Camera Systems : Fwnode Graph} Device-tree example for camera to MIPI CSI-2 bridge:\\ ~\\ \begin{minipage}[t]{0.45\textwidth} \begin{minted}[fontsize=\scriptsize]{perl} imx219: camera@10 { compatible = "sony,imx219"; ... port { camera_to_bridge: endpoint { data-lanes = <1 2>; link-frequencies = /bits/ 64 <456000000>; remote-endpoint = <&bridge_from_camera>; }; }; }; \end{minted} \end{minipage} \hfill \begin{minipage}[t]{0.45\textwidth} \begin{minted}[fontsize=\scriptsize]{perl} mipi_csi2: csi@1cb1000 { compatible = "allwinner,sun8i-v3s-mipi-csi2"; ... ports { ... port@0 { reg = <0>; bridge_from_camera: endpoint { data-lanes = <1 2>; remote-endpoint = <&camera_to_bridge>; }; }; ... }; }; \end{minted} \end{minipage} \end{frame} \begin{frame}{V4L2 Support for Complex Camera Systems : Async Subdevs} \textbf{Async registration} allows other drivers to use the subdev: \begin{itemize} \item A link between devices is described with \textbf{fwnode graph} \item An \textbf{async notifier} will match and notify when the subdev is available:\\ \code{v4l2_async_notifier_add_fwnode_remote_subdev} \item The async notifier can be used by the driver with a v4l2 device:\\ \code{v4l2_async_notifier_register(v4l2_dev, notifier);} \item Or by a subdev that needs another subdev (e.g. a bridge):\\ \code{v4l2_async_subdev_notifier_register(subdev, notifier);} \item A callback gives the requesting driver a \code{struct v4l2_subdev} \end{itemize} \end{frame} \begin{frame}{V4L2 Support for Complex Camera Systems : Media Controller} The \textbf{media controller} API provides coordination between blocks: \begin{itemize} \item Each block is an \textbf{entity} with sink/source \textbf{pads}\\ derivated from a video device or a subdev \item Entities declare a particular function\\ e.g. \code{MEDIA_ENT_F_PROC_VIDEO_PIXEL_FORMATTER} \item \textbf{Links} between pads of entities are created by drivers,\\ may allow userspace to enable/disable them \item Grouped in a media device (tied to a v4l2 device) \item Performs \textbf{runtime validation} for links, implemented in \code{struct media_entity_operations}'s \code{link_validate} \item Topology is \textbf{exposed to userspace}, usually controlled with \code{media-ctl}:\\ \code{media-ctl -l '"sun6i-csi-bridge":1 -> "sun6i-csi-capture":0[1]'} \end{itemize} \end{frame} \begin{frame}{V4L2 Support for Complex Camera Systems : Media Controller} \begin{center} \includegraphics[width=0.55\linewidth]{images/imx-media-topology.png}\\ The i.MX capture driver's media topology \end{center} \end{frame} \begin{frame}{V4L2 Support for Image Signal Processors (ISPs)} Specific aspects related to ISPs: \begin{itemize} \item Usually have an internal pipeline with \textbf{multiple blocks} \item Parameters are \textbf{highly specific} (not a good fit for V4L2 controls) \item Provide stats \textbf{information buffers} (3A, histogram) \item Exposes one or multiple \textbf{capture interfaces} \end{itemize}~\\ \pause ISPs integration in V4L2: \begin{itemize} \item Processor represented by a subdev/media entity: \code{MEDIA_ENT_F_PROC_VIDEO_ISP} \item \textbf{Capture video devices} for pixels: queues with type \code{V4L2_BUF_TYPE_VIDEO_CAPTURE} \item \textbf{Meta output video devices} for parameters: queue with type \code{V4L2_BUF_TYPE_META_OUTPUT} with dedicated (struct) buffer type \item \textbf{Meta capture video devices} for stats: queue with type \code{V4L2_BUF_TYPE_META_CAPTURE} with dedicated (struct) buffer type \end{itemize} \end{frame} \begin{frame}{V4L2 Support for Image Signal Processors (ISPs): rkisp1} \begin{minipage}{0.55\textwidth} Example driver: \textbf{rkisp1} \begin{itemize} \item \code{rkisp1_isp} subdev device to coordinate \item \code{rkisp1_mainpath}, \code{rkisp1_selfpath} giving pixels, with resizers \item \code{rkisp1_params} taking \code{struct rkisp1_params_cfg} \item \code{rkisp1_stats} giving \code{struct rkisp1_stat_buffer} \end{itemize} \end{minipage} \begin{minipage}{0.4\textwidth} \begin{center} \includegraphics[width=\linewidth]{images/rkisp1-topology.png}\\ The rkisp1 media topology \end{center} \end{minipage} \end{frame} \subsection{Accomplished Work for Advanced Camera support on Allwinner} \begin{frame}{A31/V3 and A83T MIPI CSI-2 Support} \begin{itemize} \item MIPI CSI-2 controllers feed (raw) data to the \textbf{CSI controller} \item Represented as bridges (subdevs) between CSI and the sensor \item Requires \textbf{adaptation to the CSI code} to select interface \item Needs to get sensor \textbf{pixel rate} from dedicated control: \code{V4L2_CID_PIXEL_RATE} \item Using a D-PHY block with the \textbf{generic Linux PHY API} \begin{itemize} \item \code{phy_mipi_dphy_get_default_config} helper not accounting for DDR \end{itemize} \end{itemize}~ A83T Support: \begin{itemize} \item \textbf{Reference source code} in Allwinner SDK:\\ \code{drivers/media/video/sunxi-vfe/mipi_csi/bsp_mipi_csi.c} \item Some \textbf{magic values} in registers (undocumented) \item D-PHY is mixed with controller registers \begin{itemize} \item In-driver PHY provider and consumer \end{itemize} \end{itemize}~ \end{frame} \begin{frame}{A31/V3 and A83T MIPI CSI-2 Support} A31/V3 Support: \begin{itemize} \item \textbf{Reference source code} in Allwinner SDK:\\ \code{drivers/media/video/sunxi-vfe/mipi_csi/{protocol,dphy}} \item \textbf{Documentation} available in A31 user manual \item Same D-PHY block used for MIPI DSI, in Rx mode instead of Tx \item Driver already exists for Tx, needs direction selection: \begin{itemize} \item Describe with submode? Not a run-time decision... \item Describe with different compatible? Same hardware block... \item Describe with optional device-tree property \end{itemize} \end{itemize} \end{frame} \begin{frame}[fragile]{V3 and A83T MIPI CSI-2 Support: Patch Series} \begin{itemize} \item First iteration sent out in October 2020 \item Series later integrated with ISP work \end{itemize}~ \begin{minted}[fontsize=\scriptsize]{c} arch/arm/boot/dts/sun8i-a83t.dtsi | 26 ++ arch/arm/boot/dts/sun8i-v3s.dtsi | 68 ++++ drivers/media/platform/sunxi/sun6i-csi/sun6i_csi.c | 218 ++++++++---- drivers/media/platform/sunxi/sun6i-csi/sun6i_csi.h | 65 ++-- drivers/media/platform/sunxi/sun6i-csi/sun6i_video.c | 57 +-- drivers/media/platform/sunxi/sun6i-csi/sun6i_video.h | 7 +- drivers/media/platform/sunxi/sun6i-mipi-csi2/sun6i_mipi_csi2.c | 600 +++++++++++++++++++++++++++++++ drivers/media/platform/sunxi/sun6i-mipi-csi2/sun6i_mipi_csi2.h | 117 ++++++ drivers/media/platform/sunxi/sun8i-a83t-mipi-csi2/Kconfig | 11 + drivers/media/platform/sunxi/sun8i-a83t-mipi-csi2/Makefile | 4 + drivers/media/platform/sunxi/sun8i-a83t-mipi-csi2/sun8i_a83t_dphy.c | 92 +++++ drivers/media/platform/sunxi/sun8i-a83t-mipi-csi2/sun8i_a83t_dphy.h | 39 ++ drivers/media/platform/sunxi/sun8i-a83t-mipi-csi2/sun8i_a83t_mipi_csi2.c | 666 +++++++++++++++++++++++++++++++++++ drivers/media/platform/sunxi/sun8i-a83t-mipi-csi2/sun8i_a83t_mipi_csi2.h | 197 +++++++++++ drivers/phy/allwinner/phy-sun6i-mipi-dphy.c | 164 ++++++++- 25 files changed, 2633 insertions(+), 141 deletions(-) \end{minted} \end{frame} \begin{frame}{ISP Support and Integration} Input/output aspects: \begin{itemize} \item ISP takes (raw) data from one of the \textbf{CSI controller(s)} \item DRAM input exists in theory but unable to make it work \item Input/interface part of CSI controller needs to be configured \item \textbf{Internal mux} routes data to ISP instead of CSI DMA \begin{itemize} \item Impossible to switch back to CSI DMA without reboot \end{itemize} \item \textbf{Two outputs} available: main-channel and sub-channel \end{itemize}~ \textbf{Major CSI rework} required: \begin{itemize} \item Separate bridge from DMA engine (subdev and video device) \item Register with ISP's v4l2/media devices for common topology \item Allow standalone use (both with and without ISP enabled): \code{sun6i_csi_isp_detect} helper \end{itemize} \end{frame} \begin{frame}{ISP Support and Integration: Topology} \begin{minipage}{0.45\textwidth} \begin{center} \includegraphics[width=0.6\linewidth]{images/sun6i-isp-topology.png}\\ The sun6i-isp/sun6i-csi media topology \end{center} \end{minipage} \hfill \begin{minipage}{0.45\textwidth} CSI components: \begin{itemize} \item \code{sun6i-csi-bridge} \item \code{sun6i-csi-capture} \end{itemize}~ ISP components: \begin{itemize} \item \code{sun6i-isp-proc} \item \code{sun6i-isp-params} \item \code{sun6i-isp-capture} \end{itemize}~ MIPI CSI-2 interface: \begin{itemize} \item \code{sun6i-mipi-csi2} \item \code{sun8i-a83t-mipi-csi2} \end{itemize} \end{minipage} \end{frame} \begin{frame}{ISP Support and Integration: Features and API} Parameters configure \textbf{modules of the ISP}: \begin{itemize} \item Passed via \code{sun6i-isp-params} video device \item uAPI structure: \code{struct sun6i_isp_params_config} \item Applied to next load buffer update \end{itemize}~ Supported features: \begin{itemize} \item \textbf{Bayer coefficients}, with R/GR/GB/B gain/offset:\\ \code{struct sun6i_isp_params_config_bayer} \item \textbf{2D noise filtering} (BDNF) coefficients for G and R/B: \code{struct sun6i_isp_params_config_bdnf} \item Submitted to \textbf{staging} since a stable uAPI needs all features covered \end{itemize} \end{frame} \begin{frame}[fragile]{ISP Driver and Integration: Patch Series} \begin{itemize} \item First iteration sent out in September 2021 \end{itemize}~ \begin{minted}[fontsize=\scriptsize]{c} drivers/media/platform/sunxi/sun6i-csi/sun6i_csi.c | 1051 +++++++++----------------------- drivers/media/platform/sunxi/sun6i-csi/sun6i_csi.h | 155 ++--- drivers/media/platform/sunxi/sun6i-csi/sun6i_csi_bridge.c | 895 +++++++++++++++++++++++++++ drivers/media/platform/sunxi/sun6i-csi/sun6i_csi_bridge.h | 64 ++ drivers/media/platform/sunxi/sun6i-csi/sun6i_csi_capture.c | 1094 ++++++++++++++++++++++++++++++++++ drivers/media/platform/sunxi/sun6i-csi/sun6i_csi_capture.h | 73 +++ drivers/media/platform/sunxi/sun6i-csi/sun6i_csi_reg.h | 364 ++++++----- drivers/staging/media/sunxi/sun6i-isp/sun6i_isp.c | 577 ++++++++++++++++++ drivers/staging/media/sunxi/sun6i-isp/sun6i_isp.h | 86 +++ drivers/staging/media/sunxi/sun6i-isp/sun6i_isp_capture.c | 759 +++++++++++++++++++++++ drivers/staging/media/sunxi/sun6i-isp/sun6i_isp_capture.h | 79 +++ drivers/staging/media/sunxi/sun6i-isp/sun6i_isp_params.c | 571 ++++++++++++++++++ drivers/staging/media/sunxi/sun6i-isp/sun6i_isp_params.h | 53 ++ drivers/staging/media/sunxi/sun6i-isp/sun6i_isp_proc.c | 598 +++++++++++++++++++ drivers/staging/media/sunxi/sun6i-isp/sun6i_isp_proc.h | 61 ++ drivers/staging/media/sunxi/sun6i-isp/sun6i_isp_reg.h | 275 +++++++++ drivers/staging/media/sunxi/sun6i-isp/uapi/sun6i-isp-config.h | 43 ++ 51 files changed, 8702 insertions(+), 1808 deletions(-) \end{minted} \end{frame} \subsection{Future Work and Improvements} \begin{frame}{Remaining Features to Implement} Roadmap for ISP driver completeness: \begin{itemize} \item Support \textbf{more platforms} (at least A83T) \item Declare \textbf{hardware revisions} (modules availability):\\ \code{media_dev->hw_revision} \item Support for \textbf{stats} (hist/ae/awb/af/afs) \item Support for \textbf{sub-channel, scaling and rotation} \item \textbf{Complete uAPI} that describes all modules \item Support for \textbf{all available modules} \begin{itemize} \item Start with black level correction, color matrix and gamma \end{itemize} \item Userspace \textbf{3A algorithms} support \end{itemize} \end{frame} \begin{frame}{Integration with libcamera} \begin{center} \includegraphics[width=0.3\linewidth]{images/libcamera-banner.png}\\ \end{center} \begin{itemize} \item Community-driven project for advanced camera support: \textbf{libcamera} \item Provides \textbf{abstraction} for applications, GStreamer, Android \item Implements \textbf{complex pipeline support} \item Implements \textbf{hardware-specific 3A algorithms} \item Good fit for Allwinner A31 ISP userspace support \end{itemize} \end{frame} \begin{frame}{Hardware Availability} \begin{center} \includegraphics[width=0.55\linewidth]{images/s3-olinuxino.jpg}\\ Olimex announced the S3-OLinuXino, with a RPi-compatible MIPI CSI-2 connector! \end{center} \end{frame} \questionslide \subsection{Extra Slides} \begin{frame}{Camera Optical Systems} Optical systems have multiple elements and purposes: \begin{itemize} \item \textbf{Lens} to make light converge towards sensor surface \begin{itemize} \item \textbf{Focal length} (f) indicates the amount of convergence \item Sets the angle of view, results in magnification/zoom effect \item Optional moving elements to define focus plane \end{itemize} \item Optional \textbf{focus coil} to electrically control focus adjustment \item Optional \textbf{diaphragm} to control aperture \begin{itemize} \item \textbf{F-number} (e.g. \code{f/1.8}) indicates how open the diaphragm is \item Aperture decreases with f-number (diaphragm closes) \end{itemize} \end{itemize} \end{frame} \begin{frame}{Camera Optical Systems: Illustration} \begin{center} \includegraphics[width=0.6\linewidth]{images/optical-system.png}\\ Camera optical system \end{center} \begin{center} \includegraphics[width=0.7\linewidth]{images/lens-diaphragm.jpg}\\ Diaphragm aperture variation \textit{(CC BY-SA 3.0, KoeppiK, Wikimedia Commons)} \end{center} \end{frame} \begin{frame}{Image Sensors} Components of an image sensor: \begin{enumerate} \item Color Filter Array (CFA) following a \textbf{Bayer pattern} (R/G/G/B) \item Photo-sensitive cells (\textbf{photosites}) in CMOS or CCD technology \item \textbf{Amplifier and ADC} to produce digital values \begin{itemize} \item Generally 8, 10 or 12-bit data \end{itemize} \item Configurable \textbf{shutter speed} (exposure time) \item \textbf{Clocks and timings} for frame rate \begin{itemize} \item Capture cycle repeatedly following precise timings \item External clock reference for internal PLLs \item Limits exposure time \end{itemize} \item \textbf{Processing} (more or less advanced) \item Control and \textbf{configuration} interface \begin{itemize} \item Usually configured via I2C or SPI \end{itemize} \item Data \textbf{transmission} interface \end{enumerate} \end{frame} \begin{frame}{Image Sensors: Illustration} \begin{minipage}{0.55\textwidth} \begin{center} \includegraphics[width=\linewidth]{images/ov5648-block.png}\\ OV5648 block diagram \textit{(Omnivision)} \end{center} \end{minipage} \hfill \begin{minipage}{0.4\textwidth} \begin{center} \includegraphics[width=0.9\linewidth]{images/bayer-pattern.png}\\ Bayer pattern \textit{(CC BY-SA 3.0, Cburnett, Wikimedia Commons)} \end{center} \end{minipage} \end{frame} \begin{frame}{Hardware Interfaces for Capture: Schematics} \begin{minipage}{0.55\textwidth} \begin{center} \includegraphics[width=\linewidth]{images/csi-camera.png} \end{center} \end{minipage} \hfill \begin{minipage}{0.4\textwidth} \begin{center} \includegraphics[width=\linewidth]{images/mipi-csi2-camera.png} \end{center} \end{minipage} \begin{center} Parallel and MIPI CSI-2 interfaces on the S3-OLinuXino \end{center} \end{frame} \end{document}