\documentclass[aspectratio=43,obeyspaces,spaces,hyphens,dvipsnames]{beamer}
\usepackage[utf8x]{inputenc}
\usepackage{lmodern}% http://ctan.org/pkg/lm
\usepackage{minted}
\usepackage{hyperref}
\usepackage{xcolor}
\usepackage{pgfplots}
\usepackage{tikz}
\usepackage[normalem]{ulem}
\usepackage{textcomp}

\usepackage{multimedia}

\mode<presentation>
\usetheme{Bootlin}

\def\signed #1{{\leavevmode\unskip\nobreak\hfil\penalty50\hskip2em
  \hbox{}\nobreak\hfil(#1)
  \parfillskip=0pt \finalhyphendemerits=0 \endgraf}}

\newsavebox\mybox
\newenvironment{aquote}[1]
  {\savebox\mybox{#1}\begin{quotation}}
  {\signed{\usebox\mybox}\end{quotation}}

\title{From the Camera Sensor to the User, the Journey of a Video Frame}
\conference{Embedded Linux Conference Europe, October 2020}
\authors{Maxime Chevallier}
\email{{\tiny maxime.chevallier@bootlin.com}}
\institute{Bootlin}
\slidesurl{https://bootlin.com/pub/conferences/2020}

\begin{document}

\addtocontents{toc}{\protect\setcounter{tocdepth}{-1}}
\section{From the Camera Sensor to the User\\ {\large the Journey of a Video Frame}}
\addtocontents{toc}{\protect\setcounter{tocdepth}{2}}

\begin{frame}{Maxime Chevallier}
  \begin{itemize}
    \item Linux kernel engineer at Bootlin.
      \begin{itemize}
        \item Linux kernel and driver development, system integration,
          boot time optimization, consulting\dots
        \item Embedded Linux, Linux driver development, Yocto Project
          \& OpenEmbedded and Buildroot training, with materials
          freely available under a Creative Commons license.
        \item \url{https://bootlin.com}
      \end{itemize}
    \item Contributions:
      \begin{itemize}
        \item Worked on network (MAC, PHY, switch) engines.
        \item Contributed to the Marvell EBU SoCs upstream support.
        \item Worked on Rockchip's Camera interface and Techwell's TW9900 decoder.
      \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}{Preamble - goals}
  \begin{itemize}
    \item Discover the hardware components and protocols involved in video cameras
    \item Understand how all these components chain together and how to configure them
    \item See various real-life hardware designs and use-cases
  \end{itemize}
\end{frame}

%\begin{center}
%	\includegraphics[width=0.6\textwidth]{graphics/parser.pdf}
%\end{center}

\section{Video Acquisition Hardware}

\begin{frame}{Necessary components}
	\begin{center}
		\includegraphics[width=0.6\textwidth]{graphics/hw_pipeline.pdf}
	\end{center}
	% schema
	\begin{itemize}
		\item Optical to Electrical conversion : Lenses and Sensors
		\item Signal transmission : Digital and Analog protocols
		\item Signal handling : Controllers and Signal Transcoders
		\item Image transformation : ISPs, Image Encoders/Decoders
	\end{itemize}
\end{frame}

\begin{frame}{Image capture}
	\center{Image acquisition setup}
	\begin{center}
		\includegraphics[height=6cm]{graphics/optical_sensor.pdf}
	\end{center}
\end{frame}

\begin{frame}{Lens}
	\begin{columns}
	\begin{column}{0.70\linewidth}
	\begin{itemize}
		\item Lens :
	\begin{itemize}
		\item Controls the focus of the incoming light rays
		\item Can be adjusted for manual or auto-focus
	\end{itemize}
	\item Voice Coil Actuator :
	\begin{itemize}
		\item Lens position is adjusted by a Voice Coil actuator
		\item A wire coil is attached to the Lens
		\item The Lens assembly sits in a static magnetic field
		\item The coil is drived by a DAC providing adjustable current
		\item Mostly controlled through I²C
		\item \code{MEDIA_ENT_F_LENS}
	\end{itemize}
	\end{itemize}
	\end{column}
	\begin{column}{0.30\linewidth}
	\begin{center}
		\includegraphics[width=\textwidth]{graphics/optical_sensor_lens.pdf}
	\end{center}
	\end{column}
	\end{columns}
\end{frame}

\begin{frame}{Flash}
	\begin{columns}
	\begin{column}{0.70\linewidth}
	\begin{itemize}
		\item High-power LED driver
		\item Sometimes include a \textit{privacy indicator}
		\item Controlled through I²C
		\item \code{MEDIA_ENT_F_FLASH}
	\end{itemize}
	\end{column}
	\begin{column}{0.30\linewidth}
	\begin{center}
		\includegraphics[width=\textwidth]{graphics/optical_sensor_flash.pdf}
	\end{center}
	\end{column}
	\end{columns}
\end{frame}

\begin{frame}{Sensor}
	%schema
	\begin{center}
		\includegraphics[width=0.8\textwidth]{graphics/sensor.pdf}
	\end{center}
	\begin{itemize}
		\item Converts an optical signal to an analog electrical signal
		\item Uses \code{CCD} or \code{CMOS} technologies
		\item Include internal ADCs and amplifiers
		\item Sometimes include an \code{Image Signal Processor}
		\item Data plane uses a dedicated protocol
		\item Control plane mostly uses I²C
	\end{itemize}
\end{frame}

\begin{frame}{Digital Sensors}
	%schema
	\begin{center}
		\includegraphics[width=0.6\textwidth]{graphics/bayer.png}
	\end{center}
	\begin{itemize}
		\item Sensors acquire 3 color components, arranged in a grid
		\item These colors are sometimes sent raw : \code{RGGB}, \code{RGBG}, etc.
		\item Conversion from raw sampling to pixel value : \code{debayering}
		\item Also called \code{demosaicing}, can be done in-situ
		\item On basic sensors, this need to be done in the Host
		\item Possibly costly operation, depending on the algorithm
	\end{itemize}
\end{frame}


\begin{frame}{Digital transmission protocols : Raw Parallel}
	\begin{columns}
	\begin{column}{0.70\linewidth}
	\begin{itemize}
		\item Simple approach : Transmit data and sync signals
		\item Consists of the following lines :
			\begin{itemize}
				\item Parallel data lines : 8 to 12 bits
				\item Pixel Clock : Ticks every pixel sent
				\item HSYNC : Toggles on line end
				\item VSYNC : Toggles on frame end
			\end{itemize}
		\item Often needs post-processing, such as \code{demosaicing}
	\end{itemize}
	\end{column}
	\begin{column}{0.30\linewidth}
	\begin{center}
		\includegraphics[width=\textwidth]{graphics/raw_parallel.pdf}
	\end{center}
	\end{column}
	\end{columns}
\end{frame}

\begin{frame}{Digital transmission protocols : CCP2}
	\begin{columns}
	\begin{column}{0.70\linewidth}
	\begin{center}
		\textbf{C}ompact \textbf{C}amera \textbf{P}ort \textbf{2}
	\end{center}
	\begin{itemize}
		\item Serialized interface
		\item Sync signals embedded in data
		\item Uses differential pairs to transmit data and clocks
		\item PHY Layer is based on \code{subLVDS}
		\item 4 pins needed : 2 for clk and 2 for data
		\item Up to 650 mbps
	\end{itemize}
	\end{column}
	\begin{column}{0.30\linewidth}
	\begin{center}
		\includegraphics[width=\textwidth]{graphics/ccp2.pdf}
	\end{center}
	\end{column}
	\end{columns}
\end{frame}


\begin{frame}{Transmission protocols : MIPI CSI}
	\begin{center}
		\textbf{C}amera \textbf{S}erial \textbf{I}nterface
	\end{center}
	\begin{itemize}
		\item Standard from the MIPI Alliance
		\item Multiple layers defined :
			\begin{enumerate}
				\item \textbf{PHY} : Physical transmission
				\item \textbf{Lane Management} : Lane distribution and merging
				\item \textbf{Low Level Protocol} : Checksuming, Error Correction
				\item \textbf{Application} : Pixel to Byte conversion
			\end{enumerate}
		\item CSI-2 is the most used
		\item CSI-3 : Bi-directional protocol
	\end{itemize}
\end{frame}

\begin{frame}{MIPI D-PHY Layer}
	\begin{columns}
	\begin{column}{0.70\linewidth}
		\begin{itemize}
			\item Serialized interface
			\item Sync signals embedded in data
			\item Uses differential pairs to transmit data and clocks
			\item Minimum 4 pins : 2 for clk and 2 per lane
			\item Up to 4 lanes
			\item Up to 6 Gbps
		\end{itemize}
	\end{column}
	\begin{column}{0.30\linewidth}
		\begin{center}
		\includegraphics[width=\textwidth]{graphics/mipi_dphy.pdf}
		\end{center}
	\end{column}
	\end{columns}
\end{frame}

\begin{frame}{MIPI C-PHY Layer}
	\begin{columns}
	\begin{column}{0.70\linewidth}
		\begin{itemize}
			\item Serialized interface
			\item Sync signals and clock embedded in data
			\item 3-levels signals : High, Med and Low
			\item 3 lines per lane
			\item 16 bits transmitted over 7 symbols (in quinary)
			\item Up to 3 lanes
			\item Up to 41 Gbps
		\end{itemize}
	\end{column}
	\begin{column}{0.30\linewidth}
		\begin{center}
		\includegraphics[width=\textwidth]{graphics/mipi_cphy.pdf}
		\end{center}
	\end{column}
	\end{columns}
\end{frame}


\begin{frame}{Analog transmission protocols}
		\begin{center}
		\includegraphics[width=0.3\textwidth]{graphics/PAL-NTSC-SECAM.pdf}
		\end{center}
	\begin{itemize}
		\item Video Broadcasting protocols
		\item Designed for transmission over an analog media
		\item Decomposes video into components : Y, Cr, Cb
		\item Y : Luminance ( Black and White image )
		\item U, V : Chrominance ( Color information )
		\item \textbf{PAL} : Europe
		\item \textbf{NTSC} : USA, Japan
		\item \textbf{SECAM} : France, Eastern Europe, Russia
	\end{itemize}
\end{frame}

\begin{frame}{Interlacing}
	\begin{columns}
	\begin{column}{0.70\linewidth}
		\begin{itemize}
			\item Increase the perceived framerate with the same bandwidth
			\item Transmit the odd lines, then the even lines
			\item Each part is called a \code{field}
			\item Requires a compatible display
			\item Else the video need to be \code{deinterlaced}
			\item Requires some signal processing to get correct results
		\end{itemize}
	\end{column}
	\begin{column}{0.30\linewidth}
		\begin{center}
		\includegraphics[width=\textwidth]{graphics/interlaced.jpg}
			\small{Interlacing artifacts}
		\end{center}
	\end{column}
	\end{columns}
\end{frame}

\begin{frame}{Analog to Digital Transcoders}
	\begin{center}
		\includegraphics[width=0.6\textwidth]{graphics/decoder.pdf}
	\end{center}
	\begin{itemize}
		\item Converts an analog video signal into a digital signal
		\item Can support multiple input standards
		\item Can embbed a small \code{ISP} for simple cropping/scaling
		\item Converts into a digital video standard :
		\begin{itemize}
			\item BT.656 for \code{PAL} and \code{NTSC} standards
			\item BT.1120 for higher resolution standards
		\end{itemize}
		\item Parallel or Serial interfaces supported by BT.* standards
	\end{itemize}
\end{frame}

\begin{frame}{Host Interface}
	\begin{center}
		\includegraphics[width=0.4\textwidth]{graphics/host-interface.pdf}
	\end{center}

	\begin{itemize}
		\item Hardware blocks located in the SoC
		\item Has lots of different features depending on the Hardware :
			\begin{itemize}
				\item PHY layer support
				\item Image processing (simple or advanced), including :
				\begin{itemize}
					\item Scaling, Cropping
					\item Deinterlacing
					\item Pixel format conversion
				\end{itemize}
			\end{itemize}
		\item Stores the frame into buffers using \code{DMA}
		\item The \code{V4L2} framework and the \code{media controller API} supports these blocks
	\end{itemize}
\end{frame}

\begin{frame}{Image processing}
	\begin{itemize}
		\item Cropping : Remove areas from the image
			\begin{itemize}
				\item {\footnotesize Easy to perform}
			\end{itemize}
		\item Scaling : Resize the image
			\begin{itemize}
				\item {\footnotesize Can require complex algorithms}
			\end{itemize}
		\item Deinterlacing : Recompose an interlaced stream
			\begin{itemize}
				\item {\footnotesize Joining fields is easy}
				\item {\footnotesize Removing artifacts can be complex}
			\end{itemize}
		\item Pixel format conversion
			\begin{itemize}
				\item {\footnotesize Debayering : Convert raw sensor data to a usable image format}
				\item {\footnotesize Colorspace conversion : \code{RGB} to/from \code{YUV}}
			\end{itemize}
		\item 3A Processing
			\begin{itemize}
				\item {\footnotesize Auto exposure : Adjust the brightness, control the sensor \textbf{gain}}
				\item {\footnotesize Auto focus : Adjust the focal point, control the \textbf{lens position}}
				\item {\footnotesize Auto white balance : Correct the colors}
			\end{itemize}
	\end{itemize}
	\small{See the talk on ISP drivers by Helen Koike earlier today !}
\end{frame}

\begin{frame}{Example : Nokia N900}
	\begin{columns}
	\begin{column}{0.70\linewidth}
		\begin{itemize}
			\item Smartphone with full Linux support
			\item Based on TI OMAP3 SoC
			\item Has a Flash LED driver
			\item Voice Coil controlled through a DAC
			\item CSI Sensor controlled through I2C
			\item DTS found in \code{arch/arm/boot/dts/omap3-n900.dts}
		\end{itemize}
	\end{column}
	\begin{column}{0.30\linewidth}
		\begin{center}
		\includegraphics[width=\textwidth]{graphics/n900.jpg}
			\small{Nokia N900}
		\end{center}

		\begin{center}
		\includegraphics[width=\textwidth]{graphics/n900-camera.pdf}
		\end{center}
	\end{column}
	\end{columns}
\end{frame}

\begin{frame}{Example : Custom Product}
	\begin{center}
	\includegraphics[width=0.8\textwidth]{graphics/custom-camera.pdf}
	\end{center}
	\begin{itemize}
		\item Based on Rockchip PX30 SoC
		\item Remote video source through an analog signal : \code{PAL} or \code{NTSC}
		\item Decoded through a \code{tw9900}, which auto-detects the standard
		\item BT.656 Parallel bus to transmit the digital signal
		\item Uses the \code{VIP} Camera Interface (Upstreaming in progress)
		\item Fields are simply joint, not a full de-interlacing
	\end{itemize}
\end{frame}

\begin{frame}{Linux Support}
	\begin{itemize}
		\item All the above components and more are supported by Linux
		\item Wide variety of topologies, challenging to have a full-featured infrastructure
		\item \code{V4L2} Gives the infrastructure to support Video Cameras
		\item \code{V4L2} Also deals with buffer management and interaction with userspace
		\item The \code{Media Controller} API allows configuring each block through \code{subdevs}
		\item Complex devices can be handled in userspace with \code{libcamera}
		\item Welcoming community :)
	\end{itemize}
\end{frame}

\begin{frame}{Conclusion}
	\begin{itemize}
		\item The number of technologies involved can be overwhelming
		\item Old analog terminologies and technologies still apply today
		\item However, Linux support is pretty good
		\item \code{V4L2} and the \code{media controller} API allow complex use-cases
	\end{itemize}
\end{frame}


\begin{frame}
  \begin{center}
    \Huge
    Thank you! \\
    Questions? Comments?\\
    \vspace{1cm}
    \large
    Maxime Chevallier — \code{maxime.chevallier@bootlin.com}\\
    \vspace{1cm}
    Slides under CC-BY-SA 3.0
    \scriptsize
    \url{https://bootlin.com/pub/conferences/2019/elce/chevallier-network-classification-offload/}
  \end{center}
\end{frame}

\end{document}
