\documentclass[aspectratio=169,obeyspaces,spaces,hyphens,dvipsnames]{beamer}
\usepackage[utf8]{inputenc}
\usepackage{hyperref}
\usepackage{minted}

\mode<presentation>
\usetheme{Bootlin}

\def\signed #1{{\leavevmode\unskip\nobreak\hfil\penalty50\hskip2em       
  \hbox{}\nobreak\hfil(#1)%                                              
  \parfillskip=0pt \finalhyphendemerits=0 \endgraf}}

\newsavebox\mybox
\newenvironment{aquote}[1]
  {\savebox\mybox{#1}\begin{quotation}}
  {\signed{\usebox\mybox}\end{quotation}}

\newcommand{\codehack}[1]{{\usebeamercolor[fg]{code} {\tt #1}}}

\title{Building a board farm: Continuous Integration and remote control}

\conference{Embedded Linux Conference Europe 2016}
\authors{Quentin Schulz, Antoine Ténart}
\email{quentin.schulz@bootlin.com
  antoine.tenart@bootlin.com}
\institute{Bootlin}

\newcounter{lessonno}

\begin{document}

\section{Building a board farm: Continuous Integration and remote control}
\begin{frame}{Quentin Schulz}
  \begin{itemize}
  \item quentin.schulz@bootlin.com
  \item Embedded Linux engineer at {\bf Bootlin}
  \item Former intern for creating a board farm for kernel
    continuous integration at {\bf Bootlin}
    \begin{itemize}
      \item Embedded Linux specialists.
      \item Development, consulting and training (materials freely
        available under a Creative Commons license).
      \item \url{http://bootlin.com}
    \end{itemize}
  \item Living in {\bf Toulouse}, south west of France.
  \end{itemize}
\end{frame}

\begin{frame}{Antoine Ténart}
  \begin{itemize}
  \item antoine.tenart@bootlin.com
  \item Embedded Linux engineer at {\bf Bootlin}.
  \item Contributions
    \begin{itemize}
    \item Kernel support for the Marvell Berlin ARM SoCs.
    \item Kernel support for the Annapurna Labs ARM64 Alpine v2
      platform.
    \end{itemize}
  \item Living in {\bf Toulouse}, south west of France.
  \end{itemize}
\end{frame}

\begin{frame}{Table of contents}
\tableofcontents
\end{frame}

\section{Introduction}

\begin{frame}{What is continuous integration?}
  %code base = Linux kernel
  \begin{itemize}
  \item Continuous Integration (CI) is a software engineering practice
    in which isolated changes are immediately tested and reported on
    when they are added to a larger code base. The goal of CI is to
    provide rapid feedback so that if a defect is introduced into the
    code base, it can be identified and corrected as soon as possible.
  \item Three components: continuous builds, test automation and
    processing of the test results.
  \end{itemize}
Source: \href{http://searchsoftwarequality.techtarget.com/definition/continuous-integration}{TechTarget.com}
\end{frame}

\begin{frame}{Why does the kernel need it?}
  \begin{itemize}
  \item Lots of different platforms (especially in the ARM world)
  \item Hard to test all the changes on all platforms
  \item Very frequent changes made by the community: new Linux release
    every two months, thousands of changes
  \item Need to detect regressions early
%test boot on x86 platforms, apply patches and compile other platforms
  \item Intel 0-day build bot is mainly for x86 platforms
  \end{itemize}
\end{frame}

\begin{frame}{Why do we need it?}
  \begin{itemize}
  \item Bootlin contributes to ARM platforms upstream support
  \item Cooperation with several ARM processor vendors
  \item Many Bootlin engineers are maintainers of ARM and ARM64
    platforms
    \begin{itemize}
    \item Grégory Clement: Marvell EBU
    \item Maxime Ripard: Allwinner
    \item Alexandre Belloni: Atmel
    \item Antoine Ténart: Annapurna Labs
    \end{itemize}
  \item Keep track of modifications impacting the platforms we
    maintain
  \end{itemize}
\end{frame}

\section{Components of continuous integration}
\begin{frame}{Components of continuous integration}
  \begin{center}
  \includegraphics[width=\textwidth]{ci-header.pdf}
  \end{center}
\end{frame}

\section{KernelCI}

\begin{frame}{KernelCI - Overview}
  \begin{center}
  \includegraphics[width=\textwidth]{ci-kci-header.pdf}
  \end{center}
  \begin{itemize}
    \item \url{https://kernelci.org}
    \item Detects regressions before reaching users
    \item 2.000+ boot tests per day on 200+ unique boards
  \end{itemize}
\end{frame}

\begin{frame}{KernelCI - Tracker}
  \begin{center}
  \includegraphics[width=\textwidth]{ci-kci-track-header.pdf}
  \end{center}
  \begin{itemize}
  \item Tracks $\sim$20 kernel git repositories for changes
    \begin{itemize}
    \item torvalds/linux.git
    \item arm/arm-soc.git
    \item next/linux-next.git
    \item davem/net-next.git
    \item stable/linux-stable.git
    \item ...
    \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}{KernelCI - Builder}
  \begin{center}
  \includegraphics[width=\textwidth]{ci-kci-build-header.pdf}
  \end{center}
  \begin{itemize}
  \item Builds kernels from tracked repositories
  \item Automatically builds all defconfigs for ARM, ARM64 and x86
    (and their associated device trees, if any)
  \end{itemize}
\end{frame}

\begin{frame}{KernelCI - Connection with test automation}
  \begin{center}
  \includegraphics[width=\textwidth]{ci-kci-header.pdf}
  \end{center}
  \begin{itemize}
  \item Works with contributing labs
%one boot test consists of a couple defconfig+device tree. It is per-board unique.
%results available on the website, per soc, per kernel, per board
%notify by sending a mail in the ML
  \item Sends boot tests to labs, collects result and notifies maintainers of failures
  \end{itemize}
\end{frame}

\section{Test automation - Software}

\begin{frame}{Test automation}
  \begin{center}
  \includegraphics[width=\textwidth]{ci-test-header.pdf}
  \end{center}
  \begin{itemize}
    \item Controls boards
    \item Launches tests on the boards
    \item Validates the tests and gathers the results
  \end{itemize}
\end{frame}

\subsection{LAVA}

\begin{frame}{Software - LAVA 1/2}
% self-hosted server controlling boards organized in server-dispatcher
% model
KernelCI labs should use Linaro Automated Validation Architecture
(LAVA) which:
  \begin{itemize}
% controls done via external tools (conmux/ser2net, pdudaemon/custom implementation)
  \item Controls boards
% called jobs, virtually anything is possible (even network testing between two boards with synchro)
% a job is a set of where to find an image to boot: kernel (linux or android), dtb, initramfs, which tests to run and wwhere to store the boot report.
  \item Automates boot, bootloader, user-space, ... testing
% but the same test is only run once on the same device type
  \item Runs tests simultaneously on all boards
% allow git hooks, crontab, jenkins, anything to be used with LAVA
  \item Provides API for full automation
% keep all logs of jobs, parse test results and keep them in a stream
  \item Validates tests
  \item \url{https://wiki.linaro.org/LAVA}
  \end{itemize}
\end{frame}

\begin{frame}{Software - LAVA 2/2}
  \begin{itemize}
  \item Organized in a master - dispatchers fashion
  \item Only 1 master working with N dispatchers
  \item The master controls the farm
    \begin{itemize}
    \item It handles the API and receives the tests to run
    \item It schedules the tests to run
  \end{itemize}
  \item A given dispatcher handles a set of boards
    \begin{itemize}
    \item It has the boards' configuration files
    \item It is physically connected to the boards and controls them
    \item It runs the tests
    \end{itemize}
  \item We chose to host the master and our only dispatcher on the
    same machine
  \end{itemize}
\end{frame}

\begin{frame}{LAVA inner working}
  \begin{center}
  \includegraphics[width=\textwidth]{lava-control-scheme.pdf}
  \end{center}
\end{frame}

%devicetype is representative of a type of board, let say CHIP
%boards, they share all the same bootloader commands
%device is representative of a single board, let say CHIP1, which
%has specific information on the board (how to control its power
%and where to find the serial)

\begin{frame}[fragile]{LAVA configuration files - Device}
  \begin{center}
%name of the device type configuration file
%name of the device as configured in LAVA
  \begin{minted}[fontsize=\normalsize,breaklines=true]{bash}
$ cat /etc/lava-dispatcher/devices/sun5i-r8-chip_01.conf
device_type = sun5i-r8-chip
hostname = sun5i-r8-chip_01
hard_reset_command = pduclient --daemon localhost --hostname drawer6 --command reboot --port 03 --delay 2
power_off_cmd = pduclient --daemon localhost --hostname drawer6 --command off --port 03 --delay 2
connection_command = telnet localhost 6063
  \end{minted}
  \end{center}
\end{frame}

%/!\ LAVA has two different types of configuration for a device:
%configuration files (the old way) and direct API configuration (via
%yaml, the new way). The old way uses JSON job submission and the new
%way YAML job submission. KCI is only sending JSON job submission so
%we have to keep configuring the old way until YAML job submission is
%supported by KCI.
\begin{frame}[fragile]{LAVA configuration files - Device type}
  \begin{center}
%RAM load addresses for kernel, initramfs and DTB
%commands executed in bootloader
%extends device-defaults.conf
  \begin{minted}[fontsize=\tiny]{bash}
$ cat /etc/lava-dispatcher/device-types/sun5i-r8-chip.conf
client_type = bootloader
send_char = False

z_load_addrs =
    0x42000000
    0x43300000
    0x43000000

boot_cmds_ramdisk =
    setenv autoload no,
    setenv kernel_addr_r "'{KERNEL_ADDR}'",
    setenv initrd_addr_r "'{RAMDISK_ADDR}'",
    setenv fdt_addr_r "'{DTB_ADDR}'",
    setenv ethact "'asx0'",
    setenv loadkernel "'tftp ${kernel_addr_r} {KERNEL}'",
    setenv loadinitrd "'tftp ${initrd_addr_r} {RAMDISK}; setenv initrd_size ${filesize}'",
    setenv loadfdt "'tftp ${fdt_addr_r} {DTB}'",
    setenv bootargs "'console=ttyS0,115200 earlyprintk root=/dev/ram0 ip=dhcp'",
    setenv bootcmd "'usb start; dhcp; setenv serverip {SERVER_IP}; run loadkernel; run loadinitrd; run loadfdt; {BOOTX}'",
    boot 

bootloader_prompt = =>

boot_options =
    boot_cmds

[boot_cmds]
default = boot_cmds
  \end{minted}
\end{center}
\end{frame}

\begin{frame}{Test run in LAVA}
  \begin{center}
  \includegraphics[width=0.95\textwidth]{lava-scheme.pdf}
  \end{center}
\end{frame}

%Presentation of different solutions and the solution we chose
\section{Test automation - Hardware}

\subsection{Power supply control}

\begin{frame}{Hardware - Power supply control}
We need remotely controlled power supplies:
\begin{figure}[H]                                                     
  \centering
  \begin{minipage}{0.55\textwidth}
    \center
    \includegraphics[width=0.75\textwidth]{pdu.jpg}
  %kettle cords or full-size (US) sockets
    \caption{Power Distribution Unit}
    \includegraphics[width=0.7\textwidth]{relays.jpg}
  %Network, USB, WiFi/BT controlled
    \caption{Remotely controlled relays}
  \end{minipage}
  \hfill
  \begin{minipage}{0.25\textwidth}
    \includegraphics[width=0.7\textwidth]{pdu-socket.jpg}
  %US sockets only for the 8-ports version
    \caption{Network controlled multi-sockets}
  \end{minipage}
\end{figure}
\end{frame}

\begin{frame}{Hardware - Power supply control}
\begin{minipage}{0.55\textwidth}
We chose remotely controlled relays because of:
\begin{itemize}
\item their cheap price
\item the number of ports
\item their small footprint
\item their documented TCP protocol
\item their support for virtually any power supply (you just need a
  wire)
\end{itemize}
\end{minipage}
\begin{minipage}{0.35\textwidth}
\center
\includegraphics[width=\textwidth]{relays.jpg}
\end{minipage}
\end{frame}

\subsection{Power supply}

\begin{frame}{Hardware - Power supply}
\begin{itemize}
\item Three different types of boards:
  \begin{itemize}
  %called non-ATX supplied boards
  \item 5V powered boards
  \item 12V powered boards
  %called ATX supplied boards
  \item full ATX powered boards
  \end{itemize}
  \item We separate those in two kinds:
    \begin{itemize}
    \item non-ATX supplied boards
    \item ATX supplied boards
    \end{itemize}
\end{itemize}
\end{frame}

\begin{frame}{Hardware - Power supply of non-ATX supplied boards}
  \begin{itemize}
  \item Different input voltages, two solutions:
    \begin{itemize}
    %enough amperage to power all boards and potentially hard drives or
    %other on-board devices
    \item one power supply per voltage with enough amperage
    \item one power supply for all voltages with enough amperage
    \end{itemize}
  \item We chose ATX power supplies to get all voltages from one power supply
  \end{itemize}
\end{frame}

\begin{frame}{Hardware - Power supply of non-ATX supplied boards}
\begin{figure}[H]
  \center
    \includegraphics[width=0.5\textwidth]{atx-specs.png}
    \caption{ATX specifications}
  \begin{minipage}{0.35\textwidth}
  \center
    \includegraphics[width=0.6\textwidth]{atx-power-supply.jpg}
    \caption{ATX power supply}
  \end{minipage}
  \hfill
  \begin{minipage}{0.25\textwidth}
  \center
    \includegraphics[width=0.6\textwidth]{tvs-diode.jpg}
  %supposed to absorb all extra voltage once
    \caption{TVS diode}
  \end{minipage}
\end{figure}
\end{frame}

\begin{frame}{Hardware - ATX power supply specifics}
\begin{minipage}{0.25\textwidth}
\begin{figure}[H]
  \includegraphics[width=\textwidth]{ps_on.jpg}
  \caption{24-pins ATX connector}
\end{figure}
\end{minipage}
\begin{minipage}{0.65\textwidth}
\begin{itemize}
\item does not always supply power
\item waits for a signal on \#PS\_ON or for \#PS\_ON to be put to the
ground
\item we need it to supply power all the time for non-ATX power
supplied boards (the power from ATX power supply to the boards is
controlled by per-board relays)
\item we need to control when it supplies power to ATX power supplied
boards
\end{itemize}
\end{minipage}
\end{frame}

\subsection{Interaction with boards}

\begin{frame}{Hardware - Connect to serial}
\begin{minipage}{0.45\textwidth}
  \begin{itemize}
  \item Mostly USB cable to board
  \item Lots of USB hubs
  \end{itemize}
\end{minipage}
\begin{minipage}{0.45\textwidth}
\begin{figure}
\includegraphics[width=0.8\textwidth]{usb-hub10.jpg}
\end{figure}
\begin{figure}
\includegraphics[width=0.8\textwidth]{usb-hub4.jpg}
\end{figure}
\end{minipage}
  %pics of different connectors we have in the lab?
\end{frame}

\begin{frame}{Hardware - Get and send files to boards}
\begin{minipage}{0.45\textwidth}
  \begin{itemize}
  \item TFTP protocol
  \item need of switches and Ethernet cables
  \end{itemize}
\end{minipage}
\begin{minipage}{0.45\textwidth}
\begin{figure}
\includegraphics[width=\textwidth]{switch.jpg}
\end{figure}
\end{minipage}
\end{frame}

\section{Actual building of the board farm}

\begin{frame}{Board farm - Specifications}
  \begin{itemize}
  %"requirement": pass through a door, specific place in our office
  \item Specific location (200*100*75cm)
  %antistatic, not metallic, etc...
  \item Harmless to boards (material choices)
  %as least cables as possible, drawers, scratch to reposition &
  %reorder,
  \item Easy to use
  \item Allowing evolution
  %separate big boards and small boards
  %up to 8 small boards per drawer, up to 4 big boards per drawer
  %list of 48 boards in the company
  \item As many boards as possible
  \end{itemize}
\end{frame}

\begin{frame}{Board farm - Small drawers}
  \begin{center}
  \includegraphics[width=0.6\textwidth]{small-drawer-scheme.pdf}
  \end{center}
\end{frame}

\begin{frame}{Board farm - Small drawers - IRL}
  \begin{center}
  \includegraphics[width=0.8\textwidth]{small_drawer.JPG}
  \end{center}
\end{frame}

\begin{frame}{Board farm - Big drawers}
  \begin{center}
  \includegraphics[width=0.6\textwidth]{big-drawer-scheme.pdf}
  \end{center}
\end{frame}

\begin{frame}{Board farm - Big drawers - IRL}
  \begin{center}
  \includegraphics[width=0.8\textwidth]{big_drawer.JPG}
  \end{center}
\end{frame}

\begin{frame}{Board farm}
  \begin{center}
  \includegraphics[width=0.275\textwidth]{farm.jpg}
  \end{center}
\end{frame}

\section{Board farm - Feedback}

\begin{frame}{Board farm - Some numbers}
  \begin{itemize}
  \item Launched on April 25th 2016
  \item Currently 35 boards (estimated capacity of 50)
  \item 160k+ tests run
  \item 30+ unique devices support added to KernelCI
  \end{itemize}
\end{frame}

\begin{frame}{Board farm - Some challenges}
  \begin{itemize}
  %xHCI problem, normally possible to disable it in BIOS but had to
  %recompile the kernel
  \item Many devices connected to the LAVA server which may have
    limitations. We had to recompile the kernel on this machine!
  %Marvell boards = hardware hacking (rising edge for booting)
  \item All boards are different: specific U-Boot configuration, h/w
    modifications needed to automate the boot, very old bootloaders
    (U-Boot 1.1.1 from 2004)\dots
  %ser2net LOCAL option, automatic reconnection (connect_forever
  %script), serial drops during tests, sama5d3x & sama5dx_xplained
  %have to be manually replugged
  \item Expect everything to fail: buggy serial connections, s/w
    services or machine configuration\dots
  \item LAVA assumptions may not match the hardware capabilities
  \end{itemize}
\end{frame}

\begin{frame}{Board farm - Documentation}
  \begin{itemize}
  \item LAVA: \url{https://validation.linaro.org/static/docs/index.html}
  \item KernelCI: \url{http://wiki.kernelci.org/}
  \item Configure LAVA to receive tests from KernelCI:
    \url{https://github.com/kernelci/lava-ci\#configure-lava}
  \item Adding a board to KernelCI:
    \url{https://github.com/kernelci/lava-ci\#add-board-to-kernelci}
  \item Our articles on the matter:
    \url{http://bootlin.com/blog/tag/lab/}
  \end{itemize}
\end{frame}

\section{Remote control}

\begin{frame}{Why?}
  \begin{itemize}
    \item Our farm knows how to handle boards, has a lot of them\dots
      but:
      \begin{itemize}
      %LAVA does not let users to take control of boards
      \item There is no direct access to the boards
      \item Only tests sent to LAVA can perform actions on the boards
      \end{itemize}
    %board in farm is unusable, have to take it outside the lab to work
    %on it
    \item Some boards owned only once
    %material to take in luggages
    \item Working remotely
  \end{itemize}
\end{frame}

\section{LAVA Board Overseer}

\begin{frame}{Lavabo}
  \begin{itemize}
  %KISS
  \item Reuses the same tools LAVA uses
  \item Takes full control
  %avoid concurrent access, directory per user in TFTP directory on
  %server
  \item Authenticates users
  %reserve/release boards, get status
  \item Interacts with LAVA
  \end{itemize}
\end{frame}

\begin{frame}{Lavabo reuses LAVA tools}
  \begin{center}
  %based on configuration files to get where the serial is and how to
  %control power, connect to serial connection via ssh port forwarding
  %push files to TFTP directory accessible by boards via SFTP
  %cooperate with LAVA to deny job submission to a board
  \includegraphics[width=0.85\textwidth]{lavabo-scheme.pdf}
  \end{center}
\end{frame}

\begin{frame}{Lavabo's architecture}
\begin{itemize}
\item client-server model
\item server must be on the same machine as where LAVA server is
hosted
\item no support for multi-node LAVA
\item one dedicated SSH user on the server
\item one SSH key per lavabo real user
\item LAVA's \code{connection_command} for all devices must be telnet
\item no support for rootfs on NFS
\end{itemize}
\end{frame}

\begin{frame}[fragile]{Typical workflow}
\begin{minted}[fontsize=\scriptsize]{bash}
$ lavabo list
status      job  offline_since             hostname                             offline_by
--------  -----  ------------------------  -----------------------------------  ------------
idle                                       alpine-db_01
offline          Tue Oct  4 14:23:51 2016  alpine-v2-evp_01                     antoine
idle                                       armada-370-db_01
offline          Wed Jul 13 09:13:37 2016  armada-370-rd_01                     quentin
offline          Wed Sep 21 15:46:56 2016  armada-3720-db_01                    omar
[...]
idle                                       sun5i-r8-chip_01
idle                                       sun8i-a33-sinlinx-sina33_01
idle                                       sun8i-a83t-allwinner-h8homlet-v2_01

$ lavabo reserve sun5i-r8-chip_01
$ lavabo upload mykernel sun5i-r8-chip.dtb myrootfs
File(s) successfully sent to lavabo-server.
$ lavabo reset sun5i-r8-chip_01
\end{minted}
\end{frame}

\begin{frame}[fragile]{Typical workflow}
\begin{minted}[fontsize=\scriptsize]{bash}
$ lavabo serial sun5i-r8-chip_01
Try 1 to connect to serial failed. 4 attempts remaining.
You have now access to the serial of sun5i-r8-chip_01.
Escape character is '^]'.
U-Boot SPL 2016.01-g67a66a1-dirty (Mar 09 2016 - 12:04:29)
DRAM: 512 MiB
CPU: 1008000000Hz, AXI/AHB/APB: 3/2/2
Trying to boot from NAND

U-Boot 2016.01-g67a66a1-dirty (Mar 09 2016 - 12:04:29 +0100) Allwinner Technology
CPU:   Allwinner A13 (SUN5I)
I2C:   ready
DRAM:  512 MiB
NAND:  8192 MiB
video-mode 720x480-24@60 not available, falling back to 1024x768-24@60
Setting up a 720x480i composite-ntsc console (overscan 40x20)
[...]
Hit any key to stop autoboot:  0 
=>
$ lavabo power-off sun5i-r8-chip_01
$ lavabo release sun5i-r8-chip_01
\end{minted}
\end{frame}

\begin{frame}{Lavabo - Conclusion}
  \begin{itemize}
  %only one server which serves as dispatcher also, telnet only, no
  %NFS boot
  \item Some limitations
  \item GNU GPLv2 licensed
  \item \url{https://github.com/bootlin/lavabo}
  %demo
  \item Let's play!
  \end{itemize}
\end{frame}

\begin{frame}
  \begin{center}
  \vspace{1cm}
  \Huge
  Thanks! Questions?\\
  \vspace{3cm}
  \large
  Slides under CC-BY-SA 3.0\\
  \url{http://bootlin.com/pub/conferences/2016/elce/schulz-tenart-building-boards-farm/}
  \end{center}
\end{frame}

\end{document}