\documentclass[aspectratio=169,obeyspaces,spaces,hyphens,dvipsnames]{beamer}
\usepackage[utf8]{inputenc}
\usepackage{lmodern}% http://ctan.org/pkg/lm
\usepackage{minted}
\usepackage{hyperref}
\usepackage{xcolor}
\usepackage{pgfplots}
\usepackage{tikz}
\usepackage[normalem]{ulem}

\mode<presentation>
\usetheme{Bootlin}

\def\signed #1{{\leavevmode\unskip\nobreak\hfil\penalty50\hskip2em
  \hbox{}\nobreak\hfil(#1)
  \parfillskip=0pt \finalhyphendemerits=0 \endgraf}}

\newsavebox\mybox
\newenvironment{aquote}[1]
  {\savebox\mybox{#1}\begin{quotation}}
  {\signed{\usebox\mybox}\end{quotation}}

\title{Implementing A/B System Updates with U-Boot}
\authors{Michael Opdenacker}
\email{michael.opdenacker@bootlin.com}
\slidesurl{https://bootlin.com/pub/conferences/2022/elce/}
\institute{Bootlin}
\conference{Embedded Linux Conference Europe 2022}

\begin{document}

\addtocontents{toc}{\protect\setcounter{tocdepth}{-1}}
\section{Implementing A/B System Updates with U-Boot}
\addtocontents{toc}{\protect\setcounter{tocdepth}{2}}

\begin{frame}{Michael Opdenacker}
\begin{columns}
  \column{0.75\textwidth}
  \begin{itemize}
  \item Founder and Embedded Linux engineer at Bootlin:
    \begin{itemize}
    \item Embedded Linux {\bf expertise}
    \item {\bf Development}, consulting and training
    \item Strong open-source focus
    \end{itemize}
  \item About myself:
    \begin{itemize}
      \item Always happy to learn from every new project,\\
            and share what I learn.
      \item Initial author of Bootlin's freely available embedded Linux,
	    kernel and boot time reduction training materials (\url{https://bootlin.com/docs/})
      \item Current documentation maintainer for the Yocto Project
      \item Current maintainer of the Elixir Cross Referencer, making it
            easier to study the sources of big C projects like the Linux kernel.
            See \url{https://elixir.bootlin.com}.
    \end{itemize}
  \end{itemize}
  \column{0.25\textwidth}
    \begin{center}
    \vspace{0.5cm}
    \includegraphics[width=0.5\textwidth]{graphics/logo-cc.pdf}\\
    \vspace{0.5cm}
    \includegraphics[width=\textwidth]{graphics/elixir.pdf}
    \end{center}
\end{columns}
\end{frame}


\begin{frame}{Abstract}
\small \em A popular way to implement system updates is through the A/B scheme, in
which you have two copies of the root filesystem, one which is active,
and one that is meant to contain the next update. When a new update is
successfully applied, you need to make the corresponding partition
become the new active one. That's when a number of practical questions
arise, such as how to identify the active partition, how to detect when
the new system fails to boot properly, and how to fall back to the
previous version?

\vspace{1em}
It was hard to find documentation about how U-Boot could address such
needs to implement a functional and failsafe A/B system update
mechanism. This presentation proposes to address this need by sharing
the practical solutions we found, using lesser known commands and
capabilities in U-Boot. We will also explain how the Linux side can
cooperate with the U-Boot side. Fortunately, you won't need to erase
half of your brain to get updated on this topic.
\end{frame}

\subsection{Introduction}

\begin{frame}{Context and experience}
  \begin{itemize}
     \item Had the opportunity to explore system update techniques,
	   to prepare slides and labs for a customer seminar
	   (not ready to publish in its current form).
     \item We also wanted to let participants practise with the
           {\em A/B update technique}, using the {\em swupdate}
	   solution.
     \item Though details about the theory and how to apply the
	   updates were easy to find, some practical aspects of switching
	   between between versions needed more research.
     \item This presentation aims at sharing our findings and our
           practical experience.
  \end{itemize}
\end{frame}

\begin{frame}{Full image update approaches}
  \begin{columns}
    \column{0.62\textwidth}
    \begin{itemize}
    \item Consists in deploying to devices a new complete image of the
      system.
    \item Simplest and safest approach: everything is updated at the
      same time $\rightarrow$ guarantees that all components of the
      system are consistent and well tested.
    \item Not possible to update the system while it is running.
      $\rightarrow$ two partionning approaches are commonly used:
      \begin{enumerate}
      \item {\em A/B scheme}: two full copies of the root filesystem,
        alternating between active/inactive
      \item {\em Rescue system}: one full copy of the system, and one
        smaller rescue system used to do the update
      \end{enumerate}
    \item Bootloader integration to switch between systems.
    \end{itemize}
    \column{0.38\textwidth}
    \includegraphics[width=\textwidth]{graphics/full-image-update.pdf}
  \end{columns}
\end{frame}

\begin{frame}{Full image update: typical scenario}
  With {\em A/B scheme}:
  \begin{enumerate}
  \item The system runs from active copy A
  \item An update is triggered, either by bringing an update image
    locally (SD card, USB stick) or over the network
  \item The update image is flashed/written on copy B, currently
    inactive
  \item If the flashing is successful, the bootloader configuration is
    updated to mark copy B as the active copy
  \item The system is rebooted, and boots on the new system installed
    in copy B.
  \item Watchdog used to detect a non-functional system, and fallback to
    the old copy if necessary.
  \end{enumerate}
\end{frame}

\begin{frame}{Main practical questions}
  \begin{itemize}
     \item How to identify the active partition?
	\begin{itemize}
	   \item From U-Boot to boot the right version of the system
	   \item From Linux to deploy the upgrade in the non-active partition.
	\end{itemize}
     \item How to swap the active and inactive partitions?
	\begin{itemize}
	   \item From Linux after flashing an upgrade
	   \item From U-Boot after failing to boot a new version of the system.
	\end{itemize}
     \item How to fall back to the original version if the update fails to boot properly?
  \end{itemize}
  \vspace{0.5cm}
  Here, we will address these questions in a generic way, without using
  bootloader integration implemented by software update solutions
  ({\em swupdate}, {\em RAUC}, {\em Mender}...)\\
  \vspace{1cm}
  \includegraphics[height=0.18\textheight]{graphics/SWUpdate.pdf}
  \hfill
  \includegraphics[height=0.18\textheight]{graphics/RAUC_Logo.pdf}
  \hfill
  \includegraphics[height=0.12\textheight]{graphics/mender.png}
\end{frame}

\subsection{Switching between root filesystems}

\begin{frame}{Using partition bootable flags}
   When booting on block storage, a solution to specify the "active"
   root partition is to add a \code{bootable} flag to it.
   \begin{itemize}
   \item This should be done after an update is applied successfully.
   \item The command to run in Linux will depend on the type of
         partition table.
         \begin{itemize}
         \item Using a GPT partition table:
	       \begin{itemize}
               \item You can use the \code{sgdisk} command to toggle the \code{bootable}
	             status (bit attribute \code{2}) of specified partitions, for example:\\
	             \code{sgdisk -A 4:toggle:2 -A 5:toggle:2 /dev/mmcblk0}
               \item This assumes that only one of the two partitions has the \code{bootable}
	             flag. This command makes it lose it and the other one get it.
	       \end{itemize}
	 \item Using an MBR (legacy) partition table:
	       \begin{itemize}
	       \item You can use the \code{parted} command to add the
		     \code{bootable} flag to {\bf only one partition}:\\
		     \code{parted -s /dev/mmcblk0 set 3 boot on}
	       \item Note that \code{sgdisk} cannot be used here,
	             because it would convert your MBR partition table
		     into a GPT one, which may not be supported by
		     the rom code of your SoC.
	       \end{itemize}
         \end{itemize}
   \end{itemize}
\end{frame}

\begin{frame}{Detecting the active partition in Linux}
   You can use the below commands to detect the current bootable
   partition from Linux, and deduce on which partition the next update
   should be applied.
   \begin{itemize}
   \item Using a GPT partition table, for example:\\
	 \code{sgdisk -A 4:get:2 /dev/sdc | cut -d: -f3}\\
	 This gets the bootable status (bit attribute \code{2}) of partition 4.\\
         If it is set, you will get \code{1}, otherwise \code{0}.
   \item Using an MBR partition table, for example:\\
	 \code{BOOTABLE=$(parted /dev/mmcblk0 print | grep boot | cut -d' ' -f2)}\\
	 This returns the number of the partition which is marked as
         bootable (assuming there is only one).
   \end{itemize}
\end{frame}

\begin{frame}{Detecting the active partition in U-Boot at boot time}
   Here's a solution to detect the bootable partition U-Boot
   \begin{itemize}
   \item Compile U-Boot with support for the \code{part} command
         (\projconfig{u-boot}{CONFIG_CMD_PART}).
   \item You can then run:\\
         \code{part list mmc 0 -bootable bootpart}
   \item In the \code{bootpart} environment variable, you will
	 have the number of the bootable partition
         on the first MMC device.
   \item You can then use such information to load
	 the Linux kernel and DTB from the corresponding partition,
	 and to set the Linux kernel command line accordingly.
   \end{itemize}
\end{frame}

\begin{frame}[fragile]{U-Boot: booting on the active partition (1)}
   \small
   A first solution is to have a variable part in \code{bootargs}
   \begin{itemize}
   \item You need to compile U-Boot with
         \projconfig{u-boot}{CONFIG_BOOTARGS_SUBST}
   \item Then you can use variables in \code{bootargs}:\\
	 \code{setenv bootargs 'console=ttyS0 root=/dev/mmcblk0p${bootpart}'}
   \item And of course set the boot command to load the kernel and DTB
	 from the right partition:
\begin{verbatim}
=> setenv bootcmd 'part list mmc 0 -bootable bootpart;
   load mmc 0:${bootpart} 21000000 boot/zImage;
   load mmc 0:${bootpart} 21000000 boot/dtb;
   bootz 21000000 - 22000000'
\end{verbatim}
   \end{itemize}
   \begin{center}
      \includegraphics[height=0.25\textheight]{graphics/uboot-substitute-string-bootargs.png}
   \end{center}
\end{frame}

\begin{frame}{U-Boot: booting on the active partition (2)}
   Another solution is to use U-Boot's support for {\em extlinux}
   \begin{columns}
   \column{0.6\textwidth}
   \begin{itemize}
   \item You need to compile U-Boot with
         \projconfig{u-boot}{CONFIG_CMD_SYSBOOT}
   \item This way, U-Boot can load the kernel, DTB, initramfs
	 and kernel command line from a specification in an
         \code{extlinux.conf} file.
   \item This works for booting on local storage (USB, MMC, etc)
         but also on the network (PXE and DHCP).
   \item This also allows for multiple boot options, such as
	 a recovery filesystem non-default option.
   \end{itemize}
   See \url{https://u-boot.readthedocs.io/en/latest/develop/distro.html} for details.
   \column{0.4\textwidth}
   \begin{center}
      \includegraphics[width=\textwidth]{graphics/uboot-extlinux-menu.png}
   \end{center}
   \end{columns}
\end{frame}

\begin{frame}[fragile]{U-Boot: using extlinux support (1)}
   \begin{itemize}
   \item You need a configuration file, for example
         \code{boot/extlinux/extlinux.conf}:
\begin{verbatim}
label buildroot-sysupdate
kernel /boot/zImage
devicetree /boot/at91-sama5d3_xplained.dtb
append root=/dev/mmcblk0p${bootpart} rootwait console=ttyS0
\end{verbatim}
   \item As environment variables can be used, you don't need to tweak
	 this file for each possible root partition.
   \end{itemize}
\end{frame}

\begin{frame}[fragile]{U-Boot: using extlinux support (2)}
   \small
   \begin{itemize}
   \item Several environment variables need to be set:
         \begin{itemize}
         \item \code{kernel_addr_r}: address in RAM to load the kernel image
         \item \code{ramdisk_addr_r}: address in RAM to load the initramfs image (if any)
         \item \code{fdt_addr_r}: address in RAM to load the DTB (Flattened
	       Device Tree)
         \item \code{pxefile_addr_r}: address in RAM to load the
	       configuration file (usually \code{extlinux.conf})
	 \item \code{bootfile}: the path to the configuration file,
	       for example \code{/boot/extlinux/extlinux.conf}
	 \end{itemize}
   \item You can now load the files and boot Linux, for example:
\begin{verbatim}
=> setenv bootcmd 'part list mmc 0 -bootable bootpart;
   sysboot mmc 0:${bootpart} any'
\end{verbatim}
         \code{any}: works with any filesystem supported by U-Boot.
   \item Note: according to
         \href{https://u-boot.readthedocs.io/en/latest/develop/distro.html}{U-Boot documentation},
	 U-Boot should automatically read \code{extlinux.conf} from the
	 partition with the \code{bootable} flag. It didn't seem to work
	 on Microchip SAMA5D3 Xplained. PEBKAC issue?
   \end{itemize}
\end{frame}

\subsection{Recovering from a failed upgrade}

\begin{frame}{Recovering from a failed upgrade - Challenges}
  Assuming an A/B partitioning scheme
  \begin{itemize}
  \item The case when the update fails to apply is easy to
        manage. This is easy to detect and you stick to the
        current root filesystem.
  \item In case of success, you also need to confirm that
        the new root filesystem also works correctly, and
        otherwise fall back to the older root partition.
  \end{itemize}

  Assuming a single root filesystem and a rescue filesystem
  \begin{itemize}
     \item If the update fails, the new system may not boot at all.
     \item You then also need a mechanism in the bootloader
           to fall back to the rescue filesystem, and wait for
           a new update.
  \end{itemize}
\end{frame}

\begin{frame}{U-Boot bootcount and bootlimit (1)}
  U-Boot implements a mechanism to detect fail boot attempts
  \begin{itemize}
    \item Enable this through \projconfig{u-boot}{CONFIG_BOOTCOUNT_LIMIT} 
    \item After an update is applied, set the U-Boot environment
          variables \code{upgrade_available} to \code{1}, and
          \code{bootcount} to \code{0}.
    \item When U-Boot starts, if \code{upgrade_available} is different
	  from \code{0}, \code{bootcount} is incremented and saved.
    \item If \code{bootlimit} is defined and \code{bootcount >
          bootlimit}, U-Boot will run the commands in \code{altbootcmd}
	  instead of \code{bootcmd}.
  \end{itemize}
\end{frame}

\begin{frame}{U-Boot bootcount and bootlimit (2)}
  \begin{itemize}
    \item When Linux has booted successfully, a userspace application
          can take care of setting \code{upgrade_available} and
          \code{bootcount} back to \code{0}.
    \item Otherwise, the system will be rebooted either manually
          (pressing reset) or automatically (if a watchdog is used), and \code{bootcount}
          will increase until \code{bootlimit} is reached.
    \item Note: U-Boot's configuration offers other possibilities than storing the
          boot count in the environment: in an I2C device (like an RTC),
          in an EEPROM, in SPI flash, in a file on an EXT filesystem, in
	  RAM (area persistent after resets)...
  \end{itemize}
  See \projfile{u-boot}{doc/README.bootcount} in U-Boot documentation for details.
\end{frame}

\begin{frame}{Accessing the U-Boot environment from Linux}
  To set the \code{upgrade_available} and \code{bootcount} U-Boot
  variables from Linux, to start the boot counting mechanism, or
  to disable it when the latest update booted to completion, you can use the
  the \code{fw_printenv} and \code{fw_setenv} commands.
  \begin{itemize}
     \item Such tools can be built from U-Boot's source tree:\\
	\code{make CROSS_COMPILE=arm-linux- envtools}
     \item Yocto Project: \code{libubootenv} recipe
     \item Buildroot: \code{BR2_PACKAGE_UBOOT_TOOLS_FWPRINTENV} configuration
     \item See \projfile{u-boot}{tools/env/README} and
           \url{https://elinux.org/U-boot_environment_variables_in_linux}
           for details.
  \end{itemize}
\end{frame}

\begin{frame}{Using a watchdog}
  Here's the typical use of a watchdog to handle failed updates
  \begin{itemize}
     \item You configure U-Boot to enable a watchdog on your platform
     \item When Linux is booted, a periodic userspace process should be
	   running to keep "feeding" the watchdog by writing to
           \code{/dev/watchdog}, otherwise the hardware watchdog will
           reboot the machine, and \code{bootcount} will be increased.
     \item See \kdochtml{watchdog/watchdog-api} in kernel documentation
           for details.
  \end{itemize}
\end{frame}

\begin{frame}[fragile]{Editing a partition's bootable flag from U-Boot}
  After an unsuccessful boot, here are U-Boot commands to modify partition
  tables on block storage:
  \begin{itemize}
     \item \code{mbr} command for MBR partitions (\projconfig{u-boot}{CONFIG_CMD_MBR}):
           \begin{itemize}
	   \item Example usage:
\begin{verbatim}
=> setenv mbr_parts 'name=boot,start=4M,size=128M,id=0x0e;
    name=rootfs1,size=256M,bootable,id=0x83;
    name=rootfs2,size=256M,id=0x83;
    name=data,size=-,id=0x83'
=> mbr write mmc 0
\end{verbatim}
           \item \code{mbr verify} command to check that partitions
	         are in sync with \code{mbr_parts}.
           \item See \url{https://u-boot.readthedocs.io/en/latest/usage/cmd/mbr.html} for details.
           \end{itemize}
     \item \code{gpt} command for GPT partitions (\projconfig{u-boot}{CONFIG_CMD_GPT}):
           \begin{itemize}
           \item Similar \code{gpt write} and \code{gpt verify} commands
	   \item Details on \projfile{u-boot}{doc/README.gpt}
	   \end{itemize}
  \end{itemize}
\end{frame}


\begin{frame}{Why partition flags?}
  Why didn't we store the number of the partition to use in a U-Boot
  environment variable?
  \begin{itemize}
     \item Because that's what the {\em distro boot} functionality
	   of U-Boot is supposed to rely on.
     \item Because saving the environment is far from being atomic.
	   Multiple storage blocks need to be modified. Many things
	   could go wrong.
     \item You could look for other non volatile storage on your SoC
	   or on your board to store such information.
  \end{itemize}
\end{frame}

\begin{frame}{Limitations and advice}
  \begin{itemize}
  \item We have only addressed systems booted from block devices, the
	most common ones today. Systems booting from NAND or NOR flash
	could store the partition to use on a specific non volatile
        storage.
  \item Not sure whether the provided U-Boot or even the Linux commands
	provide atomicity guarantees.
  \item At least a good idea is to use U-Boot's {\em redundant
        environment} features. If one of the two copies of the
        environment is corrupted, U-Boot will use the other one.
  \end{itemize}
\end{frame}

\begin{frame}{Strategies for updating the bootloader?}
  \begin{itemize}
     \item Need to configure the SPL to load U-Boot from a filesystem.
     \item However, it's probably difficult to implement partition
           selection and boot counting in the SPL (space limited).
     \item If you're booting from eMMC, you may flash the SPL and
           U-Boot on the special \code{boot0} and \code{boot1} hardware
	   partitions. See details about the \code{mmc partconf}
           command in \href{https://u-boot.readthedocs.io/en/latest/usage/cmd/mmc.html}{U-Boot's documentation}.
	   To be able to update the SPL too, this requires your
	   SoC to support this mechanism though. Otherwise, it's
           advisable not to update the SPL on the field.
  \end{itemize}
\end{frame}

\begin{frame}{What to remember}
  \begin{itemize}
     \item U-Boot offers a \code{part list} command to find the
	   block partition with the \code{bootable} flag.
     \item U-Boot offers \code{mbr} and \code{gpt} commands
	   to edit partitions on block devices.
     \item U-Boot can keep track of boot failures, can count them
	   (\code{bootcount} environment variable), and run an
	   alternative boot sequence (\code{altbootcmd}) when a
	   maximum number is exceeded (\code{bootlimit}).
  \end{itemize}
\end{frame}

\begin{frame}{Special thanks}
  \begin{itemize}
  \item Vyacheslav Yurkov\\
	For sharing knowledge, asking questions and reviewing these slides
  \item Thomas Petazzoni\\
	For his insightful blog post about using {\em swupdate} on stm32mp1:
	\url{https://bootlin.com/blog/building-a-linux-system-for-the-stm32mp1-remote-firmware-updates/}
  \item Köry Maincent\\
	For sharing his experience
  \end{itemize}
\end{frame}

\begin{frame}{References}
  \begin{itemize}
  \item This ELC 2022 presentation from Leon Anavi:\\
        \href{https://elinux.org/images/0/07/Leon-anavi-software-updates.pdf}{How to Choose a Software Update Mechanism for Embedded Linux Devices}
	(\href{https://youtu.be/j4o5TzMdOL4}{Video})
  \item Bootlin blog posts
    \begin{itemize}
    \item swupdate: \href{https://bootlin.com/blog/building-a-linux-system-for-the-stm32mp1-remote-firmware-updates/}{Building a Linux system for the STM32MP1: remote firmware updates}
    \item RAUC: \href{https://bootlin.com/blog/another-system-update-adventure-with-rauc-barebox-yocto-project/}{Another system update adventure with RAUC, Barebox \& Yocto Project}
    \item Mender: \href{https://bootlin.com/blog/mender-how-to-integrate-an-ota-updater/}{Mender: How to integrate an OTA updater}
    \end{itemize}
  \item Collection of {\em Embedded Linux Conference} talks on
    firmware update: \url{https://elinux.org/Upgrades_Presentations}
  \end{itemize}
\end{frame}

\questionslide

\end{document}
