\documentclass[aspectratio=169,obeyspaces,spaces,hyphens,dvipsnames]{beamer}
\usepackage[utf8]{inputenc}
\usepackage{lmodern}% http://ctan.org/pkg/lm
\usepackage{minted}
\usepackage{hyperref}
\usepackage{xcolor}
\usepackage{pgfplots}
\usepackage{tikz}
\usepackage[normalem]{ulem}

\mode<presentation>
\usetheme{Bootlin}

\def\signed #1{{\leavevmode\unskip\nobreak\hfil\penalty50\hskip2em
  \hbox{}\nobreak\hfil(#1)
  \parfillskip=0pt \finalhyphendemerits=0 \endgraf}}

\newsavebox\mybox
\newenvironment{aquote}[1]
  {\savebox\mybox{#1}\begin{quotation}}
  {\signed{\usebox\mybox}\end{quotation}}

\title{Timing Boot Time Reduction Techniques}
\authors{Michael Opdenacker}
\email{michael.opdenacker@bootlin.com}
\slidesurl{https://bootlin.com/pub/conferences/2019/elce/}
\institute{Bootlin}
\conference{Embedded Linux Conference Europe 2019}

\begin{document}

\addtocontents{toc}{\protect\setcounter{tocdepth}{-1}}
\section{Timing Boot Time Reduction Techniques}
\addtocontents{toc}{\protect\setcounter{tocdepth}{2}}

\begin{frame}{Michael Opdenacker}
  \begin{itemize}
  \item Founder and Embedded Linux engineer at Bootlin
    \begin{itemize}
    \item Embedded Linux {\bf expertise}
    \item {\bf Development}, consulting and training
    \item Strong open-source focus
    \end{itemize}
  \item Long time interest in embedded Linux boot time,
        and one of its prerequisites: small system size.
  \item Living in {\bf Orange}, France.
  \end{itemize}
\end{frame}

\subsection{Introduction}

\begin{frame}
  \frametitle{The system to optimize - Hardware}
  \begin{columns}
  \column{0.5\textwidth}
    \begin{itemize}
        \item Beagle Bone Black board (ARM Cortex A8 from TI)
        \item Beagle Bone LCD cape\\
              \url{https://elinux.org/Beagleboard:BeagleBone_LCD4}\\
        \item Standard USB webcam (supported through the \code{uvcvideo}
              driver.
	\item Booting on standard SDHC uSD card (cat 4) from Kingston
    \end{itemize}
  \column{0.5\textwidth}
     \includegraphics[width=\textwidth]{common/beaglecam.jpg}
  \end{columns}
\end{frame}

\begin{frame}
  \frametitle{The system to optimize - Software}
  \begin{columns}
  \column{0.5\textwidth}
    \begin{itemize}
	\item Root filesystem built by Buildroot, just
	      starting the \code{ffmpeg} program to show the video
	      captured by the webcam.
	\item Initial boot time (boot to first decoded frame): 9.45s
    \end{itemize}
  \column{0.5\textwidth}
     \includegraphics[width=\textwidth]{common/beaglecam.jpg}
  \end{columns}
\end{frame}

\begin{frame}
  \frametitle{Main goals for this presentation}
  \begin{itemize}
  \item Not to cover all technical details of optimization techniques
        (will share useful resources at the end)
  \item But goal to benchmark the most useful techniques on a recent
	kernel.
  \item This should give you an idea about whether each
        technique is worth trying in your system or not.
  \item Last but not least, information harder to find about
        U-Boot's {\em Falcon Mode}. Apparently, there are still
	few people taking advantage of it, otherwise there would
	be more people talking about it.
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Main optimization principles}
  \begin{itemize}
  \item Start by optimizing things that won't reduce your ability for
        further measurement and optimizations
  \item This way, good to keep slower storage, and less efficient
        compression. This amplifies the phenomena to observe.
  \item Start from the last parts of boot time, and finish by the kernel
        and at the end, by the bootloader.
  \end{itemize}
  {\em Premature optimization is the root of all evil.\\
  Donald Knuth}
\end{frame}
\subsection{Toolchain optimizations}

\begin{frame}
\frametitle{ARM vs Thumb2}
Compared the system compiled with ARM and with Thumb2
\begin{itemize}
\item Compiled with gcc 7.4, generating {\em ARM} code:\\
      Total filesystem size: 3.79 MB\\
      \code{ffmpeg} size: 227 KB
\item Compiled with gcc 7.4, generating {\em Thumb2} code:\\
      Total filesystem size: 3.10 MB (-18 \%)\\
      \code{ffmpeg} size: 183 KB (-19 \%)
\item Performance aspect: performance apparently slightly improved with Thumb2
      (approximately less than 5 \%, but there are slight variations in measured
      execution time, for one run to another).
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{musl vs uClibc}
Tried to replace {\em uClibc} by {\em musl}
\begin{itemize}
   \item musl library size: 680 KB (size of the \code{tar} archive for \code{lib/})
   \item uClibc library size: 570 KB (-16 \%)
   \item uClibc saves 110 KB (useful), but otherwise no other significant change
    in filesystem and code size. Not a surprise when the system is mostly filled
    with binaries relying on shared libraries.
\end{itemize}
We stuck to {\em uClibc}!
\end{frame}

\subsection{Application optimizations}

\begin{frame}
\frametitle{General ideas}
\begin{itemize}
  \item Reduce the size of applications by recompiling them
        with only the features needed in your system.
	Run \code{./configure --help} for applications
	packaged by the autotools.
  \item Profile your application with \code{strace} or
	more advanced tools (\code{perf}) to optimize its
	behavior, especially if you authored the code.
  \item Reducing total size: inspect the root filesystem
	to find files that don't look necessary.
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{Results on ffmpeg}
After rebuilding the system with the minimum \code{configure} options
for \code{ffmpeg}:
\begin{itemize}
   \item Total system size:\\
	 From 16.11 MB to 3.54 MB (-78 \%)
   \item Saves 150 ms in application loading + execution time
   \item Saves 120 ms in application execution time (file cache populated)
   \item Total boot time reduction: approximately 350 ms (faster mount time?)
\end{itemize}
Note: the size gains are massive. The performance gains are smaller,
because Linux only loads the code actually used by the program.
Some unnecessary code is eliminated though.
\end{frame}

\subsection{Init and root filesystem optimizations}

\begin{frame}
\frametitle{Generic ideas}
\begin{itemize}
  \item Analyze system startup with \code{bootchartd} and eliminate
        unnecessary services
  \item Group required startup scripts into a single one
        (such as \code{/etc/init.d/rcS}), critical things first.
  \item Do not mount \code{/proc} and \code{/sys} if you don't need
        them (test your application), or mount them from C code.
  \item Simplify the BusyBox configuration to the minimum
  \item You could even start your application as the {\em init}
        process (\code{init} parameter in the kernel command line)
  \item Switching to static executables
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{Usefulness of root filesystem size reduction}
\begin{itemize}
  \item A smaller filesystem may be faster to mount
  \item Really useful when you boot your root filesystem as
        an {\em initramfs}:
        \begin{itemize}
	   \item Root filesystem archive embedded in the kernel binary
	   \item A smaller initramfs will make the kernel smaller to
		 load from storage
	   \item Kernel decompression time will be faster too
	   \item The kernel and filesystem are loaded in a single
		 read operation from storage, instead of multiple
		 ones (more costly, more overhead)
	\end{itemize}
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{Detect unnecessary files}
Taking advantage of the fact that Linux filesystems record the last
access time.
\begin{itemize}
   \item Boot your system with the root filesystem mounted in read-write mode
   \item If your system has a correct time, find all files last accessed
         more than 5 minutes ago:\\
         \code{find / -atime +5 -type f}
   \item If your system doesn't have a correct time (and uses dates in
         1970), back on your PC (if your storage is removable), you
         can find files last accessed less than 1000 minutes ago, when
	 extracting the root filesystem:\\
         \code{find / -atime -1000 -type f}
   \item Remove such unwanted files with your build system (for example using
         \code{BR2_ROOTFS_POST_FAKEROOT_SCRIPT} in Buildroot)
   \item Also remove unnecessary directories with the same script.
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{Results: init and root filesystem optimizations}
\begin{itemize}
   \item Replaced all init scripts by a single script running the
         application
   \item Simplified BusyBox' configuration, only kept support
	 for \code{echo}, \code{sh}, \code{sleep} and \code{test},
	 used in the single script. Size reduced from 682 KB to 86 KB.
   \item Eliminated unused files and directories
   \item Total filesystem size reduced from 3.54 MB to 2.33 MB (-34 \%)
   \item Boot time difference: hardly noticeable, probably because
	 init scripts didn't take much time.
\end{itemize}
\end{frame}

\begin{frame}[fragile]
\frametitle{Results: switching to static executables}
\begin{columns}
\column{0.5\textwidth}
\begin{itemize}
   \item Making sense here as we only have two executables
   \item Removes all library code not used by such executables
   \item This allowed to remove more empty directories
   \item Total filesystem size reduced from 2.33 MB to 1.58 MB (-22 \%)
   \item Boot time difference: about 20 ms, probably due to the
         time saved resolving and loading shared libraries.
\end{itemize}
\column{0.5\textwidth}
\footnotesize
\begin{verbatim}
.
├── [  44]  bin
│   ├── [129K]  busybox
│   ├── [   7]  echo -> busybox
│   ├── [   7]  hush -> busybox
│   ├── [   7]  sh -> busybox
│   └── [   7]  sleep -> busybox
├── [   0]  dev
├── [ 305]  playvideo
└── [   6]  usr
    └── [  22]  bin
        ├── [  17]  [ -> ../../bin/busybox
        ├── [1.4M]  ffmpeg
        └── [  17]  test -> ../../bin/busybox

5 directories, 9 files
\end{verbatim}
\end{columns}
\end{frame}

\subsection{Filesystem optimizations}

\begin{frame}
\frametitle{Switching to an initramfs}
Multiple advantages:
\begin{itemize}
\item Possible after strong root filesystem size reduction
\item Root filesystem that can embedded in the kernel image. Only one
      access to storage required (should be faster)
\item No more need for block/storage and filesystem drivers.
      Smaller kernel, shorter load time and less
      initialization work.
\item Only constraint: mount \code{devtmpfs} from userspace
      (no longer automatic)
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{Do not compress your initramfs}
\begin{itemize}
\item If you ship your initramfs inside a compressed kernel image, don't compress
      it \\
      (enable \code{CONFIG_INITRAMFS_COMPRESSION_NONE}).
\item Otherwise, by default, your initramfs data will be compressed twice, and
      the kernel will be bigger and will take a little more time to load
      and uncompress.
\item Example on Linux 5.1 with a 1.60 MB initramfs (tar archive size)
      on Beagle Bone Black: this allowed to reduce the kernel size from 4.94
      MB to 4.74 MB (-200 KB) and save about 170 ms of boot time.
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{Switching to initramfs - Results}
\begin{itemize}
\item Initial impact on total boot time: about +300 ms, because of the
      bigger kernel and increased uncompression time.
\item After disabling \code{CONFIG_BLOCK} and \code{CONFIG_MMC},
      the compressed kernel size is reduced by 610 KB.
\item Total boot time is even slighly lower than the initial value
      (-20 ms)
\end{itemize}
\end{frame}

\subsection{Kernel optimizations}

\begin{frame}[fragile]
\frametitle{Measure - Kernel initialization functions}
To find out which kernel initialization functions are the longest to
execute, add \code{initcall_debug} to the kernel command line.
Here's what you get on the kernel log:
\begin{block}{}
\tiny
\begin{verbatim}
...
[    3.750000] calling  ov2640_i2c_driver_init+0x0/0x10 @ 1
[    3.760000] initcall ov2640_i2c_driver_init+0x0/0x10 returned 0 after 544 usecs
[    3.760000] calling  at91sam9x5_video_init+0x0/0x14 @ 1
[    3.760000] at91sam9x5-video f0030340.lcdheo1: video device registered @ 0xe0d3e340, irq = 24
[    3.770000] initcall at91sam9x5_video_init+0x0/0x14 returned 0 after 10388 usecs
[    3.770000] calling  gspca_init+0x0/0x18 @ 1
[    3.770000] gspca_main: v2.14.0 registered
[    3.770000] initcall gspca_init+0x0/0x18 returned 0 after 3966 usecs
...
\end{verbatim}
\end{block}
You might need to increase the log buffer size with
\code{CONFIG_LOG_BUF_SHIFT} in your kernel configuration. You will
also need \code{CONFIG_PRINTK_TIME} and \code{CONFIG_KALLSYMS}.
\end{frame}

\begin{frame}
\frametitle{Kernel boot graph}
With \code{initcall_debug}, you can generate a boot graph
making it easy to see which kernel initialization functions
take most time to execute.
\begin{itemize}
\item Copy and paste the output of
      the \code{dmesg} command to a file (let's call it \code{boot.log})
\item On your workstation, run the \code{scripts/bootgraph.pl} script
      in the kernel sources: \\
      \code{scripts/bootgraph.pl boot.log > boot.svg}
\item You can now open the boot graph with a vector graphics
      editor such as \code{inkscape}:
\end{itemize}
\begin{center}
    \includegraphics[width=\textwidth]{common/boot-full.pdf}
\end{center}
\end{frame}

\begin{frame}
\frametitle{Using the kernel boot graph (1)}
Start working on the functions consuming most time first. For each
function:
\begin{itemize}
\item Look for its definition in the kernel source code. You can use
      Elixir (for example \url{https://elixir.bootlin.com}).
\item For unnecessary functionality, find which kernel configuration parameter
      compiles the code, by looking at the Makefile in the corresponding
      source directory.
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{Using the kernel boot graph (2)}
\begin{itemize}
\item Postpone:
      \begin{itemize}
      \item Find which module (if any) the function belongs to.
            Load this module later if possible.
      \item Be careful: some function names don't exist, the names
	    correspond to {\em modulename}\code{_init}. Then, look for
	    initialization code in the corresponding module.
      \end{itemize}
\item Optimize necessary functionality:
      \begin{itemize}
      \item Look for parameters which could be used to reduce probe time,
            looking for the \code{module_param} macro.
      \item Look for delay loops and calls to functions containing
            \code{delay} in their name, which could take more time than
            needed. You could reduce such delays, and see whether the
            code still works or not.
      \end{itemize}
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{Removing tracing infrastructure}
\begin{itemize}
\item Corresponding to the \kfunc{tracer_init_tracefs} function call
\item Unselected \code{Tracers} in \code{Kernel hacking} (enabled
      in our default configuration)
\item Boot time savings: approximately 550 ms
\item Kernel size savings: 217 KB
\end{itemize}
\includegraphics[width=\textwidth]{common/boot-full.pdf}
\includegraphics[width=0.796\textwidth]{common/boot-notracers.pdf}
\end{frame}

\begin{frame}
\frametitle{Other features to remove}
\begin{itemize}
\item \code{omap8250_platform_driver_init (660 ms!)}\\
      Corresponds to the serial interface. No obvious reason found
      in the code. To be disabled later.
\item \code{cpsw_driver_init} (112 ms)\\
      Corresponds to the network driver. Will be disabled.
\item \code{am335x_child_init} (82 ms)\\
      Corresponds to the USB interface the camera is connected to.
      Cannot be skipped.
\item All other items are too small. Some will be disabled by removing
      as many features as possible.
\end{itemize}
\end{frame}

\begin{frame}[fragile]
\frametitle{Preset loops per jiffy}
\begin{itemize}
        \item At each boot, the Linux kernel calibrates a delay loop (for
              the \kfunc{udelay} function). This measures a number of loops per
              jiffy ({\em lpj}) value. You just need to measure this once! Find
              the \code{lpj} value in the kernel boot messages:
\begin{block}{}
\small
\begin{verbatim}
Calibrating delay loop... 996.14 BogoMIPS (lpj=4980736)
\end{verbatim}
\end{block}
        \item Now, you can add \code{lpj=<value>} to the kernel command
              line:
\begin{block}{}
\small
\begin{verbatim}
Calibrating delay loop (skipped) preset value.. 996.14 BogoMIPS (lpj=4980736)
\end{verbatim}
\end{block}
        \item Tests on BeagleBone Bloack (ARM), Linux 5.1: -82 ms\\
	Time measured at the first kernel messages... the calibration
        loop is run before the message is issued.
\end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Multiprocessing support (CONFIG\_SMP)}
  \begin{itemize}
          \item SMP is quite slow to initialize
          \item It is usually enabled in default configurations, even if
                you have a single core CPU (default configurations
                should support multiple systems).
          \item So make sure you disable it if you only have one CPU
                core.
	  \item Results on BeagleBone Black:\\
		Compressed kernel size: -188 KB (-4.6 \%)\\
		Total boot time: -126ms
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Removing kernel module support}
  If possible!
  \begin{itemize}
	\item Before turning off \code{CONFIG_MODULES}, first manually
        turn off all features selected as modules. Otherwise, modules will be
        turned into build-ins while they were not necessary.
	\item Note that \code{menuconfig} and \code{xconfig} show you
	which modules can be removed or not because of dependencies.
	\item Before this, it is easier to remove all subsystems that
	you don't need for sure.
	\item Unselect features progressively and make copies for your
	configuration files. If the system no longer boots, you may
	have to start again from the beginning otherwise.
	\item Results: -82 KB in compressed kernel size, -20 ms of boot
	time.
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Silencing the console, turning off kernel messages}
  \begin{itemize}
  \item First, booting with the \code{quiet} command line parameter:\\
        Total boot time: -577 ms
  \item After removing \code{CONFIG_PRINTK} and \code{CONFIG_BUG}: \\
	Compressed kernel size: 1939152 (-118 KB, -5.8 \%)
  \item After removing \code{CONFIG_KALLSYMS} too:\\
	Compressed kernel size: 1829168 (-107 KB, -5.7 \%)
  \item Total savings:\\
	Compressed kernel size: -225 KB (-11 \%)\\
        Total boot time: -767 ms
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Using CONFIG\_EMBEDDED and CONFIG\_EXPERT}
  \begin{itemize}
  \item Useful for keeping only the system calls that you're using in
	your system. This makes your kernel less generic, but that's
	fine when all applications are known in advance
  \item Compressed kernel size: -51 KB
  \item Total boot time: -34 ms
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Compiling the kernel in Thumb2 mode}
  Can be selected by \code{CONFIG_THUMB2_KERNEL}\\
  Results on BeagleBone Black:
  \begin{itemize}
  \item Compressed kernel size: +40 KB
  \item Total boot time: +5 ms
  \end{itemize}
  We keep compiling the kernel in ARM mode, unlike what we did for the
  root filesystem.
\end{frame}

\begin{frame}[fragile]
  \frametitle{Choosing SLAB memory allocators}
  \begin{itemize}
  \small
  \item SLAB: legacy, well proven allocator. Default choice on our board.
  \item SLOB: much simpler. More space efficient but doesn't scale
        well. Saves a few hundreds of KB in small systems (depends on
        \code{CONFIG_EXPERT}). \\
	Results on BeagleBone Black: -5 KB compressed kernel, +1,43 s total
	boot time!
  \item SLUB: more recent and simpler than
        SLAB, scaling much better (in particular for huge systems) and
        creating less fragmentation.\\
	Results on BeagleBone Black: +4 KB compressed kernel, + 2ms total
	boot time.
  \end{itemize}
  We're keeping SLAB!
  \begin{center}
    \includegraphics[height=0.2\textheight]{common/slab-screenshot.png}
  \end{center}
\end{frame}


\begin{frame}
\frametitle{Kernel Compression}
Depending on the balance between your storage reading speed and your
CPU power to decompress the kernel, you will need to benchmark
different compression algorithms.

Also recommended to experiment with compression options at the
end of the kernel optimization process, as the results may vary
according to the kernel size.
\begin{center}
    \includegraphics[width=\textwidth]{common/kernel-compression-options.pdf}
\end{center}
\end{frame}

\begin{frame}
\frametitle{Kernel compression options}
Results on TI AM335x (ARM), 1 GHz, Linux 5.1
{\fontsize{7}{10}\selectfont
\begin{tabular}{| l || c | c | c | c | c |}
\hline
Timestamp & gzip & lzma & xz & lzo & lz4 \\
\hline
Size & 2350336 & 1777000 & {\bf 1720120} & 2533872 & 2716752 \\
Copy & 0.208 s & 0.158 s & {\bf 0.154 s} & 0.224 s & 0.241 s \\
Time to userspace & 1.451 s & 2.167 s & 1.999s & {\bf 1.416 s} & 1.462 s \\
\hline
\end{tabular}
}
\vfill{}
Gzip is close. It's time to try with faster storage (SanDisk Extreme
Class A1)
{\fontsize{7}{10}\selectfont
\begin{tabular}{| l || c | c | c | c | c |}
\hline
Timestamp & gzip & lzma & xz & lzo & lz4 \\
\hline
Size & 2350336 & 1777000 & {\bf 1720120} & 2533872 & 2716752 \\
Copy & 0.150 s & 0.114 s & {\bf 0.111 s} & 0.161 s & 0.173 s \\
Time to userspace & 1.403 s & 2.132 s & 1.965 s & {\bf 1.363 s} & 1.404 s \\
\hline
\end{tabular}
}
\newline\newline
Lzo and Gzip seem the best solutions. Always benchmark as the results
depend on storage and CPU performance.
\end{frame}

\begin{frame}
\frametitle{Optimize kernel for size (1)}
\begin{itemize}
\item \code{CONFIG_CC_OPTIMIZE_FOR_SIZE}: possibility to compile the kernel
      with \code{gcc -Os} instead of \code{gcc -O2}.
\item Such optimizations give priority to code size at
      the expense of code speed.
\item Results: the initial boot time is better (smaller
      size), but the slower kernel code can offset
      the benefits. Your system will run a bit slower!
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{Optimize kernel for size (2)}
Results on BeagleBone Black, Linux 5.1, lzo compression
\begin{tabular}{| l || c | c | c |}
\hline
& O2 & Os & Diff \\
\hline
Size & 2533872 & 2390608 & -5.7 \% \\
Copy time &  0.161 s & 0.153 s & -8 ms \\
Starting kernel & 0.912 s & 0.904 s & -8 ms \\
Starting userspace & 1.363 s & 1.359 s & -4 ms \\
Total boot time & 2.961 s & 2.957s & -4 ms \\
\hline
\end{tabular}
\newline\newline
Results on Microchip SAMA5D3 Xplained, Linux 3.10, gzip compression:
\begin{tabular}{| l || c | c | c |}
\hline
Timestamp & O2 & Os & Diff \\
\hline
Starting kernel & 4.307 s & 4.213 s & -94 ms \\
Starting init & 5.593 s & 5.549 s & -44 ms \\
Login prompt & 21.085 s & 22.900 s & + 1.815 s \\
\hline
\end{tabular}
\end{frame}

\begin{frame}
\frametitle{Removing pseudo filesystems}
\begin{itemize}
\item Remove the {\em proc} filesystem:\\
      Lzo compressed kernel size: -48 KB \\
      But ffmpeg doesn't work without {\em proc}
\item Removing only proc options (\code{CONFIG_PROC_SYSCTL} and
      \code{CONFIG_PROC_PAGE_MONITOR}): -9 KB, no noticeable boot time change.
\item Removing \code{CONFIG_CONFIGFS_FS}: -7 KB
\item Removing the {\em sysfs} filesystem:\\
      Lzo compressed kernel size: -22 KB \\
      Time to userspace: -35 ms
\end{itemize}
Do it if compatible with your applications!
\end{frame}

\begin{frame}
\frametitle{Other kernel optimizations}
\begin{itemize}
\item Disable all {\em Compile-time checks and compiler options} in
      \code{Kernel hackingr}), in particular \code{CONFIG_DEBUG_INFO}:\\
      Compressed kernel size: -38 KB (-1.7 \%) \\
      Time to userspace: -6 ms
\item Replacing {\em ARM EABI stack unwinder} (\code{CONFIG_UNWINDER_ARM})
      by the default mechanism:\\
      Compressed kernel size: -24 KB (-1.1 \%)\\
      Time to user space: +0.8 ms, keeping it this option for its size
      savings.
\end{itemize}
\end{frame}

\begin{frame}[fragile]
\frametitle{Appending DTB to kernel}
\begin{itemize}
\item Goal: group two copies from SD card:
\begin{block}{}
\begin{verbatim}
0.862184 0.144682] 2205936 bytes read in 141 ms (14.9 MiB/s)
[0.873126 0.010942] 61284 bytes read in 7 ms (8.3 MiB/s)
\end{verbatim}
\end{block}
\item Using \code{CONFIG_ARM_APPENDED_DTB}, already enabled:\\
{\small
\begin{verbatim}
cat arch/arm/boot/zImage arch/arm/boot/dts/am335x-boneblack-lcd4.dtb > zImage
setenv bootcmd 'fatload mmc 0:1 81000000 zImage; bootz 81000000'
\end{verbatim}
}
\item Result: all bytes loaded at top speed
\begin{block}{}
\begin{verbatim}
[0.863885 0.149569] 2266952 bytes read in 145 ms (14.9 MiB/s)
\end{verbatim}
\end{block}
Starting kernel: 26 ms earlier\\
Userspace: 26 ms earlier
\end{itemize}
\end{frame}

\subsection{Bootloader optimizations}

\begin{frame}
\frametitle{Bootloader optimizations: strategy}
\begin{itemize}
\item It is definitely possible to improve performance in U-Boot.
      See our training slides: \url{https://bootlin.com/doc/training/boot-time/}
\item However, the best solution is to {\bf skip U-Boot}, using its {\em Falcon mode}:
      \begin{itemize}
	  \item We'll only execute the first stage of U-Boot, the SPL (Secondary Program Loader)
	  \item And will directly load the Linux kernel, instead of the U-Boot image.
          See \code{doc/README.falcon} in U-Boot sources for details.
      \end{itemize}
\item This is supported in the same way on all the boards with U-Boot
support for SPL.
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{U-Boot Falcon mode - Preparation steps}
Quite easy indeed!
\begin{itemize}
\item The Falcon mode needs a legacy kernel image:\\
\code{make uImage LOADADDR=80008000}\\
Copy it to the SD card.
\item In U-Boot’s \code{menuconfig} interface, go to the \code{SPL / TPL}
menu and unselect \code{Support an environment}. Recompile U-Boot and
update it on the SD card. In our tests, this saved 250 ms in Falcon mode!
\end{itemize}
See our training labs for details: \url{https://bootlin.com/doc/training/boot-time/}
\end{frame}

\begin{frame}
\frametitle{U-Boot Falcon mode - without DTB (embedded in uImage)}
\begin{itemize}
\item In the U-Boot command line, load the kernel image in RAM:\\
\code{load mmc 0:1 81000000 uImage}
\item Set the bootargs if needed:\\
\code{setenv bootargs console=ttyO0,115200n8 rdinit=/playvideo
lpj=4980736}
\item Simulate booting the Linux kernel:\\
\code{spl export }{\bf \code{atags}}\code{ 81000000}
\item Using the address output by this command, store the atags information to MMC (\code{args} file):\\
\code{fatwrite mmc 0:1 0x80000100 args 4000}
\item Reset and the system should boot through only the U-Boot SPL
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{U-Boot Falcon mode - with DTB}
\begin{itemize}
\item In the U-Boot command line, load the kernel image and DTB in RAM:\\
\code{load mmc 0:1 81000000 uImage}\\
\code{load mmc 0:1 82000000 dtb}\\
\item Set the bootargs if needed:\\
\code{setenv bootargs console=ttyO0,115200n8 rdinit=/playvideo
lpj=4980736}
\item Simulate booting the Linux kernel:\\
\code{spl export }{\bf \code{ftd}}\code{ 81000000 - 82000000}
\item Using the addresses output by this command, store the Flattened
Device Tree (fdt) information to MMC (\code{args} file):\\
\code{fatwrite mmc 0:1 0x8ffd9000 args 1de57}
\item Reset and the system should boot through only the U-Boot SPL
\end{itemize}
\end{frame}

\begin{frame}[fragile]
\frametitle{U-Boot Falcon mode results}
\begin{columns}
\column{0.6\textwidth}
\begin{block}{}
\tiny
\begin{verbatim}
[0.000000 0.000000]
[0.000785 0.000785] U-Boot SPL 2019.01 (Oct 27 2019 - 08:04:06 +0100)
[0.057822 0.057822] Trying to boot from MMC1
[0.378878 0.321056] fdt_root: FDT_ERR_BADMAGIC
[0.775306 0.396428] Waiting for /dev/video0 to be ready...
[1.966367 1.191061] Starting ffmpeg
...
[2.412284 0.004277] First frame decoded
\end{verbatim}
\end{block}
\column{0.4\textwidth}
\begin{itemize}
\item Time to userspace: -479 ms
\item Time to ffmpeg: -478 ms
\item Total boot time: 2.412284
\end{itemize}
\end{columns}
\end{frame}

\begin{frame}[fragile]
\frametitle{Challenges and issues}
\begin{itemize}
\item Didn't manage to boot yet without the tty layer. At least the
      application doesn't start. According to the boot graph, expecting
      to save hundreds of ms.
\item Waiting for 1.2 s for the USB camera to be enumerated.\\
      Any way around this USB asynchronousness issue?
\item Solving these issues should allow to start displaying a video in
      less than 1 s.
\end{itemize}
\begin{block}{}
\tiny
\begin{verbatim}
[0.000000 0.000000]
[0.000785 0.000785] U-Boot SPL 2019.01 (Oct 27 2019 - 08:04:06 +0100)
[0.057822 0.057822] Trying to boot from MMC1
[0.378878 0.321056] fdt_root: FDT_ERR_BADMAGIC
[0.775306 0.396428] Waiting for /dev/video0 to be ready...
[1.966367 1.191061] Starting ffmpeg
...
[2.412284 0.004277] First frame decoded
\end{verbatim}
\end{block}
\end{frame}

\begin{frame}
\frametitle{Don't miss}
\begin{itemize}
\item {\em Tuesday, October 29 - 11:30 - 12:05}\\
      We Need to Talk About Systemd:\\
      Boot Time Optimization for the New init daemon\\
      {\em Chris Simmonds, 2net}
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{Conference presentations and training materials}
\begin{itemize}
\item Andrew Murray - The Right Approach to Minimal Boot Time (2010) \\
      Video: \url{https://frama.link/nrf696Hy} - Slides: \url{https://frama.link/uCBH9jQM} \\
      Great talk about the methodology.
\item Chris Simmonds - A Pragmatic Guide to Boot-Time Optimization (2017)\\
      Video: \url{https://frama.link/Vnmj5t1m} - Slides: \url{https://frama.link/TC0YKM9N}
\item Jan Altenberg - How to Boot Linux in One Second (2015)\\
      Video: \url{https://frama.link/BztbLy9T} - Slides: \url{https://frama.link/bFkvgLFR}
\item Bootlin's Linux boot time training materials and labs:\\
      See our training slides: \url{https://bootlin.com/doc/training/boot-time/}
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{Most successful techniques in our project}
\includegraphics[width=\textwidth]{common/best-techniques.pdf}
\end{frame}

\questionslide
\end{document}

