\documentclass[aspectratio=169,obeyspaces,spaces,hyphens,dvipsnames]{beamer}
\usepackage[utf8]{inputenc}
\usepackage{lmodern}% http://ctan.org/pkg/lm
\usepackage{minted}
\usepackage{hyperref}
\usepackage{xcolor}
\usepackage{pgfplots}
\usepackage{tikz}
\usepackage[normalem]{ulem}

\mode<presentation>
\usetheme{Bootlin}

\def\signed #1{{\leavevmode\unskip\nobreak\hfil\penalty50\hskip2em
  \hbox{}\nobreak\hfil(#1)
  \parfillskip=0pt \finalhyphendemerits=0 \endgraf}}

\newsavebox\mybox
\newenvironment{aquote}[1]
  {\savebox\mybox{#1}\begin{quotation}}
  {\signed{\usebox\mybox}\end{quotation}}

\title{LLVM tools for the Linux kernel}
\authors{Michael Opdenacker}
\email{michael.opdenacker@bootlin.com}
\slidesurl{https://bootlin.com/pub/conferences/2022/lee/}
\institute{Bootlin}
\conference{Live Embedded Event \#3}

\begin{document}

\addtocontents{toc}{\protect\setcounter{tocdepth}{-1}}
\section{LLVM tools for the Linux kernel}
\addtocontents{toc}{\protect\setcounter{tocdepth}{2}}

\begin{frame}{Michael Opdenacker}
\begin{columns}
  \column{0.7\textwidth}
  \begin{itemize}
  \item Founder and Embedded Linux engineer at Bootlin:
    \begin{itemize}
    \item Embedded Linux {\bf expertise}
    \item {\bf Development}, consulting and training
    \item Strong open-source focus
    \end{itemize}
  \item Free Software contributor:
    \begin{itemize}
      \item Current maintainer of the Elixir Cross Referencer, making it
            easier to study the sources of big C projects like the Linux kernel.
            See \url{https://elixir.bootlin.com}
      \item Current documentation maintainer for the Yocto Project
      \item Co-author of Bootlin's freely available embedded Linux and
	    kernel training materials (\url{https://bootlin.com/docs/})
    \end{itemize}
  \end{itemize}
  \column{0.3\textwidth}
    \begin{center}
    \vspace{0.5cm}
    \includegraphics[width=0.5\textwidth]{graphics/logo-cc.pdf}\\
    \vspace{0.5cm}
    \includegraphics[width=\textwidth]{graphics/elixir.pdf}
    \end{center}
\end{columns}
\end{frame}

\begin{frame}{Disclaimer}
\begin{itemize}
  \item I'm neither a Clang/LLVM expert, nor involved in the project
	to build the Linux kernel with Clang.
  \item I'm just interested in the topic, and sharing my own findings
        with you.
  \item This is also why this is a short presentation!
\end{itemize}
\end{frame}

\subsection{Compiling the Linux kernel with Clang}


\begin{frame}{Definitions and conventions in this document}
\begin{itemize}
  \item Clang: compiler front-end for C, C++, Objective-C, OpenCL, CUDA...
  \item \code{clang}: the command provided by the Clang project
  \item LLVM: compiler back-end, and name of the project Clang is part of.
  \item GCC: GNU Compiler Collection.
  \item Gcc or \code{gcc}: the C compiler in GCC
\end{itemize}
\end{frame}

\begin{frame}{Why compiling the Linux kernel with Clang?}
\begin{itemize}
  \item Google reason: have only one toolchain and build all of
        Android with Clang.
  \item Good to support two different compilers, to shake out code
        that relies on undefined behavior in the compiler.
  \item Get different warnings from Clang.
  \item Access further optimizations such as {\em Link Time Optimization
        (LTO)}.
  \item Interest in LLVM static analysis tools.
  \item Linux is a big and complex project: it can also
        allow to find Clang bugs.
\end{itemize}
\end{frame}

\begin{frame}{Differences between clang and gcc}
\begin{itemize}
  \item \code{gcc} needs to be compiled for each architecture you want
        to support. Therefore, many different \code{gcc} cross-compilers
	are available.
  \item \code{clang} supports all target architectures at the same time.
        The same executable can generate code for all the architectures
        it supports.
\end{itemize}
\end{frame}

\begin{frame}[fragile]{Linux kernel: Clang support status}
  \includegraphics[height=0.8\textheight]{graphics/supported-kernel-architectures.png}\\
  \scriptsize Source: \url{https://www.kernel.org/doc/html/latest/kbuild/llvm.html}
\end{frame}

\begin{frame}[fragile]{Environment setup for Clang (Ubuntu 22.04)}
Common packages:
\begin{block}{}
\begin{verbatim}
sudo apt install build-essential flex bison libssl-dev
\end{verbatim}
\end{block}
Clang packages:
\begin{block}{}
\begin{verbatim}
sudo apt install clang llvm lld
\end{verbatim}
\end{block}
Environment for compiling the kernel:
\begin{block}{}
\begin{verbatim}
export ARCH=arm
export LLVM=1
\end{verbatim}
\end{block}
\end{frame}

\begin{frame}[fragile]{Environment setup for GCC (Ubuntu 22.04)}
Gcc packages:
\begin{block}{}
\begin{verbatim}
sudo apt install gcc-12-arm-linux-gnueabihf
sudo update-alternatives --install /usr/bin/arm-linux-gnueabihf-gcc \
     arm-linux-gnueabihf-gcc /usr/bin/arm-linux-gnueabihf-gcc-12 12
\end{verbatim}
\end{block}
Environment for compiling the kernel:
\begin{block}{}
\begin{verbatim}
export ARCH=arm
export CROSS_COMPILE=arm-linux-gnueabihf-
unset LLVM
\end{verbatim}
\end{block}
\end{frame}

\begin{frame}[fragile]{Compile time and size results}
\begin{columns}
\column{0.5\textwidth}
With Clang 14
\begin{block}{}
\begin{verbatim}
cd linux-5.18-rc6
make omap2plus_defconfig
time make -j8 zImage

real    25m59,392s
user    84m4,292s
sys     13m27,629s

du -s arch/arm/boot/zImage
4912   arch/arm/boot/zImage
\end{verbatim}
\end{block}
\column{0.5\textwidth}
With GCC 12
\begin{block}{}
\begin{verbatim}
cd linux-5.18-rc6
make omap2plus_defconfig
time make -j8 zImage

real    24m11,143s
user    80m34,624s
sys     9m55,833s

du -s arch/arm/boot/zImage
4908    arch/arm/boot/zImage
\end{verbatim}
\end{block}
\end{columns}
\end{frame}

\begin{frame}[fragile]{Compile time compared}
  \includegraphics[height=0.8\textheight]{graphics/compile-time.png}
\end{frame}

\begin{frame}[fragile]{Boot time comparisons}
On BeagleBone Black, booting Linux 5.18-rc6 on a small initramfs with
Busybox, time between \code{U-Boot SPL} and \code{Please press Enter}:
\begin{itemize}
   \item Kernel built with Clang 14:\\
	 Average boot time: 6.427 s
   \item Kernel built with gcc 12:\\
	 Average boot time: 6.422 s (-5 ms)
\end{itemize}
Conclusion: the boot time difference is negligible.
\end{frame}

\begin{frame}{Link Time Optimization (LTO)}
\begin{itemize}
  \item Clang allows to implement global optimizations at link time\\
        See \url{https://www.llvm.org/docs/LinkTimeOptimization.html}
        for details.
  \item LTO is supposed to be able to identify and delete dead code.
  \item Three LTO options in the Linux kernel:
  \begin{itemize}
     \item \kconfig{CONFIG_LTO_NONE}: no LTO (by default)
     \item \kconfig{CONFIG_LTO_CLANG_FULL}: full LTO but heavy CPU and
           RAM usage at link time. Example: needs 7.9 GB of RAM to
	   link an \code{arm64} kernel (\code{defconfig} configuration).
     \item \kconfig{CONFIG_LTO_CLANG_THIN}: much lighter than full LTO
           on RAM usage and CPU time.  See \url{https://clang.llvm.org/docs/ThinLTO.html}.
  \end{itemize}
  \item Gcc also has LTO support but is not supported for building
	the Linux kernel.
\end{itemize}
\end{frame}

\begin{frame}[fragile]{LTO tests}
\small
\begin{center}
\includegraphics[height=0.35\textheight]{graphics/kernel-size.png}\\
\includegraphics[height=0.35\textheight]{graphics/ram-usage.png}\\
\end{center}
\begin{itemize}
  \item \code{arm64} build (\code{Image.gz}): Linux 5.18-rc7, \code{defconfig} configuration
  \item \code{x86} build (\code{bzImage}): Linux 5.18-rc7, \code{x86_64_defconfig} configuration
\end{itemize}
\end{frame}


\begin{frame}[fragile]{Notes on LTO results}
\begin{itemize}
  \item Note that LTO (full and thin) is currently not enabled on \code{arm} (32 bit).
  \item The full LTO kernels are way bigger than non LTO ones
  \item This is most probably due to extra inlining, good for
        performance, but not for boot time (a bigger kernel takes more
        time to load and decompress). Topic discussed on
        \url{https://github.com/ClangBuiltLinux/linux/issues/1643}.
  \item Lacked time to run performance benchmarks on \code{x86} or
\code{arm64}
\end{itemize}
\end{frame}

\begin{frame}{clang-tidy}
\small
{\em "clang-tidy is a clang-based C++ “linter” tool. Its purpose is to
provide an extensible framework for diagnosing and fixing typical
programming errors, like style violations, interface misuse, or bugs
that can be deduced via static analysis."}
\begin{itemize}
  \item Set up your environment for compiling your kernel with Clang
  \item Configure your kernel
  \item You could even run \code{make allyesconfig} to cover the whole code
  \item Run \code{make clang-tidy} or better \code{make -j8 clang-tidy}
  \item Output: no issue reported on Linux 5.18-rc6
        (\code{omap2plus_defconfig})
\end{itemize}
\url{https://clang.llvm.org/extra/clang-tidy/}
\end{frame}

\begin{frame}{clang-analyzer}

{\em "The Clang Static Analyzer is a source code analysis tool that
finds bugs in C, C++, and Objective-C programs."}
\begin{itemize}
  \item Also set up your environment for compiling your kernel with
        Clang and configure your kernel (possibly with \code{make
        allyesconfig}).
  \item Run \code{make clang-analyzer} or better \code{make -j8
        clang-analyzer}
  \item As the volume of output is huge, suggestion to duplicate
        it to a file:
        \code{make -j8 clang-analyzer 2>&1 | tee /tmp/clang-analyzer.log}
  \item Note: static analysis takes much more time than compiling.
\end{itemize}
\url{https://clang-analyzer.llvm.org/}
\end{frame}

\begin{frame}[fragile]{clang-analyzer: many false positives}
\begin{lstlisting}[breaklines]
...
/home/tux/linux/clang/linux-5.18-rc6/kernel/reboot.c:831:9: note: Call to function 'sprintf' is insecure as it does not provide security checks introduced in the C11 standard. Replace with analogous functions that support length arguments or provides boundary checks such as 'sprintf_s' in case of C11
        return sprintf(buf, "%d\n", reboot_cpu);
               ^~~~~~~
Suppressed 40 warnings (40 in non-user code).
Use -header-filter=.* to display errors from all non-system headers. Use -system-headers to display errors from system headers as well.
15 warnings generated.
/home/tux/linux/clang/linux-5.18-rc6/kernel/async.c:125:2: warning: Value stored to 'calltime' is never read [clang-analyzer-deadcode.DeadStores]
        calltime = ktime_get();
        ^          ~~~~~~~~~~~
/home/tux/linux/clang/linux-5.18-rc6/kernel/async.c:125:2: note: Value stored to 'calltime' is never read
        calltime = ktime_get();
        ^          ~~~~~~~~~~~
...
\end{lstlisting}
\small
However:
\begin{itemize}
   \item The \code{sprintf_s()} function doesn't exist in the kernel code
   \item \code{calltime} in \kfile{kernel/async.c} can be accessed if
         the configuration enables \kfunc{pr_debug}.
   \item There are countless examples like this
\end{itemize}
\end{frame}

\begin{frame}{Improvements to the kernel code}
However, the Clang warnings have allowed to implement many
improvements to the Linux kernel code:
\begin{itemize}
     \item Nathan Chancellor's patches:
           \url{https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/log/?qt=author&q=chancellor}
     \item Nick Desaulniers' patches:
           \url{https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/log/?qt=author&q=desaulniers}
\end{itemize}
\end{frame}

\begin{frame}{Further limitations and issues}
\begin{itemize}
  \item Difficult to run \code{clang-analyzer} on a single file\\
        (see workaround on the next page)
  \item How to tweak \code{clang-analyzer} to keep only
        the warnings relevant to Linux kernel code?
\end{itemize}
\end{frame}

\begin{frame}{How to run clang-analyzer on a single file}
\begin{itemize}
  \item Make a copy of the \code{compile_commands.json} generated file, which just
        describes one file. Let's call it \code{compile_commands-1file.json}
  \item Then run:\\
        \code{python3 ./scripts/clang-tools/run-clang-tools.py
        clang-analyzer compile_commands-1file.json}
\end{itemize}
\end{frame}

\begin{frame}{Conclusions}
\begin{itemize}
  \item For the most popular CPU architectures, building the kernel with
        Clang instead of Gcc is already possible and mature.
  \item With Clang, you don't need a cross-compiler any more!
  \item The Clang warnings have already helped to improve the kernel code
  \item However, we haven't reaped all the benefits of using Clang yet:
        \begin{itemize}
            \item No size benefits of LTO yet: LTO kernels much bigger
	    \item Many \code{clang-analyzer} warnings still irrelevant
        \end{itemize}
\end{itemize}
\end{frame}

\begin{frame}{Useful resources}
\begin{itemize}
   \item ClangBuiltLinux project: \url{https://clangbuiltlinux.github.io/}\\
         Build status, bug reports, documentation, meetings.\\
         The place to join to get involved.
   \item Kernel documentation: {\em Building Linux with Clang/LLVM}\\
         \url{https://www.kernel.org/doc/html/latest/kbuild/llvm.html}
   \item LWN.net: {\em Building the kernel with Clang}\\
         \url{https://lwn.net/Articles/734071/}
\end{itemize}
Thanks to Nathan Chancellor and Nick Desaulniers (Clang/LLVM Build
Support kernel maintainers) for answering my questions!
\end{frame}


\questionslide

\end{document}
