\documentclass[aspectratio=169,obeyspaces,spaces,hyphens,dvipsnames]{beamer}
\usepackage[utf8]{inputenc}
\usepackage{lmodern}% http://ctan.org/pkg/lm
\usepackage{minted}
\usepackage{hyperref}
\usepackage{xcolor}
\usepackage{pgfplots}
\usepackage{tikz}
\usepackage[normalem]{ulem}

\mode<presentation>
\usetheme{Bootlin}

\def\signed #1{{\leavevmode\unskip\nobreak\hfil\penalty50\hskip2em
  \hbox{}\nobreak\hfil(#1)
  \parfillskip=0pt \finalhyphendemerits=0 \endgraf}}

\newsavebox\mybox
\newenvironment{aquote}[1]
  {\savebox\mybox{#1}\begin{quotation}}
  {\signed{\usebox\mybox}\end{quotation}}

\title{Yocto Project Autobuilders and the SWAT team}
\authors{Alexandre Belloni}
\email{alexandre.belloni@bootlin.com}
\slidesurl{https://bootlin.com/pub/conferences/2022/elc/belloni-yocto-autobuilders-swat/}
\institute{Bootlin}
\conference{Embedded Linux Conference}

\begin{document}

\addtocontents{toc}{\protect\setcounter{tocdepth}{-1}}
\section{Yocto Project Autobuilders and the SWAT team}
\addtocontents{toc}{\protect\setcounter{tocdepth}{2}}

\begin{frame}{Alexandre Belloni}
    \begin{itemize}
      \item Co-owner and COO at {\bf Bootlin}
        \begin{itemize}
        \item Embedded Linux experts
        \item Engineering services: Linux BSP development, kernel porting
          and drivers, Yocto/Buildroot integration, real-time, boot-time,
          security, multimedia
        \item Training services: Embedded Linux, Linux kernel drivers,
          Yocto, Buildroot, graphics stack, boot-time, real-time
        \end{itemize}
      \item Open-source contributor
        \begin{itemize}
          \item Linux kernel {\bf maintainer for the RTC and I3C subsystems}
          \item Linux kernel {\bf co-maintainer for Microchip ARM and
            MIPS processors}
          \item {\bf Yocto Project} SWAT team lead
        \end{itemize}
      \item Living in {\bf Lyon}, France
      \item \code{alexandre.belloni@bootlin.com}
    \end{itemize}
    {\small \url{https://bootlin.com/company/staff/alexandre-belloni/}}
\end{frame}

\begin{frame}{Yocto Project Autobuilders}
  \begin{center}
    \includegraphics[height=0.8\textheight]{ab-test-cluster.png}
  \end{center}
\end{frame}

\begin{frame}{Autobuilders - buildbot}
  \begin{itemize}
  \item The Yocto Project CI is based on buildbot, a python continuous
    integration framework.
  \item It is configured using python scripts, available at
    \url{https://git.yoctoproject.org/yocto-autobuilder2/}
  \item This repository lists and defines all the \code{builders}
    (available types of builds) and \code{schedulers}
  \item \code{README.md} will help you set up your own autobuilders in
    a matter of minutes.
  \item More configuration is done through the
    \url{https://git.yoctoproject.org/yocto-autobuilder-helper}
    repository, especially in \code{config.json}
  \item Nice UI at
    \url{https://autobuilder.yoctoproject.org/typhoon/}
  \end{itemize}
\end{frame}

\begin{frame}{Autobuilders - workers}
  \begin{itemize}
  \item The Yocto Project autobuilders currently have 26 workers:
    alma8-ty-1, alma8-ty-2, centos7-ty-4, debian11-ty-1, debian11-ty-3,
    fedora35-ty-1, fedora35-ty-2, fedora36-ty-1, fedora36-ty-2,
    fedora36-ty-3, opensuse153-ty-1, opensuse154-ty-1,
    opensuse154-ty-2, opensuse154-ty-3, perf-alma8, perf-debian11,
    stream8-ty-1, tumbleweed-ty-3, ubuntu1804-arm-1, ubuntu1804-ty-3,
    ubuntu2004-arm-1, ubuntu2004-ty-1, ubuntu2110-ty-2,
    ubuntu2204-ty-1, ubuntu2204-ty-2, ubuntu2204-ty-3
  \item They are necessarily heterogeneous to be able to test on many
    different distributions: AlmaLinux 8, CentOS 7, CentOS Stream 8,
    Debian 11, Fedora 35, Fedora 36, openSUSE 15.3, openSUSE 15.4,
    Tumbleweed, Ubuntu 18.04, Ubuntu 20.04, Ubuntu21.10, Ubuntu 22.04
  \item Also, two different architectures are used: x86 and aarch64
  \end{itemize}
\end{frame}

\begin{frame}{Autobuilders - workers}
  \begin{itemize}
  \item x86 workers: most are 28 cores, 56 threads Intel Xeon E5, some are
    24 cores, 48 threads with 128 to 384 GB of RAM
  \item ARM workers: one 64 core Cortex-A72 with 256GB of RAM and one 32
    cores APM X-Gene with 128 GB of RAM
  \item This creates a fair amount of maintenance as the workers need
    to be configured differently.
  \item Two workers are specifically reserved for build performance
    testing (perf-alma8 and perf-debian11)
  \item About half of the workers now have SSDs
  \end{itemize}
\end{frame}

\begin{frame}{Autobuilders - builders}
  \begin{itemize}
  \item Currently 81 different builders are defined:\\
    \fontsize{10}{10}\selectfont
    a-full, a-quick, auh, beaglebone, beaglebone-alt, bringup,
    bringup-fast, build-appliance, buildperf-alma8,
    buildperf-debian11, buildtools, check-layer, check-layer-nightly,
    docs, edgerouter, edgerouter-alt, genericx86, genericx86-64,
    genericx86-64-alt, genericx86-alt, meta-agl-core, meta-arm,
    meta-aws, meta-intel, meta-mingw, meta-oe, meta-virt, metrics,
    multilib, musl-qemux86, musl-qemux86-64, no-x11, non-gpl3,
    oe-selftest, oe-selftest-arm, oe-selftest-armhost,
    oe-selftest-centos, oe-selftest-debian, oe-selftest-fedora,
    oe-selftest-ubuntu, pkgman-deb-non-deb, pkgman-non-rpm,
    pkgman-rpm-non-rpm, poky-tiny, qa-extras, qa-extras2, qemuarm,
    qemuarm-alt, qemuarm-armhost, qemuarm-oecore, qemuarm64,
    qemuarm64-alt, qemuarm64-armhost, qemuarm64-ltp, qemuarm64-ptest,
    qemuarm64-ptest-fast, qemumips, qemumips-alt, qemumips64, qemuppc,
    qemuppc-alt, qemuppc64, qemuriscv32, qemuriscv64,
    qemuriscv64-ptest, qemux86, qemux86-64, qemux86-64-alt,
    qemux86-64-ltp, qemux86-64-ptest, qemux86-64-ptest-fast,
    qemux86-64-x32, qemux86-alt, qemux86-world, qemux86-world-alt,
    reproducible, reproducible-centos, reproducible-debian,
    reproducible-fedora, reproducible-ubuntu, wic
  \end{itemize}
\end{frame}

\begin{frame}[fragile]{Autobuilders - builders}
  What is getting built?\\
  Most of the builders build \code{core-image-sato} using the
  \code{poky} distro. Look at \code{config.json}:
  \begin{block}{config.json}
    \fontsize{7}{6}\selectfont
    \begin{minted}{JSON}
    "overrides" : {
        "qemux86-64" : {
            "MACHINE" : "qemux86-64",
            "TEMPLATE" : "arch-qemu",
            "step1" : {
                "extravars" : [
                     "IMAGE_FSTYPES:append = ' wic wic.bmap'"
                ]
            }
        },
    }
     \end{minted}
  \end{block}
\end{frame}

\begin{frame}[fragile]{Autobuilders - builders}
  \begin{block}{config.json}
    \fontsize{7}{6}\selectfont
    \begin{minted}{JSON}
    "templates" : {
        "arch-qemu" : {
            "BUILDINFO" : true,
            "BUILDHISTORY" : true,
            "extravars" : [
                "IMAGE_INSTALL:append = ' ssh-pregen-hostkeys'"
            ],
            "step1" : {
                "BBTARGETS" : "core-image-sato core-image-sato-sdk core-image-minimal core-image-minimal-dev core-image-sato:do_populate_sdk",
                "SANITYTARGETS" : "core-image-minimal:do_testimage core-image-sato:do_testimage core-image-sato-sdk:do_testimage core-image-sato:do_testsdk"
            },
            "step2" : {
                "SDKMACHINE" : "x86_64",
                "BBTARGETS" : "core-image-sato:do_populate_sdk core-image-minimal:do_populate_sdk_ext core-image-sato:do_populate_sdk_ext",
                "SANITYTARGETS" : "core-image-sato:do_testsdk core-image-minimal:do_testsdkext core-image-sato:do_testsdkext"
            },
            "step3" : {
                "shortname" : "Machine oe-selftest",
                "BUILDHISTORY" : false,
                "EXTRACMDS" : ["${SCRIPTSDIR}/checkvnc; DISPLAY=:1 oe-selftest ${HELPERSTMACHTARGS} -j 15"],
                "ADDLAYER" : ["${BUILDDIR}/../meta-selftest"]
            }
        },
    }
     \end{minted}
  \end{block}
\end{frame}

\begin{frame}{Autobuilders - builders}
  \begin{itemize}
  \item Two parent builders: \code{a-full} and \code{a-quick}. They will request
    builds from other builders.
  \item auh: Tries to upgrade all the recipes to their latest upstream
    version
  \item Machine specific builders, building for Yocto Project members
    machines: beaglebone, beaglebone-alt, edgerouter, edgerouter-alt,
    genericx86, genericx86-64, genericx86-64-alt, genericx86-alt
  \item Qemu based machines: musl-qemux86, musl-qemux86-64, qemuarm, qemuarm-alt, qemuarm-armhost, qemuarm-oecore, qemuarm64, qemuarm64-alt, qemuarm64-armhost
  \item Performance builders, recording the build time and other
    performance related metrics: buildperf-alma8, buildperf-debian11
  \item Documentation: docs
  \item Yocto Project members layers: meta-agl-core, meta-arm,
    meta-aws, meta-intel
  \item check-layer, check-layer-nightly: checks whether the included
    layer are Yocto Project compatible
  \end{itemize}
\end{frame}

\begin{frame}{Autobuilders - builders}
  \begin{itemize}
  \item metrics: checks the packages for existing CVEs
  \item selftests: oe-selftest, oe-selftest-centos, oe-selftest-debian, oe-selftest-fedora, oe-selftest-ubuntu
  \item ptests: qemuarm64-ptest, qemuarm64-ptest-fast, qemux86-64-ptest, qemux86-64-ptest-fast
  \item ltp: qemuarm64-ltp, qemux86-64-ltp
  \item reproducible: Ensures the generated packages are bit for bit
    reproducible
  \item wic: tests \code{wic} by generating multiple disk images
  \item non-gpl3: builds with \code{INCOMPATIBLE_LICENSE = '*GPLv3'}
    and \code{meta-gplv2}
  \end{itemize}
\end{frame}

\begin{frame}{Autobuilders - schedulers}
  \begin{itemize}
  \item Any builder can be triggered manually through the buildbot
    interface
  \item \code{a-quick} runs at 1am each day Mon-Sat using the \code{master}
    branch
  \item \code{a-full} runs at 1am Sun each week using the \code{master} 
    branch
  \item \code{check-layer-nightly} runs  each day for \code{master}
  \item \code{metrics} runs at 7am each day
  \item \code{check-layer-nightly} runs twice a week for \code{kirkstone}
  \item \code{check-layer-nightly} runs twice a week for \code{dunfell}
  \item \code{AUH} twice a month on 1st and 15th
  \item build performance tests run at 3am, 9am, 3pm and 9pm
  \item \code{docs} runs on every commit
  \end{itemize}
\end{frame}

\begin{frame}{Autobuilders - manual buildrequests}
  \begin{itemize}
  \item An \code{a-full} build takes from 5 to 9 hours and will load
    most of the workers.
  \item It is then not practical to start a build automatically for
    every commit on the \code{master} or LTS branches(\code{dunfell},
    \code{kirkstone})
  \item It is even less practical to do so for every patch series sent
    on the mailing list
  \item So, build testing is a manual process
  \end{itemize}
\end{frame}

\begin{frame}{Build testing workflow}
  \begin{itemize}
  \item The process starts by reviewing and collecting patches from
    the mailing-lists in the appropriate repositories: \code{bitbake},
    \code{meta-yocto}, \code{openembedded-core} and \code{yocto-docs}
  \item A \code{poky} branch is then created from these repositories
    using \code{combo-layer}
  \item This branch is pushed upstream and the autobuilders can be
    instructed to build \code{a-full} using it.
  \item In case of build failures, the incriminated patches are
    removed from the branch and the process starts over
  \item If the \code{a-full} build is successful, a final review is
    done and patches are merged by Richard Purdie.
  \end{itemize}
\end{frame}

\begin{frame}{Tests - selftests}
  The selftest builders run:
  \begin{itemize}
  \item \code{bitbake-selftest}: tests BitBake and its APIs, including the
    parser and fetchers.
  \item \code{oe-selftest}, skipping the \code{reproducible} test:
    \code{openembedded-core} unit tests from
    \code{meta/lib/oeqa/selftest/}. Tests \code{devtool},
    \code{recipetool}, \code{archiver}, \code{bitbake-layers},
    CVE checks, \code{INCOMPATIBLE_LICENSE}, \code{runqemu},
    \code{wic}
  \item \code{oe-pylint}: runs \code{pylint3} when available on the
    python modules.
  \end{itemize}
\end{frame}

\begin{frame}{Tests - SDK}
  \begin{itemize}
  \item Both the regular SDK and the extensible SDK are tested, using
    the \code{testsdk} and \code{testsdk_ext} tasks for the built
    image.
  \item Tests are in \code{meta/lib/oeqa/sdk} and
    \code{meta/lib/oeqa/sdkext}
  \item They assume the SDK environment is already setup.
  \item Tests whether the compiler can generate proper binaries for the
    platform, also tests make, cmake, perl, python,...
  \end{itemize}
\end{frame}

\begin{frame}{Tests - images}
  \begin{itemize}
  \item The autobuilders don't have a board farm however, the qemu
    images are run.
  \item The \code{testimage} task of the built image is used
    \begin{itemize}
    \item Uses \code{runqemu} to boot the kernel and use the generated
      rootfs
    \item Tests network connectivity, \code{apt}, \code{dnf},
      \code{stap}, \code{systemd}, \code{weston}, \code{xorg}
    \end{itemize}
  \item \code{-ltp} builders add LTP in the rootfs and runs the test
    suite.
  \item \code{-ptest} builders add \code{ptest} packages in the rootfs
    and will run the various tests
    \begin{itemize}
    \item \code{ptests} are unit tests or test suites that are in the
      upstream release
    \item they are always built but have to be explicitly installed
    \item e.g.: openssl-ptest, sed-ptest, glibc-tests-ptest,
      lttng-tools-ptest, python3-ptest
    \item full list is in
      \code{meta/conf/distro/include/ptest-packagelists.inc}
    \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}{Tests - reproducible}
  The output of the build is bit for bit reproducible (apart from Go
  and ruby docs)
  \begin{itemize}
  \item Does a first build with \code{Shared State} allowed (but not
    necessarily present)
  \item Then a second build with \code{Shared State} disabled,
    ensuring binaries will get built
  \item Both output are then compared
  \item All package types are tested (\code{ipk}, \code{deb},
    \code{rpm})
  \item Failures, including binaries and diffoscope output are
    available at \url{https://autobuilder.yocto.io/pub/repro-fail/}
  \item Results are at
    \url{https://www.yoctoproject.org/reproducible-build-results/}
  \end{itemize}
\end{frame}

\begin{frame}{Saved results}
  Some output of the build is saved and archived, in particular:
  \begin{itemize}
  \item \code{stdio} output, including separate files for
    warnings and errors.
  \item \code{Shared State} as all the autobuilders are populating the
    same sstate-cache
  \item \code{Hash equivalency} is exported through
    \url{http://typhoon.yocto.io:8687}
  \item \code{buildhistory} is pushed to
    \url{https://git.yoctoproject.org/poky-buildhistory/}
  \item \code{testresults} are committed to
    \url{https://git.yoctoproject.org/yocto-testresults/}
  \item \code{buildstats} from the performance test builds are
    committed to \url{https://git.yoctoproject.org/yocto-buildstats/}
  \item Most of those are also available with their logs on
    \url{https://autobuilder.yocto.io/pub/non-release/}
  \end{itemize}
\end{frame}

\begin{frame}{Buildperf output example}
  \url{https://autobuilder.yocto.io/pub/non-release/20220621-14/testresults/buildperf-alma8/perf-alma8_master_20220621150024_9694ef314c.html}
\end{frame}

\begin{frame}{SWAT team}
  \begin{itemize}
  \item The SWAT team looks at the build failures so none are missed
  \item Its goal is not directly to solve them but to raise attention
    to the various failures
  \item A specific tool allows to triage those failure: swatbot
    \url{https://swatbot.yoctoproject.org/mainindex/swat/}
  \item The autobuilders are feeding results to swatbot
  \item Person on SWAT duty will triage failures:
    \begin{itemize}
    \item For a patch that is under testing and not in \code{master}
      yet, reply on the mailing list, pointing to the build failure.
    \item For a commit that is on \code{master}, open a bug on
      bugzilla
    \item For an intermittent issue (\code{AB-INT}), a bug needs to be
      open or comment added to an existing bug to allow tracking the
      frequency of the issue
    \item Sometimes the issue has already been handled by the time the
      failure is triaged
    \end{itemize}
  \item Statistics are also maintained
  \end{itemize}
\end{frame}

\begin{frame}[fragile]{AB-INT issues - solved}
  230 tracked AB-INT have been closed
  \begin{itemize}
  \item Some are infrastructure related:
    \begin{itemize}
    \item 14551 runtime\_test.TestImage.test\_testimage\_virgl\_headless failure
      \begin{block}{}
        \fontsize{8}{8}\selectfont
        \begin{verbatim}
runqemu - ERROR - Failed to run qemu: qemu-system-x86_64: -qmp unix:./.ce0t9x2n,server,wait: info: QEMU waiting for connection on: disconnected:unix:./.ce0t9x2n,server=on
qemu-system-x86_64: egl: no drm render node available
qemu-system-x86_64: egl: render node init failed
        \end{verbatim}
      \end{block}
      This was a permission issue following an OpenSUSE upgrade
    \item 14096 perl install race (pod2text)
      \begin{block}{}
        \fontsize{8}{8}\selectfont
        \begin{verbatim}
Couldn't copy cpan/podlators/blib/script/pod2text to /.../usr/bin/pod2text: No such file or directory
Couldn't chmod 755 /.../usr/bin/pod2text: No such file or directory
|   /usr/local/oe-sdk-hardcoded-buildpath/sysroots/aarch64-pokysdk-linux/usr/bin/pod2text
|   /usr/local/oe-sdk-hardcoded-buildpath/sysroots/aarch64-pokysdk-linux/usr/bin/pod2usage
        \end{verbatim}
      \end{block}
      Reported upstream \url{https://github.com/arsv/perl-cross/issues/75}\\
      This ended up being a \code{make} issue, happening only on an
      old version, shipped with ubuntu 16.04 and ubuntu 18.04
    \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}[fragile]{AB-INT issues - solved}
  \begin{itemize}
  \item 
    \begin{itemize}
    \item 14712 build hangs: host make is not collecting its own children, turning them into zombies\\
      \code{make} issue in version 4.2.1 shipped by centOS, Alma,
      Stream and openSUSE.
      Reported upstream
      \url{https://bugs.centos.org/view.php?id=18432}
      Fixed by disallowing make 4.2.1
    \end{itemize}
  \item Many were performance related: multiple builds are allowed to
    run in parallel on a single worker, increasing the load.
    \begin{itemize}
    \item e.g. Kernel RCU stalls in qemu due to I/Os and host CPU load
    \item \code{make} load awareness is now used: \code{PARALLEL_MAKE = '-j
      16 -l 52'}
    \item \code{xz} limits were lowered:
      \begin{verbatim}
      XZ_MEMLIMIT = '5%'
      XZ_THREADS = '8'
      \end{verbatim}
    \item the qemu rootfs image is copied to tmpfs to avoid I/Os while
      running
    \item the workers are getting switched to SSDs
    \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}[fragile]{AB-INT issues - solved}
  \begin{itemize}
  \item Some upstream tests are not very well written:
    \begin{itemize}
    \item 14840 AB-INT PTEST ARM: python3 testSockName ptest failure
      \begin{block}{}
        \fontsize{8}{8}\selectfont
        \begin{verbatim}
ERROR: testSockName (test.test_socket.GeneralModuleTests)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/usr/lib/python3.10/test/test_socket.py", line 1380, in testSockName
    sock.bind(("0.0.0.0", port))
OSError: [Errno 98] Address already in use
        \end{verbatim}
      \end{block}
      \begin{block}{test\_socket.py}
        \fontsize{7}{6}\selectfont
        \begin{minted}{Python}
        port = socket_helper.find_unused_port()
        sock.bind(("0.0.0.0", port))
        \end{minted}
      \end{block}
    \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}[fragile]{AB-INT issues - solved}
  \begin{itemize}
  \item Some upstream tests are not very well written:
    \begin{itemize}
    \item 14507 AB-INT PTEST: libevent monotonic\_prc\_fallback\_FAILED\\
      Reported upstream
      \url{https://github.com/libevent/libevent/issues/1193}
      \begin{block}{}
        \fontsize{8}{8}\selectfont
        \begin{verbatim}
regress:
  FAIL ../libevent-2.1.12-stable/test/regress_util.c:1478: assert(diff.tv_sec == 0): 1 vs 0 util/monotonic_prc_fallback:
  [monotonic_prc_fallback FAILED]
 1/312 TESTS FAILED. (33 skipped)
        \end{verbatim}
      \end{block}
      Fixed by allowing the test to rerun and marking a test failed
      only when all runs failed. Patch upstreamed
      \url{https://github.com/libevent/libevent/pull/1214}.
    \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}{AB-INT issues - current}
  49 AB-INT are still present. The most pressing or worrying are:
  \begin{itemize}
  \item 14018 efibootpartition.GenericEFITest.test\_boot\_efi selftest failed
  \item 14201 Bitbake server intermittent timeout
  \item 14263 lttng-tools ptest intermittent failure
  \item 14401 Test unable to login to serial console
  \item 14522 qemuppc doesn't shutdown within timeout (serial console issues)
  \item 14665 prservice.BitbakePrTests.test\_import\_export\_replace\_db failure
  \item 14775 SDK preparation failure: SState: cannot test file://[...] TimeoutError('timed out')
  \item 14787 systemd.SystemdServiceTests.test\_systemd\_status failure
  \end{itemize}
\end{frame}

\begin{frame}{AB-INT issues - help needed}
  What needs to be worked on?
  \begin{itemize}
  \item Better logging is needed, especially collecting the relevant
    output when something fails:
    \begin{itemize}
    \item dmesg output of the target kernel
    \item bitbake-cooker.log
    \end{itemize}
  \item More tests are also needed, both for \code{oe-selftest} but also
    upstream tests to run as \code{ptests}
  \item Some tests would benefit from being more robust and especially
    less timing dependent.
  \end{itemize}
\end{frame}

\questionslide

\end{document}
