\documentclass[aspectratio=169,obeyspaces,spaces,hyphens,dvipsnames]{beamer}
\usepackage[utf8]{inputenc}
\usepackage{lmodern}% http://ctan.org/pkg/lm
\usepackage{minted}
\usepackage{newunicodechar}
\usepackage{hyperref}
\usepackage{xcolor}
\usepackage{pgfplots}
\usepackage{tikz}
\usepackage[normalem]{ulem}

\mode<presentation>
\usetheme{Bootlin}

\def\signed #1{{\leavevmode\unskip\nobreak\hfil\penalty50\hskip2em
  \hbox{}\nobreak\hfil(#1)
  \parfillskip=0pt \finalhyphendemerits=0 \endgraf}}

\newsavebox\mybox
\newenvironment{aquote}[1]
  {\savebox\mybox{#1}\begin{quotation}}
  {\signed{\usebox\mybox}\end{quotation}}


\title{OpenEmbedded and Yocto Project best practices}
\authors{Alexandre Belloni}
\email{alexandre.belloni@bootlin.com}
\slidesurl{http://bootlin.com/pub/conferences/2020/elce/belloni-yocto-best-practices/}
\institute{Bootlin}
\conference{Embedded Linux Conference Europe 2020}

\begin{document}

\addtocontents{toc}{\protect\setcounter{tocdepth}{-1}}
\section{OpenEmbedded and Yocto Project best practices}
\addtocontents{toc}{\protect\setcounter{tocdepth}{2}}

\begin{frame}{Alexandre Belloni}
  \begin{columns}
    \column{0.6\textwidth}
    \begin{itemize}
    \item Embedded Linux engineer at Bootlin
      \begin{itemize}
      \item Embedded Linux {\bf expertise}
      \item {\bf Development}, consulting and training
      \item Bootloader, Linux kernel, Yocto Project, Buildroot
      \item Complete Linux BSP development
      \item Hardware support in bootloader/Linux
      \item Strong open-source focus: upstreaming and contributions
      \item Freely available training materials
      \end{itemize}
    \item Open-source contributor
      \begin{itemize}
      \item Maintainer for the Linux kernel {\bf RTC subsystem}
      \item Co-Maintainer of {\bf kernel support for Microchip (ARM and MIPS)
        processors}
      \end{itemize}
    \end{itemize}
    \column{0.4\textwidth}
    \includegraphics[width=\textwidth]{common/logo-square.pdf}\\
  \end{columns}
\end{frame}

\subsection{Distributions}

\begin{frame}{Poky}
  \begin{center}
    \includegraphics[height=0.8\textheight]{yocto-overview-poky.pdf}
  \end{center}
\end{frame}

\begin{frame}[fragile]{Poky}
  \begin{itemize}
  \item Yocto Project is an entity, not something you can use.
  \item Poky is the reference distribution, the code that is
    downladed and used.
  \item As a reference distribution, it is not tailored to your
    system (e.g. it always includes opengl)
  \item It can generate demo images but is not meant to be used
    as-is on production systems.
  \item The included features are not stable (e.g. it switched from
    xorg to wayland)
  \item Poky bundles Openembedded-core, bitbake and two very small
    layers:
    \begin{itemize}
    \item \code{meta-yocto-bsp} is a {\bf BSP} layer for reference boards from
      the Yocto Project members
    \item \code{meta-poky} is a {\bf distro} layer with four
      distributions: \code{poky}, \code{poky-tiny}, \code{poky-bleeding},
      \code{poky-altcfg}
    \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}[fragile]{Poky}
  \begin{itemize}
  \item For your project, not using Poky has some advantages:
    \begin{itemize}
    \item when reporting bugs, it is necessary to reproduce with a
      \code{nodistro} build
    \item it is easier to start from \code{nodistro} and create a
      distribution than tuning a distribution including
      \code{poky.conf}
    \item it is easier to work with the oe-core repository when
      sending patches upstream
    \item Confidentiality, Poky defines \code{PREMIRRORS} that point
      to \url{http://downloads.yoctoproject.org/mirror/sources/},
      it will leak the name of everything that is fetched using
      version control.
    \end{itemize}
  \item The main drawback is having to match the oe-core and bitbake
    branches manually.
  \end{itemize}
\end{frame}

\begin{frame}[fragile]{Creating your own distribution}
  \begin{itemize}
  \item Not that difficult, simply have
    \code{conf/distro/<distro_name>.conf}
  \item Used to define the distribution wide policies:
    \begin{itemize}
    \item Toolchain (including libc) selection
    \item init selection
    \item \code{DISTRO_FEATURES}
    \item \code{PREFERRED_PROVIDERS}
    \item \code{PACKAGE_CLASSES}
    \item QA checks with \code{WARN_QA} and \code{ERROR_QA}
    \end{itemize}
  \end{itemize}
\end{frame}

\subsection{local.conf}

\begin{frame}[fragile]{local.conf}
  \begin{itemize}
  \item \code{local.conf} is really for local configuration (CPU
    number, disk space).
  \item Avoid the numerous tutorials saying otherwise
  \item The main reason is distribution of the changes and
    reproducibility of the build.
  \item Also huge drawback, a change in \code{local.conf} makes bitbake
    parse all the recipes again.
  \item It is fine to carry changes in \code{local.conf} for
    development/testing.
  \end{itemize}
\end{frame}

\begin{frame}[fragile]{local.conf - site.conf}
  \begin{itemize}
  \item \code{site.conf} is for site wide configuration (proxies,
    mirrors, shared sstate-cache location).
  \item Unfortunately, it suffers from the same \code{local.conf}
    distribution drawback.
  \end{itemize}
\end{frame}

\begin{frame}[fragile]{local.conf - image recipes}
  \begin{itemize}
  \item The most abused variable in \code{local.conf} is
    \code{IMAGE_INSTALL_append} (seen in tutorials from SoM vendors).
  \item This is not even easy for beginners due to parse order.
  \item The solution is simply to create your own image recipe as soon
    as the \code{core-image-*.bb} recipes are not enough anymore.
  \end{itemize}
\end{frame}

\begin{frame}[fragile]{local.conf - machine configuration}
  All the machine related varibales should go in the machine
  configuration:
  \begin{itemize}
  \item \code{PREFERRED_PROVIDER_virtual/kernel}
  \item \code{PREFERRED_PROVIDER_virtual/bootloader}
  \item \code{PREFERRED_VERSION_linux-*}
  \item \code{IMAGE_FSTYPES}
  \item In a few cases, \code{IMAGE_INSTALL_append}, for example, to
    actually install the kernel in the root filesystem.
  \end{itemize}
\end{frame}

\begin{frame}[fragile]{local.conf - distro configuration}
  The other variables should go in the distro configuration:
  \begin{itemize}
  \item \code{PREFERRED_PROVIDER_*}
  \item \code{PREFERRED_VERSION_*}
  \item \code{PACKAGECONFIG_pn-*}
  \item \code{INCOMPATIBLE_LICENSE}
  \item \code{LICENSE_FLAGS_WHITELIST}
  \end{itemize}
\end{frame}

\subsection{Release management}

\begin{frame}[fragile]{Release management}
  There are multiple tasks that OE/bitbake based projects let you do
  on your own to ensure build reproducibility:
  \begin{itemize}
  \item Code distribution and project setup.
  \item Release tagging
  \end{itemize}
  A separate tool is needed for that, usual solutions are:
  \begin{itemize}
  \item combo-layer, as done by Poky:
    \url{https://wiki.yoctoproject.org/wiki/Combo-layer}
  \item git submodules + setup script. Great example in YOE:
    \url{https://github.com/YoeDistro/yoe-distro}
  \item repo and \code{templateconf} or setup script
  \item kas
  \end{itemize}
\end{frame}

\begin{frame}[fragile]{repo}
  \begin{itemize}
  \item \code{repo} is used in Android to distribute its source code,
    which is split into many \code{git} repositories. It's a wrapper
    to handle several \code{git} repositories at once.
  \item The \code{repo} configuration is stored in \code{manifest}
      file, usually available in its own \code{git} repository.
  \item It could also be in a specific branch of your custom layer.
  \item It only handles fetching code, handling \code{local.conf} and
    \code{bblayers.conf} is done separately
  \end{itemize}
\end{frame}

\begin{frame}[fragile]{Manifest example}
  \begin{block}{}
  \fontsize{9}{9}\selectfont
  \begin{minted}{xml}
<?xml version="1.0" encoding="UTF-8"?>
<manifest>
  <default sync-j="4" revision="dunfell"/>

  <remote fetch="https://github.com/openembedded" name="oe"/>
  <remote fetch="https://github.com/Freescale" name="freescale"/>
  <remote fetch="ssh://git@server.com" name="private"/>

  <project remote="freescale" name="meta-freescale" path="sources/meta-freescale"/>
  <project remote="oe" name="openembedded-core" path="sources/openembedded-core"/>
  <project remote="oe" name="bitbake" path="sources/openembedded-core/bitbake"
           revision="1.46" />
  <project remote="oe" name="meta-openembedded" path="sources/meta-openembedded"/>

  <project remote="private" name="meta-custom" path="sources/meta-custom">
    <copyfile dest="setup-environment" src="buildconf/setup-environment"/>
  </project>
</manifest>
  \end{minted}
  \end{block}
\end{frame}

\begin{frame}[fragile]{repo: release}
  To tag a release, a few steps have to be taken:
  \begin{itemize}
  \item Optionally tag the custom layers
  \item For each project entry in the manifest, set the revision
    parameter to either a tag or a commit hash.
  \item Commit and tag this version of the manifest.
  \end{itemize}
\end{frame}

\begin{frame}[fragile]{kas}
  \begin{itemize}
  \item Specific tool developed by Siemens for OpenEmbedded:
    \url{https://github.com/siemens/kas}
  \item Will fetch layers and build the image in a single command
  \item Uses a single JSON or YAML configuration file part of the
    custom layer
  \item Can generate and run inside a Docker container
  \item Can setup \code{local.conf} and \code{bblayers.conf}
  \end{itemize}
\end{frame}

\begin{frame}[fragile]{kas configuration}
  \begin{block}{}
  \fontsize{10}{10}\selectfont
  \begin{minted}{yaml}
header:
  version: 8
machine: mymachine
distro: mydistro
target:
  - myimage

repos:
  meta-custom:

  bitbake:
    url: "https://git.openembedded.org/bitbake"
    refspec: "1.46"

  openembedded-core:
    url: "https://git.openembedded.org/openembedded-core"
    refspec: dunfell
    layers:
      meta:
  \end{minted}
  \end{block}
\end{frame}

\begin{frame}[fragile]{kas configuration}
  \fontsize{10}{10}\selectfont
  \begin{block}{}
  \begin{minted}{yaml}
  meta-freescale:
    url: "https://github.com/Freescale/meta-freescale"
    refspec: dunfell

  meta-openembedded:
   url: http://git.openembedded.org/meta-openembedded
   refspec: dunfell
   layers:
     meta-oe:
     meta-python:
     meta-networking:
  \end{minted}
  \end{block}
\end{frame}

\begin{frame}[fragile]{Network access}
  Another task when creating a release is to ensure all the code is
  available internally, either on the local build machine or on local
  mirrors.
  \begin{itemize}
  \item Ensure there is no \code{SRCREV = "${AUTOREV}"} in any recipe.
  \item Set \code{BB_GENERATE_MIRROR_TARBALLS = "1"} to generate
    tarballs of the git repositories in \code{DL_DIR}.
  \item Fetch all the source (e.g using \code{bitbake -c fetchall
    <target>}).
  \item Archive \code{DL_DIR}, make the tarballs available internally.
  \item Optionally build once with \code{BB_NO_NETWORK = "1"} to check
    for missing tarballs or remaining \code{AUTOREV}.
  \item Point bitbake to your internal mirrors, using
    \code{PREMIRRORS} or \code{INHERIT += "own-mirrors"} with
    \code{SOURCE_MIRROR_URL}
  \item Build the release, from scratch using
    \code{BB_FETCH_PREMIRRORONLY = "1"}.
  \end{itemize}
\end{frame}

\subsection{Build optimization}

\begin{frame}[fragile]{Sharing the sstate-cache}
  It is possible to share the shared state cache across multiple
  build machines:
  \begin{itemize}
  \item Set up CI or nightly builds.
  \item Use the \code{DL_DIR} to populate the \code{PREMIRRORS}.
  \item Share the sstate-cache (\code{SSTATE_DIR}) over NFS or HTTP.
  \item Setup \code{SSTATE_MIRRORS} to point to that share
  \end{itemize}
  This works well if all the hosts are similar as this influence
  checksums. Containers will help.
\end{frame}

\begin{frame}[fragile]{Cleaning the sstate-cache}
  The sstate-cache is growing over time. It is possible to clean old
  data with:
  \begin{block}{}
    \begin{minted}[fontsize=\footnotesize]{console}
$ ./scripts/sstate-cache-management.sh --remove-duplicated -d \
  --cache-dir=<SSTATE_DIR>
    \end{minted}
  \end{block}
\end{frame}

\subsection{License compliance}

\begin{frame}[fragile]{Listing licenses}
  OpenEmbbedded will generate a manifest of all the licenses of the
  software present on the target image in
  \code{LICENSE_DIRECTORY/IMAGE_NAME/license.manifest}
  \begin{block}{}
    \fontsize{9}{9}\selectfont
    \begin{minted}{console}
PACKAGE NAME: busybox
PACKAGE VERSION: 1.31.1
RECIPE NAME: busybox
LICENSE: GPLv2 & bzip2-1.0.6

PACKAGE NAME: dropbear
PACKAGE VERSION: 2019.78
RECIPE NAME: dropbear
LICENSE: MIT & BSD-3-Clause & BSD-2-Clause & PD
    \end{minted}
  \end{block}
\end{frame}

\begin{frame}[fragile]{Providing license text}
  To include the license text in the root filesystem either:
  \begin{itemize}
  \item Use \code{COPY_LIC_DIRS = "1"} and \code{COPY_LIC_MANIFEST =
    "1"}
  \item or use \code{LICENSE_CREATE_PACKAGE = "1"} to generate
    packages including the license and install the required license
    packages.
  \end{itemize}
\end{frame}

\begin{frame}[fragile]{Providing sources}
  OpenEmbbedded provides the \code{archiver} class to generate
  tarballs of the source code:
  \begin{itemize}
  \item Use \code{INHERIT += "archiver"}
  \item Set the \code{ARCHIVER_MODE} variable, the default is to
    provide patched sources. To provide configured sources:
  \end{itemize}
    \begin{block}{}
      \fontsize{9}{9}\selectfont
      \begin{minted}{bash}
ARCHIVER_MODE[src] = "configured"
      \end{minted}
    \end{block}

\end{frame}

\questionslide

\end{document}
