\documentclass[obeyspaces,spaces,hyphens,dvipsnames]{beamer}
\usepackage[utf8]{inputenc}
\usepackage{minted}

\mode<presentation>
\usetheme{Bootlin}

\newcommand{\codecomment}[1]{{\color{red} \textit{#1}}}

\title{SMP bring up on ARM SoCs}
\authors{Gregory CLEMENT}
\institute{Bootlin}

\usepackage{hyperref}

\begin{document}

\begin{frame}{Embedded Linux Conference 2014}
  \begin{columns}[c]
   \column{0.6\textwidth}
     \huge
     SMP bring up on ARM SoCs\\
     \vspace{1cm}
     \normalsize
     Gregory CLEMENT\\
     {\bf Bootlin}\\
     {\em gregory.clement@bootlin.com}
     \column{0.3\textwidth}
     \includegraphics[width=\textwidth]{ARM-SMP-chip.jpg}
   \end{columns}
\end{frame}

\begin{frame}{Gregory CLEMENT}
  \begin{itemize}
  \item Embedded Linux engineer and trainer at Bootlin since
    2010
    \begin{itemize}
    \item Embedded Linux {\bf development}: kernel and driver
      development, system integration, boot time and power
      consumption optimization, consulting, etc.
    \item Embedded Linux {\bf training}, Linux driver development
      training and Android system development training, with materials
      freely available under a Creative Commons license.
    \item \url{http://bootlin.com}
    \end{itemize}
  \item Contributing to {\bf kernel support for the Armada 370, 375, 38x and
      Armada XP} ARM SoCs from Marvell.
  \item Co-maintainer of mvebu sub-architecture (SoCs from Marvell
    Engineering Business Unit)
  \item Living near {\bf Lyon}, France
  \end{itemize}
\end{frame}

\begin{frame}{Motivation and Overview}
  \begin{itemize}
  \item Motivation
    \begin{itemize}
    \item All the new ARM CPUs are now SMP capable.
    \item Most of the new ARM SoCs actually uses multiple CPUs.
    \item Did SMP bring up for two kinds of ARM SoCs during past year.
    \item Documentation I would have like to find.
    \end{itemize}
  \item Overview
    \begin{itemize}
    \item SMP operations and functions.
    \item Implementation of these functions.
    \item Issues encountered during SMP bring up.
    \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}{SMP on ARM SoC?}
  \begin{itemize}
  \item A symmetric multiprocessor system (SMP) is a multiprocessor
    system with centralized shared memory called main memory (MM)
    operating under a single operating system with two or more
    homogeneous processors. (Wikipedia)
  \item Most of the SMP code is not architecture dependent (in kernel
    directory).
  \item Most of the hardware support is related to the ARM specification.
  \item SoC level: all that is beyond the CPU, mainly initialization
    and power down the CPUs.
  \end{itemize}
\end{frame}

\begin{frame}{Assumptions}
  \begin{itemize}
  \item SoC support is working on one CPU core.
  \item Support is following the last requirements

    {\scriptsize (see ``Your new ARM SoC Linux support check-list”
      from Thomas Petazzoni:
      \url{http://bootlin.com/pub/conferences/2013/elc/arm-soc-checklist/})}
  \item Datasheet is available.
  \item No hardware bug (or at least they are documented).
  \end{itemize}
\end{frame}

\begin{frame}[fragile]{SMP operations}
  \begin{itemize}
  \item Few SMP functions related to the SoC:
  \footnotesize
  \begin{block}{}
    \begin{minted}{c}
struct smp_operations {
	void (*smp_init_cpus)(void);
	void (*smp_prepare_cpus)(unsigned int max_cpus);
	void (*smp_secondary_init)(unsigned int cpu);
	int  (*smp_boot_secondary)(unsigned int cpu,
                                struct task_struct *idle);
[...]
};
    \end{minted}
  \end{block}
 \normalsize
\item Only \code{smp_boot_secondary()} is mandatory, others are optional
  and depend on the need of the SoCs.
\end{itemize}
\end{frame}


\begin{frame}{Role of each SMP operations 1/2}
  \begin{itemize}
  \item\code{smp_init_cpus()}:
    \begin{itemize}
    \item Setup the set of possible CPUs (via \code{cpu_possible()}).
    \item Can be removed if the CPU topology is up to date in the
      device tree.
    \item Called {\bf very early} during the boot process (from
      \code{setup_arch()}).
    \end{itemize}
  \item\code{smp_prepare_cpus()}:
    \begin{itemize}
    \item Enables coherency.
    \item Initializes \code{cpu_possible} map.
    \item Prepares the resources (power, ram, clock...).
    \item Called early during the boot process (before the initcalls
      but after \code{setup_arch()}).
    \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}{Role of each SMP operations 2/2}
  \begin{itemize}
  \item\code{smp_secondary_init()}:
    \begin{itemize}
    \item Supposed to ``perform platform specific initialization of
      the specified CPU''.
    \item Mostly dealing with pen release stuff which seems to be
      wrong. Russell King: {\em The \code{pen_release} stuff is only
        there for platforms where there's no proper way of controlling
        the secondary CPUs except by using a software method.}
    \item Called from \code{secondary_start_kernel()} on the CPU which
      has just been started.
    \end{itemize}
  \item\code{smp_boot_secondary()}:
    \begin{itemize}
    \item Actually boots a secondary CPU identified by the CPU number given
      in parameter.
    \item Called from \code{cpu_up()} on the booting CPU.
    \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}{Booting sequence of the SMP operations}
  \begin{center}
\includegraphics[width=\textwidth]{boot-seq-smp-ops.pdf}
\end{center}
\end{frame}

\begin{frame}[fragile]{Hotplug SMP operations}
  \begin{itemize}
  \item When \code{CONFIG_HOTPLUG_CPU} is selected three more
    functions are added to \code{smp_operations}:
    \footnotesize
  \begin{block}{}
    \begin{minted}{c}
struct smp_operations {
[...]
	int  (*cpu_kill)(unsigned int cpu);
	void (*cpu_die)(unsigned int cpu);
	int  (*cpu_disable)(unsigned int cpu);
};
    \end{minted}
  \end{block}
 \normalsize
\item To support the hotplug features \code{cpu_kill()} and
  \code{cpu_die()} must be provided.
\end{itemize}
\end{frame}


\begin{frame}{Role of each SMP hotplug operations 1/2}
  \begin{itemize}
  \item \code{cpu_kill()}:
    \begin{itemize}
    \item Supposed to do the powering off and/or cutting off clocks
      to the dying CPU.
    \item Need to be synchronized with \code{cpu_die()}.
    \item Needed for kexec (only user of
      \code{platform_can_cpu_hotplug()}, which checks the presence of
      this function).
    \item Called on the thread which is asking for a CPU to be shutdown.
    \end{itemize}
  \item \code{cpu_die()}:
    \begin{itemize}
    \item Supposed to ensure death of the CPU.
    \item Not supposed to return from this function, returns {\bf only}
      if the SoC can't power down the CPU.
    \item Either power down the CPU or at least call \code{cpu_do_idle()}.
    \item Called from the idle thread for the CPU which has been shutdown.
    \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}{Role of each SMP hotplug operations 2/2}
  \begin{itemize}
  \item \code{cpu_disable()}:
    \begin{itemize}
    \item Common ARM code takes care of the shutdown of a CPU: no more
      interrupts can be handled by the kernel after the routine
      returns, including local timers which are shutdown.
    \item By default, the common code allows disabling all CPUs except
      the first one (the boot CPU). This function can be used to change
      this behavior.
    \item Runs on the CPU to be shutdown.
    \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}{SMP related features: nice to have}
    Not mandatory features, but they are usually added in the same time
    that SMP support.
  \begin{itemize}
  \item IRQ affinity: allows to dedicate one or several CPUs to an
    interrupt.
    \begin{itemize}
    \item Part of the IRQ chip driver.
    \item Implemented in the following handler:
      \code{int (*irq_set_affinity)(struct irq_data *data, const struct cpumask *dest, bool force)}
    \end{itemize}
  \item Local timer: timer dedicated to a CPU, thanks to this, no more
    need to broadcast a tick to all the CPUs.
    \begin{itemize}
    \item No more a specific API to register it since 3.12.
    \item Still needs to use the \code{percpu} API and the notifiers
      to implement it.
    \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}{Use of the PSCI for SMP}
    PSCI stands for Power State Coordination Interface.
  \begin{itemize}
  \item Needed for virtualization.
  \item Used to coordinate OSes and hypervisors.
  \item Only solution to deal with secure mode.
  \item Provides functions used for SMP bring-up such as \code{CPU_ON}
    or \code{CPU_OFF}.
  \item Function provided by the firmware, the SMP operations are no
    more used.
  \item ID of these functions provided through the device tree.
  \end{itemize}
\end{frame}

\begin{frame}{Implementing the SMP operations 1/2}
  \begin{itemize}
  \item \code{smp_init_cpus()}
    \begin{itemize}
    \item Typically if an SCU unit is available the number of CPUs is
      retrieved using \code{scu_get_core_count()}.
    \item And then the CPU possible map is built
      using\code{ set_cpu_possible()}.
    \item However as the cpu topology is supposed to be described in
      the device tree, the whole map is already built in the function
      \code{arm_dt_init_cpu_maps}.
  \end{itemize}
  \item\code{smp_prepare_cpus()}
    \begin{itemize}
%MVEBU: set_secondary_cpus_clock(); flush_cache_all();  set_cpu_coherent
% exynos: 	scu_enable(scu_base_addr()); call_firmware_op(set_cpu_boot_addr, phys_cpu, boot_addr))
% MVENBU A9: nothing except quirk
% socfpga: scu_enable(socfpga_scu_base_addr);
% sunxi/ iomap
% hsi scu_enable / iomap
% power domain / scu enable
% qcom:  Synchronise with the boot thread.: spin_lock(&boot_lock);; 	spin_unlock(&boot_lock);

    \item If SCU is present then the coherency is enabled by using
      \code{scu_enable()}.
    \item Most of the time resources are allocated and the registers
      are mapped using \code{ioremap()} and its friends.
    \item The way to enable (if needed) the power and the clocks is
      highly platform specific and does not involve common code.
  \end{itemize}
  \end{itemize}
\end{frame}


\begin{frame}{Implementing the SMP operations 2/2}
  \begin{itemize}
  \item\code{smp_secondary_init()}
%exynos: pen_release
%tegea cpumask_set_cpu
% sti: pen_release and trace_hardirqs_off
% prima: pen_release
    \begin{itemize}
    \item No common code used here but a lot of copy/paste of the
      \code{pen_release} mechanism.
    \end{itemize}
  \item\code{smp_boot_secondary()}
% sunxi: poweroff deassert reset
% socfpga release CPU from reset
% mvebu release CPU from reset
%exynos CPU have already started so release them from pen_release
    \begin{itemize}
    \item Releases the CPU from reset by writing on a power management
      register.
    \item Some time writes ``magic'' pattern read by the {\em bootrom} which
      will release the CPU from reset.
    \item Uses the \code{pen_release} if the secondary CPU has
      already been released from reset.
    \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}{Implementing the SMP hotplug operations 1/2}
  \begin{itemize}
  \item \code{cpu_kill()}
% imx: wait for the cpu died by polling his starting address set to 0
% shmobile-scu ans -apmu: same
% hisi same and then
% hisi: write wifi mask, disable core, reset and specific case for cpu 2 and 3
% tegra: xai reset state (clk) then disable the clock

    \begin{itemize}
    \item Most of the platforms check that the CPU is going to die, by
      reading a register of the SoC wrote by the dying CPU.
    \item Some platforms really check that the dying CPU went in the
      reset state.
    \item Then it actually powers down the CPU if the dying CPU has not
      done it by itself.
  \end{itemize}
  \item \code{cpu_die()}
% tegra go to shutdown
% other cpu_idle
% zynq, imx, hisi got to low_power before cpu_idle
% shmobile: low_power then kind of idle (call wfi)
    \begin{itemize}
    \item On most of the platforms, the dying CPU writes a register of
      the SoC, usually by setting to 0 the jump address of the CPU.
    \item Some platforms switch in low power mode.
    \item The others just call \code{cpu_do_idle()}.
    \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}{Implementing the SMP hotplug operations 2/2}
  \begin{itemize}
  \item \code{cpu_disable()}
    \begin{itemize}
    \item Only implemented if the behavior of the platforms is
      different of the default (ie all CPUs except the first one can
      be killed).
    \item Currently only used by shmobile where any CPU can be
      killed.
    \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}{Registering the SMP operations}
  \begin{itemize}
  \item Used to be part of the machine description. The
    \code{smp_operations} structure of the SoC was associated to the
    \code{.smp} field using the \code{smp_ops()} helper.
  \item From 3.16, they should be directly registered using the
    \code{CPU_METHOD_OF_DECLARE()} helper.
  \item Then the operation will be fetched from the device tree.
  \end{itemize}
\end{frame}


\begin{frame}{Implementing a local timer}
  \begin{itemize}
  \item Should be part of a clock source driver in
    \code{drivers/clocksource/}.
  \item Each local timer will be called during the CPU bring up
    (\code{CPU_STARTING}), so the setup of the timer must be
    associated to a notifier using \code{register_cpu_notifier()}.
  \item The timer must be stopped when the CPU is shutting
    down (\code{CPU_DYING}), so this function will be also associated
    to the notifier.
  \item The interrupt has to be registered with
    \code{request_percpu_irq()}.
  \item And the \code{clock_event_device} has to be allocated with
    \code{alloc_percpu()}.
  \end{itemize}
\end{frame}

\begin{frame}{Implementing the IRQ affinity}
  \begin{itemize}
  \item Should be part of an IRQ chip driver in
    \code{drivers/irqchip/}.
  \item The interrupts associated to \code{(struct irq_data *data}
    received should be treated only by the CPU represented by
    \code{struct cpumask *dest}.
  \item The cpumask can be manipulated using the helpers located in
    \code{include/linux/cpumask.h}.
  \item It is valid, depending of the hardware capabilities, to set
    the IRQ affinity of only of a subset of the group of the CPUs.
  \end{itemize}
\end{frame}

\begin{frame}[fragile]{SMP and device tree: CPU topology - "cpus"}
  From \code{Documentation/devicetree/bindings/arm/cpus.txt}: {\em The
    device tree allows to describe the layout of CPUs in a system
    through the "cpus" node, which in turn contains a number of
    subnodes (ie "cpu) defining properties for every cpu.}

  The \code{cpus} nodes are composed of:
  \begin{itemize}
  \item \code{#address-cells}: 1 for 32-bit and 2 for 64-bit
  \item \code{#size-cells}: Always 0
  \item \code{enable-method}:
    \begin{itemize}
    \item Optional for ARM 32-bit, but highly recommended for new
      implementations from 3.16.
    \item For ARM 32 bits, the string is the one used for registering
      the SMP operations with \code{CPU_METHOD_OF_DECLARE()}.
    \item Required for ARM v8 and must be \code{"spin-table"} or
      \code{"psci"}
    \item Possible to use \code{"psci"} for ARM 32-bit too.
    \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}[fragile]{SMP and device tree: CPU bindings - "cpu"}
  The \code{cpu} subnodes are composed of:
  \begin{itemize}
  \item \code{device_type}: Always \code{"cpu"}.
  \item \code{reg}: Related to the MPIDR, more or less the ID of the
    CPU.
  \item \code{compatible}: The CPU type for example:
    \code{"arm,cortex-a9"} or \code{"marvell,pj4b"}.
  \item \code{enable-method}: Supposed to be defined at the CPU level
    but can be inherit from the CPUs level.
  \end{itemize}
\end{frame}

\begin{frame}[fragile]{SMP and device tree: CPU bindings - Example}
  \begin{block}{}
    \begin{minted}{perl}
cpus {
        #address-cells = <1>;
        #size-cells = <0>;
        enable-method = "marvell,armada-380-smp";

        cpu@0 {
                device_type = "cpu";
                compatible = "arm,cortex-a9";
                reg = <0>;
        };
        cpu@1 {
                device_type = "cpu";
                compatible = "arm,cortex-a9";
                reg = <1>;
        };
};
    \end{minted}
  \end{block}
\end{frame}

\begin{frame}[fragile]{SMP and device tree: PSCI}
  Fully defined in
  \code{Documentation/devicetree/bindings/arm/psci.txt}.

  It is more that just supporting SMP.
  \begin{itemize}
  \item \code{compatible}: Always \code{"arm,psci"}
  \item \code{device_type}: Always \code{"cpu"}.
  \item \code{method}: \code{"smc"} or \code{"hvc"} depending on the
    method used to call the PSCI firmware.
  \item \code{cpu_off}: Function ID for \code{CPU_OFF} operation.
  \item \code{cpu_on}: Function ID for \code{CPU_ON} operation.
  \item There are other functions defined but they are not used as SMP
    operations.
  \end{itemize}
\end{frame}


\begin{frame}[fragile]{SMP and device tree: PSCI - Example}
\scriptsize
  \begin{block}{}
    \begin{minted}{perl}
psci {
    compatible	= "arm,psci";
    method		= "smc";
    cpu_suspend	= <0x84000002>;
    cpu_off		= <0x84000004>;
    cpu_on		= <0x84000006>;
};
cpus {
        #address-cells = <1>;
        #size-cells = <0>;

        cpu@0 {
                compatible = "arm,cortex-a15";
                device_type = "cpu";
                reg = <0>;
        };
[...]
        cpu@3 {
                compatible = "arm,cortex-a15";
                device_type = "cpu";
                reg = <3>;
        };
};
    \end{minted}
  \end{block}
\end{frame}

\begin{frame}{Issues encountered during SMP Bring-up: coherency}
  \begin{itemize}
  \item {\em Symptom}: The kernel booted on the CPU 0 but the other
    CPUs failed to boot.
    \begin{itemize}
    \item On the secondary CPUs the L1 cache were corrupted.
    \item The L1 cache needed to be invalidated before starting a
      secondary CPU.
    \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}{Issues encountered during SMP Bring-up: coherency}
  \begin{itemize}
  \item {\em Symptom}: The kernel crashed pretty early during the SMP
    initialization.
    \begin{itemize}
    \item The registers controlling the coherency were not mapped yet.
    \item The initialization of the resources associated to the
      coherency were done in an initcall, however the coherency for
      SMP is called before the initcalls.
    \item The solution was to move this initialization in the
      \code{.init_time} handler called directly from
      \code{start_kernel}.
    \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}{Issues encountered during SMP Bring-up: timers}
  \begin{itemize}
  \item {\em Symptom}: The kernel booted but then nothing happened in
    userspace
    \begin{itemize}
    \item No timer ticked at all so no schedule happened.
    \item Here the problem was that the field \code{irq} of the timer
      was not filled.
    \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}{Issues encountered during SMP Bring-up: timers}
  \begin{itemize}
  \item {\em Symptom}: Sometimes the kernel hanged, just after
    booting, sometimes later when we have already switched to
    userspace.
    \begin{itemize}
    \item There was no local timer neither broadcast timer: no tick on
      the secondary CPUs. As soon as the task ran on a secondary
      CPU, then it hanged, as no schedule happened.
    \item The reason in our case was that \code{CONFIG_HAVE_TWD} was
      not selected.
    \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}
  \begin{center}
  \Huge
  Questions?\\
  \vspace{1.5cm}
  \huge
  Gregory CLEMENT\\
  \large
  \vspace{0.5cm}
  \code{gregory.clement@bootlin.com}
  \vspace{0.5cm}
  \newline Slides under CC-BY-SA 3.0\\
  \scriptsize
  \url{http://bootlin.com/pub/conferences/2014/elc/clement-smp-bring-up-on-arm-soc}
  \end{center}
\end{frame}

\end{document}
