\documentclass[aspectratio=169,obeyspaces,spaces,hyphens,dvipsnames]{beamer}
\usepackage[utf8]{inputenc}
\usepackage{lmodern}% http://ctan.org/pkg/lm
\usepackage{minted}
\usepackage{hyperref}
\usepackage{xcolor}
\usepackage[normalem]{ulem}
\usepackage[T1]{fontenc}

\mode<presentation>
\usetheme{Bootlin}

\def\signed #1{{\leavevmode\unskip\nobreak\hfil\penalty50\hskip2em
  \hbox{}\nobreak\hfil(#1)
  \parfillskip=0pt \finalhyphendemerits=0 \endgraf}}

\newsavebox\mybox
\newenvironment{aquote}[1]
  {\savebox\mybox{#1}\begin{quotation}}
  {\signed{\usebox\mybox}\end{quotation}}

\title{Linux Power Management features, their relationships and interactions}
\authors{Théo Lebrun}
\email{theo.lebrun@bootlin.com}
\slidesurl{https://bootlin.com/pub/conferences/}
\institute{Bootlin}
\conference{Embedded Linux Conference Europe 2024}

\begin{document}

\addtocontents{toc}{\protect\setcounter{tocdepth}{-1}}
\section{Linux Power Management features, their relationships and interactions}
\addtocontents{toc}{\protect\setcounter{tocdepth}{2}}

\begin{frame}{Théo Lebrun}
  \begin{itemize}
    \item Embedded Linux engineer at Bootlin
    \begin{itemize}
      \item Embedded Linux {\bf expertise}
      \item {\bf Development}, consulting and training
      \item Strong open-source focus
    \end{itemize}
    \item Linux kernel device driver developer
    \begin{itemize}
      \item Suspend-to-RAM support for a TI SoC
      \item Upstreaming of Mobileye SoCs
    \end{itemize}
    \item Open-source contributor (kernel, PipeWire ecosystem, etc.)
    \item Current maintainer of \url{https://elixir.bootlin.com}
    \item Living in {\bf Lyon}, France
    \item \code{theo.lebrun@bootlin.com}
  \end{itemize}
  {\small \url{https://bootlin.com/company/staff/theo-lebrun/}}
\end{frame}

\begin{frame}{Table of Contents}
  \begin{enumerate}
    \item {\bf System-wide suspend}
      \begin{itemize}
        \item How different modes work?
        \item Device callbacks involved
        \item Suspend modes: their differences and tradeoffs
        \item Case study: s2idle \& \code{CLOCK_MONOTONIC} quirk
      \end{itemize}
    \bigskip
    \item {\bf Runtime Power Management}
      \begin{itemize}
        \item The overall concept
        \item Features of runtime PM
        \item Interactions with system-wide suspend, with case study
      \end{itemize}
  \end{enumerate}
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\subsection{System-wide\\
suspend states}

\begin{frame}{System-wide suspend}
  \begin{itemize}
    \item First, stop the world by freezing all tasks.

% TODO: does it disable all IRQs or is it part of the below section?

    \item Then suspend individual devices in four steps:
      \begin{enumerate}
        \item Prepare,
        \item Suspend,
        \item Late suspend,
        \item No-IRQ suspend.
      \end{enumerate}

    \item Finally, go into {\em \guillemotleft{} suspend \guillemotright{}}.

    \pause
    \medskip

    \item Things to think about:
      \begin{itemize}
        \item Desired {\bf suspend type}: suspend-to-idle, standby, suspend-to-RAM, hibernation;
        \item {\bf Wakeup source};
        \item Targeted individual {\bf device states} during suspend;
        \item Entry \& exit {\bf latency goals}.
      \end{itemize}
    \item Doc: \kdochtml{admin-guide/pm/sleep-states} \& \kdochtml{admin-guide/pm/suspend-flows}
    \item Code: \kfile{kernel/power/suspend.c}
  \end{itemize}
\end{frame}

\begin{frame}[fragile]{System-wide suspend: entering suspend}
  \begin{block}{}
    \fontsize{10}{10}\selectfont
    \begin{minted}{console}
$ cat /sys/power/mem_sleep
[s2idle] deep
$ echo deep > /sys/power/mem_sleep
$ echo mem > /sys/power/state

Unable to handle kernel paging request at virtual address ...
(if you are lucky)
    \end{minted}
  \end{block}

  \pause

  \begin{block}{}
    \fontsize{10}{10}\selectfont
    \begin{minted}{console}
$ # Need debugging help?
$ echo 0 > /sys/module/printk/parameters/console_suspend
$ echo 8 > /proc/sys/kernel/printk
$ echo 1 > /sys/power/pm_print_times     # ifdef CONFIG_PM_SLEEP_DEBUG
$ echo 1 > /sys/power/pm_debug_messages  # same
    \end{minted}
  \end{block}

\end{frame}

\begin{frame}[fragile]{System-wide suspend: device PM callbacks}
  \begin{itemize}
    \item See \kstruct{dev_pm_ops} \& doc \kdochtml{driver-api/pm/devices}.
  \end{itemize}

  \begin{columns}
    \column{0.65\textwidth}
      \begin{block}{\href{https://elixir.bootlin.com/linux/v6.10.10/source/include/linux/pm.h\#L62-L310}{include/linux/pm.h}}
        \fontsize{10}{10}\selectfont
        \begin{minted}{c}
struct dev_pm_ops {
    int (*prepare)(struct device *dev);
    int (*suspend)(struct device *dev);
    int (*suspend_late)(struct device *dev);
    int (*suspend_noirq)(struct device *dev);

    int (*resume_noirq)(struct device *dev);
    int (*resume_early)(struct device *dev);
    int (*resume)(struct device *dev);
    void (*complete)(struct device *dev);

    /* and more (hibernation and runtime PM)... */
};
        \end{minted}
      \end{block}

    \pause

    \column{0.35\textwidth}
      \begin{block}{}
        \fontsize{10}{10}\selectfont
        \begin{minted}{python}
# Pseudocode (ie Python).

# Each function is called,
# one after the other.
for dev in devices_topdown:
    prepare(dev)
for dev in devices_downtop:
    suspend(dev)
for dev in devices_downtop:
    suspend_late(dev)
for dev in devices_downtop:
    suspend_noirq(dev)
        \end{minted}
      \end{block}
  \end{columns}

% TODO: do I want to expand on the above? Give an example? Give overview of what
% is usually done? Talk about expectations of subsystems. Give example of PCI
% device at suspend_noirq with GPIO stuff.

\end{frame}

\begin{frame}{System-wide suspend: device PM callbacks}
  \begin{itemize}
    \item See \kstruct{dev_pm_ops} \& doc \kdochtml{driver-api/pm/devices}.
    \item \code{->prepare()}: do not register new children devices.
    \pause \item \code{->suspend()}: please stop doing I/O.
    \pause \item \code{->suspend_late()}: please stop.
    \pause \item \code{->suspend_noirq()}: please.
    \begin{itemize}
      \item Additional guarantee: IRQ handlers will not be called.
    \end{itemize}

    \pause\medskip

    \item Any of the \code{->suspend*()} callbacks
      {\em can/might/should/must}, depending on subsystem:
    \begin{enumerate}
      \item save device state, for later restore, and,
      \item put individual device into low-power state.
    \end{enumerate}

    \medskip

    \item Summary: {\bf behavior is device specific}. \\
      No guarantees about device states are provided, and no information
      is {\em exported} (apart from potential error codes).
  \end{itemize}
\end{frame}

\begin{frame}[fragile]{System-wide suspend: device PM callbacks}
  \begin{itemize}
    \item Two GPIO controllers with implementations at different stages.
    \item Implication: \code{pinctrl-nomadik} pins must be configured at
      \code{->suspend()} or before.
  \end{itemize}

  \begin{block}{\href{https://elixir.bootlin.com/linux/v6.10.10/source/drivers/pinctrl/renesas/pinctrl-rzg2l.c\#L2730-L2732}{\codelink{drivers/pinctrl/renesas/pinctrl-rzg2l.c}}}
    \fontsize{10}{10}\selectfont
    \begin{minted}{c}
static const struct dev_pm_ops rzg2l_pinctrl_pm_ops = {
    NOIRQ_SYSTEM_SLEEP_PM_OPS(rzg2l_pinctrl_suspend_noirq,
                              rzg2l_pinctrl_resume_noirq)
};
    \end{minted}
  \end{block}

  \begin{block}{\href{https://elixir.bootlin.com/linux/v6.10.10/source/drivers/pinctrl/nomadik/pinctrl-nomadik.c\#L1268-L1270}{\codelink{drivers/pinctrl/nomadik/pinctrl-nomadik.c}}}
    \fontsize{10}{10}\selectfont
    \begin{minted}{c}
static SIMPLE_DEV_PM_OPS(nmk_pinctrl_pm_ops,
                         nmk_pinctrl_suspend,
                         nmk_pinctrl_resume);
    \end{minted}
  \end{block}
\end{frame}

\begin{frame}{System-wide suspend: device PM callbacks}
  \begin{itemize}
    \item Careful! Moving everything to \code{->suspend_noirq()} is
      {\bf not} the solution.
    \pause
    \item Do you need interrupts for your suspend process?
    \item Do the actions you provide, eg \kfunc{pinctrl_select_state}, require
      interrupts to work?
    \item Goto 1: recursively think about your dependencies. \\
      They must work as long as you do.
  \end{itemize}
\end{frame}

\begin{frame}{System-wide suspend: device PM callbacks}
    \centering\includegraphics[height=1.0\textheight]{tleb-pm-i2c-gpio-expander.png}
\end{frame}

% TODO: give various examples of subsystems / devices and the common behaviors.
% Maybe it should wait until we talk about runtime PM

\begin{frame}[fragile]{System-wide suspend: suspend-to-idle mode}
  \begin{itemize}
    \item Always available, if \code{CONFIG_SUSPEND=y}.
    \item Piggyback on platforms' idle loop support.
  \end{itemize}

  \pause

  \begin{block}{\href{https://elixir.bootlin.com/linux/v6.10.6/source/kernel/power/suspend.c\#L90-L124}{\codelink{kernel/power/suspend.c}}}
    \fontsize{10}{10}\selectfont
    \begin{minted}{c}
static void s2idle_enter(void) /* abbreviated! */
{
    s2idle_state = S2IDLE_STATE_ENTER;
    /* Push all the CPUs into the idle loop. */
    wake_up_all_idle_cpus();
    /* Put current CPU in idle as well, waiting for wakeup event. */
    swait_event_exclusive(s2idle_wait_head, s2idle_state == S2IDLE_STATE_WAKE);
    /* Wake up all CPUs for them to restore their state. */
    wake_up_all_idle_cpus();
    s2idle_state = S2IDLE_STATE_NONE;
}
    \end{minted}
  \end{block}

  \begin{itemize}
    \item \code{s2idle_state} is set to \code{S2IDLE_STATE_WAKE} inside
      \href{https://elixir.bootlin.com/linux/v6.10.6/C/ident/pm_system_wakeup
      }{interrupt handlers}.
  \end{itemize}
\end{frame}

\begin{frame}[fragile]{System-wide suspend: platform-provided modes (standby, S2R)}
  \begin{itemize}
    \item Next states are calling into platform code.
    \item States might not be supported!
  \end{itemize}

  \begin{block}{\kline{v6.10.10}{kernel/power/suspend.c}{L90-L124}}
    \fontsize{10}{10}\selectfont
    \begin{minted}{c}
typedef int __bitwise suspend_state_t;
#define PM_SUSPEND_ON       ((__force suspend_state_t) 0)
#define PM_SUSPEND_TO_IDLE  ((__force suspend_state_t) 1)
#define PM_SUSPEND_STANDBY  ((__force suspend_state_t) 2) /* Standby */
#define PM_SUSPEND_MEM      ((__force suspend_state_t) 3) /* Suspend-to-RAM */
#define PM_SUSPEND_MIN      PM_SUSPEND_TO_IDLE
#define PM_SUSPEND_MAX      ((__force suspend_state_t) 4)

struct platform_suspend_ops {
    int (*valid)(suspend_state_t state);
    int (*enter)(suspend_state_t state);
    /* ... */
};
extern void suspend_set_ops(const struct platform_suspend_ops *ops);
    \end{minted}
  \end{block}
\end{frame}

  % \begin{itemize}

  %   \pause \item Behavior? It stops all but one CPU. It disables IRQs.

  %   \pause \item Else, no one knows. It might:
  %     \begin{itemize}
  %     \item do an idle loop on the remaining CPU,
  %     \item stop none/some/all CPU caches,
  %     \item stop none/some clocks,
  %     \item put DDR into self-refresh (though that would be unexpected),
  %     \item etc.
  %     \end{itemize}

    % arch/arm/mach-at91/pm.c: TODO AT91_PM_STANDBY / AT91_PM_ULP0 / AT91_PM_ULP0_FAST / AT91_PM_ULP1 / AT91_PM_BACKUP
    % arch/arm/mach-imx/pm-imx5.c: TODO WAIT_UNCLOCKED_POWER_OFF
    % arch/arm/mach-imx/pm-imx6.c: TODO imx6q_pm_enter
    % arch/arm/mach-milbeaut/platsmp.c: asm("wfi");
    % arch/arm/mach-mvebu/kirkwood-pm.c: TODO kirkwood_low_power
    % arch/arm/mach-mvebu/pm.c: cpu_do_idle
    % arch/arm/mach-omap2/pm33xx-core.c: TODO pm_ops->soc_suspend(suspend_state, am33xx_do_wfi_sram, suspend_wfi_flags);
    % arch/arm/mach-pxa/pm.c: TODO pxa27x_cpu_pm_enter / pxa3xx_cpu_pm_enter
    % arch/arm/mach-ux500/pm.c: cpu_do_idle
    % arch/mips/loongson2ef/common/pm.c: TODO loongson_pm_enter
    % arch/powerpc/platforms/44x/cpm.c: TODO cpm_idle_sleep
    % arch/powerpc/platforms/52xx/mpc52xx_pm.c: TODO mpc52xx_pm_enter
    % arch/powerpc/platforms/83xx/suspend.c: TODO mpc83xx_suspend_enter
    % arch/powerpc/sysdev/fsl_pmc.c: use cpu feature for sleep (pmc_suspend_enter)
    % drivers/acpi/sleep.c: ACPI_STATE_S1
    % drivers/soc/bcm/brcmstb/pm/pm-mips.c: S3 for deep, S2 for standby (brcmstb_pm_standby)

    % TODO: note that regulators have builtin mechanism to act differently based
    % on suspend targets! Done at ->suspend(). Undocumented OF props.


    % TODO: talk about pm_message_t in include/linux/pm.h

  %   \pause

  %   \item There is no contract signed by platform drivers. They are free to do
  %     whatever they want. Take \code{arm/mach-at91} as example: standby can be \href{
  %     https://elixir.bootlin.com/linux/v6.10.6/source/arch/arm/mach-at91/pm.c\#L1649
  %     }{configured} using a bootargs parameter.
  % \end{itemize}

\begin{frame}{System-wide suspend: platform-provided modes (standby, S2R)}
  \begin{itemize}
    \item Expected behavior of standby and suspend-to-RAM? {\em No one knows.}
    \item S2R {\em should} lower the memory frequency and put it in self-refresh.

    \pause \medskip

    \item Else?
    \begin{itemize}
      \item Standby {\em could} be implemented using an idle loop (\code{cpu_do_idle()}, WFI).
      \item Whole SoC {\em could} be turned off.
      \item None/some/all CPU caches {\em could} be stopped.
      \item None/some clocks {\em could} be stopped.
      \item Few drivers customize their behavior using the \ksym{pm_suspend_target_state} global.
      \item The regulator subsystem exposes OF properties for picking suspend state:
        \href{https://elixir.bootlin.com/linux/v6.10.10/source/drivers/regulator/of_regulator.c\#L18-L22}{\codelink{regulator-state-*}}.
    \end{itemize}

    \medskip

    \item Summary: {\bf behavior is platform specific}.

  \end{itemize}
\end{frame}

\begin{frame}{System-wide suspend: platform-provided modes examples}
  \begin{itemize}
    \item S2R often implies code running from SRAM.
      See \href{https://elixir.bootlin.com/linux/v6.10.10/source/arch/arm/mach-mvebu/pm.c\#L67}{arm/mach-mvebu} or \href{https://elixir.bootlin.com/linux/v6.10.10/source/arch/arm/mach-at91/pm.c\#L589}{arm/mach-at91}
      for examples fully handled inside Linux.

    \pause \item PSCI: look into \kfile{drivers/firmware/psci/psci.c}.
      No standby support, only S2R. Offloaded to firmware with a
      \code{PSCI_1_0_FN64_SYSTEM_SUSPEND} call.

    \pause \item \href{https://elixir.bootlin.com/linux/v6.10.10/source/arch/arm/mach-at91/pm.c}{arm/mach-at91}
      has 5 different suspend modes: \\
      from \code{AT91_PM_STANDBY} (WFI + reduce DRAM power) \\
      to \code{AT91_PM_BACKUP} (SoC off + DDR self-refresh + many clocks disabled). \\
      Standby \& S2R can be configured to any of those using a module parameter:
      \code{atmel.pm_modes=ulp0,backup}.
  \end{itemize}
\end{frame}

\begin{frame}{System-wide suspend: hibernation}
  \begin{itemize}
    \item The most {\em efficient} mode: shutdown!
    \item Beforehand, save all state to disk.
    \item Hibernation is not shutdown:
      \begin{itemize}
      \item Some peripherals might be configured as wakeup sources.
      \item It is useful if userspace takes a long time to initialize at boot.
      \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}{System-wide suspend: tradeoff}
    \centering\includegraphics{tleb-pm-suspend-tradeoff.drawio.pdf}
\end{frame}

\begin{frame}{System-wide suspend: s2idle \& \code{CLOCK_MONOTONIC} quirk}
  \begin{itemize}
    \item What is the expected behavior of \\
      \code{clock_gettime(CLOCK_MONOTONIC, tp)} \\
      across suspend?
      \begin{enumerate}
      \item It should continue ticking,
      \item It should be stopped.
      \item It depends on the suspend type.
      \end{enumerate}
  \end{itemize}
\end{frame}

\begin{frame}[fragile]{System-wide suspend: s2idle \& \code{CLOCK_MONOTONIC} quirk}
  \begin{block}{}
    \fontsize{10}{10}\selectfont
    \begin{minted}{text}
$ man clock_gettime.2
...
 CLOCK_MONOTONIC
    A nonsettable system-wide clock that represents monotonic time since—
    as  described  by  POSIX—"some  unspecified  point  in the past".  On
    Linux, that point corresponds to the number of seconds that the  sys‐
    tem has been running since it was booted.

    The  CLOCK_MONOTONIC  clock is not affected by discontinuous jumps in
    the system time (e.g., if the system administrator  manually  changes
    the  clock),  but  is  affected by frequency adjustments.  This clock
    does not count time that the system is suspended.  All  CLOCK_MONOTO‐
    NIC  variants  guarantee  that the time returned by consecutive calls
    will not go backwards, but successive calls may—depending on the  ar‐
    chitecture—return identical (not-increased) time values.
...
    \end{minted}
  \end{block}
\end{frame}

\begin{frame}{System-wide suspend: s2idle \& \code{CLOCK_MONOTONIC} quirk}
  \begin{itemize}
    \item For a working s2idle, interrupts must be kept enabled (as wakeup source).
    \item Interrupt handlers make one assumption: the timekeeping subsystem is running.
    \item \code{CLOCK_MONOTONIC} is driven by the timekeeping subsystem.
    \item $\implies$ In s2idle, the kernel breaks its promise.
  \end{itemize}
\end{frame}

\begin{frame}[fragile]{System-wide suspend: s2idle \& \code{CLOCK_MONOTONIC} quirk}
  \begin{itemize}
    \item Tip: use tiny tool \href{https://github.com/tleb/clkdump}{clkdump} to dump clocks.
    \item It prints all clocks then \code{sleep(1)}, and loop.
    \item Guess when the s2idle happened:
  \end{itemize}

  \begin{block}{}
    \fontsize{10}{10}\selectfont
    \begin{minted}{console}
$ ./clkdump | grep 'MONOTONIC\s'  # output is abbreviated
Sat, 1 Jan 2000 00:35:10 GMT    CLOCK_MONOTONIC    1.000084 s
Sat, 1 Jan 2000 00:35:11 GMT    CLOCK_MONOTONIC    1.000083 s
Sat, 1 Jan 2000 00:35:12 GMT    CLOCK_MONOTONIC    1.000080 s
Sat, 1 Jan 2000 00:35:55 GMT    CLOCK_MONOTONIC   43.095237 s
Sat, 1 Jan 2000 00:35:56 GMT    CLOCK_MONOTONIC    1.000138 s
Sat, 1 Jan 2000 00:35:57 GMT    CLOCK_MONOTONIC    1.000097 s
Sat, 1 Jan 2000 00:35:58 GMT    CLOCK_MONOTONIC    1.000128 s
    \end{minted}
  \end{block}
\end{frame}

\begin{frame}{System-wide suspend: s2idle \& \code{CLOCK_MONOTONIC} quirk}
  \begin{itemize}
    \item {\bf Except} if the kernel offloads the idle loop to a cpuidle device,
       that can enter s2idle with interrupts disabled. In that case, timekeeping
       is suspended then resumed and \code{CLOCK_MONOTONIC} behaves as expected.
    \item The code path is completely different inbetween s2idle and s2idle + cpuidle.
    \item {\bf Most platforms are safe.} Try disabling your cpuidle driver!
  \end{itemize}
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\subsection{Runtime Power Management\\ {\em (pm\_runtime)}}

\begin{frame}[fragile]{Runtime PM}
  \begin{columns}
    \column{0.35\textwidth}
      \begin{itemize}
        \item Individual device suspend and resume
        \item Doc: \kdochtml{power/runtime_pm}
      \end{itemize}

    \column{0.65\textwidth}
      \begin{block}{\href{https://elixir.bootlin.com/linux/v6.10.10/source/include/linux/pm.h\#L62-L310}{\codelink{include/linux/pm.h}}}
        \fontsize{10}{10}\selectfont
        \begin{minted}{c}
struct dev_pm_ops {
    /* Device is active but not needed anymore. */
    int (*runtime_suspend)(struct device *dev);

    /* Device is suspended but needed. */
    int (*runtime_resume)(struct device *dev);

    /* ... */
};
        \end{minted}
      \end{block}
  \end{columns}
\end{frame}

\begin{frame}[fragile]{Runtime PM}

  \begin{columns}
    \column{0.45\textwidth}

      \begin{itemize}
        \item Devices don't suspend \& resume themselves manually.
        \item Think of the device model as a tree of devices.
        \item Device users touch a usage reference counter.
        % \item Think of the device model as a tree of devices.
      \end{itemize}

  \column{0.55\textwidth}
  \begin{block}{}
    \fontsize{10}{10}\selectfont
    \begin{minted}{c}
/* Pseudocode for mental model! */

void pm_runtime_get(struct device *dev) {
    dev.power.usage_count++;
    if (dev->parent)
        pm_runtime_get(dev->parent);
    if (dev->power.usage_count == 1)
        runtime_resume(dev);
}

void pm_runtime_put(struct device *dev) {
    dev.power.usage_count--;
    if (dev.power.usage_count == 0)
        runtime_suspend(dev);
    if (dev->parent)
        pm_runtime_put(dev->parent);
}
    \end{minted}
  \end{block}

  \end{columns}
\end{frame}

\begin{frame}{Runtime PM: features}
  \begin{itemize}
    \item {\bf Kind of!} This code is {\em slightly} oversimplified.

    \pause\medskip

    \item What happens when calling \code{pm_runtime_get|put()} inside an IRQ?
    \begin{itemize}
      \item \code{->runtime_suspend|resume()} is marked IRQ safe or,
      \item The call {\bf will be done} async (put request into workqueue).
      \item See \kfunc{pm_runtime_irq_safe} and \ksym{RPM_ASYNC}.
    \end{itemize}

    \pause\medskip

    \item Devices can be disabled.
    \begin{itemize}
      \item \code{dev->power.disable_depth}, yet another refcount.
      \item Devices' default state is disabled (refcount=1).
      \item Disabling a device runtime PM does not force suspend it! \\
        You can disable a device while it is active, and it will stay put.
      \item See \kfunc{pm_runtime_enable} and \kfunc{pm_runtime_disable}.
    \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}{Runtime PM: features (bis)}
  \begin{itemize}
    \pause

    \item Devices can be allowed/forbidden from runtime PM.
    \begin{itemize}
      \item \code{dev->power.runtime_auto} boolean.
      \item This is different from disabling!
      \item Default state is \code{true}.
      \item See \kfunc{pm_runtime_allow} and \kfunc{pm_runtime_forbid}.
    \end{itemize}

    \pause\medskip

    \item A device can ask to ignore its \code{->runtime_suspend|resume()} callbacks.
    \begin{itemize}
      \item You are a minor, ie your parent (device) handles PM for you.
      \item See \kfunc{pm_runtime_no_callbacks}.
    \end{itemize}

    \pause\medskip

    \item {\bf Autosuspend!}
    \begin{itemize}
      \item Don't suspend as soon as \code{dev->power.usage_count == 0}, wait a bit.
      \item Think storage device that you do not want to toggle on/off all the time.
      \item See \kfunc{pm_runtime_use_autosuspend} and
        \kfunc{pm_runtime_set_autosuspend_delay}.
      \item Userspace might play with it: \href{
        https://elixir.bootlin.com/linux/v6.10.7/source/Documentation/ABI/testing/sysfs-devices-power\#L186
        }{\codelink{/sys/devices/.../power/autosuspend_delay_ms}}.
    \end{itemize}

  \end{itemize}
\end{frame}

\begin{frame}{Runtime PM vs system-wide suspend}
  \begin{itemize}
    \item Implicit \code{pm_runtime_disable()} before suspend-late.
    \item {\em Almost.} Standard \code{pm_runtime_disable()} wakes up the
      device if there is a resume request pending.
    \pause \item Implicit \code{pm_runtime_enable()} after resume-early.
    \pause \item Each driver must take explicit action!
    \begin{enumerate}
      \item Do nothing, the default;
      \item \kfunc{pm_runtime_force_suspend} \& \kfunc {pm_runtime_force_resume};
      \item Custom behavior otherwise.
    \end{enumerate}
  \end{itemize}
\end{frame}

\begin{frame}[fragile]{Runtime PM vs system-wide suspend: example issue}
  \begin{block}{}
    \fontsize{10}{10}\selectfont
    \begin{minted}{text}
commit 7da7fd7e66ac9b0d4287aefba516795145f3c722
Author: Thomas Richard <thomas.richard@bootlin.com>
Date:   Thu Jun 13 15:13:28 2024 +0200

i2c: omap: wakeup the controller during suspend() callback

A device may need the controller up during suspend_noirq() or resume_noirq().
But if the controller is autosuspended, there is no way to wake it up during
suspend_noirq() or resume_noirq() because runtime PM is disabled.

The suspend() callback wakes up the controller, so it is available until
its suspend_noirq() callback (pm_runtime_force_suspend()). During the resume,
it is restored by resume_noirq() callback (pm_runtime_force_resume()).
Then resume() callback enables autosuspend.

So the controller is up during a little time slot in suspend and resume
sequences even if it is not used.
    \end{minted}
  \end{block}
\end{frame}

\begin{frame}{Runtime PM vs system-wide suspend: example issue}
    \centering\includegraphics[width=1.5\textheight]{tleb-pm-i2c-suspend.drawio.pdf}
\end{frame}

\begin{frame}{Summary}
  \begin{itemize}
    \item {\bf Behavior is device and platform specific.} \\
      {\em \guillemotleft{} X is suspended \guillemotright{}} does not tell you much.

    \pause \item Issues arise when subsystems, each with their suspend assumptions, \\
      come in contact.

    \pause \item Beware of code paths that differ from one suspend type to another.

    \pause \bigskip
    \item {\bf To be continued...} \\
      genpd, QoS, wakeup sources, and more (?).
  \end{itemize}
\end{frame}



% TODO: example of custom behavior you might want to implement. Either without
% RPM or with RPM and see the diff with its behavior.

% TODO: issue from Thomas of RPM that was disabled and that meant it couldn't
% wake the control up afterwards.

% \begin{frame}{Runtime PM vs system-wide suspend: PCI example}
%   \begin{itemize}
%     \item \code{->prepare()}:
%       \begin{itemize}
%       \item implicit \code{pm_runtime_resume(dev)} beforehands,
%       \end{itemize}
%     \item \code{->suspend()}:
%   \end{itemize}

%   \begin{block}{\href{https://elixir.bootlin.com/linux/master/source/Documentation/admin-guide/cgroup-v1/memory.rst\#L602}{Documentation/admin-guide/cgroup-v1/memory.rst}}
%   \end{block}

% \end{frame}

% TODO: extract anecdotes from past projects into suspend + runtime PM conflicts.

% TODO: maybe avoid PCI which is complex and pick something else (GPIO expander
% on I2C bus?)

% TODO: why is this mmio-mux stuff being done at suspend_noirq?

% TODO: talk about timekeeping: guaranteed disabled in platform-provided
% suspends. Not guaranteed in s2idle (only if cpuidle device can enter_s2idle)

% TODO: maybe end with something along the lines of "it is difficult /
% impossible to know for sure device states".
%
% We might rely on clock tree but that isn't available as tasks are suspended.
%
% And then there are more complex setups incoming, with TI that has automotive
% platforms with many cores, some not running Linux. The type of suspend might
% depend on given latency tolerances from each device, and so Linux won't be
% responsible anymore.

% TODO: talk about latency tolances wrt suspend+wakeup? I am pretty sure it
% isn't well supported (can platforms report latency?).

% Summary for suspend: next time, don't say "the board is in s2idle", that is
% ambiguous.
% Say: processor is WFI from TF-a, L3 is disabled, DDR has seen its frequency
% lowered and is in self-refresh, peripherals in the SoC are not being reset but
% external peripherals are.

\questionslide

\end{document}
