\documentclass[aspectratio=169,obeyspaces,spaces,hyphens,dvipsnames]{beamer}
\usepackage[utf8]{inputenc}
\usepackage{lmodern}% http://ctan.org/pkg/lm
\usepackage{minted}
\usepackage{hyperref}
\usepackage{xcolor}
\usepackage{pgfplots}
\usepackage{tikz}
\usepackage{typicons}
\usepackage[normalem]{ulem}

\mode<presentation>
\usetheme{Bootlin}

\def\signed #1{{\leavevmode\unskip\nobreak\hfil\penalty50\hskip2em
  \hbox{}\nobreak\hfil(#1)
  \parfillskip=0pt \finalhyphendemerits=0 \endgraf}}

\newsavebox\mybox
\newenvironment{aquote}[1]
  {\savebox\mybox{#1}\begin{quotation}}
  {\signed{\usebox\mybox}\end{quotation}}

\title{Inspecting and optimizing memory usage in Linux}
\authors{João Marcos Costa}
\email{joaomarcos.costa@bootlin.com}
\slidesurl{https://bootlin.com/pub/conferences/}
\institute{Bootlin}
\conference{Embedded Linux Conference Europe 2024}

\begin{document}

\addtocontents{toc}{\protect\setcounter{tocdepth}{-1}}
\section{Inspecting and optimizing memory usage in Linux}
\addtocontents{toc}{\protect\setcounter{tocdepth}{2}}

\begin{frame}{João Marcos Costa}
  \begin{itemize}
  \item Embedded Linux engineer and trainer at Bootlin since 2023
    \begin{itemize}
    \item Embedded Linux experts
    \item Engineering services: Linux BSP development, kernel porting
      and drivers, Yocto/Buildroot integration, real-time, boot-time,
      security, multimedia
    \item Training services: Embedded Linux, Linux kernel drivers,
      Yocto, Buildroot, graphics stack, boot-time, real-time
    \end{itemize}
  \item Open-source contributor
  \item Living in {\bf Lyon}, France
  \item \code{joaomarcos.costa@bootlin.com}
  \end{itemize}
  {\small \url{https://bootlin.com/company/staff/joao-marcos-costa/}}
\end{frame}

% \begin{frame}{Frame title}
%   \begin{itemize}
%   \item lorem ipsum
%   \end{itemize}
% \end{frame}

\begin{frame}{Background}
  \begin{itemize}
  \item Project's requirements:
      \begin{itemize}
      \item use as little memory as possible
      \item furthermore, understand how the memory is being used
      \end{itemize}
  \item iMX93 Evaluation Kit, with 2 GiB of LPDDR4X RAM
  \item a humble enthusiast, not a memory guru!
  \end{itemize}
\end{frame}

\begin{frame}{What is this talk about?}
  \begin{itemize}
  \item Virtual memory
  \item Memory used by the kernel vs. by the programs
  \item Is it leaking?
  \item What if I don't have enough of it?
  \end{itemize}
\end{frame}

\subsection{Introduction}

\begin{frame}{Virtual memory: the basics}
  \begin{itemize}
  \item Physical memory is not directly referred to. Virtual addresses are used
      instead
  \item The address translation is handled by the Memory Management Unit (MMU)
  \item The virtual address space is typically divided in 4KiB-long pages
  \item Other page sizes do exist (see \href{https://wiki.debian.org/Hugepages}{Huge pages})
  \item Those pages are indexed in the MMU's Page Tables
  \end{itemize}
\end{frame}

\begin{frame}[fragile]{Virtual memory: representing pages}
    \begin{block}{\kline{v6.10.10}{include/linux/mm_types.h}{74}}
    \fontsize{9}{9}\selectfont
    \begin{minted}{c}

struct page {
    unsigned long flags; /* Page status, see <include/linux/page-flags.h>*/
    struct list_head lru;
    struct address_space *mapping;
    pgoff_t index;
    atomic_t _refcount; /* Usage count */
    void *virtual;    /* Kernel virtual address */
};
    \end{minted}
  \end{block}
  \begin{itemize}
  \item This struct represents physical page, not a virtual one
  \item Example: 2GiB of RAM $\equiv$ 524288 4KiB-long pages
      \begin{itemize}
          \item \textit{struct page} is $\approx$ 64-bytes long
          \item $524288 \times 64 = 32 MiB$ to represent physical memory pages
      \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}[fragile]{Virtual memory: representing pages}
  \begin{block}{\kline{v6.10.10}{include/linux/mm_types.h}{648}}
    \fontsize{9}{9}\selectfont
    \begin{minted}{c}

struct vm_area_struct {
    /* VMA covers [vm_start; vm_end) addresses within mm */
    unsigned long vm_start;
    unsigned long vm_end;
    struct mm_struct *vm_mm;    /* The address space we belong to. */
    const vm_flags_t vm_flags;
    struct list_head anon_vma_chain;
    struct anon_vma *anon_vma;
    /* Function pointers to deal with this struct. */
    const struct vm_operations_struct *vm_ops;
    struct file * vm_file;      /* File we map to (can be NULL). */
    void * vm_private_data;     /* was vm_pte (shared mem) */
};
    \end{minted}
  \end{block}
  \begin{itemize}
  \item There is not a single struct to represent a virtual page
  \item Instead, Linux refers to a range of virtual addresses, or \textit{Virtual Memory Area (VMA)}
  \end{itemize}
\end{frame}

\begin{frame}{Memory zones}
  \begin{itemize}
  \item Physical memory is divided in zones, rather than being an homogeneous pool of addresses
  \item \textbf{ZONE\_DMA}: the lower 16 MiB, for Direct Memory Access (legacy?)
  \item \textbf{ZONE\_DMA32}: between 16 MiB, and below 4 GiB (64-bit Linux only)
  \item \textbf{ZONE\_NORMAL}:
      \begin{itemize}
          \item For 32-bit machines, between 16 MiB and 896 MiB
          \item For 64-bit machines, all memory above 4 GiB
      \end{itemize}
  \item \textbf{ZONE\_HIGHMEM}: the memory above 896 MiB, only in 32-bit machines
  \end{itemize}
\end{frame}

\begin{frame}{Zones and nodes}
  \centering\includegraphics[scale=0.25]{mypictures/zones.pdf}
  \begin{itemize}
      \item Zones are attached to nodes, and nodes to CPUs
      \item One node per CPU
      \item Each node is aware of its zones and their available memory pages
      \item Non-Uniform Memory Access (NUMA)
  \end{itemize}
\end{frame}

\begin{frame}[fragile]{Zones and nodes}
  \begin{block}{512 MiB RAM, 32-bits machine}
    \fontsize{7}{7}\selectfont
    \begin{minted}{shell}
# cat /proc/buddyinfo
Node 0, zone   Normal     28     13      8      3      3      1      2      2      2      3      2     51
    \end{minted}
  \end{block}
  \begin{itemize}
      \item Each column represents the number of available consecutive memory chunks of a certain size
      \item Every chunk's (i.e. column) size is defined by: $PAGESIZE \times K \times 2^{n}$
      \item e.g.: the first column ($K=28, n=0)$ stands for 28 chunks of 4096 bytes

  \end{itemize}
  \begin{block}{32 GiB RAM, 64-bits machine}
    \fontsize{7}{7}\selectfont
    \begin{minted}{shell}
# cat /proc/buddyinfo
Node 0, zone      DMA      0      0      0      0      0      0      0      0      1      1      2 
Node 0, zone    DMA32   2241   1787   1400   1356    935    485    157     39      7      5      0 
Node 0, zone   Normal  26143  18051  12270   8620   5536   3278   1564    620    206     40      3 
    \end{minted}
  \end{block}
\end{frame}

\begin{frame}{Virtual memory: what does it provide?}
  \centering\includegraphics[scale=0.25]{mypictures/virtual-memory.pdf}
  \begin{itemize}
  \item Memory is represented in a simpler way: as a uniform and continguous address space
  \item Each process will run in its own isolated addres space
  \item Primary and secondary memory (i.e., the disk) are abstracted as one
  \item Processes can share memory segments (e.g., shared libraries, and IPC)
  \end{itemize}
\end{frame}

\subsection{Memory usage in kernel-space}

\begin{frame}{Memory usage in kernel-space}
  \begin{itemize}
      \item Memory allocations (\textit{vmalloc()} and \textit{kmalloc()})
      \item Modules
      \item The kernel's binary itself
      \item Low level allocations (simply not tracked)
  \end{itemize}
\end{frame}

\begin{frame}[fragile]{Examining the boot logs}
  \begin{itemize}
  \item The starting point for our analysis
  \item \code{Memory: A/B available (...)}
  \item \code{B}: The total physical memory (minus \textit{OPTEE})
  \item \code{A}: \code{B} minus the memory reserved by/for the kernel (i.e.,
      \code{reserved} field)
  \end{itemize}
  \begin{block}{Kernel logs from early boot}
    \fontsize{9}{9}\selectfont
    \begin{minted}{shell}
[    0.000000] Memory: 2012000K/2064384K available (7936K kernel code, 572K rwdata, 1708K
rodata, 1280K init, 328K bss, 52384K reserved, 0K cma-reserved)
    \end{minted}
  \end{block}
\end{frame}

\begin{frame}{Examining the boot logs}
  \begin{itemize}
      \item \code{kernel code}: \code{.text} section of the kernel binary
      \item \code{rwdata}: initialized (and writable) global and static variables
      \item \code{rodata}: read-only kernel data as constants and strings
      \item \code{init}: initialization code, reclaimed later
      \item \code{bss}: uninitialized data
      \item \code{reserved}: an overall metric for memory reserved by/for the
          kernel including code, data, and the physical pages (32 MiB, in this
          case)
      \item \code{cma-reserved}: Contiguous Memory Allocator
  \end{itemize}
\end{frame}

\begin{frame}[fragile]{Examining the boot logs}
    \begin{block}{\kline{v6.10.10}{mm/mm_init.c}{2634}}
    \fontsize{5}{5}\selectfont
    \begin{minted}{c}
static void __init mem_init_print_info(void)
{
    [...]

        pr_info("Memory: %luK/%luK available (%luK kernel code, %luK rwdata, %luK rodata, %luK init, %luK bss, %luK reserved, %luK cma-reserved"
#ifdef  CONFIG_HIGHMEM
        ", %luK highmem"
#endif
        ")\n",
        K(nr_free_pages()), K(physpages),
        codesize / SZ_1K, datasize / SZ_1K, rosize / SZ_1K,
        (init_data_size + init_code_size) / SZ_1K, bss_size / SZ_1K,
        K(physpages - totalram_pages() - totalcma_pages),
        K(totalcma_pages)
#ifdef  CONFIG_HIGHMEM
        , K(totalhigh_pages())
#endif
        );

    [...]
    \end{minted}
  \end{block}
\end{frame}

\begin{frame}[fragile]{Kernel memory usage, but from user-space}
  \begin{block}{Kernel logs from early boot}
    \fontsize{9}{9}\selectfont
    \begin{minted}{shell}
[    0.000000] Memory: 2012000K/2064384K available (...1280K init...
    \end{minted}
  \end{block}
    \begin{block}{\code{/proc/meminfo}}
    \fontsize{9}{9}\selectfont
    \begin{minted}{shell}
root@xxxx-imx93:~# grep MemTotal /proc/meminfo
MemTotal:        2013280 kB
    \end{minted}
  \end{block}
  \begin{itemize}
      \item Total memory slightly increased after \code{init} segment was retrieved
      \item \code{/proc/meminfo}: significant information about the kernel's memory usage
  \end{itemize}
\end{frame}

\begin{frame}[fragile]{Examining \code{/proc/meminfo}}
  \begin{itemize}
      \item \code{Slab}: total memory utilized for caching in-kernel data structures, managed by the Slab allocator
      \item \code{KernelStack}: memory allocated to kernel stacks
      \item \code{PageTables}: lowest level arrays of pages for address translation
      \item \code{VmallocUsed}: the used portion of \code{vmalloc} memory area
  \end{itemize}
    \begin{block}{\code{/proc/meminfo}}
    \fontsize{9}{9}\selectfont
    \begin{minted}{shell}
# cat /proc/meminfo
Slab:               9440 kB
KernelStack:         752 kB
PageTables:          628 kB
VmallocUsed:       15204 kB
[...]
    \end{minted}
  \end{block}
\end{frame}

\begin{frame}{The kernel binary}
  \includegraphics[scale=0.25]{mypictures/kernel-binary.pdf}
  \includegraphics[scale=0.25]{mypictures/kernel-binary-drivers.pdf}
  \begin{itemize}
      \item How much space each component takes in the final binary
      \item Rough numbers, based on the disk usage in the build directory
      \item The less code, the better!
  \end{itemize}
\end{frame}

\subsection{Memory usage in user-space}

\begin{frame}[fragile]{First approach: using \code{ps} command}
    \begin{block}{}
    \fontsize{8}{8}\selectfont
    \begin{minted}{shell}
# ps aux
USER       PID %CPU %MEM    VSZ   RSS TTY      STAT START   TIME COMMAND
root         1  0.0  0.6  22452  3080 ?        Ss   Jan05   0:17 /sbin/init
root       161  0.0  0.3   5440  1728 ?        Ss   Jan05   0:02 /lib/systemd/systemd-udevd
root       169  0.0  1.1  13120  6024 ?        Ss   Jan05   1:19 /lib/systemd/systemd-journald
root       214  0.0  0.1   2776   844 ?        Ss   Jan05   0:00 /sbin/klogd -n
    \end{minted}
  \end{block}
  \begin{itemize}
      \item Virtual Set Size and Resident Set Size
      \item \code{VSZ}: total virtual memory size, as listed in \code{/proc/<PID>/maps}
      \item \code{RSS}: the memory that is actually mapped to physical pages
      \item Equivalent to \code{VIRT} and \code{RES} in \code{top} command
      \item \code{RSS} repeatedly accounts for shared memory areas \tiWarningOutline
  \end{itemize}
\end{frame}

\begin{frame}[fragile]{Second approach: using \code{smem} command}
    \begin{block}{}
    \fontsize{7}{7}\selectfont
    \begin{minted}{shell}
# smem -t
  PID User     Command                         Swap      USS      PSS      RSS 
  290 0        /sbin/getty 38400 tty1             0       80      240     1380 
  281 0        /sbin/syslogd -n -O /var/lo        0       76      249     1440 
  284 0        /sbin/klogd -n                     0      100      273     1464 
    1 0        init [5]                           0      176      286     1172 
  289 0        /bin/sh /bin/start_getty 11        0       84      310     1576 
  325 0        smemcap                            0      164      350     1528 
  292 0        -sh                                0      216      442     1708 
  273 0        /usr/sbin/dropbear -r /etc/        0      344      455     1340 
  162 0        /sbin/udevd -d                     0      792      902     1784 
-------------------------------------------------------------------------------
    9 1                                           0     2032     3507    13392 
    \end{minted}
  \end{block}
  \begin{itemize}
      \item Unique Set Size and Proportional Set Size
      \item \code{USS}: unique (private) memory per process
      \item \code{PSS}: proportional fraction of a shared memory zone
      \item if a 120Kb memory zone is shared among 3 proccesses, the \code{PSS} will be of 40Kb for each
      \item more details in \code{/proc/<PID>/smaps}
  \end{itemize}
\end{frame}

\begin{frame}{Plotting the memory usage with \code{smem}} % linuxatemyram
  \begin{itemize}
      \item Record the data on the target: \code{smemcap > smemcap_imx93.tar}
      \item Plot it on the host: \code{smem --pie name -S smemcap_imx93.tar -s [uss|pss]}
  \end{itemize}
  \includegraphics[scale=0.4]{mypictures/smem-uss.pdf}
  \includegraphics[scale=0.4]{mypictures/smem-pss.pdf}
\end{frame}

\begin{frame}[fragile]{\code{free} command} % linuxatemyram
    \begin{block}{}
    \fontsize{7}{7}\selectfont
    \begin{minted}{shell}
# free -h
               total        used        free      shared  buff/cache   available
Mem:            31Gi        18Gi       652Mi       3.5Gi        16Gi        12Gi
    \end{minted}
  \end{block}
  \begin{itemize}
      \item \code{used}: unavailable memory
      \item \code{available}: used, but can be reclaimed
      \item \code{free}: not used for anything (wasted?)
      \item \code{shared} and \code{buff/cache}: \code{tmpfs}, kernel buffers, page cache, etc.
      \item check \href{https://www.linuxatemyram.com/}{linuxatemyram.com}
  \end{itemize}
\end{frame}

\subsection{Optimization} % zram + swap + small kernel + small libc + oom

\begin{frame}{Smaller code}
  \begin{itemize}
      \item Thinking about features in a wider perspective (graphics, networking,
          filesystems, buses, virtualization, etc) then disabling the corresponding configs
      \item Compiler optimization level (size, rather than performance)
      \item Disable debugging (\code{CONFIG_DEBUG_KERNEL})
      \item In iMX93: went from a 32MiB kernel to 11 MiB
  \end{itemize}
\end{frame}

\begin{frame}[fragile]{Catching memory leaks}
  \begin{itemize}
      \item Monitoring \textit{procfs}
      \begin{itemize}
          \item \textit{free} command, for a global view
          \item per-process \textit{USS} and/or \textit{RSS}, to find the leak source
          \item \code{/proc/meminfo}, and \code{/proc/<PID>/smaps} respectively
      \end{itemize}
  \end{itemize}

  \centering\includegraphics[scale=0.48]{mypictures/catch-memleak.pdf}

\end{frame}

\begin{frame}[fragile]{Catching memory leaks}
  \begin{itemize}
  \item Valgrind
      \begin{itemize}
          \item Detects memory management errors: leaks, invalid accesses, bad
              freeing of heap blocks, etc.
          \item No need to rebuild your program, but it works better with \code{-g}
      \end{itemize}
  \end{itemize}
  \begin{block}{}
    \fontsize{6}{6}\selectfont
    \begin{minted}{c}
#include <stdlib.h>

int main() {
    char *string_a, *string_b;

    string_a = malloc(10); /* 10 bytes leaking */
    free(string_b); /* Invalid free */

    return 0;
}
    \end{minted}
  \end{block}
\end{frame}

\begin{frame}[fragile]{Catching memory leaks}
    \begin{block}{}
    \fontsize{6}{6}\selectfont
    \begin{minted}{shell}
valgrind --leak-check=full --show-leak-kinds=all ./memleak
==941479== Memcheck, a memory error detector
==941479== Copyright (C) 2002-2022, and GNU GPL'd, by Julian Seward et al.
==941479== Using Valgrind-3.22.0 and LibVEX; rerun with -h for copyright info
==941479== Command: ./memleak
==941479== 
==941479== Conditional jump or move depends on uninitialised value(s)
==941479==    at 0x4845ADF: free (vg_replace_malloc.c:985)
==941479==    by 0x401157: main (memleak.c:7)
==941479== 
==941479== 
==941479== HEAP SUMMARY:
==941479==     in use at exit: 10 bytes in 1 blocks
==941479==   total heap usage: 1 allocs, 0 frees, 10 bytes allocated
==941479== 
==941479== 10 bytes in 1 blocks are definitely lost in loss record 1 of 1
==941479==    at 0x484280F: malloc (vg_replace_malloc.c:442)
==941479==    by 0x401147: main (memleak.c:6)
==941479== 
==941479== LEAK SUMMARY:
==941479==    definitely lost: 10 bytes in 1 blocks
==941479==    indirectly lost: 0 bytes in 0 blocks
==941479==      possibly lost: 0 bytes in 0 blocks
==941479==    still reachable: 0 bytes in 0 blocks
==941479==         suppressed: 0 bytes in 0 blocks
==941479== 
==941479== Use --track-origins=yes to see where uninitialised values come from
==941479== For lists of detected and suppressed errors, rerun with: -s
==941479== ERROR SUMMARY: 2 errors from 2 contexts (suppressed: 0 from 0)
    \end{minted}
    \end{block}
\end{frame}

\begin{frame}[fragile]{Swap}
  \begin{itemize}
  \item Freeing memory by moving pages to the disk
  \item Not quite suited for embedded systems: wearing out flash disks, unresponsiveness (\textit{thrashing}), etc.
  \item Can be tuned with \code{/proc/sys/vm/swappiness}, ranging from 0 to 200
  \item Use it to balance swapping and filesystem paging (e.g., 100 means equal IO cost)
  \item Setting it to 0...
      \begin{itemize}
          \item Disables it in a memory control group context
          \item Does *not* disable it in a system-wide context
      \end{itemize}
  \end{itemize}
    \begin{block}{\kline{v6.10.10}{Documentation/admin-guide/cgroup-v1/memory.rst}{614}}
    \fontsize{7}{7}\selectfont
    \begin{minted}{rst}
Please note that unlike during the global reclaim, limit reclaim enforces that 0 swappiness really prevents from
any swapping even if there is a swap storage available. This might lead to memcg OOM killer if there are no file
pages to reclaim.
    \end{minted}
    \end{block}
\end{frame}

\begin{frame}[fragile]{ZRAM: like swap, but better}
  \begin{itemize}
  \item Pages are compressed then stored into a RAM-based block device
  \item Faster than disk-based swap
  \end{itemize}
      \begin{block}{}
    \fontsize{7}{7}\selectfont
    \begin{minted}{shell}
Device Drivers  --->
    [*] Block devices  --->
        <M> Compressed RAM block device support
        [*] Write back incompressible or idle page to backing device
        [*] Track zRam block status
    \end{minted}
  \end{block}
\end{frame}

\begin{frame}[fragile]{Out of memory killer}
  \begin{itemize}
  \item Last resource: a process is killed to reclaim memory
  \item Heuristic choice: the more memory the process takes, the higher its \textit{badness}
  \end{itemize}
    \begin{block}{\kline{v6.10.10}{mm/oom_kill.c}{202}}
    \fontsize{7}{7}\selectfont
    \begin{minted}{c}
long oom_badness(struct task_struct *p, unsigned long totalpages)
{

    /* (...) */

    /*
     * The baseline for the badness score is the proportion of RAM that each
     * task's rss, pagetable and swap space use.
     */
    points = get_mm_rss(p->mm) + get_mm_counter(p->mm, MM_SWAPENTS) +
        mm_pgtables_bytes(p->mm) / PAGE_SIZE;

    /* Normalize to oom_score_adj units */
    adj *= totalpages / 1000;
    points += adj;

    return points;
}
    \end{minted}
    \end{block}

\end{frame}

\begin{frame}[fragile]{Out of memory killer}
  \begin{itemize}
  \item Tuning OOM Killer: \code{/proc/<PID>/oom_score_adj}
  \item This adjustment parameter ranges from -1000 to 1000
    \begin{block}{\kfile{include/uapi/linux/oom.h}}
    \fontsize{5}{5}\selectfont
    \begin{minted}{c}
/*
 * /proc/<pid>/oom_score_adj set to OOM_SCORE_ADJ_MIN disables oom killing for
 * pid.
 */
#define OOM_SCORE_ADJ_MIN   (-1000)
#define OOM_SCORE_ADJ_MAX   1000
    \end{minted}
    \end{block}
  \item Example: \code{echo -1000 > /proc/`pgrep myapp`/oom_score_adj}

    \begin{block}{}
    \fontsize{5}{5}\selectfont
    \begin{minted}{shell}
# cat /proc/`pgrep firefox`/oom_score
738
# echo -1000 > /proc/`pgrep firefox`/oom_score_adj
# cat /proc/`pgrep firefox`/oom_score
0
    \end{minted}
    \end{block}
  \item Firefox is now immune to the OOM Killer!
  \end{itemize}
\end{frame}

% \begin{frame}[fragile]{Frame with code}
%   \begin{block}{\code{filename}}
%     \fontsize{9}{9}\selectfont
%     \begin{minted}{c}
% #include <stdio.h>

% int main(void)
% {
%     printf("hello, world\n");
% }
%     \end{minted}
%   \end{block}
% \end{frame}

\questionslide

\end{document}
