\documentclass[aspectratio=169,obeyspaces,spaces,hyphens,dvipsnames]{beamer}
\usepackage[utf8]{inputenc}
\usepackage{lmodern}% http://ctan.org/pkg/lm
\usepackage{minted}
\usepackage{hyperref}
\usepackage{xcolor}
\usepackage{pgfplots}
\usepackage{tikz}
\usepackage[normalem]{ulem}

\mode<presentation>
\usetheme{Bootlin}

\def\signed #1{{\leavevmode\unskip\nobreak\hfil\penalty50\hskip2em
  \hbox{}\nobreak\hfil(#1)
  \parfillskip=0pt \finalhyphendemerits=0 \endgraf}}

\newsavebox\mybox
\newenvironment{aquote}[1]
  {\savebox\mybox{#1}\begin{quotation}}
  {\signed{\usebox\mybox}\end{quotation}}

\title{Ethernet switch support in the Linux kernel}
\authors{Alexandre Belloni}
\email{alexandre.belloni@bootlin.com}
\slidesurl{https://bootlin.com/pub/conferences/2018/elc/}
\institute{Bootlin}
\conference{ELC 2018}

\begin{document}

\addtocontents{toc}{\protect\setcounter{tocdepth}{-1}}
\section{Ethernet switch support in the Linux kernel}
\addtocontents{toc}{\protect\setcounter{tocdepth}{2}}

\begin{frame}{Alexandre Belloni}
  \begin{itemize}
  \item Embedded Linux engineer at Bootlin
    \begin{itemize}
    \item Embedded Linux {\bf expertise}
    \item {\bf Development}, consulting and training
    \item Strong open-source focus
    \end{itemize}
  \item Open-source contributor
      \begin{itemize}
      \item Maintainer for the Linux kernel {\bf RTC subsystem}
      \item Co-Maintainer of {\bf kernel support for Atmel ARM
        processors}
      \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}{Switch}
  \begin{center}
    \includegraphics[width=0.8\textwidth]{pictures/Switch_HW.pdf}
  \end{center}
\end{frame}

\begin{frame}{Microsemi VSC7514}
  \begin{columns}[c]
    \column{0.5\textwidth}
    \begin{center}
      \includegraphics[width=\textwidth]{pictures/VSC7514_Block_Diagram.png}
    \end{center}
    \column{0.5\textwidth}
    \begin{itemize}
    \item 500MHz MIPS CPU
    \item Usual controllers (UART, I2C, SPI)
    \item 2 MDIO controllers
    \item 10 port gigabit Ethernet switch
    \item 4 integrated PHYs
    \item Currently supported using an SDK running in userspace using
      UIO
    \end{itemize}
  \end{columns}
\end{frame}

\begin{frame}{Features}
  Usual hardware switch features include:
  \begin{itemize}
  \item Bridging
  \item STP
  \item MAC filtering
  \item IGMP snooping
  \item VLAN tagging/untagging
  \end{itemize}
\end{frame}

\begin{frame}{Linux switch support}
  \begin{itemize}
  \item Switch ports are Linux network interfaces
  \item Standard Linux tools are used:
    \begin{itemize}
    \item \code{ip}, \code{ifconfig} for interfaces
    \item \code{ip}, \code{bridge}, \code{brctl} for bridging
    \item Linux bonding for port trunks
    \end{itemize}
  \item The switch can then accelerate what Linux can do in software
  \item \code{switchdev} is the Linux framework to offload features to
    the device
  \end{itemize}
\end{frame}

\begin{frame}{Switchdev}
  \begin{itemize}
  \item Stateless framework, not using the device driver model
  \item \code{switchdev_ops} are attached to a \code{net_device} that
    has to be registered by the driver
  \item \code{switchdev_ops} implement offloading operations
  \item \code{switchdev_obj} abstracts objects (VLANS, MDB) to be used
    by the device
  \end{itemize}
\end{frame}

\subsection{Front ports}

\begin{frame}{Host mode}
  \begin{itemize}
  \item Linux expects each port to be a network interface
  \item The VSC7514 doesn't have an Ethernet controller
  \item However, it is possible to extract or inject frames to/from
    the CPU
  \item Most of the initial configuration is to configure the ports to
    not forward frames and set up the CPU port for extraction and
    injection
  \end{itemize}
\end{frame}

\begin{frame}{Network devices, registration}
  \begin{itemize}
  \item Each port is registered with \code{register_netdev} after
    setting the \code{struct net_device} members: \code{.netdev_ops},
    \code{.ethtool_ops}, \code{.switchdev_ops}
  \item The interface MAC address is added to the switch MAC table
  \item If necessary, the phy is looked up and probed.
  \end{itemize}
\end{frame}

\begin{frame}{Network devices, \code{.ndo_open}}
  \begin{itemize}
  \item \code{ifconfig sw0p0 up} or \code{ip link set dev sw0p0 up}
  \item Enable frame reception on the port and auto learning of MAC
    addresses
  \item Attach and start the phy.
  \end{itemize}
\end{frame}

\begin{frame}{Network devices, \code{.ndo_start_xmit}}
  \begin{itemize}
  \item The frames are injected on the CPU port and configured to be
    forwarded to the switch port
  \item There is a 128-bit header to specify what to do with the
    frame, in particular the port on which the frame has to be
    injected
  \item Frames can be transmitted using:
    \begin{itemize}
    \item PIO with one register
    \item DMA to DDR memory
    \item DMA to 16 KB registers
    \end{itemize}
  \end{itemize}
\end{frame}

\subsection{Bridging and STP}

\begin{frame}{Software bridging}
  \begin{center}
    \includegraphics[width=0.8\textwidth]{pictures/Switch_SW.pdf}
  \end{center}
\end{frame}

\begin{frame}{Hardware offloading}
  \begin{center}
    \includegraphics[width=0.8\textwidth]{pictures/Switch_accel.pdf}
  \end{center}
\end{frame}

\begin{frame}[fragile]{Bridging}
  \begin{itemize}
  \item Setting up a bridge can be done using \code{ip}:
  \begin{block}{}
    \fontsize{8}{8}\selectfont
    \begin{minted}{console}
ip link add name br0 type bridge
ip link set dev sw0p0 master br0
ip link set dev sw0p1 master br0
ip link set dev sw0p2 master br0
ip link set dev sw0p3 master br0
    \end{minted}
  \end{block}
  \end{itemize}
\end{frame}

\begin{frame}[fragile]{handling bridging}
  \begin{itemize}
  \item Handling interface addition and removal is done through a
    netdevice notifier callback registered using
    \code{register_netdevice_notifier}
  \item The event is \code{NETDEV_CHANGEUPPER}. It is necessary to
    check the upper device is a bridge with
    \code{netif_is_bridge_master}
  \begin{block}{}
    \fontsize{8}{8}\selectfont
    \begin{minted}{c}
struct netdev_notifier_changeupper_info {
        struct netdev_notifier_info info; /* must be first */
        struct net_device *upper_dev; /* new upper dev */
        bool master; /* is upper dev master */
        bool linking; /* is the notification for link or unlink */
        void *upper_info; /* upper dev info */
};
    \end{minted}
  \end{block}
  \item Check \code{info->linking} to discriminate between interface
    addition and removal
  \end{itemize}
\end{frame}

\begin{frame}{Forwarding database}
  \begin{center}
    \includegraphics[width=0.8\textwidth]{pictures/Switch_fdb.pdf}
  \end{center}
\end{frame}

\begin{frame}[fragile]{FDB - Userspace}
  \begin{itemize}
  \item Dumping the current bridge fdb table:
  \begin{block}{}
    \fontsize{8}{8}\selectfont
    \begin{minted}{console}
bridge fdb show
    \end{minted}
  \end{block}
  \item Adding an FDB entry
  \begin{block}{}
    \fontsize{8}{8}\selectfont
    \begin{minted}{console}
bridge fdb add 00:00:05:00:01:00 dev sw0p0 static
    \end{minted}
  \end{block}
  \end{itemize}
\end{frame}

\begin{frame}{FDB - kernel}
  \begin{itemize}
  \item The bridge core and the switch FDB have to be kept in sync
  \item Handled using \code{.ndo_fdb_*} callbacks of the
    \code{net_device_ops} structure.
  \item At first, they were set to \code{switchdev_port_fdb_*} but
    they were removed in v4.14
  \item \code{.ndo_fdb_add} and \code{.ndo_fdb_del} are simple to
    implement, simply adds or removes a MAC entry
  \item \code{.ndo_fdb_dump} is more complicated as it has to handle
    netlink messaging. Taken mostly from DSA.
  \item This was necessary because the HW is not able to send
    interrupts when it learns a new MAC so the driver is not able to
    send an event to the bridge driver to maintain the FDB table.
    \code{call_switchdev_notifiers(SWITCHDEV_FDB_ADD_TO_BRIDGE, ...)}
  \end{itemize}
\end{frame}

\begin{frame}[fragile]{Ageing}
  \begin{itemize}
  \item Ageing time can be changed at setup using \code{ip}:
  \begin{block}{}
    \fontsize{8}{8}\selectfont
    \begin{minted}{console}
ip link set dev br0 type bridge ageing_time 1000
    \end{minted}
  \end{block}
  \item or with \code{brctl}:
  \begin{block}{}
    \fontsize{8}{8}\selectfont
    \begin{minted}{console}
brctl setageing br0 1000
    \end{minted}
  \end{block}
  \item The bridge core will call the \code{.switchdev_port_attr_set}
    callback of the registered \code{switchdev_ops}
  \end{itemize}
\end{frame}

\begin{frame}[fragile]{\code{switchdev_port_attr_set}}
  \begin{block}{}
    \fontsize{8}{8}\selectfont
    \begin{minted}{c}
int (*switchdev_port_attr_set)(struct net_device *dev,
                               const struct switchdev_attr *attr,
                               struct switchdev_trans *trans);
    \end{minted}
  \end{block}
  \begin{block}{}
    \fontsize{8}{7}\selectfont
    \begin{minted}{c}
struct switchdev_attr {
        struct net_device *orig_dev;
        enum switchdev_attr_id id;
        u32 flags;
        void *complete_priv;
        void (*complete)(struct net_device *dev, int err, void *priv);
        union {
                struct netdev_phys_item_id ppid;        /* PORT_PARENT_ID */
                u8 stp_state;                           /* PORT_STP_STATE */
                unsigned long brport_flags;             /* PORT_BRIDGE_FLAGS */
                unsigned long brport_flags_support;     /* PORT_BRIDGE_FLAGS_SUPPORT */
                bool mrouter;                           /* PORT_MROUTER */
                clock_t ageing_time;                    /* BRIDGE_AGEING_TIME */
                bool vlan_filtering;                    /* BRIDGE_VLAN_FILTERING */
                bool mc_disabled;                       /* MC_DISABLED */
        } u;
};
    \end{minted}
  \end{block}
\end{frame}

\begin{frame}[fragile]{Ageing}
  \begin{itemize}
  \item \code{attr->id} is the attribute to set, for ageing, it will
    be \code{SWITCHDEV_ATTR_ID_BRIDGE_AGEING_TIME}
  \item \code{attr->u.ageing_time} holds the ageing time in jiffies
  \item \code{.switchdev_port_attr_set} is called twice to allow to
    change the configuration atomically. Use
    \code{switchdev_trans_ph_prepare(trans)} or
    \code{switchdev_trans_ph_commit(trans)} to know which step of the
    transaction this is.
  \end{itemize}
\end{frame}

\begin{frame}{STP}
  \begin{itemize}
  \item Enabling STP is done with \code{brctl stp br0 on} or with
    \code{ip link set dev br0 type bridge stp_state 1}
  \item Handling STP is done through the
    \code{.switchdev_port_attr_set} callback
  \item \code{attr->id} will be
    \code{SWITCHDEV_ATTR_ID_PORT_STP_STATE}
  \item \code{attr->u.stp_state} hold the target STP state
  \item The various states are:
    \begin{itemize}
    \item \code{BR_STATE_DISABLED}
    \item \code{BR_STATE_LISTENING}
    \item \code{BR_STATE_LEARNING}
    \item \code{BR_STATE_FORWARDING}
    \item \code{BR_STATE_BLOCKING}
    \end{itemize}
  \end{itemize}
\end{frame}

\subsection{Link aggregation}

\begin{frame}[fragile]{configuration}
  \begin{block}{}
    \fontsize{9}{9}\selectfont
    \begin{minted}{console}
ip link add name aggr0 type bond
ip link set dev eth_yellow master aggr0
ip link set dev eth_blue master aggr0
    \end{minted}
  \end{block}
\end{frame}

\begin{frame}{Link aggregation}
  \begin{itemize}
  \item As for bridging, it uses the \code{NETDEV_CHANGEUPPER} in the
    netdevice notifier callback
  \item Check the upper device is a bond with
    \code{netif_is_lag_master}
  \item Check \code{info->linking} to discriminate between interface
    addition and removal
  \end{itemize}
\end{frame}

\subsection{IGMP snooping}

\begin{frame}{IGMP snooping}
  \begin{center}
    \includegraphics[width=0.5\textwidth]{pictures/Switch_IGMP.pdf}
  \end{center}
\end{frame}

\begin{frame}{IGMP snooping}
  \begin{itemize}
  \item Linux can install MDBs so the switch avoids flooding multicast
    traffic on all ports.
  \item The switch needs to forward IGMP packets to the CPU.
    \code{netdev_for_each_mc_addr} will provide all the multicast
    addresses to be installed in the MAC table.
  \item The bridge core will call the \code{.switchdev_port_obj_add}
    callback of the registered \code{switchdev_ops}
  \end{itemize}
\end{frame}

\begin{frame}[fragile]{\code{switchdev_port_obj_add}}
  \begin{block}{}
    \fontsize{8}{8}\selectfont
    \begin{minted}{c}
int (*switchdev_port_obj_add)(struct net_device *dev,
                              const struct switchdev_obj *obj,
                              struct switchdev_trans *trans);
    \end{minted}
  \end{block}
  \begin{block}{}
    \fontsize{8}{7}\selectfont
    \begin{minted}{c}
struct switchdev_obj {
        struct net_device *orig_dev;
        enum switchdev_obj_id id;
        u32 flags;
        void *complete_priv;
        void (*complete)(struct net_device *dev, int err, void *priv);
};
    \end{minted}
  \end{block}
  \begin{block}{}
    \fontsize{8}{7}\selectfont
    \begin{minted}{c}
struct switchdev_obj_port_mdb {
        struct switchdev_obj obj;
        unsigned char addr[ETH_ALEN];
        u16 vid;
};

#define SWITCHDEV_OBJ_PORT_MDB(obj) \
        container_of(obj, struct switchdev_obj_port_mdb, obj)
    \end{minted}
  \end{block}
\end{frame}

\begin{frame}{installing MDBs}
  \begin{itemize}
  \item \code{obj->id} will be \code{SWITCHDEV_OBJ_ID_PORT_MDB}
  \item Then cast to an mdb with \code{SWITCHDEV_OBJ_PORT_MDB()}
  \item The address and VLAN id are now available in \code{mdb->addr}
    and \code{mdb->vid}
  \end{itemize}
\end{frame}

\subsection{VLAN filtering}

\begin{frame}[fragile]{configuration}
  \begin{block}{}
    \fontsize{9}{9}\selectfont
    \begin{minted}{console}
ip link add name br0 type bridge
ip link set dev br0 type bridge vlan_filtering 1
ip link set dev sw0p0 master br0
ip link set dev sw0p1 master br0
ip link set dev sw0p2 master br0
ip link set dev sw0p3 master br0
bridge vlan add dev sw0p0 vid 1 pvid untagged
bridge vlan add dev sw0p1 vid 1
bridge vlan add dev sw0p2 vid 1
bridge vlan add dev sw0p3 vid 1
bridge vlan add dev sw0p0 vid 10
bridge vlan add dev sw0p1 vid 10 pvid untagged
bridge vlan add dev sw0p2 vid 20 pvid untagged
bridge vlan add dev sw0p3 vid 20
bridge vlan add dev sw0p0 vid 30
bridge vlan add dev sw0p1 vid 30
bridge vlan add dev sw0p2 vid 30
    \end{minted}
  \end{block}
\end{frame}


\begin{frame}[fragile]{VLAN}
  \begin{itemize}
  \item The bridge core will call the \code{.switchdev_port_obj_add}
    callback of the registered \code{switchdev_ops}
  \item This time, \code{obj->id} will be \code{SWITCHDEV_OBJ_ID_PORT_VLAN}
  \end{itemize}
  \begin{block}{}
    \fontsize{9}{9}\selectfont
    \begin{minted}{c}
struct switchdev_obj_port_vlan {
        struct switchdev_obj obj;
        u16 flags;
        u16 vid_begin;
        u16 vid_end;
};

#define SWITCHDEV_OBJ_PORT_VLAN(obj) \
        container_of(obj, struct switchdev_obj_port_vlan, obj)
    \end{minted}
  \end{block}
\end{frame}

\begin{frame}[fragile]{VLAN}
  \begin{itemize}
  \item Cast to a \code{switchdev_obj_port_vlan} with \code{SWITCHDEV_OBJ_PORT_VLAN}
  \item All VLAN ids to install are from \code{vid_begin} to
    \code{vid_end}
  \item \code{flags} will be a combination of:
  \begin{block}{}
    \fontsize{9}{9}\selectfont
    \begin{minted}{c}
#define BRIDGE_VLAN_INFO_MASTER        (1<<0) /* Operate on Bridge device as well */
#define BRIDGE_VLAN_INFO_PVID          (1<<1) /* VLAN is PVID, ingress untagged */
#define BRIDGE_VLAN_INFO_UNTAGGED      (1<<2) /* VLAN egresses untagged */
#define BRIDGE_VLAN_INFO_RANGE_BEGIN   (1<<3) /* VLAN is start of vlan range */
#define BRIDGE_VLAN_INFO_RANGE_END     (1<<4) /* VLAN is end of vlan range */
#define BRIDGE_VLAN_INFO_BRENTRY       (1<<5) /* Global bridge VLAN entry */
    \end{minted}
  \end{block}
  \end{itemize}
\end{frame}

\subsection{DSA}

\begin{frame}{DSA}
  \begin{center}
    \includegraphics[width=0.8\textwidth]{pictures/DSA.pdf}
  \end{center}
\end{frame}

\begin{frame}{DSA}
  \begin{itemize}
  \item Distributed Switch Architecture
  \item Handles chaining switches through Ethernet ports
  \item Handles the vendor specific switch tagging protocol
  \item Integrates nicely in the device model
  \item As a defined device tree binding
  \end{itemize}
\end{frame}

\begin{frame}{DSA vs switchdev}
  \begin{itemize}
  \item Is the switch connected to the CPU through an Ethernet
    interface?
  \item Can that interface absorb all the traffic from the switch?
  \item Does the switch use switch tags?
  \end{itemize}
\end{frame}

\begin{frame}{Other challenges}
  \begin{itemize}
  \item The switch can be used by the internal CPU using MMIO or by an
    external CPU using PCIe. Device tree is used to describe the
    switch when using MMIO. This becomes quite impractical when using
    the same switch connected through PCIe on x86. The current
    solution is to have an MFD driver registering all the necessary
    drivers as platform\_devices.
  \item All the registers are packed in the register space, this
    complicates support for similar switches but with a different
    number of ports
  \end{itemize}
\end{frame}

\begin{frame}{Next steps}
  \begin{itemize}
  \item Sending patches Upstream
  \item DMA
  \item PTP, IEEE1588 support
  \item QoS
  \item SyncE support
  \item Rework promiscuous support
  \end{itemize}
\end{frame}

\questionslide

\end{document}