\documentclass[aspectratio=169,obeyspaces,spaces,hyphens,dvipsnames]{beamer}
\usepackage[utf8x]{inputenc}
\usepackage{lmodern}% http://ctan.org/pkg/lm
\usepackage{minted}
\usepackage{hyperref}
\usepackage{xcolor}
\usepackage{pgfplots}
\usepackage{tikz}
\usepackage[normalem]{ulem}
\usepackage{textcomp}

\usepackage{multimedia}

\mode<presentation>
\usetheme{Bootlin}

\def\signed #1{{\leavevmode\unskip\nobreak\hfil\penalty50\hskip2em
  \hbox{}\nobreak\hfil(#1)
  \parfillskip=0pt \finalhyphendemerits=0 \endgraf}}

\newsavebox\mybox
\newenvironment{aquote}[1]
  {\savebox\mybox{#1}\begin{quotation}}
  {\signed{\usebox\mybox}\end{quotation}}

\title{Precision time protocol (PTP) and packet timestamping in Linux}
\conference{Embedded Linux Conference Europe, October 2020}
\authors{Antoine Ténart}
\email{{\tiny atenart+elce2020@kernel.org}}
\institute{Bootlin}
\slidesurl{https://bootlin.com/pub/conferences/2020/elce/tenart-timestamping-and-ptp-in-linux/}

\begin{document}

\addtocontents{toc}{\protect\setcounter{tocdepth}{-1}}
\section{Precision time protocol (PTP) and packet timestamping in Linux}
\addtocontents{toc}{\protect\setcounter{tocdepth}{2}}

\section{Introduction}

\begin{frame}{Preamble - goals}
  \begin{itemize}
    \item Understand what is the precision time protocol (PTP) and its modes of
      operation.
    \item Have a first glance at what is a packet timestamping and how the
      kernel supports it.
      \begin{itemize}
        \item {\bf Disclaimer:} packet timestamping can be used in various
          applications, we'll only cover it in regard to PTP.
      \end{itemize}
    \item Understand why hardware timestamping of packets is beneficial.
    \item See how PTP offloading support (hardware timestamping and PTP
      hardware clock) can be provided by device drivers.
  \end{itemize}
\end{frame}

\begin{frame}{Agenda}
  \begin{enumerate}
    \item Background
    \item Overview of the precision time protocol (PTP)
    \item Packet timestamping
    \item PTP offloading support in Linux
    \item User-space PTP implementation
  \end{enumerate}
\end{frame}

\section{Background}

\begin{frame}{Event ordering}
  \begin{itemize}
    \item Event ordering is essential to some applications:
      \begin{itemize}
        \item Transactions
        \item Logs
        \item Debugging and performance analysis
      \end{itemize}
    \item Ordering is based on timestamps\dots
    \item \dots collected from large ranges of machines.
    \item Need for clocks synchronization on (local) networks (frequency, phase
      and time).
  \end{itemize}
\end{frame}

\begin{frame}{NTP}
  \begin{itemize}
    \item Network time protocol (NTP) is a network protocol for clock
      synchronization.
    \item Provides accuracy within a few milliseconds (best case scenario).
    \item Not precise enough for some applications: events can occur within the
      same millisecond.
    \item Need for a higher accuracy.
  \end{itemize}
\end{frame}

\section{Overview of the precision time protocol (PTP)}

\begin{frame}{Overview}
  \begin{itemize}
    \item Precision time protocol (PTP) is a network protocol for clock
      synchronization.
    \item Down to sub-microsecond accuracy on local networks.
    \item Standardized by \code{IEEE 1588-2002}, \code{IEEE 1588-2008} and
      \code{IEEE 1588-2019}.
    \item Hierarchical leader/follower architecture for clock distribution.
      \begin{itemize}
        \item Leader ("grandmaster"), boundary and follower ("slave") clocks.
      \end{itemize}
    \item PTP packets may be transmitted over Ethernet or UDP over IPv4/IPv6,
      using multicast or unicast addresses.
      \begin{itemize}
        \item Depending on the publication used as reference, not all modes are
          available.
      \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}{PTP hierarchy 1/3}
  \begin{columns}
    \begin{column}{0.4\linewidth}
      \includegraphics[width=1.2\textwidth]{images/ptp-hierarchy-1.jpg}
    \end{column}
    \begin{column}{0.6\linewidth}
      \begin{itemize}
        \item Leader clock:
          \begin{itemize}
            \item Time source for the PTP network.
            \item Usually synchronize its clock to an external source (GNSS,
              etc\dots).
            \item Is an "ordinary clock" (has a single PTP network connection).
          \end{itemize}
      \end{itemize}
    \end{column}
  \end{columns}
\end{frame}

\begin{frame}{PTP hierarchy 2/3}
  \begin{columns}
    \begin{column}{0.4\linewidth}
      \includegraphics[width=1.2\textwidth]{images/ptp-hierarchy-2.jpg}
    \end{column}
    \begin{column}{0.6\linewidth}
      \begin{itemize}
        \item Boundary clock:
          \begin{itemize}
            \item Has multiple PTP network connections and relay accurate time:
              \begin{itemize}
                \item Synchronizes its clock against the leader.
                \item Acts as a clock source for the followers.
              \end{itemize}
            \item May become the leader if the current leader disappear.
            \item Having a boundary clock is optional.
          \end{itemize}
      \end{itemize}
    \end{column}
  \end{columns}
\end{frame}

\begin{frame}{PTP hierarchy 3/3}
  \begin{columns}
    \begin{column}{0.4\linewidth}
      \includegraphics[width=1.2\textwidth]{images/ptp-hierarchy-3.jpg}
    \end{column}
    \begin{column}{0.6\linewidth}
      \begin{itemize}
        \item Follower clock:
          \begin{itemize}
            \item Synchronize its clock to a leader (here, the boundary clock).
            \item May become the leader if the leader disappear.
            \item Is an "ordinary clock" (has a single PTP network connection).
          \end{itemize}
      \end{itemize}
    \end{column}
  \end{columns}
\end{frame}

\begin{frame}{PTP synchronization mechanism 1/2}
  \begin{columns}
    \begin{column}{0.5\linewidth}
      \includegraphics[width=\textwidth]{images/ptp-synchronization.jpg}
    \end{column}
    \begin{column}{0.5\linewidth}
      \begin{itemize}
        \item Time offset is computed based on timestamps of packet sent and
          received.
        \item Done by the follower.
        \item Timestamps made on the leader side are sent to the follower by
          follow up packets (\code{follow_up}, \code{delay_resp}).
      \end{itemize}
    \end{column}
  \end{columns}
\end{frame}

\begin{frame}{PTP synchronization mechanism 2/2}
  \begin{columns}
    \begin{column}{0.5\linewidth}
      \includegraphics[width=\textwidth]{images/ptp-synchronization.jpg}
    \end{column}
    \begin{column}{0.5\linewidth}
      The trip time include the transmit time and the delta between the two
      clocks:
      \[ d_{lf} = t_{1} - t_{0} + {\delta}t \]
      \[ d_{fl} = t_{3} - t_{2} + {\delta}t \]
      We assume the two trip times are equal, hence we have:
      \[ {\delta}t = \frac{1}{2}(t_{1} - t_{0} + t_{2} - t_{3}) \]
    \end{column}
  \end{columns}
\end{frame}

\begin{frame}{One-step vs two-step}
  \begin{itemize}
    \item PTP can work in two operating modes: \code{1-step} and \code{2-step}.
    \item \code{1-step} includes \( t_{0} \) in the \code{sync} packet. There is
      no \code{follow_up} packet.
      \begin{itemize}
        \item The difference lies in the leader side. It needs a hardware
          enabled device to include \( t_{0} \) in the \code{sync} packets.
        \item All followers (should) support both modes: there is no hardware
          requirement for receiving \code{1-step sync} packets.
      \end{itemize}
  \end{itemize}
\end{frame}

\section{Packet timestamping}

\begin{frame}{Timestamp accuracy}
  \begin{itemize}
    \item Packet timestamps, when used for PTP, must be {\bf accurate}: they
      play a critical role in the time offset computation.
    \item Ideally we would like a timestamp issued at the exact time of
      transmission, when the packet leave the device.
    \item Not possible in the real world, the timestamp has to occur before.
    \item Two possibilities: in software and in hardware.
  \end{itemize}
\end{frame}

\begin{frame}{Software timestamping}
  \begin{columns}
    \begin{column}{0.5\linewidth}
      \includegraphics[width=\textwidth]{images/software-timestamping.jpg}
    \end{column}
    \begin{column}{0.5\linewidth}
      \begin{itemize}
        \item Timestamp is done in the application, or in the kernel.
        \item Uses the system clock.
        \item Error and deltas are big:
          \begin{itemize}
            \item Timestamp is done far away from the actual transmission.
            \item A lot can interfere: scheduling, queuing, interrupts\dots
          \end{itemize}
      \end{itemize}
    \end{column}
  \end{columns}
\end{frame}

\begin{frame}{Hardware timestamping}
  \begin{columns}
    \begin{column}{0.5\linewidth}
      \includegraphics[width=\textwidth]{images/hardware-timestamping.jpg}
    \end{column}
    \begin{column}{0.5\linewidth}
      \begin{itemize}
        \item Timestamp is done in the hardware.
          \begin{itemize}
            \item Can be done in the \code{MAC},
            \item in a \code{PHY},
            \item or using a dedicated controller.
          \end{itemize}
        \item Uses a PTP hardware clock (PHC).
        \item Error and deltas are small.
          \begin{itemize}
            \item Timestamp occurs close to the actual transmission.
            \item The packet is already in the hardware.
          \end{itemize}
      \end{itemize}
    \end{column}
  \end{columns}
\end{frame}

\section{PTP offloading support in Linux}

\begin{frame}{Overview}
  \begin{itemize}
    \item Two mechanisms are combined to provide support for offloading PTP
      packets timestamping:
      \begin{itemize}
        \item The \ksym{SO_TIMESTAMPING} socket option.
        \item The PTP hardware clock (PHC) infrastructure.
      \end{itemize}
    \item Read the full documentation at
      \kfile{Documentation/networking/timestamping.rst}.
  \end{itemize}
\end{frame}

\begin{frame}{\ksym{SO_TIMESTAMPING} (1/3)}
  \begin{itemize}
    \item Configured using \code{setsockopt}.
    \item Generates timestamps on reception, transmission or both.
    \item Works for streams and datagrams.
    \item Supports multiple timestamp sources:
      \begin{itemize}
        \item \ksym{SOF_TIMESTAMPING_RX_SOFTWARE}: timestamp is generated just
          after the network device driver hands the packet to the Rx stack.
        \item \ksym{SOF_TIMESTAMPING_TX_SOFTWARE}: timestamp is generated in the
          network device driver, as close as possible to passing the packet to
          the hardware. Requires driver support and may not be available for all
          devices.
        \item \ksym{SOF_TIMESTAMPING_RX_HARDWARE}: requires driver support.
        \item \ksym{SOF_TIMESTAMPING_TX_HARDWARE}: requires driver support.
        \item and two other options not used for PTP applications:
          \ksym{SOF_TIMESTAMPING_TX_SCHED} and \ksym{SOF_TIMESTAMPING_TX_ACK}.
      \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}{\ksym{SO_TIMESTAMPING} (2/3)}
  \begin{itemize}
    \item Hardware timestamping must be initialized for each device driver
      expected to be used.
    \item Configuration passed using the \ksym{SIOCSHWTSTAMP} ioctl. Must choose
      a \code{tx_type} and an \code{rx_filter}.
    \item Possible values for \code{tx_type}:
      \begin{itemize}
        \item \ksym{HWTSTAMP_TX_OFF}
        \item \ksym{HWTSTAMP_TX_ON}: report timestamps through the socket error
          queue.
        \item \ksym{HWTSTAMP_TX_ONESTEP_SYNC}: insert timestamps directly into
          \code{sync} packets.
        \item \ksym{HWTSTAMP_TX_ONESTEP_P2P}: same as before but also insert
          timestamps into \code{delay_resp} packets.
      \end{itemize}
    \item Possible values for \code{rx_filter}:
      \begin{itemize}
        \item \ksym{HWTSTAMP_FILTER_NONE}
        \item \ksym{HWTSTAMP_FILTER_ALL}
        \item \ksym{HWTSTAMP_FILTER_PTP_V2_L2_EVENT}: PTP v2, Ethernet, all
          event packets.
        \item \ksym{HWTSTAMP_FILTER_PTP_V2_L4_SYNC}: PTP v2, UDP sync packets.
        \item For the full list, see \kfile{include/uapi/linux/net_tstamp.h}
      \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}[fragile]{\ksym{SO_TIMESTAMPING} (3/3)}
  \begin{minted}[fontsize=\footnotesize]{text}
# ethtool -T eth0
Time stamping parameters for eth0:
Capabilities:
        hardware-transmit     (SOF_TIMESTAMPING_TX_HARDWARE)
        software-transmit     (SOF_TIMESTAMPING_TX_SOFTWARE)
        hardware-receive      (SOF_TIMESTAMPING_RX_HARDWARE)
        software-receive      (SOF_TIMESTAMPING_RX_SOFTWARE)
        software-system-clock (SOF_TIMESTAMPING_SOFTWARE)
        hardware-raw-clock    (SOF_TIMESTAMPING_RAW_HARDWARE)
PTP Hardware Clock: 0
Hardware Transmit Timestamp Modes:
        off                   (HWTSTAMP_TX_OFF)
        on                    (HWTSTAMP_TX_ON)
        one-step-sync         (HWTSTAMP_TX_ONESTEP_SYNC)
Hardware Receive Filter Modes:
        none                  (HWTSTAMP_FILTER_NONE)
        all                   (HWTSTAMP_FILTER_ALL)
  \end{minted}
\end{frame}

\begin{frame}{Supporting \ksym{SO_TIMESTAMPING} in a device driver (1/2)}
  \begin{itemize}
    \item In a networking Ethernet driver (MAC), implementing:
      \begin{itemize}
        \item \ksym{get_ts_info} in \code{struct} \ksym{ethtool_ops}
        \item \ksym{ndo_do_ioctl} in \code{struct} \ksym{net_device_ops}, for
          \ksym{SIOCSHWTSTAMP} and \ksym{SIOCGHWTSTAMP}.
        \item Filling \ksym{hwtstamps} in \code{struct} \ksym{skbuff} with Rx
          timestamps.
        \item Calling \kfunc{skb_tstamp_tx} when a Tx timestamp is reported by
          the hardware.
      \end{itemize}
    \item In a networking PHY or other dedicated engines driver: implementing
      the \code{struct} \ksym{mii_timestamper} callbacks, in \code{struct}
      \ksym{phy_device}:
      \begin{itemize}
        \item \ksym{ts_info}
        \item \ksym{hwtstamp}
        \item \ksym{rxtstamp}
        \item \ksym{txtstamp} and calling \kfunc{skb_complete_tx_timestamp}
      \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}{Supporting \ksym{SO_TIMESTAMPING} in a device driver (2/2)}
  \begin{itemize}
    \item Both interfaces allow us to:
      \begin{enumerate}
        \item Report the timestamping capabilities (\ksym{ts_info} and
          \ksym{get_ts_info}).
        \item Configure the mode to use (\ksym{hwtstamp} and
          \ksym{ndo_do_ioctl}).
        \item Report Rx timestamps (\ksym{rxtstamp} and \ksym{hwtstamps}).
        \item Report Tx timestamps
          (\ksym{txtstamp}/\kfunc{skb_complete_tx_timestamp} and
          \ksym{skb_tstamp_tx}).
      \end{enumerate}
  \end{itemize}
\end{frame}

\begin{frame}{PTP hardware clock}
  \begin{itemize}
    \item PHC are used by hardware engines to timestamp packets.
    \item The PHC must be synchronized.
    \item Described by \code{struct} \ksym{ptp_clock_info}, which embeds operation
      callbacks:
      \begin{itemize}
        \item \ksym{gettimex64}: reports the current time from the hardware
          clock by filling a \code{timespec64} structure.
        \item \ksym{settime64}: sets the time on the hardware clock.
        \item \ksym{adjfine}: adjusts the frequency of the hardware clock, using
          an "offset from nominal frequency in parts per million, but with a 16
          bit binary fractional field".
        \item \ksym{adjtime}: shifts the time of the hardware clock by an
          \code{s64 delta}.
        \item \ksym{adjphase}: adjusts the phase of the hardware clock by an
          \code{s32 phase}.
      \end{itemize}
    \item The structure also contains a few parameters, including \ksym{max_adj}
      which defines the maximum frequency adjustment in parts per billion.
  \end{itemize}
\end{frame}

\begin{frame}{Driver examples}
  \begin{itemize}
    \item At Bootlin, we had the opportunity to introduce PTP offloading support
      for some hardware engines.
    \item For the Microsemi Ocelot network switch:
      \begin{itemize}
        \item \kfile{drivers/net/ethernet/mscc/}
      \end{itemize}
    \item For the Microsemi VSC85xx PHYs:
      \begin{itemize}
        \item \kfile{drivers/net/phy/mscc/mscc_ptp.c}
      \end{itemize}
  \end{itemize}
\end{frame}

\section{User-space PTP implementation}

\begin{frame}{Linux PTP}
  \begin{itemize}
    \item The Linux PTP project is an implementation of the Precision Time
      Protocol according to \code{IEEE-1588}, for Linux.
    \item Maintained by Richard Cochran, who also maintains PTP support in
      Linux.
    \item Provides a reliable implementation of PTP for Linux, and correctly
      uses the kernel interfaces for PHC and timestamping.
    \item Provides a few utilities including \code{ptp4l} and \code{phc2sys}.
  \end{itemize}
\end{frame}

\begin{frame}[fragile]{The \code{ptp4l} command}
  \begin{itemize}
    \item Implementation of PTP, for ordinary and boundary clocks.
    \item Can use software or hardware timestamping.
    \item Can perform PTP operations on top of UDP (IPv4/IPv6) and Ethernet.
    \item Can optionally use a configuration file.
  \end{itemize}
  \begin{minted}[fontsize=\footnotesize]{text}
# ptp4l -i eth0 -H -2 -m
selected /dev/ptp4 as PTP clock
port 1: INITIALIZING to LISTENING on INIT_COMPLETE
port 0: INITIALIZING to LISTENING on INIT_COMPLETE
port 1: new foreign master 7e7618.fffe.b52b26-1
selected best master clock 7e7618.fffe.b52b26
port 1: LISTENING to UNCALIBRATED on RS_SLAVE
master offset       1949 s0 freq    -552 path delay      2807
master offset       1953 s2 freq    -548 path delay      2807
port 1: UNCALIBRATED to SLAVE on MASTER_CLOCK_SELECTED
master offset       1974 s2 freq   +1426 path delay      2807
master offset        629 s2 freq    +673 path delay      2807
  \end{minted}
\end{frame}

\begin{frame}{The \code{phc2sys} command}
  \begin{itemize}
    \item Synchronizes two (or more) clocks.
    \item Typically used to keep the system clock in sync with the PHC.
    \item When using hardware timestamping, \code{ptp4l} adjusts the PHC and
      then \code{phc2sys} adjusts the system clock.
    \item When using software timestamping, \code{phc2sys} isn't used; the
      system clock is directly adjusted by \code{ptp4l}.
  \end{itemize}
\end{frame}

\begin{frame}
  \begin{center}
    \Huge
    Thank you! \\
    Questions? Comments? \\
    \vspace{1cm}
    \large
    \code{info@bootlin.com} \\
    \vspace{1cm}
    Slides under CC-BY-SA 3.0
  \end{center}
\end{frame}

\end{document}
