\documentclass[aspectratio=169,obeyspaces,spaces,hyphens,dvipsnames]{beamer}
\usepackage[utf8]{inputenc}
\usepackage{minted}
\usepackage{hyperref}

\mode<presentation>
\usetheme{Bootlin}

\def\signed #1{{\leavevmode\unskip\nobreak\hfil\penalty50\hskip2em
  \hbox{}\nobreak\hfil(#1)%
  \parfillskip=0pt \finalhyphendemerits=0 \endgraf}}

\newsavebox\mybox
\newenvironment{aquote}[1]
  {\savebox\mybox{#1}\begin{quotation}}
  {\signed{\usebox\mybox}\end{quotation}}

\title{Modernizing the NAND framework: The big picture}
\conference{Embedded Linux Conference Europe 2016}
\authors{Boris Brezillon}
\email{boris@bootlin.com}
\institute{Bootlin}
\slidesurl{http://bootlin.com/pub/conferences/2016/elce/brezillon-nand-framework}

\begin{document}

\addtocontents{toc}{\protect\setcounter{tocdepth}{-1}}
\section{Modernizing the NAND framework: The big picture}
\addtocontents{toc}{\protect\setcounter{tocdepth}{2}}

\begin{frame}{Boris Brezillon}
  \begin{itemize}
  \item Embedded Linux engineer and trainer at Bootlin
    \begin{itemize}
    \item Embedded Linux {\bf development}: kernel and driver
      development, system integration, boot time and power
      consumption optimization, consulting, etc.
    \item Embedded Linux, Linux driver development, Yocto Project / OpenEmbedded
      and Buildroot {\bf training courses}, with materials freely
      available under a Creative Commons license.
    \item \url{http://bootlin.com}
    \end{itemize}
  \item Contributions
    \begin{itemize}
    \item {\bf Kernel support for the AT91 SoCs} ARM SoCs from Atmel
    \item {\bf Kernel support for the sunXi SoCs} ARM SoCs from Allwinner
    \item {\bf Maintainer of the NAND subsystem}
    \end{itemize}
  \item Living in {\bf Toulouse}, south west of France
  \end{itemize}
\end{frame}

\begin{frame}{What is this talk about?}
  \begin{itemize}
  \item Detailing the current state of the NAND framework, its
	inconsistencies and limitations
  \item Proposing some approaches to rework it
  \item Sharing ideas and getting feedback
  \end{itemize}
\end{frame}

\subsection{Introduction to the NAND framework}

\begin{frame}{NAND framework position in the flash stack}
  \begin{center}
    \includegraphics[scale=0.2]{linux-flash-stack.pdf}
  \end{center}
\end{frame}

\begin{frame}{NAND subsystem architecture}
  \begin{itemize}
  \item Designed to interface with 'raw' NANDs
  \item (Thin?) layer between MTD subsystem and NAND controller drivers
  \item All the abstraction is handled through \code{struct nand_chip}
  \end{itemize}
\end{frame}

\subsection{Summarizing NAND framework limitations}

\begin{frame}{An ancient framework}
  \begin{itemize}
  \item Introduced in linux 2.4.6
  \item Was meant to support first NAND chips and simple NAND controllers
  \item Has evolved from time to time to support features exposed by
        newer NAND chips
  \item Disruptive growth
  \item Lot of code duplicated in NAND controller drivers
  \item Too open to enforce consistency across NAND controller drivers
        on some aspects...
  \item ... and too restrictive on other aspects to have really good
        performance
  \item Now lacks many functionalities to make raw NAND chips
        really attractive compared to eMMC
  \end{itemize}
\end{frame}

\begin{frame}{Too open to be easily maintainable}
  \begin{itemize}
  \item Mainly comes from the way the framework has grown
    \begin{itemize}
    \item Creation of a new hook in \code{struct nand_chip} for each new
	  feature
    \item Allows for high customization in each NAND controller driver...
    \item ... but also becomes a burden to maintain
    \item ... and prevents any transverse refactoring of the framework
    \end{itemize}
  \item Some core methods (which should be controller agnostic) can be
	overloaded by NAND controller drivers 
  \item Results in hardly maintainable code base, where some NAND
	controller drivers are only compatible with a limited number
	of NAND chips.
  \item This liberty at the controller driver level also penalizes MTD users:
	inconsistent behavior from one controller to another.
  \end{itemize}
\end{frame}

\begin{frame}{Too restrictive to allow controller side optimizations}
  \begin{itemize}
  \item Most modern NAND controllers are able to pipeline NAND commands and
        data transfers...
  \item ... but the framework splits commands and data transfer requests
  \item Some controllers are even capable of queuing several NAND commands
        thus allowing for maximum NAND throughput with minimum CPU usage...
  \item ... but the framework is serializing the page read/write requests
  \end{itemize}
\end{frame}

\begin{frame}{Does not take advantage of advanced NAND features}
  \begin{itemize}
  \item Modern NANDs provide different solutions to optimize read/write
	accesses
    \begin{itemize}
    \item Cached accesses
    \item Multi-plane accesses
    \item Multi-die accesses
    \item ... and probably other funky stuff invented by NAND vendors :)
    \end{itemize}
  \item An attempt was made to support cached program operations when doing
	sequential write accesses...
  \item ... the code is still there but the feature is always disabled
	because it was not providing a significant improvement at the time
	it was added
  \item Tested it recently on a DDR NAND, and the results are really
        encouraging (20\% improvement)
  \item Combined with NAND controller optimizations we could even get better
	results
  \end{itemize}
\end{frame}

\subsection{Clarifying NAND framework concepts/interfaces}

\begin{frame}{Needs to be redesigned to clarify the different concepts}
  \begin{itemize}
  \item The NAND framework is not providing a clear separation of the NAND
	controller and NAND chip concepts
  \item This leads to bad design in many NAND controller drivers
  \item Reworking the API to expose NAND controllers and automate NAND
	chip registration attached to those controllers should clarify
	the separation
  \item Some \code{struct nand_chip} hooks should be moved to the new
	\code{struct nand_controller} (not necessarily in their current
	form)
  \item However, we should not rush on moving methods from
	\code{struct nand_chip} to \code{struct nand_controller},
	otherwise we will end up with the same hardly maintainable mess
	but in a different place
  \end{itemize}
\end{frame}

\begin{frame}{NAND components association}
  \begin{center}
    \includegraphics[scale=0.25]{nand-elem-assoc.pdf}
  \end{center}
\end{frame}

\begin{frame}{Fuzzy design leads to erroneous driver implementation}
  \begin{itemize}
  \item The NAND chip interface is not clear
  \item Some methods are abused (either intentionally or unintentionally)
  \item Not clear when the methods should be implemented/overloaded
    \begin{itemize}
    \item Some are meant to be implemented by simple controllers
    \item Some are meant to be implemented by advanced controllers,
	  thus leaving more control to the NAND controller, but when they
	  are not implemented the core provides generic wrappers around
	  the simple controller methods
    \item Some should be implemented by NAND vendor specific code, but are
	  overloaded by the NAND controller driver
    \end{itemize}
  \item Everything is mixed up in \code{struct nand_chip}
  \item It's easy to get it wrong
  \end{itemize}
\end{frame}

\begin{frame}{Proof by example: \code{->cmdfunc()} misunderstanding}
  \begin{itemize}
  \item One key hook in \code{struct nand_chip} is \code{->cmdfunc()}
  \item Used to tell the NAND chip to launch a NAND operation
  \item Let's first detail the NAND protocol to understand what
	\code{->cmdfunc()} is supposed to do
  \end{itemize}
\end{frame}

\begin{frame}{NAND protocol (1)}
  \begin{center}
    \includegraphics[scale=0.4]{nand-protocol.pdf}
  \end{center}
\end{frame}

\begin{frame}{NAND protocol (2)}
  \begin{center}
    \includegraphics[scale=0.4]{nand-operations.pdf}
  \end{center}
\end{frame}

\begin{frame}{NAND protocol implementation}
  \begin{itemize}
  \item \code{->cmdfunc()} is only partially handling NAND operations
  \item Only takes care of sending \code{CMD} and \code{ADDR} cycles
  \item Data are transferred with \code{->read/write_buf/byte/word()}
  \item The core provides a default implementation of \code{->cmdfunc()}
  \item Default implementation relies on \code{->cmd_ctrl()} to actually
	send the \code{CMD} and \code{ADDR} cycles on the bus
  \end{itemize}
\end{frame}

\begin{frame}{NAND protocol implementation}
  \begin{center}
    \includegraphics[scale=0.4]{nand-proto-implem.pdf}
  \end{center}
\end{frame}

\begin{frame}{NAND protocol implementation limitation}
  \begin{itemize}
  \item This implementation worked well for simple controllers
	implementing \code{->cmd_ctrl()}
  \item But some controllers are now able to handle full NAND
	operations (including the data transfer)
  \item Hence the ability for NAND controller drivers to overload
	\code{->cmdfunc()}
  \item Introduces a few problems:
    \begin{itemize}
    \item \code{->cmdfunc()} does not provide enough information to predict
	  how many bytes the core will actually transfer
    \item Not all controllers are able to dissociate data transfers from the
	  \code{CMD} and \code{ADDR} cycles which forces some drivers to guess
	  the number of bytes the core will ask directly in \code{->cmdfunc()}
    \item Implies non-uniform support across NAND controllers (some controllers
	  will only support a subset of commands, thus preventing the use of
	  advanced features)
    \item Encourages people to support only a minimal set of operations
    \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}{Addressing this limitation}
  \begin{itemize}
  \item \code{struct nand_controller} should have a method to send full NAND
	operations (\code{->exec_op()} ?)
  \item We can still provide helpers to make this method rely on mechanisms
	similar to \code{->cmd_ctrl()} and \code{->read/write_byte/buf/word()}
  \item NAND controller drivers should not do any guessing depending on the
	operation type
  \item Should improve a bit the performance on controllers supporting a
	single interrupt event for the whole operation
  \item Will allow NAND core to implement different NAND operations in a
	generic manner
  \item Only limitation: some NAND controllers implement dedicated logic for
	some commands (READID, RESET, ...)
  \item We may need to define an higher level interface for these
	controllers
  \end{itemize}
\end{frame}

\subsection{Other problems}

\begin{frame}{Smartness is not necessarily good}
  \begin{itemize}
  \item NAND core code tries to fill the gaps between what is supposed to
	be implemented and what the NAND controller driver really implements
  \item Set its own default implementations to fill the missing
	\code{struct nand_chip} hooks...
  \item ... but it's not necessarily a good idea
    \begin{itemize}
    \item Sometimes the implementation is missing because the developer did not
	  care implementing it
    \item Blindly assigning a default implementation in this case can be worst
	  than reporting \code{-ENOTSUPP}
    \end{itemize}
  \item Solution:
    \begin{itemize}
    \item Export default implementations, and let NAND controller drivers fill
	  \code{struct nand_chip} appropriately
    \item Fail and complain when mandatory methods are missing
    \item Implement a dummy method returning \code{-ENOTSUPP} when the hook is
	  optional
    \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}{Adapting the NAND subsystem to developer needs}
  \begin{itemize}
  \item Nothing new here, but it is worth mentioning that this applies
	to the NAND subsystem too
  \item Already happening for some aspects
    \begin{itemize}
    \item Helpers to fix bitflips in erased pages (added in 4.4)
    \item DT parsing automation (added in 4.2 by Brian Norris)
    \item Timing setting automation (added in 4.9 by Sascha Hauer)
    \end{itemize}
  \item Need to be pushed even further
  \item The introduction of the NAND controller concept should help removing
	more boilerplate code
  \item As showed earlier with the \code{->cmdfunc()} example,
	the \code{nand_controller} interface has to take modern NAND controller
	capabilities and constraints into account
  \end{itemize}
\end{frame}

\begin{frame}{Isolating the NAND layer from the MTD layer}
  \begin{itemize}
  \item NAND and MTD concepts are currently mixed all over the NAND
	subsystem
  \item NAND controller drivers have to deal with the MTD struct
  \item Provide a better separation to hide MTD
    \begin{itemize}
    \item Ease maintenance (future NAND refactoring should not
	  impact the MTD layer and the other way around)
    \item Control driver accesses (allowing drivers to mess with
	  \code{mtd_info} is not a good idea)
    \item Clarify the code (only one object passed to NAND controller
	  methods instead of 2)
    \end{itemize}
  \item Rework has already started:
    \begin{itemize}
    \item Now \code{struct nand_chip} directly embeds a \code{struct mtd_info}
	  object
    \item New NAND controller methods should only be passed a \code{nand_chip}
	  object
    \end{itemize}
  \end{itemize}
\end{frame}

\subsection{Improving NAND performance}

\begin{frame}{Raw NAND flash bus}
  \begin{center}
    \includegraphics[scale=0.28]{rawnand-interface.pdf}
  \end{center}
\end{frame}

\begin{frame}{Controller side optimization}
  \begin{itemize}
  \item Modern NAND controllers support advanced features like:
    \begin{itemize}
    \item Pipelining NAND operation
    \item Advanced CS control, allowing interleaved operations
	  (do not have to wait for an operation on a NAND die to
	  finish before launching an operation on a different die)
    \item And probably other hacks to improve NAND traffic
    \end{itemize}
  \item The problem is, the NAND framework design does not allow the
	controller to do this kind of optimization.
    \begin{itemize}
    \item NAND chip accesses are serialized at the core level (prevents
	  pipelining NAND operations)
    \item Accesses to a given NAND controller are also serialized, thus
	  preventing multi-die or multi-chip accesses
    \end{itemize}
  \end{itemize}
\end{frame}


\begin{frame}{Controller side optimization: \code{->read()} example}
  \begin{center}
    \includegraphics[scale=0.28]{nand-read-seq.pdf}
  \end{center}
\end{frame}

\begin{frame}{Controller side optimization: \code{->read()} example}
  \begin{center}
    \includegraphics[scale=0.28]{nand-read-seq-reworked.pdf}
  \end{center}
\end{frame}

\begin{frame}{Chip side optimizations}
  \begin{itemize}
  \item We have different ways to optimize NAND performance
  \item But they pretty much all depend on NAND controller implementation
	to be relevant
  \item Some examples:
    \begin{itemize}
    \item Cached accesses
    \item Multi-plane accesses
    \item Multi-die accesses
    \item DDR or higher SDR mode support
    \end{itemize}
  \item Not all chips support those features
    \begin{itemize}
    \item We need a way to expose which features the chip supports
    \item ONFI and JEDEC already have a standard solution to expose that
	  (should be extended for non-ONFI/non-JEDEC chips)
    \item Optimization features are exposed by the mean of new NAND operations
	  (or extension of existing NAND operations)
    \item NAND specific hooks should be added to let the chip build a NAND
	  operation description
    \item In the end, the decision to use those optimizations should be left
	  to the controller
    \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}{Chip side optimizations: cached accesses}
  \begin{itemize}
  \item Cached accesses: the chip has 2 levels of cache, one is exposed
	to the user, and the other one is used to retrieve or push
	data to/from another NAND page
  \item Cached accesses are relevant when the time spent in ECC calculation
	and I/O transfer operations is not negligible compared to the
	READPAGE, PROGPAGE time.
  \item Particularly true for modern NAND chips with huge pages (greater
	than 4k)
  \end{itemize}
\end{frame}

\begin{frame}{Optimizing even more: NAND IO scheduling}
  \begin{itemize}
  \item NAND I/Os can be dispatched on different plane/dies
  \item Most controllers support accessing several dies in parallel
    \begin{itemize}
    \item While one operation is taking place on a specific die the
	  controller can interact with other dies
    \item The only contention is on the NAND bus: only a single die can
	  be addressed at a time, but once the command is sent and the
	  chip is busy, the controller can use the NAND bus to interact
	  with another die
    \end{itemize}
  \item Getting an efficient usage of the NAND bus requires scheduling
	I/O requests
    \begin{itemize}
    \item Queue NAND requests per-die, and not per-chip
    \item If we want multi-plane support, we should even have a per-plane
	  queue
    \item The scheduling algorithm should be generic
    \item The NAND controller should only de-queue requests from the different
	  queues
    \end{itemize}
  \item Still just an idea, not sure about the implementation details yet
  \end{itemize}
\end{frame}

\subsection{Supporting modern/non-ONFI/non-JEDEC chips}

\begin{frame}{Supporting modern/non-ONFI/non-JEDEC NANDs}
  \begin{itemize}
  \item MLC/TLC NAND chips need some specific features that are not
	accessible through standard commands (read-retry, SLC mode, ...)
  \item We need vendor specific code
  \item Doing that should also simplify the 'not so generic' chip detection
	code in \code{drivers/mtd/nand/nand_base.c}
  \item Better isolate vendor specific handling in vendor specific \code{.c}
	files
  \item Some \code{struct nand_chip} hooks should be filled in there
  \item Vendor specific tweaking is also required for ONFI or JEDEC compliant
	chips
  \item This approach should limit the number of full-id NAND entries in
	\code{nand_ids} table (and we may even be able to completely get rid
	of it)
  \item Proposal sent a few months ago: \url{http://lkml.iu.edu/hypermail/linux/kernel/1606.2/02892.html}
  \end{itemize}
\end{frame}

\subsection{Sharing code between different NAND based devices}

\begin{frame}{Sharing code between different NAND based devices}
  \begin{itemize}
  \item Linux kernel currently supports different NAND based devices using
	different interfaces
    \begin{itemize}
    \item Raw NANDs (those we are talking about in this talk)
    \item OneNAND NANDs
    \item SPI NANDs
    \end{itemize}
  \item Even if they don't use the same physical and logical interfaces,
	they may share enough to allow some factorization
    \begin{itemize}
    \item Memory array organization
    \item NAND type (SLC/MLC/TLC), and associated constraints
    \item Bad block table handling
    \item ECC handling?
    \item ...
    \end{itemize}
  \item Isolating those similarities and factorizing common code sounds
	like a good idea
  \end{itemize}
\end{frame}

\begin{frame}{Sharing code between different NAND based devices}
  \begin{itemize}
  \item Work already started by Brian Norris and Peter Pan
	(making bad block table code reusable for OneNAND and SPI NAND)
\item I proposed to go further (submitted an RFC):
    \begin{itemize}
    \item Create \code{struct nand_device}
    \item Make \code{struct nand_chip}, \code{struct onenand_chip} and
	  \code{struct spinand_chip} inherit from \code{struct nand_device}
    \item Move all the code under \code{drivers/mtd/nand} and create one
	  subdirectory per interface type
    \item See what else could be factorized
    \end{itemize}
  \end{itemize}
\end{frame}

\subsection{Any help is welcome!}

\begin{frame}{Come help if you want!}
  \begin{itemize}
  \item Lot of work to be done to make NAND devices attractive again
  \item At the core level:
    \begin{itemize}
    \item Share your ideas, or your vision of what would be an ideal
	  interface
    \item Propose implementations and/or review others proposals
    \item Test the new approach on your controllers/chips
    \end{itemize}
  \item At the driver level:
    \begin{itemize}
    \item Convert your driver(s) to the new infrastructures
    \item Review submissions from others
    \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}
  \begin{center}
  \Huge
  Questions? Suggestions? Comments?\\
  \vspace{1.5cm}
  \huge
  Boris Brezillon\\
  \large
  \vspace{0.5cm}
  \code{boris.brezillon@bootlin.com}
  \vspace{0.5cm}
  \newline Slides under CC-BY-SA 3.0\\
  \scriptsize
  \url{http://bootlin.com/pub/conferences/2016/elc/brezillon-nand-framework/}
  \end{center}
\end{frame}

\end{document}
