\subsubsection{#1}}
\newcommand{{FsaNet~}}{{FsaNet~}}
\newcommand{{state-of-the-art~}}{{state-of-the-art~}}
\newcommand{{FCN~\cite{long2015fully}}}{{FCN~\cite{long2015fully}}}
\newcommand{{EncNet~\cite{zhang2018context}}}{{EncNet~\cite{zhang2018context}}}
\newcommand{{APCNet~\cite{he2019adaptive}}}{{APCNet~\cite{he2019adaptive}}}
\newcommand{{ANN~\cite{zhu2019asymmetric}}}{{ANN~\cite{zhu2019asymmetric}}}
\newcommand{{GCNet~\cite{cao2019gcnet}}}{{GCNet~\cite{cao2019gcnet}}}
\newcommand{{DMNet~\cite{he2019dynamic}}}{{DMNet~\cite{he2019dynamic}}}
\newcommand{{DNLNet~\cite{yin2020disentangled}}}{{DNLNet~\cite{yin2020disentangled}}}
\newcommand{{CCNet~\cite{huang2020ccnet}}}{{CCNet~\cite{huang2020ccnet}}}
\newcommand{{DANet~\cite{fu2019dual}}}{{DANet~\cite{fu2019dual}}}
\newcommand{{ISANet~\cite{huang2019interlaced}}}{{ISANet~\cite{huang2019interlaced}}}
\newcommand{{UPerNet~\cite{xiao2018unified}}}{{UPerNet~\cite{xiao2018unified}}}
\newcommand{{Non-local~Network~\cite{wang2018non}}}{{Non-local~Network~\cite{wang2018non}}}
\newcommand{{EMANet~\cite{li2019expectation}}}{{EMANet~\cite{li2019expectation}}}
\newcommand{{OCRNet~\cite{yuan2020object}}}{{OCRNet~\cite{yuan2020object}}}
\newcommand{{PSANet~\cite{zhao2018psanet}}}{{PSANet~\cite{zhao2018psanet}}}
\newcommand{{PSPNet~\cite{zhao2017pyramid}}}{{PSPNet~\cite{zhao2017pyramid}}}
\newcommand{{DeepLabV3~\cite{chen2017rethinking}}}{{DeepLabV3~\cite{chen2017rethinking}}}
\newcommand{{DecoupledSegNets~\cite{li2020improving}}}{{DecoupledSegNets~\cite{li2020improving}}}
\newcommand{{DGCNet~\cite{zhang2019dual}}}{{DGCNet~\cite{zhang2019dual}}}
\newcommand{{HANet~\cite{choi2020cars}}}{{HANet~\cite{choi2020cars}}}
\newcommand{\mathbf{q}}{\mathbf{q}}
\newcommand{\mathbf{k}}{\mathbf{k}}
\newcommand{\mathbf{v}}{\mathbf{v}}
\newcommand{\mathrm{Softmax}}{\mathrm{Softmax}}
\def\emph{vs.}{\emph{vs.}}
\def\emph{i.e.,~}{\emph{i.e.,~}}
\def\emph{e.g.,~}{\emph{e.g.,~}}
\def\emph{etc}{\emph{etc}}
\def{\em et al.}{{\em et al.}}

\begin{document}
\title{FsaNet: Frequency Self-attention for Semantic Segmentation}
\author{
Fengyu Zhang,~
Ashkan Panahi,~
and Guangjun Gao
\thanks{F. Zhang and G. Gao are with the School of Traffic \& Transportation Engineering, Central South University, Changsha, 410075, China (e-mail: fengyuzhang@csu.edu.cn; gjgao@csu.edu.cn).  A. Panahi and also F. Zhang are with the Department of Computer Science and Engineering, Chalmers University of Technology, SE-412 96 Göteborg, Sweden.
}}

\markboth{IEEE Transactions on Image Processing,~Vol.~xx, No.~xx, November~2022}%
{Zhang \MakeLowercase{\textit{et al.}}:
FsaNet: Frequency Self-attention for Semantic Segmentation}

\maketitle
\begin{abstract}
Considering the spectral properties of images, we propose a new self-attention mechanism with highly reduced computational complexity, up to a linear rate. To better preserve edges while promoting similarity within objects, we propose individualized processes over different frequency bands. In particular, we study a case where the process is merely over low-frequency components. By ablation study, we show that low frequency self-attention can achieve very close or better performance relative to full frequency even without retraining the network. Accordingly, we design and embed novel plug-and-play modules to the head of a CNN network that we refer to as FsaNet. The frequency self-attention 1) takes low frequency coefficients as input, 2) can be mathematically equivalent to spatial domain self-attention with linear structures, 3) simplifies token mapping ($1\times1$ convolution) stage and token mixing stage simultaneously. We show that the frequency self-attention requires $87.29\% \sim 90.04\%$ less memory, $96.13\% \sim 98.07\%$ less FLOPs, and $97.56\% \sim 98.18\%$ in run time than the regular self-attention. Compared to other ResNet101-based self-attention networks, {FsaNet~} achieves a new {state-of-the-art~} result ($83.0\%$ mIoU) on Cityscape test dataset and competitive results on ADE20k and VOCaug. Code is accessible at
\url{https://github.com/zfy-csu/FsaNet}.
\end{abstract}

\begin{IEEEkeywords}
Self-attention, low frequency, linear complexity, frequency decoupling, semantic segmentation.
\end{IEEEkeywords}

\maketitle
\IEEEdisplaynontitleabstractindextext
\IEEEpeerreviewmaketitle

\section{Introduction}\label{sec:introduction}
\IEEEPARstart{S}{emantic} segmentation is the task of classifying different compartments of an image at a fine level of pixels. It is widely used in the fields of autonomous driving, industrial defect detection, and medical tissue segmentation. To improve the performance of semantic segmentation, two main principles are improving the  inner consistency of the objects in an image and refining their detail along their boundaries. From the perspective of traditional image processing, the inner body and the edge correspond to the low and high frequency components, respectively. Self-attention is a newly developed technique that can effectively improve the object's inner consistency by modeling the non-local context. However, it adopts identical learnable parameters to process all frequency components, which makes it difficult to achieve these two goals at the same time. On the other hand, the computational cost of self-attention is extremely high and the network integrated with self-attention have very high requirements on the hardware platform. This is a huge obstacle for real-world applications, especially in mobile scenarios such as autonomous driving.
To solve the above problems, we propose a new mechanism that treats different frequency components in a decoupled manner. We strive to design a more ingenious attention mechanism that can effectively improve the segmentation accuracy while reducing the complexity by exploiting the sparsity in the frequency domain.

Our study focuses on CNN-based self-attention networks. In this scenario, the input is a sequence of vectors obtained by row-wise unrolling the feature maps extracted from a convolutional network. A vector in the input is often referred to as a token. Denote the size of the feature maps by $H\times W\times C$. Then, the number of tokens $N$ is equal to $HW$. Each output token is a weighted sum of all input tokens, where the weights are a measure of similarity between individual tokens. The similarity between all tokens needs to be calculated and stored in a so-called attention map of size $N\times N$. Both memory and computation are quadratic with respect to $N$. The situation is even worse as dense prediction tasks tend to have higher resolution feature maps.

In response, we design a self-attention mechanism that heavily reduces the computational complexity, while improving the quality of the attention mechanism. We achieve this goal by first transforming the tokens into the frequency domain and alternatively taking frequency coefficients as input.  As a result, we can easily decouple different frequencies and simplify the computation by taking advantage of the sparsity in the frequency domain.  While the easiest decoupling method is to transform the input to the frequency domain, retain the specified frequency coefficients and then reconstruct a new input, in this way, the sparsity in the frequency domain cannot be effectively utilized. Hence, we find an equivalent method,  while ensuring that the result of reconstructing the new frequency coefficients are as consistent as possible with direct calculation in the spatial domain. Some aspects of our method is briefly explained in the sequel.

{\bf Replacing the input of self-attention} The input of a conventional self-attention is a one-dimensional vector extracted row-wise from a feature map. However, if the frequency domain transformation is performed directly on the vector, the similarity in both vertical and horizontal directions cannot be effectively utilized, the original spatial relations are lost, and the resulting frequency sparsity will be insignificant. Therefore, we restore the vector as a feature map, perform a two-dimensional frequency domain transform, retain the specified frequency block and expand it row by row as a new input. To simplify the calculation, we equivalently replace the above series of operations by multiplying a projection matrix. With fixed crop size, the matrix only needs to be calculated once during the entire process of training or inference.

\textbf{Modifying the structure of self-attention} The transformation of the input from the spatial domain to the frequency domain is linear. If the expression of self-attention also has a linear computational structure, it will be possible to transfer the self-attention operations from the spatial domain to the frequency domain by the associative law of matrix multiplication. In the standard approach, non-linearity is solely introduced by the exponential function in Softmax. We replace the exponential function with a linear approximation to obtain two self-attention mechanisms with linear structure. The replacement also facilitates the acquisition of global contextual information due to a more uniform distribution of attention weights.

\textbf{The benefits of frequency self-attention} Based on the above, self-attention applying to spatial position tokens is transformed into self-attention applying to frequency coefficients representing different scales. Thanks to the orthonormal nature of DCT transform, each frequency coefficient is independent and global. As such, removing frequency coefficient tokens will not do harm to receptive field. Benefit from the sparsity of the frequency domain, the token number can be greatly reduced. The method of compressing the computation in the frequency domain is tailored to the general characteristics of images. Even if the spatial self-attention is replaced directly by low frequency self-attention, it will only cause trivial difference on results. Therefore, the number of retained frequency components can be quickly determined through ablation study on input frequency components. In addition, we move the mapping convolution behind dimension reduction, which further reduces computational costs.

\textbf{Contributions}
 We highlight the novelties below:\\
 •	We develop two new self-attention modules, individually processing frequency components, which can preserve edge details while promoting subject similarity.\\
 •	We translate a series of operations of reshape, 2D-DCT, and 2D-IDCT to a linear map, yielding a simple transformation form between spatial position tokens and frequency tokens.\\
 •	We show the proposed frequency self-attention is mathematically equivalent to spatial self-attention with linear structure.\\
 •	We experimentally verify that {FsaNet~} outperforms or is comparable to the {state-of-the-art~} methods and has the lowest computational cost.

The remainder of this article proceeds as follows: \secref{sec:relatedwork} gives a brief review of related works. Our method is discussed in \secref{sec:Methodology}.
\secref{sec:experiments} analyzes ablation study on frequency components of self-attention and reports the performance in segmentation tasks on 3 datasets. The complexity analysis of our technique is presented in \secref{sec:Experiments_Complex}. Finally, concluding remarks and future works are described in \secref{sec:futurework}.

\section{Related Work}\label{sec:relatedwork}
In this section, we first briefly describe several popular semantic segmentation algorithms. Then, we review relevant simplified self-attention solutions in CNN and transformer.
\subsection{Semantic segmentation}\label{sec:Current_works}
At present, the most successful semantic segmentation methods employ a general-purpose backbones to encode features, such as ResNet\cite{he2016deep}, and Vit\cite{dosovitskiy2020image}. Then the features will be decoded to the pixel space by a decode-head network. {FCN~\cite{long2015fully}}~is the first encoder-decoder method. Since then, many improvements have been proposed by backbone enhancement, adding context modules, and decoupling body and edge. Our work is an improvement on semantic segmentation from the latter two perspectives.

\textbf{Backbone enhancement} CNN networks are widely used traditional visual backbones. For better performance, HRNet\cite{wang2020deep} designs a wide backbone to maintain high resolution; U-Net\cite{li2018h} utilizes feature in multi-stage with long-range skip connections. For real-time semantic segmentation, some lightweight backbones are proposed, such as bisenetv2\cite{yu2021bisenet} and CGNet\cite{wu2020cgnet}. After Vit\cite{dosovitskiy2020image} introduced Transformer from natural language processing (NLP) to computer vision (CV), Transformer backbones are becoming popular. SETR\cite{zheng2021rethinking}, Segmenter\cite{strudel2021segmenter}, SegFormer\cite{xie2021segformer}, SWIN\cite{liu2021swin}, and TWINs\cite{chu2021twins} outperformed CNN-based methods. Subsequently, some CNN-based methods achieved new state-of-the-art results, such as DWNet\cite{han2021demystifying} and ConvNeXt\cite{liu2022convnet}. CNN-Transformer hybrid backbone is the latest trend, such as Cvt\cite{wu2021cvt}, Next-ViT\cite{li2022next}.

\textbf{Context module design} Multiple types of context modules have been proposed to increase the receptive field size. {PSPNet~\cite{zhao2017pyramid}}~presented a pyramid pooling module by adaptively pooling at different scales. Compared with {PSPNet~\cite{zhao2017pyramid}}~, {HANet~\cite{choi2020cars}}~ introduces a height prior to avoid confusion in predicting the elevation of the objects; {UPerNet~\cite{xiao2018unified}}~ fuses different scale information at different stages of upsampling. {DeepLabV3~\cite{chen2017rethinking}}~proposed a spatial pyramid pooling module by pooling and atrous convolution with different dilated rate. Based on {DeepLabV3~\cite{chen2017rethinking}}, DeepLabv3+\cite{chen2018encoder} fuses low-level features with high-level features to improve the accuracy of segmentation boundaries. {DMNet~\cite{he2019dynamic}}~perform depth-wise convolution with filters of different sizes, which are dynamically generated through adaptive pooling at different scales. {EncNet~\cite{zhang2018context}}~ adopts channel attention to highlight class-dependent feature, and introduces a loss to de-emphasize the probability of a vehicle to appear in an indoor scene.
After {Non-local~Network~\cite{wang2018non}}~introduce self-attention as a context module, a lot of follow up studies has been proposed. {DANet~\cite{fu2019dual}}~adopts an extra channel attention module which is placed in parallel. Similarly, {DGCNet~\cite{zhang2019dual}}~model contextual information in spatial and channel dimensions by two branches of graph convolutional network. {PSANet~\cite{zhao2018psanet}}~has collect branch and distribute branch for adaptively learning attention maps, but requires more parameters and memory. There is also plenty of research devoted to simplifying self-attention, and related work is detailed in \secref{Self-attention simplification in CNN}.

\textbf{Decouple body and edge} The two tasks of improving edge contrast and consistency within objects are contradictory in nature. Therefore, some methods decouple these two processes. Gated-SCNN\cite{takikawa2019gated} considers an extra shape stream branch, which only only processes boundary information. {DecoupledSegNets~\cite{li2020improving}}~split the body feature and edge feature by flow filed learning. RPCNet\cite{zhen2020joint} enhance edge by supervising the boundary obtained by a spatial derivative. STDC-Seg\cite{fan2021rethinking} adopts the Laplacian convolution kernel to generate edges to assist the supervision of the segmentation. {DNLNet~\cite{yin2020disentangled}}~divides attention map into two independent terms, which tend to focus on the within-category and boundary respectively. Although many studies put forward the concept that the body and edge correspond to the high and low frequency components, our work is the first decouple method based on frequency domain transformation in signal processing field.

\subsection{Self-attention simplification}
Self-attention simplification in various scenarios has been extensively studied due to its quadratic complexity. Our work can also be regarded as a self-attention simplification.

\textbf{Self-attention simplification in CNN}\label{Self-attention simplification in CNN}
Some methods treat keys and values as bases and reduce their numbers. In {CCNet~\cite{huang2020ccnet}}, every pixel only collect information from the criss-cross path and obtain global information by recurrent processing. {ISANet~\cite{huang2019interlaced}}~adopts a short-range attention implemented within a block to capture information from nearby pixels and a long-range attention implemented between blocks to capture information from the pixels with a fix spatial interval distance. {ANN~\cite{zhu2019asymmetric}}~introduces pyramid pooling into key and value branch. ACFNet\cite{zhang2019acfnet} and {OCRNet~\cite{yuan2020object}}~characterizes a pixel based on the object class features, which calculated by pre-evaluated coarse segmentation results. {EMANet~\cite{li2019expectation}}~removes the convolution for query and iteratively estimate a compact set of Gaussian bases by expectation-maximization.
Some linear methods remove the restriction of Softmax to avoid explicitly modeling the attention map. Linear Attention\cite{li2020linear} considers a linear attention module, based on the first-order Taylor expansion, and A2-Net\cite{chen20182} gathers a compact features set through second-order attention pooling and modified the structure of self-attention into implement Softmax separately on query and key. Furthermore, {GCNet~\cite{cao2019gcnet}}~shares same attention map for each query to compress the attention map.

\textbf{Self-attention simplification in Transformer}
Some methods reduce the length of the key-value pairs. In CSWin\cite{dong2021cswin}, query tokens only concentrate on a cross-shaped window with a certain width. Twins\cite{chu2021twins} uses both a local self-attention and a global self-attention implemented on tokens achieved by downsampling each local window. Sparse Transformer\cite{child2019generating} retains the top-$k$ contributive elements in each attention map. Reformer\cite{kitaev2019reformer} searches and computes only some large attention values via locality sensitive hashing.
There are also linear methods, such as the Fast Autoregressive Transformers\cite{katharopoulos2020transformers}, Performer\cite{choromanski2020rethinking}, UFO-ViT\cite{song2021ufo}, and SOFT\cite{lu2021soft}. Furthermore, AFT\cite{zhai2021attention} replaces matrix products with Hadamard products.
Some methods employ self-attention locally. HaloNet\cite{vaswani2021scaling} implements self-attention on non-overlapping blocks internally, and padding the blocks with boundary pixels to compensate for severe global information loss. Swin\cite{liu2021swin} computes self-attention within a local window and exchanges information between different windows by a shifted window method. T2T-ViT\cite{yuan2021tokens} structurize the image by gradually by recursively aggregating adjacent Tokens.

\textbf{{FsaNet~} vs. other simplification methods}
From dimension reduction perspective, {FsaNet~} not only reduces the dimension for key and value, but also reduces the dimension for query at the same time. Compared with the above ways of selecting bases, we leverage the frequency domain sparsity to generate bases, which has better interpretability. Unlike the current linear attention methods, which first calculates the product of the key and value matrices, we combines dimension reduction and cleverly proposes a better order of computation so that the mapping convolution is also simplified. In addition, our method can also be viewed as local self-attention in the frequency domain, but requires no additional operations to compensate global information.

\section{Methodology}\label{sec:Methodology}
\CheckRmv{
\begin{figure}[t]
  \centering
  \begin{overpic}[width=0.6\linewidth]{structure/networkstruct.pdf}
  \end{overpic}
  \caption{The structure of frequency self-attention}
  \label{fig:networkstruct}
\end{figure}
}
The overall structure of our proposed network is shown in \figref{fig:networkstruct}. The raw input images are first processed by a backbone network to obtain the feature maps $X\in R^{C\times H\times W}$. In a standard architecture, each 2D map is row-wise expanded (vectorized), yielding the input $X^{'}\in R^{C\times HW}$ of a conventional spatial self-attention unit. Spatial self-attention treats all frequency components indiscriminately, which may cause great distortion on the edges while promoting the consistency of low-frequency components. Therefore, we first remove the high frequency components from $X$ and then feed the result, containing low frequency components, to the self-attention unit. We keep $k$ horizontal low frequency components and $k$ vertical low frequency components, where $k<H$ and $k<W$. A conceptual depiction of this procedure is presented by the red path in \figref{fig:idealgoal}, where each channel in the input $X$ undergoes a 2D-DCT transformation to achieve $F\in R^{C\times H\times W}$. Next, only the specified low frequency block $f\in R^{C\times k\times k}$ are retained, from which new feature maps $X_{f}\in R^{C\times H\times W}$ is reconstructed by an inverse 2D-DCT transform. Finally, $X_{f}$ is row-wise expanded to achieve the input $X_{f}^{'}\in R^{C\times HW}$ of a spatial self-attention unit and the output is $O^{'}\in R^{C\times HW}$. The above processes are depicted in the red path in \figref{fig:idealgoal}.\\
\CheckRmv{
\begin{figure}[t]
  \centering
  \begin{overpic}[width=\linewidth]{structure/idealgoal.pdf}
  \end{overpic}
  \caption{The ideal goal}
  \label{fig:idealgoal}
\end{figure}
}
\newcommand{\addFig}[1]{{\includegraphics[height=.087\textwidth]{#1.pdf}}}
\CheckRmv{
\begin{figure}[t]
  \centering
  \renewcommand{\arraystretch}{0.5}
  \setlength{\tabcolsep}{0.00mm}
  \begin{tabular}{cc}
    \addFig{structure/assumption2}&
    \addFig{structure/assumption3}\\
    (a) Equivalence path for $P$ & (b) Equivalence path for $G$\\
  \end{tabular}
  \caption{Conversion between spatial position tokens and frequency tokens}
  \label{fig:assumption23}
\end{figure}
}
The red path in \figref{fig:idealgoal} is not ideal. Existing self-attention simplification methods show that there is a lot of redundant computation in the spatial domain, suggesting that the frequency blocks $f$ that need to be preserved tend to be small. Therefore, we design the operations in the green path in \figref{fig:idealgoal} to be equivalent to the result of the red path, which takes advantage of sparsity to simplify computation while decoupling frequencies. $X$ is reshaped to $X^{'}$, and then transformed by Linear operation 1 to $f^{'}\in R^{C\times k^{2}}$, exclusively containing the desired frequency components. Next, $f'$ is processed by a frequency self-attention unit to achieve $ o^{'}\in R^{C\times k^{2}}$, which is then transformed by Linear operation 2 to achieve the self attention output $O^{'}$ in the spatial domain. In this way, the complexity of self-attention is reduced from quadratic in $HW$ to quadratic in $k^{2}$. For this purpose, we make $f'$ from row-wise expanding $f$ and propose a linear projection map $P\in R^{HW\times k^{2}}$ which makes $f'=X^{'}P$ (Linear operation 1 in the green path), as shown in \figref{fig:assumption23}(a). We also propose a $G\in R^{k^{2}\times HW}$ for dimensionality expansion which makes $X_{f}^{'}=f'G$, as shown in \figref{fig:assumption23}(b). The exact description of $P,G$ will be discussed in Section \ref{sec:PG}.
Note that the frequency self-attention in the green path does not have as clear intuition as spatial self attention. It is difficult to propose an independent design for its architecture. Hence, we start with spatial self-attention analysis and assume that frequency self-attention share identical learnable parameters, which enables them transfer directly without even retraining. The spatial self-attention consists of two stages: token mapping and token mixing. Token mapping is common among most self-attention mechanisms. $X^{'}$ is first linearly mapped into matrices $Q$, $K$, $V$. The learnable parameters of self-attention are introduced in the token mapping stage, including: $W_{q}\in R^{d_{k},C}$, $B_{q}\in R^{d_{k},1} $, $W_{k}\in R^{d_{k},C}$, $B_{k}\in R^{d_{k},1}$, $W_{v}\in R^{d_{v },C}$ and $B_{v}\in R^{d_{v},1}$. The relation between these parameters can be described by $Q=W_{q}X^{'}+B_{q}$, $K=W_{k}X^{'}+B_{k}$ and $V=W_{v}X^{'}+B_{v}$.
Token mixing refers to the evaluation of the output. Our analysis is based on Non-local Embedding Gaussian self-attention\cite{wang2018non}, in which the process can be described as:
\begin{eqnarray}\label{eq:Non-local Embedded Gaussian}
O^{'}=V\mathrm{Softmax}(K^{T}Q).
\end{eqnarray}

In $Q$, $K$, and $V$, each column corresponds to a q(uery), k(ey), and v(alue) vector, respectively. Each column of $K^{T}Q$ corresponds to the similarity between one $\mathbf{q}$ vector (a column of $Q$) and all $\mathbf{k}$ vectors (a column of $K$). Next, Softmax is applied to each column to ensure that the weights are positive and sum to $1$. By doing this, each output vector is computed as a weighted sum of all $\mathbf{v}$ vectors. It is worth noting that by experiments in \secref{sec:experiments:bias ablation} we found that $B_{q}$, $B_{k}$, $B_{v}$ can be removed in a well-trained network, with no harm to the performance. Therefore, we removed these biases in subsequent analyses. To take low frequency feature maps as input, we replace $X^{'}$ with $X^{'}_{f}=f'G$. Correspondingly, the $Q$, $K$ and $V$ can be replaced with $Q_f=W_{q}X^{'}_{f}=W_{q}f^{'}G$, $K_f=W_{k}X^{'}_{f}=W_{k}f^{'}G$ and $V_f=W_{v}X^{'}_f=W_{v}f^{'}G$. Further, the replacement corresponds to the red path and yields:
\begin{eqnarray}\label{eq:lowfrequency_attention}
    \begin{small}
    O^{'}=W_{v}f^{'}G\mathrm{Softmax}[(W_{k}f^{'}G)^{T}(W_{q}f^{'}G)]
    \end{small}
\end{eqnarray}

Consider the following grouping:
\begin{eqnarray}\label{eq:green_path_grouping}
    \begin{small}  O^{'}=\underbrace{(W_{v}f^{'})\underbrace{G\mathrm{Softmax[G^{T}}}\underbrace{(W_{k}f^{'})^{T}(W_{q}f^{'})}}G].
    \end{small}
\end{eqnarray}

Even if the Softmax operator is nonlinear, the expression in \eqref{eq:lowfrequency_attention} can be viewed as a composition of a series of operations, which can be grouped in different ways based on the associative law of function composition. The grouping indicates that we can achieve the output by reconstructing the new coefficients achieved by frequency self-attention, as depicted in the green path. However, due to the Softmax, the proposed grouping does not provide a significant computational benefit. Hence, a proper spatial self-attention structure need to be sought with suitably approximated Softmax and next replace its input with the low-frequency components.

\subsection{What's a proper spatial self-attention structure for FsaNet?}
Let us start by the definition of Softmax, which consists of applying an exponential function and the sum normalization:
$\mathrm{Softmax}(x)_i = e^{x_i}/\sum\limits_{j}e^{x_j}$.
In \eqref{eq:Non-local Embedded Gaussian}, $e^{K^{T}Q}$ denote the element-wise exponent of $K^{T}Q$ and define $\lambda\in R^{1\times HW}$ as the element-wise inverse of the column-wise sum of $e^{K^{T}Q}$. We may also write  $\lambda=1./\left(\mathbf{1}_{HW}^{T} e^{K^{T}Q}\right)$. Note that $\mathrm{Softmax}(K^{T}Q)$ is the column-wise application of the rule, which corresponds to the sum normalization of each column of $e^{K^{T}Q}$. This can be implemented by multiplying $e^{K^{T}}Q$ from right to a diagonal matrix $\rho$ with the elements of $\lambda$ as its diagonal. Hence, \eqref{eq:Non-local Embedded Gaussian} can be reformulated as:
\begin{eqnarray}
  \begin{aligned}
     O'=V e^{K^{T}Q} \rho =V e^{K^{T}Q} \left(1./\mathrm{diag}\left(\mathbf{1}_{HW}^{T} e^{K^{T}Q}\right)\right).
  \end{aligned}
\end{eqnarray}

Now we observe that the nonlinearity of the exponential function prevents a further simplification, and an approximation is hence sought. It is not difficult to think of approximating the exponential function by its first-order Taylor series at $0$. Then, the entire procedure will include only matrix multiplication and addition, as we elaborate. Based on the approximation of the exponential terms, we obtain an approximation of Softmax, which we refer to as LinSoftmax:
\newcommand{\mathrm{LinSoftmax}}{\mathrm{LinSoftmax}}
\begin{eqnarray}
\mathrm{LinSoftmax}(x)_{i} = (1+x_i)/\sum\limits_{j}(1+x_j)
\end{eqnarray}

In order to reduce the error caused by approximation, we modify the attention module to ensure each element in $K^TQ$ fall in $[-1\ 1]$. In this interval, $1+x$ is close to $e^{x}$. When $x>1$, $e^{x}$ grows much faster than $1+x$. When $x< -1$, $e^{x}$ approaches $0$ but $1+x$ decreases unboundedly. The modification should not introduce substantial computational limitations. Our main strategy is to introduce two affine maps $\rho_{k}\in R^{HW\times HW}$ and $\rho_{q}\in R^{HW\times HW}$, respectively normalizing $K$ and $Q$. To avoid arbitrary matrix operations of size $HW\times HW$, we let $\rho_{k}$ and $\rho_{q}$ be diagonal. Thus, the corresponding matrix multiplications can be replaced by the Hadamard product of their diagonal vectors $\lambda_k\in R^{1\times HW}$ and $\lambda_q\in R^{1\times HW}$, which is significantly faster. In summary, we modify \eqref{eq:Non-local Embedded Gaussian} to:
\begin{eqnarray}\label{linsoftmax_rho_kq}
O'= V\mathrm{LinSoftmax}((K\rho_k)^T  Q\rho_q).
\end{eqnarray}

In the following, we consider two choices for $\rho_q$ and $\rho_k$ and consequently achieve two kinds of self-attention structure for FsaNet. The first one has a highly simplistic design and the second one is more complex in design.

\myPara{Non-local Dot product self-attention structure}\label{sec:startegy1}
The first strategy is to choose $\rho_q$ and $\rho_k$ as identity matrices when the self-attention is placed after the CNN backbone. In the following, we provide a combination of analytical and empirical evidence to support this choice. We conduct experiments on Cityscapes\cite{cordts2016cityscapes} val dataset based on four trained CNN networks with the module in \eqref{eq:Non-local Embedded Gaussian} provided by a semantic segmentation toolbox MMSegmentation\cite{mmseg2020}. After training, we studied the elements of $K^T Q$ and observed that only about 0.005\% of the elements are out of $[-1\ 1]$. We also observe that the distribution of dot product values for all points is similar, which is visualized in \figref{fig:distribution}.
\CheckRmv{
\begin{figure}[t]
  \centering
  \begin{overpic}[width=0.7\linewidth]{linsoftmax/distribution.pdf}
  \end{overpic}
  \caption{Distribution of dot product values for one point}
  \label{fig:distribution}
\end{figure}
}
As seen, most dot product values are close to $0$. Based on this observation, we may ignore normalization by setting $\lambda_q$ and $\lambda_v$ to all-one vectors. Correspondingly, $\rho_q$ and $\rho_k$ are identity matrices. As a result, we directly replace the Softmax with LinSoftmax. Our numerical studies show that the linear replacement will lead to only 0.1\% performance drop in mIoU. In this case, \eqref{eq:Non-local Embedded Gaussian} can be simply approximated by $O'=V\mathrm{LinSoftmax}(K^T Q)$. With $\lambda=1./(HW\mathbf{1}_{HW}^{T}+\mathbf{1}_{HW}^{T}K^T Q)$ and $\rho=\mathrm{diag}(\lambda)$, it can be equivalent to:
\begin{eqnarray}\label{eq:nonlocalsimpl}
  \begin{aligned}
   O'=V(\mathbf{11^{T}}+K^T Q)\rho.
  \end{aligned}
\end{eqnarray}

This linear form is similar to Non-local Dot product module:
\begin{eqnarray}\label{eq:Non-local Dot product}
 O'=VK^TQ/(HW).
\end{eqnarray}

The two different places between them are: $\lambda$ includes an additional term $\mathbf{1}_{HW}^{T}K^T Q$ and $O'$ includes a term $V\mathbf{11^{T}}\rho$. To understand the effect of the first term, we conduct experiments where we set $\lambda=1./(HW\mathbf{1}_{HW}^{T})$. We found out that the mIoU fluctuates just about 0.02\%, which means that altering $\lambda$ is not critical on the performance. Based on this, a simpler approximation expression can be obtained:
\begin{eqnarray}\label{eq:attention_linear_1}
   O' =V\mathbf{11^{T}}/(HW)+V K^T Q/(HW).
\end{eqnarray}
\CheckRmv{
\begin{figure}[t]
  \begin{overpic}[width=\linewidth]{linsoftmax/Softmax_noconstraintLinSoftmax.pdf}
  \end{overpic}
  \caption{The left figure corresponds to $Softmax(K^TQ)$, the right figure corresponds to $(\mathbf{11^{T}}+K^{T}Q)/(HW)$. For each figure, the attention maps of all points is displayed in the lower right corner, where each row represents the attention map of one single point against all other points. The attention map of the point marked by the red line is displayed in the upper left corner. The $\ell_2$ norm of the $HW$ columns in $Q$ is shown in the lower left corner. The $\ell_2$ norm of the $HW$ columns in $K$ is shown in the upper right corner. }\label{fig:Softmax_noconstraintLinSoftmax}
\end{figure}
}
To further understand the difference of approximating \eqref{eq:Non-local Embedded Gaussian} with \eqref{eq:attention_linear_1}, we visualize the two kinds of attention maps in \figref{fig:Softmax_noconstraintLinSoftmax}, calculated by $\mathrm{Softmax}(K^TQ)$ and $(\mathbf{11^{T}}+K^{T}Q)/(HW)$ based on the same $Q$, $K$ and $V$. The learned parameters in $Q$, $K$ and $V$ are achieved from a well-trained CNN network with Non-local Embedded Gaussian module. It is seen that there is a slight change among the entire attention map, but for each point, the attention map is almost unchanged. This suggests that the assignment of attention weights is hardly affected by the approximate expression in \eqref{eq:attention_linear_1}. Next, the global average pooling term $V\mathbf{11^{T}}/(HW)$ is removed. In this case, we observe that mIoU will decrease $3.3\%$. However, this reduction is a result of the learned parameters being trained with the module in \eqref{eq:Non-local Embedded Gaussian} rather than \eqref{eq:Non-local Dot product}. In fact, noting that removing the global average pooling term $V\mathbf{11^{T}}/(HW)$ simply leads to the non-local dot product module in \eqref{eq:Non-local Dot product} and according to the previous studies on the latter \cite{wang2018non}, we know that retraining the network with the model in \eqref{eq:Non-local Dot product} will recover the loss. In summary, all observations indicate that \eqref{eq:Non-local Dot product} can be used as a structure base to obtain our framework in \eqref{eq:green_path_grouping} by a series of empirically justified simplifications.
As a final note, the purely multi-linear form in \eqref{eq:Non-local Dot product} brings another benefit. It can be simultaneously viewed as both spatial attention and channel attention depending on the order of operations. Computing the product of the leftmost matrices first leads to channel self attention, while computing the product of the rightmost matrices first leads to spatial self attention. This observation, makes the non-local dot product module adaptable to different forms of attention by parameter adjustment.
\myPara{Normalized LinSoftmax self-attention structure}
\label{sec:strategy2}
Although the empirical performance of the non-local dot product form is often satisfactory, its stability in a network other than CNN may not be guaranteed. To overcome this limitation, we consider another strategy of selecting $\lambda_k$ and $\lambda_q$ as the element-wise inverse of the $\ell_{2}$ norms of the $\mathbf{q}$ and $\mathbf{k}$ vectors. In this case, we make sure that $\mathbf{q}\cdot \mathbf{k}$ is not only strictly restricted to $[-1\ 1]$, but also represents a cosine similarity metric. This form of self-attention has also been considered in Linear attention\cite{li2020linear}. Based on the pre-implemented $\ell_2$ normalization, self-attention can also be expressed as the product of matrices, as we elaborate.
First, we denote by $\|Q\|_2,\|K\|_2$ the vectors respectively containing the $\ell_2$ norms of the columns of $Q$ and $K$ (i.e. along the channel dimension)\footnote{Note that $\|\ldotp\|_2$ does not denote the operator norm of a matrix and its output is a vector.}, and then define:
\newcommand{\mathrm{diag}}{\mathrm{diag}}
\begin{eqnarray}
\lambda_{q}=1./\left\|Q\right\|_2,\quad \lambda_{k}=1./\left\|K\right\|_2.
\end{eqnarray}

We further define $\rho_{q}=\mathrm{diag}(\lambda_{q}), \quad \rho_{k}=\mathrm{diag}(\lambda_{k})$. In order to implement summation normalization, we remind:
\begin{eqnarray}
\rho=\mathrm{diag}(1./\left(\mathbf{1}_{HW}^{T}[\mathbf{1}\mathbf{1}^{T}+(K\rho_{k})^{T}(Q\rho_{q})])
\right)
\end{eqnarray}

Then, we can calculate the output by:
\begin{eqnarray}\label{eq:LinSoftmax_l2_reg}
\begin{aligned}
   O^{'}=V[\mathbf{1}\mathbf{1}^{T}+(K\rho_{k})^{T}(Q\rho_{q})]\rho.
\end{aligned}
\end{eqnarray}

To understand the difference caused by the approximation, we visualize the two kinds of attention maps in \figref{fig:Softmax_normLinSoftmax},  calculated by $\mathrm{Softmax}((K\rho_{k})^{T}(Q\rho_{q}))$ and $\mathrm{LinSoftmax}((K\rho_{k})^{T}(Q\rho_{q}))$ based on the same $Q$, $K$ and $V$. The $Q$, $K$ and $V$ are achieved from a well-trained CNN network with $\ell2$ normalized LinSoftmax self-attention.
\CheckRmv{
\begin{figure}[t]
  \begin{overpic}[width=\linewidth]{linsoftmax/Softmax_normLinSoftmax.pdf}
  \end{overpic}
  \caption{ The left figure corresponds to $\mathrm{Softmax}((K\rho_{k})^{T}(Q\rho_{q}))$, the right figure corresponds to $\mathrm{LinSoftmax}((K\rho_{k})^{T}(Q\rho_{q}))$. The implications of each subfigure are identical with \figref{fig:Softmax_noconstraintLinSoftmax}.}\label{fig:Softmax_normLinSoftmax}
  \label{fig:NormSoftmax_NormLinSoftmax}
\end{figure}
}
It is seen that the weight distribution of attention is basically unchanged, although the difference between the weights has narrowed somewhat.
\newcommand{\mathbb{R}}{\mathbb{R}}
\subsection{Replace the input with its low frequency components}
Base on the above, two structures for frequency self-attention are obtained. Then we need to replace the input with its low frequency components. For this purpose, we introduce $P$ and $G$ as shown in \figref{fig:assumption23}. We also find that decomposition and reconstruction with $P$ and $G$ have similar properties to the decomposition and reconstruction of 1D-DCT, namely $G=P^{T}$ and $P^{T}P=E$. These good properties can further simplify the computation. Therefore, we can replace the $Q$, $K$ and $V$ with their low-pass filtered version $Q_f=W_{q}f'P^T$, $K_f=W_{k}f'P^T$ and $V_f=W_{v}f'P^T$.
By implementing the replacement and applying $P^{T}P=E$ to the Non-local Dot-product structure in \eqref{eq:Non-local Dot product}, a series of procedures depicted in the green path in \figref{fig:idealgoal} can be obtained by:
\begin{eqnarray}\label{eq:fsa_dot_PG}
O'=(W_{v}f')(W_{k}f')^{T}(W_{q}f')/(HW) P^T
\end{eqnarray}

For this situation, $f'=X'P$ corresponds to Linear operation 1, $(W_{v}f')(W_{k}f')^{T}(W_{q}f')/(HW)$ corresponds to the frequency self-attention and the operation of multiplying $P^T$ corresponds to Linear operation 2.

Similarly, we replace the $Q$, $K$, $V$ in the Normalized LinSoftmax self-attention structure in \eqref{eq:Non-local Dot product}, which leads to a sequence of steps depicted in the green path in \figref{fig:idealgoal} by:
\begin{eqnarray}\label{eq:lowfre_spation_attention_lin}
\begin{small}
\begin{aligned}
&\rho_{q}=\mathrm{diag}(1./\left\|W_{q}f^{'}P^T\right\|_2),\quad \rho_{k}=\mathrm{diag}(1./\left\|W_{k}f^{'}P^T\right\|_2)\\
&\rho=\mathrm{diag}(1./\left(HW\mathbf{1}_{HW}^{T}+\mathbf{1}_{HW}^{T}\rho_{k}P(W_{k}f^{'})^{T}(W_{q}f^{'})P^{T}\rho_{q}\right))\\
&O^{'}=(W_{v}f^{'}(P^{T}\rho_{k}P)(W_{k}f^{'})^{T}(W_{q}f^{'})P^{T}\rho_{q}+W_{v}f^{'}P^{T}\mathbf{1}\mathbf{1}_{HW}^{T})\rho.
\end{aligned}
\end{small}
\end{eqnarray}

In this case, $f'=X'P$ corresponds to Linear operation 1, $W_{v}f^{'}(P^{T}\rho_{k}P)(W_{k}f^{'})^{T}(W_{q}f^{'})$ corresponds to the frequency self-attention unit and the operations of multiplying $P^{T}\rho_{q}$, adding $W_{v}f^{'}G\mathbf{1}\mathbf{1}_{HW}^{T}$, and multiplying $\rho$ corresponds to Linear operation 2.
So far, we have finished the design of frequency self-attention in the green path and it remains to the exact expression for $P,G$ and prove the above properties. Hence, we provide a detailed discussion in the following part.
\myPara{Definition of \texorpdfstring{$P$}. and \texorpdfstring{$G$}.}\label{sec:PG}
we first define two reshape functions and two 2D-DCT transform functions.
We denote by $\psi$ a reshape  function of a 3d-tensor into a 2D-tensor. $\psi$ acts channel wise, and on every channel, row-wise vectorizes a 2D map to a 1D map. Note that we use the same notation $\psi$ for any matrix size. $\psi^{-1}$ represents the inverse reshape function from a 2d-tensor into a 3d-tensor,  converting 1D map to a 2D map on every channel. Further, we denote by $\phi$ the 2D-DCT transform of the selecting $k$ lowest horizontal and $k$ lowest vertical bases. It transforms a 3D-tensor in $\mathbb{R}^{C\times  H\times W}$ into $\mathbb{R}^{C\times  k\times k}$. $\phi^{-1}$ represents 2D-IDCT reconstruction based on $k$ horizontal and $k$ vertical low-frequency bases and zero padding. It transform a 3D-tensor in $R^{C\times  k\times k}$ to $R^{C\times  H\times W}$.
Since the above operation does not introduce any non-linearity, there exists a matrix $P\in R^{HW\times k^{2}}$ such that $f^{'}=\psi(\phi(\psi^{-1}(X^{'})))=X^{'}P$, as shown in \figref{fig:assumption23} (a). Similarly, we observe that there is a matrix $G\in R^{k^{2}\times HW}$ such that $X_{f}^{'}=\psi(\phi^{-1}(\psi^{-1}(f^{'})))=f^{'}G$, as shown in (b) of \figref{fig:assumption23}. Based on these definitions, the complicated series of operations are translated to a linear map, for which an explicit formula can be established. For any matrix $A\in R^{C_{1}\times C_{2}}$,$B\in R^{C_{2}\times N}$, each channel (row) in $AB$ is a linear combination among the channels (rows) in $B$. On the other hand, $\psi(\phi(\psi^{-1}(\ldotp)))$ is a linear operation of the feature map acting individually on each channel. Now we consider the composition $\psi(\phi(\psi^{-1}(AB)))=(AB)P$ of these two operations and note that $(AB)P=A(BP)=A\psi(\phi(\psi^{-1}(B)))$. In other words, $A$ acts on the rows of $B$ while $\psi(\phi(\psi^{-1}(\ldotp)))$ acts on columns. Then, we have $\psi(\phi(\psi^{-1}(AB)))=A\psi(\phi(\psi^{-1}(B)))$. Similarly, we obtain: $\psi(\phi^{-1}(\psi^{-1}(AB)))=A\psi(\phi^{-1}(\psi^{-1}(B)))$. Let $E_{1}^{'} \in {HW\times HW}$ be the identity matrix, we get: $f^{'} =\psi(\phi(\psi^{-1}(X^{'}E_{1}^{'})))=X^{'}\psi(\phi(\psi^{-1}(E_{1}^{'})))$. Similarly, let $E_{2}^{'} \in R^{hw\times hw}$ be another identity matrix and combine $X_{f}^{'}=\psi(\phi^{-1}(\psi^{-1}(f^{'}E_{2}^{'})))=f^{'}\psi(\phi^{-1}(\psi^{-1}(E_{2}^{'})))$. Hence, we conclude that:
\begin{eqnarray}
\begin{aligned}
P=\psi(\phi(\psi^{-1}(E_{1}^{'}))),\quad G=\psi(\phi^{-1}(\psi^{-1}(E_{2}^{'}))).
\end{aligned}
\end{eqnarray}
\myPara{Properties of \texorpdfstring{$P$}. and \texorpdfstring{$G$}.}
In this section, we provide the proof for the properties $G=P^{T}$ and $P^{T}P=E$. To start, we calculate the elements of $P$ and $G$. Let $D_n$ denote a complete $n$-dimensional DCT matrix, and each column represent a DCT basis corresponding to a single frequency. We denote $D_n=\left[d_{i,j}:=\cos\left(\frac\pi n\left(i+\frac 12\right)j\right)\right]$ for $i=0,1,\ldots,n-1$ and $j=0,1,\ldots,n-1$\footnote{In this paper, we index an $n-$array by $0,1,\ldots,n-1$. This holds true for multi-dimensional arrays.}.
Suppose $D_{n,k}$ consists of the set of basis (columns) in $D_n$  corresponding to the $k\leq n$ lowest frequencies. In other words, $D_{n,k}=[d_{i,j}]$ for $i=1,2,\ldots,n$ and $j=1,2\ldots,k$. For a 1d signal $\mathbf{ x^{T}}$, $\mathbf{ x^{T}} D_{n,k}$ represents the low frequency coefficient vectors. As such, $\mathbf{ x^{T}}D_{n,k}D_{n,k}^{T}$ is the reconstructed signal that only contains the lowest $k$ frequencies, i.e. a low-pass filtered signal. Since the columns of $D_n$ are orthonormal, $D^T_{n,k}D_{n,k}$ is always the identity matrix. Next, we note that the 2D-DCT transform of a $H\times W$ signal $X$ (with a single channel $C=1$) can be obtained by performing $D_H^TXD_W$. In particular, the $k\times k$ coefficients of the $k$ lowest horizontal and vertical frequencies is given by $X_f=\phi(X)=D_{H,k}^TXD_{W,k}$. For multiple channels, this procedure is repeated individually for each channel. Now, we compute each element in $P$ and $G$ individually.

\textbf{Calculating $P$:} Let $E_{1}^{'}\in R^{HW\times HW}$ be an identity matrix. We recall $P=\psi(\phi(\psi^{-1}(E_{1}^{'})))$ and hence reshape $E_{1}^{'}$ to $\psi^{-1}(E_{1}^{'}) \in R^{HW\times H \times W}$. For $m=0,1,\ldots, HW-1$, $\psi^{-1}(E_{1}^{'})[m,:,:]$ represents the $m^{\mathrm{th}}$ matrix, in which only includes one non-zero element. To express the index of the non-zero element, the integer quotient ($//$) and remainder ($\%$) operators of integer division are adopted. All the elements in $\psi^{-1}(E_{1}^{'})[m,:,:]$ are zeros but $\psi^{-1}(E_{1}^{'})[m,m//W,m\%W]$ is one. Next, we perform 2D-DCT transform $\phi$ on the $m^{\mathrm{th}}$ matrix by left multiplying $D_{H,k}^{T}$ and right multiplying $D_{W,k}$, which provides $\phi(\psi^{-1}(E_{1}^{'})[m,:,:])=D_{H,k}^{T}\psi^{-1}(E_{1}^{'})[m,:,:]D_{W,k}=D_{H,k}[m//W,:]^{T}D_{W,k}[m\%W,:]$. We may also write it as:
\begin{eqnarray}
    \begin{tiny}
        \phi(\psi^{-1}(E_{1}^{'}))[m,:,:]
        =\begin{bmatrix}
        D_{H,k}[m//W,0]D_{W,k}[m\%W,:],\\
        D_{H,k}[m//W,1]D_{W,k}[m\%W,:],\\
        \cdots,\\
        D_{H,k}[m//W,k-1]D_{W,k}[m\%W,:]\\
        \end{bmatrix}
    \end{tiny}
\end{eqnarray}

Expanding $\phi(\psi^{-1}(E_{1}^{'})[m,:,:]$ row by row, we obtain the $m^{\mathrm{th}}$ row of $\psi(\phi((\psi^{-1}(E_{1}^{'}))$, which is also the $m^{\mathrm{th}}$ row of $P$. So the element in the $m^{\mathrm{th}}$ row and $n^{\mathrm{th}}$ column of $P$ for $m\in\{0,1,\ldots, HW-1\}$ and $n\in\{0,1,\ldots,k^2-1\}$ is:
\begin{eqnarray}
    P[m,n]=D_{H,k}[m//W,n//k]D_{W,k}[m\%W,n\%k].
\end{eqnarray}

\textbf{Calculating $G$:} Let $E_{2}^{'} \in R^{k^{2}\times k^{2}}$ be an identity matrix and remember that $G=\psi(\phi^{-1}(\psi^{-1}(E_{2}^{'})))$. Accordingly, we reshape $E_{2}^{'}$ to $\psi^{-1}(E_{2}^{'}) \in R^{k^{2}\times k \times k}$. For $m\in \{0,1,\ldots, k^2-1\}$, $\psi^{-1}(E_{2}^{'})[m,:,:]$ represents the $m^{\mathrm{th}}$ matrix, in which all elements are zeros but $\psi^{-1}(E_{2}^{'})[m,m//k,m\%k]$ is one. Next, we perform 2D-IDCT transform on the $m^{\mathrm{th}}$ matrix by left multiplying $D_{H,k}$ and right multiplying $D_{W,k}^{T}$, which provides $\phi^{-1}(\psi^{-1}(E_{2}^{'})[m,:,:])=D_{H,k}\psi^{-1}(E_{2}^{'})[m,:,:]D_{W,k}^{T}=D_{H,k}[:,m//k]D_{W,k}[:,m\%k]^{T}$. It can also be written as:
\begin{eqnarray}
    \begin{tiny}
        \phi^{-1}(\psi^{-1}(E_{2}^{'}))[m,:,:]
        =\begin{bmatrix}
        D_{H,k}[0,m//k]D_{W,k}[:,m\% k]^{T},\\
        D_{H,k}[1,m//k]D_{W,k}[:,m\%k]^{T},\\
        \cdots,\\
        D_{H,k}[H-1,m//k]D_{W,k}[:,m\%k]^{T}\\
        \end{bmatrix}
    \end{tiny}
\end{eqnarray}

Expanding $\phi^{-1}(\psi^{-1}(E_{2}^{'}))[m,:,:]$ row by row, we obtain the $m^\mathrm{th}$ row of $\psi(\phi^{-1}(\psi^{-1}(E_{2}^{'})))$, which is also the $m^\mathrm{th}$ row of $G$. So the elements in the the $m^\mathrm{th}$ row and the $n^\mathrm{th}$ column of $G$ for $m\in\{0,1,\ldots,k^2-1\}$ and $n\in\{0,1,\ldots,HW-1\}$ is:
\begin{eqnarray}
    G[m,n]=D_{H,k}[n//W,m//k]D_{W,k}[n\%W,m\%k].
\end{eqnarray}

Now, we verify the two aforementioned properties of $P,G$.
\textbf{First property,} $\mathbf{G=P^{T}}:$ We immediately observe this property by comparing the computed values of $P[m,n]$ and $G[m,n]$ and directly verifying that $P[m,n]=G[n,m]$.
\textbf{Second property,} $\mathbf{P^{T}P=E_{k^{2}}}$: We verify this condition by direct calculation. We observe $P^{T}P[m,n]=\sum_{r=0}^{H^{2}-1}P^{T}[m,r]P[r,n]$ and it can be explicitly written as $P^{T}P[m,n]=\sum_{r=0}^{H^{2}-1} D_{H,k}[r//H,m//k]D_{W,k}[r\%H,m\%k]\\D_{H,k}[r//H,n//k]D_{W,k}[r\%H,n\%k]$. When $r$ increases from $0$ to $H^{2}-1$, $r//H$ does not change during every period of $H$ values. Hence, $P^{T}P[m,n]=D_{H,k}[0,m//k]D_{H,k}[0,n//k]\\ \sum_{r=0}^{H-1}D_{W,k}[r\%H,m\%k]D_{W,k}[r\%H,n\%k]+\hdots+D_{H,k}[H-1,m//k]D_{H,k}[H-1,n//k] \sum_{r=H^{2}-H}^{H^{2}-1}D_{W,k}[r\%H,m\%k]\\D_{W,k}[r\%H,n\%k]$. On the other hand, $r\%H$ assumes similar values in every period of $H$ values. Accordingly, we can conclude that $\sum_{r=0}^{H-1}D_{W,k}[r\%H,m\%k]D_{W,k}[r\%H,n\%k]\\=\hdots=\sum_{r=H^{2}-H}^{H^{2}-1}D_{W,k}[r\%H,m\%k]D_{W,k}[r\%H,n\%k]$. Combining the above two formulas we can get: $P^{T}P[m,n]\\=\sum_{r=0}^{H-1}D_{H,k}[r,m//k]D_{H,k}[r,n//k]\sum_{r=0}^{H-1}D_{W,k}[r,m\%k]\\D_{W,k}[r,n\%k]=D_{H,k}[:,m//k]^{T}D_{H,k}[:,n//k]D_{W,k}[:,m\%k]^{T}D_{W,k}[:,n\%k]$. When $m=n$, $P^{T}P[m,n]=1$. When $m\neq n$, $P^{T}P[m,n]=0$. This concludes the result.

\subsection{Implementation for Frequency Self-attention}
Base on the above discussion, we observe that there exist three different methods for implementing the transformation to the frequency domain and retaining low-frequency coefficients. First, we may directly multiply the vectorized input to the matrix $P$.
Second, multiply each channel respectively from left and right to  $D^T_{H,k}$ and $D_{W,k}$, and then reshape it into a vector. Third, obtain the complete frequency block of size $(H, W)$ by a fast 2D-DCT algorithm, retain the specified frequency block of size $(k, k)$, and then reshape it into a vector. The complexity of these three methods are  $O(HWk^{2})$, $O(k(HW+k\min(H,W)))$ and $O(HW\log_{2}(HW))$, respectively\footnote{Here, both addition and multiplication are considered as a single unit of computation and the complexity of reshaping and permitting is not considered.}. This confirms that for large $k$ value, fast DCT is the most efficient approach. However, for smaller dimensions, this might not be the case, as unlike the two other strategies, fast DCT calculates the entire frequency components. Hence, we conduct additional empirical studies for selecting the fastest algorithm in our setup.
Taking $H=W=97$, we experimentally test the running time of these methods when $k$ change from 1 to 97 and show the curves in \figref{fig:dcttime}.
\CheckRmv{
\begin{figure}[t]
  \centering
  \begin{overpic}[width=0.85\linewidth]{timefig/dcttime.pdf}
  \end{overpic}
  \caption{The running time of the three methods under different $k$ values.}
  \label{fig:dcttime}
\end{figure}
}
As shown in \figref{fig:dcttime}, the running time of the first method drops rapidly as $k$ decreases. Note that the necessary permutations in the second method is not considered  in the above complexity analysis, but affects the results in the figure. When $k$ is less than 14, the first method runs fastest. The running time of the third method has the longest running time when $k$ is less than 64. The main reason is that in these experiments parallelization by a GPU is considered, which is highly effective for matrix multiplication but has a marginal effect on fast DCT. Accordingly, we choose the first method by $P$, based on which we present the flow chart of this calculation process in \figref{fig:flowchartnonlocal} and \figref{fig:flowchartlinsoftmax}.
\CheckRmv{
\begin{figure}[t]
  \begin{overpic}[width=0.9\linewidth]{structure/flowchartnonlocal.pdf}
  \end{overpic}
  \caption{Data flow chart for Non-local dot-product frequency self-attention}
  \label{fig:flowchartnonlocal}
\end{figure}
}
\CheckRmv{
\begin{figure}[t]
  \begin{overpic}[width=\linewidth]{structure/flowchartlinsoftmax.pdf}
  \end{overpic}
  \caption{Data flow chart for normalized LinSoftmax frequency self-attention}
  \label{fig:flowchartlinsoftmax}
\end{figure}
}

\section{Experiments}\label{sec:experiments}
 In the following subsections, we first introduce the datasets and a semantic segmentation toolbox. Then, we introduce ablation experiments on the high-frequency components to justify our methodology and further determine the required number $k$ of low-frequency components. Finally, we show the implementation details and performance experiments.
\subsection{Datasets and Toolbox}
We carry out comprehensive experiments on the Cityscapes~\cite{cordts2016cityscapes}, the ADE20K dataset\cite{zhou2019semantic}, and the VOCaug dataset\cite{pascal-voc-2012} with mIoU (mean intersection over union) as evaluation metric. We adopt MMSegmentation~\cite{mmseg2020} toolbox to maintain a fair performance comparison. \\
• Cityscapes is a dataset of urban street scenes with 34 categories. However, existing studies mainly focus on only 19 categories. There are 5,000 finely annotated images, which are divided into 2,975/500/1,525 images for training, validation, and testing, respectively. Besides, extra 19997 images are provided with only a coarse annotation.\\
• ADE20K is a scene dataset containing dense labels of 150 stuff/object categories. The scenes are relatively rich: indoor, outdoor, natural scenes, etc. The dataset includes 20k/2k images for training and validation, respectively. \\
• VOCaug (PASCAL VOC 2012 Augmented Dataset) mainly contains complex daily scenes with 20 categories. Most recent works on Pascal VOC 2012 usually exploit extra augmentation data. The augmented dataset includes 10,582/1,449 images for training and validation, respectively. \\
• The Mmsegmentation toolbox provides a unified framework for comparing the {state-of-the-art~} and popular methods. For fairness, all hyperparameters remain the same in all supported methods, except for the difference in their own structural characteristics. Note that the cityscapes dataset suggest a slightly different approach for calculating the mIoU than the conventional method. There is often a small gap ($\sim0.1\%$) between the two, and we adopt the Cityscapes instructions.

\subsection{Ablation study for self-attention/Non-local module}\label{sec:experiments:ablation}

\myPara{Ablation study on biases for Non-local module}\label{sec:experiments:bias ablation}
Here we establish that the biases in the three $1\times 1$ convolutions to obtain $Q$, $K$ and $V$ have no substantial effect, which is also the reason why they are removed from the previous formulas. In order to verify the above points, we implemented an ablation study based on a series of trained Non-local models officially provided by MMsegmentation. These twelve models are all in the Embedded Gaussian mode, and trained under different datasets, Backbones and learning rate update schemes. Usually, we need to retrain the model to perform the ablation study, because even if useless parameters or frequency components are removed from the model, the performance may be severely degraded without retraining. On the other hand, if some parameters or frequency components are removed but have little effect on performance without retraining, they can be removed. All ablation studies in this paper are performed without retraining, still demonstrating the ineffectiveness of the 3 biases and high-frequency components.
We directly remove the three biases at the same time, and the comparison with the original results is shown in Table\ref{biastab}. We observe that the performance of no model drop by more than 0.05, of which 8 models do not exceed 0.02, 1 model has the same performance, and 2 even have improved performance.
\CheckRmv{
    \begin{table}[tbp]
    \scriptsize
    \centering \renewcommand{\arraystretch}{1.05}
    \setlength{\tabcolsep}{2.2mm}
    \caption{Ablation study on Biases.
    }
	\begin{tabular}{lcccc}
	\toprule
	Dataset & Backbone & Schedule & Bias & Nobias\\
	\midrule
	\multirow{4}{*}{Cityscapes} & ResNet-50 & 40k & 78.21 & \textbf{78.23} \\
	~ & ResNet-50 & 80k & 78.86 & \textbf{78.88} \\
	~ & ResNet-101 & 40k & \textbf{78.51} & 78.49 \\
	~ & ResNet-101 & 80k & \textbf{79.35} & \textbf{79.35} \\
	\midrule
	\multirow{4}*{ADE20k} & ResNet-50 & 80k & \textbf{40.75} & 40.72 \\
	~ & ResNet-50 & 160k & \textbf{42.03} & 42.01 \\
	~ & ResNet-101 & 80k & \textbf{42.90} & 42.86 \\
	~ & ResNet-101 & 160k & \textbf{44.63} & 44.62 \\
	\midrule
	\multirow{4}*{VOCaug} & ResNet-50 & 20k & \textbf{76.20} & 76.15 \\
	~ & ResNet-50 & 40k & \textbf{76.65} & 76.62 \\
	~ & ResNet-101 & 20k & \textbf{78.15} & 78.12 \\
	~ & ResNet-101 & 40k & \textbf{78.27} & 78.24 \\
	\bottomrule
    \end{tabular}
    \label{biastab}
    \end{table}
}
\myPara{Ablation study on input frequency components for Non-local module}\label{sec:experiments:frequency ablation}
\renewcommand{\addFig}[1]{{\includegraphics[width=.33\linewidth]{#1.pdf}}}
\CheckRmv{
\begin{figure*}[t]
  \centering
  \tiny
  \renewcommand{\arraystretch}{0.5}
  \setlength{\tabcolsep}{0.2mm}
  \begin{tabular}{ccc}
    \addFig{visualfig/city}&
    \addFig{visualfig/ade}&
    \addFig{visualfig/voc}\\
  \end{tabular}
  \caption{mIoU varies with preserved low frequencies on Cityscapes, ADE20K and VOCaug.}
  \label{fig:cityfre}
\end{figure*}
}
We conducted follow-up experiments with biases removed. To analyze the effect of retaining different low-frequency components on the performance, we transform the input to the frequency domain, and take the low frequency blocks of size $(k, k)$ and reconstruct it as the new input for self-attention. The curves of performance versus $k$ value on three different datasets are shown in \figref{fig:cityfre}. We use two scales to display the curves and also marked the percentage of retained low-frequency components corresponding to different $k$ values.
Embedded Gaussian module contains nonlinear operations, hence crudely using low-frequency components as new input is not conducive to performance preservation. However, with the reduction of the retained frequency components, the performance drops very slowly, and sometimes even slightly rises. This implies that the structure of the Embedded Gaussian has been made close to linear through parameter adaptive adjustment. And the learned parameters tend to cater to the low-frequency components even the network is trained with full frequency. Moreover, we observe that the curves with high performance declined more slowly than the curves with low performance. This also shows that the parameters with good performance tend to adapt to low frequencies, so the performance is easier to maintain when few low-frequency components are retained. As seen from the above figures, there is considerable redundancy in the calculation of spatial self-attention, so it is feasible to reduce the number of tokens from $HW$ to $k$.
To visually observe the changes of the low and high frequency components of the feature map before and after self-attention processing, we visualize the process in \figref{fig:highlowfreprocess}. We observe that the coherence of the subject is enhanced after the self-attention processing of the low-frequency components, while the edges of the high-frequency components become more blurred after the identity processing.
\renewcommand{\addFig}[1]{{\includegraphics[width=.115\textwidth]{#1.png}}}
\CheckRmv{
\begin{figure}[t]
  \centering
  \tiny
  \renewcommand{\arraystretch}{0.5}
  \setlength{\tabcolsep}{0.05mm}
  \begin{tabular}{lcccc}
    \rotatebox[origin=l]{90}{before on Cityscapes}&
    \addFig{visualfig/city/orign}&
    \addFig{visualfig/city/full}&
    \addFig{visualfig/city/low}&
    \addFig{visualfig/city/high}\\
    \rotatebox[origin=l]{90}{after on Cityscapes}&
    \addFig{visualfig/city/label}&
    \addFig{visualfig/city/fullprocessed}&
    \addFig{visualfig/city/lowprocessed}&
    \addFig{visualfig/city/highprocessed}
    \\
    \rotatebox[origin=l]{90}{before on ADE20K}&
    \addFig{visualfig/ade/orign}&
    \addFig{visualfig/ade/full}&
    \addFig{visualfig/ade/low}&
    \addFig{visualfig/ade/high}\\
    \rotatebox[origin=l]{90}{after on ADE20K}&
    \addFig{visualfig/ade/label}&
    \addFig{visualfig/ade/fullprocessed}&
    \addFig{visualfig/ade/lowprocessed}&
    \addFig{visualfig/ade/highprocessed}
    \\
    \rotatebox[origin=l]{90}{before on VOCaug}&
    \addFig{visualfig/voc/orign}&
    \addFig{visualfig/voc/full}&
    \addFig{visualfig/voc/low}&
    \addFig{visualfig/voc/high} \\
    \rotatebox[origin=l]{90}{after on VOCaug}&
    \addFig{visualfig/voc/label}&
    \addFig{visualfig/voc/fullprocessed}&
    \addFig{visualfig/voc/lowprocessed}&
    \addFig{visualfig/voc/highprocessed}\\
   &Image and GT & Full frequency & Low frequency & High frequency \\
  \end{tabular}
  \caption{High and low frequencies are separately processed.}
  \label{fig:highlowfreprocess}
\end{figure}
}
\subsection{Implementation Details}
All experiments are implemented on 4 Tesla V100 GPUs with ResNet101 as Backbone. We realise {FsaNet~} by replacing the non-local module with our frequency self-attention module in Non-local neural network implemented by Mmsegmentation. The poly learning rate is adopted and calculated by
\newcommand{\mathrm{lr}_\mathrm{base}}{\mathrm{lr}_\mathrm{base}}
\newcommand{\mathrm{lr}_\mathrm{min}}{\mathrm{lr}_\mathrm{min}}
\newcommand{\mathrm{iter}}{\mathrm{iter}}
\newcommand{\mathrm{iter}_\mathrm{max}}{\mathrm{iter}_\mathrm{max}}
$(\mathrm{lr}_\mathrm{base} - \mathrm{lr}_\mathrm{min})*(1-\frac{\mathrm{iter}}{\mathrm{iter}_\mathrm{max}})^p+ \mathrm{lr}_\mathrm{min}$ with $\mathrm{lr}_\mathrm{base}=1e{-2}$, $p=0.9$ and $\mathrm{lr}_\mathrm{min}=1e{-4}$. For Cityscapes and ADE20K, We set $\mathrm{iter}_\mathrm{max}=8000$, and set $\mathrm{iter}_\mathrm{max}=4000$ for VOC. The momentum parameter is 0.9 and weight decay is 0.0005. The crop size is $769\times 769$ for Cityscapes, and is $512\times 512$ for ADE20K and VOC. The batch size is 8 for Cityscapes, and is 16 for ADE20K and VOC. We follow {CCNet~\cite{huang2020ccnet}}~so that the input is cyclically processed by one self-attention module $R$ times.
Further, we perform an  experiment on Cityscapes test set. To improve our method compared to other ResNet101-based methods, we adopt similar procedural steps to {CCNet~\cite{huang2020ccnet}}. For convenience, we replaced the  criss-cross module in {CCNet~\cite{huang2020ccnet}} with our frequency self-attention module. In this way, the continuity loss in {CCNet~\cite{huang2020ccnet}}~is applied in {FsaNet~}. The backbone is still ResNet-101 but its last two down-sampling operations are removed, and dilated convolutions are employed in the subsequent convolutional layers. We also employ coarsely labeled data, increase the crop size to $849\times 849$ and set $R=2$ and $k=8$. The poly learning rate is calculated by $\mathrm{lr}_\mathrm{base}*(1-\frac{\mathrm{iter}}{\mathrm{iter}_\mathrm{max}})^{p}$ with $\mathrm{lr}_\mathrm{base}=1e{-2}$, $p=0.9$. After 6000 steps of training on the finely labeled data set, the ohem(Online Hard Example Mining) is utilized, and then the coarsely labeled data is added for mixed training for 30k steps. The momentum parameter is again 0.9 and weight decay is 0.0005. Notably, we add a set of learnable biases to the output of self-attention, as we discovered this modification to be beneficial to performance. In the implementation of Mmsegmentaion, this bias already exists since the output of self-attention is processed by an extra output convolution.

\subsection{FsaNet Performance on Cityscapes}\label{sec:Experiments_City}
\textbf{Performance on Cityscapes val set}
Results of {FsaNet~} and other methods on Cityscapes val set in mmsegmentation are summarized in \tabref{citymiouTab}. We refer to the frequency self-attention based on the Non-local Dot product version at $R=1$ and $R=2$ as FsaNet-Dot-R1 and FsaNet-Dot-R2 respectively. We refer to the frequency self-attention based on $\ell_{2}$ normalized LinSoftmax version at $R=1$ and $R=2$ as FsaNet-Lin-R1 and FsaNet-Lin-R2 respectively. We achieved better results with $R=2$. Considering the influence of random factors, FsaNet-Dot and FsaNet-LinSoftmax have small differences, which is consistent with the conclusion that different versions of the non-local modules achieve similar results. Compared to the {FCN~\cite{long2015fully}}~without context module, {FsaNet~} has an improvement of 4.62$\sim$5.29\% in mIoU, which fully demonstrates the effectiveness. {FsaNet~} can easily outperform {Non-local~Network~\cite{wang2018non}}~by a large margin of 0.62$\sim$1.29\%. {FsaNet~} also showed apparent advantages against {DNLNet~\cite{yin2020disentangled}}~by 0.69$\sim$1.36\%, which is also a {Non-local~Network~\cite{wang2018non}}~improved method by decoupling boundary and the body.
\begin{table}[!tbp]

        \scriptsize
        \setlength{\tabcolsep}{0mm}
	\begin{minipage}[t]{0.28\linewidth}
        \vspace{0pt}
		\makeatletter\def\@captype{table}\makeatother
		\caption{Performance on Cityscapes (val).}
		\begin{tabular}{lc}
        \toprule
         Method & mIoU(\%)\\
        \midrule
        {FCN~\cite{long2015fully}}	 & 75.35\\
        {EncNet~\cite{zhang2018context}} & 75.97\\
        {APCNet~\cite{he2019adaptive}} & 78.32\\
        {ANN~\cite{zhu2019asymmetric}} & 78.61\\
        {GCNet~\cite{cao2019gcnet}} & 79.05\\
        {DMNet~\cite{he2019dynamic}} & 79.06\\
        {DNLNet~\cite{yin2020disentangled}} & 79.28\\
        {CCNet~\cite{huang2020ccnet}} & 79.31\\
        Non-local\cite{wang2018non} & 79.35\\
        {EMANet~\cite{li2019expectation}} & 79.48\\
        {PSANet~\cite{zhao2018psanet}}& 79.49\\
        {DeepLabV3~\cite{chen2017rethinking}} & 79.59\\
        {PSPNet~\cite{zhao2017pyramid}} & 79.69\\
        {DANet~\cite{fu2019dual}} & 80.29\\
        {OCRNet~\cite{yuan2020object}} & 80.33\\
        {ISANet~\cite{huang2019interlaced}} & 80.47\\
        \midrule
        {FsaNet~}-Dot-R1 & 79.97\\
        {FsaNet~}-Lin-R1 & 80.04\\
        {FsaNet~}-Dot-R2 & 80.36\\
        {FsaNet~}-Lin-R2 & \textbf{80.64}\\
        \bottomrule
	\end{tabular}
	\label{citymiouTab}
	\end{minipage}
        \quad\begin{minipage}[t]{0.28\linewidth}
        \vspace{0.17em}
        \makeatletter\def\@captype{table}\makeatother
		\caption{Performance on ADE20k.}
		\begin{tabular}{lc}
        \toprule
        Method  & mIoU(\%)\\
        \midrule
        {FCN~\cite{long2015fully}} & 39.61\\
        {EncNet~\cite{zhang2018context}} & 42.11\\
        {GCNet~\cite{cao2019gcnet}} & 42.82\\
        {OCRNet~\cite{yuan2020object}} & 42.87\\
        Non-local\cite{wang2018non} & 42.90\\
        {UPerNet~\cite{xiao2018unified}} & 42.91\\
        {ANN~\cite{zhu2019asymmetric}}	 & 42.94\\
        {ISANet~\cite{huang2019interlaced}}	 & 43.51\\
        {PSPNet~\cite{zhao2017pyramid}}	 & 43.57\\
        {DANet~\cite{fu2019dual}}	 & 43.64\\
        {CCNet~\cite{huang2020ccnet}}	 & 43.71\\
        {PSANet~\cite{zhao2018psanet}}	 & 43.80\\
        {DeepLabV3~\cite{chen2017rethinking}} & 44.08\\
        \midrule
        FsaNet-Dot-R1 & 43.04\\
        FsaNet-Lin-R1 & 43.05\\
        FsaNet-Dot-R2 & 43.53\\
        FsaNet-Lin-R2 &\textbf{44.10}\\
	\bottomrule
	\label{ADEmiouTab}
		\end{tabular}
	\end{minipage}
        \quad\begin{minipage}[t]{0.28\linewidth}
        \vspace{0.16em}
        \makeatletter\def\@captype{table}\makeatother
		\caption{Performance on VOCaug.}
		\begin{tabular}{lc}
        \toprule
        Method & mIoU(\%)\\
        \midrule
        {FCN~\cite{long2015fully}}& 69.91\\
        {DANet~\cite{fu2019dual}} & 76.51\\
        {ANN~\cite{zhu2019asymmetric}}& 76.70\\
        {UPerNet~\cite{xiao2018unified}} & 77.43\\
        {PSANet~\cite{zhao2018psanet}} & 77.73\\
        {GCNet~\cite{cao2019gcnet}} & 77.84\\
        {CCNet~\cite{huang2020ccnet}} & 77.87\\
        {DeepLabV3~\cite{chen2017rethinking}} & 77.92\\
        {ISANet~\cite{huang2019interlaced}} & 78.12\\
        Non-local\cite{wang2018non} & 78.27\\
        \midrule
        FsaNet-Dot-R1 & 78.31\\
        FsaNet-Lin-R1 & 78.33\\
        FsaNet-Dot-R2 & 78.67\\
        FsaNet-Lin-R2 & \textbf{78.70}\\
        \bottomrule
        \label{VOCmiouTab}
		\end{tabular}
	\end{minipage}
\vspace{-15pt}
\end{table}
We visualize the segmentation results of Non-local Network, {OCRNet~\cite{yuan2020object}}, and our approaches on the Cityscapes validation set in \figref{fig:cityseg}. In the first row, Some pixels of the truck were wrongly attributed to bus or car in {Non-local~Network~\cite{wang2018non}}, {OCRNet~\cite{yuan2020object}}, and {FsaNet~}(R=1). However, when {FsaNet~} is applied with $R=2$, the consistency within the truck is better promoted, and the whole truck is correctly identified. In the second row, person and rider are marked in dark red and bright red, respectively. In {Non-local~Network~\cite{wang2018non}} and {OCRNet~\cite{yuan2020object}}, the head, chest and arms of the rider on the far right are misclassified as person. In {FsaNet~}(R=1), only a few pixels of face and chest are misclassified. In {FsaNet~}(R=2), all the pixels of the rider are correctly identified. In the third row, {FsaNet~} is less prone to omissions in identifying yellow traffic signs. This shows that on small objects, {FsaNet~} better maintains  the details.
\renewcommand{\addFig}[1]{{\includegraphics[height=.06\textwidth]{#1.png}}}
\CheckRmv{
\begin{figure}[t]
  \centering
  \tiny
  \renewcommand{\arraystretch}{1}
  \setlength{\tabcolsep}{0.0mm}
  \begin{tabular}{cccc}
   \\
   \addFig{cityseg/010351/Image}&
   \addFig{cityseg/010351/non}&
   \addFig{cityseg/010351/ocr}&
   \addFig{cityseg/010351/GT}
   \\
   \addFig{cityseg/016462/Image}&
   \addFig{cityseg/016462/non}&
   \addFig{cityseg/016462/ocr}&
   \addFig{cityseg/016462/GT}
   \\
   \addFig{cityseg/000538/Image}&
   \addFig{cityseg/000538/non}&
   \addFig{cityseg/000538/ocr}&
   \addFig{cityseg/000538/GT}
   \\
   Image & Non-local & OCRNet & Ground Truth
   \\
   \addFig{cityseg/010351/r1lin}&
   \addFig{cityseg/010351/r1dot}&
   \addFig{cityseg/010351/r2lin}&
   \addFig{cityseg/010351/r2dot}
   \\
   \addFig{cityseg/016462/r1lin}&
   \addFig{cityseg/016462/r1dot}&
   \addFig{cityseg/016462/r2lin}&
   \addFig{cityseg/016462/r2dot}
   \\
   \addFig{cityseg/000538/r1lin}&
   \addFig{cityseg/000538/r1dot}&
   \addFig{cityseg/000538/r2lin}&
   \addFig{cityseg/000538/r2dot}
   \\
   FsaNet-Lin-R1 & FsaNet-Dot-R1 & FsaNet-Lin-R2 & FsaNet-Dot-R2\\
  \end{tabular}
  \caption{Visualization of segmentation for the validation
set of Cityscapes.}
  \label{fig:cityseg}
\end{figure}
}

\textbf{Performance on Cityscapes test set}
We also compare {FsaNet~} based on ResNet-101 with other recent approaches on the Cityscapes test set. All approaches share dilated ResNet101 as backbone but adopt different enhanced strategies. We enhance FsaNet by continuous loss, ohem training, extra data, and bigger crop size to get better result. {CCNet~\cite{huang2020ccnet}}~adopts continuous loss and ohem training. {PSANet~\cite{zhao2018psanet}}~adopts bigger batch size and extra data. {DANet~\cite{fu2019dual}}~and {DGCNet~\cite{zhang2019dual}}~adopts both spatial and channel attention. {HANet~\cite{choi2020cars}}~introduces structural priors with respect to the height in urban-scene data. {OCRNet~\cite{yuan2020object}}~adopts extra data. {DecoupledSegNets~\cite{li2020improving}}~adopts context module in DeepLabV3+\cite{chen2018encoder}, decouples the body feature and edge feature which are supervised separately, increases crop size, integrates multi-stage information, uses class uniforms sampling to enhance data, and uses ohem training. As seen from \tabref{citytestmIou}, FsaNet-Lin-R2 and FsaNet-Dot-R2 achieve the mIoU of 82.81\% and 83.05\%, which surpasses all the other well-designed models with ResNet-101 of previous {state-of-the-art~} work.\\
\begin{table}[tbp]
    \scriptsize
    \centering \renewcommand{\arraystretch}{1.05}
    \setlength{\tabcolsep}{1mm}
    \caption{Comparison with state-of-the-arts on Cityscapes (test). ‡: trained on fine and coarse annotated data.
    }
    \begin{tabular}{lcccc}
    \toprule
    Method & Backbone &Batch size& Crop size & mIoU(\%)\\
    \midrule
    {CCNet~\cite{huang2020ccnet}} &  ResNet101& 8 & 769 & 81.40\\
    {PSANet~\cite{zhao2018psanet}}‡ & ResNet101 & 16 & 705 & 81.40\\
    {DANet~\cite{fu2019dual}} & ResNet101 & 8 & 768 & 81.50\\
    {DGCNet~\cite{zhang2019dual}} & ResNet101& 8 & 769 & 82.00\\
    {HANet~\cite{choi2020cars}} & ResNet101 & 8 & 768 & 82.10\\
    {OCRNet~\cite{yuan2020object}}‡ & ResNet101 & 8 & 769 &82.40\\
    {DecoupledSegNets~\cite{li2020improving}} & ResNet101 & 8 & 832 & 82.80\\
    FsaNet-Lin-R2‡ & ResNet101 & 8 & 849 & 82.81\\
    FsaNet-Dot-R2‡ & ResNet101 & 8 & 849 & \textbf{83.05}\\
    \bottomrule
    \end{tabular}
    \label{citytestmIou}
\end{table}

\subsection{FsaNet Performance on ADE20K}\label{sec:Experiments_ADE}
We conduct experiments on the AED20K in mmsegmentation framework and show the results in \tabref{ADEmiouTab}. The FsaNet achieve significant performance gains over the {FCN~\cite{long2015fully}}~with 3.43$\sim$4.49\% by mIoU. This significant improvement demonstrates the effectiveness of proposed frequency self-attention modules on ADE20K. Compared with the {Non-local~Network~\cite{wang2018non}}, {FsaNet~} has increased mIoU by 0.14$\sim$1.2\%. When $R=1$, {FsaNet~} can slightly outperform the {Non-local~Network~\cite{wang2018non}}. When $R=2$, {FsaNet~} can achieve performance that exceeds or is comparable to previous {state-of-the-art~} methods based on ResNet101.
We also show {FsaNet~} segmentation results compared to {Non-local~Network~\cite{wang2018non}}~and {OCRNet~\cite{yuan2020object}}~on ADE20K in \figref{fig:adeseg}. In the first row it can be observed that the four versions of FsaNet can better maintain the integrity and internal consistency of the wall than {Non-local~Network~\cite{wang2018non}}~and {OCRNet~\cite{yuan2020object}}. The climbing plants on the house in FsaNet are also more completely segmented. Although FsaNet mistakes plants for trees, such misunderstandings may also occur in human perception. In the second row, we observe that FsaNet better promotes the consistency inside lamps and pool tables. In the result of {Non-local~Network~\cite{wang2018non}}, some parts of the table lamp are identified as chandeliers and in {OCRNet~\cite{yuan2020object}}~result some pixels on the left leg of the pool table are misattributed to a person. In the third row, in the result of {Non-local~Network~\cite{wang2018non}}, some pixels of the bed are misattributed to pillow and in {OCRNet~\cite{yuan2020object}}~result the box and paintings were ignored on the chest of drawers. This indicates that {FsaNet~} can better promote the internal consistency of large objects and is also beneficial to the recognition of small objects.
 \renewcommand{\addFig}[1]{{\includegraphics[width=.06\textwidth]{#1.jpg}}}
\CheckRmv{
\begin{figure}[t]
  \centering
  \tiny
  \renewcommand{\arraystretch}{0.5}
  \setlength{\tabcolsep}{0.0mm}
  \begin{tabular}{cccccccc}
   \addFig{adeseg/00000001/Image}&
   \addFig{adeseg/00000001/non}&
   \addFig{adeseg/00000001/ocr}&
   \addFig{adeseg/00000001/GT}&
   \addFig{adeseg/00000001/r1lin}&
   \addFig{adeseg/00000001/r1dot}&
   \addFig{adeseg/00000001/r2lin}&
   \addFig{adeseg/00000001/r2dot}
   \\
   \addFig{adeseg/00000988/Image}&
   \addFig{adeseg/00000988/non}&
   \addFig{adeseg/00000988/ocr}&
   \addFig{adeseg/00000988/GT}&
   \addFig{adeseg/00000988/r1lin}&
   \addFig{adeseg/00000988/r1dot}&
   \addFig{adeseg/00000988/r2lin}&
   \addFig{adeseg/00000988/r2dot}
   \\
   \addFig{adeseg/00000152/Image}&
   \addFig{adeseg/00000152/non}&
   \addFig{adeseg/00000152/ocr}&
   \addFig{adeseg/00000152/GT}&
    \addFig{adeseg/00000152/r1lin}&
   \addFig{adeseg/00000152/r1dot}&
   \addFig{adeseg/00000152/r2lin}&
   \addFig{adeseg/00000152/r2dot}
   \\
  Image & Non-local & OCRNet & Ground Truth& FsaNet-Lin-R1 & FsaNet-Dot-R1 & FsaNet-Lin-R2 & FsaNet-Dot-R2
  \end{tabular}
  \caption{Visualization of segmentation for ADE20K.}
  \label{fig:adeseg}
\end{figure}
}

\subsection{FsaNet Performance on VOCaug}\label{sec:Experiments_VOC}
We also compare {FsaNet~} on the VOCaug\cite{pascal-voc-2012} with {state-of-the-art~} methods in Mmsegmentation in \tabref{VOCmiouTab}. The {FsaNet~} has an improvement of 7.73$\sim$9.07\% in mIoU over {FCN~\cite{long2015fully}}, demonstrating the effectiveness of our modules. We observe that the {Non-local~Network~\cite{wang2018non}}~outperformed previous approaches and {FsaNet~} achieves better results with less computational consumption.
Visual comparisons of semantic segmentation results on VOC are illustrated in \figref{fig:vocseg}. In the three rows of images, we observe that {FsaNet~} tends to accurately preserve the outline details for bicycles, cars and airplanes, separately. In the second line, the number of cars is hard to be counted because the segmentation of {Non-local~Network~\cite{wang2018non}}~and {OCRNet~\cite{yuan2020object}}~is not fine.
 \renewcommand{\addFig}[1]{{\includegraphics[width=.06\textwidth]{#1.jpg}}}
 \newcommand{\addFigpng}[1]{{\includegraphics[width=.06\textwidth]{#1.png}}}
\CheckRmv{
\begin{figure}[t]
  \centering
  \tiny
  \renewcommand{\arraystretch}{0.5}
  \setlength{\tabcolsep}{0.1mm}
  \begin{tabular}{cccccccc}
   \addFig{vocseg/2007_000129/Image}&
   \addFig{vocseg/2007_000129/non}&
   \addFig{vocseg/2007_000129/ocr}&
   \addFigpng{vocseg/2007_000129/GT}&
   \addFig{vocseg/2007_000129/r1lin}&
   \addFig{vocseg/2007_000129/r1dot}&
   \addFig{vocseg/2007_000129/r2lin}&
   \addFig{vocseg/2007_000129/r2dot}
   \\
   \addFig{vocseg/2007_005331/Image}&
   \addFig{vocseg/2007_005331/non}&
   \addFig{vocseg/2007_005331/ocr}&
   \addFigpng{vocseg/2007_005331/GT}&
   \addFig{vocseg/2007_005331/r1lin}&
   \addFig{vocseg/2007_005331/r1dot}&
   \addFig{vocseg/2007_005331/r2lin}&
   \addFig{vocseg/2007_005331/r2dot}
   \\
   \addFig{vocseg/2009_002888/Image}&
   \addFig{vocseg/2009_002888/non}&
   \addFig{vocseg/2009_002888/ocr}&
   \addFigpng{vocseg/2009_002888/GT}&
   \addFig{vocseg/2009_002888/r1lin}&
   \addFig{vocseg/2009_002888/r1dot}&
   \addFig{vocseg/2009_002888/r2lin}&
   \addFig{vocseg/2009_002888/r2dot}
   \\
  Image & Non-local & OCRNet & Ground Truth& FsaNet-Lin-R1 & FsaNet-Dot-R1 & FsaNet-Lin-R2 & FsaNet-Dot-R2\\
  \end{tabular}
  \caption{Visualization of segmentation for VOCaug.}
  \label{fig:vocseg}
\end{figure}
}
\section{Complexity Analysis}\label{sec:Experiments_Complex}
\subsection{Memory, FLOPs and Run time}
We report the memory, FLOPs (Floating Point Operations) and run time in \tabref{Tab:FLOPs} on Cityscapes with $769\times 769$ crop size. The input is a feature map of size $512\times 97\times 97$ and the embedding dimension is $64$. We counted FLOPs for all operations including: matrix multiplication, Hadamard product, 'fro' norm, summation along a dimension, Softmax. A multiply/add/exponential/square operation is counted as a FLOP. Compared to {Non-local~Network~\cite{wang2018non}}, FsaNet-Dot reduces computation by 90.04\% in memory, 98.18\% in run time, and 98.07\% in FLOPs; FsaNet-Lin reduces the computation by 87.29\% in memory, 97.56\% in run time, and 96.13\% in FLOPs. Considering that the input and output already account for 9.57\% of the non-local memory, the memory costs in self-attention processing drops even greater.
\CheckRmv{
\begin{table}[tbp]
    \scriptsize
    \centering \renewcommand{\arraystretch}{1.05}
    \setlength{\tabcolsep}{2.2mm}
    \caption{Comparison of Non-local self-attention.}
    \begin{tabular}{lccc}
    \toprule
    Method &           GPU memory(M) & FLOPs(G) &Run time(ms)\\
    \midrule
    Non-local\cite{wang2018non}&          402.82 & 25.30 & 35.1\\
    FsaNet-Dot& 40.14  & 0.49  & 0.64\\
    FsaNet-Lin&  51.19  & 0.98  & 0.86\\
    \bottomrule
    \end{tabular}
    \label{Tab:FLOPs}
\end{table}
}
\subsection{Theoretical complexity Comparison}
We give the overall $N$-dependent complexity of self-Attention simplification methods. For horizontal comparison, we also give a more precise representation of the complexity. The original implementation of self-attention includes two parts: token mapping ($1\times 1$ convolution) and token mixing. To simplify self-attention, some methods add dimension reduction operations. The total complexity consists of the sum of these three parts. {ISANet~\cite{huang2019interlaced}}~and {CCNet~\cite{huang2020ccnet}}~reduce the sequence length of key and query by dimension permutation and reshaping to reduce dimension, which are not generally considered in the theoretical complexity but certainly consumes time. {ANN~\cite{zhu2019asymmetric}}~uses pooling to reduce dimension, where $S=110$. {EMANet~\cite{li2019expectation}}~obtains attention maps and $K$ Gaussian bases through $T$ iterations, where $T=3$, $K=64$. {OCRNet~\cite{yuan2020object}}~obtains $K$ soft object region bases to achieve dimensionality reduction, and applies $1\times1$ convolution after dimensionality reduction. Here, $K$ is the number of taxonomic categories. Almost all of the computation of {GCNet~\cite{cao2019gcnet}}~is composed of $1\times 1$ convolution, because the attention map is degenerated from a matrix to a vector.
{FsaNet~} is the only method that reduces the dimension of query, key and value at the same time, and adopts the method of applying $1\times1$ convolution after dimension reduction. This not only reduces the complexity of the token mixing part sharply, but also reduces the complexity of convolution, hence a sharp decline in computation. We show the methods in \tabref{tab:Complex} in order of descending complexity from top to bottom. When $N=HW=97^{2}$, the number of channels and embedding dimension are $C=512$, even if $k$ in FsaNet takes the maximum value of $14$, the complexity is still only about half of {GCNet~\cite{cao2019gcnet}}, which shows our approach is much more efficient than all the other approaches.
\CheckRmv{
    \begin{table}[tbp]
    \scriptsize
    \centering
    \centering \renewcommand{\arraystretch}{1.05}
    \setlength{\tabcolsep}{0.3mm}
    \caption{Comparision with other methods.
    }
    \begin{tabular}{ lcccc }
        \toprule
        Method    & Overall & $1\times 1$ Conv & Reduce dim & Token mix\\
        \midrule
        Nonlocal\cite{wang2018non} & $O(N^{2})$     & $O(3NC^{2})$         & $0$          & $O(2N^{2}C)$\\
        {ISANet~\cite{huang2019interlaced}}    & $O(N\sqrt{N})$ & $O(6NC^{2})$         & $0$          & $O(4N\sqrt{N}C)$\\
        {CCNet~\cite{huang2020ccnet}}     & $O(N\sqrt{N})$ & $O(3NC^{2})$         & $0$          & $O(4N\sqrt{N}C)$\\
        {ANN~\cite{zhu2019asymmetric}}       & $O(N)$         & $O(3NC^{2})$         & $O(4NC)$     & $O(2NCS)$\\
        {EMANet~\cite{li2019expectation}}    & $O(N)$         & $O(2NC^{2})$         & $O(2NCKT)$   & $O(NKC)$\\
        {OCRNet~\cite{yuan2020object}}    & $O(N)$         & $O(2NC^{2}+2KC^{2})$ & $O(NCK)$     & $O(2NCK)$\\
        {GCNet~\cite{cao2019gcnet}}     & $O(N)$         & $O(NC^{2}+NC)$       & $0$          & $O(NC)$\\
        FsaNet    & $O(N)$         & $O(3k^{2}C^{2})$     & $O(NCk^{2})$ & $O(2Ck^{4})$\\
        \bottomrule
    \end{tabular}
    \label{tab:Complex}
    \end{table}
}
\section{Conclusion and Future Work}\label{sec:futurework}
Inspired by the idea of decoupling high and low frequencies, we performed ablation experiments on high-frequency components of self-attention input, and discovered that up to 98\% of the high-frequency components do not significantly affect the performance. Based on this, we transform the input to the frequency domain, and exploit the sparsity of the frequency domain to propose a frequency self-attention mechanism with lowest computational cost. With CNN-based backbone, our~{FsaNet~} consistently achieves an outstanding performance on the well-known semantic segmentation datasets, namely Cityscapes\cite{cordts2016cityscapes}, ADE20K\cite{zhou2019semantic} and VOCaug\cite{pascal-voc-2012}. Although ~{FsaNet~} still has some performance gap compared with transformer-based methods, pure CNN-based approaches still dominate in real-time semantic segmentation field, which makes~{FsaNet~} a promising real-time solution.
\section*{Acknowledgments}
We are grateful to the High Performance Computing Center of Central South University for partial support of this work. Fengyu Zhang acknowledges the financial support from China Scholarship Council, grant number: 202006370295.

{\small