با سلام
بهتر است از روشی که در زیر آمده استفاده کنید:
\documentclass{article}
\usepackage{amsmath,pgfkeys,diagbox,tikz}
\usepackage{mathabx}
\usepackage{xepersian}
\settextfont{XB Yas}
\setdigitfont{XB Yas}
\begin{document}
\usetikzlibrary{positioning,decorations.pathreplacing,matrix}
\usetikzlibrary{mindmap,trees,shadows}
\usetikzlibrary{patterns}
\tikzstyle{state}=[thick,scale=1]
\begin{tikzpicture}[ decoration={brace,amplitude=5pt}]
\draw[->] (0,0) -- (5.5,0) node[right] {$X$};
\draw[->] (0,0) -- (0,5.5) node[above] {$Y$};
\draw (0,1pt) -- (0,-1pt) node[below] {$0$};
\draw (2,1pt) -- (2,-1pt) node[below] {};
\draw (0pt,2) -- (-2pt,2) node[left] {};
\draw (0pt,3) -- (-2pt,3) node[left] {};
\draw[style = dashed] (2,0) -- (2,2);
\draw[style = dashed] (0,2) -- (2,2);
\draw[style = dashed] (3,0) -- (3,3);
\draw[style = dashed] (0,3) -- (3,3);
\draw[style=smooth]plot coordinates{(1.6,1.2)(1.8,1.55)(1.9,1.8)(2,2)(2.35,2.65)(3,3)(4.1,3.1)(5,4)};
\node [state](A)[yshift=5cm,xshift=4cm] {$y=f(x)$};
\node [state](B)[yshift=-1cm,xshift=2cm] {$\text{ب}$};
\draw [decorate,color=red] (-0.2,2) -- (-0.2,3)
node [midway,anchor=east,inner sep=5pt, outer sep=5pt]{$A$};
\draw [decorate,color=green!80!black] (3.0,-0.2)--(2,-0.2)
node [midway,anchor=north,inner sep=5pt, outer sep=5pt]{$B=f(A)$};
\end{tikzpicture}
\end{document}
نتیجه به صورت زیر خواهد بود:
