presentation.tex

%===============================================================
% Template author: Martin Malý.
% Template URL: https://www.overleaf.com/latex/templates/czech-technical-university-colors-beamer-template-lab-of-structure-of-biomolecules/xmgmzfhtyrxt
%===============================================================

\documentclass{beamer}
% \documentclass[aspectratio=169]{beamer} % Uncomment for 16:9
\usepackage[utf8]{inputenc}

\usetheme{Madrid}

\definecolor{cvut_navy}{HTML}{0065BD}
\definecolor{cvut_blue}{HTML}{6AADE4}
\definecolor{cvut_gray}{HTML}{156570}

\setbeamercolor{section in toc}{fg=black,bg=white}
\setbeamercolor{alerted text}{fg=cvut_blue}
\setbeamercolor*{palette primary}{bg=cvut_navy,fg=gray!20!white}
\setbeamercolor*{palette secondary}{bg=cvut_blue,fg=white}
\setbeamercolor*{palette tertiary}{parent=palette primary}
\setbeamercolor*{palette quaternary}{fg=green,bg=gray!5!white}

\setbeamercolor*{sidebar}{fg=cvut_navy,bg=gray!15!white}


\setbeamercolor{titlelike}{parent=palette primary}
\setbeamercolor{frametitle}{parent=palette primary}

\setbeamercolor*{separation line}{}
\setbeamercolor*{fine separation line}{}

\setbeamertemplate{navigation symbols}{}

% Change itemize and enumerate style.
\setbeamertemplate{itemize item}{%
  \textcolor{cvut_navy}{\raisebox{.45ex}{\rule{.8ex}{.8ex}}}
}
\setbeamertemplate{enumerate item}{%
  \textcolor{cvut_navy}{\arabic{enumi}.}
}

\usepackage{eqnarray,amsmath}
\usepackage{amsfonts}
\usepackage{amssymb}
\usepackage{graphicx}
\usepackage{lmodern} % for bold and italic at the same time
\usepackage{bm} % for bold and italic at the same time
\usepackage{epstopdf}
\usepackage{changepage}
\usepackage{array,booktabs}
\usepackage[dua, nolist]{acronym}

%% | -------------------------- tikz -------------------------- |

\usepackage{tikz}
\usepackage{pgfplots}
\pgfplotsset{compat=1.14}
\usetikzlibrary{backgrounds,arrows,automata,shapes,positioning,calc,through,spy,shapes,shapes.geometric,shapes.multipart,fit,patterns,fadings}
\pgfdeclarelayer{background}
\pgfdeclarelayer{foreground}
\pgfsetlayers{background,main,foreground}
\tikzset{
    use page relative coordinates/.style={
        shift={(current page.south west)},
        x={(current page.south east)},
        y={(current page.north west)}
    },
}


% Change color of Figure and Table labels.
\usepackage[labelfont={color=cvut_navy}]{caption}

% Define logo condition
\newif\ifplacelogo
\logo{\ifplacelogo\includegraphics[height=1cm]{src/fig/pdfs/ctu_logo_blue_filled.pdf}\fi}


%====================================================
%========== DEFINITION OF AUTHORS ETC...=============
%====================================================
\author[Yauheni Zviazdou]{Yauheni Zviazdou}
\institute[]{Czech Technical University in Prague \\ Faculty of Electrical Engineering \\ Department of Cybernetics \\}
\title[Text representation models. RAG.]{From FastText to Transformer Models, and their Application in Retrieval-Augmented Generation}
\date[Bachelor's thesis presentation]{Bachelor's thesis presentation\\Supervisor: Ing. Jan Šedivý, CSc.}

%====================================================
%========== BEGINNING OF DOCUMENT ===================
%====================================================
\begin{document}


% Title slide
\begin{frame}
  \titlepage
  \begin{center}
    \includegraphics[height=2cm]{src/fig/pdfs/ctu_logo_blue_filled.pdf}
  \end{center}
  \input{src/abbreviations.tex}
\end{frame}


% Motivation
\placelogotrue
\begin{frame}
  \frametitle{Motivation}
  \begin{columns}
    \begin{column}{0.5\textwidth}
      \begin{itemize}
        \item LLM chatbots integrate external documents, enhancing context-aware responses.
        \item Answer quality depends on the context provided by the technical document.
        \item Context quality depends on text representation.
      \end{itemize}
    \end{column}
    \begin{column}{0.5\textwidth}
      \includegraphics[scale=0.15]{src/fig/imgs/RAG_example.png}
    \end{column}
  \end{columns}

  % Adding cross and check marks.
  % https://tex.stackexchange.com/questions/42619/xmark-that-complements-the-ams-checkmark
  \newcommand{\tikzxmark}{%
  \tikz[scale=0.02] {
      \draw[line width=1.2,line cap=round] (0,0) to [bend left=6] (1,1);
      \draw[line width=1.2,line cap=round] (0.2,0.95) to [bend right=3] (0.8,0.05);
  }}
  \newcommand{\tikzcmark}{%
  \tikz[scale=0.02] {
      \draw[line width=1.2,line cap=round] (0.25,0) to [bend left=10] (1,1);
      \draw[line width=1.3,line cap=round] (0,0.35) to [bend right=1] (0.23,0);
  }}
  \begin{tikzpicture}[remember picture, overlay,use page relative coordinates]
    \node at (0.90,0.63) {\tikzxmark};
    \node at (0.90,0.27) {\tikzcmark};
      
  \end{tikzpicture}
\end{frame}


% Work objectives
\begin{frame}
  \frametitle{Work objectives}
  \textcolor{cvut_navy}{\textbf{Task 1.}} Review and compare text representation methods.
  \begin{itemize}
    \item Review text representation methods.
    \item Compare traditional and transformer-based methods.
  \end{itemize}
  \textcolor{cvut_navy}{\textbf{Task 2.}} RAG Optimization
  \begin{itemize}
    \item Choose optimal models for RAG.
    \item Choose optimal chunk size and number of context chunks.
  \end{itemize}
  \bigskip
  \input{src/fig/tikz/text_encode.tex}
\end{frame}


% Text representation methods
\begin{frame}[t]
  \frametitle{Text representation methods}
  \begin{columns}[onlytextwidth]
    \begin{column}{0.5\textwidth}
      \begin{figure}
        \input{src/fig/tikz/evolution_text_representaion_v2.tex}
      \end{figure}
    \end{column}
    \begin{column}{0.55\textwidth}
      \textcolor{cvut_navy}{\textbf{FastText}}
      \begin{itemize}
        \item Neural Network with few layers.      
        \item Pre-trained embeddings
        \item Independent of context.
        \item Limited word order understanding.        
      \end{itemize}
      \textcolor{cvut_navy}{\textbf{Transformer models}}
      \begin{itemize}
        \item Deep Neural Network with self-attention mechanism.
        \item Real-time embeddings generation.
        \item Contextual embeddings.        
        \item Captures word relationships.
      \end{itemize}
    \end{column}
  \end{columns}
\end{frame}


% Methodology
\begin{frame}
  \frametitle{Methodology}
  \begin{columns}[onlytextwidth]
    \begin{column}{0.4\textwidth}
      \textcolor{cvut_navy}{\textbf{Corpus information}}
      \begin{itemize}
        \item Czech language
        \item Diacritics, diacriticless
      \end{itemize}
      \textcolor{cvut_navy}{\textbf{Baseline}}
      \begin{itemize}
        \item FastText
        \item Architecture: CBOW
        \item Dimensionality: 300
      \end{itemize}
      \textcolor{cvut_navy}{\textbf{Benchmark}}
      \begin{itemize}
        \item UPV FAQ
      \end{itemize}
      \textcolor{cvut_navy}{\textbf{Chosen models}}
      \begin{itemize}
        \item 15 groups
        \item 37 models
        \item 13M - 560M parameters
      \end{itemize}
    \end{column}
    \begin{column}{0.6\textwidth}
      \begin{figure}
        \includegraphics[scale=0.75]{src/fig/pdfs/tikz/diacritics_diacriticless.pdf}
        \caption{Creation of diacriticless corpus.}
      \end{figure}     
    \end{column}
  \end{columns}
\end{frame}


% Balanced models
\begin{frame}
  \frametitle{Results: Balanced models}
  \textcolor{cvut_navy}{\textbf{Results analysis}}
  \begin{itemize}
    \item Supervised learning enhance model performance.
    \item English models work well for Czech too.    
    \item Task-specified fine-tuning boosts performance. 
  \end{itemize}
  \begin{table}
    \centering
    \includegraphics[scale=0.8]{src/fig/pdfs/tables/balanced.pdf}
    \caption{Balanced models compared to baseline.}
  \end{table}
\end{frame}


% Retrieval-Augmented Generation (RAG)
\begin{frame}
  \frametitle{Retrieval-Augmented Generation (RAG)}
  \begin{columns}[onlytextwidth,T]
    \begin{column}{0.75\textwidth}
      \begin{figure}[h]
        \includegraphics[scale=0.6]{src/fig/pdfs/tikz/RAG_scheme.pdf}
        \caption{RAG architecture.}
       \end{figure}
    \end{column}

    \begin{column}{0.75\textwidth}
      \vspace{10px}
      \textcolor{cvut_navy}{\textbf{Factors}}
      \begin{itemize}
        \item Embedding \\model
        \item Chunk size
      \end{itemize}
    \end{column}
  \end{columns}
\end{frame}


% Optimizing RAG
\begin{frame}
  \frametitle{Results: RAG optimization}
  \textcolor{cvut_navy}{\textbf{RAG Components}}
  \begin{itemize}
    \item Embedding model: GTE\textsubscript{Small}.
    \item Chunks: 256, 512, 1024, 2048, 4096 symbols.
    \item Almost same context size (3072 - 4096 symbols).
    \item Answers generation: GPT-3.5-turbo.
    \item Answers quality measurement: GPT-4o.
  \end{itemize}
  
  \begin{table}
    \input{src/tables/RAG_evaluation_presentation.tex}
    \caption{RAG evaluation.}
  \end{table}
\end{frame}


% Summary
\placelogofalse 
\begin{frame}
  \frametitle{Summary}
  \textcolor{cvut_navy}{\textbf{Summary}}
  \begin{itemize}
    \item Transformer models have better performance than traditional embedding models.    
    \item Best embedding models: SimCSE-RetroMAE-Small, GTE\textsubscript{Small}, all mE5 verions.
    \item Best embedding model overall: mE5\textsubscript{Large}.
    \item Best chunk size for RAG: 4096 symbols.
  \end{itemize}
  \vspace{10px}
  \begin{center}
    \huge \textcolor{cvut_navy}{\textbf{Thank you for your attention!}}  
  \end{center}
  \vspace{10px}
  \begin{center}
    \includegraphics[height=2cm]{src/fig/pdfs/ctu_logo_blue_filled.pdf}
  \end{center}
\end{frame}

% Questions
\placelogotrue
\begin{frame}
  \frametitle{Questions}
  \textcolor{cvut_navy}{\textbf{Opponent}}: Ing. Luboš Král, Ph.D.

  \vspace{20px}
  
  \textcolor{cvut_navy}{\textbf{Question 1.}} What is the main use for RAG systems?
  \begin{itemize}
    \item Improved accuracy for question answering.
    \item Up-to-date information from external sources for question answering.
  \end{itemize}
  \textcolor{cvut_navy}{\textbf{Question 2.}} By how much is the accuracy of RAG systems lower for Czech than for English?  
  \begin{itemize}
    \item Accuracy can be 15-20\% lower\footnote{Based on analysis of multilingual models performance from MTEB, \url{https://huggingface.co/spaces/mteb/leaderboard}.}.
    \item Focus on English language for research community.
    \item Lack of training data.
  \end{itemize}
\end{frame}

\end{document}
% =============================================================
% =========================== END =============================
% =============================================================