\documentclass[12pt, unicode]{beamer} \usetheme{Warsaw} \usepackage{luatexja} \usepackage{color} \usepackage{listings} \lstset{ basicstyle=\ttfamily\bfseries, commentstyle=\color{red}\itshape, stringstyle=\color{black!30!green}, showstringspaces=false, keywordstyle=\color{blue}\bfseries} %color \definecolor{battleshipgrey}{rgb}{0.52, 0.52, 0.51} \title{Fluentd forward protocol modes in v0} \subtitle{... and how to implement minimal Fluentd logger.} \author{Hiroshi Hatake} \date[2016/04/08]{Technical information sharing seminar} \begin{document} \frame{\maketitle} \begin{frame}{Introduction} \begin{block}{What is Fluentd?} Fluentd is a fully free open-source log collector for unified logging layer. \end{block} \onslide<1-> \begin{itemize} \item<2-> Unified logging with JSON and msgpack \item<3-> Pluggable architecture \item<4-> Minimum resources required \item<5-> Built-in reliability \end{itemize} \end{frame} \begin{frame}{Forward protocol - Introduction} \begin{block}{Forward protocol\footnote[frame]{https://github.com/fluent/fluentd/wiki/Forward-Protocol-Specification-v0}} Fluentd has three modes in forward protocol. \end{block} \begin{itemize} \item<1-> Message Mode \item<2-> Forward Mode \item<3-> PackedForward Mode \end{itemize} \end{frame} \begin{frame}{Message Mode protocol} \begin{block}{Message Mode\footnote[frame]{https://github.com/fluent/fluentd/wiki/Forward-Protocol-Specification-v0\#message-mode}} It carries just an event. \end{block} \begin{itemize} \item \uncover<1->{\textbf {tag} is a string separated with '.' (e.g. myapp.access) to categorize events.} \item \uncover<1->{\textbf {time} is a number of seconds since Unix epoch.} \item \uncover<1->{\textbf {record} is key-value pairs of the event record.} \item \uncover<2->{\color{battleshipgrey} \textbf {option} is optional key-value pairs, to bring data to control servers' behavior.} \end{itemize} \end{frame} \begin{frame}{Message Mode protocol} \begin{block}{Message Mode\footnote[frame]{https://github.com/fluent/fluentd/wiki/Forward-Protocol-Specification-v0\#message-mode}} It carries just an event. \end{block} \begin{table}[htb] \resizebox{\textwidth}{!}{% \begin{tabular}{l | c | c | c } \hline name & type & content & mandatory? \\ \hline \hline tag & str & tag name & yes \\ \hline time & Integer/EventTime(ext type) & Unix Time/Unix Time with nano seconds & yes \\ \hline record & Object(like HashMap) & pairs of keys(String) and values(Object) & yes \\ \hline option & Object(like HashMap) & pairs of keys(String) and values(Object) & no \\ \hline \end{tabular}} \end{table} \end{frame} \begin{frame}{Forward Mode protocol} \begin{block}{Forward Mode\footnote[frame]{https://github.com/fluent/fluentd/wiki/Forward-Protocol-Specification-v0\#forward-mode}} It carries a series of events as a msgpack array on a single request. \end{block} \begin{itemize} \item \uncover<1->{\textbf {tag} is a string separated with '.' (e.g. myapp.access) to categorize events.} \item \uncover<1->{\textbf {entries}\footnote[frame]{https://github.com/fluent/fluentd/wiki/Forward-Protocol-Specification-v0\#entry} is an array of Entries which are pairs of Unix epoch and record.} \item \uncover<2->{\color{battleshipgrey} \textbf {option} is optional key-value pairs, to bring data to control servers' behavior.} \end{itemize} \end{frame} \begin{frame}{Forward Mode protocol} \begin{block}{Forward Mode\footnote[frame]{https://github.com/fluent/fluentd/wiki/Forward-Protocol-Specification-v0\#forward-mode}} It carries a series of events as a msgpack array on a single request. \end{block} \begin{table}[htb] \resizebox{\textwidth}{!}{% \begin{tabular}{l | c | c | c } \hline name & type & content & mandatory? \\ \hline \hline tag & str & tag name & yes \\ \hline entries & Array & an array of Entries which are pairs of Unix epoch and record. & yes \\ \hline option & Object(like HashMap) & pairs of keys(String) and values(Object) & no \\ \hline \end{tabular}} \end{table} \end{frame} \begin{frame}{PackedForward Mode protocol} \begin{block}{PackedForward Mode\footnote[frame]{https://github.com/fluent/fluentd/wiki/Forward-Protocol-Specification-v0\#packedforward-mode}} It carries a series of events as a msgpack binary on a single request. \end{block} \begin{itemize} \item \uncover<1->{\textbf {tag} is a string separated with '.' (e.g. myapp.access) to categorize events.} \item \uncover<1->{\textbf {entries} is a msgpack stream of Entry which contains pairs of Unix epoch and record msgpack binary.} \item \uncover<2->{\color{battleshipgrey} \textbf {option} is optional key-value pairs, to bring data to control servers' behavior.} \end{itemize} \end{frame} \begin{frame}{PackedForward Mode protocol} \begin{block}{PackedForward Mode\footnote[frame]{https://github.com/fluent/fluentd/wiki/Forward-Protocol-Specification-v0\#packedforward-mode}} It carries a series of events as a msgpack binary on a single request. \end{block} \begin{table}[htb] \resizebox{\textwidth}{!}{% \begin{tabular}{l | c | c | c } \hline name & type & content & mandatory? \\ \hline \hline tag & str & tag name & yes \\ \hline entries & str | bin & a msgpack stream of Entry which contains pairs of Unix epoch and record. & yes \\ \hline option & Object(like HashMap) & pairs of keys(String) and values(Object) & no \\ \hline \end{tabular}} \end{table} \end{frame} \begin{frame}{How to implement minimal Fluentd logger?} \begin{block}{Minimal Fluentd logger implementation strategy} \begin{itemize} \item \uncover<1-> {{\textbf {Fluentd logger}} needs to send an event to Fluentd.} \begin{itemize} \item \uncover<2-> {{\textbf {Fluentd logger}} need {\textbf {not}} to add optional elements in sending events to Fluentd.} \item \uncover<3-> {{\textbf {Fluentd logger}} {\textbf {should}} use message mode to send events to Fluentd.} \end{itemize} \end{itemize} \end{block} \end{frame} \begin{frame}{How to implement minimal Fluentd logger?} \begin{block}{Minimal Fluentd logger implementation} \begin{itemize} \item \uncover<1-> {{\textbf {Fluentd logger}} needs to connect to Fluentd with TCP.} \item \uncover<2-> {{\textbf {Fluentd logger}} send an event which contains tag, timespec, and record, to Fluentd.} \item \uncover<3-> {{\textbf {Fluentd logger}} should disconnect connection against Fluentd immediately when sending an event.} \end{itemize} \end{block} \end{frame} \begin{frame}{A short characteristics of Rust\footnote[frame]{https://www.rust-lang.org/} language} \begin{block}{Characteristics} Rust is a systems programming language that runs blazingly fast, prevents segfaults, and guarantees thread safety. \end{block} \begin{itemize} \item \uncover<1-> {Guaranteed memory safety} \item \uncover<2-> {Trait-based generics} \item \uncover<3-> {Type inference etc.} \end{itemize} \end{frame} \begin{frame}{About Rust language -- what is the weakness points of Rust language?} \begin{block}{Weakness of Rust language} Rust does not have... \end{block} \begin{itemize} \item \uncover<1-> {Dynamic typing} \item \uncover<2-> {Code simplicity} \item \uncover<3-> {Easy to write code} \end{itemize} \end{frame} % Set font size for annotation. \newcommand\smallannotate{\fontsize{9}{9.2}\selectfont} \begin{frame}{A short exploration of Rust language} \begin{block}{Fun things of Rust} Rust makes you to benefit static type system because ... \end{block} \begin{itemize} \item \uncover<1-> {Flexible type by trait based generics} \item \uncover<2-> {To clarify software specification} \item \uncover<3-> {Well documented ecosystem etc. {\smallannotate(e.g. You don't create document when there is wrong code. Amazing!)}} \end{itemize} \end{frame} %% Define highlighting for Rust. \lstdefinelanguage{Rust} { morecomment = [l]{//}, morecomment = [l]{///}, morecomment = [s]{/*}{*/}, morestring=[b]", sensitive = true, morekeywords = {extern, crate, fn, let, use, mut} } % Set font size for lstlisting. \newcommand\Small{\fontsize{9}{9.2}\selectfont} \begin{frame}[fragile] fruently example: \begin{lstlisting}[language={Rust},basicstyle=\ttfamily\Small] extern crate fruently; use fruently::fluent::Fluent; use std::collections::HashMap; use fruently::forwardable::JsonForwardable; fn main() { // create record let mut obj: HashMap = HashMap::new(); obj.insert("name".to_string(), "fruently".to_string()); // establish a TCP connection with Fluentd. let fruently = Fluent::new("0.0.0.0:24224", "test"); // send json-encoded record to Fluentd // and disconnecting automatically. let _ = fruently.post(&obj); } \end{lstlisting} \end{frame} \frame{\centering \Large Demo} \frame{\centering \Large Any questions?} \end{document}