diff --git a/thesis/Main.pdf b/thesis/Main.pdf index 5342ed4..53cdb99 100644 Binary files a/thesis/Main.pdf and b/thesis/Main.pdf differ diff --git a/thesis/Main.tex b/thesis/Main.tex index 739e0ab..24c7966 100755 --- a/thesis/Main.tex +++ b/thesis/Main.tex @@ -359,7 +359,6 @@ In the main training step, DeepSAD's network is trained using SGD backpropagatio \fig{deepsad_procedure}{diagrams/deepsad_procedure/deepsad_procedure}{(WORK IN PROGRESS) Depiction of DeepSAD's training procedure, including data flows and tweakable hyperparameters.} - To infer if a previously unknown data sample is normal or anomalous, the sample is fed in a forward-pass through the fully trained network. During inference, the centroid $\mathbf{c}$ needs to be known, to calculate the geometric distance of the samples latent representation to $\mathbf{c}$. This distance is tantamount to an anomaly score, which correlates with the likelihood of the sample being anomalous. Due to differences in input data type, training success and latent space dimensionality, the anomaly score's magnitude has to be judged on an individual basis for each trained network. This means, scores produced by one network that signify normal data, may very well clearly indicate an anomaly for another network. The geometric distance between two points in space is a scalar analog value, therefore post-processing of the score is necessary to achieve a binary classification of normal and anomalous if desired. DeepSAD's full training and inference procedure is visualized in figure~\ref{fig:deepsad_procedure}, which gives a comprehensive overview of the dataflows, tuneable hyperparameters and individual steps involved. diff --git a/thesis/diagrams/deepsad_procedure/deepsad_procedure.pdf b/thesis/diagrams/deepsad_procedure/deepsad_procedure.pdf index 764a98d..e254220 100644 Binary files a/thesis/diagrams/deepsad_procedure/deepsad_procedure.pdf and b/thesis/diagrams/deepsad_procedure/deepsad_procedure.pdf differ diff --git a/thesis/diagrams/deepsad_procedure/deepsad_procedure.tex b/thesis/diagrams/deepsad_procedure/deepsad_procedure.tex index 5b81638..bbe2343 100644 --- a/thesis/diagrams/deepsad_procedure/deepsad_procedure.tex +++ b/thesis/diagrams/deepsad_procedure/deepsad_procedure.tex @@ -1,5 +1,6 @@ \documentclass[tikz,border=10pt]{standalone} \usepackage{tikz} +\usepackage{amsfonts} \usetikzlibrary{positioning, shapes.geometric, fit, arrows, arrows.meta, backgrounds} % Define box styles @@ -7,9 +8,9 @@ databox/.style={rectangle, align=center, draw=black, fill=blue!50, thick, rounded corners},%, inner sep=4}, procbox/.style={rectangle, align=center, draw=black, fill=orange!30, thick, rounded corners}, hyperbox/.style={rectangle, align=center, draw=black, fill=green!30, thick, rounded corners}, - stepsbox/.style={rectangle, align=left, draw=black,fill=white, rounded corners, minimum width=6cm, minimum height=1.5cm, font=\small}, - outputbox/.style={rectangle, align=center, draw=red!80, fill=red!20, rounded corners, minimum width=6cm, minimum height=1.5cm, font=\small}, - hlabelbox/.style={rectangle, align=center, draw=black,fill=white, rounded corners, minimum width=6cm, minimum height=1.5cm, font=\small}, + stepsbox/.style={rectangle, align=left, draw=black,fill=white, rounded corners, minimum width=5.2cm, minimum height=1.5cm, font=\small}, + outputbox/.style={rectangle, align=center, draw=red!80, fill=red!20, rounded corners, minimum width=5.2cm, minimum height=1.5cm, font=\small}, + hlabelbox/.style={rectangle, align=center, draw=black,fill=white, rounded corners, minimum width=5.2cm, minimum height=1.5cm, font=\small}, vlabelbox/.style={rectangle, align=center, draw=black,fill=white, rounded corners, minimum width=3cm, minimum height=1.8cm, font=\small}, arrow/.style={-{Latex[length=3mm]}}, arrowlabel/.style={fill=white,inner sep=2pt,midway} @@ -25,11 +26,11 @@ \begin{tikzpicture}[node distance=1cm and 2cm] \node (data) {Data}; - \node[right=7 of data] (process) {Procedure}; - \node[right=7 of process] (hyper) {Hyperparameters}; + \node[right=4.9 of data] (process) {Procedure}; + \node[right=4.1 of process] (hyper) {Hyperparameters}; \begin{pgfonlayer}{foreground} - \node[hlabelbox, below=of data] (unlabeled) {\boxtitle{Unlabeled Data} More normal than \\ anomalous samples required}; + \node[hlabelbox, below=1.29 of data] (unlabeled) {\boxtitle{Unlabeled Data} Significantly more normal than \\ anomalous samples required}; \node[hlabelbox, below=.1 of unlabeled] (labeled) {\boxtitle{Labeled Data} No requirement regarding ratio \\ +1 = normal, -1 = anomalous}; \end{pgfonlayer} \begin{pgfonlayer}{background} @@ -39,16 +40,16 @@ %\draw[arrow] (latent.east) -- node{} (autoenc.west); \begin{pgfonlayer}{foreground} - \node[stepsbox, below=of process] (pretrainproc) {Train Autoencoder for $E_A$ Epochs \\ with $L_A$ Learning Rate \\ No Labels Used}; - \node[outputbox, below=.1 of pretrainproc] (pretrainout) {\boxtitle{Outputs} Encoder Network \\ $\mathbf{w}$: Network Weights}; + \node[stepsbox, below=of process] (pretrainproc) {Train Autoencoder $\mathcal{W}_{E}$ \\ optimize Autoencoding Objective \\ for $E_A$ Epochs \\ with $L_A$ Learning Rate \\ No Labels Used / Required}; + \node[outputbox, below=.1 of pretrainproc] (pretrainout) {\boxtitle{Outputs} $\mathcal{W}$: Encoder / DeepSAD Network \\ $\mathbf{w_{E}}$: Encoder Network Weights}; \end{pgfonlayer} \begin{pgfonlayer}{background} \node[procbox, fit=(pretrainproc) (pretrainout), label={[label distance = 1, name=pretrainlab]above:{\textbf{Pre-Training of Autoencoder}}}] (pretrain) {}; \end{pgfonlayer} \begin{pgfonlayer}{foreground} - \node[hlabelbox, below=of hyper] (autoencarch) {\boxtitle{Autoencoder Architecture} Choose based on data type \\ Latent Space Size (based on complexity)}; - \node[hlabelbox, below=.1 of autoencarch] (pretrainhyper) {\boxtitle{Hyperparameters} $E_A$: Number of Epochs \\ $L_A$: Learning Rate}; + \node[hlabelbox, below=1.26 of hyper] (autoencarch) {\boxtitle{Autoencoder Architecture} $\mathcal{W}_{E}$: Autoencoder Network \\ $\mathbb{R}^d$: Latent Space Size }; + \node[hlabelbox, below=.1 of autoencarch] (pretrainhyper) {\boxtitle{Hyperparameters} $E_A$: Number of Epochs \\ $L_A$: Learning Rate AE}; \end{pgfonlayer} \begin{pgfonlayer}{background} \node[hyperbox, fit=(autoencarch) (pretrainhyper), label={[label distance = 1, name=autoenclabel]above:{\textbf{Pre-Training Hyperparameters}}}] (pretrainhyp) {}; @@ -61,7 +62,7 @@ % \draw[arrow] (node cs:name=autoenc,angle=196) |- (node cs:name=pretrain,angle=5); \begin{pgfonlayer}{foreground} - \node[stepsbox, below=1.4 of pretrain] (calccproc) {1. Init Encoder with $\mathbf{w}$ \\ 2. Forward Pass on all data \\ 3. $\mathbf{c}$ = Mean Latent Representation}; + \node[stepsbox, below=1.4 of pretrain] (calccproc) {Init Network $\mathcal{W}$ with $\mathbf{w_{E}}$ \\ Forward Pass on all data \\ Hypersphere center $\mathbf{c}$ is mean \\ of all Latent Representation}; \node[outputbox, below=.1 of calccproc] (calccout) {\boxtitle{Outputs} $\mathbf{c}$: Hypersphere Center}; \end{pgfonlayer} \begin{pgfonlayer}{background} @@ -76,21 +77,21 @@ %\draw[arrow] (node cs:name=traindata,angle=-45) |- node[arrowlabel]{all training data, labels removed} (node cs:name=calcc,angle=200); \begin{pgfonlayer}{foreground} - \node[stepsbox, below=1.4 of calcc] (maintrainproc) {Train Network for $E_M$ Epochs \\ with $L_M$ Learning Rate \\ Considers Labels with $\eta$ strength}; - \node[outputbox, below=.1 of maintrainproc] (maintrainout) {\boxtitle{Outputs} Encoder Network \\ $\mathbf{w}$: Network Weights \\ $\mathbf{c}$: Hypersphere Center}; + \node[stepsbox, below=1.4 of calcc] (maintrainproc) {Init Network $\mathcal{W}$ with $\mathbf{w_{E}}$ \\ Train Network $\mathcal{W}$ \\ optimize DeepSAD Objective\\ for $E_M$ Epochs \\ with $L_M$ Learning Rate \\ Considers Labels with $\eta$ strength}; + \node[outputbox, below=.1 of maintrainproc] (maintrainout) {\boxtitle{Outputs} $\mathcal{W}$: DeepSAD Network \\ $\mathbf{w}$: DeepSAD Network Weights \\ $\mathbf{c}$: Hypersphere Center}; \end{pgfonlayer} \begin{pgfonlayer}{background} \node[procbox, fit=(maintrainproc) (maintrainout), label={[label distance = 1, name=maintrainlab]above:{\textbf{Main Training}}}] (maintrain) {}; \end{pgfonlayer} \begin{pgfonlayer}{foreground} - \node[hlabelbox, below=11.25 of hyper] (maintrainhyper) {$E_M$: Number of Epochs \\ $L_M$: Learning Rate \\ $\eta$: Strength Labeled/Unlabeled}; + \node[hlabelbox, below=12.48 of hyper] (maintrainhyper) {$E_M$: Number of Epochs \\ $L_M$: Learning Rate \\ $\eta$: Weight Labeled/Unlabeled}; \end{pgfonlayer} \begin{pgfonlayer}{background} \node[hyperbox, fit=(maintrainhyper), label={[label distance = 1, name=autoenclabel]above:{\textbf{Main-Training Hyperparameters}}}] (maintrainhyp) {}; \end{pgfonlayer} - \draw[arrow] (node cs:name=pretrain,angle=-20) -- +(1, 0) |- (node cs:name=maintrain,angle=20); + \draw[arrow] (node cs:name=pretrain,angle=-50) |- +(1.5, -0.55) -- +(1.5,-5.4) -| (node cs:name=maintrain,angle=50); %\draw[arrow] (pretrainoutput.south) -- (node cs:name=maintrain,angle=22); @@ -101,7 +102,7 @@ \begin{pgfonlayer}{foreground} - \node[stepsbox, below=1.4 of maintrain] (inferenceproc) {Forward Pass through Network = $\mathbf{p}$ \\ Calculate Geometric Distance $\mathbf{p} \rightarrow \mathbf{c}$ \\ Anomaly Score = Geometric Distance}; + \node[stepsbox, below=1.4 of maintrain] (inferenceproc) {Init Network $\mathcal{W}$ with $\mathbf{w}$ \\Forward Pass on sample = $\mathbf{p}$ \\ Calculate Distance $\mathbf{p} \rightarrow \mathbf{c}$ \\ Distance = Anomaly Score}; \node[outputbox, below=.1 of inferenceproc] (inferenceout) {\boxtitle{Outputs} Anomaly Score (Analog Value) \\ Higher for Anomalies}; \end{pgfonlayer} \begin{pgfonlayer}{background} @@ -109,7 +110,7 @@ \end{pgfonlayer} \begin{pgfonlayer}{foreground} - \node[hlabelbox, below=11.8 of traindata] (newdatasample) {\boxtitle{New Data Sample} Same data type as training data}; + \node[hlabelbox, below=13.32 of traindata] (newdatasample) {\boxtitle{New Data Sample} Same data type as training data}; \end{pgfonlayer} \begin{pgfonlayer}{background} \node[databox, fit=(newdatasample), label={[label distance = 1] above:{\textbf{Unseen Data}}}] (newdata) {};