%\VignetteIndexEntry{Manipulation of data-frame data with dutility functions}
%\VignetteEngine{R.rsp::tex}
%\VignetteKeyword{R}
%\VignetteKeyword{package}
%\VignetteKeyword{vignette}
%\VignetteKeyword{LaTeX}
\PassOptionsToPackage{usenames}{xcolor}
\PassOptionsToPackage{hidelinks,colorlinks,linkcolor={blue!50!black},urlcolor={blue!50!black},citecolor={blue!50!black}}{hyperref}
\documentclass[a4paper]{tufte-handout}
\usepackage{etex}
\usepackage{etoolbox}
\usepackage[utf8]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{mathtools}
\usepackage[strict=true]{csquotes}
\usepackage{xcolor}
\usepackage{natbib}
\usepackage{amsmath,amssymb,textcomp}
\usepackage{bm}
\usepackage{graphicx}
\usepackage{wrapfig}
\usepackage{rotating}
\usepackage{capt-of}
\usepackage{listings}
\usepackage{booktabs}
\usepackage{multicol}
\usepackage{longtable}
\usepackage{zlmtt}
\usepackage{float}
\usepackage{fancyvrb}
\usepackage{verbatim}
\usepackage[english]{babel}
\usepackage{hyperref}
\usepackage{environ}
\NewEnviron{mnote}{\marginnote{\BODY}}
\NewEnviron{snote}{\sidenote{\BODY}}
\usepackage{xspace}
\usepackage{url}
\usepackage{amsthm}
\newtheorem{thm}{Theorem.}
\newtheorem{prop}{Proposition.}
\theoremstyle{definition}
\newtheorem{defn}[thm]{Definition.}
\newtheorem{exa}[thm]{Example}
\newcommand{\R}{\ensuremath{\mathbb{R}}} 
\newcommand{\Real}{\ensuremath{\mathbb{R}}} 
\newcommand{\Complex}{\ensuremath{\mathbb{C}}} 
\newcommand{\Z}{\ensuremath{\mathbb{Z}}} 
\newcommand{\N}{\ensuremath{\mathbb{N}}} 
\newcommand{\norm}[1]{\ensuremath{\left\Vert#1\right\Vert}} 
\newcommand{\var}{\ensuremath{\mathbb{V}\text{ar}}} 
\newcommand{\cov}{\ensuremath{\mathbb{C}\text{ov}}} 
\newcommand{\cor}{\ensuremath{\mathbb{C}\text{or}}} 
\newcommand{\E}{\ensuremath{\mathbb{E}}} 
\newcommand{\pr}{\ensuremath{\mathbb{P}}} 
\newcommand{\from}{\leftarrow} 
\newcommand{\To}[2]{\ensuremath{#1\!\to\!#2}}
\newcommand{\From}[2]{\ensuremath{#1\!\from\!#2}}
\newcommand{\chain}[3]{\ensuremath{#1\!\to\!#2\to\!#3}}
\newcommand{\ichain}[3]{\ensuremath{#1\!\from\!#2\from\!#3}}
\newcommand{\fork}[3]{\ensuremath{#1\!\from\!#2\to\!#3}}
\newcommand{\ifork}[3]{\ensuremath{#1\!\to\!#2\from\!#3}}
\newcommand{\pa}{\text{pa}}
\newcommand{\abs}[1]{\ensuremath{\left\vert#1\right\vert}} 
\newcommand{\ipr}[1]{\langle#1\rangle} 
\newcommand{\set}[1]{\left{#1\right}} 
\newcommand{\seq}[1]{\left<#1\right>} 
\renewcommand{\subset}{\subseteq}  
\renewcommand{\supset}{\supseteq} 
\DeclareMathOperator*{\argmax}{arg\,max}
\DeclareMathOperator*{\argmin}{arg\,min}
\DeclareMathOperator{\sgn}{sgn}
\DeclareMathOperator{\mvec}{vec}
\DeclareMathOperator{\tr}{tr}
\DeclareMathOperator{\im}{im}
\DeclareMathOperator{\diag}{diag}
\DeclareMathOperator{\bias}{bias}
\DeclareMathOperator{\rank}{rank}
\DeclareMathOperator{\logit}{logit}
\DeclareMathOperator{\expit}{expit}
\makeatletter
 \def\mathcolor#1#{\@mathcolor{#1}}
 \def\@mathcolor#1#2#3{%
   \protect\leavevmode
   \begingroup
     \color#1{#2}#3%
  \endgroup
 }
\newcommand{\Dto}{\overset{\mathcal{D}}{\longrightarrow}}
\newcommand{\Pto}{\overset{P}{\longrightarrow}}
\newcommand{\Wto}{\overset{W}{\longrightarrow}}
\newcommand{\VV}{\bm{\Omega}_\theta}
\newcommand{\independenT}[2]{\mathrel{\setbox0\hbox{$#1#2$}\copy0\kern-\wd0\mkern4mu\box0}} 
\newcommand{\indep}{\protect\mathpalette{\protect\independenT}{\perp}}
\DefineVerbatimEnvironment{verbatim}{Verbatim}{xleftmargin=0.5em,xrightmargin=0em,numbers=none,frame=none,fontsize=\footnotesize,formatcom={\color[rgb]{0.2,0.2,0.2}}}
\setlength{\parindent}{0pt} % Kills annoying indents. 
\let\iint\relax 
\let\iiint\relax

\lstset{basicstyle=\ttfamily\small,keywordstyle=\color{black},commentstyle=\color{gray}\ttfamily\itshape,stringstyle=\color[rgb]{0,0,0.5},columns=fullflexible,alsoletter=.,texcl=true,escapeinside={*@}{@*)},escapebegin=\lst@commentstyle\,,breaklines=true,breakatwhitespace=false,numbers=left,numberstyle=\ttfamily\tiny\color{gray},stepnumber=1,numbersep=10pt,backgroundcolor=\color{white},tabsize=4,showspaces=false,showstringspaces=false,xleftmargin=.23in,frame=lines
,rulesepcolor=\color[rgb]{0.85,0.85,0.85},basewidth={0.5em,0.42em},language=r,label=
,caption= ,captionpos=b}
\lstset{basicstyle=\ttfamily\small,keywordstyle=\color{black},commentstyle=\color{gray}\ttfamily\itshape,stringstyle=\color[rgb]{0,0,0.5},columns=fullflexible,alsoletter=.,texcl=true,escapeinside={*@}{@*)},escapebegin=\lst@commentstyle\,,breaklines=true,breakatwhitespace=false,numbers=left,numberstyle=\ttfamily\tiny\color{gray},stepnumber=1,numbersep=10pt,backgroundcolor=\color{white},tabsize=4,showspaces=false,showstringspaces=false,xleftmargin=.23in,frame=lines
,rulesepcolor=\color[rgb]{0.85,0.85,0.85},basewidth={0.5em,0.42em},language=python,label=
,caption= ,captionpos=b}

\setlength{\parindent}{0pt} % Kills annoying indents.
\newcommand{\n}{}
\author{Klaus Holst \& Thomas Scheike}
\date{\today}
\title{Manipulation of data-frame data with dutility functions}
\hypersetup{
 pdfauthor={Klaus Holst \& Thomas Scheike},
 pdftitle={Manipulation of data-frame data with dutility functions},
 pdfkeywords={},
 pdfsubject={},
 pdfcreator={Emacs 26.1 (Org mode 9.1.14)}, 
 pdflang={English}}
\begin{document}

\maketitle
\noindent\rule{\textwidth}{0.5pt}


\section*{Simple data manipulation for data-frames}
\label{sec:orge9cc1d9}

\begin{itemize}
\item Renaming variables, Deleting variables
\item Looking at the data
\item Making new variales for the analysis
\item Making factors (groupings)
\item Working with factors
\item Making a factor from existing numeric variable and vice versa
\end{itemize}


Here are some key data-manipulation steps on a data-frame which is how we typically 
organize our data in R. 
After having read the data into R it will typically be a data-frame, if not we 
can force it to be a data-frame.   The basic idea of the utility functions is 
to get a simple and easy to type way of making simple data-manipulation on a 
data-frame much like what is possible in SAS or STATA.  

The functions, say,  dcut, dfactor and so on are all functions that basically does what 
the base R cut, factor do, but are easier to use in the context of data-frames 
and have additional functionality. 

\lstset{numbers=left,numberstyle=\ttfamily\tiny\color{gray},language=r,label= ,caption= ,captionpos=b}
\begin{lstlisting}
library(mets)
data(melanoma)
\end{lstlisting}


\lstset{numbers=left,numberstyle=\ttfamily\tiny\color{gray},language=r,label= ,caption= ,captionpos=b}
\begin{lstlisting}
is.data.frame(melanoma)
\end{lstlisting}

\begin{verbatim}
[1] TRUE
\end{verbatim}

Here we work on the melanoma data that is already read into R and is 
a data-frame.


\section*{dUtility functions}
\label{sec:org995ef5a}

The structure for all functions 
is  

\begin{itemize}
\item dfunction(dataframe,y\textasciitilde{}x|ifcond,\ldots{})
\end{itemize}

to use the function on y in a dataframe grouped by x if 
condition ifcond is valid. The basic functions are 

Data processing
\begin{itemize}
\item dsort
\item dreshape
\item dcut
\item drm, drename, ddrop, dkeep, dsubset
\item drelevel
\item dlag
\item dfactor, dnumeric
\end{itemize}
Data aggregation
\begin{itemize}
\item dby, dby2
\item dscalar, deval, daggregate
\item dmean, dsd, dsum, dquantile, dcor
\item dtable, dcount
\end{itemize}
Data summaries
\begin{itemize}
\item dhead, dtail,
\item dsummary,
\item dprint, dlist, dlevels, dunique
\end{itemize}


A generic function daggregate, daggr,  can be called with a function as the 
argument 


\begin{itemize}
\item daggregate(dataframe,y\textasciitilde{}x|ifcond,fun=function,\ldots{})
\end{itemize}

without the grouping variable (x)

\begin{itemize}
\item daggregate(dataframe,\textasciitilde{}y|ifcond,fun=function,\ldots{})
\end{itemize}

A useful feature is that y and x as well as the subset condition
can be specified using regular-expressions or by wildcards (default).
Here to illustrate this, we compute the means of certain variables. 

First just oveall 
\lstset{numbers=left,numberstyle=\ttfamily\tiny\color{gray},language=r,label= ,caption= ,captionpos=b}
\begin{lstlisting}
dmean(melanoma,~thick+I(log(thick)))
\end{lstlisting}

\begin{verbatim}
     thick I(log(thick)) 
291.985366      5.223341
\end{verbatim}

now only when days>500
\lstset{numbers=left,numberstyle=\ttfamily\tiny\color{gray},language=r,label= ,caption= ,captionpos=b}
\begin{lstlisting}
dmean(melanoma,~thick+I(log(thick))|I(days>500))
\end{lstlisting}

\begin{verbatim}
     thick I(log(thick)) 
271.582011      5.168691
\end{verbatim}

and now after sex but only when days>500
\lstset{numbers=left,numberstyle=\ttfamily\tiny\color{gray},language=r,label= ,caption= ,captionpos=b}
\begin{lstlisting}
dmean(melanoma,thick+I(log(thick))~sex|I(days>500))
\end{lstlisting}

\begin{verbatim}
  sex    thick I(log(thick))
1   0 242.9580      5.060086
2   1 320.2429      5.353321
\end{verbatim}

and finally after quartiles of days (via the dcut function)
\lstset{numbers=left,numberstyle=\ttfamily\tiny\color{gray},language=r,label= ,caption= ,captionpos=b}
\begin{lstlisting}
dmean(melanoma,thick+I(log(thick))~I(dcut(days)))
\end{lstlisting}

\begin{verbatim}
        I(dcut(days))    thick I(log(thick))
1       [10,1.52e+03] 482.1731      5.799525
2    (1.52e+03,2e+03] 208.5490      4.987652
3    (2e+03,3.04e+03] 223.2941      4.974759
4 (3.04e+03,5.56e+03] 250.1961      5.120129
\end{verbatim}

or summary of all variables starting with "s" and that contains "a"

\lstset{numbers=left,numberstyle=\ttfamily\tiny\color{gray},language=r,label= ,caption= ,captionpos=b}
\begin{lstlisting}
dmean(melanoma,"s*"+"*a*"~sex|I(days>500))
\end{lstlisting}

\begin{verbatim}
  sex   status     days
1   0 1.831933 2399.143
2   1 1.714286 2169.800
\end{verbatim}


\section*{Renaming, deleting, keeping, dropping  variables}
\label{sec:orgb2de04c}

\lstset{numbers=left,numberstyle=\ttfamily\tiny\color{gray},language=r,label= ,caption= ,captionpos=b}
\begin{lstlisting}
melanoma=drename(melanoma,tykkelse~thick)
names(melanoma)
\end{lstlisting}

\begin{verbatim}
[1] "no"       "status"   "days"     "ulc"      "tykkelse" "sex"
\end{verbatim}


Deleting variables

\lstset{numbers=left,numberstyle=\ttfamily\tiny\color{gray},language=r,label= ,caption= ,captionpos=b}
\begin{lstlisting}
data(melanoma)
melanoma=drm(melanoma,~thick+sex)
names(melanoma)
\end{lstlisting}

\begin{verbatim}
[1] "no"     "status" "days"   "ulc"
\end{verbatim}

or sas style

\lstset{numbers=left,numberstyle=\ttfamily\tiny\color{gray},language=r,label= ,caption= ,captionpos=b}
\begin{lstlisting}
data(melanoma)
melanoma=ddrop(melanoma,~thick+sex)
names(melanoma)
\end{lstlisting}

\begin{verbatim}
[1] "no"     "status" "days"   "ulc"
\end{verbatim}

alternatively we can also keep certain variables

\lstset{numbers=left,numberstyle=\ttfamily\tiny\color{gray},language=r,label= ,caption= ,captionpos=b}
\begin{lstlisting}
data(melanoma)
melanoma=dkeep(melanoma,~thick+sex+status+days)
names(melanoma)
\end{lstlisting}

\begin{verbatim}
[1] "thick"  "sex"    "status" "days"
\end{verbatim}


This can also be done with direct asignment 

\lstset{numbers=left,numberstyle=\ttfamily\tiny\color{gray},language=r,label= ,caption= ,captionpos=b}
\begin{lstlisting}
data(melanoma)
ddrop(melanoma) <- ~thick+sex
names(melanoma)
\end{lstlisting}

\begin{verbatim}
[1] "no"     "status" "days"   "ulc"
\end{verbatim}


\section*{Looking at the data}
\label{sec:orgae42906}

\lstset{numbers=left,numberstyle=\ttfamily\tiny\color{gray},language=r,label= ,caption= ,captionpos=b}
\begin{lstlisting}
data(melanoma)
dstr(melanoma)
\end{lstlisting}

\begin{verbatim}
'data.frame':	205 obs. of  6 variables:
 $ no    : int  789 13 97 16 21 469 685 7 932 944 ...
 $ status: int  3 3 2 3 1 1 1 1 3 1 ...
 $ days  : int  10 30 35 99 185 204 210 232 232 279 ...
 $ ulc   : int  1 0 0 0 1 1 1 1 1 1 ...
 $ thick : int  676 65 134 290 1208 484 516 1288 322 741 ...
 $ sex   : int  1 1 1 0 1 1 1 1 0 0 ...
\end{verbatim}

The data can in Rstudio be seen as a data-table but to list certain parts of 
the data in output window 

\lstset{numbers=left,numberstyle=\ttfamily\tiny\color{gray},language=r,label= ,caption= ,captionpos=b}
\begin{lstlisting}
dlist(melanoma)
\end{lstlisting}

\begin{verbatim}
    no  status days ulc thick sex
1   789 3       10  1    676  1  
2    13 3       30  0     65  1  
3    97 2       35  0    134  1  
4    16 3       99  0    290  0  
5    21 1      185  1   1208  1  
---                              
201 317 2      4492 1   706   1  
202 798 2      4668 0   612   0  
203 806 2      4688 0    48   0  
204 606 2      4926 0   226   0  
205 328 2      5565 0   290   0
\end{verbatim}

\lstset{numbers=left,numberstyle=\ttfamily\tiny\color{gray},language=r,label= ,caption= ,captionpos=b}
\begin{lstlisting}
dlist(melanoma, ~.|sex==1)
\end{lstlisting}

\begin{verbatim}
    no  status days ulc thick
1   789 3       10  1    676 
2    13 3       30  0     65 
3    97 2       35  0    134 
5    21 1      185  1   1208 
6   469 1      204  1    484 
---                          
191 445 2      3909 1   806  
195 415 2      4119 0    65  
197 175 2      4207 0    65  
198 493 2      4310 0   210  
201 317 2      4492 1   706
\end{verbatim}

\lstset{numbers=left,numberstyle=\ttfamily\tiny\color{gray},language=r,label= ,caption= ,captionpos=b}
\begin{lstlisting}
dlist(melanoma, ~ulc+days+thick+sex|sex==1)
\end{lstlisting}

\begin{verbatim}
    ulc days thick sex
1   1    10   676  1  
2   0    30    65  1  
3   0    35   134  1  
5   1   185  1208  1  
6   1   204   484  1  
---                   
191 1   3909 806   1  
195 0   4119  65   1  
197 0   4207  65   1  
198 0   4310 210   1  
201 1   4492 706   1
\end{verbatim}


Getting summaries

\lstset{numbers=left,numberstyle=\ttfamily\tiny\color{gray},language=r,label= ,caption= ,captionpos=b}
\begin{lstlisting}
dsummary(melanoma)
\end{lstlisting}

\begin{verbatim}
      no            status          days           ulc            thick     
Min.   :  2.0   Min.   :1.00   Min.   :  10   Min.   :0.000   Min.   :  10  
1st Qu.:222.0   1st Qu.:1.00   1st Qu.:1525   1st Qu.:0.000   1st Qu.:  97  
Median :469.0   Median :2.00   Median :2005   Median :0.000   Median : 194  
Mean   :463.9   Mean   :1.79   Mean   :2153   Mean   :0.439   Mean   : 292  
3rd Qu.:731.0   3rd Qu.:2.00   3rd Qu.:3042   3rd Qu.:1.000   3rd Qu.: 356  
Max.   :992.0   Max.   :3.00   Max.   :5565   Max.   :1.000   Max.   :1742  
     sex        
Min.   :0.0000  
1st Qu.:0.0000  
Median :0.0000  
Mean   :0.3854  
3rd Qu.:1.0000  
Max.   :1.0000
\end{verbatim}

or for specfic variables

\lstset{numbers=left,numberstyle=\ttfamily\tiny\color{gray},language=r,label= ,caption= ,captionpos=b}
\begin{lstlisting}
dsummary(melanoma,~thick+status+sex)
\end{lstlisting}

\begin{verbatim}
    thick          status          sex        
Min.   :  10   Min.   :1.00   Min.   :0.0000  
1st Qu.:  97   1st Qu.:1.00   1st Qu.:0.0000  
Median : 194   Median :2.00   Median :0.0000  
Mean   : 292   Mean   :1.79   Mean   :0.3854  
3rd Qu.: 356   3rd Qu.:2.00   3rd Qu.:1.0000  
Max.   :1742   Max.   :3.00   Max.   :1.0000
\end{verbatim}

Summaries in different groups (sex)

\lstset{numbers=left,numberstyle=\ttfamily\tiny\color{gray},language=r,label= ,caption= ,captionpos=b}
\begin{lstlisting}
dsummary(melanoma,thick+days+status~sex)
\end{lstlisting}

\begin{verbatim}
sex: 0
     thick             days          status     
 Min.   :  10.0   Min.   :  99   Min.   :1.000  
 1st Qu.:  97.0   1st Qu.:1636   1st Qu.:2.000  
 Median : 162.0   Median :2059   Median :2.000  
 Mean   : 248.6   Mean   :2283   Mean   :1.833  
 3rd Qu.: 306.0   3rd Qu.:3131   3rd Qu.:2.000  
 Max.   :1742.0   Max.   :5565   Max.   :3.000  
------------------------------------------------------------ 
sex: 1
     thick             days          status     
 Min.   :  16.0   Min.   :  10   Min.   :1.000  
 1st Qu.: 105.0   1st Qu.:1052   1st Qu.:1.000  
 Median : 258.0   Median :1860   Median :2.000  
 Mean   : 361.1   Mean   :1946   Mean   :1.722  
 3rd Qu.: 484.0   3rd Qu.:2784   3rd Qu.:2.000  
 Max.   :1466.0   Max.   :4492   Max.   :3.000
\end{verbatim}

and only among those with thin-tumours or only females (sex==1)

\lstset{numbers=left,numberstyle=\ttfamily\tiny\color{gray},language=r,label= ,caption= ,captionpos=b}
\begin{lstlisting}
dsummary(melanoma,thick+days+status~sex|thick<97)
\end{lstlisting}

\begin{verbatim}
sex: 0
     thick            days          status     
 Min.   :10.00   Min.   : 355   Min.   :1.000  
 1st Qu.:32.00   1st Qu.:1762   1st Qu.:2.000  
 Median :64.00   Median :2227   Median :2.000  
 Mean   :51.48   Mean   :2425   Mean   :2.034  
 3rd Qu.:65.00   3rd Qu.:3185   3rd Qu.:2.000  
 Max.   :81.00   Max.   :4688   Max.   :3.000  
------------------------------------------------------------ 
sex: 1
     thick            days          status     
 Min.   :16.00   Min.   :  30   Min.   :1.000  
 1st Qu.:30.00   1st Qu.:1820   1st Qu.:2.000  
 Median :65.00   Median :2886   Median :2.000  
 Mean   :55.75   Mean   :2632   Mean   :1.875  
 3rd Qu.:81.00   3rd Qu.:3328   3rd Qu.:2.000  
 Max.   :81.00   Max.   :4207   Max.   :3.000
\end{verbatim}

\lstset{numbers=left,numberstyle=\ttfamily\tiny\color{gray},language=r,label= ,caption= ,captionpos=b}
\begin{lstlisting}
dsummary(melanoma,thick+status~+1|sex==1)
\end{lstlisting}

\begin{verbatim}
    thick            status     
Min.   :  16.0   Min.   :1.000  
1st Qu.: 105.0   1st Qu.:1.000  
Median : 258.0   Median :2.000  
Mean   : 361.1   Mean   :1.722  
3rd Qu.: 484.0   3rd Qu.:2.000  
Max.   :1466.0   Max.   :3.000
\end{verbatim}


or 

\lstset{numbers=left,numberstyle=\ttfamily\tiny\color{gray},language=r,label= ,caption= ,captionpos=b}
\begin{lstlisting}
dsummary(melanoma,~thick+status|sex==1)
\end{lstlisting}

\begin{verbatim}
    thick            status     
Min.   :  16.0   Min.   :1.000  
1st Qu.: 105.0   1st Qu.:1.000  
Median : 258.0   Median :2.000  
Mean   : 361.1   Mean   :1.722  
3rd Qu.: 484.0   3rd Qu.:2.000  
Max.   :1466.0   Max.   :3.000
\end{verbatim}


To make more complex conditions need to use the I() 

\lstset{numbers=left,numberstyle=\ttfamily\tiny\color{gray},language=r,label= ,caption= ,captionpos=b}
\begin{lstlisting}
dsummary(melanoma,thick+days+status~sex|I(thick<97 & sex==1))
\end{lstlisting}

\begin{verbatim}
sex: 1
     thick            days          status     
 Min.   :16.00   Min.   :  30   Min.   :1.000  
 1st Qu.:30.00   1st Qu.:1820   1st Qu.:2.000  
 Median :65.00   Median :2886   Median :2.000  
 Mean   :55.75   Mean   :2632   Mean   :1.875  
 3rd Qu.:81.00   3rd Qu.:3328   3rd Qu.:2.000  
 Max.   :81.00   Max.   :4207   Max.   :3.000
\end{verbatim}



Tables between variables

\lstset{numbers=left,numberstyle=\ttfamily\tiny\color{gray},language=r,label= ,caption= ,captionpos=b}
\begin{lstlisting}
dtable(melanoma,~status+sex)
\end{lstlisting}

\begin{verbatim}

       sex  0  1
status          
1          28 29
2          91 43
3           7  7
\end{verbatim}

All bivariate tables

\lstset{numbers=left,numberstyle=\ttfamily\tiny\color{gray},language=r,label= ,caption= ,captionpos=b}
\begin{lstlisting}
dtable(melanoma,~status+sex+ulc,level=2)
\end{lstlisting}

\begin{verbatim}

   status
sex  1  2  3
  0 28 91  7
  1 29 43  7

   status
ulc  1  2  3
  0 16 92  7
  1 41 42  7

   sex
ulc  0  1
  0 79 36
  1 47 43
\end{verbatim}

All univariate tables

\lstset{numbers=left,numberstyle=\ttfamily\tiny\color{gray},language=r,label= ,caption= ,captionpos=b}
\begin{lstlisting}
dtable(melanoma,~status+sex+ulc,level=1)
\end{lstlisting}

\begin{verbatim}

status
  1   2   3 
 57 134  14 

sex
  0   1 
126  79 

ulc
  0   1 
115  90
\end{verbatim}

and with new variables 

\lstset{numbers=left,numberstyle=\ttfamily\tiny\color{gray},language=r,label= ,caption= ,captionpos=b}
\begin{lstlisting}
dtable(melanoma,~status+sex+ulc+dcut(days)+I(days>300),level=1)
\end{lstlisting}

\begin{verbatim}

status
  1   2   3 
 57 134  14 

sex
  0   1 
126  79 

ulc
  0   1 
115  90 

dcut(days)
      [10,1.52e+03]    (1.52e+03,2e+03]    (2e+03,3.04e+03] (3.04e+03,5.56e+03] 
                 52                  51                  51                  51 

I(days > 300)
FALSE  TRUE 
   11   194
\end{verbatim}

\section*{Sorting the data}
\label{sec:org4adbc9e}

To sort the data 

\lstset{numbers=left,numberstyle=\ttfamily\tiny\color{gray},language=r,label= ,caption= ,captionpos=b}
\begin{lstlisting}
data(melanoma)
mel= dsort(melanoma,~days)
dsort(melanoma) <- ~days
head(mel)
\end{lstlisting}

\begin{verbatim}
   no status days ulc thick sex
1 789      3   10   1   676   1
2  13      3   30   0    65   1
3  97      2   35   0   134   1
4  16      3   99   0   290   0
5  21      1  185   1  1208   1
6 469      1  204   1   484   1
\end{verbatim}

and to sort after multiple variables increasing and decreasing 

\lstset{numbers=left,numberstyle=\ttfamily\tiny\color{gray},language=r,label= ,caption= ,captionpos=b}
\begin{lstlisting}
dsort(melanoma) <- ~days-status
head(melanoma)
\end{lstlisting}

\begin{verbatim}
   no status days ulc thick sex
1 789      3   10   1   676   1
2  13      3   30   0    65   1
3  97      2   35   0   134   1
4  16      3   99   0   290   0
5  21      1  185   1  1208   1
6 469      1  204   1   484   1
\end{verbatim}



\section*{Making new variales for the analysis}
\label{sec:orgb492e14}

To define a bunch of new covariates within a data-frame

\lstset{numbers=left,numberstyle=\ttfamily\tiny\color{gray},language=r,label= ,caption= ,captionpos=b}
\begin{lstlisting}
data(melanoma)
melanoma= transform(melanoma, thick2=thick^2, lthick=log(thick) ) 
dhead(melanoma)
\end{lstlisting}

\begin{verbatim}
   no status days ulc thick sex  thick2   lthick
1 789      3   10   1   676   1  456976 6.516193
2  13      3   30   0    65   1    4225 4.174387
3  97      2   35   0   134   1   17956 4.897840
4  16      3   99   0   290   0   84100 5.669881
5  21      1  185   1  1208   1 1459264 7.096721
6 469      1  204   1   484   1  234256 6.182085
\end{verbatim}


When the above definitions are done using a condition this can be achieved 
using the dtransform function that extends transform with a possible condition  

\lstset{numbers=left,numberstyle=\ttfamily\tiny\color{gray},language=r,label= ,caption= ,captionpos=b}
\begin{lstlisting}
melanoma=dtransform(melanoma,ll=thick*1.05^ulc,sex==1)  
melanoma=dtransform(melanoma,ll=thick,sex!=1)  
dmean(melanoma,ll~sex+ulc)
\end{lstlisting}

\begin{verbatim}
  sex ulc       ll
1   0   0 173.7342
2   1   0 197.3611
3   0   1 374.5532
4   1   1 523.1198
\end{verbatim}



\section*{Making factors (groupings)}
\label{sec:orgca893e7}

On the melanoma data the variable thick gives the thickness of the melanom tumour. For some analyses we would like to make a factor depending on the thickness. This can be done in several different ways

\lstset{numbers=left,numberstyle=\ttfamily\tiny\color{gray},language=r,label= ,caption= ,captionpos=b}
\begin{lstlisting}
melanoma=dcut(melanoma,~thick,breaks=c(0,200,500,800,2000))
\end{lstlisting}

New variable is named thickcat.0 by default.

To see levels of factors in data-frame
\lstset{numbers=left,numberstyle=\ttfamily\tiny\color{gray},language=r,label= ,caption= ,captionpos=b}
\begin{lstlisting}
dlevels(melanoma)
\end{lstlisting}

\begin{verbatim}
thickcat.0 #levels=:4 
[1] "[0,200]"     "(200,500]"   "(500,800]"   "(800,2e+03]"
-----------------------------------------
\end{verbatim}

Checking group sizes 
\lstset{numbers=left,numberstyle=\ttfamily\tiny\color{gray},language=r,label= ,caption= ,captionpos=b}
\begin{lstlisting}
dtable(melanoma,~thickcat.0)
\end{lstlisting}

\begin{verbatim}

thickcat.0
    [0,200]   (200,500]   (500,800] (800,2e+03] 
        109          64          20          12
\end{verbatim}

With adding to the data-frame directly 
\lstset{numbers=left,numberstyle=\ttfamily\tiny\color{gray},language=r,label= ,caption= ,captionpos=b}
\begin{lstlisting}
dcut(melanoma,breaks=c(0,200,500,800,2000)) <- gr.thick1~thick
dlevels(melanoma)
\end{lstlisting}

\begin{verbatim}
thickcat.0 #levels=:4 
[1] "[0,200]"     "(200,500]"   "(500,800]"   "(800,2e+03]"
-----------------------------------------
gr.thick1 #levels=:4 
[1] "[0,200]"     "(200,500]"   "(500,800]"   "(800,2e+03]"
-----------------------------------------
\end{verbatim}

new variable is named thickcat.0 (after first cut-point), 
or to get quartiles with default names thick.cat.4 

\lstset{numbers=left,numberstyle=\ttfamily\tiny\color{gray},language=r,label= ,caption= ,captionpos=b}
\begin{lstlisting}
dcut(melanoma) <- ~ thick  # new variable is thickcat.4
dlevels(melanoma)
\end{lstlisting}

\begin{verbatim}
thickcat.0 #levels=:4 
[1] "[0,200]"     "(200,500]"   "(500,800]"   "(800,2e+03]"
-----------------------------------------
gr.thick1 #levels=:4 
[1] "[0,200]"     "(200,500]"   "(500,800]"   "(800,2e+03]"
-----------------------------------------
thickcat.4 #levels=:4 
[1] "[10,97]"        "(97,194]"       "(194,356]"      "(356,1.74e+03]"
-----------------------------------------
\end{verbatim}

or median groups, here starting again with the original data, 

\lstset{numbers=left,numberstyle=\ttfamily\tiny\color{gray},language=r,label= ,caption= ,captionpos=b}
\begin{lstlisting}
data(melanoma)
dcut(melanoma,breaks=2) <- ~ thick  # new variable is thick.2
dlevels(melanoma)
\end{lstlisting}

\begin{verbatim}
thickcat.2 #levels=:2 
[1] "[10,194]"       "(194,1.74e+03]"
-----------------------------------------
\end{verbatim}



to control new names

\lstset{numbers=left,numberstyle=\ttfamily\tiny\color{gray},language=r,label= ,caption= ,captionpos=b}
\begin{lstlisting}
data(melanoma)
mela= dcut(melanoma,thickcat4+dayscat4~thick+days,breaks=4)
dlevels(mela)
\end{lstlisting}

\begin{verbatim}
thickcat4 #levels=:4 
[1] "[10,97]"        "(97,194]"       "(194,356]"      "(356,1.74e+03]"
-----------------------------------------
dayscat4 #levels=:4 
[1] "[10,1.52e+03]"       "(1.52e+03,2e+03]"    "(2e+03,3.04e+03]"   
[4] "(3.04e+03,5.56e+03]"
-----------------------------------------
\end{verbatim}

or

\lstset{numbers=left,numberstyle=\ttfamily\tiny\color{gray},language=r,label= ,caption= ,captionpos=b}
\begin{lstlisting}
data(melanoma)
dcut(melanoma,breaks=4) <- thickcat4+dayscat4~thick+days
dlevels(melanoma)
\end{lstlisting}

\begin{verbatim}
thickcat4 #levels=:4 
[1] "[10,97]"        "(97,194]"       "(194,356]"      "(356,1.74e+03]"
-----------------------------------------
dayscat4 #levels=:4 
[1] "[10,1.52e+03]"       "(1.52e+03,2e+03]"    "(2e+03,3.04e+03]"   
[4] "(3.04e+03,5.56e+03]"
-----------------------------------------
\end{verbatim}

This can also be typed out more specifically

\lstset{numbers=left,numberstyle=\ttfamily\tiny\color{gray},language=r,label= ,caption= ,captionpos=b}
\begin{lstlisting}
melanoma$gthick = cut(melanoma$thick,breaks=c(0,200,500,800,2000))
melanoma$gthick = cut(melanoma$thick,breaks=quantile(melanoma$thick),include.lowest=TRUE)
\end{lstlisting}


\section*{Working with factors}
\label{sec:orgb002bf7}

To see levels of covariates in data-frame

\lstset{numbers=left,numberstyle=\ttfamily\tiny\color{gray},language=r,label= ,caption= ,captionpos=b}
\begin{lstlisting}
data(melanoma)
dcut(melanoma,breaks=4) <- thickcat4~thick
dlevels(melanoma) 
\end{lstlisting}

\begin{verbatim}
thickcat4 #levels=:4 
[1] "[10,97]"        "(97,194]"       "(194,356]"      "(356,1.74e+03]"
-----------------------------------------
\end{verbatim}

To relevel the factor

\lstset{numbers=left,numberstyle=\ttfamily\tiny\color{gray},language=r,label= ,caption= ,captionpos=b}
\begin{lstlisting}
dtable(melanoma,~thickcat4)
melanoma = drelevel(melanoma,~thickcat4,ref="(194,356]")
dlevels(melanoma)
\end{lstlisting}

\begin{verbatim}

thickcat4
       [10,97]       (97,194]      (194,356] (356,1.74e+03] 
            56             53             45             51
thickcat4 #levels=:4 
[1] "[10,97]"        "(97,194]"       "(194,356]"      "(356,1.74e+03]"
-----------------------------------------
thickcat4.(194,356] #levels=:4 
[1] "(194,356]"      "[10,97]"        "(97,194]"       "(356,1.74e+03]"
-----------------------------------------
\end{verbatim}

or to take the third level in the list of levels, same as above, 

\lstset{numbers=left,numberstyle=\ttfamily\tiny\color{gray},language=r,label= ,caption= ,captionpos=b}
\begin{lstlisting}
melanoma = drelevel(melanoma,~thickcat4,ref=2)
dlevels(melanoma)
\end{lstlisting}

\begin{verbatim}
thickcat4 #levels=:4 
[1] "[10,97]"        "(97,194]"       "(194,356]"      "(356,1.74e+03]"
-----------------------------------------
thickcat4.(194,356] #levels=:4 
[1] "(194,356]"      "[10,97]"        "(97,194]"       "(356,1.74e+03]"
-----------------------------------------
thickcat4.2 #levels=:4 
[1] "(97,194]"       "[10,97]"        "(194,356]"      "(356,1.74e+03]"
-----------------------------------------
\end{verbatim}


To combine levels of a factor (first combinining first 3 groups into one)

\lstset{numbers=left,numberstyle=\ttfamily\tiny\color{gray},language=r,label= ,caption= ,captionpos=b}
\begin{lstlisting}
melanoma = drelevel(melanoma,~thickcat4,newlevels=1:3)
dlevels(melanoma)
\end{lstlisting}

\begin{verbatim}
thickcat4 #levels=:4 
[1] "[10,97]"        "(97,194]"       "(194,356]"      "(356,1.74e+03]"
-----------------------------------------
thickcat4.(194,356] #levels=:4 
[1] "(194,356]"      "[10,97]"        "(97,194]"       "(356,1.74e+03]"
-----------------------------------------
thickcat4.2 #levels=:4 
[1] "(97,194]"       "[10,97]"        "(194,356]"      "(356,1.74e+03]"
-----------------------------------------
thickcat4.1:3 #levels=:2 
[1] "[10,97]-(194,356]" "(356,1.74e+03]"   
-----------------------------------------
\end{verbatim}

or to combine groups 1 and 2 into one group and 3 and 4 into another

\lstset{numbers=left,numberstyle=\ttfamily\tiny\color{gray},language=r,label= ,caption= ,captionpos=b}
\begin{lstlisting}
dkeep(melanoma) <- ~thick+thickcat4
melanoma = drelevel(melanoma,gthick2~thickcat4,newlevels=list(1:2,3:4))
dlevels(melanoma)
\end{lstlisting}

\begin{verbatim}
thickcat4 #levels=:4 
[1] "[10,97]"        "(97,194]"       "(194,356]"      "(356,1.74e+03]"
-----------------------------------------
gthick2 #levels=:2 
[1] "[10,97]-(97,194]"         "(194,356]-(356,1.74e+03]"
-----------------------------------------
\end{verbatim}


Changing order of factor levels 

\lstset{numbers=left,numberstyle=\ttfamily\tiny\color{gray},language=r,label= ,caption= ,captionpos=b}
\begin{lstlisting}
dfactor(melanoma,levels=c(3,1,2,4)) <-  thickcat4.2~thickcat4
dlevel(melanoma,~ "thickcat4*")
dtable(melanoma,~thickcat4+thickcat4.2)
\end{lstlisting}

\begin{verbatim}
thickcat4 #levels=:4 
[1] "[10,97]"        "(97,194]"       "(194,356]"      "(356,1.74e+03]"
-----------------------------------------
thickcat4.2 #levels=:4 
[1] "(194,356]"      "[10,97]"        "(97,194]"       "(356,1.74e+03]"
-----------------------------------------

               thickcat4.2 (194,356] [10,97] (97,194] (356,1.74e+03]
thickcat4                                                           
[10,97]                            0      56        0              0
(97,194]                           0       0       53              0
(194,356]                         45       0        0              0
(356,1.74e+03]                     0       0        0             51
\end{verbatim}

Combine levels but now control factor-level names 
\lstset{numbers=left,numberstyle=\ttfamily\tiny\color{gray},language=r,label= ,caption= ,captionpos=b}
\begin{lstlisting}
melanoma=drelevel(melanoma,gthick3~thickcat4,newlevels=list(group1.2=1:2,group3.4=3:4))
dlevels(melanoma)
\end{lstlisting}

\begin{verbatim}
thickcat4 #levels=:4 
[1] "[10,97]"        "(97,194]"       "(194,356]"      "(356,1.74e+03]"
-----------------------------------------
gthick2 #levels=:2 
[1] "[10,97]-(97,194]"         "(194,356]-(356,1.74e+03]"
-----------------------------------------
thickcat4.2 #levels=:4 
[1] "(194,356]"      "[10,97]"        "(97,194]"       "(356,1.74e+03]"
-----------------------------------------
gthick3 #levels=:2 
[1] "group1.2" "group3.4"
-----------------------------------------
\end{verbatim}


\section*{Making a factor from existing numeric variable and vice versa}
\label{sec:orgdcd4633}

A numeric variable "status" with values 1,2,3 into a factor by

\lstset{numbers=left,numberstyle=\ttfamily\tiny\color{gray},language=r,label= ,caption= ,captionpos=b}
\begin{lstlisting}
data(melanoma)
melanoma = dfactor(melanoma,~status, labels=c("malignant-melanoma","censoring","dead-other"))
melanoma = dfactor(melanoma,sexl~sex,labels=c("females","males"))
dtable(melanoma,~sexl+status.f)
\end{lstlisting}

\begin{verbatim}

        status.f malignant-melanoma censoring dead-other
sexl                                                    
females                          28        91          7
males                            29        43          7
\end{verbatim}

A gender factor with values "M", "F" can be converted into numerics by

\lstset{numbers=left,numberstyle=\ttfamily\tiny\color{gray},language=r,label= ,caption= ,captionpos=b}
\begin{lstlisting}
melanoma = dnumeric(melanoma,~sexl)
dstr(melanoma,"sex*")
dtable(melanoma,~'sex*',level=2)
\end{lstlisting}

\begin{verbatim}
'data.frame':	205 obs. of  3 variables:
 $ sex   : int  1 1 1 0 1 1 1 1 0 0 ...
 $ sexl  : Factor w/ 2 levels "females","males": 2 2 2 1 2 2 2 2 1 1 ...
 $ sexl.n: num  2 2 2 1 2 2 2 2 1 1 ...

         sex
sexl        0   1
  females 126   0
  males     0  79

      sex
sexl.n   0   1
     1 126   0
     2   0  79

      sexl
sexl.n females males
     1     126     0
     2       0    79
\end{verbatim}
\end{document}