%% LyX 1.6.7 created this file.  For more info, see http://www.lyx.org/.
%% Do not edit unless you really know what you are doing.
\documentclass[10pt,english,nogin]{book}
\usepackage{lmodern}
\renewcommand{\sfdefault}{lmss}
\renewcommand{\ttdefault}{lmtt}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{listings}
\lstset{basicstyle={\ttfamily},
breaklines=true,
language=R}
\usepackage[paperwidth=7.44in,paperheight=9.69in]{geometry}
\geometry{verbose,tmargin=1in,bmargin=1in,lmargin=1in,rmargin=1in}
\pagestyle{headings}
\setcounter{secnumdepth}{2}
\setcounter{tocdepth}{1}
\usepackage{color}
\usepackage{babel}

\usepackage{rotating}
\usepackage{varioref}
\usepackage{float}
\usepackage{url}
\usepackage{amsthm}
\usepackage{amsmath}
\makeindex
\usepackage{amssymb}
\usepackage[unicode=true, 
 bookmarks=true,bookmarksnumbered=true,bookmarksopen=true,bookmarksopenlevel=0,
 breaklinks=true,pdfborder={0 0 0},backref=page,colorlinks=true]
 {hyperref}
\hypersetup{pdftitle={Introduction to Probability and Statistics Using R},
 pdfauthor={G. Jay Kerns},
 linkcolor=blue,  citecolor=black, urlcolor=blue}

\makeatletter

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% LyX specific LaTeX commands.
\providecommand{\LyX}{L\kern-.1667em\lower.25em\hbox{Y}\kern-.125emX\@}
\newcommand{\noun}[1]{\textsc{#1}}
%% Because html converters don't know tabularnewline
\providecommand{\tabularnewline}{\\}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% Textclass specific LaTeX commands.
\usepackage{Sweave}
\newcommand{\Rcode}[1]{{\texttt{#1}}}
\newcommand{\Robject}[1]{{\texttt{#1}}}
\newcommand{\Rcommand}[1]{{\texttt{#1}}}
\newcommand{\Rfunction}[1]{{\texttt{#1}}}
\newcommand{\Rfunarg}[1]{{\textit{#1}}}
\newcommand{\Rpackage}[1]{{\textit{#1}}}
\newcommand{\Rmethod}[1]{{\textit{#1}}}
\newcommand{\Rclass}[1]{{\textit{#1}}}
\numberwithin{equation}{section}
\numberwithin{figure}{section}
\theoremstyle{plain}
\ifx\thechapter\undefined
\newtheorem{thm}{Theorem}
\else
\newtheorem{thm}{Theorem}[chapter]
\fi
 \theoremstyle{definition}
  \newtheorem{example}[thm]{Example}
  \theoremstyle{plain}
  \newtheorem{fact}[thm]{Fact}
\newenvironment{lyxcode}
{\par\begin{list}{}{
\setlength{\rightmargin}{\leftmargin}
\setlength{\listparindent}{0pt}% needed for AMS classes
\raggedright
\setlength{\itemsep}{0pt}
\setlength{\parsep}{0pt}
\normalfont\ttfamily}%
 \item[]}
{\end{list}}
  \theoremstyle{definition}
  \newtheorem{xca}[thm]{Exercise}
  \theoremstyle{remark}
  \newtheorem{note}[thm]{Note}
  \theoremstyle{plain}
  \newtheorem{ax}[thm]{Axiom}
  \theoremstyle{plain}
  \newtheorem{prop}[thm]{Proposition}
  \theoremstyle{definition}
  \newtheorem{defn}[thm]{Definition}
  \theoremstyle{remark}
  \newtheorem{rem}[thm]{Remark}
  \theoremstyle{plain}
  \newtheorem{cor}[thm]{Corollary}
  \theoremstyle{plain}
  \newtheorem{assumption}[thm]{Assumption}
  \theoremstyle{remark}
  \newtheorem*{note*}{Note}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% User specified LaTeX commands.
% Meta information - fill between {} and do not remove %
% \VignetteIndexEntry{An Introduction to Probability and Statistics Using R}
% \VignetteDepends{}
% \VignetteKeywords{}
% \VignettePackage{IPSUR}
% Package

% user-loaded packages
\usepackage{amssymb}
\usepackage{amsmath}
\usepackage{latexsym}
%\usepackage{theorem}
\usepackage{subfig}
\usepackage{graphics}
\usepackage{epsfig}
\usepackage{makeidx}
%\usepackage{showidx}
\usepackage{multicol}
%\usepackage{floatflt}
\usepackage{url}
\usepackage{srcltx}
\usepackage{txfonts}
\usepackage{stmaryrd}
\usepackage{textcomp}
%\usepackage{pslatex}
\usepackage{upgreek}
\usepackage[subfigure]{tocloft}
\setlength{\cftfignumwidth}{1.5cm}


%  user defined commands
% special operators
\renewcommand{\P}{%
  \mathop{\operator@font I\hspace{-1.5pt}P\hspace{.13pt}}}
\newcommand{\E}{%
  \mathop{\operator@font I\hspace{-1.5pt}E\hspace{.13pt}}}
\newcommand{\VAR}{\operatorname{var}}
\newcommand{\COV}{\operatorname{cov}}
\newcommand{\COR}{\operatorname{cor}}
\renewcommand{\vec}[1]{\mbox{\boldmath$#1$}}

% special symbols
\newcommand{\me}{\mathrm{e}}
\newcommand{\R}{\mathbb{R}}
\newcommand{\diff}{\mathrm{d}}
\newcommand{\ybar}{\overline{y}}
\newcommand{\xbar}{\overline{x}}
\newcommand{\Xbar}{\overline{X}}
\newcommand{\Ybar}{\overline{Y}}

% special logos
\providecommand{\IPSUR}
{\textsc{I\kern 0ex\lower-.3ex\hbox{\small P}\kern -.5ex\lower.4ex\hbox{\footnotesize S}\kern -.25exU}\kern -.1ex\lower .15ex\hbox{\textsf{\large R}}\@}

\providecommand{\IPSURtitle}
{\fontsize{30}{35}\selectfont \textsc{I\kern -.16ex\lower-.5ex\hbox{\Large P}\kern -.5ex\lower.4ex\hbox{\Large S}\kern -.25exU}\kern -.1ex\lower .15ex\hbox{\textsf{\Huge R}}\@}


%%  Sweave specific commands
% make the input blue, output red
\DefineVerbatimEnvironment{Soutput}{Verbatim}{formatcom=\color{blue}}
\DefineVerbatimEnvironment{Sinput}{Verbatim}{fontshape=sl, formatcom=\color{red}}
% make the input/output black
%\DefineVerbatimEnvironment{Soutput}{Verbatim}{formatcom=\color{black}}
%\DefineVerbatimEnvironment{Sinput}{Verbatim}{fontshape=sl, formatcom=\color{black}}


% get rid of extra Sweave space
\fvset{listparameters={\setlength{\topsep}{0pt}}}
\renewenvironment{Schunk}{\vspace{\topsep}}{\vspace{\topsep}}  

% settings for program listings
\lstset{
language=R,
breaklines=true,
breakatwhitespace=true,
keywordstyle={\ttfamily},
numberstyle = {\ttfamily},
morestring=[b]"
}

% Turn on questions and answers
\newcommand{\question}[1]{#1}
\newcommand{\answer}[1]{#1}
% Turn off questions and answers
%\newcommand{\question}[1]{}
%\newcommand{\answer}[1]{}

\@ifundefined{showcaptionsetup}{}{%
 \PassOptionsToPackage{caption=false}{subfig}}
\usepackage{subfig}
\AtBeginDocument{
  \def\labelitemii{\(\circ\)}
}

\makeatother

\begin{document}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%                                                                  %%%
%%%  IPSUR.Rnw - Introduction to Probability and Statistics Using R  %%%
%%%  Copyright (C) 2011  G. Jay Kerns, <gkerns@ysu.edu>              %%%
%%%  This program is free software: you can redistribute it and/or   %%%
%%%  modify it under the terms of the GNU General Public License as  %%%
%%%  published by the Free Software Foundation, either version 3     %%%
%%%  of the License, or (at your option) any later version.  This    %%%
%%%  program is distributed in the hope that it will be useful,      %%%
%%%  but WITHOUT ANY WARRANTY; without even the implied warranty     %%%
%%%  of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.         %%%
%%%  See the GNU General Public License for more details.  You       %%%
%%%  should have received a copy of the GNU General Public License   %%%
%%%  along with this program.  If not, see                           %%%
%%%  <http://www.gnu.org/licenses/>                                  %%%
%%%                                                                  %%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

<<echo = FALSE>>=
###  IPSUR.R - Introduction to Probability and Statistics Using R
###  Copyright (C) 2011  G. Jay Kerns, <gkerns@ysu.edu>
###  This program is free software: you can redistribute it and/or modify
###  it under the terms of the GNU General Public License as published by
###  the Free Software Foundation, either version 3 of the License, or
###  (at your option) any later version.
###  This program is distributed in the hope that it will be useful,
###  but WITHOUT ANY WARRANTY; without even the implied warranty of
###  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
###  GNU General Public License for more details.
###  You should have received a copy of the GNU General Public License
###  along with this program.  If not, see <http://www.gnu.org/licenses/>
###################################################
@

<<echo = FALSE>>=
seed <- 42
set.seed(seed)
options(width = 75)
#library(random)
#i_seed <- randomNumbers(n = 624, col = 1, min = -1e+09, max = 1e+09)
#.Random.seed[2:626] <- as.integer(c(1, i_seed))
#save.image(file = "seed.RData")
@

<<echo = FALSE>>=
options(useFancyQuotes = FALSE)
#library(prob)
library(RcmdrPlugin.IPSUR)
# Generate RcmdrTestDrive
n <- 168
# generate order 
order <- 1:n
# generate race 
race <- sample(c("White","AfAmer","Asian","Other"), size=n, prob=c(76,13,5,6), replace = TRUE)
race <- factor(race)
# generate gender and smoke 
tmp <- sample(4, size=n, prob=c(12,38,9,41), replace = TRUE) 
gender <- factor(ifelse(tmp < 3,"Male", "Female")) 
smoke <- factor(ifelse(tmp %in% c(1,3), "Yes", "No"))
# generate parking
parking <- rgeom(n, prob = 0.4) + 1
# generate salary 
m <- 17 + (as.numeric(gender)-1) 
s <- 1 + (2 - as.numeric(gender)) 
salary <- rnorm(n, mean = m, sd = s)
# simulate reduction 
x <- arima.sim(list(order=c(1,0,0), ar=.9), n=n) 
reduction <- as.numeric((20*x + order)/n + 5)
# simulate before and after
before <- rlogis(n, location = 68, scale = 3) 
m <- (as.numeric(smoke)-1)*2.5 
after <- before - rnorm(n, mean = m, sd=0.1)
RcmdrTestDrive <- data.frame(order = order, race = race, smoke = smoke, gender = gender, salary = salary, reduction = reduction, before = before, after = after, parking = parking)
# clean up
remove(list = names(RcmdrTestDrive))
remove(x, n, m, s, tmp)
@

<<echo = FALSE>>=
plot.htest <- function (x, hypoth.or.conf = 'Hypoth',...) { 
require("HH") 
if (x$method == "1-sample proportions test with continuity correction" || x$method == "1-sample proportions test without continuity correction"){
mu <- x$null.value
obs.mean <- x$estimate
n <- NA
std.dev <- abs(obs.mean - mu)/sqrt(x$statistic)
deg.freedom <- NA
if(x$alternative == "two.sided"){
alpha.right <- (1 - attr(x$conf.int, "conf.level"))/2
Use.alpha.left <- TRUE
Use.alpha.right <- TRUE
} else if (x$alternative == "less") {
alpha.right <- 1 - attr(x$conf.int, "conf.level")
Use.alpha.left <- TRUE
Use.alpha.right <- FALSE
} else {
alpha.right <- 1 - attr(x$conf.int, "conf.level")
Use.alpha.left <- FALSE
Use.alpha.right <- TRUE
}
} else if (x$method == "One Sample z-test"){
mu <- x$null.value
obs.mean <- x$estimate
n <- x$parameter[1]
std.dev <- x$parameter[2]
deg.freedom <- NA
if(x$alternative == "two.sided"){
alpha.right <- (1 - attr(x$conf.int, "conf.level"))/2
Use.alpha.left <- TRUE
Use.alpha.right <- TRUE
} else if (x$alternative == "less") {
alpha.right <- 1 - attr(x$conf.int, "conf.level")
Use.alpha.left <- TRUE
Use.alpha.right <- FALSE
} else {
alpha.right <- 1 - attr(x$conf.int, "conf.level")
Use.alpha.left <- FALSE
Use.alpha.right <- TRUE
} 
} else if (x$method == "One Sample t-test" || x$method == "Paired t-test"){
mu <- x$null.value
obs.mean <- x$estimate
n <- x$parameter + 1
std.dev <- x$estimate/x$statistic*sqrt(n)
deg.freedom <- x$parameter
if(x$alternative == "two.sided"){
alpha.right <- (1 - attr(x$conf.int, "conf.level"))/2
Use.alpha.left <- TRUE
Use.alpha.right <- TRUE
} else if (x$alternative == "less") {
alpha.right <- 1 - attr(x$conf.int, "conf.level")
Use.alpha.left <- TRUE
Use.alpha.right <- FALSE
} else {
alpha.right <- 1 - attr(x$conf.int, "conf.level")
Use.alpha.left <- FALSE
Use.alpha.right <- TRUE
}
} else if (x$method == "Welch Two Sample t-test"){
mu <- x$null.value
obs.mean <- -diff(x$estimate)
n <- x$parameter + 2
std.dev <- obs.mean/x$statistic*sqrt(n)
deg.freedom <- x$parameter
if(x$alternative == "two.sided"){
alpha.right <- (1 - attr(x$conf.int, "conf.level"))/2
Use.alpha.left <- TRUE
Use.alpha.right <- TRUE
} else if (x$alternative == "less") {
alpha.right <- 1 - attr(x$conf.int, "conf.level")
Use.alpha.left <- TRUE
Use.alpha.right <- FALSE
} else {
alpha.right <- 1 - attr(x$conf.int, "conf.level")
Use.alpha.left <- FALSE
Use.alpha.right <- TRUE
} 
} else if (x$method == " Two Sample t-test"){
mu <- x$null.value
obs.mean <- -diff(x$estimate)
n <- x$parameter + 2
std.dev <- obs.mean/x$statistic*sqrt(n)
deg.freedom <- x$parameter
if(x$alternative == "two.sided"){
alpha.right <- (1 - attr(x$conf.int, "conf.level"))/2
Use.alpha.left <- TRUE
Use.alpha.right <- TRUE
} else if (x$alternative == "less") {
alpha.right <- 1 - attr(x$conf.int, "conf.level")
Use.alpha.left <- TRUE
Use.alpha.right <- FALSE
} else {
alpha.right <- 1 - attr(x$conf.int, "conf.level")
Use.alpha.left <- FALSE
Use.alpha.right <- TRUE
}
}
return(normal.and.t.dist(mu.H0 = mu, obs.mean = obs.mean, std.dev = std.dev, n = n, deg.freedom = deg.freedom, alpha.right = alpha.right, Use.obs.mean = TRUE, Use.alpha.left = Use.alpha.left, Use.alpha.right = Use.alpha.right, hypoth.or.conf = hypoth.or.conf))
}
@


\title{\fontsize{30}{35}\selectfont Introduction to Probability\\ and
Statistics Using \textsf{R}}


\date{\fontsize{24}{28}\selectfont \noun{First Edition}}


\author{\fontsize{24}{28}\selectfont G.~Jay Kerns}

\maketitle
\pagenumbering{roman}
\setcounter{page}{2}

\noindent \IPSUR: Introduction to Probability and Statistics Using
\textsf{R}

\noindent Copyright \textcopyright~2010 G.~Jay Kerns

\noindent ISBN: 978-0-557-24979-4

\medskip{}


\noindent Permission is granted to copy, distribute and/or modify
this document under the terms of the GNU Free Documentation License,
Version 1.3 or any later version published by the Free Software Foundation;
with no Invariant Sections, no Front-Cover Texts, and no Back-Cover
Texts. A copy of the license is included in the section entitled ``GNU
Free Documentation License''.

\noindent \bigskip{}


\noindent Date: \today

\noindent \vfill{}


\cleardoublepage
\phantomsection
\pdfbookmark[1]{Contents}{table}

\tableofcontents{}

\cleardoublepage
\phantomsection
\addcontentsline{toc}{chapter}{Preface}


\chapter*{Preface}

This book was expanded from lecture materials I use in a one semester
upper-division undergraduate course entitled \emph{Probability and
Statistics} at Youngstown State University. Those lecture materials,
in turn, were based on notes that I transcribed as a graduate student
at Bowling Green State University. The course for which the materials
were written is 50-50 Probability and Statistics, and the attendees
include mathematics, engineering, and computer science majors (among
others). The catalog prerequisites for the course are a full year
of calculus.

The book can be subdivided into three basic parts. The first part
includes the introductions and elementary \emph{descriptive statistics};
I want the students to be knee-deep in data right out of the gate.
The second part is the study of \emph{probability}, which begins at
the basics of sets and the equally likely model, journeys past discrete/continuous
random variables, and continues through to multivariate distributions.
The chapter on sampling distributions paves the way to the third part,
which is \emph{inferential statistics}. This last part includes point
and interval estimation, hypothesis testing, and finishes with introductions
to selected topics in applied statistics.

I usually only have time in one semester to cover a small subset of
this book. I cover the material in Chapter 2 in a class period that
is supplemented by a take-home assignment for the students. I spend
a lot of time on Data Description, Probability, Discrete, and Continuous
Distributions. I mention selected facts from Multivariate Distributions
in passing, and discuss the meaty parts of Sampling Distributions
before moving right along to Estimation (which is another chapter
I dwell on considerably). Hypothesis Testing goes faster after all
of the previous work, and by that time the end of the semester is
in sight. I normally choose one or two final chapters (sometimes three)
from the remaining to survey, and regret at the end that I did not
have the chance to cover more.

In an attempt to be correct I have included material in this book
which I would normally not mention during the course of a standard
lecture. For instance, I normally do not highlight the intricacies
of measure theory or integrability conditions when speaking to the
class. Moreover, I often stray from the matrix approach to multiple
linear regression because many of my students have not yet been formally
trained in linear algebra. That being said, it is important to me
for the students to hold something in their hands which acknowledges
the world of mathematics and statistics beyond the classroom, and
which may be useful to them for many semesters to come. It also mirrors
my own experience as a student.

The vision for this document is a more or less self contained, essentially
complete, correct, introductory textbook. There should be plenty of
exercises for the student, with full solutions for some, and no solutions
for others (so that the instructor may assign them for grading). By
\inputencoding{latin9}\lstinline[showstringspaces=false]!Sweave!\inputencoding{utf8}'s
dynamic nature it is possible to write randomly generated exercises
and I had planned to implement this idea already throughout the book.
Alas, there are only 24 hours in a day. Look for more in future editions.

Seasoned readers will be able to detect my origins: \emph{Probability
and Statistical Inference} by Hogg and Tanis \cite{Hogg2006}, \emph{Statistical
Inference} by Casella and Berger \cite{Casella2002}, and \emph{Theory
of Point Estimation/Testing Statistical Hypotheses} by Lehmann \cite{Lehmann1998,Lehmann1986}.
I highly recommend each of those books to every reader of this one.
Some \textsf{R} books with {}``introductory'' in the title that
I recommend are \emph{Introductory Statistics with }\textsf{\emph{R}}
by Dalgaard \cite{Dalgaard2008} and \emph{Using }\textsf{\emph{R}}\emph{
for Introductory Statistics} by Verzani \cite{Verzani2005}. Surely
there are many, many other good introductory books about \textsf{R},
but frankly, I have tried to steer clear of them for the past year
or so to avoid any undue influence on my own writing.

I would like to make special mention of two other books: \emph{Introduction
to Statistical Thought} by Michael Lavine \cite{Lavine2009} and \emph{Introduction
to Probability} by Grinstead and Snell \cite{Grinstead1997}. Both
of these books are \emph{free} and are what ultimately convinced me
to release \IPSUR\ under a free license, too.

Please bear in mind that the title of this book is {}``Introduction
to Probability and Statistics Using \textsf{R}'', and not {}``Introduction
to \textsf{R} Using Probability and Statistics'', nor even {}``Introduction
to Probability and Statistics and \textsf{R} Using Words''. The people
at the party are Probability and Statistics; the handshake is \textsf{R}.
There are several important topics about \textsf{R} which some individuals
will feel are underdeveloped, glossed over, or wantonly omitted. Some
will feel the same way about the probabilistic and/or statistical
content. Still others will just want to learn \textsf{R} and skip
all of the mathematics.

Despite any misgivings: here it is, warts and all. I humbly invite
said individuals to take this book, with the GNU Free Documentation
License (GNU-FDL) in hand, and make it better. In that spirit there
are at least a few ways in my view in which this book could be improved.
\begin{description}
\item [{Better~data.}] The data analyzed in this book are almost entirely
from the \inputencoding{latin9}\lstinline[showstringspaces=false]!datasets!\inputencoding{utf8}
package in base \textsf{R}, and here is why:

\begin{enumerate}
\item I made a conscious effort to minimize dependence on contributed packages,
\item The data are instantly available, already in the correct format, so
we need not take time to manage them, and
\item The data are \emph{real}.
\end{enumerate}
I made no attempt to choose data sets that would be interesting to
the students; rather, data were chosen for their potential to convey
a statistical point. Many of the data sets are decades old or more
(for instance, the data used to introduce simple linear regression
are the speeds and stopping distances of cars in the 1920's).

In a perfect world with infinite time I would research and contribute
recent, \emph{real} data in a context crafted to engage the students
in \emph{every} example. One day I hope to stumble over said time.
In the meantime, I will add new data sets incrementally as time permits.

\item [{More~proofs.}] I would like to include more proofs for the sake
of completeness (I understand that some people would not consider
more proofs to be improvement). Many proofs have been skipped entirely,
and I am not aware of any rhyme or reason to the current omissions.
I will add more when I get a chance. 
\item [{More~and~better~graphics:}] I have not used the \inputencoding{latin9}\lstinline[basicstyle={\ttfamily},breaklines=true,language=R]!ggplot2!\inputencoding{utf8}
package \cite{Wickam2009} because I do not know how to use it yet.
It is on my to-do list.
\item [{More~and~better~exercises:}] There are only a few exercises
in the first edition simply because I have not had time to write more.
I have toyed with the \inputencoding{latin9}\lstinline[basicstyle={\ttfamily},breaklines=true,language=R]!exams!\inputencoding{utf8}
package \cite{exams} and I believe that it is a right way to move
forward. As I learn more about what the package can do I would like
to incorporate it into later editions of this book.
\end{description}

\section*{About This Document}

\IPSUR\ contains many interrelated parts: the \textbf{\emph{Document}},
the \textbf{\emph{Program}}, the \textbf{\emph{Package}}, and the
\textbf{\emph{Ancillaries}}. In short, the \emph{Document} is what
you are reading right now. The \emph{Program} provides an efficient
means to modify the Document. The \emph{Package} is an \textsf{R}
package that houses the Program and the Document. Finally, the \emph{Ancillaries}
are extra materials that reside in the Package and were produced by
the Program to supplement use of the Document. We briefly describe
each of them in turn.


\subsection*{The Document}

The \emph{Document} is that which you are reading right now -- \IPSUR's
\emph{raison d'\^etre}. There are transparent copies (nonproprietary
text files) and opaque copies (everything else). See the GNU-FDL in
Appendix \ref{cha:GNU-Free-Documentation} for more precise language
and details.
\begin{description}
\item [{\texttt{IPSUR.tex}}] is a transparent copy of the Document to be
typeset with a \LaTeX{} distribution such as Mik\TeX{} or \TeX{} Live.
Any reader is free to modify the Document and release the modified
version in accordance with the provisions of the GNU-FDL. Note that
this file cannot be used to generate a randomized copy of the Document.
Indeed, in its released form it is only capable of typesetting the
exact version of \IPSUR\ which you are currently reading. Furthermore,
the \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!.tex!\inputencoding{utf8}
file is unable to generate any of the ancillary materials. 
\item [{\texttt{IPSUR-xxx.eps},~\texttt{IPSUR-xxx.pdf}}] are the image
files for every graph in the Document. These are needed when typesetting
with \LaTeX{}.
\item [{\texttt{IPSUR.pdf}}] is an opaque copy of the Document. This is
the file that instructors would likely want to distribute to students.
\item [{\texttt{IPSUR.dvi}}] is another opaque copy of the Document in
a different file format.
\end{description}

\subsection*{The Program}

The \emph{Program} includes \inputencoding{latin9}\lstinline[showstringspaces=false]!IPSUR.lyx!\inputencoding{utf8}
and its nephew \inputencoding{latin9}\lstinline[showstringspaces=false]!IPSUR.Rnw!\inputencoding{utf8};
the purpose of each is to give individuals a way to quickly customize
the Document for their particular purpose(s).
\begin{description}
\item [{\texttt{IPSUR.lyx}}] is the source \LyX{} file for the Program,
released under the GNU General Public License (GNU GPL) Version 3.
This file is opened, modified, and compiled with \LyX{}, a sophisticated
open-source document processor, and may be used (together with \inputencoding{latin9}\lstinline[showstringspaces=false]!Sweave!\inputencoding{utf8})
to generate a randomized, modified copy of the Document with brand
new data sets for some of the exercises and the solution manuals (in
the Second Edition). Additionally, \LyX{} can easily activate/deactivate
entire blocks of the document, \emph{e.g.~}the \textsf{proofs} of
the theorems, the student \textsf{solutions} to the exercises, or
the instructor \textsf{answers} to the problems, so that the new author
may choose which sections (s)he would like to include in the final
Document (again, Second Edition). The \inputencoding{latin9}\lstinline[showstringspaces=false]!IPSUR.lyx!\inputencoding{utf8}
file is all that a person needs (in addition to a properly configured
system -- see Appendix \ref{cha:Instructions-for-Instructors}) to
generate/compile/export to all of the other formats described above
and below, which includes the ancillary materials \inputencoding{latin9}\lstinline[showstringspaces=false]!IPSUR.Rdata!\inputencoding{utf8}
and \inputencoding{latin9}\lstinline[showstringspaces=false]!IPSUR.R!\inputencoding{utf8}.
\item [{\texttt{IPSUR.Rnw}}] is another form of the source code for the
Program, also released under the GNU GPL Version 3. It was produced
by exporting \inputencoding{latin9}\lstinline[showstringspaces=false]!IPSUR.lyx!\inputencoding{utf8}
into\textsf{ R}/Sweave format (\inputencoding{latin9}\lstinline[showstringspaces=false]!.Rnw!\inputencoding{utf8}).
This file may be processed with Sweave to generate a randomized copy
of \inputencoding{latin9}\lstinline[showstringspaces=false]!IPSUR.tex!\inputencoding{utf8}
-- a transparent copy of the Document -- together with the ancillary
materials \inputencoding{latin9}\lstinline[showstringspaces=false]!IPSUR.Rdata!\inputencoding{utf8}
and \inputencoding{latin9}\lstinline[showstringspaces=false]!IPSUR.R!\inputencoding{utf8}.
Please note, however, that \inputencoding{latin9}\lstinline[showstringspaces=false]!IPSUR.Rnw!\inputencoding{utf8}
is just a simple text file which does not support many of the extra
features that \LyX{} offers such as WYSIWYM editing, instantly (de)activating
branches of the manuscript, and more. 
\end{description}

\subsection*{The Package}

There is a contributed package on \inputencoding{latin9}\lstinline[showstringspaces=false]!CRAN!\inputencoding{utf8},
called \inputencoding{latin9}\lstinline[showstringspaces=false]!IPSUR!\inputencoding{utf8}.
The package affords many advantages, one being that it houses the
Document in an easy-to-access medium. Indeed, a student can have the
Document at his/her fingertips with only three commands:

<<eval = FALSE>>=
install.packages("IPSUR")
library(IPSUR)
read(IPSUR)
@

Another advantage goes hand in hand with the Program's license; since
\IPSUR\ is free, the source code must be freely available to anyone
that wants it. A package hosted on \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!CRAN!\inputencoding{utf8}
allows the author to obey the license by default.

A much more important advantage is that the excellent facilities at
\textsf{R}-Forge are building and checking the package daily against
patched and development versions of the absolute latest pre-release
of \textsf{R}. If any problems surface then I will know about it within
24 hours.

And finally, suppose there is some sort of problem. The package structure
makes it \emph{incredibly} easy for me to distribute bug-fixes and
corrected typographical errors. As an author I can make my corrections,
upload them to the repository at \textsf{R}-Forge, and they will be
reflected \emph{worldwide} within hours. We aren't in Kansas anymore,
Toto.


\subsection*{Ancillary Materials}

These are extra materials that accompany \IPSUR. They reside in the
\inputencoding{latin9}\lstinline[showstringspaces=false]!/etc!\inputencoding{utf8}
subdirectory of the package source.
\begin{description}
\item [{\texttt{IPSUR.R}}] is the exported \textsf{R} code from \inputencoding{latin9}\lstinline[showstringspaces=false]!IPSUR.Rnw!\inputencoding{utf8}.
With this script, literally every \textsf{R} command from the entirety
of \IPSUR\ can be resubmitted at the command line.
\end{description}

\section*{Notation}

We use the notation \inputencoding{latin9}\lstinline[showstringspaces=false]!x!\inputencoding{utf8}
or \inputencoding{latin9}\lstinline[showstringspaces=false]!stem.leaf!\inputencoding{utf8}
notation to denote objects, functions, \emph{etc}.. The sequence {}``\textsf{Statistics}
\textsf{$\triangleright$} \textsf{Summaries} \textsf{$\triangleright$}
\textsf{Active Dataset}'' means to click the \textsf{Statistics}
menu item, next click the \textsf{Summaries} submenu item, and finally
click \textsf{Active Dataset}.


\section*{Acknowledgements}

This book would not have been possible without the firm mathematical
and statistical foundation provided by the professors at Bowling Green
State University, including Drs.~G\'{a}bor Sz\'{e}kely, Craig Zirbel,
Arjun K.~Gupta, Hanfeng Chen, Truc Nguyen, and James Albert. I would
also like to thank Drs.~Neal Carothers and Kit Chan.

I would also like to thank my colleagues at Youngstown State University
for their support. In particular, I would like to thank Dr.~G.~Andy
Chang for showing me what it means to be a statistician.

I would like to thank Richard Heiberger for his insightful comments
and improvements to several points and displays in the manuscript. 

Finally, and most importantly, I would like to thank my wife for her
patience and understanding while I worked hours, days, months, and
years on a \emph{free book}. In retrospect, I can't believe I ever
got away with it.

\vfill{}


\cleardoublepage
\phantomsection
\addcontentsline{toc}{chapter}{List of Figures}
\listoffigures

\vfill{}


\cleardoublepage
\phantomsection
\addcontentsline{toc}{chapter}{List of Tables}
\listoftables


\chapter{An Introduction to Probability and Statistics}

\pagenumbering{arabic} 

\noindent This chapter has proved to be the hardest to write, by far.
The trouble is that there is so much to say -- and so many people
have already said it so much better than I could. When I get something
I like I will release it here.

In the meantime, there is a lot of information already available to
a person with an Internet connection. I recommend to start at Wikipedia,
which is not a flawless resource but it has the main ideas with links
to reputable sources.

In my lectures I usually tell stories about Fisher, Galton, Gauss,
Laplace, Quetelet, and the Chevalier de Mere.


\section{Probability}

The common folklore is that probability has been around for millennia
but did not gain the attention of mathematicians until approximately
1654 when the Chevalier de Mere had a question regarding the fair
division of a game's payoff to the two players, if the game had to
end prematurely.


\section{Statistics}

Statistics concerns data; their collection, analysis, and interpretation.
In this book we distinguish between two types of statistics: descriptive
and inferential. 

Descriptive statistics concerns the summarization of data. We have
a data set and we would like to describe the data set in multiple
ways. Usually this entails calculating numbers from the data, called
descriptive measures, such as percentages, sums, averages, and so
forth.

Inferential statistics does more. There is an inference associated
with the data set, a conclusion drawn about the population from which
the data originated.

I would like to mention that there are two schools of thought of statistics:
frequentist and bayesian. The difference between the schools is related
to how the two groups interpret the underlying probability (see Section
\ref{sec:Interpreting-Probabilities}). The frequentist school gained
a lot of ground among statisticians due in large part to the work
of Fisher, Neyman, and Pearson in the early twentieth century. That
dominance lasted until inexpensive computing power became widely available;
nowadays the bayesian school is garnering more attention and at an
increasing rate.

This book is devoted mostly to the frequentist viewpoint because that
is how I was trained, with the conspicuous exception of Sections \ref{sec:Bayes'-Rule}
and \ref{sec:Conditional-Distributions}. I plan to add more bayesian
material in later editions of this book.


\section*{Chapter Exercises}

\addcontentsline{toc}{section}{Chapter Exercises}
\setcounter{thm}{0}


\chapter{An Introduction to \textsf{R\label{cha:introduction-to-R}}}


\section{Downloading and Installing \textsf{R\label{sec:download-install-R}}}

The instructions for obtaining \textsf{R} largely depend on the user's
hardware and operating system. The \textsf{R} Project has written
an \textsf{R} Installation and Administration manual with complete,
precise instructions about what to do, together with all sorts of
additional information. The following is just a primer to get a person
started.


\subsection{Installing \textsf{R}}

Visit one of the links below to download the latest version of \textsf{R}
for your operating system:
\begin{description}
\item [{Microsoft~Windows:}] \url{http://cran.r-project.org/bin/windows/base/} 
\item [{MacOS:}] \url{http://cran.r-project.org/bin/macosx/}
\item [{Linux:}] \url{http://cran.r-project.org/bin/linux/}
\end{description}
On Microsoft Windows, click the \inputencoding{latin9}\lstinline[showstringspaces=false]!R-x.y.z.exe!\inputencoding{utf8}
installer to start installation. When it asks for \textquotedbl{}Customized
startup options\textquotedbl{}, specify \textsf{Yes}. In the next
window, be sure to select the SDI (single document interface) option;
this is useful later when we discuss three dimensional plots with
the \inputencoding{latin9}\lstinline[showstringspaces=false]!rgl!\inputencoding{utf8}
package \cite{rgl}.


\paragraph*{Installing \textsf{R} on a USB drive (Windows)}

With this option you can use \textsf{R} portably and without administrative
privileges. There is an entry in the \textsf{R} for Windows FAQ about
this. Here is the procedure I use: 
\begin{enumerate}
\item Download the Windows installer above and start installation as usual.
When it asks \emph{where} to install, navigate to the top-level directory
of the USB drive instead of the default \inputencoding{latin9}\lstinline[showstringspaces=false]!C!\inputencoding{utf8}
drive.
\item When it asks whether to modify the Windows registry, uncheck the box;
we do NOT want to tamper with the registry. 
\item After installation, change the name of the folder from \inputencoding{latin9}\lstinline[showstringspaces=false]!R-x.y.z!\inputencoding{utf8}
to just plain \inputencoding{latin9}\lstinline[showstringspaces=false]!R!\inputencoding{utf8}.
(Even quicker: do this in step 1.) 
\item Download the following shortcut to the top-level directory of the
USB drive, right beside the \inputencoding{latin9}\lstinline[showstringspaces=false]!R!\inputencoding{utf8}
folder, not inside the folder.


\begin{center}
\url{http://ipsur.r-forge.r-project.org/book/download/R.exe}
\par\end{center}

Use the downloaded shortcut to run \textsf{R}.

\end{enumerate}
Steps 3 and 4 are not required but save you the trouble of navigating
to the \inputencoding{latin9}\lstinline[showstringspaces=false]!R-x.y.z/bin!\inputencoding{utf8}
directory to double-click \inputencoding{latin9}\lstinline[showstringspaces=false]!Rgui.exe!\inputencoding{utf8}
every time you want to run the program. It is useless to create your
own shortcut to \inputencoding{latin9}\lstinline[showstringspaces=false]!Rgui.exe!\inputencoding{utf8}.
Windows does not allow shortcuts to have relative paths; they always
have a drive letter associated with them. So if you make your own
shortcut and plug your USB drive into some \emph{other} machine that
happens to assign your drive a different letter, then your shortcut
will no longer be pointing to the right place. 


\subsection{Installing and Loading Add-on Packages\label{sub:installing-loading-packages}}

There are \emph{base} packages (which come with \textsf{R} automatically),
and \emph{contributed} packages (which must be downloaded for installation).
For example, on the version of \textsf{R} being used for this document
the default base packages loaded at startup are 

<<keep.source = TRUE>>=
getOption("defaultPackages")
@

The base packages are maintained by a select group of volunteers,
called {}``\textsf{R} Core''. In addition to the base packages,
there are literally thousands of additional contributed packages written
by individuals all over the world. These are stored worldwide on mirrors
of the Comprehensive \textsf{R} Archive Network, or \inputencoding{latin9}\lstinline[showstringspaces=false]!CRAN!\inputencoding{utf8}
for short. Given an active Internet connection, anybody is free to
download and install these packages and even inspect the source code.

To install a package named \inputencoding{latin9}\lstinline[showstringspaces=false]!foo!\inputencoding{utf8},
open up \textsf{R} and type \inputencoding{latin9}\lstinline[showstringspaces=false]!install.packages("foo")!\inputencoding{utf8}\index{install.packages@\texttt{install.packages}}.
To install \inputencoding{latin9}\lstinline[showstringspaces=false]!foo!\inputencoding{utf8}
and additionally install all of the other packages on which \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!foo!\inputencoding{utf8}
depends, instead type \inputencoding{latin9}\lstinline[showstringspaces=false]!install.packages("foo", depends = TRUE)!\inputencoding{utf8}.

The general command \inputencoding{latin9}\lstinline[showstringspaces=false]!install.packages()!\inputencoding{utf8}
will (on most operating systems) open a window containing a huge list
of available packages; simply choose one or more to install.

No matter how many packages are installed onto the system, each one
must first be loaded for use with the \inputencoding{latin9}\lstinline[showstringspaces=false]!library!\inputencoding{utf8}\index{library@\texttt{library}}
function. For instance, the \inputencoding{latin9}\lstinline[showstringspaces=false]!foreign!\inputencoding{utf8}
package \cite{foreign} contains all sorts of functions needed to
import data sets into \textsf{R} from other software such as SPSS,
SAS, \emph{etc}.. But none of those functions will be available until
the command \inputencoding{latin9}\lstinline[showstringspaces=false]!library(foreign)!\inputencoding{utf8}
is issued. 

Type \inputencoding{latin9}\lstinline[showstringspaces=false]!library()!\inputencoding{utf8}
at the command prompt (described below) to see a list of all available
packages in your library.

For complete, precise information regarding installation of \textsf{R}
and add-on packages, see the \textsf{R} Installation and Administration
manual, \url{http://cran.r-project.org/manuals.html}.


\section{Communicating with \textsf{R\label{sec:Communicating-with-R}}}


\paragraph*{One line at a time}

This is the most basic method and is the first one that beginners
will use.
\begin{description}
\item [{\textsf{RGui}~(Microsoft$\circledR$~Windows)}]~
\item [{Terminal}]~
\item [{Emacs/ESS,~XEmacs}]~
\item [{JGR}]~
\end{description}

\paragraph*{Multiple lines at a time}

For longer programs (called \emph{scripts}) there is too much code
to write all at once at the command prompt. Furthermore, for longer
scripts it is convenient to be able to only modify a certain piece
of the script and run it again in \textsf{R}. Programs called \emph{script
editors} are specially designed to aid the communication and code
writing process. They have all sorts of helpful features including
\textsf{R} syntax highlighting, automatic code completion, delimiter
matching, and dynamic help on the \textsf{R} functions as they are
being written. Even more, they often have all of the text editing
features of programs like Microsoft$\circledR$ Word. Lastly, most
script editors are fully customizable in the sense that the user can
customize the appearance of the interface to choose what colors to
display, when to display them, and how to display them.
\begin{description}
\item [{\textsf{R~}Editor~(Windows):\index{R Editor@\textsf{R} Editor}}] In
Microsoft$\circledR$ Windows, \textsf{R}Gui has its own built-in
script editor, called \textsf{R} Editor. From the console window,
select \textsf{File}\emph{ }\textsf{$\triangleright$}\emph{ }\textsf{New}\textsf{\emph{
}}\textsf{Script}\emph{.} A script window opens, and the lines of
code can be written in the window. When satisfied with the code, the
user highlights all of the commands and presses \textsf{Ctrl+R}. The
commands are automatically run at once in \textsf{R} and the output
is shown. To save the script for later, click \textsf{File}\emph{
}\textsf{$\triangleright$}\emph{ }\textsf{Save as...} in \textsf{R}
Editor. The script can be reopened later with \textsf{File}\emph{
}\textsf{$\triangleright$}\emph{ }\textsf{Open Script...} in \textsf{RGui}.
Note that \textsf{R} Editor does not have the fancy syntax highlighting
that the others do.
\item [{\textsf{R}WinEdt:\index{RWinEdt@\textsf{R}WinEdt}}] This option
is coordinated with WinEdt for \LaTeX{} and has additional features
such as code highlighting, remote sourcing, and a ton of other things.
However, one first needs to download and install a shareware version
of another program, WinEdt, which is only free for a while -- pop-up
windows will eventually appear that ask for a registration code. \textsf{R}WinEdt
is nevertheless a very fine choice if you already own WinEdt or are
planning to purchase it in the near future.
\item [{Tinn-\textsf{R}/Sciviews-K:\index{Tinn-R@Tinn-\textsf{R}}\index{Sciviews-K}}] This
one is completely free and has all of the above mentioned options
and more. It is simple enough to use that the user can virtually begin
working with it immediately after installation. But Tinn-\textsf{R}
proper is only available for Microsoft$\circledR$ Windows operating
systems. If you are on MacOS or Linux, a comparable alternative is
Sci-Views - Komodo Edit.
\item [{Emacs/ESS:\index{Emacs}\index{ESS}}] Emacs is an all purpose
text editor. It can do absolutely anything with respect to modifying,
searching, editing, and manipulating, text. And if Emacs can't do
it, then you can write a program that extends Emacs to do it. Once
such extension is called \emph{ESS}, which stands for \emph{E}macs
\emph{S}peaks \emph{S}tatistics. With ESS a person can speak to \textsf{R,}
do all of the tricks that the other script editors offer, and much,
much, more. Please see the following for installation details, documentation,
reference cards, and a whole lot more:
\end{description}
\begin{center}
\url{http://ess.r-project.org}
\par\end{center}

\emph{Fair warning}: if you want to try Emacs and if you grew up with
Microsoft$\circledR$ Windows or Macintosh, then you are going to
need to relearn everything you thought you knew about computers your
whole life. (Or, since Emacs is completely customizable, you can reconfigure
Emacs to behave the way you want.) I have personally experienced this
transformation and I will never go back.
\begin{description}
\item [{JGR~(read~{}``Jaguar''):\index{JGR}}] This one has the bells
and whistles of \textsf{RGui} plus it is based on Java, so it works
on multiple operating systems. It has its own script editor like \textsf{R}
Editor but with additional features such as syntax highlighting and
code-completion. If you do not use Microsoft$\circledR$ Windows (or
even if you do) you definitely want to check out this one.
\item [{Kate,~Bluefish,~\emph{etc}.}] There are literally dozens of other
text editors available, many of them free, and each has its own (dis)advantages.
I only have mentioned the ones with which I have had substantial personal
experience and have enjoyed at some point. Play around, and let me
know what you find.
\end{description}

\paragraph*{Graphical User Interfaces (GUIs)}

By the word {}``GUI'' I mean an interface in which the user communicates
with \textsf{R} by way of points-and-clicks in a menu of some sort.
Again, there are many, many options and I only mention ones that I
have used and enjoyed. Some of the other more popular script editors
can be downloaded from the\textsf{ R}-Project website at \url{http://www.sciviews.org/_rgui/}.
On the left side of the screen (under \textbf{Projects}) there are
several choices available.
\begin{description}
\item [{\textsf{R~}Commander}] provides\index{R Commander@\textsf{R} Commander}
a point-and-click interface to many basic statistical tasks. It is
called the {}``Commander'' because every time one makes a selection
from the menus, the code corresponding to the task is listed in the
output window. One can take this code, copy-and-paste it to a text
file, then re-run it again at a later time without the \textsf{R}
Commander's assistance. It is well suited for the introductory level.
\inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!Rcmdr!\inputencoding{utf8}
also allows for user-contributed {}``Plugins'' which are separate
packages on \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!CRAN!\inputencoding{utf8}
that add extra functionality to the \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!Rcmdr!\inputencoding{utf8}
package. The plugins are typically named with the prefix \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!RcmdrPlugin!\inputencoding{utf8}
to make them easy to identify in the \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!CRAN!\inputencoding{utf8}
package list. One such plugin is the \\  \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!RcmdrPlugin.IPSUR!\inputencoding{utf8}
package which accompanies this text.
\item [{Poor~Man's~GUI}] \index{Poor Man's GUI} is an alternative to
the \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!Rcmdr!\inputencoding{utf8}
which is based on GTk instead of Tcl/Tk. It has been a while since
I used it but I remember liking it very much when I did. One thing
that stood out was that the user could drag-and-drop data sets for
plots. See here for more information: \url{http://wiener.math.csi.cuny.edu/pmg/}.
\item [{Rattle\index{Rattle}}] is a data mining toolkit which was designed
to manage/analyze very large data sets, but it provides enough other
general functionality to merit mention here. See \cite{rattle} for
more information.
\item [{Deducer\index{Deducer}}] is relatively new and shows promise from
what I have seen, but I have not actually used it in the classroom
yet.
\end{description}

\section{Basic \textsf{R} Operations and Concepts\label{sec:Basic-R-Operations}}

The \textsf{R} developers have written an introductory document entitled
{}``An Introduction to \textsf{R}''. There is a sample session included
which shows what basic interaction with \textsf{R} looks like. I recommend
that all new users of \textsf{R} read that document, but bear in mind
that there are concepts mentioned which will be unfamiliar to the
beginner.

Below are some of the most basic operations that can be done with
\textsf{R}. Almost every book about \textsf{R} begins with a section
like the one below; look around to see all sorts of things that can
be done at this most basic level.


\subsection{Arithmetic\label{sub:Arithmetic}}

<<keep.source = TRUE>>=
2 + 3       # add
4 * 5 / 6   # multiply and divide
7^8         # 7 to the 8th power
@

Notice the comment character \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!#!\inputencoding{utf8}\index{#@\texttt{\#}}.
Anything typed after a \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!#!\inputencoding{utf8}
symbol is ignored by \textsf{R}. We know that $20/6$ is a repeating
decimal, but the above example shows only 7 digits. We can change
the number of digits displayed with \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!options!\inputencoding{utf8}\index{options@\texttt{options}}:

<<keep.source = TRUE>>=
options(digits = 16)
10/3                 # see more digits
sqrt(2)              # square root
exp(1)               # Euler's constant, e
pi       
options(digits = 7)  # back to default
@

Note that it is possible to set \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!digits!\inputencoding{utf8}\index{digits@\texttt{digits}}
up to 22, but setting them over 16 is not recommended (the extra significant
digits are not necessarily reliable). Above notice the \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!sqrt!\inputencoding{utf8}\index{sqrt@\texttt{sqrt}}
function for square roots and the \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!exp!\inputencoding{utf8}\index{exp@\texttt{exp}}
function for powers of $\me$, Euler's number.


\subsection{Assignment, Object names, and Data types\label{sub:Assignment-Object-names}}

It is often convenient to assign numbers and values to variables (objects)
to be used later. The proper way to assign values to a variable is
with the \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!<-!\inputencoding{utf8}
operator (with a space on either side). The \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!=!\inputencoding{utf8}
symbol works too, but it is recommended by the \textsf{R} masters
to reserve \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!=!\inputencoding{utf8}
for specifying arguments to functions (discussed later). In this book
we will follow their advice and use \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!<-!\inputencoding{utf8}
for assignment. Once a variable is assigned, its value can be printed
by simply entering the variable name by itself.

<<keep.source = TRUE>>=
x <- 7*41/pi   # don't see the calculated value
x              # take a look
@

When choosing a variable name you can use letters, numbers, dots {}``\texttt{.}'',
or underscore {}``\texttt{\_}'' characters. You cannot use mathematical
operators, and a leading dot may not be followed by a number. Examples
of valid names are: \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!x!\inputencoding{utf8},
\inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!x1!\inputencoding{utf8},
\inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!y.value!\inputencoding{utf8},
and \inputencoding{latin9}\lstinline[basicstyle={\ttfamily},language=Caml]!y_hat!\inputencoding{utf8}.
(More precisely, the set of allowable characters in object names depends
on one's particular system and locale; see An Introduction to \textsf{R}
for more discussion on this.)

Objects can be of many \emph{types}, \emph{modes}, and \emph{classes}.
At this level, it is not necessary to investigate all of the intricacies
of the respective types, but there are some with which you need to
become familiar:
\begin{description}
\item [{integer:}] the values $0$, $\pm1$, $\pm2$, \ldots{}; these
are represented exactly by \textsf{R}.
\item [{double:}] real numbers (rational and irrational); these numbers
are not represented exactly (save integers or fractions with a denominator
that is a power of 2, see \cite{Venables2010}).
\item [{character:}] elements that are wrapped with pairs of \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!"!\inputencoding{utf8}
or \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!'!\inputencoding{utf8};
\item [{logical:}] includes \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!TRUE!\inputencoding{utf8},
\inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!FALSE!\inputencoding{utf8},
and \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!NA!\inputencoding{utf8}
(which are reserved words); the \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!NA!\inputencoding{utf8}\index{NA@\texttt{NA}}
stands for {}``not available'', \emph{i.e.}, a missing value.
\end{description}
You can determine an object's type with the \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!typeof!\inputencoding{utf8}\index{typeof@\texttt{typeof}}
function. In addition to the above, there is the \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!complex!\inputencoding{utf8}\index{complex@\texttt{complex}}\index{as.complex@\texttt{as.complex}}
data type:

<<five, keep.source = TRUE>>=
sqrt(-1)              # isn't defined
sqrt(-1+0i)           # is defined
sqrt(as.complex(-1))  # same thing
(0 + 1i)^2            # should be -1
typeof((0 + 1i)^2)
@

Note that you can just type \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!(1i)^2!\inputencoding{utf8}
to get the same answer. The \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!NaN!\inputencoding{utf8}\index{NaN@\texttt{NaN}}
stands for {}``not a number''; it is represented internally as \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!double!\inputencoding{utf8}\index{double}. 


\subsection{Vectors\label{sub:Vectors}}

All of this time we have been manipulating vectors of length 1. Now
let us move to vectors with multiple entries.


\paragraph*{Entering data vectors}
\begin{enumerate}
\item \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!c!\inputencoding{utf8}\index{c@\texttt{c}}:
If you would like to enter the data \texttt{74,31,95,61,76,34,23,54,96}
into \textsf{R}, you may create a data vector with the \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!c!\inputencoding{utf8}
function (which is short for \emph{concatenate}).


<<>>=
x <- c(74, 31, 95, 61, 76, 34, 23, 54, 96)
x
@

The elements of a vector are usually coerced by \textsf{R} to the
the most general type of any of the elements, so if you do \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!c(1, "2")!\inputencoding{utf8}
then the result will be \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!c("1", "2")!\inputencoding{utf8}.

\item \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!scan!\inputencoding{utf8}\index{scan@\texttt{scan}}:
This method is useful when the data are stored somewhere else. For
instance, you may type \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!x <- scan()!\inputencoding{utf8}
at the command prompt and \textsf{R} will display \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!1:!\inputencoding{utf8}
to indicate that it is waiting for the first data value. Type a value
and press \textsf{Enter}, at which point \textsf{R} will display \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!2:!\inputencoding{utf8},
and so forth. Note that entering an empty line stops the scan. This
method is especially handy when you have a column of values, say,
stored in a text file or spreadsheet. You may copy and paste them
all at the \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!1:!\inputencoding{utf8}
prompt, and \textsf{R} will store all of the values instantly in the
vector \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!x!\inputencoding{utf8}.
\item repeated data; regular patterns: the \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!seq!\inputencoding{utf8}\index{seq@\texttt{seq}}
function will generate all sorts of sequences of numbers. It has the
arguments \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!from!\inputencoding{utf8},
\inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!to!\inputencoding{utf8},
\inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!by!\inputencoding{utf8},
and \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!length.out!\inputencoding{utf8}
which can be set in concert with one another. We will do a couple
of examples to show you how it works.


<<>>=
seq(from = 1, to = 5)
seq(from = 2, by = -0.1, length.out = 4)
@

Note that we can get the first line much quicker with the colon operator
\inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!:!\inputencoding{utf8}

<<>>=
1:5
@

The vector \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!LETTERS!\inputencoding{utf8}\index{LETTERS@\texttt{LETTERS}}
has the 26 letters of the English alphabet in uppercase and \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!letters!\inputencoding{utf8}\index{letters@\texttt{letters}}
has all of them in lowercase.

\end{enumerate}

\paragraph*{Indexing data vectors}

Sometimes we do not want the whole vector, but just a piece of it.
We can access the intermediate parts with the \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]![]!\inputencoding{utf8}\index{[]@\texttt{{[}{]}}}
operator. Observe (with \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!x!\inputencoding{utf8}
defined above)

<<>>=
x[1]
x[2:4]
x[c(1,3,4,8)]
x[-c(1,3,4,8)]
@

Notice that we used the minus sign to specify those elements that
we do \emph{not} want. 

<<>>=
LETTERS[1:5]
letters[-(6:24)]

@


\subsection{Functions and Expressions\label{sub:Functions-and-Expressions}}

A function takes arguments as input and returns an object as output.
There are functions to do all sorts of things. We show some examples
below.

<<keep.source = TRUE>>=
x <- 1:5
sum(x)
length(x)
min(x)
mean(x)      # sample mean
sd(x)        # sample standard deviation
@

It will not be long before the user starts to wonder how a particular
function is doing its job, and since \textsf{R} is open-source, anybody
is free to look under the hood of a function to see how things are
calculated. For detailed instructions see the article {}``Accessing
the Sources'' by Uwe Ligges \cite{Ligges2006}. In short:
\begin{enumerate}
\item Type the name of the function without any parentheses or arguments.
If you are lucky then the code for the entire function will be printed,
right there looking at you. For instance, suppose that we would like
to see how the \inputencoding{latin9}\lstinline[showstringspaces=false]!intersect!\inputencoding{utf8}\index{intersect@\texttt{intersect}}
function works:


<<>>=
intersect
@

\item If instead it shows \inputencoding{latin9}\lstinline[showstringspaces=false]!UseMethod("!\inputencoding{utf8}\emph{something}\inputencoding{latin9}\lstinline[showstringspaces=false]!")!\inputencoding{utf8}\index{UseMethod@\texttt{UseMethod}}
then you will need to choose the \emph{class} of the object to be
inputted and next look at the \emph{method} that will be \emph{dispatched}
to the object. For instance, typing \inputencoding{latin9}\lstinline[showstringspaces=false]!rev!\inputencoding{utf8}\index{rev@\texttt{rev}}
says 


<<>>=
rev
@

The output is telling us that there are multiple methods associated
with the \inputencoding{latin9}\lstinline[showstringspaces=false]!rev!\inputencoding{utf8}
function. To see what these are, type

<<>>=
methods(rev)
@

Now we learn that there are two different \inputencoding{latin9}\lstinline[showstringspaces=false]!rev(x)!\inputencoding{utf8}
functions, only one of which being chosen at each call depending on
what \inputencoding{latin9}\lstinline[showstringspaces=false]!x!\inputencoding{utf8}
is. There is one for \inputencoding{latin9}\lstinline[showstringspaces=false]!dendrogram!\inputencoding{utf8}
objects and a \inputencoding{latin9}\lstinline[showstringspaces=false]!default!\inputencoding{utf8}
method for everything else. Simply type the name to see what each
method does. For example, the \inputencoding{latin9}\lstinline[showstringspaces=false]!default!\inputencoding{utf8}
method can be viewed with

<<>>=
rev.default
@

\item Some functions are hidden by a \emph{namespace} (see An Introduction
to \textsf{R} \cite{Venables2010}), and are not visible on the first
try. For example, if we try to look at the code for \inputencoding{latin9}\lstinline[showstringspaces=false]!wilcox.test!\inputencoding{utf8}\index{wilcox.test@\texttt{wilcox.test}}
(see Chapter \ref{cha:Nonparametric-Statistics}) we get the following:


<<>>=
wilcox.test
methods(wilcox.test)
@

If we were to try \inputencoding{latin9}\lstinline[showstringspaces=false]!wilcox.test.default!\inputencoding{utf8}
we would get a {}``not found'' error, because it is hidden behind
the namespace for the package \inputencoding{latin9}\lstinline[showstringspaces=false]!stats!\inputencoding{utf8}
(shown in the last line when we tried \inputencoding{latin9}\lstinline[showstringspaces=false]!wilcox.test!\inputencoding{utf8}).
In cases like these we prefix the package name to the front of the
function name with three colons; the command \inputencoding{latin9}\lstinline[showstringspaces=false]!stats:::wilcox.test.default!\inputencoding{utf8}
will show the source code, omitted here for brevity.

\item If it shows \inputencoding{latin9}\lstinline[showstringspaces=false]!.Internal(!\inputencoding{utf8}\emph{something}\inputencoding{latin9}\lstinline[showstringspaces=false]!)!\inputencoding{utf8}\index{.Internal@\texttt{.Internal}}
or \inputencoding{latin9}\lstinline[showstringspaces=false]!.Primitive("!\inputencoding{utf8}\emph{something}\inputencoding{latin9}\lstinline[showstringspaces=false]!")!\inputencoding{utf8}\index{.Primitive@\texttt{.Primitive}},
then it will be necessary to download the source code of \textsf{R}
(which is \emph{not} a binary version with an \inputencoding{latin9}\lstinline[showstringspaces=false]!.exe!\inputencoding{utf8}
extension) and search inside the code there. See Ligges \cite{Ligges2006}
for more discussion on this. An example is \inputencoding{latin9}\lstinline[showstringspaces=false]!exp!\inputencoding{utf8}:


<<>>=
exp
@

Be warned that most of the \inputencoding{latin9}\lstinline[showstringspaces=false]!.Internal!\inputencoding{utf8}
functions are written in other computer languages which the beginner
may not understand, at least initially.

\end{enumerate}

\section{Getting Help\label{sec:Getting-Help}}

When you are using \textsf{R}, it will not take long before you find
yourself needing help. Fortunately, \textsf{R} has extensive help
resources and you should immediately become familiar with them. Begin
by clicking \textsf{Help} on \textsf{Rgui}. The following options
are available.
\begin{itemize}
\item \textbf{Console}: gives useful shortcuts, for instance, \textsf{Ctrl+L},
to clear the \textsf{R} console screen.
\item \textbf{FAQ on }\textsf{\textbf{R}}: frequently asked questions concerning
general \textsf{R} operation.
\item \textbf{FAQ on }\textsf{\textbf{R}}\textbf{ for Windows}: frequently
asked questions about \textsf{R}, tailored to the Microsoft Windows
operating system.
\item \textbf{Manuals}: technical manuals about all features of the \textsf{R}
system including installation, the complete language definition, and
add-on packages.
\item \textsf{\textbf{R}}\textbf{ functions (text)\ldots{}}: use this if
you know the \emph{exact} name of the function you want to know more
about, for example, \inputencoding{latin9}\lstinline[showstringspaces=false]!mean!\inputencoding{utf8}
or \inputencoding{latin9}\lstinline[showstringspaces=false]!plot!\inputencoding{utf8}.
Typing \inputencoding{latin9}\lstinline[showstringspaces=false]!mean!\inputencoding{utf8}
in the window is equivalent to typing \inputencoding{latin9}\lstinline[showstringspaces=false]!help("mean")!\inputencoding{utf8}\index{help@\texttt{help}}
at the command line, or more simply, \inputencoding{latin9}\lstinline[showstringspaces=false]!?mean!\inputencoding{utf8}\index{?@\texttt{?}}.
Note that this method only works if the function of interest is contained
in a package that is already loaded into the search path with \inputencoding{latin9}\lstinline[showstringspaces=false]!library!\inputencoding{utf8}.
\item \textbf{HTML Help}: use this to browse the manuals with point-and-click
links. It also has a Search Engine \& Keywords for searching the help
page titles, with point-and-click links for the search results. This
is possibly the best help method for beginners. It can be started
from the command line with the command \inputencoding{latin9}\lstinline[showstringspaces=false]!help.start()!\inputencoding{utf8}\index{help.start@\texttt{help.start}}.
\item \textbf{Search help\ldots{}}: use this if you do not know the exact
name of the function of interest, or if the function is in a package
that has not been loaded yet. For example, you may enter \inputencoding{latin9}\lstinline[showstringspaces=false]!plo!\inputencoding{utf8}
and a text window will return listing all the help files with an alias,
concept, or title matching `\inputencoding{latin9}\lstinline[showstringspaces=false]!plo!\inputencoding{utf8}'
using regular expression matching; it is equivalent to typing \inputencoding{latin9}\lstinline[showstringspaces=false]!help.search("plo")!\inputencoding{utf8}\index{help.search@\texttt{help.search}}
at the command line. The advantage is that you do not need to know
the exact name of the function; the disadvantage is that you cannot
point-and-click the results. Therefore, one may wish to use the HTML
Help search engine instead. An equivalent way is \inputencoding{latin9}\lstinline[showstringspaces=false]!??plo!\inputencoding{utf8}\index{??@\texttt{??}}
at the command line.
\item \textbf{search.r-project.org\ldots{}}: this will search for words
in help lists and email archives of the \textsf{R} Project. It can
be very useful for finding other questions that other users have asked.
\item \textbf{Apropos\ldots{}}: use this for more sophisticated partial
name matching of functions. See \inputencoding{latin9}\lstinline[showstringspaces=false]!?apropos!\inputencoding{utf8}\index{apropos@\texttt{apropos}}
for details.
\end{itemize}
On the help pages for a function there are sometimes {}``Examples''
listed at the bottom of the page, which will work if copy-pasted at
the command line (unless marked otherwise). The \inputencoding{latin9}\lstinline[showstringspaces=false]!example!\inputencoding{utf8}\index{example@\texttt{example}}
function will run the code automatically, skipping the intermediate
step. For instance, we may try \inputencoding{latin9}\lstinline[showstringspaces=false]!example(mean)!\inputencoding{utf8}
to see a few examples of how the \inputencoding{latin9}\lstinline[showstringspaces=false]!mean!\inputencoding{utf8}
function works.


\subsection{\textsf{R} Help Mailing Lists}

There are several mailing lists associated with \textsf{R}, and there
is a huge community of people that read and answer questions related
to \textsf{R}. See here \url{http://www.r-project.org/mail.html}
for an idea of what is available. Particularly pay attention to the
bottom of the page which lists several special interest groups (SIGs)
related to \textsf{R}.

Bear in mind that \textsf{R} is free software, which means that it
was written by volunteers, and the people that frequent the mailing
lists are also volunteers who are not paid by customer support fees.
Consequently, if you want to use the mailing lists for free advice
then you must adhere to some basic etiquette, or else you may not
get a reply, or even worse, you may receive a reply which is a bit
less cordial than you are used to. Below are a few considerations:
\begin{enumerate}
\item Read the FAQ (\url{http://cran.r-project.org/faqs.html}). Note that
there are different FAQs for different operating systems. You should
read these now, even without a question at the moment, to learn a
lot about the idiosyncrasies of \textsf{R}.
\item Search the archives. Even if your question is not a FAQ, there is
a very high likelihood that your question has been asked before on
the mailing list. If you want to know about topic \inputencoding{latin9}\lstinline[showstringspaces=false]!foo!\inputencoding{utf8},
then you can do \inputencoding{latin9}\lstinline[showstringspaces=false]!RSiteSearch("foo")!\inputencoding{utf8}\index{RSiteSearch@\texttt{RSiteSearch}}
to search the mailing list archives (and the online help) for it. 
\item Do a Google search and an \texttt{RSeek.org} search.
\end{enumerate}
If your question is not a FAQ, has not been asked on \textsf{R}-help
before, and does not yield to a Google (or alternative) search, then,
and only then, should you even consider writing to \textsf{R}-help.
Below are a few additional considerations. 
\begin{enumerate}
\item \textbf{Read the posting guide (\url{http://www.r-project.org/posting-guide.html})
before posting.} This will save you a lot of trouble and pain.
\item Get rid of the command prompts (\inputencoding{latin9}\lstinline[showstringspaces=false]!>!\inputencoding{utf8})
from output. Readers of your message will take the text from your
mail and copy-paste into an \textsf{R} session. If you make the readers'
job easier then it will increase the likelihood of a response.
\item Questions are often related to a specific data set, and the best way
to communicate the data is with a \inputencoding{latin9}\lstinline[showstringspaces=false]!dump!\inputencoding{utf8}\index{dump@\texttt{dump}}
command. For instance, if your question involves data stored in a
vector \inputencoding{latin9}\lstinline[showstringspaces=false]!x!\inputencoding{utf8},
you can type \inputencoding{latin9}\lstinline[showstringspaces=false]!dump("x","")!\inputencoding{utf8}
at the command prompt and copy-paste the output into the body of your
email message. Then the reader may easily copy-paste the message from
your email into \textsf{R} and \inputencoding{latin9}\lstinline[showstringspaces=false]!x!\inputencoding{utf8}
will be available to him/her.
\item Sometimes the answer the question is related to the operating system
used, the attached packages, or the exact version of \textsf{R} being
used. The \inputencoding{latin9}\lstinline[showstringspaces=false]!sessionInfo()!\inputencoding{utf8}\index{sessionInfo@\texttt{sessionInfo}}
command collects all of this information to be copy-pasted into an
email (and the Posting Guide requests this information). See Appendix
\ref{cha:R-Session-Information} for an example.
\end{enumerate}

\section{External Resources}

There is a mountain of information on the Internet about \textsf{R}.
Below are a few of the important ones.
\begin{description}
\item [{The~\textsf{R~}Project~for~Statistical~Computing:\index{The R-Project@The \textsf{R}-Project}}] (\url{http://www.r-project.org/})
Go here first.
\item [{The~Comprehensive~\textsf{R~}Archive~Network:\index{CRAN}}] (\url{http://cran.r-project.org/})
This is where \textsf{R} is stored along with thousands of contributed
packages. There are also loads of contributed information (books,
tutorials, \emph{etc}.). There are mirrors all over the world with
duplicate information.
\item [{\textsf{R}-Forge:\index{R-Forge@\textsf{R}-Forge}}] (\url{http://r-forge.r-project.org/})
This is another location where \textsf{R} packages are stored. Here
you can find development code which has not yet been released to \textsf{CRAN}.
\item [{\textsf{R~}Wiki:\index{R-Wiki@\textsf{R}-Wiki}}] (\url{http://wiki.r-project.org/rwiki/doku.php})
There are many tips and tricks listed here. If you find a trick of
your own, login and share it with the world.
\item [{Other:}] the \textsf{R} Graph Gallery\index{R Graph Gallery@\textsf{R} Graph Gallery}
(\url{http://addictedtor.free.fr/graphiques/}) and \textsf{R} Graphical
Manual\index{R Graphical Manual@\textsf{R} Graphical Manual} (\url{http://bm2.genes.nig.ac.jp/RGM2/index.php})
have literally thousands of graphs to peruse. \textsf{R}Seek (\url{http://www.rseek.org})
is a search engine based on Google specifically tailored for \textsf{R}
queries. 
\end{description}

\section{Other Tips}

It is unnecessary to retype commands repeatedly, since \textsf{R}
remembers what you have recently entered on the command line. On the
Microsoft$\circledR$ Windows \textsf{R}Gui, to cycle through the
previous commands just push the $\uparrow$ (up arrow) key. On Emacs/ESS
the command is \inputencoding{latin9}\lstinline[showstringspaces=false]!M-p!\inputencoding{utf8}
(which means hold down the \textsf{Alt} button and press {}``\inputencoding{latin9}\lstinline[showstringspaces=false]!p!\inputencoding{utf8}'').
More generally, the command \inputencoding{latin9}\lstinline[showstringspaces=false]!history()!\inputencoding{utf8}\index{history@\texttt{history}}
will show a whole list of recently entered commands.
\begin{itemize}
\item To find out what all variables are in the current work environment,
use the commands \inputencoding{latin9}\lstinline[showstringspaces=false]!objects()!\inputencoding{utf8}\index{objects@\texttt{objects}}
or \inputencoding{latin9}\lstinline[showstringspaces=false]!ls()!\inputencoding{utf8}\index{ls@\texttt{ls}}.
These list all available objects in the workspace. If you wish to
remove one or more variables, use \inputencoding{latin9}\lstinline[showstringspaces=false]!remove(var1, var2, var3)!\inputencoding{utf8}\index{remove@\texttt{remove}},
or more simply use \inputencoding{latin9}\lstinline[showstringspaces=false]!rm(var1, var2, var3)!\inputencoding{utf8},
and to remove all objects use \inputencoding{latin9}\lstinline[showstringspaces=false]!rm(list = ls())!\inputencoding{utf8}.
\item Another use of \inputencoding{latin9}\lstinline[showstringspaces=false]!scan!\inputencoding{utf8}
is when you have a long list of numbers (separated by spaces or on
different lines) already typed somewhere else, say in a text file.
To enter all the data in one fell swoop, first highlight and copy
the list of numbers to the Clipboard with \textsf{Edit}\emph{ }\textsf{$\triangleright$}\emph{
}\textsf{Copy} (or by right-clicking and selecting \textsf{Copy}).
Next type the \inputencoding{latin9}\lstinline[showstringspaces=false]!x <- scan()!\inputencoding{utf8}
command in the \textsf{R} console, and paste the numbers at the \inputencoding{latin9}\lstinline[showstringspaces=false]!1:!\inputencoding{utf8}
prompt with \textsf{Edit}\emph{ }\textsf{$\triangleright$}\emph{
}\textsf{Paste}. All of the numbers will automatically be entered
into the vector \inputencoding{latin9}\lstinline[showstringspaces=false]!x!\inputencoding{utf8}.
\item The command \inputencoding{latin9}\lstinline[showstringspaces=false]!Ctrl+l!\inputencoding{utf8}
clears the screen in the Microsoft$\circledR$ Windows \textsf{R}Gui.
The comparable command for Emacs/ESS is 
\item Once you use \textsf{R} for awhile there may be some commands that
you wish to run automatically whenever \textsf{R} starts. These commands
may be saved in a file called \inputencoding{latin9}\lstinline[showstringspaces=false]!Rprofile.site!\inputencoding{utf8}\index{Rprofile.site@\texttt{Rprofile.site}}
which is usually in the \inputencoding{latin9}\lstinline[showstringspaces=false]!etc!\inputencoding{utf8}
folder, which lives in the \textsf{R} home directory (which on Microsoft$\circledR$
Windows usually is \inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false]!C:\Program Files\R!\inputencoding{utf8}).
Alternatively, you can make a file \inputencoding{latin9}\lstinline[showstringspaces=false]!.Rprofile!\inputencoding{utf8}\index{.Rprofile@\texttt{.Rprofile}}
to be stored in the user's home directory, or anywhere \textsf{R}
is invoked. This allows for multiple configurations for different
projects or users. See {}``Customizing the Environment'' of \emph{An
Introduction to }\textsf{\emph{R}} for more details.
\item When exiting \textsf{R} the user is given the option to {}``save
the workspace''. I recommend that beginners DO NOT save the workspace
when quitting. If \textsf{Yes} is selected, then all of the objects
and data currently in \textsf{R}'s memory is saved in a file located
in the working directory called \inputencoding{latin9}\lstinline[showstringspaces=false]!.RData!\inputencoding{utf8}\index{.RData@\texttt{.RData}}.
This file is then automatically loaded the next time \textsf{R} starts
(in which case \textsf{R} will say \inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false]![previously saved workspace restored]!\inputencoding{utf8}).
This is a valuable feature for experienced users of \textsf{R}, but
I find that it causes more trouble than it saves with beginners.
\end{itemize}
\newpage{}


\section*{Chapter Exercises}

\addcontentsline{toc}{section}{Chapter Exercises}
\setcounter{thm}{0}


\chapter{Data Description \label{cha:Describing-Data-Distributions}}

\noindent In this chapter we introduce the different types of data
that a statistician is likely to encounter, and in each subsection
we give some examples of how to display the data of that particular
type. Once we see how to display data distributions, we next introduce
the basic properties of data distributions. We qualitatively explore
several data sets. Once that we have intuitive properties of data
sets, we next discuss how we may numerically measure and describe
those properties with descriptive statistics.


\paragraph*{What do I want them to know?}
\begin{itemize}
\item different data types, such as quantitative versus qualitative, nominal
versus ordinal, and discrete versus continuous
\item basic graphical displays for assorted data types, and some of their
(dis)advantages 
\item fundamental properties of data distributions, including center, spread,
shape, and crazy observations
\item methods to describe data (visually/numerically) with respect to the
properties, and how the methods differ depending on the data type
\item all of the above in the context of grouped data, and in particular,
the concept of a factor
\end{itemize}

\section{Types of Data\label{sec:Types-of-Data}}

Loosely speaking, a datum is any piece of collected information, and
a data set is a collection of data related to each other in some way.
We will categorize data into five types and describe each in turn:
\begin{description}
\item [{Quantitative}] data associated with a measurement of some quantity
on an observational unit,
\item [{Qualitative}] data associated with some quality or property of
the observational unit,
\item [{Logical}] data to represent true or false and which play an important
role later,
\item [{Missing}] data that should be there but are not, and
\item [{Other~types}] everything else under the sun.
\end{description}
In each subsection we look at some examples of the type in question
and introduce methods to display them.


\subsection{Quantitative data\label{sub:Quantitative-Data}}

Quantitative data are any data that measure or are associated with
a measurement of the quantity of something. They invariably assume
numerical values. Quantitative data can be further subdivided into
two categories. 
\begin{itemize}
\item \emph{Discrete data} take values in a finite or countably infinite
set of numbers, that is, all possible values could (at least in principle)
be written down in an ordered list. Examples include: counts, number
of arrivals, or number of successes. They are often represented by
integers, say, 0, 1, 2, \emph{etc}..
\item \emph{Continuous data} take values in an interval of numbers. These
are also known as scale data, interval data, or measurement data.
Examples include: height, weight, length, time, \emph{etc}. Continuous
data are often characterized by fractions or decimals: 3.82, 7.0001,
4~$\frac{5}{8}$, \emph{etc}..
\end{itemize}
Note that the distinction between discrete and continuous data is
not always clear-cut. Sometimes it is convenient to treat data as
if they were continuous, even though strictly speaking they are not
continuous. See the examples.
\begin{example}
\textbf{Annual Precipitation in US Cities.} The vector \inputencoding{latin9}\lstinline[showstringspaces=false]!precip!\inputencoding{utf8}\index{Data sets!precip@\texttt{precip}}
contains average amount of rainfall (in inches) for each of 70 cities
in the United States and Puerto Rico. Let us take a look at the data:

<<>>=
str(precip)
precip[1:4]
@

The output shows that \inputencoding{latin9}\lstinline[showstringspaces=false]!precip!\inputencoding{utf8}
is a numeric vector which has been \emph{named}, that is, each value
has a name associated with it (which can be set with the \inputencoding{latin9}\lstinline[showstringspaces=false]!names!\inputencoding{utf8}\index{names@\texttt{names}}
function). These are quantitative continuous data.

\end{example}

\begin{example}
\textbf{Lengths of Major North American Rivers.} The U.S.~Geological
Survey recorded the lengths (in miles) of several rivers in North
America. They are stored in the vector \inputencoding{latin9}\lstinline[showstringspaces=false]!rivers!\inputencoding{utf8}\index{Data sets!rivers@\texttt{rivers}}
in the \inputencoding{latin9}\lstinline[showstringspaces=false]!datasets!\inputencoding{utf8}
package (which ships with base \textsf{R}). See \inputencoding{latin9}\lstinline[showstringspaces=false]!?rivers!\inputencoding{utf8}.
Let us take a look at the data with the \inputencoding{latin9}\lstinline[showstringspaces=false]!str!\inputencoding{utf8}\index{str@\texttt{str}}
function.

<<>>=
str(rivers)
@

The output says that \inputencoding{latin9}\lstinline[showstringspaces=false]!rivers!\inputencoding{utf8}
is a numeric vector of length 141, and the first few values are 735,
320, 325, \emph{etc}. These data are definitely quantitative and it
appears that the measurements have been rounded to the nearest mile.
Thus, strictly speaking, these are discrete data. But we will find
it convenient later to take data like these to be continuous for some
of our statistical procedures. 
\end{example}

\begin{example}
\textbf{Yearly Numbers of Important Discoveries.} The vector \inputencoding{latin9}\lstinline[showstringspaces=false]!discoveries!\inputencoding{utf8}\index{Data sets!discoveries@\texttt{discoveries}}
contains numbers of “great” inventions/discoveries in each year from
1860 to 1959, as reported by the 1975 World Almanac. Let us take a
look at the data:

<<>>=
str(discoveries)
discoveries[1:4]
@

\end{example}
The output is telling us that \inputencoding{latin9}\lstinline[showstringspaces=false]!discoveries!\inputencoding{utf8}
is a \emph{time series} (see Section \ref{sub:other-data-types} for
more) of length 100. The entries are integers, and since they represent
counts this is a good example of discrete quantitative data. We will
take a closer look in the following sections.


\subsection*{Displaying Quantitative Data\label{sub:Displaying-Quantitative-Data}}

One of the first things to do when confronted by quantitative data
(or any data, for that matter) is to make some sort of visual display
to gain some insight into the data's structure. There are almost as
many display types from which to choose as there are data sets to
plot. We describe some of the more popular alternatives. 


\paragraph*{Strip~charts\index{strip chart} (also known as Dot plots)\index{dot plot| see\{strip chart\}}\label{par:Strip-charts}}

These can be used for discrete or continuous data, and usually look
best when the data set is not too large. Along the horizontal axis
is a numerical scale above which the data values are plotted. We can
do it in \textsf{R} with a call to the \inputencoding{latin9}\lstinline[showstringspaces=false]!stripchart!\inputencoding{utf8}\index{stripchart@\texttt{stripchart}}
function. There are three available methods.
\begin{description}
\item [{overplot}] plots ties covering each other. This method is good
to display only the distinct values assumed by the data set.
\item [{jitter}] adds some noise to the data in the $y$ direction in which
case the data values are not covered up by ties.
\item [{stack}] plots repeated values stacked on top of one another. This
method is best used for discrete data with a lot of ties; if there
are no repeats then this method is identical to overplot.
\end{description}
See Figure \ref{fig:Various-stripchart-methods,}, which is produced
by the following code.

<<eval = FALSE>>=
stripchart(precip, xlab="rainfall")
stripchart(rivers, method="jitter", xlab="length")
stripchart(discoveries, method="stack", xlab="number")
@

The leftmost graph is a strip chart of the \inputencoding{latin9}\lstinline[showstringspaces=false]!precip!\inputencoding{utf8}
data. The graph shows tightly clustered values in the middle with
some others falling balanced on either side, with perhaps slightly
more falling to the left. Later we will call this a symmetric distribution,
see Section \ref{sub:Shape}. The middle graph is of the \inputencoding{latin9}\lstinline[showstringspaces=false]!rivers!\inputencoding{utf8}
data, a vector of length 141. There are several repeated values in
the rivers data, and if we were to use the overplot method we would
lose some of them in the display. This plot shows a what we will later
call a right-skewed shape with perhaps some extreme values on the
far right of the display. The third graph strip charts \inputencoding{latin9}\lstinline[showstringspaces=false]!discoveries!\inputencoding{utf8}
data which are literally a textbook example of a right skewed distribution.

%
\begin{figure}
\begin{centering}
<<echo = FALSE, fig=true, height = 4.5, width = 6>>=
par(mfrow = c(1,3)) # 3 plots: 1 row, 3 columns
stripchart(precip, xlab="rainfall")
stripchart(rivers, method="jitter", xlab="length")
stripchart(discoveries, method="stack", xlab="number")
par(mfrow = c(1,1)) # back to normal
@
\par\end{centering}

\caption{Strip charts of the \texttt{precip}, \texttt{rivers}, and \texttt{discoveries}
data\label{fig:Various-stripchart-methods,}}


~

{\small The first graph uses the }\texttt{\small overplot}{\small{}
method, the second the }\texttt{\small jitter}{\small{} method, and
the third the }\texttt{\small stack}{\small{} method.}
\end{figure}


The \inputencoding{latin9}\lstinline[showstringspaces=false]!DOTplot!\inputencoding{utf8}\index{DOTplot@\texttt{DOTplot}}
function in the \inputencoding{latin9}\lstinline[showstringspaces=false]!UsingR!\inputencoding{utf8}\index{R packages!UsingR@\texttt{UsingR}}
package \cite{usingr} is another alternative.


\paragraph*{Histogram\index{Histogram}}

These are typically used for continuous data. A histogram is constructed
by first deciding on a set of classes, or bins, which partition the
real line into a set of boxes into which the data values fall. Then
vertical bars are drawn over the bins with height proportional to
the number of observations that fell into the bin. 

These are one of the most common summary displays, and they are often
misidentified as {}``Bar Graphs'' (see below.) The scale on the
$y$ axis can be frequency, percentage, or density (relative frequency).
The term histogram was coined by Karl Pearson in 1891, see \cite{Miller}.
\begin{example}
\textbf{Annual Precipitation in US Cities.} We are going to take another
look at the \inputencoding{latin9}\lstinline[showstringspaces=false]!precip!\inputencoding{utf8}\index{Data sets!precip@\texttt{precip}}
data that we investigated earlier. The strip chart in Figure \ref{fig:Various-stripchart-methods,}
suggested a loosely balanced distribution; let us now look to see
what a histogram says. 

There are many ways to plot histograms in \textsf{R}, and one of the
easiest is with the \inputencoding{latin9}\lstinline[showstringspaces=false]!hist!\inputencoding{utf8}\index{hist@\texttt{hist}}
function. The following code produces the plots in Figure \ref{fig:histograms}.

<<eval = FALSE>>=
hist(precip, main = "")
hist(precip, freq = FALSE, main = "")
@

Notice the argument \inputencoding{latin9}\lstinline[showstringspaces=false]!main = ""!\inputencoding{utf8},
which suppresses the main title from being displayed -- it would have
said {}``Histogram of \texttt{precip}'' otherwise. The plot on the
left is a frequency histogram (the default), and the plot on the right
is a relative frequency histogram (\inputencoding{latin9}\lstinline[showstringspaces=false]!freq = FALSE!\inputencoding{utf8}). 

%
\begin{figure}
\begin{centering}
<<echo = FALSE, fig=true, height = 4.5, width = 6>>=
par(mfrow = c(1,2)) # 2 plots: 1 row, 2 columns
hist(precip, main = "")
hist(precip, freq = FALSE, main = "")
par(mfrow = c(1,1)) # back to normal
@
\par\end{centering}

\caption{(Relative) frequency histograms of the \texttt{precip} data\label{fig:histograms}}

\end{figure}


\end{example}
Please be careful regarding the biggest weakness of histograms: the
graph obtained strongly depends on the bins chosen. Choose another
set of bins, and you will get a different histogram. Moreover, there
are not any definitive criteria by which bins should be defined; the
best choice for a given data set is the one which illuminates the
data set's underlying structure (if any). Luckily for us there are
algorithms to automatically choose bins that are likely to display
well, and more often than not the default bins do a good job. This
is not always the case, however, and a responsible statistician will
investigate many bin choices to test the stability of the display.
\begin{example}
Recall that the strip chart in Figure \ref{fig:Various-stripchart-methods,}
suggested a relatively balanced shape to the \inputencoding{latin9}\lstinline[showstringspaces=false]!precip!\inputencoding{utf8}
data distribution. Watch what happens when we change the bins slightly
(with the \inputencoding{latin9}\lstinline[showstringspaces=false]!breaks!\inputencoding{utf8}
argument to \inputencoding{latin9}\lstinline[showstringspaces=false]!hist!\inputencoding{utf8}).
See Figure \ref{fig:histograms-bins} which was produced by the following
code.

<<eval = FALSE>>=
hist(precip, breaks = 10, main = "")
hist(precip, breaks = 200, main = "")
@

%
\begin{figure}
\begin{centering}
<<echo = FALSE, fig=true, height = 4.5, width = 6>>=
par(mfrow = c(1,2)) # 2 plots: 1 row, 2 columns
hist(precip, breaks = 10, main = "")
hist(precip, breaks = 200, main = "")
par(mfrow = c(1,1)) # back to normal
@
\par\end{centering}

\caption{More histograms of the \texttt{precip} data\label{fig:histograms-bins}}

\end{figure}
The leftmost graph (with \inputencoding{latin9}\lstinline[showstringspaces=false]!breaks = 10!\inputencoding{utf8})
shows that the distribution is not balanced at all. There are two
humps: a big one in the middle and a smaller one to the left. Graphs
like this often indicate some underlying group structure to the data;
we could now investigate whether the cities for which rainfall was
measured were similar in some way, with respect to geographic region,
for example.

The rightmost graph in Figure \ref{fig:histograms-bins} shows what
happens when the number of bins is too large: the histogram is too
grainy and hides the rounded appearance of the earlier histograms.
If we were to continue increasing the number of bins we would eventually
get all observed bins to have exactly one element, which is nothing
more than a glorified strip chart.

\end{example}

\paragraph*{Stemplots (more to be said in Section \ref{sec:Exploratory-Data-Analysis})}

Stemplots have two basic parts: \emph{stems} and \emph{leaves}. The
final digit of the data values is taken to be a \emph{leaf}, and the
leading digit(s) is (are) taken to be \emph{stems}. We draw a vertical
line, and to the left of the line we list the stems. To the right
of the line, we list the leaves beside their corresponding stem. There
will typically be several leaves for each stem, in which case the
leaves accumulate to the right. It is sometimes necessary to round
the data values, especially for larger data sets.
\begin{example}
\label{exa:-ukdriverdeaths-first}\inputencoding{latin9}\lstinline[showstringspaces=false]!UKDriverDeaths!\inputencoding{utf8}\index{Data sets!UKDriverDeaths@\texttt{UKDriverDeaths}}
is a time series that contains the total car drivers killed or seriously
injured in Great Britain monthly from Jan 1969 to Dec 1984. See \inputencoding{latin9}\lstinline[showstringspaces=false]!?UKDriverDeaths!\inputencoding{utf8}.
Compulsory seat belt use was introduced on January 31, 1983. We construct
a stem and leaf diagram in \textsf{R} with the \inputencoding{latin9}\lstinline[showstringspaces=false]!stem.leaf!\inputencoding{utf8}\index{stem.leaf@\texttt{stem.leaf}}
function from the \inputencoding{latin9}\lstinline[showstringspaces=false]!aplpack!\inputencoding{utf8}\index{R packages@\textsf{R} packages!aplpack@\texttt{aplpack}}
package \cite{aplpack}.
\end{example}
<<>>=
library(aplpack)
stem.leaf(UKDriverDeaths, depth = FALSE)
@

The display shows a more or less balanced mound-shaped distribution,
with one or maybe two humps, a big one and a smaller one just to its
right. Note that the data have been rounded to the tens place so that
each datum gets only one leaf to the right of the dividing line.

Notice that the \inputencoding{latin9}\lstinline[showstringspaces=false]!depth!\inputencoding{utf8}s\index{depths}
have been suppressed. To learn more about this option and many others,
see Section \ref{sec:Exploratory-Data-Analysis}. Unlike a histogram,
the original data values may be recovered from the stemplot display
-- modulo the rounding -- that is, starting from the top and working
down we can read off the data values 1050, 1070, 1110, 1130, \emph{etc}. 


\paragraph*{Index plot}

Done with the \inputencoding{latin9}\lstinline[showstringspaces=false]!plot!\inputencoding{utf8}\index{plot@\texttt{plot}}
function. These are good for plotting data which are ordered, for
example, when the data are measured over time. That is, the first
observation was measured at time 1, the second at time 2, \emph{etc}.
It is a two dimensional plot, in which the index (or time) is the
$x$ variable and the measured value is the $y$ variable. There are
several plotting methods for index plots, and we discuss two of them:
\begin{description}
\item [{spikes:}] draws a vertical line from the $x$-axis to the observation
height (\inputencoding{latin9}\lstinline[showstringspaces=false]!type = "h"!\inputencoding{utf8}).
\item [{points:}] plots a simple point at the observation height (\inputencoding{latin9}\lstinline[showstringspaces=false]!type = "p"!\inputencoding{utf8}).\end{description}
\begin{example}
\textbf{Level of Lake Huron 1875-1972.} Brockwell and Davis \cite{Brockwell1991}
give the annual measurements of the level (in feet) of Lake Huron
from 1875--1972. The data are stored in the time series \inputencoding{latin9}\lstinline[showstringspaces=false]!LakeHuron!\inputencoding{utf8}\index{Data sets!LakeHuron@\texttt{LakeHuron}}.
See \inputencoding{latin9}\lstinline[showstringspaces=false]!?LakeHuron!\inputencoding{utf8}.
Figure \ref{fig:indpl-lakehuron} was produced with the following
code:

<<>>=
plot(LakeHuron, type = "h")
plot(LakeHuron, type = "p")
@

The plots show an overall decreasing trend to the observations, and
there appears to be some seasonal variation that increases over time.

%
\begin{figure}
\begin{centering}
<<echo = FALSE, fig=true, height = 8, width = 6>>=
par(mfrow = c(2,1)) # 2 plots: 1 row, 2 columns
plot(LakeHuron, type = "h")
plot(LakeHuron, type = "p")
par(mfrow = c(1,1)) # back to normal
@
\par\end{centering}

\caption{Index plots of the \texttt{LakeHuron} data\label{fig:indpl-lakehuron}}

\end{figure}


\end{example}

\paragraph*{Density estimates}


\subsection{Qualitative Data, Categorical Data, and Factors\label{sub:Qualitative-Data}}

Qualitative data are simply any type of data that are not numerical,
or do not represent numerical quantities. Examples of qualitative
variables include a subject's name, gender, race/ethnicity, political
party, socioeconomic status, class rank, driver's license number,
and social security number (SSN).

Please bear in mind that some data \emph{look} to be quantitative
but are \emph{not,} because they do not represent numerical quantities
and do not obey mathematical rules. For example, a person's shoe size
is typically written with numbers: 8, or 9, or 12, or $12\,\frac{1}{2}$.
Shoe size is not quantitative, however, because if we take a size
8 and combine with a size 9 we do not get a size 17.

Some qualitative data serve merely to \emph{identify} the observation
(such a subject's name, driver's license number, or SSN). This type
of data does not usually play much of a role in statistics. But other
qualitative variables serve to \emph{subdivide} the data set into
categories; we call these \emph{factors}. In the above examples, gender,
race, political party, and socioeconomic status would be considered
factors (shoe size would be another one). The possible values of a
factor are called its \emph{levels}. For instance, the factor \emph{gender}
would have two levels, namely, male and female. Socioeconomic status
typically has three levels: high, middle, and low.

Factors may be of two types: \emph{nominal}\index{nominal data} and
\emph{ordinal}\index{ordinal data}. Nominal factors have levels that
correspond to names of the categories, with no implied ordering. Examples
of nominal factors would be hair color, gender, race, or political
party. There is no natural ordering to {}``Democrat'' and {}``Republican'';
the categories are just names associated with different groups of
people. 

In contrast, ordinal factors have some sort of ordered structure to
the underlying factor levels. For instance, socioeconomic status would
be an ordinal categorical variable because the levels correspond to
ranks associated with income, education, and occupation. Another example
of ordinal categorical data would be class rank.

Factors have special status in \textsf{R}. They are represented internally
by numbers, but even when they are written numerically their values
do not convey any numeric meaning or obey any mathematical rules (that
is, Stage III cancer is not Stage I cancer + Stage II cancer).
\begin{example}
The \inputencoding{latin9}\lstinline[showstringspaces=false]!state.abb!\inputencoding{utf8}\index{Data sets!state.abb@\texttt{state.abb}}
vector gives the two letter postal abbreviations for all 50 states.

<<>>=
str(state.abb)
@

These would be ID data. The \inputencoding{latin9}\lstinline[showstringspaces=false]!state.name!\inputencoding{utf8}\index{Data sets!state.name@\texttt{state.name}}
vector lists all of the complete names and those data would also be
ID.

\end{example}

\begin{example}
\textbf{U.S.~State Facts and Features.} The U.S.~Department of Commerce
of the U.S.~Census Bureau releases all sorts of information in the
\emph{Statistical Abstract of the United States}, and the \inputencoding{latin9}\lstinline[showstringspaces=false]!state.region!\inputencoding{utf8}\index{Data sets!state.region@\texttt{state.region}}
data lists each of the 50 states and the region to which it belongs,
be it Northeast, South, North Central, or West. See \inputencoding{latin9}\lstinline[showstringspaces=false]!?state.region!\inputencoding{utf8}.

<<>>=
str(state.region)
state.region[1:5]
@

The \inputencoding{latin9}\lstinline[showstringspaces=false]!str!\inputencoding{utf8}\index{str@\texttt{str}}
output shows that \inputencoding{latin9}\lstinline[showstringspaces=false]!state.region!\inputencoding{utf8}
is already stored internally as a factor and it lists a couple of
the factor levels. To see all of the levels we printed the first five
entries of the vector in the second line.need to print a piece of
the from 
\end{example}



\subsection*{Displaying Qualitative Data\label{sub:Displaying-Qualitative-Data}}


\paragraph*{Tables\label{par:Tables}}

One of the best ways to summarize qualitative data is with a table
of the data values. We may count frequencies with the \inputencoding{latin9}\lstinline[showstringspaces=false]!table!\inputencoding{utf8}
function or list proportions with the \inputencoding{latin9}\lstinline[showstringspaces=false]!prop.table!\inputencoding{utf8}\index{prop.table@\texttt{prop.table}}
function (whose input is a frequency table). In the \textsf{R} Commander
you can do it with \textsf{Statistics} \textsf{$\triangleright$}
\textsf{Frequency Distribution\ldots{}.} Alternatively, to look at
tables for all factors in the \inputencoding{latin9}\lstinline[showstringspaces=false]!Active data set!\inputencoding{utf8}\index{Active data set@\texttt{Active data set}}
you can do \textsf{Statistics} \textsf{$\triangleright$} \textsf{Summaries}
\textsf{$\triangleright$} \textsf{Active Dataset}.

<<keep.source = TRUE>>=
Tbl <- table(state.division)
Tbl               # frequencies
Tbl/sum(Tbl)      # relative frequencies
prop.table(Tbl)   # same thing
@


\paragraph*{Bar Graphs\label{par:Bar-Graphs}}

A bar graph is the analogue of a histogram for categorical data. A
bar is displayed for each level of a factor, with the heights of the
bars proportional to the frequencies of observations falling in the
respective categories. A disadvantage of bar graphs is that the levels
are ordered alphabetically (by default), which may sometimes obscure
patterns in the display. 
\begin{example}
\textbf{U.S.~State Facts and Features.} The \inputencoding{latin9}\lstinline[showstringspaces=false]!state.region!\inputencoding{utf8}
data lists each of the 50 states and the region to which it belongs,
be it Northeast, South, North Central, or West. See \inputencoding{latin9}\lstinline[showstringspaces=false]!?state.region!\inputencoding{utf8}.
It is already stored internally as a factor. We make a bar graph with
the \inputencoding{latin9}\lstinline[showstringspaces=false]!barplot!\inputencoding{utf8}\index{barplot@\texttt{barplot}}
function: 

<<eval = FALSE>>=
barplot(table(state.region), cex.names = 0.50)
barplot(prop.table(table(state.region)), cex.names = 0.50)
@

See Figure \ref{fig:bar-gr-stateregion}. The display on the left
is a frequency bar graph because the $y$ axis shows counts, while
the display on the left is a relative frequency bar graph. The only
difference between the two is the scale. Looking at the graph we see
that the majority of the fifty states are in the South, followed by
West, North Central, and finally Northeast. Over 30\% of the states
are in the South.

Notice the \inputencoding{latin9}\lstinline[showstringspaces=false]!cex.names!\inputencoding{utf8}\index{cex.names@\texttt{cex.names}}
argument that we used, above. It shrinks the names on the $x$ axis
by 50\% which makes them easier to read. See \inputencoding{latin9}\lstinline[showstringspaces=false]!?par!\inputencoding{utf8}\index{par@\texttt{par}}
for a detailed list of additional plot parameters.

%
\begin{figure}
\begin{centering}
<<echo = FALSE, fig=true, height = 4.5, width = 6>>=
par(mfrow = c(1,2)) # 2 plots: 1 row, 2 columns
barplot(table(state.region), cex.names = 0.50)
barplot(prop.table(table(state.region)), cex.names = 0.50)
par(mfrow = c(1,1))
@
\par\end{centering}

\caption{Bar graphs of the \texttt{state.region} data\label{fig:bar-gr-stateregion}}


~

{\small The left graph is a frequency barplot made with }\texttt{\small table}{\small{}
and the right is a relative frequency barplot made with }\texttt{\small prop.table}{\small .}
\end{figure}


\end{example}

\paragraph*{Pareto Diagrams\label{par:Pareto-Diagrams}}

A pareto diagram is a lot like a bar graph except the bars are rearranged
such that they decrease in height going from left to right. The rearrangement
is handy because it can visually reveal structure (if any) in how
fast the bars decrease -- this is much more difficult when the bars
are jumbled. 
\begin{example}
U.S.~State Facts and Features. The \inputencoding{latin9}\lstinline[showstringspaces=false]!state.division!\inputencoding{utf8}\index{Data sets!state.division@\texttt{state.division}}
data record the division (New England, Middle Atlantic, South Atlantic,
East South Central, West South Central, East North Central, West North
Central, Mountain, and Pacific) of the fifty states. We can make a
pareto diagram with either the \inputencoding{latin9}\lstinline[showstringspaces=false]!RcmdrPlugin.IPSUR!\inputencoding{utf8}\index{R packages@\textsf{R} packages!RcmdrPlugin.IPSUR@\texttt{RcmdrPlugin.IPSUR}}
package or with the \inputencoding{latin9}\lstinline[showstringspaces=false]!pareto.chart!\inputencoding{utf8}\index{pareto.chart@\texttt{pareto.chart}}
function from the \inputencoding{latin9}\lstinline[showstringspaces=false]!qcc!\inputencoding{utf8}\index{R packages@\textsf{R} packages!qcc@\texttt{qcc}}
package \cite{qcc}. See Figure \ref{fig:Pareto-chart}. The code
follows.

<<eval = FALSE>>=
library(qcc)
pareto.chart(table(state.division), ylab="Frequency")
@

%
\begin{figure}
\begin{centering}
<<echo = FALSE, fig=true, height = 4.5, width = 6>>=
library(qcc)
pareto.chart(table(state.division), ylab="Frequency")
@
\par\end{centering}

\caption{Pareto chart of the \texttt{state.division} data\label{fig:Pareto-chart}}

\end{figure}


\end{example}

\paragraph*{Dot Charts\label{par:Dotcharts}}

These are a lot like a bar graph that has been turned on its side
with the bars replaced by dots on horizontal lines. They do not convey
any more (or less) information than the associated bar graph, but
the strength lies in the economy of the display. Dot charts are so
compact that it is easy to graph very complicated multi-variable interactions
together in one graph. See Section \ref{sec:Comparing-Data-Sets}.
We will give an example here using the same data as above for comparison.
The graph was produced by the following code.

<<eval = FALSE>>=
x <- table(state.region)
dotchart(as.vector(x), labels = names(x))
@

%
\begin{figure}
\begin{centering}
<<echo = FALSE, fig=true, height = 4.5, width = 6>>=
x <- table(state.region)
dotchart(as.vector(x), labels = names(x))
@
\par\end{centering}

\caption{Dot chart of the \texttt{state.region} data\label{fig:dot-charts}}

\end{figure}


See Figure \ref{fig:dot-charts}. Compare it to Figure \ref{fig:bar-gr-stateregion}.


\paragraph*{Pie Graphs\label{par:Pie-Graphs}}

These can be done with \textsf{R} and the \textsf{R} Commander, but
they fallen out of favor in recent years because researchers have
determined that while the human eye is good at judging linear measures,
it is notoriously bad at judging relative areas (such as those displayed
by a pie graph). Pie charts are consequently a very bad way of displaying
information. A bar chart or dot chart is a preferable way of displaying
qualitative data. See \inputencoding{latin9}\lstinline[showstringspaces=false]!?pie!\inputencoding{utf8}\index{pie@\texttt{pie}}
for more information.

We are not going to do any examples of a pie graph and discourage
their use elsewhere. 


\subsection{Logical Data\label{sub:Logical-Data}}

There is another type of information recognized by \textsf{R} which
does not fall into the above categories. The value is either \inputencoding{latin9}\lstinline[showstringspaces=false]!TRUE!\inputencoding{utf8}
or \inputencoding{latin9}\lstinline[showstringspaces=false]!FALSE!\inputencoding{utf8}
(note that equivalently you can use \inputencoding{latin9}\lstinline[showstringspaces=false]!1 = TRUE!\inputencoding{utf8},
\inputencoding{latin9}\lstinline[showstringspaces=false]!0 = FALSE!\inputencoding{utf8}).
Here is an example of a logical vector:

<<>>=
x <- 5:9
y <- (x < 7.3)
y
@

Many functions in \textsf{R} have options that the user may or may
not want to activate in the function call. For example, the \inputencoding{latin9}\lstinline[showstringspaces=false]!stem.leaf!\inputencoding{utf8}
function has the \inputencoding{latin9}\lstinline[showstringspaces=false]!depths!\inputencoding{utf8}
argument which is \inputencoding{latin9}\lstinline[showstringspaces=false]!TRUE!\inputencoding{utf8}
by default. We saw in Section \ref{sub:Quantitative-Data} how to
turn the option off, simply enter \inputencoding{latin9}\lstinline[showstringspaces=false]!stem.leaf(x, depths = FALSE)!\inputencoding{utf8}
and they will not be shown on the display.

We can swap \inputencoding{latin9}\lstinline[showstringspaces=false]!TRUE!\inputencoding{utf8}
with \inputencoding{latin9}\lstinline[showstringspaces=false]!FALSE!\inputencoding{utf8}
with the exclamation point \inputencoding{latin9}\lstinline[showstringspaces=false]*!*\inputencoding{utf8}.

<<>>=
!y
@


\subsection{Missing Data\label{sub:Missing-Data}}

Missing data are a persistent and prevalent problem in many statistical
analyses, especially those associated with the social sciences. \textsf{R}
reserves the special symbol \inputencoding{latin9}\lstinline[showstringspaces=false]!NA!\inputencoding{utf8}
to representing missing data.

Ordinary arithmetic with \inputencoding{latin9}\lstinline[showstringspaces=false]!NA!\inputencoding{utf8}
values give \inputencoding{latin9}\lstinline[showstringspaces=false]!NA!\inputencoding{utf8}'s
(addition, subtraction, \emph{etc}.) and applying a function to a
vector that has an \inputencoding{latin9}\lstinline[showstringspaces=false]!NA!\inputencoding{utf8}
in it will usually give an \inputencoding{latin9}\lstinline[showstringspaces=false]!NA!\inputencoding{utf8}.

<<>>=
x <- c(3, 7, NA, 4, 7)
y <- c(5, NA, 1, 2, 2)
x + y
@

Some functions have a \inputencoding{latin9}\lstinline[showstringspaces=false]!na.rm!\inputencoding{utf8}
argument which when \inputencoding{latin9}\lstinline[showstringspaces=false]!TRUE!\inputencoding{utf8}
will ignore missing data as if it were not there (such as \inputencoding{latin9}\lstinline[showstringspaces=false]!mean!\inputencoding{utf8},
\inputencoding{latin9}\lstinline[showstringspaces=false]!var!\inputencoding{utf8},
\inputencoding{latin9}\lstinline[showstringspaces=false]!sd!\inputencoding{utf8},
\inputencoding{latin9}\lstinline[showstringspaces=false]!IQR!\inputencoding{utf8},
\inputencoding{latin9}\lstinline[showstringspaces=false]!mad!\inputencoding{utf8},
\ldots{}). 

<<>>=
sum(x)
sum(x, na.rm = TRUE)
@

Other functions do not have a \inputencoding{latin9}\lstinline[showstringspaces=false]!na.rm!\inputencoding{utf8}
argument and will return \inputencoding{latin9}\lstinline[showstringspaces=false]!NA!\inputencoding{utf8}
or an error if the argument has \inputencoding{latin9}\lstinline[showstringspaces=false]!NA!\inputencoding{utf8}s.
In those cases we can find the locations of any \inputencoding{latin9}\lstinline[showstringspaces=false]!NA!\inputencoding{utf8}s
with the \inputencoding{latin9}\lstinline[showstringspaces=false]!is.na!\inputencoding{utf8}
function and remove those cases with the \inputencoding{latin9}\lstinline[showstringspaces=false]![]!\inputencoding{utf8}
operator.

<<>>=
is.na(x)
z <- x[!is.na(x)]
sum(z)
@

The analogue of \inputencoding{latin9}\lstinline[showstringspaces=false]!is.na!\inputencoding{utf8}
for rectangular data sets (or data frames) is the \inputencoding{latin9}\lstinline[showstringspaces=false]!complete.cases!\inputencoding{utf8}
function. See Appendix \ref{sec:Editing-Data-Sets}.


\subsection{Other Data Types\label{sub:other-data-types}}


\section{Features of Data Distributions\label{sec:features-of-data}}

Given that the data have been appropriately displayed, the next step
is to try to identify salient features represented in the graph. The
acronym to remember is \emph{C}enter, \emph{U}nusual features, \emph{S}pread,
and \emph{S}hape. (CUSS).


\subsection{Center\label{sub:Center}}

One of the most basic features of a data set is its center. Loosely
speaking, the center of a data set is associated with a number that
represents a middle or general tendency of the data. Of course, there
are usually several values that would serve as a center, and our later
tasks will be focused on choosing an appropriate one for the data
at hand. Judging from the histogram that we saw in Figure \ref{fig:histograms-bins},
a measure of center would be about \Sexpr{round(mean(precip))}. 


\subsection{Spread\label{sub:Spread}}

The spread of a data set is associated with its variability; data
sets with a large spread tend to cover a large interval of values,
while data sets with small spread tend to cluster tightly around a
central value. 


\subsection{Shape\label{sub:Shape}}

When we speak of the \emph{shape} of a data set, we are usually referring
to the shape exhibited by an associated graphical display, such as
a histogram. The shape can tell us a lot about any underlying structure
to the data, and can help us decide which statistical procedure we
should use to analyze them.


\paragraph*{Symmetry and Skewness}

A distribution is said to be \emph{right-skewed} (or \emph{positively
skewed}) if the right tail seems to be stretched from the center.
A \emph{left-skewed} (or \emph{negatively skewed}) distribution is
stretched to the left side. A symmetric distribution has a graph that
is balanced about its center, in the sense that half of the graph
may be reflected about a central line of symmetry to match the other
half.

We have already encountered skewed distributions: both the discoveries
data in Figure \ref{fig:Various-stripchart-methods,} and the precip
data in Figure \ref{fig:histograms-bins} appear right-skewed. The
\inputencoding{latin9}\lstinline[showstringspaces=false]!UKDriverDeaths!\inputencoding{utf8}
data in Example \ref{exa:-ukdriverdeaths-first} is relatively symmetric
(but note the one extreme value 2654 identified at the bottom of the
stemplot).


\paragraph*{Kurtosis}

Another component to the shape of a distribution is how {}``peaked''
it is. Some distributions tend to have a flat shape with thin tails.
These are called \emph{platykurtic}, and an example of a platykurtic
distribution is the uniform distribution; see Section \ref{sec:The-Continuous-Uniform}.
On the other end of the spectrum are distributions with a steep peak,
or spike, accompanied by heavy tails; these are called \emph{leptokurtic}.
Examples of leptokurtic distributions are the Laplace distribution
and the logistic distribution. See Section \ref{sec:Other-Continuous-Distributions}.
In between are distributions (called \emph{mesokurtic}) with a rounded
peak and moderately sized tails. The standard example of a mesokurtic
distribution is the famous bell-shaped curve, also known as the Gaussian,
or normal, distribution, and the binomial distribution can be mesokurtic
for specific choices of $p$. See Sections \ref{sec:binom-dist} and
\ref{sec:The-Normal-Distribution}.


\subsection{Clusters and Gaps\label{sub:clusters-and-gaps}}

Clusters or gaps are sometimes observed in quantitative data distributions.
They indicate clumping of the data about distinct values, and gaps
may exist between clusters. Clusters often suggest an underlying grouping
to the data. For example, take a look at the \inputencoding{latin9}\lstinline[showstringspaces=false]!faithful!\inputencoding{utf8}
data which contains the duration of \inputencoding{latin9}\lstinline[showstringspaces=false]!eruptions!\inputencoding{utf8}
and the \inputencoding{latin9}\lstinline[showstringspaces=false]!waiting!\inputencoding{utf8}
time between eruptions of the Old Faithful geyser in Yellowstone National
Park. (Do not be frightened by the complicated information at the
left of the display for now; we will learn how to interpret it in
Section \ref{sec:Exploratory-Data-Analysis}).\label{exa:stemleaf-multiple-lines-stem}

<<>>=
library(aplpack)
stem.leaf(faithful$eruptions)
@

There are definitely two clusters of data here; an upper cluster and
a lower cluster. 


\subsection{Extreme Observations and other Unusual Features\label{sub:Extreme-Observations-and}}

Extreme observations fall far from the rest of the data. Such observations
are troublesome to many statistical procedures; they cause exaggerated
estimates and instability. It is important to identify extreme observations
and examine the source of the data more closely. There are many possible
reasons underlying an extreme observation:
\begin{itemize}
\item \textbf{Maybe the value is a typographical error.} Especially with
large data sets becoming more prevalent, many of which being recorded
by hand, mistakes are a common problem. After closer scrutiny, these
can often be fixed.
\item \textbf{Maybe the observation was not meant for the study}, because
it does not belong to the population of interest. For example, in
medical research some subjects may have relevant complications in
their genealogical history that would rule out their participation
in the experiment. Or when a manufacturing company investigates the
properties of one of its devices, perhaps a particular product is
malfunctioning and is not representative of the majority of the items.
\item \textbf{Maybe it indicates a deeper trend or phenomenon}. Many of
the most influential scientific discoveries were made when the investigator
noticed an unexpected result, a value that was not predicted by the
classical theory. Albert Einstein, Louis Pasteur, and others built
their careers on exactly this circumstance.
\end{itemize}

\section{Descriptive Statistics\label{sec:Descriptive-Statistics}}


\subsection{Frequencies and Relative Frequencies\label{sub:Frequencies-and-Relative}}

These are used for categorical data. The idea is that there are a
number of different categories, and we would like to get some idea
about how the categories are represented in the population. For example,
we may want to see how the 


\subsection{Measures of Center\label{sub:Measures-of-Center}}

The \emph{sample mean} is denoted $\xbar$ (read {}``$x$-bar'')
and is simply the arithmetic average of the observations:\begin{equation}
\xbar=\frac{x_{1}+x_{2}+\cdots+x_{n}}{n}=\frac{1}{n}\sum_{i=1}^{n}x_{i}.\end{equation}

\begin{itemize}
\item Good: natural, easy to compute, has nice mathematical properties
\item Bad: sensitive to extreme values
\end{itemize}
It is appropriate for use with data sets that are not highly skewed
without extreme observations.

The \emph{sample median} is another popular measure of center and
is denoted $\tilde{x}$. To calculate its value, first sort the data
into an increasing sequence of numbers. If the data set has an odd
number of observations then $\tilde{x}$ is the value of the middle
observation, which lies in position $(n+1)/2$; otherwise, there are
two middle observations and $\tilde{x}$ is the average of those middle
values.
\begin{itemize}
\item Good: resistant to extreme values, easy to describe
\item Bad: not as mathematically tractable, need to sort the data to calculate
\end{itemize}
One desirable property of the sample median is that it is resistant
to extreme observations, in the sense that the value of $\tilde{x}$
depends only the values of the middle observations, and is quite unaffected
by the actual values of the outer observations in the ordered list.
The same cannot be said for the sample mean. Any significant changes
in the magnitude of an observation $x_{k}$ results in a corresponding
change in the value of the mean. Hence, the sample mean is said to
be sensitive to extreme observations.

The \emph{trimmed mean} is a measure designed to address the sensitivity
of the sample mean to extreme observations. The idea is to {}``trim''
a fraction (less than 1/2) of the observations off each end of the
ordered list, and then calculate the sample mean of what remains.
We will denote it by $\xbar_{t=0.05}$.
\begin{itemize}
\item Good: resistant to extreme values, shares nice statistical properties
\item Bad: need to sort the data
\end{itemize}

\subsection{How to do it with \textsf{R}}
\begin{itemize}
\item You can calculate frequencies or relative frequencies with the \inputencoding{latin9}\lstinline[showstringspaces=false]!table!\inputencoding{utf8}
function, and relative frequencies with \inputencoding{latin9}\lstinline[showstringspaces=false]!prop.table(table())!\inputencoding{utf8}.
\item You can calculate the sample mean of a data vector \inputencoding{latin9}\lstinline[showstringspaces=false]!x!\inputencoding{utf8}
with the command \inputencoding{latin9}\lstinline[showstringspaces=false]!mean(x)!\inputencoding{utf8}. 
\item You can calculate the sample median of \inputencoding{latin9}\lstinline[showstringspaces=false]!x!\inputencoding{utf8}
with the command \inputencoding{latin9}\lstinline[showstringspaces=false]!median(x)!\inputencoding{utf8}.
\item You can calculate the trimmed mean with the \inputencoding{latin9}\lstinline[showstringspaces=false]!trim!\inputencoding{utf8}
argument; \inputencoding{latin9}\lstinline[showstringspaces=false]!mean(x, trim = 0.05)!\inputencoding{utf8}.
\end{itemize}

\subsection{Order Statistics and the Sample Quantiles\label{sub:Order-Statistics-and}}

A common first step in an analysis of a data set is to sort the values.
Given a data set $x_{1}$, $x_{2}$, \ldots{},$x_{n}$, we may sort
the values to obtain an increasing sequence\begin{equation}
x_{(1)}\leq x_{(2)}\leq x_{(3)}\leq\cdots\leq x_{(n)}\end{equation}
and the resulting values are called the \emph{order statistics}. The
$k$$^{\text{th}}$ entry in the list, $x_{(k)}$, is the $k$$^{\text{th}}$
order statistic, and approximately $100(k/n)$\% of the observations
fall below $x_{(k)}$. The order statistics give an indication of
the shape of the data distribution, in the sense that a person can
look at the order statistics and have an idea about where the data
are concentrated, and where they are sparse.

The \emph{sample quantiles} are related to the order statistics. Unfortunately,
there is not a universally accepted definition of them. Indeed, \textsf{R}
is equipped to calculate quantiles using nine distinct definitions!
We will describe the default method (\inputencoding{latin9}\lstinline[showstringspaces=false]!type = 7!\inputencoding{utf8}),
but the interested reader can see the details for the other methods
with \inputencoding{latin9}\lstinline[showstringspaces=false]!?quantile!\inputencoding{utf8}.

Suppose the data set has $n$ observations. Find the sample quantile
of order $p$ ($0<p<1$), denoted $\tilde{q}_{p}$ , as follows:
\begin{description}
\item [{First~step:}] sort the data to obtain the order statistics $x_{(1)}$,
$x_{(2)}$, \ldots{},$x_{(n)}$. 
\item [{Second~step:}] calculate $(n-1)p+1$ and write it in the form
$k.d$, where $k$ is an integer and $d$ is a decimal.
\item [{Third~step:}] The sample quantile $\tilde{q}_{p}$ is\begin{equation}
\tilde{q}_{p}=x_{(k)}+d(x_{(k+1)}-x_{(k)}).\end{equation}

\end{description}
The interpretation of $\tilde{q}_{p}$ is that approximately $100p$\%
of the data fall below the value $\tilde{q}_{p}$ . 

Keep in mind that there is not a unique definition of percentiles,
quartiles, \emph{etc}. Open a different book, and you'll find a different
procedure. The difference is small and seldom plays a role except
in small data sets with repeated values. In fact, most people do not
even notice in common use.

Clearly, the most popular sample quantile is $\tilde{q}_{0.50}$,
also known as the sample median, $\tilde{x}$. The closest runners-up
are the \emph{first quartile} $\tilde{q}_{0.25}$ and the \emph{third
quartile} $\tilde{q}_{0.75}$ (the \emph{second quartile} is the median). 


\subsection{How to do it with \textsf{R}}


\paragraph*{At the command prompt}

We can find the order statistics of a data set stored in a vector
\inputencoding{latin9}\lstinline[showstringspaces=false]!x!\inputencoding{utf8}
with the command \inputencoding{latin9}\lstinline[showstringspaces=false]!sort(x)!\inputencoding{utf8}. 

You can calculate the sample quantiles of any order $p$ where $0<p<1$
for a data set stored in a data vector \inputencoding{latin9}\lstinline[showstringspaces=false]!x!\inputencoding{utf8}
with the \inputencoding{latin9}\lstinline[showstringspaces=false]!quantile!\inputencoding{utf8}
function, for instance, the command \inputencoding{latin9}\lstinline[showstringspaces=false]!quantile(x, probs = c(0, 0.25, 0.37))!\inputencoding{utf8}
will return the smallest observation, the first quartile, $\tilde{q}_{0.25}$,
and the 37th sample quantile, $\tilde{q}_{0.37}$. For $\tilde{q}_{p}$
simply change the values in the \inputencoding{latin9}\lstinline[showstringspaces=false]!probs!\inputencoding{utf8}
argument to the value $p$.


\paragraph*{With the \textsf{R} Commander}

In \inputencoding{latin9}\lstinline[showstringspaces=false]!Rcmdr!\inputencoding{utf8}
we can find the order statistics of a variable in the \inputencoding{latin9}\lstinline[showstringspaces=false]!Active data set!\inputencoding{utf8}
by doing \textsf{Data $\triangleright$ Manage variables in Active
data set\ldots{} $\triangleright$ Compute new variable}\textsf{\emph{\ldots{}}}.
In the \textsf{Expression to compute} dialog simply type \inputencoding{latin9}\lstinline[showstringspaces=false]!sort(varname)!\inputencoding{utf8},
where \inputencoding{latin9}\lstinline[showstringspaces=false]!varname!\inputencoding{utf8}
is the variable that it is desired to sort.

In \inputencoding{latin9}\lstinline[showstringspaces=false]!Rcmdr!\inputencoding{utf8},
we can calculate the sample quantiles for a particular variable with
the sequence \textsf{Statistics $\triangleright$ Summaries $\triangleright$
Numerical Summaries\ldots{}}. We can automatically calculate the
quartiles for all variables in the \inputencoding{latin9}\lstinline[showstringspaces=false]!Active data set!\inputencoding{utf8}
with the sequence \textsf{Statistics $\triangleright$ Summaries $\triangleright$
Active Dataset}.


\subsection{Measures of Spread\label{sub:Measures-of-Spread}}


\paragraph*{Sample Variance and Standard Deviation}

The \emph{sample variance} is denoted $s^{2}$ and is calculated with
the formula\begin{equation}
s^{2}=\frac{1}{n-1}\sum_{i=1}^{n}(x_{i}-\xbar)^{2}.\end{equation}
The \emph{sample standard deviation} is $s=\sqrt{s^{2}}$. Intuitively,
the sample variance is approximately the average squared distance
of the observations from the sample mean. The sample standard deviation
is used to scale the estimate back to the measurement units of the
original data. 
\begin{itemize}
\item Good: tractable, has nice mathematical/statistical properties
\item Bad: sensitive to extreme values
\end{itemize}
We will spend a lot of time with the variance and standard deviation
in the coming chapters. In the meantime, the following two rules give
some meaning to the standard deviation, in that there are bounds on
how much of the data can fall past a certain distance from the mean.
\begin{fact}
Chebychev's Rule: The proportion of observations within $k$ standard
deviations of the mean is at least $1-1/k^{2}$, \emph{i.e.}, at least
75\%, 89\%, and 94\% of the data are within 2, 3, and 4 standard deviations
of the mean, respectively.
\end{fact}


Note that Chebychev's Rule does not say anything about when $k=1$,
because $1-1/1^{2}=0$, which states that at least 0\% of the observations
are within one standard deviation of the mean (which is not saying
much).

Chebychev's Rule applies to any data distribution, \emph{any} list
of numbers, no matter where it came from or what the histogram looks
like. The price for such generality is that the bounds are not very
tight; if we know more about how the data are shaped then we can say
more about how much of the data can fall a given distance from the
mean.
\begin{fact}
\label{fac:Empirical-Rule}Empirical Rule: If data follow a bell-shaped
curve, then approximately 68\%, 95\%, and 99.7\% of the data are within
1, 2, and 3 standard deviations of the mean, respectively. 
\end{fact}

\paragraph*{Interquartile Range}

Just as the sample mean is sensitive to extreme values, so the associated
measure of spread is similarly sensitive to extremes. Further, the
problem is exacerbated by the fact that the extreme distances are
squared. We know that the sample quartiles are resistant to extremes,
and a measure of spread associated with them is the \emph{interquartile
range} ($IQR$) defined by $IQR=q_{0.75}-q_{0.25}$. 
\begin{itemize}
\item Good: stable, resistant to outliers, robust to nonnormality, easy
to explain
\item Bad: not as tractable, need to sort the data, only involves the middle
50\% of the data.
\end{itemize}

\paragraph*{Median Absolute Deviation }

A measure even more robust than the $IQR$ is the \emph{median absolute
deviation} ($MAD$). To calculate it we first get the median $\widetilde{x}$,
next the \emph{absolute deviations} $|x_{1}-\tilde{x}|$, $|x_{2}-\tilde{x}|$,
\ldots{}, $|x_{n}-\tilde{x}|$, and the $MAD$ is proportional to
the median of those deviations:\begin{equation}
MAD\propto\mbox{median}(|x_{1}-\tilde{x}|,\ |x_{2}-\tilde{x}|,\ldots,|x_{n}-\tilde{x}|).\end{equation}
That is, the $MAD=c\cdot\mbox{median}(|x_{1}-\tilde{x}|,\ |x_{2}-\tilde{x}|,\ldots,|x_{n}-\tilde{x}|)$,
where $c$ is a constant chosen so that the $MAD$ has nice properties.
The value of $c$ in \textsf{R} is by default $c=1.4286$. This value
is chosen to ensure that the estimator of $\sigma$ is correct, on
the average, under suitable sampling assumptions (see Section \ref{sec:Point-Estimation-1}).
\begin{itemize}
\item Good: stable, very robust, even more so than the $IQR$.
\item Bad: not tractable, not well known and less easy to explain.
\end{itemize}

\subsubsection*{Comparing Apples to Apples}

We have seen three different measures of spread which, for a given
data set, will give three different answers. Which one should we use?
It depends on the data set. If the data are well behaved, with an
approximate bell-shaped distribution, then the sample mean and sample
standard deviation are natural choices with nice mathematical properties.
However, if the data have an unusual or skewed shape with several
extreme values, perhaps the more resistant choices among the $IQR$
or $MAD$ would be more appropriate.

However, once we are looking at the three numbers it is important
to understand that the estimators are not all measuring the same quantity,
on the average. In particular, it can be shown that when the data
follow an approximately bell-shaped distribution, then on the average,
the sample standard deviation $s$ and the $MAD$ will be the approximately
the same value, namely, $\sigma$, but the $IQR$ will be on the average
1.349 times larger than $s$ and the $MAD$. See \ref{cha:Sampling-Distributions}
for more details.


\subsection{How to do it with \textsf{R}}


\paragraph*{At the command prompt}

From the console we may compute the sample range with \inputencoding{latin9}\lstinline[showstringspaces=false]!range(x)!\inputencoding{utf8}
and the sample variance with \inputencoding{latin9}\lstinline[showstringspaces=false]!var(x)!\inputencoding{utf8},
where \inputencoding{latin9}\lstinline[showstringspaces=false]!x!\inputencoding{utf8}
is a numeric vector. The sample standard deviation is \inputencoding{latin9}\lstinline[showstringspaces=false]!sqrt(var(x))!\inputencoding{utf8}
or just \inputencoding{latin9}\lstinline[showstringspaces=false]!sd(x)!\inputencoding{utf8}.
The $IQR$ is \inputencoding{latin9}\lstinline[showstringspaces=false]!IQR(x)!\inputencoding{utf8}
and the median absolute deviation is \inputencoding{latin9}\lstinline[showstringspaces=false]!mad(x)!\inputencoding{utf8}.


\paragraph*{In \textsf{R} Commander}

In \inputencoding{latin9}\lstinline[showstringspaces=false]!Rcmdr!\inputencoding{utf8}
we can calculate the sample standard deviation with the \textsf{Statistics
$\triangleright$ Summaries $\triangleright$ Numerical Summaries}\ldots{}
combination. \textsf{R} Commander does not calculate the $IQR$ or
$MAD$ in any of the menu selections, by default.


\subsection{Measures of Shape\label{sub:Measures-of-Shape}}


\paragraph*{Sample Skewness}

The \emph{sample skewness}, denoted by $g_{1}$, is defined by the
formula\begin{equation}
g_{1}=\frac{1}{n}\frac{\sum_{i=1}^{n}(x_{i}-\xbar)^{3}}{s^{3}}.\end{equation}
The sample skewness can be any value $-\infty<g_{1}<\infty$. The
sign of $g_{1}$ indicates the direction of skewness of the distribution.
Samples that have $g_{1}>0$ indicate right-skewed distributions (or
positively skewed), and samples with $g_{1}<0$ indicate left-skewed
distributions (or negatively skewed). Values of $g_{1}$ near zero
indicate a symmetric distribution. These are not hard and fast rules,
however. The value of $g_{1}$ is subject to sampling variability
and thus only provides a suggestion to the skewness of the underlying
distribution. 

We still need to know how big is {}``big'', that is, how do we judge
whether an observed value of $g_{1}$ is far enough away from zero
for the data set to be considered skewed to the right or left? A good
rule of thumb is that data sets with skewness larger than $2\sqrt{6/n}$
in magnitude are substantially skewed, in the direction of the sign
of $g_{1}$. See Tabachnick \& Fidell \cite{Tabachnick2006} for details.


\paragraph*{Sample Excess Kurtosis}

The \emph{sample excess kurtosis}, denoted by $g_{2}$, is given by
the formula\begin{equation}
g_{2}=\frac{1}{n}\frac{\sum_{i=1}^{n}(x_{i}-\xbar)^{4}}{s^{4}}-3.\end{equation}
The sample excess kurtosis takes values $-2\leq g_{2}<\infty$. The
subtraction of 3 may seem mysterious but it is done so that mound
shaped samples have values of $g_{2}$ near zero. Samples with $g_{2}>0$
are called \emph{leptokurtic}, and samples with $g_{2}<0$ are called
\emph{platykurtic}. Samples with $g_{2}\approx0$ are called \emph{mesokurtic}.

As a rule of thumb, if $|g_{2}|>4\sqrt{6/n}$ then the sample excess
kurtosis is substantially different from zero in the direction of
the sign of $g_{2}$. See Tabachnick \& Fidell \cite{Tabachnick2006}
for details.

Notice that both the sample skewness and the sample kurtosis are invariant
with respect to location and scale, that is, the values of $g_{1}$
and $g_{2}$ do not depend on the measurement units of the data.


\subsection{How to do it with \textsf{R}}

The \inputencoding{latin9}\lstinline[showstringspaces=false]!e1071!\inputencoding{utf8}
package \cite{Dimitriadoue1071} has the \inputencoding{latin9}\lstinline[showstringspaces=false]!skewness!\inputencoding{utf8}
function for the sample skewness and the \inputencoding{latin9}\lstinline[showstringspaces=false]!kurtosis!\inputencoding{utf8}
function for the sample excess kurtosis. Both functions have a \inputencoding{latin9}\lstinline[showstringspaces=false]!na.rm!\inputencoding{utf8}
argument which is \inputencoding{latin9}\lstinline[showstringspaces=false]!FALSE!\inputencoding{utf8}
by default.
\begin{example}
We said earlier that the \inputencoding{latin9}\lstinline[showstringspaces=false]!discoveries!\inputencoding{utf8}
data looked positively skewed; let's see what the statistics say:
\end{example}
<<>>=
library(e1071)
skewness(discoveries)
2*sqrt(6/length(discoveries))
@

The data are definitely skewed to the right. Let us check the sample
excess kurtosis of the \inputencoding{latin9}\lstinline[showstringspaces=false]!UKDriverDeaths!\inputencoding{utf8}
data:

<<>>=
kurtosis(UKDriverDeaths)
4*sqrt(6/length(UKDriverDeaths))
@

so that the \inputencoding{latin9}\lstinline[showstringspaces=false]!UKDriverDeaths!\inputencoding{utf8}
data appear to be mesokurtic, or at least not substantially leptokurtic.


\section{Exploratory Data Analysis\label{sec:Exploratory-Data-Analysis}}

This field was founded (mostly) by John Tukey (1915-2000). Its tools
are useful when not much is known regarding the underlying causes
associated with the data set, and are often used for checking assumptions.
For example, suppose we perform an experiment and collect some data\ldots{}
now what? We look at the data using exploratory visual tools.


\subsection{More About Stemplots}

There are many bells and whistles associated with stemplots, and the
\inputencoding{latin9}\lstinline[showstringspaces=false]!stem.leaf!\inputencoding{utf8}
function can do many of them.
\begin{description}
\item [{Trim~Outliers:}] Some data sets have observations that fall far
from the bulk of the other data (in a sense made more precise in Section
\ref{sub:Outliers}). These extreme observations often obscure the
underlying structure to the data and are best left out of the data
display. The \inputencoding{latin9}\lstinline[showstringspaces=false]!trim.outliers!\inputencoding{utf8}
argument (which is \inputencoding{latin9}\lstinline[showstringspaces=false]!TRUE!\inputencoding{utf8}
by default) will separate the extreme observations from the others
and graph the stemplot without them; they are listed at the bottom
(respectively, top) of the stemplot with the label \inputencoding{latin9}\lstinline[showstringspaces=false]!HI!\inputencoding{utf8}
(respectively \inputencoding{latin9}\lstinline[showstringspaces=false]!LO!\inputencoding{utf8}). 
\item [{Split~Stems:}] The standard stemplot has only one line per stem,
which means that all observations with first digit \inputencoding{latin9}\lstinline[showstringspaces=false]!3!\inputencoding{utf8}
are plotted on the same line, regardless of the value of the second
digit. But this gives some stemplots a {}``skyscraper'' appearance,
with too many observations stacked onto the same stem. We can often
fix the display by increasing the number of lines available for a
given stem. For example, we could make two lines per stem, say, \inputencoding{latin9}\lstinline[showstringspaces=false]!3*!\inputencoding{utf8}
and \inputencoding{latin9}\lstinline[showstringspaces=false]!3.!\inputencoding{utf8}.
Observations with second digit 0 through 4 would go on the upper line,
while observations with second digit 5 through 9 would go on the lower
line. (We could do a similar thing with five lines per stem, or even
ten lines per stem.) The end result is a more spread out stemplot
which often looks better. A good example of this was shown on page
\pageref{exa:stemleaf-multiple-lines-stem}.
\item [{Depths:}] these are used to give insight into the balance of the
observations as they accumulate toward the median. In a column beside
the standard stemplot, the frequency of the stem containing the sample
median is shown in parentheses. Next, frequencies are accumulated
from the outside inward, including the outliers. Distributions that
are more symmetric will have better balanced depths on either side
of the sample median.
\end{description}

\subsection{How to do it with \textsf{R}}


\paragraph*{At the command prompt}

The basic command is \inputencoding{latin9}\lstinline[showstringspaces=false]!stem(x)!\inputencoding{utf8}
or a more sophisticated version written by Peter Wolf called \inputencoding{latin9}\lstinline[showstringspaces=false]!stem.leaf(x)!\inputencoding{utf8}
in the \textsf{R} Commander. We will describe \inputencoding{latin9}\lstinline[showstringspaces=false]!stem.leaf!\inputencoding{utf8}
since that is the one used by \textsf{R} Commander.


\paragraph*{With the \textsf{R} Commander}

WARNING: Sometimes when making a stem plot the result will not be
what you expected. There are several reasons for this:
\begin{itemize}
\item Stemplots by default will trim extreme observations (defined in Section
\ref{sub:Outliers}) from the display. This in some cases will result
in stemplots that are not as wide as expected.
\item The leafs digit is chosen automatically by \inputencoding{latin9}\lstinline[showstringspaces=false]!stem.leaf!\inputencoding{utf8}
according to an algorithm that the computer believes will represent
the data well. Depending on the choice of the digit, \inputencoding{latin9}\lstinline[showstringspaces=false]!stem.leaf!\inputencoding{utf8}
may drop digits from the data or round the values in unexpected ways.
\end{itemize}
Let us take a look at the \inputencoding{latin9}\lstinline[showstringspaces=false]!rivers!\inputencoding{utf8}
data set\label{ite:stemplot-rivers}.

<<>>=
stem.leaf(rivers)
@

The stemplot shows a right-skewed shape to the \inputencoding{latin9}\lstinline[showstringspaces=false]!rivers!\inputencoding{utf8}
data distribution. Notice that the last digit of each of the data
values were dropped from the display. Notice also that there were
eight extreme observations identified by the computer, and their exact
values are listed at the bottom of the stemplot. Look at the scale
on the left of the stemplot and try to imagine how ridiculous the
graph would have looked had we tried to include enough stems to include
these other eight observations; the stemplot would have stretched
over several pages. Notice finally that we can use the depths to approximate
the sample median for these data. The median lies in the row identified
by \inputencoding{latin9}\lstinline[showstringspaces=false]!(18)!\inputencoding{utf8},
which means that the median is the average of the ninth and tenth
observation on that row. Those two values correspond to \inputencoding{latin9}\lstinline[showstringspaces=false]!43!\inputencoding{utf8}
and \inputencoding{latin9}\lstinline[showstringspaces=false]!43!\inputencoding{utf8},
so a good guess for the median would be 430. (For the record, the
sample median is $\widetilde{x}=425$. Recall that stemplots round
the data to the nearest stem-leaf pair.) 

Next let us see what the \inputencoding{latin9}\lstinline[showstringspaces=false]!precip!\inputencoding{utf8}
data look like.

<<>>=
stem.leaf(precip)
@

Here is an example of split stems, with two lines per stem. The final
digit of each datum has been dropped for the display. The data appear
to be left skewed with four extreme values to the left and one extreme
value to the right. The sample median is approximately 37 (it turns
out to be 36.6).


\subsection{Hinges and the Five Number Summary\label{sub:hinges-and-5NS} }

Given a data set $x_{1}$, $x_{2}$, \ldots{}, $x_{n}$, the hinges
are found by the following method: 
\begin{itemize}
\item Find the order statistics $x_{(1)}$, $x_{(2)}$, \ldots{}, $x_{(n)}$. 
\item The \emph{lower hinge} $h_{L}$ is in position $L=\left\lfloor (n+3)/2\right\rfloor /2$,
where the symbol $\left\lfloor x\right\rfloor $ denotes the largest
integer less than or equal to $x$. If the position $L$ is not an
integer, then the hinge $h_{L}$ is the average of the adjacent order
statistics.
\item The \emph{upper hinge} $h_{U}$ is in position $n+1-L$.
\end{itemize}
Given the hinges, the \emph{five number summary} ($5NS$) is \begin{equation}
5NS=(x_{(1)},\ h_{L},\ \tilde{x},\ h_{U},\ x_{(n)}).\end{equation}
An advantage of the $5NS$ is that it reduces a potentially large
data set to a shorter list of only five numbers, and further, these
numbers give insight regarding the shape of the data distribution
similar to the sample quantiles in Section \ref{sub:Order-Statistics-and}.


\subsection{How to do it with \textsf{R}}

If the data are stored in a vector \inputencoding{latin9}\lstinline[showstringspaces=false]!x!\inputencoding{utf8},
then you can compute the $5NS$ with the \inputencoding{latin9}\lstinline[showstringspaces=false]!fivenum!\inputencoding{utf8}
function.


\subsection{Boxplots\label{sub:boxplots} }

A boxplot is essentially a graphical representation of the $5NS$.
It can be a handy alternative to a stripchart when the sample size
is large.

A boxplot is constructed by drawing a box alongside the data axis
with sides located at the upper and lower hinges. A line is drawn
parallel to the sides to denote the sample median. Lastly, whiskers
are extended from the sides of the box to the maximum and minimum
data values (more precisely, to the most extreme values that are not
potential outliers, defined below).

Boxplots are good for quick visual summaries of data sets, and the
relative positions of the values in the $5NS$ are good at indicating
the underlying shape of the data distribution, although perhaps not
as effectively as a histogram. Perhaps the greatest advantage of a
boxplot is that it can help to objectively identify extreme observations
in the data set as described in the next section.

Boxplots are also good because one can visually assess multiple features
of the data set simultaneously:
\begin{description}
\item [{Center}] can be estimated by the sample median, $\tilde{x}$.
\item [{Spread}] can be judged by the width of the box, $h_{U}-h_{L}$.
We know that this will be close to the $IQR$, which can be compared
to $s$ and the $MAD$, perhaps after rescaling if appropriate.
\item [{Shape}] is indicated by the relative lengths of the whiskers, and
the position of the median inside the box. Boxes with unbalanced whiskers
indicate skewness in the direction of the long whisker. Skewed distributions
often have the median tending in the opposite direction of skewness.
Kurtosis can be assessed using the box and whiskers. A wide box with
short whiskers will tend to be platykurtic, while a skinny box with
wide whiskers indicates leptokurtic distributions.
\item [{Extreme~observations}] are identified with open circles (see below).
\end{description}

\subsection{Outliers\label{sub:Outliers} }

A \emph{potential outlier} is any observation that falls beyond 1.5
times the width of the box on either side, that is, any observation
less than $h_{L}-1.5(h_{U}-h_{L})$ or greater than $h_{U}+1.5(h_{U}-h_{L})$.
A \emph{suspected outlier} is any observation that falls beyond 3
times the width of the box on either side. In \textsf{R}, both potential
and suspected outliers (if present) are denoted by open circles; there
is no distinction between the two. 

When potential outliers are present, the whiskers of the boxplot are
then shortened to extend to the most extreme observation that is not
a potential outlier. If an outlier is displayed in a boxplot, the
index of the observation may be identified in a subsequent plot in
\inputencoding{latin9}\lstinline[showstringspaces=false]!Rcmdr!\inputencoding{utf8}
by clicking the \textsf{Identify outliers with mouse} option in the
\textsf{Boxplot} dialog.

What do we do about outliers? They merit further investigation. The
primary goal is to determine why the observation is outlying, if possible.
If the observation is a typographical error, then it should be corrected
before continuing. If the observation is from a subject that does
not belong to the population of interest, then perhaps the datum should
be removed. Otherwise, perhaps the value is hinting at some hidden
structure to the data.


\subsection{How to do it with \textsf{R}}

The quickest way to visually identify outliers is with a boxplot,
described above. Another way is with the \inputencoding{latin9}\lstinline[showstringspaces=false]!boxplot.stats!\inputencoding{utf8}
function.
\begin{example}
The \inputencoding{latin9}\lstinline[showstringspaces=false]!rivers!\inputencoding{utf8}
data. We will look for potential outliers in the \inputencoding{latin9}\lstinline[showstringspaces=false]!rivers!\inputencoding{utf8}
data.

<<>>=
boxplot.stats(rivers)$out
@

We may change the \inputencoding{latin9}\lstinline[showstringspaces=false]!coef!\inputencoding{utf8}
argument to 3 (it is 1.5 by default) to identify suspected outliers.

<<>>=
boxplot.stats(rivers, coef = 3)$out
@

\end{example}

\subsection{Standardizing variables}

It is sometimes useful to compare data sets with each other on a scale
that is independent of the measurement units. Given a set of observed
data $x_{1}$, $x_{2}$, \ldots{}, $x_{n}$ we get $z$ scores, denoted
$z_{1}$, $z_{2}$, \ldots{}, $z_{n}$, by means of the following
formula\[
z_{i}=\frac{x_{i}-\xbar}{s},\quad i=1,\,2,\,\ldots,\, n.\]



\subsection{How to do it with \textsf{R}}

The \inputencoding{latin9}\lstinline[showstringspaces=false]!scale!\inputencoding{utf8}
function will rescale a numeric vector (or data frame) by subtracting
the sample mean from each value (column) and/or by dividing each observation
by the sample standard deviation.


\section{Multivariate Data and Data Frames\label{sec:multivariate-data}}

We have had experience with vectors of data, which are long lists
of numbers. Typically, each entry in the vector is a single measurement
on a subject or experimental unit in the study. We saw in Section
\ref{sub:Vectors} how to form vectors with the \inputencoding{latin9}\lstinline[showstringspaces=false]!c!\inputencoding{utf8}
function or the \inputencoding{latin9}\lstinline[showstringspaces=false]!scan!\inputencoding{utf8}
function. 

However, statistical studies often involve experiments where there
are two (or more) measurements associated with each subject. We display
the measured information in a rectangular array in which each row
corresponds to a subject, and the columns contain the measurements
for each respective variable. For instance, if one were to measure
the height and weight and hair color of each of 11 persons in a research
study, the information could be represented with a rectangular array.
There would be 11 rows. Each row would have the person's height in
the first column and hair color in the second column.

The corresponding objects in \textsf{R} are called \emph{data frames},
and they can be constructed with the \inputencoding{latin9}\lstinline[showstringspaces=false]!data.frame!\inputencoding{utf8}
function. Each row is an observation, and each column is a variable.
\begin{example}
Suppose we have two vectors \inputencoding{latin9}\lstinline[showstringspaces=false]!x!\inputencoding{utf8}
and \inputencoding{latin9}\lstinline[showstringspaces=false]!y!\inputencoding{utf8}
and we want to make a data frame out of them.

<<>>=
x <- 5:8
y <- letters[3:6]
A <- data.frame(v1 = x, v2 = y)
@

\end{example}
Notice that \inputencoding{latin9}\lstinline[showstringspaces=false]!x!\inputencoding{utf8}
and \inputencoding{latin9}\lstinline[showstringspaces=false]!y!\inputencoding{utf8}
are the same length. This is \emph{necessary}. Also notice that \inputencoding{latin9}\lstinline[showstringspaces=false]!x!\inputencoding{utf8}
is a numeric vector and \inputencoding{latin9}\lstinline[showstringspaces=false]!y!\inputencoding{utf8}
is a character vector. We may choose numeric and character vectors
(or even factors) for the columns of the data frame, but each column
must be of exactly one type. That is, we can have a column for \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!height!\inputencoding{utf8}
and a column for \inputencoding{latin9}\lstinline[showstringspaces=false]!gender!\inputencoding{utf8},
but we will get an error if we try to mix function \inputencoding{latin9}\lstinline[showstringspaces=false]!height!\inputencoding{utf8}
(numeric) and \inputencoding{latin9}\lstinline[showstringspaces=false]!gender!\inputencoding{utf8}
(character or factor) information in the same column.

Indexing of data frames is similar to indexing of vectors. To get
the entry in row $i$ and column $j$ do \inputencoding{latin9}\lstinline[showstringspaces=false]!A[i,j]!\inputencoding{utf8}.
We can get entire rows and columns by omitting the other index. 

<<>>=
A[3,]
A[1, ]
A[ ,2]
@

There are several things happening above. Notice that \inputencoding{latin9}\lstinline[showstringspaces=false]!A[3,]!\inputencoding{utf8}
gave a data frame (with the same entries as the third row of \inputencoding{latin9}\lstinline[showstringspaces=false]!A!\inputencoding{utf8})
yet \inputencoding{latin9}\lstinline[showstringspaces=false]!A[1, ]!\inputencoding{utf8}
is a numeric vector. \inputencoding{latin9}\lstinline[showstringspaces=false]!A[ ,2]!\inputencoding{utf8}
is a factor vector because the default setting for \inputencoding{latin9}\lstinline[showstringspaces=false]!data.frame!\inputencoding{utf8}
is \inputencoding{latin9}\lstinline[showstringspaces=false]!stringsAsFactors = TRUE!\inputencoding{utf8}.

Data frames have a \inputencoding{latin9}\lstinline[showstringspaces=false]!names!\inputencoding{utf8}
attribute and the names may be extracted with the \inputencoding{latin9}\lstinline[showstringspaces=false]!names!\inputencoding{utf8}
function. Once we have the names we may extract given columns by way
of the dollar sign.

<<>>=
names(A)
A$v1
@

The above is identical to \inputencoding{latin9}\lstinline[showstringspaces=false]!A[ ,1]!\inputencoding{utf8}. 


\subsection{Bivariate Data\label{sub:Bivariate-Data}}
\begin{itemize}
\item Stacked bar charts
\item odds ratio and relative risk
\item Introduce the sample correlation coefficient.
\end{itemize}
The \textbf{sample Pearson product-moment }\textbf{\emph{correlation
coefficient}}:\[
r=\frac{\sum_{i=1}^{n}(x_{i}-\xbar)(y_{i}-\ybar)}{\sqrt{\sum_{i=1}^{n}(x_{i}-\xbar)}\sqrt{\sum_{i=1}^{n}(y_{i}-\ybar)}}\]

\begin{itemize}
\item independent of scale
\item $-1<r<1$
\item measures \emph{strength} and \emph{direction} of linear association
\item Two-Way Tables. Done with \inputencoding{latin9}\lstinline[showstringspaces=false]!table!\inputencoding{utf8},
or in the \textsf{R} Commander by following \textsf{Statistics $\triangleright$
Contingency Tables $\triangleright$} \textsf{Two-way Tables}. You
can also enter and analyze a two-way table.

\begin{itemize}
\item table
\item prop.table
\item addmargins
\item rowPercents (Rcmdr)
\item colPercents (Rcmdr)
\item totPercents(Rcmdr)
\item A <- xtabs(\textasciitilde{} gender + race, data = RcmdrTestDrive)
\item xtabs( Freq \textasciitilde{} Class + Sex, data = Titanic) \# from
built in table
\item barplot(A, legend.text=TRUE) 
\item barplot(t(A), legend.text=TRUE) 
\item barplot(A, legend.text=TRUE, beside = TRUE)
\item spineplot(gender \textasciitilde{} race, data = RcmdrTestDrive)
\item Spine plot: plots categorical versus categorical
\end{itemize}
\item Scatterplot: look for linear association and correlation. 

\begin{itemize}
\item carb \textasciitilde{} optden, data = Formaldehyde (boring)
\item conc \textasciitilde{} rate, data = Puromycin
\item xyplot(accel \textasciitilde{} dist, data = attenu) nonlinear association
\item xyplot(eruptions \textasciitilde{} waiting, data = faithful) (linear,
two groups)
\item xyplot(Petal.Width \textasciitilde{} Petal.Length, data = iris)
\item xyplot(pressure \textasciitilde{} temperature, data = pressure) (exponential
growth)
\item xyplot(weight \textasciitilde{} height, data = women) (strong positive
linear)
\end{itemize}
\end{itemize}

\subsection{Multivariate Data\label{sub:Multivariate-Data}}

Multivariate Data Display
\begin{itemize}
\item Multi-Way Tables. You can do this with \inputencoding{latin9}\lstinline[showstringspaces=false]!table!\inputencoding{utf8},
or in \textsf{R} Commander by following \textsf{Statistics} \textsf{$\triangleright$}
\textsf{Contingency Tables} \textsf{$\triangleright$} \textsf{Multi-way
Tables}.
\item Scatterplot matrix. used for displaying pairwise scatterplots simultaneously.
Again, look for linear association and correlation.
\item 3D Scatterplot. See Figure \pageref{fig:3D-scatterplot-trees}
\item \inputencoding{latin9}\lstinline[showstringspaces=false]!plot(state.region, state.division)!\inputencoding{utf8} 
\item \inputencoding{latin9}\lstinline[showstringspaces=false]!barplot(table(state.division,state.region), legend.text=TRUE)!\inputencoding{utf8}
\end{itemize}

\section{Comparing Populations\label{sec:Comparing-Data-Sets}}

Sometimes we have data from two or more groups (or populations) and
we would like to compare them and draw conclusions. Some issues that
we would like to address:
\begin{itemize}
\item Comparing centers and spreads: variation within versus between groups
\item Comparing clusters and gaps
\item Comparing outliers and unusual features
\item Comparing shapes.
\end{itemize}

\subsection{Numerically}

I am thinking here about the \textsf{Statistics} \textsf{$\triangleright$}
\textsf{Numerical Summaries} \textsf{$\triangleright$ Summarize by
groups} option or the \textsf{Statistics} \textsf{$\triangleright$}
\textsf{Summaries} \textsf{$\triangleright$Table of Statistics} option. 


\subsection{Graphically}
\begin{itemize}
\item Boxplots

\begin{itemize}
\item Variable width: the width of the drawn boxplots are proportional to
$\sqrt{n_{i}}$, where $n_{i}$ is the size of the $i^{\text{th}}$
group. Why? Because many statistics have variability proportional
to the reciprocal of the square root of the sample size.
\item Notches: extend to $1.58\cdot(h_{U}-h_{L})/\sqrt{n}$. The idea is
to give roughly a 95\% confidence interval for the difference in two
medians. See Chapter \ref{cha:Hypothesis-Testing}.
\end{itemize}
\item Stripcharts

\begin{itemize}
\item stripchart(weight \textasciitilde{} feed, method=\textquotedbl{}stack\textquotedbl{},
data=chickwts)
\end{itemize}
\item Bar Graphs

\begin{itemize}
\item barplot(xtabs(Freq \textasciitilde{} Admit + Gender, data = UCBAdmissions))
\# stacked bar chart
\item barplot(xtabs(Freq \textasciitilde{} Admit, data = UCBAdmissions))
\item barplot(xtabs(Freq \textasciitilde{} Gender + Admit, data = UCBAdmissions),
legend = TRUE, beside = TRUE) \# oops, discrimination.
\item barplot(xtabs(Freq \textasciitilde{} Admit+Dept, data = UCBAdmissions),
legend = TRUE, beside = TRUE) \# different departments have different
standards
\item barplot(xtabs(Freq \textasciitilde{} Gender+Dept, data = UCBAdmissions),
legend = TRUE, beside = TRUE) \# men mostly applied to easy departments,
women mostly applied to difficult departments
\item barplot(xtabs(Freq \textasciitilde{} Gender+Dept, data = UCBAdmissions),
legend = TRUE, beside = TRUE)
\item barchart(Admit \textasciitilde{} Freq, data = C)
\item barchart(Admit \textasciitilde{} Freq|Gender, data = C)
\item barchart(Admit \textasciitilde{} Freq | Dept, groups = Gender, data
= C)
\item barchart(Admit \textasciitilde{} Freq | Dept, groups = Gender, data
= C, auto.key = TRUE)
\end{itemize}
\item Histograms

\begin{itemize}
\item \textasciitilde{} breaks | wool{*}tension, data = warpbreaks
\item \textasciitilde{} weight | feed, data = chickwts
\item \textasciitilde{} weight | group, data = PlantGrowth 
\item \textasciitilde{} count | spray, data = InsectSprays
\item \textasciitilde{} len | dose, data = ToothGrowth
\item \textasciitilde{} decrease | treatment, data = OrchardSprays (or rowpos
or colpos)
\end{itemize}
\item Scatterplots

\begin{itemize}
\item xyplot(Petal.Width \textasciitilde{} Petal.Length, data = iris, group
= Species)
\end{itemize}
<<eval = FALSE>>=
library(lattice)
xyplot()
@

\item Scatterplot matrices

\begin{itemize}
\item splom(\textasciitilde{} cbind(GNP.deflator,GNP,Unemployed,Armed.Forces,Population,Year,Employed),
data = longley)
\item splom(\textasciitilde{} cbind(pop15,pop75,dpi), data = LifeCycleSavings)
\item splom(\textasciitilde{} cbind(Murder, Assault, Rape), data = USArrests)
\item splom(\textasciitilde{} cbind(CONT, INTG, DMNR), data = USJudgeRatings)
\item splom(\textasciitilde{} cbind(area,peri,shape,perm), data = rock)
\item splom(\textasciitilde{} cbind(Air.Flow, Water.Temp, Acid.Conc., stack.loss),
data = stackloss)
\item splom(\textasciitilde{} cbind(Fertility,Agriculture,Examination,Education,Catholic,Infant.Mortality),
data = swiss)
\item splom(\textasciitilde{} cbind(Fertility,Agriculture,Examination),
data = swiss) (positive and negative)
\end{itemize}
\item Dot charts

\begin{itemize}
\item dotchart(USPersonalExpenditure)
\item dotchart(t(USPersonalExpenditure))
\item dotchart(WorldPhones) (transpose is no good)
\item freeny.x is no good, neither is volcano
\item dotchart(UCBAdmissions{[},,1{]})
\item dotplot(Survived \textasciitilde{} Freq | Class, groups = Sex, data
= B)
\item dotplot(Admit \textasciitilde{} Freq | Dept, groups = Gender, data
= C)
\end{itemize}
\item Mosaic plot

\begin{itemize}
\item mosaic(\textasciitilde{} Survived + Class + Age + Sex, data = Titanic)
(or just mosaic(Titanic))
\item mosaic(\textasciitilde{} Admit + Dept + Gender, data = UCBAdmissions)
\end{itemize}
\item Spine plots

\begin{itemize}
\item spineplot(xtabs(Freq \textasciitilde{} Admit + Gender, data = UCBAdmissions))
\# rescaled barplot
\end{itemize}
\item Quantile-quantile plots: There are two ways to do this. One way is
to compare two independent samples (of the same size). qqplot(x,y).
Another way is to compare the sample quantiles of one variable to
the theoretical quantiles of another distribution.
\end{itemize}
Given two samples $\left\{ x_{1},\, x_{2},\,\ldots,\, x_{n}\right\} $
and $\left\{ y_{1},\, y_{2},\,\ldots,\, y_{n}\right\} $, we may find
the order statistics $x_{(1)}\leq x_{(2)}\leq\cdots\leq x_{(n)}$
and $y_{(1)}\leq y_{(2)}\leq\cdots\leq y_{(n)}$. Next, plot the $n$
points $(x_{(1)},y_{(1)})$, $(x_{(2)},y_{(2)})$ ,\ldots{},$(x_{(n)},y_{(n)})$.

It is clear that if $x_{(k)}=y_{(k)}$ for all $k=1,2,\ldots,n$,
then we will have a straight line. It is also clear that in the real
world, a straight line is NEVER observed, and instead we have a scatterplot
that hopefully had a general linear trend. What do the rules tell
us?
\begin{itemize}
\item If the $y$-intercept of the line is greater (less) than zero, then
the center of the $Y$ data is greater (less) than the center of the
$X$ data.
\item If the slope of the line is greater (less) than one, then the spread
of the $Y$ data is greater (less) than the spread of the $X$ data..
\end{itemize}

\subsection{Lattice Graphics\label{sub:Lattice-Graphics}}

The following types of plots are useful when there is one variable
of interest and there is a factor in the data set by which the variable
is categorized. 

It is sometimes nice to set \inputencoding{latin9}\lstinline[showstringspaces=false]!lattice.options(default.theme = "col.whitebg")!\inputencoding{utf8}


\paragraph*{Side by side boxplots}

<<eval = FALSE>>=
library(lattice)
bwplot(~weight | feed, data = chickwts)
@

%
\begin{figure}[H]
\begin{centering}
<<echo = FALSE, fig=true, height = 4, width = 4>>=
library(lattice)
print(bwplot(~ weight | feed, data = chickwts))
@
\par\end{centering}

\caption{Boxplots of \texttt{weight} by \texttt{feed} type in the \texttt{chickwts}
data}

\end{figure}





\paragraph*{Histograms}

<<eval = FALSE>>=
histogram(~age | education, data = infert)
@

%
\begin{figure}[H]
\begin{centering}
<<echo = FALSE, fig=true, height = 4, width = 4>>=
library(lattice)
print(histogram(~age | education, data = infert))
@
\par\end{centering}

\caption{Histograms of \texttt{age} by \texttt{education} level from the \texttt{infert}
data}

\end{figure}



\paragraph*{Scatterplots}

<<eval = FALSE>>=
xyplot(Petal.Length ~ Petal.Width | Species, data = iris)
@

%
\begin{figure}[H]
\begin{centering}
<<echo = FALSE, fig=true, height = 4, width = 4>>=
library(lattice)
print(xyplot(Petal.Length ~ Petal.Width | Species, data = iris))
@
\par\end{centering}

\caption{An \texttt{xyplot} of \texttt{Petal.Length} versus \texttt{Petal.Width}
by \texttt{Species} in the \texttt{iris} data}

\end{figure}



\paragraph*{Coplots}

<<eval = FALSE>>=
coplot(conc ~ uptake | Type * Treatment, data = CO2)
@

%
\begin{figure}[H]
\begin{centering}
<<echo = FALSE, fig=true, height = 4, width = 4>>=
library(lattice)
print(coplot(conc ~ uptake | Type * Treatment, data = CO2))
@
\par\end{centering}

\caption{A \texttt{coplot} of \texttt{conc} versus \texttt{uptake} by \texttt{Type}
and \texttt{Treatment} in the \texttt{CO2} data}

\end{figure}


\newpage{}


\section*{Chapter Exercises}

\addcontentsline{toc}{section}{Chapter Exercises}
\setcounter{thm}{0}


\paragraph*{Directions: }

Open \textsf{R} and issue the following commands at the command line
to get started. Note that you need to have the \inputencoding{latin9}\lstinline[showstringspaces=false]!RcmdrPlugin.IPSUR!\inputencoding{utf8}
package installed, and for some exercises you need the \inputencoding{latin9}\lstinline[showstringspaces=false]!e1071!\inputencoding{utf8}
package.
\begin{lyxcode}
library(RcmdrPlugin.IPSUR)~\\
data(RcmdrTestDrive)~~\\
attach(RcmdrTestDrive)~~\\
names(RcmdrTestDrive)~~\#~shows~names~of~variables
\end{lyxcode}
<<echo = FALSE, results = hide>>=
attach(RcmdrTestDrive)
names(RcmdrTestDrive)
@

To load the data in the \textsf{R} Commander (\inputencoding{latin9}\lstinline[showstringspaces=false]!Rcmdr!\inputencoding{utf8}),
click the \textsf{Data Set} button, and select \inputencoding{latin9}\lstinline[showstringspaces=false]!RcmdrTestDrive!\inputencoding{utf8}
as the active data set. To learn more about the data set and where
it comes from, type \inputencoding{latin9}\lstinline[showstringspaces=false]!?RcmdrTestDrive!\inputencoding{utf8}
at the command line.
\begin{xca}
\label{xca:summary-RcmdrTestDrive}Perform a summary of all variables
in \inputencoding{latin9}\lstinline[showstringspaces=false]!RcmdrTestDrive!\inputencoding{utf8}.
You can do this with the command \inputencoding{latin9}
\begin{lstlisting}[showstringspaces=false]
summary(RcmdrTestDrive)
\end{lstlisting}
\inputencoding{utf8}

Alternatively, you can do this in the \inputencoding{latin9}\lstinline[showstringspaces=false]!Rcmdr!\inputencoding{utf8}
with the sequence \textsf{Statistics} \textsf{$\triangleright$ Summaries}
\textsf{$\triangleright$ Active Data Set}. Report the values of the
summary statistics for each variable.


\paragraph*{Answer:}

<<"Find summary statistics">>=
summary(RcmdrTestDrive)
@
\end{xca}

\begin{xca}
Make a table of the \emph{race} variable. Do this with \textsf{Statistics}
\textsf{$\triangleright$ Summaries} \textsf{$\triangleright$ IPSUR
- Frequency Distributions}\emph{...}
\begin{enumerate}
\item Which ethnicity has the highest frequency?
\item Which ethnicity has the lowest frequency?
\item Include a bar graph of \emph{race}. Do this with \textsf{Graphs} \textsf{$\triangleright$}
\textsf{IPSUR - Bar Graph}...
\end{enumerate}
\end{xca}

\paragraph*{Solution:}

First we will make a table of the \emph{race} variable with the \inputencoding{latin9}\lstinline[showstringspaces=false]!table!\inputencoding{utf8}
function.

<<>>=
table(race)
@
\begin{enumerate}
\item For these data, \Sexpr{names(table(race))[which(table(race)==max(table(race)))]}
has the highest frequency.
\item For these data, \Sexpr{names(table(race))[which(table(race)==min(table(race)))]}
has the lowest frequency.
\item The graph is shown below.
\end{enumerate}
\begin{center}
<<echo = FALSE, fig=true, height = 4, width = 6>>=
barplot(table(RcmdrTestDrive$race), main="", xlab="race", ylab="Frequency", legend.text=FALSE, col=NULL) 
@
\par\end{center}


\begin{xca}
Calculate the average \emph{salary} by the factor \emph{gender}. Do
this with \textsf{Statistics} \textsf{$\triangleright$ Summaries}
\textsf{$\triangleright$ Table of Statistics}\emph{...}
\begin{enumerate}
\item Which \emph{gender} has the highest mean \emph{salary}? 
\item Report the highest mean \emph{salary}.
\item Compare the spreads for the genders by calculating the standard deviation
of \emph{salary} by \emph{gender}. Which \emph{gender} has the biggest
standard deviation?
\item Make boxplots of \emph{salary} by \emph{gender} with the following
method:

\begin{quote}
On the \inputencoding{latin9}\lstinline[showstringspaces=false]!Rcmdr!\inputencoding{utf8},
click \textsf{Graphs} \textsf{$\triangleright$} \textsf{IPSUR - Boxplot}...\\
In the \textsf{Variable} box, select \emph{salar}y.\\
Click the \textsf{Plot by groups}... box and select \emph{gender}.
Click \textsf{OK}.\\
Click \textsf{OK} to graph the boxplot.
\end{quote}
How does the boxplot compare to your answers to (1) and (3)?

\end{enumerate}
\end{xca}
\noindent 
\paragraph*{Solution:}

We can generate a table listing the average salaries by gender with
two methods. The first uses \inputencoding{latin9}\lstinline[showstringspaces=false]!tapply!\inputencoding{utf8}:

<<keep.source = TRUE>>=
x <- tapply(salary, list(gender = gender), mean)
x
@

The second method uses the \inputencoding{latin9}\lstinline[showstringspaces=false]!by!\inputencoding{utf8}
function:

<<keep.source = TRUE>>=
by(salary, gender, mean, na.rm = TRUE)
@

Now to answer the questions:
\begin{enumerate}
\item Which gender has the highest mean salary? 


We can answer this by looking above. For these data, the gender with
the highest mean salary is \Sexpr{names(x)[which(x==max(x))]}.

\item Report the highest mean salary.


Depending on our answer above, we would do something like \inputencoding{latin9}
\begin{lstlisting}[showstringspaces=false]
mean(salary[gender == Male])
\end{lstlisting}
\inputencoding{utf8} for example. For these data, the highest mean salary is 

<<>>=
x[which(x==max(x))]
@

\item Compare the spreads for the genders by calculating the standard deviation
of \emph{salary} by \emph{gender}. Which gender has the biggest standard
deviation?


<<>>=
y <- tapply(salary, list(gender = gender), sd)
y
@

For these data, the the largest standard deviation is approximately
\Sexpr{round(y[which(y==max(y))],2)} which was attained by the \Sexpr{names(y)[which(y==max(y))]}
gender.

\item Make boxplots of \emph{salary} by \emph{gender}. How does the boxplot
compare to your answers to (1) and (3)?


The graph is shown below.

\begin{center}
<<echo = FALSE, fig=true, height = 4.5, width = 6>>=
boxplot(salary~gender, xlab="salary", ylab="gender", main="", notch=FALSE, varwidth=TRUE, horizontal=TRUE, data=RcmdrTestDrive) 
@
\par\end{center}

Answers will vary. There should be some remarks that the center of
the box is farther to the right for the \Sexpr{names(x)[which(x==max(x))]}
gender, and some recognition that the box is wider for the \Sexpr{names(y)[which(y==max(y))]}
gender.\end{enumerate}



\begin{xca}
For this problem we will study the variable \emph{reduction}.
\begin{enumerate}
\item Find the order statistics and store them in a vector \inputencoding{latin9}\lstinline[showstringspaces=false]!x!\inputencoding{utf8}.
\emph{Hint: }\inputencoding{latin9}\lstinline[showstringspaces=false]!x <- sort(reduction)!\inputencoding{utf8}
\item Find $x_{(137)}$, the 137$^{\text{th}}$ order statistic.
\item Find the IQR.
\item Find the Five Number Summary (5NS).
\item Use the 5NS to calculate what the width of a boxplot of \emph{reduction}
would be.
\item Compare your answers (3) and (5). Are they the same? If not, are they
close?
\item Make a boxplot of \emph{reduction}, and include the boxplot in your
report. You can do this with the \inputencoding{latin9}\lstinline[showstringspaces=false]!boxplot!\inputencoding{utf8}
function, or in \inputencoding{latin9}\lstinline[showstringspaces=false]!Rcmdr!\inputencoding{utf8}
with \textsf{Graphs} \textsf{$\triangleright$} \textsf{IPSUR - Boxplot}...
\item Are there any potential/suspected outliers? If so, list their values.
\emph{Hint:} use your answer to (a).
\item Using the rules discussed in the text, classify answers to (8), if
any, as \emph{potential} or \emph{suspected} outliers.
\end{enumerate}
\end{xca}

\paragraph*{Answers:}

<<echo = FALSE, results = hide>>=
x = sort(reduction)
@

<<>>=
x[137]
IQR(x)
fivenum(x)
fivenum(x)[4] - fivenum(x)[2]
@

\noindent Compare your answers (3) and (5). Are they the same? If
not, are they close?

Yes, they are close, within \Sexpr{abs(IQR(x)-(fivenum(x)[4] - fivenum(x)[2]))}
of each other.

\noindent The boxplot of \emph{reduction} is below.

\begin{center}
<<echo = FALSE, fig=true, height = 4, width = 6>>=
boxplot(reduction, xlab="reduction", main="", notch=FALSE, varwidth=TRUE, horizontal=TRUE, data=RcmdrTestDrive) 
@
\par\end{center}

<<>>=
temp <- fivenum(x)
inF <- 1.5 * (temp[4] - temp[2]) + temp[4]
outF <- 3 * (temp[4] - temp[2]) + temp[4]
which(x > inF)
which(x > outF)
@

Observations \Sexpr{which(x > inF)} would be considered potential
outliers, while observation(s) \Sexpr{which(x > outF)} would be considered
a suspected outlier.


\begin{xca}
In this problem we will compare the variables \emph{before} and \emph{after}.
Don't forget \inputencoding{latin9}\lstinline[showstringspaces=false]!library(e1071)!\inputencoding{utf8}.
\begin{enumerate}
\item Examine the two measures of center for both variables. Judging from
these measures, which variable has a higher center?
\item Which measure of center is more appropriate for \emph{before}? (You
may want to look at a boxplot.) Which measure of center is more appropriate
for \emph{after}?
\item Based on your answer to (2), choose an appropriate measure of spread
for each variable, calculate it, and report its value. Which variable
has the biggest spread? (Note that you need to make sure that your
measures are on the same scale.) 
\item Calculate and report the skewness and kurtosis for \emph{before}.
Based on these values, how would you describe the shape of \emph{before}?
\item Calculate and report the skewness and kurtosis for \emph{after}. Based
on these values, how would you describe the shape of \emph{after}?
\item Plot histograms of \emph{before} and \emph{after} and compare them
to your answers to (4) and (5).
\end{enumerate}
\end{xca}

\paragraph*{Solution:}
\begin{enumerate}
\item Examine the two measures of center for both variables that you found
in problem 1. Judging from these measures, which variable has a higher
center?


We may take a look at the \inputencoding{latin9}\lstinline[showstringspaces=false]!summary(RcmdrTestDrive)!\inputencoding{utf8}
output from Exercise \ref{xca:summary-RcmdrTestDrive}. Here we will
repeat the relevant summary statistics.

<<>>=
c(mean(before), median(before))
c(mean(after), median(after))
@

The idea is to look at the two measures and compare them to make a
decision. In a nice world, both the mean and median of one variable
will be larger than the other which sends a nice message. If We get
a mixed message, then we should look for other information, such as
extreme values in one of the variables, which is one of the reasons
for the next part of the problem.

\item Which measure of center is more appropriate for \emph{before}? (You
may want to look at a boxplot.) Which measure of center is more appropriate
for \emph{after}?


The boxplot of \emph{before} is shown below.

\begin{center}
<<echo = FALSE, fig=true, height = 4.5, width = 6>>=
boxplot(before, xlab="before", main="", notch=FALSE, varwidth=TRUE, horizontal=TRUE, data=RcmdrTestDrive) 
@
\par\end{center}

We want to watch out for extreme values (shown as circles separated
from the box) or large departures from symmetry. If the distribution
is fairly symmetric then the mean and median should be approximately
the same. But if the distribution is highly skewed with extreme values
then we should be skeptical of the sample mean, and fall back to the
median which is resistant to extremes. By design, the before variable
is set up to have a fairly symmetric distribution.

A boxplot of \emph{after} is shown next.

\begin{center}
<<echo = FALSE, fig=true, height = 4.5, width = 6>>=
boxplot(after, xlab="after", notch=FALSE, varwidth=TRUE, horizontal=TRUE, data=RcmdrTestDrive) 
@
\par\end{center}

The same remarks apply to the \emph{after} variable. The \emph{after}
variable has been designed to be left-skewed\ldots{} thus, the median
would likely be a good choice for this variable.

\item Based on your answer to (2), choose an appropriate measure of spread
for each variable, calculate it, and report its value. Which variable
has the biggest spread? (Note that you need to make sure that your
measures are on the same scale.) 


Since \emph{before} has a symmetric, mound shaped distribution, an
excellent measure of center would be the sample standard deviation.
And since \emph{after} is left-skewed, we should use the median absolute
deviation. It is also acceptable to use the IQR, but we should rescale
it appropriately, namely, by dividing by 1.349. The exact values are
shown below.

<<>>=
sd(before)
mad(after)
IQR(after)/1.349
@

Judging from the values above, we would decide which variable has
the higher spread. Look at how close the \inputencoding{latin9}\lstinline[showstringspaces=false]!mad!\inputencoding{utf8}
and the \inputencoding{latin9}\lstinline[showstringspaces=false]!IQR!\inputencoding{utf8}
(after suitable rescaling) are; it goes to show why the rescaling
is important.

\item Calculate and report the skewness and kurtosis for \emph{before}.
Based on these values, how would you describe the shape of \emph{before}?


The values of these descriptive measures are shown below.

<<>>=
library(e1071)
skewness(before)
kurtosis(before)
@

We should take the sample skewness value and compare it to $2\sqrt{6/n}\approx$\Sexpr{round(2*sqrt(6/length(before)),3)}
in absolute value to see if it is substantially different from zero.
The direction of skewness is decided by the sign (positive or negative)
of the skewness value. 

We should take the sample kurtosis value and compare it to $2\cdot\sqrt{24/168}\approx$\Sexpr{round(4*sqrt(6/length(before)),3)}),
in absolute value to see if the excess kurtosis is substantially different
from zero. And take a look at the sign to see whether the distribution
is platykurtic or leptokurtic.

\item Calculate and report the skewness and kurtosis for \emph{after}. Based
on these values, how would you describe the shape of \emph{after}?


The values of these descriptive measures are shown below.

<<>>=
skewness(after)
kurtosis(after)
@

We should do for this one just like we did previously. We would again
compare the sample skewness and kurtosis values (in absolute value)
to \Sexpr{round(2*sqrt(6/length(after)),3)} and \Sexpr{round(4*sqrt(6/length(after)),3)},
respectively.

\item Plot histograms of \emph{before} and \emph{after} and compare them
to your answers to (4) and (5).


The graphs are shown below.

\begin{center}
<<echo = FALSE, fig=true, height = 4.5, width = 6>>=
hist(before, xlab="before", data=RcmdrTestDrive) 
@
\par\end{center}

\begin{center}
<<echo = FALSE, fig=true, height = 4.5, width = 6>>=
hist(after, xlab="after", data=RcmdrTestDrive) 
@
\par\end{center}

Answers will vary. We are looking for visual consistency in the histograms
to our statements above.\end{enumerate}

\begin{xca}
Describe the following data sets just as if you were communicating
with an alien, but one who has had a statistics class. Mention the
salient features (data type, important properties, anything special).
Support your answers with the appropriate visual displays and descriptive
statistics.
\begin{enumerate}
\item Conversion rates of Euro currencies stored in \inputencoding{latin9}\lstinline[showstringspaces=false]!euro!\inputencoding{utf8}.
\item State abbreviations stored in \inputencoding{latin9}\lstinline[showstringspaces=false]!state.abb!\inputencoding{utf8}.
\end{enumerate}
\end{xca}

\chapter{Probability\label{cha:Probability}}

\noindent In this chapter we define the basic terminology associated
with probability and derive some of its properties. We discuss three
interpretations of probability. We discuss conditional probability
and independent events, along with Bayes' Theorem. We finish the chapter
with an introduction to random variables, which paves the way for
the next two chapters.

In this book we distinguish between two types of experiments: \emph{deterministic}
and \emph{random}. A \emph{deterministic} experiment is one whose
outcome may be predicted with certainty beforehand, such as combining
Hydrogen and Oxygen, or adding two numbers such as $2+3$. A \emph{random}
experiment is one whose outcome is determined by chance. We posit
that the outcome of a random experiment may not be predicted with
certainty beforehand, even in principle. Examples of random experiments
include tossing a coin, rolling a die, and throwing a dart on a board,
how many red lights you encounter on the drive home, how many ants
traverse a certain patch of sidewalk over a short period, \emph{etc}.


\paragraph*{What do I want them to know?}
\begin{itemize}
\item that there are multiple interpretations of probability, and the methods
used depend somewhat on the philosophy chosen
\item nuts and bolts of basic probability jargon: sample spaces, events,
probability functions, \emph{etc}.
\item how to count
\item conditional probability and its relationship with independence
\item Bayes' Rule and how it relates to the subjective view of probability
\item what we mean by 'random variables', and where they come from
\end{itemize}
%
\begin{figure}
\begin{centering}
<<echo = FALSE, fig = TRUE, width = 4.5, height = 2.75>>=
require(diagram)
par(mex = 0.2, cex = 0.5)
openplotmat(frame.plot=TRUE)
straightarrow(from = c(0.46,0.74), to = c(0.53,0.71), arr.pos = 1)
straightarrow(from = c(0.3,0.65), to = c(0.3,0.51), arr.pos = 1)
textellipse(mid = c(0.74,0.55), box.col = grey(0.95), radx = 0.24, rady = 0.22, lab = c(expression(bold(underline(DETERMINISTIC))), expression(2*H[2]+O[2] %->% H[2]*O), "3 + 4 = 7"), cex = 2 )
textrect(mid = c(0.3, 0.75), radx = 0.15, rady = 0.1, lab = c("Experiments"), cex = 2 )
textellipse(mid = c(0.29,0.25), box.col = grey(0.95), radx = 0.27, rady = 0.22, lab = c(expression(bold(underline(RANDOM))), "toss coin, roll die", "count ants on sidewalk", "measure rainfall" ), cex = 2 )
@
\par\end{centering}

\caption{Two types of experiments}

\end{figure}



\section{Sample Spaces\label{sec:Sample-Spaces}}

For a random experiment $E$, the set of all possible outcomes of
$E$ is called the \emph{sample space\index{sample space}} and is
denoted by the letter $S$. For a coin-toss experiment, $S$ would
be the results {}``Head'' and {}``Tail'', which we may represent
by $S=\left\{ H,T\right\} $. Formally, the performance of a random
experiment is the unpredictable selection of an outcome in $S$.


\subsection{How to do it with \textsf{R}}

Most of the probability work in this book is done with the \inputencoding{latin9}\lstinline[showstringspaces=false]!prob!\inputencoding{utf8}
package \cite{Kernsprob}. A sample space is (usually) represented
by a \emph{data frame}, that is, a rectangular collection of variables
(see Section \ref{sub:Multivariate-Data}). Each row of the data frame
corresponds to an outcome of the experiment. The data frame choice
is convenient both for its simplicity and its compatibility with the
\textsf{R} Commander. Data frames alone are, however, not sufficient
to describe some of the more interesting probabilistic applications
we will study later; to handle those we will need to consider a more
general \emph{list} data structure. See Section \ref{sub:howto-ps-objects}
for details.
\begin{example}
Consider the random experiment of dropping a Styrofoam cup onto the
floor from a height of four feet. The cup hits the ground and eventually
comes to rest. It could land upside down, right side up, or it could
land on its side. We represent these possible outcomes of the random
experiment by the following.

<<>>=
S <- data.frame(lands = c("down","up","side"))
S
@

The sample space \inputencoding{latin9}\lstinline[showstringspaces=false]!S!\inputencoding{utf8}
contains the column \inputencoding{latin9}\lstinline[showstringspaces=false]!lands!\inputencoding{utf8}
which stores the outcomes \inputencoding{latin9}\lstinline[showstringspaces=false]!"down"!\inputencoding{utf8},
\inputencoding{latin9}\lstinline[showstringspaces=false]!"up"!\inputencoding{utf8},
and \inputencoding{latin9}\lstinline[showstringspaces=false]!"side"!\inputencoding{utf8}. 

\end{example}
Some sample spaces are so common that convenience wrappers were written
to set them up with minimal effort. The underlying machinery that
does the work includes the \inputencoding{latin9}\lstinline[showstringspaces=false]!expand.grid!\inputencoding{utf8}
function in the \inputencoding{latin9}\lstinline[showstringspaces=false]!base!\inputencoding{utf8}
package, \inputencoding{latin9}\lstinline[showstringspaces=false]!combn!\inputencoding{utf8}
in the \inputencoding{latin9}\lstinline[showstringspaces=false]!combinat!\inputencoding{utf8}
package \cite{combinat}, and \inputencoding{latin9}\lstinline[showstringspaces=false]!permsn!\inputencoding{utf8}
in the \inputencoding{latin9}\lstinline[showstringspaces=false]!prob!\inputencoding{utf8}
package%
\footnote{The seasoned \textsf{R} user can get the job done without the convenience
wrappers. I encourage the beginner to use them to get started, but
I also recommend that introductory students wean themselves as soon
as possible. The wrappers were designed for ease and intuitive use,
not for speed or efficiency.%
}.

Consider the random experiment of tossing a coin. The outcomes are
$H$ and $T$. We can set up the sample space quickly with the \inputencoding{latin9}\lstinline[showstringspaces=false]!tosscoin!\inputencoding{utf8}
function:

<<>>= 
library(prob)
tosscoin(1) 
@

The number \inputencoding{latin9}\lstinline[showstringspaces=false]!1!\inputencoding{utf8}
tells \inputencoding{latin9}\lstinline[showstringspaces=false]!tosscoin!\inputencoding{utf8}
that we only want to toss the coin once. We could toss it three times: 

<<echo=TRUE,print=TRUE>>= 
tosscoin(3) 
@ 

Alternatively we could roll a fair die: 

<<echo=TRUE,print=TRUE>>= 
rolldie(1) 
@

The \inputencoding{latin9}\lstinline[showstringspaces=false]!rolldie!\inputencoding{utf8}
function defaults to a 6-sided die, but we can specify others with
the \inputencoding{latin9}\lstinline[showstringspaces=false]!nsides!\inputencoding{utf8}
argument. The command \inputencoding{latin9}\lstinline[showstringspaces=false]!rolldie(3, nsides = 4)!\inputencoding{utf8}
would be used to roll a 4-sided die three times.

Perhaps we would like to draw one card from a standard set of playing
cards (it is a long data frame):

<<echo=TRUE,print=TRUE>>= 
head(cards()) 
@

The \inputencoding{latin9}\lstinline[showstringspaces=false]!cards!\inputencoding{utf8}
function that we just used has optional arguments \inputencoding{latin9}\lstinline[showstringspaces=false]!jokers!\inputencoding{utf8}
(if you would like Jokers to be in the deck) and \inputencoding{latin9}\lstinline[showstringspaces=false]!makespace!\inputencoding{utf8}
which we will discuss later. There is also a \inputencoding{latin9}\lstinline[showstringspaces=false]!roulette!\inputencoding{utf8}
function which returns the sample space associated with one spin on
a roulette wheel. There are EU and USA versions available. Interested
readers may contribute any other game or sample spaces that may be
of general interest.


\subsection{Sampling from Urns\label{sub:sampling-from-urns}}

This is perhaps the most fundamental type of random experiment. We
have an urn that contains a bunch of distinguishable objects (balls)
inside. We shake up the urn, reach inside, grab a ball, and take a
look. That's all.

But there are all sorts of variations on this theme. Maybe we would
like to grab more than one ball -- say, two balls. What are all of
the possible outcomes of the experiment now? It depends on how we
sample. We could select a ball, take a look, put it back, and sample
again. Another way would be to select a ball, take a look -- but do
not put it back -- and sample again (equivalently, just reach in and
grab two balls). There are certainly more possible outcomes of the
experiment in the former case than in the latter. In the first (second)
case we say that sampling is done \emph{with} (\emph{without}) \emph{replacement}.

There is more. Suppose we do not actually keep track of which ball
came first. All we observe are the two balls, and we have no idea
about the order in which they were selected. We call this \emph{unordered
sampling} (in contrast to \emph{ordered}) because the order of the
selections does not matter with respect to what we observe. We might
as well have selected the balls and put them in a bag before looking.

Note that this one general class of random experiments contains as
a special case all of the common elementary random experiments. Tossing
a coin twice is equivalent to selecting two balls labeled $H$ and
$T$ from an urn, with replacement. The die-roll experiment is equivalent
to selecting a ball from an urn with six elements, labeled 1 through
6.


\subsection{How to do it with \textsf{R} }

The \inputencoding{latin9}\lstinline[showstringspaces=false]!prob!\inputencoding{utf8}
package accomplishes sampling from urns with the \inputencoding{latin9}\lstinline[showstringspaces=false]!urnsamples!\inputencoding{utf8}\index{urnsamples@\texttt{urnsamples}}
function, which has arguments \inputencoding{latin9}\lstinline[showstringspaces=false]!x!\inputencoding{utf8},
\inputencoding{latin9}\lstinline[showstringspaces=false]!size!\inputencoding{utf8},
\inputencoding{latin9}\lstinline[showstringspaces=false]!replace!\inputencoding{utf8},
and \inputencoding{latin9}\lstinline[showstringspaces=false]!ordered!\inputencoding{utf8}.
The argument \inputencoding{latin9}\lstinline[showstringspaces=false]!x!\inputencoding{utf8}
represents the urn from which sampling is to be done. The \inputencoding{latin9}\lstinline[showstringspaces=false]!size!\inputencoding{utf8}
argument tells how large the sample will be. The \inputencoding{latin9}\lstinline[showstringspaces=false]!ordered!\inputencoding{utf8}
and \inputencoding{latin9}\lstinline[showstringspaces=false]!replace!\inputencoding{utf8}
arguments are logical and specify how sampling will be performed.
We will discuss each in turn.
\begin{example}
\label{exa:sample-urn-two-from-three}Let our urn simply contain three
balls, labeled 1, 2, and 3, respectively. We are going to take a sample
of size 2 from the urn. 

\subsubsection*{Ordered, With Replacement}

If sampling is with replacement, then we can get any outcome 1, 2,
or 3 on any draw. Further, by {}``ordered'' we mean that we shall
keep track of the order of the draws that we observe. We can accomplish
this in \textsf{R} with

<<echo=TRUE,print=TRUE>>=
urnsamples(1:3, size = 2, replace = TRUE, ordered = TRUE)
@ 

Notice that rows 2 and 4 are identical, save for the order in which
the numbers are shown. Further, note that every possible pair of the
numbers 1 through 3 are listed. This experiment is equivalent to rolling
a 3-sided die twice, which we could have accomplished with \inputencoding{latin9}\lstinline[showstringspaces=false]!rolldie(2, nsides = 3)!\inputencoding{utf8}.


\subsubsection*{Ordered, Without Replacement}

Here sampling is without replacement, so we may not observe the same
number twice in any row. Order is still important, however, so we
expect to see the outcomes \inputencoding{latin9}\lstinline[showstringspaces=false]!1,2!\inputencoding{utf8}
and \inputencoding{latin9}\lstinline[showstringspaces=false]!2,1!\inputencoding{utf8}
somewhere in our data frame. 

<<echo=TRUE,print=TRUE>>= 
urnsamples(1:3, size = 2, replace = FALSE, ordered = TRUE)
@ 

This is just as we expected. Notice that there are less rows in this
answer due to the more restrictive sampling procedure. If the numbers
1, 2, and 3 represented {}``Fred'', {}``Mary'', and {}``Sue'',
respectively, then this experiment would be equivalent to selecting
two people of the three to serve as president and vice-president of
a company, respectively, and the sample space shown above lists all
possible ways that this could be done.


\subsubsection*{Unordered, Without Replacement}

Again, we may not observe the same outcome twice, but in this case,
we will only retain those outcomes which (when jumbled) would not
duplicate earlier ones. 

<<echo=TRUE,print=TRUE>>= 
urnsamples(1:3, size = 2, replace = FALSE, ordered = FALSE) 
@ 

This experiment is equivalent to reaching in the urn, picking a pair,
and looking to see what they are. This is the default setting of \inputencoding{latin9}\lstinline[showstringspaces=false]!urnsamples!\inputencoding{utf8},
so we would have received the same output by simply typing \inputencoding{latin9}\lstinline[showstringspaces=false]!urnsamples(1:3, 2)!\inputencoding{utf8}.


\subsubsection*{Unordered, With Replacement}

The last possibility is perhaps the most interesting. We replace the
balls after every draw, but we do not remember the order in which
the draws came. 

<<echo=TRUE,print=TRUE>>= 
urnsamples(1:3, size = 2, replace = TRUE, ordered = FALSE) 
@ 

We may interpret this experiment in a number of alternative ways.
One way is to consider this as simply putting two 3-sided dice in
a cup, shaking the cup, and looking inside -- as in a game of \emph{Liar's
Dice}, for instance. Each row of the sample space is a potential pair
we could observe. Another way is to view each outcome as a separate
method to distribute two identical golf balls into three boxes labeled
1, 2, and 3. Regardless of the interpretation, \inputencoding{latin9}\lstinline[showstringspaces=false]!urnsamples!\inputencoding{utf8}
lists every possible way that the experiment can conclude.

\end{example}
Note that the urn does not need to contain numbers; we could have
just as easily taken our urn to be \inputencoding{latin9}\lstinline[showstringspaces=false]!x = c("Red","Blue","Green")!\inputencoding{utf8}.
But, there is an \textbf{important} point to mention before proceeding.
Astute readers will notice that in our example, the balls in the urn
were \textit{distinguishable} in the sense that each had a unique
label to distinguish it from the others in the urn. A natural question
would be, {}``What happens if your urn has indistinguishable elements,
for example, what if \inputencoding{latin9}\lstinline[showstringspaces=false]!x = c("Red","Red","Blue")!\inputencoding{utf8}?''
The answer is that \inputencoding{latin9}\lstinline[showstringspaces=false]!urnsamples!\inputencoding{utf8}
behaves as if each ball in the urn is distinguishable, regardless
of its actual contents. We may thus imagine that while there are two
red balls in the urn, the balls are such that we can tell them apart
(in principle) by looking closely enough at the imperfections on their
surface.

In this way, when the \inputencoding{latin9}\lstinline[showstringspaces=false]!x!\inputencoding{utf8}
argument of \inputencoding{latin9}\lstinline[showstringspaces=false]!urnsamples!\inputencoding{utf8}
has repeated elements, the resulting sample space may appear to be
\inputencoding{latin9}\lstinline[showstringspaces=false]!ordered = TRUE!\inputencoding{utf8}
even when, in fact, the call to the function was \inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false]!urnsamples(..., ordered = FALSE)!\inputencoding{utf8}.
Similar remarks apply for the \inputencoding{latin9}\lstinline[showstringspaces=false]!replace!\inputencoding{utf8}
argument. 


\section{Events\label{sec:Events}}

An \emph{event\index{event}} $A$ is merely a collection of outcomes,
or in other words, a subset of the sample space%
\footnote{This naive definition works for finite or countably infinite sample
spaces, but is inadequate for sample spaces in general. In this book,
we will not address the subtleties that arise, but will refer the
interested reader to any text on advanced probability or measure theory.%
}. After the performance of a random experiment $E$ we say that the
event $A$ \emph{occurred} if the experiment's outcome belongs to
$A$. We say that a bunch of events $A_{1}$, $A_{2}$, $A_{3}$,
\ldots{} are \emph{mutually exclusive\index{mutually exclusive}}
or \emph{disjoint} if $A_{i}\cap A_{j}=\emptyset$ for any distinct
pair $A_{i}\neq A_{j}$. For instance, in the coin-toss experiment
the events $A=\left\{ \mbox{Heads}\right\} $ and $B=\left\{ \mbox{Tails}\right\} $
would be mutually exclusive. Now would be a good time to review the
algebra of sets in Appendix \ref{sec:The-Algebra-of}.


\subsection{How to do it with \textsf{R}}

Given a data frame sample/probability space \inputencoding{latin9}\lstinline[showstringspaces=false]!S!\inputencoding{utf8},
we may extract rows using the \inputencoding{latin9}\lstinline[showstringspaces=false]![]!\inputencoding{utf8}
operator: 

<<echo=TRUE,print=TRUE>>= 
S <- tosscoin(2, makespace = TRUE) 
S[1:3, ] 
S[c(2,4), ] 
@ 

and so forth. We may also extract rows that satisfy a logical expression
using the \inputencoding{latin9}\lstinline[showstringspaces=false]!subset!\inputencoding{utf8}
function, for instance 

<<echo=TRUE,print=FALSE>>= 
S <- cards() 
@ 

<<echo=TRUE,print=TRUE>>= 
subset(S, suit == "Heart") 
subset(S, rank %in% 7:9)
@ 

We could continue indefinitely. Also note that mathematical expressions
are allowed: 

<<echo=TRUE,print=TRUE>>= 
subset(rolldie(3), X1+X2+X3 > 16) 
@


\subsection{Functions for Finding Subsets}

It does not take long before the subsets of interest become complicated
to specify. Yet the main idea remains: we have a particular logical
condition to apply to each row. If the row satisfies the condition,
then it should be in the subset. It should not be in the subset otherwise.
The ease with which the condition may be coded depends of course on
the question being asked. Here are a few functions to get started.


\subsubsection*{The \texttt{\%in\%} function}

The function \inputencoding{latin9}\lstinline[showstringspaces=false]!%in%!\inputencoding{utf8}
helps to learn whether each value of one vector lies somewhere inside
another vector. 

<<echo=TRUE,print=FALSE>>= 
x <- 1:10 
y <- 8:12 
y %in% x
@ 

Notice that the returned value is a vector of length 5 which tests
whether each element of \inputencoding{latin9}\lstinline[showstringspaces=false]!y!\inputencoding{utf8}
is in \inputencoding{latin9}\lstinline[showstringspaces=false]!x!\inputencoding{utf8},
in turn.


\subsubsection*{The \texttt{isin} function}

It is more common to want to know whether the \emph{whole} vector
\inputencoding{latin9}\lstinline[showstringspaces=false]!y!\inputencoding{utf8}
is in \inputencoding{latin9}\lstinline[showstringspaces=false]!x!\inputencoding{utf8}.
We can do this with the \inputencoding{latin9}\lstinline[showstringspaces=false]!isin!\inputencoding{utf8}
function. 

<<echo=TRUE,print=TRUE>>= 
isin(x,y) 
@

Of course, one may ask why we did not try something like \inputencoding{latin9}\lstinline[showstringspaces=false]!all(y %in% x)!\inputencoding{utf8},
which would give a single result, \inputencoding{latin9}\lstinline[showstringspaces=false]!TRUE!\inputencoding{utf8}.
The reason is that the answers are different in the case that \inputencoding{latin9}\lstinline[showstringspaces=false]!y!\inputencoding{utf8}
has repeated values. Compare: 

<<echo=TRUE,print=FALSE>>= 
x <- 1:10 
y <- c(3,3,7) 
@ 

<<echo=TRUE,print=TRUE>>= 
all(y %in% x)
isin(x,y) 
@ 

The reason for the above is of course that \inputencoding{latin9}\lstinline[showstringspaces=false]!x!\inputencoding{utf8}
contains the value 3, but \inputencoding{latin9}\lstinline[showstringspaces=false]!x!\inputencoding{utf8}
does not have \emph{two} 3's. The difference is important when rolling
multiple dice, playing cards, \emph{etc}. Note that there is an optional
argument \inputencoding{latin9}\lstinline[showstringspaces=false]!ordered!\inputencoding{utf8}
which tests whether the elements of \inputencoding{latin9}\lstinline[showstringspaces=false]!y!\inputencoding{utf8}
appear in \inputencoding{latin9}\lstinline[showstringspaces=false]!x!\inputencoding{utf8}
in the order in which they are appear in \inputencoding{latin9}\lstinline[showstringspaces=false]!y!\inputencoding{utf8}.
The consequences are 

<<echo=TRUE,print=TRUE>>= 
isin(x, c(3,4,5), ordered = TRUE) 
isin(x, c(3,5,4), ordered = TRUE) 
@ 

The connection to probability is that have a data frame sample space
and we would like to find a subset of that space. A \inputencoding{latin9}\lstinline[showstringspaces=false]!data.frame!\inputencoding{utf8}
method was written for \inputencoding{latin9}\lstinline[showstringspaces=false]!isin!\inputencoding{utf8}
that simply applies the function to each row of the data frame. We
can see the method in action with the following: 

<<echo=TRUE,print=FALSE>>= 
S <- rolldie(4) 
subset(S, isin(S, c(2,2,6), ordered = TRUE)) 
@

There are a few other functions written to find useful subsets, namely,
\inputencoding{latin9}\lstinline[showstringspaces=false]!countrep!\inputencoding{utf8}
and \inputencoding{latin9}\lstinline[showstringspaces=false]!isrep!\inputencoding{utf8}.
Essentially these were written to test for (or count) a specific number
of designated values in outcomes. See the documentation for details.


\subsection{Set Union, Intersection, and Difference}

Given subsets $A$ and $B$, it is often useful to manipulate them
in an algebraic fashion. To this end, we have three set operations
at our disposal: union, intersection, and difference. Below is a table
that summarizes the pertinent information about these operations.

\begin{center}
\begin{tabular}{|l|c|l|l|}
\hline 
% after \\: \hline or \cline{col1-col2} \cline{col3-col4} ...
Name  & Denoted  & Defined by elements  & Code \tabularnewline
\hline 
Union  & $A\cup B$  & in $A$ or $B$ or both  & \texttt{union(A,B)} \tabularnewline
Intersection  & $A\cap B$  & in both $A$ and $B$  & \texttt{intersect(A,B)} \tabularnewline
Difference  & $A\textbackslash B$  & in $A$ but not in $B$  & \texttt{setdiff(A,B)} \tabularnewline
\hline
\end{tabular}
\par\end{center}

Some examples follow. 

<<echo=TRUE,print=FALSE>>= 
S = cards() 
A = subset(S, suit == "Heart") 
B = subset(S, rank %in% 7:9)
@ 

We can now do some set algebra: 

<<echo=TRUE,print=TRUE>>= 
union(A,B) 
intersect(A,B) 
setdiff(A,B) 
setdiff(B,A) 
@ 

Notice that \inputencoding{latin9}\lstinline[showstringspaces=false]!setdiff!\inputencoding{utf8}
is not symmetric. Further, note that we can calculate the \emph{complement}
of a set $A$, denoted $A^{c}$ and defined to be the elements of
$S$ that are not in $A$ simply with \inputencoding{latin9}\lstinline[showstringspaces=false]!setdiff(S,A)!\inputencoding{utf8}.

There have been methods written for \inputencoding{latin9}\lstinline[showstringspaces=false]!intersect!\inputencoding{utf8},
\inputencoding{latin9}\lstinline[showstringspaces=false]!setdiff!\inputencoding{utf8},
\inputencoding{latin9}\lstinline[showstringspaces=false]!subset!\inputencoding{utf8},
and \inputencoding{latin9}\lstinline[showstringspaces=false]!union!\inputencoding{utf8}
in the case that the input objects are of class \inputencoding{latin9}\lstinline[showstringspaces=false]!ps!\inputencoding{utf8}.
See Section \ref{sub:howto-ps-objects}.
\begin{note}
When the \inputencoding{latin9}\lstinline[showstringspaces=false]!prob!\inputencoding{utf8}
package loads you will notice a message: {}``\inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false]!The!\inputencoding{utf8}
\inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false]!following!\inputencoding{utf8}
\inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false]!object(s)!\inputencoding{utf8}
\inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false]!are masked from package:base!\inputencoding{utf8}
\inputencoding{latin9}\lstinline[showstringspaces=false]!: intersect!\inputencoding{utf8},
\inputencoding{latin9}\lstinline[showstringspaces=false]!setdiff!\inputencoding{utf8},
\inputencoding{latin9}\lstinline[showstringspaces=false]!union!\inputencoding{utf8}''.
The reason for this message is that there already exist methods for
the functions \inputencoding{latin9}\lstinline[showstringspaces=false]!intersect!\inputencoding{utf8},
\inputencoding{latin9}\lstinline[showstringspaces=false]!setdiff!\inputencoding{utf8},
\inputencoding{latin9}\lstinline[showstringspaces=false]!subset!\inputencoding{utf8},
and \inputencoding{latin9}\lstinline[showstringspaces=false]!union!\inputencoding{utf8}
in the \inputencoding{latin9}\lstinline[showstringspaces=false]!base!\inputencoding{utf8}
package which ships with \textsf{R}. However, these methods were designed
for when the arguments are vectors of the same mode. Since we are
manipulating sample spaces which are data frames and lists, it was
necessary to write methods to handle those cases as well. When the
\inputencoding{latin9}\lstinline[showstringspaces=false]!prob!\inputencoding{utf8}
package is loaded, \textsf{R} recognizes that there are multiple versions
of the same function in the search path and acts to shield the new
definitions from the existing ones. But there is no cause for alarm,
thankfully, because the \inputencoding{latin9}\lstinline[showstringspaces=false]!prob!\inputencoding{utf8}
functions have been carefully defined to match the usual \inputencoding{latin9}\lstinline[showstringspaces=false]!base!\inputencoding{utf8}
package definition in the case that the arguments are vectors.
\end{note}

\section{Model Assignment\label{sec:Interpreting-Probabilities}}

Let us take a look at the coin-toss experiment more closely. What
do we mean when we say {}``the probability of Heads'' or write $\P(\mbox{Heads})$?
Given a coin and an itchy thumb, how do we go about finding what $\P(\mbox{Heads})$
should be?


\subsection{The Measure Theory Approach}

This approach states that the way to handle $\P(\mbox{Heads})$ is
to define a mathematical function, called a \emph{probability measure},
on the sample space. Probability measures satisfy certain axioms (to
be introduced later) and have special mathematical properties, so
not just any mathematical function will do. But in any given physical
circumstance there are typically all sorts of probability measures
from which to choose, and it is left to the experimenter to make a
reasonable choice -- usually based on considerations of objectivity.
For the tossing coin example, a valid probability measure assigns
probability $p$ to the event $\left\{ \mbox{Heads}\right\} $, where
$p$ is some number $0\leq p\leq1$. An experimenter that wishes to
incorporate the symmetry of the coin would choose $p=1/2$ to balance
the likelihood of $\left\{ \mbox{Heads}\right\} $ and $\left\{ \mbox{Tails}\right\} $.

Once the probability measure is chosen (or determined), there is not
much left to do. All assignments of probability are made by the probability
function, and the experimenter needs only to plug the event $\left\{ \mbox{Heads}\right\} $
into to the probability function to find $\P(\mbox{Heads})$. In this
way, the probability of an event is simply a calculated value, nothing
more, nothing less. Of course this is not the whole story; there are
many theorems and consequences associated with this approach that
will keep us occupied for the remainder of this book. The approach
is called \emph{measure theory} because the measure (probability)
of a set (event) is associated with how big it is (how likely it is
to occur).

The measure theory approach is well suited for situations where there
is symmetry to the experiment, such as flipping a balanced coin or
spinning an arrow around a circle with well-defined pie slices. It
is also handy because of its mathematical simplicity, elegance, and
flexibility. There are literally volumes of information that one can
prove about probability measures, and the cold rules of mathematics
allow us to analyze intricate probabilistic problems with vigor. 

The large degree of flexibility is also a disadvantage, however. When
symmetry fails it is not always obvious what an {}``objective''
choice of probability measure should be; for instance, what probability
should we assign to $\left\{ \mbox{Heads}\right\} $ if we spin the
coin rather than flip it? (It is not $1/2$.) Furthermore, the mathematical
rules are restrictive when we wish to incorporate subjective knowledge
into the model, knowledge which changes over time and depends on the
experimenter, such as personal knowledge about the properties of the
specific coin being flipped, or of the person doing the flipping.

The mathematician who revolutionized this way to do probability theory
was Andrey Kolmogorov, who published a landmark monograph in 1933.
See 

\begin{center}
\url{http://www-history.mcs.st-andrews.ac.uk/Mathematicians/Kolmogorov.html}
\par\end{center}

\noindent for more information.


\subsection{Relative Frequency Approach}

This approach states that the way to determine $\P(\mbox{Heads})$
is to flip the coin repeatedly, in exactly the same way each time.
Keep a tally of the number of flips and the number of Heads observed.
Then a good approximation to $\P(\mbox{Heads})$ will be\begin{equation}
\P(\mbox{Heads})\approx\frac{\mbox{number of observed Heads}}{\mbox{total number of flips}}.\end{equation}


The mathematical underpinning of this approach is the celebrated \textbf{Law
of Large Numbers}, which may be loosely described as follows. Let
$E$ be a random experiment in which the event $A$ either does or
does not occur. Perform the experiment repeatedly, in an identical
manner, in such a way that the successive experiments do not influence
each other. After each experiment, keep a running tally of whether
or not the event $A$ occurred. Let $S_{n}$ count the number of times
that $A$ occurred in the $n$ experiments. Then the law of large
numbers says that \begin{equation}
\frac{S_{n}}{n}\to\P(A)\mbox{ as }n\to\infty.\end{equation}


As the reasoning goes, to learn about the probability of an event
$A$ we need only repeat the random experiment to get a reasonable
estimate of the probability's value, and if we are not satisfied with
our estimate then we may simply repeat the experiment more times all
the while confident that with more and more experiments our estimate
will stabilize to the true value. 

The frequentist approach is good because it is relatively light on
assumptions and does not worry about symmetry or claims of objectivity
like the measure-theoretic approach does. It is perfect for the spinning
coin experiment. One drawback to the method is that one can never
know the exact value of a probability, only a long-run approximation.
It also does not work well with experiments that can not be repeated
indefinitely, say, the probability that it will rain today, the chances
that you get will get an A in your Statistics class, or the probability
that the world is destroyed by nuclear war.

This approach was espoused by Richard von Mises in the early twentieth
century, and some of his main ideas were incorporated into the measure
theory approach. See 

\begin{center}
\url{http://www-history.mcs.st-andrews.ac.uk/Biographies/Mises.html}
\par\end{center}

\noindent for more.


\subsection{The Subjective Approach}

The subjective approach interprets probability as the experimenter's
\emph{degree of belief} that the event will occur. The estimate of
the probability of an event is based on the totality of the individual's
knowledge at the time. As new information becomes available, the estimate
is modified accordingly to best reflect his/her current knowledge.
The method by which the probabilities are updated is commonly done
with Bayes' Rule, discussed in Section \ref{sec:Bayes'-Rule}. 

So for the coin toss example, a person may have $\P(\mbox{Heads})=1/2$
in the absence of additional information. But perhaps the observer
knows additional information about the coin or the thrower that would
shift the probability in a certain direction. For instance, parlor
magicians may be trained to be quite skilled at tossing coins, and
some are so skilled that they may toss a fair coin and get nothing
but Heads, indefinitely. I have \emph{seen} this. It was similarly
claimed in \emph{Bringing Down the House} \cite{Mezrich2003} that
MIT students were accomplished enough with cards to be able to cut
a deck to the same location, every single time. In such cases, one
clearly should use the additional information to assign $\P(\mbox{Heads})$
away from the symmetry value of $1/2$.

This approach works well in situations that cannot be repeated indefinitely,
for example, to assign your probability that you will get an A in
this class, the chances of a devastating nuclear war, or the likelihood
that a cure for the common cold will be discovered.

The roots of subjective probability reach back a long time. See 

\begin{center}
\url{http://en.wikipedia.org/wiki/Subjective_probability} 
\par\end{center}

\noindent for a short discussion and links to references about the
subjective approach.


\subsection{Equally Likely Model (ELM)}

We have seen several approaches to the assignment of a probability
model to a given random experiment and they are very different in
their underlying interpretation. But they all cross paths when it
comes to the equally likely model which assigns equal probability
to all elementary outcomes of the experiment.

The ELM appears in the measure theory approach when the experiment
boasts symmetry of some kind. If symmetry guarantees that all outcomes
have equal {}``size'', and if outcomes with equal {}``size'' should
get the same probability, then the ELM is a logical objective choice
for the experimenter. Consider the balanced 6-sided die, the fair
coin, or the dart board with equal-sized wedges.

The ELM appears in the subjective approach when the experimenter resorts
to indifference or ignorance with respect to his/her knowledge of
the outcome of the experiment. If the experimenter has no prior knowledge
to suggest that (s)he prefer Heads over Tails, then it is reasonable
for the him/her to assign equal subjective probability to both possible
outcomes.

The ELM appears in the relative frequency approach as a fascinating
fact of Nature: when we flip balanced coins over and over again, we
observe that the proportion of times that the coin comes up Heads
tends to $1/2$. Of course if we assume that the measure theory applies
then we can prove that the sample proportion must tend to 1/2 as expected,
but that is putting the cart before the horse, in a manner of speaking.

The ELM is only available when there are finitely many elements in
the sample space.


\subsection{How to do it with \textsf{R}}

In the \inputencoding{latin9}\lstinline[showstringspaces=false]!prob!\inputencoding{utf8}
package, a probability space is an object of outcomes \inputencoding{latin9}\lstinline[showstringspaces=false]!S!\inputencoding{utf8}
and a vector of probabilities (called {}``\inputencoding{latin9}\lstinline[showstringspaces=false]!probs!\inputencoding{utf8}'')
with entries that correspond to each outcome in \inputencoding{latin9}\lstinline[showstringspaces=false]!S!\inputencoding{utf8}.
When \inputencoding{latin9}\lstinline[showstringspaces=false]!S!\inputencoding{utf8}
is a data frame, we may simply add a column called \inputencoding{latin9}\lstinline[showstringspaces=false]!probs!\inputencoding{utf8}
to \inputencoding{latin9}\lstinline[showstringspaces=false]!S!\inputencoding{utf8}
and we will be finished; the probability space will simply be a data
frame which we may call \inputencoding{latin9}\lstinline[showstringspaces=false]!S!\inputencoding{utf8}.
In the case that S is a list, we may combine the \inputencoding{latin9}\lstinline[showstringspaces=false]!outcomes!\inputencoding{utf8}
and \inputencoding{latin9}\lstinline[showstringspaces=false]!probs!\inputencoding{utf8}
into a larger list, \inputencoding{latin9}\lstinline[showstringspaces=false]!space!\inputencoding{utf8};
it will have two components: \inputencoding{latin9}\lstinline[showstringspaces=false]!outcomes!\inputencoding{utf8}
and \inputencoding{latin9}\lstinline[showstringspaces=false]!probs!\inputencoding{utf8}.
The only requirements we need are for the entries of \inputencoding{latin9}\lstinline[showstringspaces=false]!probs!\inputencoding{utf8}
to be nonnegative and \inputencoding{latin9}\lstinline[showstringspaces=false]!sum(probs)!\inputencoding{utf8}
to be one.

To accomplish this in \textsf{R}, we may use the \inputencoding{latin9}\lstinline[showstringspaces=false]!probspace!\inputencoding{utf8}
function. The general syntax is \inputencoding{latin9}\lstinline[showstringspaces=false]!probspace(x, probs)!\inputencoding{utf8},
where \inputencoding{latin9}\lstinline[showstringspaces=false]!x!\inputencoding{utf8}
is a sample space of outcomes and \inputencoding{latin9}\lstinline[showstringspaces=false]!probs!\inputencoding{utf8}
is a vector (of the same length as the number of outcomes in \inputencoding{latin9}\lstinline[showstringspaces=false]!x!\inputencoding{utf8}).
The specific choice of \inputencoding{latin9}\lstinline[showstringspaces=false]!probs!\inputencoding{utf8}
depends on the context of the problem, and some examples follow to
demonstrate some of the more common choices.
\begin{example}
The Equally Likely Model asserts that every outcome of the sample
space has the same probability, thus, if a sample space has $n$ outcomes,
then \inputencoding{latin9}\lstinline[showstringspaces=false]!probs!\inputencoding{utf8}
would be a vector of length $n$ with identical entries $1/n$. The
quickest way to generate \inputencoding{latin9}\lstinline[showstringspaces=false]!probs!\inputencoding{utf8}
is with the \inputencoding{latin9}\lstinline[showstringspaces=false]!rep!\inputencoding{utf8}
function. We will start with the experiment of rolling a die, so that
$n=6$. We will construct the sample space, generate the \inputencoding{latin9}\lstinline[showstringspaces=false]!probs!\inputencoding{utf8}
vector, and put them together with \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!probspace!\inputencoding{utf8}. 

<<echo=TRUE,print=TRUE>>= 
outcomes <- rolldie(1) 
p <- rep(1/6, times = 6) 
probspace(outcomes, probs = p) 
@ 

The \inputencoding{latin9}\lstinline[showstringspaces=false]!probspace!\inputencoding{utf8}
function is designed to save us some time in many of the most common
situations. For example, due to the especial simplicity of the sample
space in this case, we could have achieved the same result with only
(note the name change for the first column) 

<<echo=TRUE,print=TRUE>>= 
probspace(1:6, probs = p) 
@ 

Further, since the equally likely model plays such a fundamental role
in the study of probability the \inputencoding{latin9}\lstinline[showstringspaces=false]!probspace!\inputencoding{utf8}
function will assume that the equally model is desired if no \inputencoding{latin9}\lstinline[showstringspaces=false]!probs!\inputencoding{utf8}
are specified. Thus, we get the same answer with only 

<<echo=TRUE,print=TRUE>>= 
probspace(1:6) 
@ 

And finally, since rolling dice is such a common experiment in probability
classes, the \inputencoding{latin9}\lstinline[showstringspaces=false]!rolldie!\inputencoding{utf8}
function has an additional logical argument \inputencoding{latin9}\lstinline[showstringspaces=false]!makespace!\inputencoding{utf8}
that will add a column of equally likely \inputencoding{latin9}\lstinline[showstringspaces=false]!probs!\inputencoding{utf8}
to the generated sample space: 

<<echo=TRUE,print=TRUE>>= 
rolldie(1, makespace = TRUE)
@ 

\noindent or just \inputencoding{latin9}\lstinline[showstringspaces=false]!rolldie(1, TRUE)!\inputencoding{utf8}.
Many of the other sample space functions (\inputencoding{latin9}\lstinline[showstringspaces=false]!tosscoin!\inputencoding{utf8},
\inputencoding{latin9}\lstinline[showstringspaces=false]!cards!\inputencoding{utf8},
\inputencoding{latin9}\lstinline[showstringspaces=false]!roulette!\inputencoding{utf8},
\textit{etc}.) have similar \inputencoding{latin9}\lstinline[showstringspaces=false]!makespace!\inputencoding{utf8}
arguments. Check the documentation for details.

\end{example}
One sample space function that does NOT have a \inputencoding{latin9}\lstinline[showstringspaces=false]!makespace!\inputencoding{utf8}
option is the \inputencoding{latin9}\lstinline[showstringspaces=false]!urnsamples!\inputencoding{utf8}
function. This was intentional. The reason is that under the varied
sampling assumptions the outcomes in the respective sample spaces
are NOT, in general, equally likely. It is important for the user
to carefully consider the experiment to decide whether or not the
outcomes are equally likely and then use \inputencoding{latin9}\lstinline[showstringspaces=false]!probspace!\inputencoding{utf8}
to assign the model.
\begin{example}
\label{exa:unbalanced-coin}\textbf{An unbalanced coin.} While the
\inputencoding{latin9}\lstinline[showstringspaces=false]!makespace!\inputencoding{utf8}
argument to \inputencoding{latin9}\lstinline[showstringspaces=false]!tosscoin!\inputencoding{utf8}
is useful to represent the tossing of a \emph{fair} coin, it is not
always appropriate. For example, suppose our coin is not perfectly
balanced, for instance, maybe the {}``$H$'' side is somewhat heavier
such that the chances of a $H$ appearing in a single toss is 0.70
instead of 0.5. We may set up the probability space with 

<<echo=TRUE,print=TRUE>>= 
probspace(tosscoin(1), probs = c(0.70, 0.30)) 
@ 

The same procedure can be used to represent an unbalanced die, roulette
wheel, \textit{etc}.

\end{example}

\subsection{Words of Warning}

It should be mentioned that while the splendour of \textsf{R }is uncontested,
it, like everything else, has limits both with respect to the sample/probability
spaces it can manage and with respect to the finite accuracy of the
representation of most numbers (see the \textsf{R} FAQ 7.31). When
playing around with probability, one may be tempted to set up a probability
space for tossing 100 coins or rolling 50 dice in an attempt to answer
some scintillating question. (Bear in mind: rolling a die just 9 times
has a sample space with over \emph{10 million} outcomes.)

Alas, even if there were enough RAM to barely hold the sample space
(and there were enough time to wait for it to be generated), the infinitesimal
probabilities that are associated with \emph{so many} outcomes make
it difficult for the underlying machinery to handle reliably. In some
cases, special algorithms need to be called just to give something
that holds asymptotically. User beware.


\section{Properties of Probability\label{sec:Properties-of-Probability}}


\subsection{Probability Functions\label{sub:Probability-Functions}}

A \emph{probability function} is a rule that associates with each
event $A$ of the sample space a unique number $\P(A)=p$, called
the probability of $A$. Any probability function $\P$ satisfies
the following three Kolmogorov Axioms:
\begin{ax}
\label{ax:prob-nonnegative}$\P(A)\geq0$ for any event $A\subset S$.
\end{ax}

\begin{ax}
\label{ax:total-mass-one}$\P(S)=1$.
\end{ax}

\begin{ax}
\label{ax:countable-additivity}If the events $A_{1}$, $A_{2}$,
$A_{3}$\ldots{} are disjoint then\begin{equation}
\P\left(\bigcup_{i=1}^{n}A_{i}\right)=\sum_{i=1}^{n}\P(A_{i})\mbox{ for every }n,\end{equation}
and furthermore,\begin{equation}
\P\left(\bigcup_{i=1}^{\infty}A_{i}\right)=\sum_{i=1}^{\infty}\P(A_{i}).\end{equation}

\end{ax}
The intuition behind the axioms: first, the probability of an event
should never be negative. And since the sample space contains all
possible outcomes, its probability should be one, or 100\%. The final
axiom may look intimidating, but it simply means that for a sequence
of disjoint events (in other words, sets that do not overlap), their
total probability (measure) should equal the sum of its parts. For
example, the chance of rolling a 1 or a 2 on a die is the chance of
rolling a 1 plus the chance of rolling a 2. The connection to measure
theory could not be more clear.


\subsection{Properties}

For any events $A$ and $B$,
\begin{enumerate}
\item $\P(A^{c})=1-\P(A)$.\label{enu:prop-prob-complement} \begin{proof}
Since $A\cup A^{c}=S$ and $A\cap A^{c}=\emptyset$, we have\[
1=\P(S)=\P(A\cup A^{c})=\P(A)+\P(A^{c}).\]

\end{proof}

\item $\P(\emptyset)=0$.

\begin{proof}
Note that $\emptyset=S^{c}$, and use Property 1.
\end{proof}
\item If $A\subset B$ , then $\P(A)\leq\P(B)$.

\begin{proof}
Write $B=A\cup\left(B\cap A^{c}\right)$, and notice that $A\cap\left(B\cap A^{c}\right)=\emptyset$;
thus\[
\P(B)=\P(A\cup\left(B\cap A^{c}\right))=\P(A)+\P\left(B\cap A^{c}\right)\geq\P(A),\]
since $\P\left(B\cap A^{c}\right)\ge0$. 
\end{proof}
\item $0\leq\P(A)\leq1$.

\begin{proof}
The left inequality is immediate from Axiom \ref{ax:prob-nonnegative},
and the second inequality follows from Property 3 since $A\subset S$.
\end{proof}
\item \textbf{\emph{The General Addition Rule.}}\begin{equation}
\P(A\cup B)=\P(A)+\P(B)-\P(A\cap B).\label{eq:general-addition-rule-1}\end{equation}



More generally, for events $A_{1}$, $A_{2}$, $A_{3}$,\ldots{},
$A_{n}$,\begin{equation}
\P\left(\bigcup_{i=1}^{n}A_{i}\right)=\sum_{i=1}^{n}\P(A_{i})-\sum_{i=1}^{n-1}\sum_{j=i+1}^{n}\P(A_{i}\cap A_{j})+\cdots+(-1)^{n-1}\P\left(\bigcap_{i=1}^{n}A_{i}\right)\end{equation}


\item \textbf{\emph{The Theorem of Total Probability.}} Let $B_{1}$, $B_{2}$,
\ldots{}, $B_{n}$ be mutually exclusive and exhaustive. Then\begin{equation}
\P(A)=\P(A\cap B_{1})+\P(A\cap B_{2})+\cdots+\P(A\cap B_{n}).\label{eq:theorem-total-probability}\end{equation}

\end{enumerate}

\subsection{Assigning Probabilities}

A model of particular interest is the \emph{equally likely model}.
The idea is to divide the sample space $S$ into a finite collection
of elementary events $\left\{ a_{1},\ a_{2},\ldots,a_{N}\right\} $
that are equally likely in the sense that each $a_{i}$ has equal
chances of occurring. The probability function associated with this
model must satisfy $\P(S)=1$, by Axiom 2. On the other hand, it must
also satisfy\[
\P(S)=\P(\left\{ a_{1},\ a_{2},\ldots,a_{N}\right\} )=\P(a_{1}\cup a_{2}\cup\cdots\cup a_{N})=\sum_{i=1}^{N}\P(a_{i}),\]
by Axiom 3. Since $\P(a_{i})$ is the same for all $i$, each one
necessarily equals $1/N$. 

For an event $A\subset S$, we write it as a collection of elementary
outcomes: if $A=\left\{ a_{i_{1}},a_{i_{2}},\ldots,a_{i_{k}}\right\} $
then $A$ has $k$ elements and \begin{align*}
\P(A) & =\P(a_{i_{1}})+\P(a_{i_{2}})+\cdots+\P(a_{i_{k}}),\\
 & =\frac{1}{N}+\frac{1}{N}+\cdots+\frac{1}{N},\\
 & =\frac{k}{N}=\frac{\#(A)}{\#(S)}.\end{align*}
In other words, under the equally likely model, the probability of
an event $A$ is determined by the number of elementary events that
$A$ contains.
\begin{example}
Consider the random experiment $E$ of tossing a coin. Then the sample
space is $S=\{H,T\}$, and under the equally likely model, these two
outcomes have $\P(H)=\P(T)=1/2$. This model is taken when it is reasonable
to assume that the coin is fair.
\end{example}

\begin{example}
Suppose the experiment $E$ consists of tossing a fair coin twice.
The sample space may be represented by $S=\{HH,\, HT,\, TH,\, TT\}$.
Given that the coin is fair and that the coin is tossed in an independent
and identical manner, it is reasonable to apply the equally likely
model. 

What is $\P(\mbox{at least 1 Head})$? Looking at the sample space
we see the elements $HH$, $HT$, and $TH$ have at least one Head;
thus, $\P(\mbox{at least 1 Head})=3/4$. 

What is $\P(\mbox{no Heads})$? Notice that the event $\left\{ \mbox{no Heads}\right\} =\left\{ \mbox{at least one Head}\right\} ^{c}$,
which by Property \ref{enu:prop-prob-complement} means $\P(\mbox{no Heads})=1-\P(\mbox{at least one Head})=1-3/4=1/4$.
It is obvious in this simple example that the only outcome with no
Heads is $TT$, however, this complementation trick is useful in more
complicated circumstances.
\end{example}

\begin{example}
\label{exa:three-child-family}Imagine a three child family, each
child being either Boy ($B$) or Girl ($G$). An example sequence
of siblings would be $BGB$. The sample space may be written\[
S=\left\{ \begin{array}{cccc}
BBB, & BGB, & GBB, & GGB,\\
BBG, & BGG, & GBG, & GGG\end{array}\right\} .\]
Note that for many reasons (for instance, it turns out that girls
are slightly more likely to be born than boys), this sample space
is \emph{not} equally likely. For the sake of argument, however, we
will assume that the elementary outcomes each have probability $1/8$.

What is $\P(\mbox{exactly 2 Boys})$? Inspecting the sample space
reveals three outcomes with exactly two boys: $\left\{ BBG,\, BGB,\, GBB\right\} $.
Therefore $\P(\mbox{exactly 2 Boys})=3/8$. 

What is $\P(\mbox{at most 2 Boys})$? One way to solve the problem
would be to count the outcomes that have 2 or less Boys, but a quicker
way would be to recognize that the only way that the event $\left\{ \mbox{at most 2 Boys}\right\} $
does \emph{not} occur is the event $\left\{ \mbox{all Girls}\right\} $.
Thus\[
\P(\mbox{at most 2 Boys})=1-\P(GGG)=1-1/8=7/8.\]

\end{example}

\begin{example}
Consider the experiment of rolling a six-sided die, and let the outcome
be the face showing up when the die comes to rest. Then $S=\left\{ 1,2,3,4,5,6\right\} $.
It is usually reasonable to suppose that the die is fair, so that
the six outcomes are equally likely.
\end{example}

\begin{example}
Consider a standard deck of 52 cards. These are usually labeled with
the four \emph{suits}: Clubs, Diamonds, Hearts, and Spades, and the
13 \emph{ranks}: 2, 3, 4, \ldots{}, 10, Jack (J), Queen (Q), King
(K), and Ace (A). Depending on the game played, the Ace may be ranked
below 2 or above King. 

Let the random experiment $E$ consist of drawing exactly one card
from a well-shuffled deck, and let the outcome be the face of the
card. Define the events $A=\left\{ \mbox{draw an Ace}\right\} $ and
$B=\left\{ \mbox{draw a Club}\right\} $. Bear in mind: we are only
drawing one card.

Immediately we have $\P(A)=4/52$ since there are four Aces in the
deck; similarly, there are $13$ Clubs which implies $\P(B)=13/52$.

What is $\P(A\cap B)$? We realize that there is only one card of
the 52 which is an Ace and a Club at the same time, namely, the Ace
of Clubs. Therefore $\P(A\cap B)=1/52$.

To find $\P(A\cup B)$ we may use the above with the General Addition
Rule to get\begin{eqnarray*}
\P(A\cup B) & = & \P(A)+\P(B)-\P(A\cap B),\\
 & = & 4/52+13/52-1/52,\\
 & = & 16/52.\end{eqnarray*}

\end{example}

\begin{example}
Staying with the deck of cards, let another random experiment be the
selection of a five card stud poker hand, where {}``five card stud''
means that we draw exactly five cards from the deck without replacement,
no more, and no less. It turns out that the sample space $S$ is so
large and complicated that we will be obliged to settle for the trivial
description $S=\left\{ \mbox{all possible 5 card hands}\right\} $
for the time being. We will have a more precise description later.

What is $\P(\mbox{Royal Flush})$, or in other words, $\P(\mbox{A, K, Q, J, 10 all in the same suit})$? 

It should be clear that there are only four possible royal flushes.
Thus, if we could only count the number of outcomes in $S$ then we
could simply divide four by that number and we would have our answer
under the equally likely model. This is the subject of Section \ref{sec:Methods-of-Counting}.
\end{example}

\subsection{How to do it with \textsf{R}}

Probabilities are calculated in the \inputencoding{latin9}\lstinline[showstringspaces=false]!prob!\inputencoding{utf8}
package with the \inputencoding{latin9}\lstinline[showstringspaces=false]!prob!\inputencoding{utf8}
function.

Consider the experiment of drawing a card from a standard deck of
playing cards. Let's denote the probability space associated with
the experiment as \inputencoding{latin9}\lstinline[showstringspaces=false]!S!\inputencoding{utf8},
and let the subsets \inputencoding{latin9}\lstinline[showstringspaces=false]!A!\inputencoding{utf8}
and \inputencoding{latin9}\lstinline[showstringspaces=false]!B!\inputencoding{utf8}
be defined by the following: 

<<echo=TRUE,print=FALSE>>= 
S <- cards(makespace = TRUE) 
A <- subset(S, suit == "Heart") 
B <- subset(S, rank %in% 7:9)
@ 

Now it is easy to calculate 

<<echo=TRUE,print=TRUE>>= 
prob(A) 
@ 

Note that we can get the same answer with 

<<echo=TRUE,print=TRUE>>= 
prob(S, suit == "Heart") 
@ 

We also find \inputencoding{latin9}\lstinline[showstringspaces=false]!prob(B) = 0.23!\inputencoding{utf8}
(listed here approximately, but 12/52 actually) and \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!prob(S) = 1!\inputencoding{utf8}.
Internally, the \inputencoding{latin9}\lstinline[showstringspaces=false]!prob!\inputencoding{utf8}
function operates by summing the \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!probs!\inputencoding{utf8}
column of its argument. It will find subsets on-the-fly if desired.

We have as yet glossed over the details. More specifically, \inputencoding{latin9}\lstinline[showstringspaces=false]!prob!\inputencoding{utf8}
has three arguments: \inputencoding{latin9}\lstinline[showstringspaces=false]!x!\inputencoding{utf8},
which is a probability space (or a subset of one), \inputencoding{latin9}\lstinline[showstringspaces=false]!event!\inputencoding{utf8},
which is a logical expression used to define a subset, and \inputencoding{latin9}\lstinline[showstringspaces=false]!given!\inputencoding{utf8},
which is described in Section \ref{sec:Conditional-Probability}.

\emph{WARNING}. The \inputencoding{latin9}\lstinline[showstringspaces=false]!event!\inputencoding{utf8}
argument is used to define a subset of \inputencoding{latin9}\lstinline[showstringspaces=false]!x!\inputencoding{utf8},
that is, the only outcomes used in the probability calculation will
be those that are elements of \inputencoding{latin9}\lstinline[showstringspaces=false]!x!\inputencoding{utf8}
and satisfy \inputencoding{latin9}\lstinline[showstringspaces=false]!event!\inputencoding{utf8}
simultaneously. In other words, \inputencoding{latin9}\lstinline[showstringspaces=false]!prob(x, event)!\inputencoding{utf8}
calculates 

\inputencoding{latin9}
\begin{lstlisting}[breaklines=true,showstringspaces=false]
prob(intersect(x, subset(x, event)))
\end{lstlisting}
\inputencoding{utf8}

Consequently, \inputencoding{latin9}\lstinline[showstringspaces=false]!x!\inputencoding{utf8}
should be the entire probability space in the case that \inputencoding{latin9}\lstinline[showstringspaces=false]!event!\inputencoding{utf8}
is non-null.


\section{Counting Methods\label{sec:Methods-of-Counting}}

The equally-likely model is a convenient and popular way to analyze
random experiments. And when the equally likely model applies, finding
the probability of an event $A$ amounts to nothing more than counting
the number of outcomes that $A$ contains (together with the number
of events in $S$). Hence, to be a master of probability one must
be skilled at counting outcomes in events of all kinds.
\begin{prop}
The Multiplication Principle. Suppose that an experiment is composed
of two successive steps. Further suppose that the first step may be
performed in $n_{1}$ distinct ways while the second step may be performed
in $n_{2}$ distinct ways. Then the experiment may be performed in
$n_{1}n_{2}$ distinct ways.

More generally, if the experiment is composed of $k$ successive steps
which may be performed in $n_{1}$, $n_{2}$, \ldots{}, $n_{k}$
distinct ways, respectively, then the experiment may be performed
in $n_{1}n_{2}\cdots n_{k}$ distinct ways.\end{prop}
\begin{example}
We would like to order a pizza. It will be sure to have cheese (and
marinara sauce), but we may elect to add one or more of the following
five (5) available toppings:\[
\mbox{pepperoni, sausage, anchovies, olives, and green peppers.}\]
How many distinct pizzas are possible?

There are many ways to approach the problem, but the quickest avenue
employs the Multiplication Principle directly. We will separate the
action of ordering the pizza into a series of stages. At the first
stage, we will decide whether or not to include pepperoni on the pizza
(two possibilities). At the next stage, we will decide whether or
not to include sausage on the pizza (again, two possibilities). We
will continue in this fashion until at last we will decide whether
or not to include green peppers on the pizza.

At each stage we will have had two options, or ways, to select a pizza
to be made. The Multiplication Principle says that we should multiply
the 2's to find the total number of possible pizzas: $2\cdot2\cdot2\cdot2\cdot2=2^{5}=32$.
\end{example}

\begin{example}
We would like to buy a desktop computer to study statistics. We go
to a website to build our computer our way. Given a line of products
we have many options to customize our computer. In particular, there
are 2 choices for a processor, 3 different operating systems, 4 levels
of memory, 4 hard drives of differing sizes, and 10 choices for a
monitor. How many possible types of computer must Gell be prepared
to build? \textbf{Answer:} $2\cdot3\cdot4\cdot4\cdot10=960$.
\end{example}

\subsection{Ordered Samples}

Imagine a bag with $n$ distinguishable balls inside. Now shake up
the bag and select $k$ balls at random. How many possible sequences
might we observe?
\begin{prop}
The number of ways in which one may select an ordered sample of $k$
subjects from a population that has $n$ distinguishable members is
\begin{itemize}
\item $n^{k}$ if sampling is done with replacement,
\item $n(n-1)(n-2)\cdots(n-k+1)$ if sampling is done without replacement.
\end{itemize}
\end{prop}
Recall from calculus the notation for \emph{factorials}: \begin{eqnarray*}
1! & = & 1,\\
2! & = & 2\cdot1=2,\\
3! & = & 3\cdot2\cdot1=6,\\
 & \vdots\\
n! & = & n(n-1)(n-2)\cdots3\cdot2\cdot1.\end{eqnarray*}

\begin{fact}
The number of permutations of $n$ elements is $n!$.\end{fact}
\begin{example}
Take a coin and flip it 7 times. How many sequences of Heads and Tails
are possible? \textbf{Answer:} $2^{7}=128$.
\end{example}

\begin{example}
In a class of 20 students, we randomly select a class president, a
class vice-president, and a treasurer. How many ways can this be done?
\textbf{Answer:} $20\cdot19\cdot18=6840$.
\end{example}

\begin{example}
We rent five movies to watch over the span of two nights. We wish
to watch 3 movies on the first night. How many distinct sequences
of 3 movies could we possibly watch? \textbf{Answer:} $5\cdot4\cdot3=60$.
\end{example}

\subsection{Unordered Samples}

The number of ways in which one may select an unordered sample of
$k$ subjects from a population that has $n$ distinguishable members
is
\begin{itemize}
\item $(n-1+k)!/[(n-1)!k!]$ if sampling is done with replacement,
\item $n!/[k!(n-k)!]$ if sampling is done without replacement.
\end{itemize}
The quantity $n!/[k!(n-k)!]$ is called a \emph{binomial coefficient}
and plays a special role in mathematics; it is denoted\begin{equation}
{n \choose k}=\frac{n!}{k!(n-k)!}\label{eq:binomial-coefficient}\end{equation}
and is read {}``$n$ choose $k$''.
\begin{example}
You rent five movies to watch over the span of two nights, but only
wish to watch 3 movies the first night. Your friend, Fred, wishes
to borrow some movies to watch at his house on the first night. You
owe Fred a favor, and allow him to select 2 movies from the set of
5. How many choices does Fred have? \textbf{Answer:} ${5 \choose 2}=10$.
\end{example}

\begin{example}
Place 3 six-sided dice into a cup. Next, shake the cup well and pour
out the dice. How many distinct rolls are possible? \textbf{Answer:}
$(6-1+3)!/[(6-1)!3!]={8 \choose 5}=56$. 
\end{example}

\subsection{How to do it with \textsf{R}}

The factorial $n!$ is computed with the command \inputencoding{latin9}\lstinline[showstringspaces=false]!factorial(n)!\inputencoding{utf8}
and the binomial coefficient ${n \choose k}$ with the command \inputencoding{latin9}\lstinline[showstringspaces=false]!choose(n,k)!\inputencoding{utf8}.

The sample spaces we have computed so far have been relatively small,
and we can visually study them without much trouble. However, it is
\emph{very} easy to generate sample spaces that are prohibitively
large. And while \textsf{R} is wonderful and powerful and does almost
everything except wash windows, even \textsf{R} has limits of which
we should be mindful.

But we often do not need to actually generate the sample space; it
suffices to count the number of outcomes. The \inputencoding{latin9}\lstinline[showstringspaces=false]!nsamp!\inputencoding{utf8}
function will calculate the number of rows in a sample space made
by \inputencoding{latin9}\lstinline[showstringspaces=false]!urnsamples!\inputencoding{utf8}
without actually devoting the memory resources necessary to generate
the space. The arguments are \inputencoding{latin9}\lstinline[showstringspaces=false]!n!\inputencoding{utf8},
the number of (distinguishable) objects in the urn, \inputencoding{latin9}\lstinline[showstringspaces=false]!k!\inputencoding{utf8},
the sample size, and \inputencoding{latin9}\lstinline[showstringspaces=false]!replace!\inputencoding{utf8},
\inputencoding{latin9}\lstinline[showstringspaces=false]!ordered!\inputencoding{utf8},
as above.

\begin{center}
%
\begin{table}
\begin{centering}
\begin{tabular}{l|cc}
 & \inputencoding{latin9}\lstinline[basicstyle={\ttfamily},showstringspaces=false,tabsize=4]!ordered = TRUE!\inputencoding{utf8} & \inputencoding{latin9}\lstinline[showstringspaces=false,tabsize=4]!ordered = FALSE!\inputencoding{utf8}\tabularnewline
\hline
 &  & \tabularnewline
\inputencoding{latin9}\lstinline[showstringspaces=false,tabsize=4]!replace = TRUE!\inputencoding{utf8} & $n^{k}$ & $\frac{(n-1+k)!}{(n-1)!k!}$\tabularnewline
 &  & \tabularnewline
\inputencoding{latin9}\lstinline[showstringspaces=false,tabsize=4]!replace = FALSE!\inputencoding{utf8} & $\frac{n!}{(n-k)!}$ & ${n \choose k}$\tabularnewline
\multicolumn{1}{l}{} &  & \tabularnewline
\end{tabular}
\par\end{centering}

\caption{Sampling $k$ from $n$ objects with \texttt{urnsamples}\label{tab:Sampling-k-from-n}}

\end{table}

\par\end{center}
\begin{example}
We will compute the number of outcomes for each of the four \inputencoding{latin9}\lstinline[showstringspaces=false]!urnsamples!\inputencoding{utf8}
examples that we saw in Example \ref{exa:sample-urn-two-from-three}.
Recall that we took a sample of size two from an urn with three distinguishable
elements.
\end{example}
<<echo=TRUE,print=TRUE>>= 
nsamp(n=3, k=2, replace = TRUE, ordered = TRUE) 
nsamp(n=3, k=2, replace = FALSE, ordered = TRUE) 
nsamp(n=3, k=2, replace = FALSE, ordered = FALSE) 
nsamp(n=3, k=2, replace = TRUE, ordered = FALSE) 
@ 

Compare these answers with the length of the data frames generated
above.


\subsubsection*{The Multiplication Principle}

A benefit of \inputencoding{latin9}\lstinline[showstringspaces=false]!nsamp!\inputencoding{utf8}
is that it is \emph{vectorized} so that entering vectors instead of
numbers for \inputencoding{latin9}\lstinline[showstringspaces=false]!n!\inputencoding{utf8},
\inputencoding{latin9}\lstinline[showstringspaces=false]!k!\inputencoding{utf8},
\inputencoding{latin9}\lstinline[showstringspaces=false]!replace!\inputencoding{utf8},
and \inputencoding{latin9}\lstinline[showstringspaces=false]!ordered!\inputencoding{utf8}
results in a vector of corresponding answers. This becomes particularly
convenient for combinatorics problems.
\begin{example}
There are 11 artists who each submit a portfolio containing 7 paintings
for competition in an art exhibition. Unfortunately, the gallery director
only has space in the winners' section to accommodate 12 paintings
in a row equally spread over three consecutive walls. The director
decides to give the first, second, and third place winners each a
wall to display the work of their choice. The walls boast 31 separate
lighting options apiece. How many displays are possible?

Answer: The judges will pick 3 (ranked) winners out of 11 (with \inputencoding{latin9}\lstinline[showstringspaces=false]!rep = FALSE!\inputencoding{utf8},
\inputencoding{latin9}\lstinline[showstringspaces=false]!ord = TRUE!\inputencoding{utf8}).
Each artist will select 4 of his/her paintings from 7 for display
in a row (\inputencoding{latin9}\lstinline[showstringspaces=false]!rep = FALSE!\inputencoding{utf8},
\inputencoding{latin9}\lstinline[showstringspaces=false]!ord = TRUE!\inputencoding{utf8}),
and lastly, each of the 3 walls has 31 lighting possibilities (\inputencoding{latin9}\lstinline[showstringspaces=false]!rep = TRUE!\inputencoding{utf8},
\inputencoding{latin9}\lstinline[showstringspaces=false]!ord = TRUE!\inputencoding{utf8}).
These three numbers can be calculated quickly with 

<<echo=TRUE,print=FALSE>>= 
n <- c(11,7,31) 
k <- c(3,4,3) 
r <- c(FALSE,FALSE,TRUE) 
@ 

<<echo=TRUE,print=TRUE>>= 
x <- nsamp(n, k, rep = r, ord = TRUE) 
@ 

(Notice that \inputencoding{latin9}\lstinline[showstringspaces=false]!ordered!\inputencoding{utf8}
is always \inputencoding{latin9}\lstinline[showstringspaces=false]!TRUE!\inputencoding{utf8};
\inputencoding{latin9}\lstinline[showstringspaces=false]!nsamp!\inputencoding{utf8}
will recycle \inputencoding{latin9}\lstinline[showstringspaces=false]!ordered!\inputencoding{utf8}
and \inputencoding{latin9}\lstinline[showstringspaces=false]!replace!\inputencoding{utf8}
to the appropriate length.) By the Multiplication Principle, the number
of ways to complete the experiment is the product of the entries of
\inputencoding{latin9}\lstinline[showstringspaces=false]!x!\inputencoding{utf8}: 

<<echo=TRUE,print=TRUE>>= 
prod(x) 
@ 

Compare this with the some other ways to compute the same thing: 

<<echo=TRUE,print=TRUE>>= 
(11*10*9)*(7*6*5*4)*313 
@

or alternatively 

<<echo=TRUE,print=TRUE>>= 
prod(9:11)*prod(4:7)*313 
@ 

or even 

<<echo=TRUE,print=TRUE>>= 
prod(factorial(c(11,7))/factorial(c(8,3)))*313 
@ 

\end{example}
As one can guess, in many of the standard counting problems there
aren't substantial savings in the amount of typing; it is about the
same using \inputencoding{latin9}\lstinline[showstringspaces=false]!nsamp!\inputencoding{utf8}
versus \inputencoding{latin9}\lstinline[showstringspaces=false]!factorial!\inputencoding{utf8}
and \inputencoding{latin9}\lstinline[showstringspaces=false]!choose!\inputencoding{utf8}.
But the virtue of \inputencoding{latin9}\lstinline[showstringspaces=false]!nsamp!\inputencoding{utf8}
\texttt{l}ies in its collecting the relevant counting formulas in
a one-stop shop. Ultimately, it is up to the user to choose the method
that works best for him/herself. 
\begin{example}
\textbf{The Birthday Problem.} Suppose that there are $n$ people
together in a room. Each person announces the date of his/her birthday
in turn. The question is: what is the probability of at least one
match? If we let the event $A$ represent $\left\{ \mbox{there is at least one match}\right\} $,
then would like to know $\P(A)$, but as we will see, it is more convenient
to calculate $\P(A^{c})$.

For starters we will ignore leap years and assume that there are only
365 days in a year. Second, we will assume that births are equally
distributed over the course of a year (which is not true due to all
sorts of complications such as hospital delivery schedules). See \url{http://en.wikipedia.org/wiki/Birthday_problem}
for more.

Let us next think about the sample space. There are 365 possibilities
for the first person's birthday, 365 possibilities for the second,
and so forth. The total number of possible birthday sequences is therefore
$\#(S)=365^{n}$.

Now we will use the complementation trick we saw in Example \ref{exa:three-child-family}.
We realize that the only situation in which $A$ does \emph{not} occur
is if there are \emph{no} matches among all people in the room, that
is, only when everybody's birthday is different, so\[
\P(A)=1-\P(A^{c})=1-\frac{\#(A^{c})}{\#(S)},\]
since the outcomes are equally likely. Let us then suppose that there
are no matches. The first person has one of 365 possible birthdays.
The second person must not match the first, thus, the second person
has only 364 available birthdays from which to choose. Similarly,
the third person has only 363 possible birthdays, and so forth, until
we reach the $n$$^{\text{th}}$ person, who has only $365-n+1$ remaining
possible days for a birthday. By the Multiplication Principle, we
have $\#(A^{c})=365\cdot364\cdots(365-n+1)$, and\begin{equation}
\P(A)=1-\frac{365\cdot364\cdots(365-n+1)}{365^{n}}=1-\frac{364}{365}\cdot\frac{363}{365}\cdots\frac{(365-n+1)}{365}.\end{equation}


As a surprising consequence, consider this: how many people does it
take to be in the room so that the probability of at least one match
is at least 0.50? Clearly, if there is only $n=1$ person in the room
then the probability of a match is zero, and when there are $n=366$
people in the room there is a 100\% chance of a match (recall that
we are ignoring leap years). So how many people does it take so that
there is an equal chance of a match and no match?

When I have asked this question to students, the usual response is
{}``somewhere around $n=180$ people'' in the room. The reasoning
seems to be that in order to get a 50\% chance of a match, there should
be 50\% of the available days to be occupied. The number of students
in a typical classroom is 25, so as a companion question I ask students
to estimate the probability of a match when there are $n=25$ students
in the room. Common estimates are a 1\%, or 0.5\%, or even 0.1\% chance
of a match. After they have given their estimates, we go around the
room and each student announces their birthday. More often than not,
we observe a match in the class, to the students' disbelief.

Students are usually surprised to hear that, using the formula above,
one needs only $n=23$ students to have a greater than 50\% chance
of at least one match. Figure \ref{fig:The-Birthday-Problem} shows
a graph of the birthday probabilities:
\end{example}
%
\begin{figure}
\begin{centering}
<<echo = FALSE, fig=true, height = 4, width = 4>>=
g <- Vectorize(pbirthday.ipsur)
plot(1:50, g(1:50), xlab = "Number of people in room", ylab = "Prob(at least one match)")
abline(h = 0.5)
abline(v = 23, lty = 2)
remove(g)
@
\par\end{centering}

\caption{The birthday problem\label{fig:The-Birthday-Problem}}
{\small The horizontal line is at $p=0.50$ and the vertical line
is at $n=23$.}
\end{figure}





\subsection{How to do it with \textsf{R}}

We can make the plot in Figure \ref{fig:The-Birthday-Problem} with
the following sequence of commands.

\inputencoding{latin9}
\begin{lstlisting}[basicstyle={\ttfamily},breaklines=true,showstringspaces=false,tabsize=2]
	g <- Vectorize(pbirthday.ipsur)
	plot(1:50, g(1:50),
			xlab = "Number of people in room",
			ylab = "Prob(at least one match)",
			main = "The Birthday Problem")
	abline(h = 0.5)
	abline(v = 23, lty = 2)	# dashed line
\end{lstlisting}
\inputencoding{utf8}

There is a \textsf{Birthday problem} item in the \textsf{Probability}
menu of \inputencoding{latin9}\lstinline[showstringspaces=false]!RcmdrPlugin.IPSUR!\inputencoding{utf8}.

In the base \textsf{R} version, one can compute approximate probabilities
for the more general case of probabilities other than 1/2, for differing
total number of days in the year, and even for more than two matches.


\section{Conditional Probability\label{sec:Conditional-Probability}}

Consider a full deck of 52 standard playing cards. Now select two
cards from the deck, in succession. Let $A=\left\{ \mbox{first card drawn is an Ace}\right\} $
and $B=\left\{ \mbox{second card drawn is an Ace}\right\} $. Since
there are four Aces in the deck, it is natural to assign $\P(A)=4/52$.
Suppose we look at the first card. What now is the probability of
$B$? Of course, the answer depends on the value of the first card.
If the first card is an Ace, then the probability that the second
also is an Ace should be $3/51$, but if the first card is not an
Ace, then the probability that the second is an Ace should be $4/51$.
As notation for these two situations we write\[
\P(B|A)=3/51,\quad\P(B|A^{c})=4/51.\]

\begin{defn}
The conditional probability of $B$ given $A$, denoted $\P(B|A)$,
is defined by\begin{equation}
\P(B|A)=\frac{\P(A\cap B)}{\P(A)},\quad\mbox{if }\P(A)>0.\end{equation}
We will not be discussing a conditional probability of $B$ given
$A$ when $\P(A)=0$, even though this theory exists, is well developed,
and forms the foundation for the study of stochastic processes%
\footnote{Conditional probability in this case is defined by means of \emph{conditional
expectation}, a topic that is well beyond the scope of this text.
The interested reader should consult an advanced text on probability
theory, such as Billingsley, Resnick, or Ash Dooleans-Dade.%
}. \end{defn}
\begin{example}
Toss a coin twice. The sample space is given by $S=\left\{ HH,\ HT,\ TH,\ TT\right\} $.
Let $A=\left\{ \mbox{a head occurs}\right\} $ and $B=\left\{ \mbox{a head and tail occur}\right\} $.
It should be clear that $\P(A)=3/4$, $\P(B)=2/4$, and $\P(A\cap B)=2/4$.
What now are the probabilities $\P(A|B)$ and $\P(B|A)$?\[
\P(A|B)=\frac{\P(A\cap B)}{\P(B)}=\frac{2/4}{2/4}=1,\]
in other words, once we know that a Head and Tail occur, we may be
certain that a Head occurs. Next\[
\P(B|A)=\frac{\P(A\cap B)}{\P(A)}=\frac{2/4}{3/4}=\frac{2}{3},\]
which means that given the information that a Head has occurred, we
no longer need to account for the outcome $TT$, and the remaining
three outcomes are equally likely with exactly two outcomes lying
in the set $B$. 
\end{example}

\begin{example}
\label{exa:Toss-a-six-sided-die-twice}Toss a six-sided die twice.
The sample space consists of all ordered pairs $(i,j)$ of the numbers
$1,2,\ldots,6$, that is, $S=\left\{ (1,1),\ (1,2),\ldots,(6,6)\right\} $.
We know from Section \ref{sec:Methods-of-Counting} that $\#(S)=6^{2}=36$.
Let $A=\left\{ \mbox{outcomes match}\right\} $ and $B=\left\{ \mbox{sum of outcomes at least 8}\right\} $.
The sample space may be represented by a matrix:

%
\begin{table}
\begin{centering}
\begin{tabular}{c}
\begin{sideways}
First Roll%
\end{sideways}\tabularnewline
\end{tabular}\begin{tabular}{c|cccccc|}
\multicolumn{1}{c}{} & \multicolumn{6}{c}{Second Roll}\tabularnewline
\multicolumn{1}{c}{} & 1 & 2 & 3 & 4 & 5 & \multicolumn{1}{c}{6}\tabularnewline
\cline{2-7} 
1 & $\varprod$ &  &  &  &  & \tabularnewline
2 &  & $\varprod$ &  &  &  & {\Large $\bigcirc$}\tabularnewline
3 &  &  & $\varprod$ &  & {\Large $\bigcirc$} & {\Large $\bigcirc$}\tabularnewline
4 &  &  &  & {\huge $\otimes$} & {\Large $\bigcirc$} & {\Large $\bigcirc$}\tabularnewline
5 &  &  & {\Large $\bigcirc$} & {\Large $\bigcirc$} & {\huge $\otimes$} & {\Large $\bigcirc$}\tabularnewline
6 &  & {\Large $\bigcirc$} & {\Large $\bigcirc$} & {\Large $\bigcirc$} & {\Large $\bigcirc$} & {\huge $\otimes$}\tabularnewline
\cline{2-7} 
\end{tabular}
\par\end{centering}

\caption{Rolling two dice\label{tab:Rolling-two-dice}}

\end{table}


The outcomes lying in the event $A$ are marked with the symbol {}``$\varprod$'',
the outcomes falling in $B$ are marked with {}``$\bigcirc$'',
and those in both $A$ and $B$ are marked {}``$\otimes$''. Now
it is clear that $\P(A)=6/36$, $\P(B)=15/36$, and $\P(A\cap B)=3/36$.
Finally, \[
\P(A|B)=\frac{3/36}{15/36}=\frac{1}{5},\quad\P(B|A)=\frac{3/36}{6/36}=\frac{1}{2}.\]
Again, we see that given the knowledge that $B$ occurred (the 15
outcomes in the lower right triangle), there are 3 of the 15 that
fall into the set $A$, thus the probability is $3/15$. Similarly,
given that $A$ occurred (we are on the diagonal), there are 3 out
of 6 outcomes that also fall in $B$, thus, the probability of $B$
given $A$ is 1/2. 
\end{example}

\subsection{How to do it with \textsf{R}}

Continuing with Example \ref{exa:Toss-a-six-sided-die-twice}, the
first thing to do is set up the probability space with the \inputencoding{latin9}\lstinline[showstringspaces=false]!rolldie!\inputencoding{utf8}
function.

<<keep.source = TRUE>>=
library(prob)
S <- rolldie(2, makespace = TRUE)  # assumes ELM
head(S)                            #  first few rows
@

Next we define the events

<<keep.source = TRUE>>=
A <- subset(S, X1 == X2)
B <- subset(S, X1 + X2 >= 8)
@

And now we are ready to calculate probabilities. To do conditional
probability, we use the \inputencoding{latin9}\lstinline[showstringspaces=false]!given!\inputencoding{utf8}
argument of the \inputencoding{latin9}\lstinline[showstringspaces=false]!prob!\inputencoding{utf8}
function:

<<keep.source = TRUE>>=
prob(A, given = B)
prob(B, given = A)
@

Note that we do not actually need to define the events $A$ and $B$
separately as long as we reference the original probability space
$S$ as the first argument of the \inputencoding{latin9}\lstinline[showstringspaces=false]!prob!\inputencoding{utf8}
calculation:

<<keep.source = TRUE>>=
prob(S, X1==X2, given = (X1 + X2 >= 8) )
prob(S, X1+X2 >= 8, given = (X1==X2) )
@


\subsection{Properties and Rules}

The following theorem establishes that conditional probabilities behave
just like regular probabilities when the conditioned event is fixed. 
\begin{thm}
For any fixed event $A$ with $\P(A)>0$,
\begin{enumerate}
\item $\P(B|A)\geq0,$ for all events $B\subset S$,
\item $\P(S|A)=1$, and
\item If $B_{1}$, $B_{2}$, $B_{3}$,\ldots{} are disjoint events, then\begin{equation}
\P\left(\left.\bigcup_{k=1}^{\infty}B_{k}\:\right|A\right)=\sum_{k=1}^{\infty}\P(B_{k}|A).\end{equation}

\end{enumerate}
\end{thm}
In other words, $\P(\cdot|A)$ is a legitimate probability function.
With this fact in mind, the following properties are immediate:
\begin{prop}
For any events $A$, $B$, and $C$ with $\P(A)>0$,
\begin{enumerate}
\item $\P(B^{c}|A)=1-\P(B|A).$
\item If $B\subset C$ then $\P(B|A)\leq\P(C|A)$.
\item $\P[(B\cup C)|A]=\P(B|A)+\P(C|A)-\P[(B\cap C|A)].$
\item \textbf{The Multiplication Rule.} For any two events $A$ and $B$,\begin{equation}
\P(A\cap B)=\P(A)\P(B|A).\label{eq:multiplication-rule-short}\end{equation}
And more generally, for events $A_{1}$, $A_{2}$, $A_{3}$,\ldots{},
$A_{n}$,\begin{equation}
\P(A_{1}\cap A_{2}\cap\cdots\cap A_{n})=\P(A_{1})\P(A_{2}|A_{1})\cdots\P(A_{n}|A_{1}\cap A_{2}\cap\cdots\cap A_{n-1}).\label{eq:multiplication-rule-long}\end{equation}

\end{enumerate}
\end{prop}


The Multiplication Rule is very important because it allows us to
find probabilities in random experiments that have a sequential structure,
as the next example shows.
\begin{example}
\label{exa:two-cards-both-aces}At the beginning of the section we
drew two cards from a standard playing deck. Now we may answer our
original question, what is $\P(\mbox{both Aces})$?\[
\P(\mbox{both Aces})=\P(A\cap B)=\P(A)\P(B|A)=\frac{4}{52}\cdot\frac{3}{51}\approx0.00452.\]

\end{example}

\subsection{How to do it with \textsf{R\label{sub:howto-ps-objects}}}

Continuing Example \ref{exa:two-cards-both-aces}, we set up the probability
space by way of a three step process. First we employ the \inputencoding{latin9}\lstinline[showstringspaces=false]!cards!\inputencoding{utf8}
function to get a data frame \inputencoding{latin9}\lstinline[showstringspaces=false]!L!\inputencoding{utf8}
with two columns: \inputencoding{latin9}\lstinline[showstringspaces=false]!rank!\inputencoding{utf8}
and \inputencoding{latin9}\lstinline[showstringspaces=false]!suit!\inputencoding{utf8}.
Both columns are stored internally as factors with 13 and 4 levels,
respectively.

Next we sample two cards randomly from the \inputencoding{latin9}\lstinline[showstringspaces=false]!L!\inputencoding{utf8}
data frame by way of the \inputencoding{latin9}\lstinline[showstringspaces=false]!urnsamples!\inputencoding{utf8}
function. It returns a list \inputencoding{latin9}\lstinline[showstringspaces=false]!M!\inputencoding{utf8}
which contains all possible pairs of rows from \inputencoding{latin9}\lstinline[showstringspaces=false]!L!\inputencoding{utf8}
(there are \inputencoding{latin9}\lstinline[showstringspaces=false]!choose(52,2)!\inputencoding{utf8}
of them). The sample space for this experiment is exactly the list
\inputencoding{latin9}\lstinline[showstringspaces=false]!M!\inputencoding{utf8}.

At long last we associate a probability model with the sample space.
This is right down the \inputencoding{latin9}\lstinline[showstringspaces=false]!probspace!\inputencoding{utf8}
function's alley. It assumes the equally likely model by default.
We call this result \inputencoding{latin9}\lstinline[showstringspaces=false]!N!\inputencoding{utf8}
which is an object of class \inputencoding{latin9}\lstinline[showstringspaces=false]!ps!\inputencoding{utf8}
-- short for {}``probability space''. 

But do not be intimidated. The object \inputencoding{latin9}\lstinline[showstringspaces=false]!N!\inputencoding{utf8}
is nothing more than a list with two elements: \inputencoding{latin9}\lstinline[showstringspaces=false]!outcomes!\inputencoding{utf8}
and \inputencoding{latin9}\lstinline[showstringspaces=false]!probs!\inputencoding{utf8}.
The \inputencoding{latin9}\lstinline[showstringspaces=false]!outcomes!\inputencoding{utf8}
element is itself just another list, with \inputencoding{latin9}\lstinline[showstringspaces=false]!choose(52,2)!\inputencoding{utf8}
entries, each one a data frame with two rows which correspond to the
pair of cards chosen. The \inputencoding{latin9}\lstinline[showstringspaces=false]!probs!\inputencoding{utf8}
element is just a vector with \inputencoding{latin9}\lstinline[showstringspaces=false]!choose(52,2)!\inputencoding{utf8}
entries all the same: \inputencoding{latin9}\lstinline[showstringspaces=false]!1/choose(52,2)!\inputencoding{utf8}.

Putting all of this together we do

<<>>=
library(prob)
L <- cards()
M <- urnsamples(L, size = 2)
N <- probspace(M)
@

Now that we have the probability space \inputencoding{latin9}\lstinline[showstringspaces=false]!N!\inputencoding{utf8}
we are ready to do some probability. We use the \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!prob!\inputencoding{utf8}
function, just like before. The only trick is to specify the event
of interest correctly, and recall that we were interested in $\P(\mbox{both Aces})$.
But if the cards are both Aces then the \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!rank!\inputencoding{utf8}
of both cards should be \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!"A"!\inputencoding{utf8},
which sounds like a job for the \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!all!\inputencoding{utf8}
function:

<<>>=
prob(N, all(rank == "A"))
@

Note that this value matches what we found in Example \ref{exa:two-cards-both-aces},
above. We could calculate all sorts of probabilities at this point;
we are limited only by the complexity of the event's computer representation. 


\begin{example}
\label{exa:urn-7-red-3-green}Consider an urn with 10 balls inside,
7 of which are red and 3 of which are green. Select 3 balls successively
from the urn. Let $A=\left\{ 1^{\text{st}}\mbox{ ball is red}\right\} $,
$B=\left\{ 2^{\text{nd}}\mbox{ ball is red}\right\} $, and $C=\left\{ 3^{\text{rd}}\mbox{ ball is red}\right\} $.
Then\[
\P(\mbox{all 3 balls are red})=\P(A\cap B\cap C)=\frac{7}{10}\cdot\frac{6}{9}\cdot\frac{5}{8}\approx0.2917.\]

\end{example}

\subsection{How to do it with \textsf{R}}

Example \ref{exa:urn-7-red-3-green} is similar to Example \ref{exa:two-cards-both-aces},
but it is even easier. We need to set up an urn (vector \inputencoding{latin9}\lstinline[showstringspaces=false]!L!\inputencoding{utf8})
to hold the balls, we sample from \inputencoding{latin9}\lstinline[showstringspaces=false]!L!\inputencoding{utf8}
to get the sample space (data frame \inputencoding{latin9}\lstinline[showstringspaces=false]!M!\inputencoding{utf8}),
and we associate a probability vector (column \inputencoding{latin9}\lstinline[showstringspaces=false]!probs!\inputencoding{utf8})
with the outcomes (rows of \inputencoding{latin9}\lstinline[showstringspaces=false]!M!\inputencoding{utf8})
of the sample space. The final result is a probability space (an ordinary
data frame \inputencoding{latin9}\lstinline[showstringspaces=false]!N!\inputencoding{utf8}).

It is easier for us this time because our urn is a vector instead
of a \inputencoding{latin9}\lstinline[showstringspaces=false]!cards()!\inputencoding{utf8}
data frame. Before there were two dimensions of information associated
with the outcomes (rank and suit) but presently we have only one dimension
(color).

<<>>=
library(prob)
L <- rep(c("red","green"), times = c(7,3))
M <- urnsamples(L, size = 3, replace = FALSE, ordered = TRUE)
N <- probspace(M)
@

Now let us think about how to set up the event $\left\{ \mbox{all 3 balls are red}\right\} $.
Rows of \inputencoding{latin9}\lstinline[showstringspaces=false]!N!\inputencoding{utf8}
that satisfy this condition have \inputencoding{latin9}\lstinline[showstringspaces=false]!X1=="red" & X2=="red" & X3=="red"!\inputencoding{utf8},
but there must be an easier way. Indeed, there is. The \inputencoding{latin9}\lstinline[showstringspaces=false]!isrep!\inputencoding{utf8}
function (short for {}``is repeated'') in the \inputencoding{latin9}\lstinline[showstringspaces=false]!prob!\inputencoding{utf8}
package was written for this purpose. The command \inputencoding{latin9}\lstinline[showstringspaces=false]!isrep(N,"red",3)!\inputencoding{utf8}
will test each row of \inputencoding{latin9}\lstinline[showstringspaces=false]!N!\inputencoding{utf8}
to see whether the value \inputencoding{latin9}\lstinline[showstringspaces=false]!"red"!\inputencoding{utf8}
appears \inputencoding{latin9}\lstinline[showstringspaces=false]!3!\inputencoding{utf8}
times. The result is exactly what we need to define an event with
the \inputencoding{latin9}\lstinline[showstringspaces=false]!prob!\inputencoding{utf8}
function. Observe

<<>>=
prob(N, isrep(N, "red", 3))
@

Note that this answer matches what we found in Example \ref{exa:urn-7-red-3-green}.
Now let us try some other probability questions. What is the probability
of getting two \inputencoding{latin9}\lstinline[showstringspaces=false]!"red"!\inputencoding{utf8}s?

<<>>=
prob(N, isrep(N, "red", 2))
@

Note that the exact value is $21/40$; we will learn a quick way to
compute this in Section \ref{sec:other-discrete-distributions}. What
is the probability of observing \inputencoding{latin9}\lstinline[showstringspaces=false]!"red"!\inputencoding{utf8},
then \inputencoding{latin9}\lstinline[showstringspaces=false]!"green"!\inputencoding{utf8},
then \inputencoding{latin9}\lstinline[showstringspaces=false]!"red"!\inputencoding{utf8}?

<<>>=
prob(N, isin(N, c("red","green","red"), ordered = TRUE))
@

Note that the exact value is $7/20$ (do it with the Multiplication
Rule). What is the probability of observing \inputencoding{latin9}\lstinline[showstringspaces=false]!"red"!\inputencoding{utf8},
\inputencoding{latin9}\lstinline[showstringspaces=false]!"green"!\inputencoding{utf8},
and \inputencoding{latin9}\lstinline[showstringspaces=false]!"red"!\inputencoding{utf8},
in no particular order?

<<>>=
prob(N, isin(N, c("red","green","red")))
@

We already knew this. It is the probability of observing two \inputencoding{latin9}\lstinline[showstringspaces=false]!"red"!\inputencoding{utf8}s,
above.


\begin{example}
Consider two urns, the first with 5 red balls and 3 green balls, and
the second with 2 red balls and 6 green balls. Your friend randomly
selects one ball from the first urn and transfers it to the second
urn, without disclosing the color of the ball. You select one ball
from the second urn. What is the probability that the selected ball
is red? Let $A=\left\{ \mbox{transferred ball is red}\right\} $ and
$B=\left\{ \mbox{selected ball is red}\right\} $. Write\begin{align*}
B & =S\cap B\\
 & =(A\cup A^{c})\cap B\\
 & =(A\cap B)\cup(A^{c}\cap B)\end{align*}
and notice that $A\cap B$ and $A^{c}\cap B$ are disjoint. Therefore\begin{align*}
\P(B) & =\P(A\cap B)+\P(A^{c}\cap B)\\
 & =\P(A)\P(B|A)+\P(A^{c})\P(B|A^{c})\\
 & =\frac{5}{8}\cdot\frac{3}{9}+\frac{3}{8}\cdot\frac{2}{9}\\
 & =\frac{21}{72}\ \end{align*}
(which is 7/24 in lowest terms).
\end{example}

\begin{example}
We saw the \inputencoding{latin9}\lstinline[showstringspaces=false]!RcmdrTestDrive!\inputencoding{utf8}
data set in Chapter \ref{cha:introduction-to-R} in which a two-way
table of the smoking status versus the gender was

<<echo = FALSE>>=
.Table <- xtabs(~smoke+gender, data=RcmdrTestDrive)
addmargins(.Table) # Table with Marginal Distributions
remove(.Table)
@

If one person were selected at random from the data set, then we see
from the two-way table that $\P(\mbox{Female})=70/168$ and $\P(\mbox{Smoker})=32/168$.
Now suppose that one of the subjects quits smoking, but we do not
know the person's gender. If we select one subject at random, what
now is $\P(\mbox{Female})$? Let $A=\left\{ \mbox{the quitter is a female}\right\} $
and $B=\left\{ \mbox{selected person is a female}\right\} $. Write\begin{align*}
B & =S\cap B\\
 & =(A\cup A^{c})\cap B\\
 & =(A\cap B)\cup(A^{c}\cap B)\end{align*}
and notice that $A\cap B$ and $A^{c}\cap B$ are disjoint. Therefore\begin{align*}
\P(B) & =\P(A\cap B)+\P(A^{c}\cap B),\\
 & =\P(A)\P(B|A)+\P(A^{c})\P(B|A^{c}),\\
 & =\frac{5}{8}\cdot\frac{3}{9}+\frac{3}{8}\cdot\frac{2}{9},\\
 & =\frac{21}{72},\end{align*}
(which is 7/24 in lowest terms).

\end{example}
Using the same reasoning, we can return to the example from the beginning
of the section and show that\[
\P(\left\{ \mbox{second card is an Ace}\right\} )=4/52.\]
. 


\section{Independent Events\label{sec:Independent-Events}}

Toss a coin twice. The sample space is $S=\left\{ HH,\ HT,\ TH,\ TT\right\} $.
We know that $\P(1^{\text{st}}\mbox{ toss is }H)=2/4$, $\P(2^{\text{nd}}\mbox{ toss is }H)=2/4$,
and $\P(\mbox{both }H)=1/4$. Then\begin{align*}
\P(2^{\text{nd}}\mbox{ toss is }H\ |\ 1^{\text{st}}\mbox{ toss is }H) & =\frac{\P(\mbox{both }H)}{\P(1^{\text{st}}\mbox{ toss is }H)},\\
 & =\frac{1/4}{2/4},\\
 & =\P(2^{\text{nd}}\mbox{ toss is }H).\end{align*}
Intuitively, this means that the information that the first toss is
$H$ has no bearing on the probability that the second toss is $H$.
The coin does not remember the result of the first toss. 
\begin{defn}
Events $A$ and $B$ are said to be \emph{independent} if \begin{equation}
\P(A\cap B)=\P(A)\P(B).\end{equation}
Otherwise, the events are said to be \emph{dependent}. 
\end{defn}
The connection with the above example stems from the following. We
know from Section \ref{sec:Conditional-Probability} that when $\P(B)>0$
we may write \begin{equation}
\P(A|B)=\frac{\P(A\cap B)}{\P(B)}.\end{equation}
In the case that $A$ and $B$ are independent, the numerator of the
fraction factors so that $\P(B)$ cancels with the result:\begin{equation}
\P(A|B)=\P(A)\quad\mbox{when \mbox{\emph{A},\emph{ B}} are independent.}\end{equation}
The interpretation in the case of independence is that the information
that the event $B$ occurred does not influence the probability of
the event $A$ occurring. Similarly, $\P(B|A)=\P(B)$, and so the
occurrence of the event $A$ likewise does not affect the probability
of event $B$. It may seem more natural to define $A$ and $B$ to
be independent when $\P(A|B)=\P(A)$; however, the conditional probability
$\P(A|B)$ is only defined when $\P(B)>0$. Our definition is not
limited by this restriction. It can be shown that when $\P(A),\ \P(B)>0$
the two notions of independence are equivalent.
\begin{prop}
If the events $A$ and $B$ are independent then
\begin{itemize}
\item $A$ and $B^{c}$ are independent,
\item $A^{c}$ and $B$ are independent,
\item $A^{c}$ and $B^{c}$ are independent.
\end{itemize}
\end{prop}
\begin{proof}
Suppose that $A$ and $B$ are independent. We will show the second
one; the others are similar. We need to show that\[
\P(A^{c}\cap B)=\P(A^{c})\P(B).\]
To this end, note that the Multiplication Rule, Equation \ref{eq:multiplication-rule-short}
implies \begin{eqnarray*}
\P(A^{c}\cap B) & = & \P(B)\P(A^{c}|B),\\
 & = & \P(B)[1-\P(A|B)],\\
 & = & \P(B)\P(A^{c}).\end{eqnarray*}
\end{proof}
\begin{defn}
The events $A$, $B$, and $C$ are \emph{mutually independent} if
the following four conditions are met: \begin{eqnarray*}
\P(A\cap B) & = & \P(A)\P(B),\\
\P(A\cap C) & = & \P(A)\P(C),\\
\P(B\cap C) & = & \P(B)\P(C),\end{eqnarray*}
and\[
\P(A\cap B\cap C)=\P(A)\P(B)\P(C).\]
If only the first three conditions hold then $A$, $B$, and $C$
are said to be independent \emph{pairwise}. Note that pairwise independence
is not the same as mutual independence when the number of events is
larger than two.
\end{defn}
We can now deduce the pattern for $n$ events, $n>3$. The events
will be mutually independent only if they satisfy the product equality
pairwise, then in groups of three, in groups of four, and so forth,
up to all $n$ events at once. For $n$ events, there will be $2^{n}-n-1$
equations that must be satisfied (see Exercise \ref{xca:numb-cond-indep}).
Although these requirements for a set of events to be mutually independent
may seem stringent, the good news is that for most of the situations
considered in this book the conditions will all be met (or at least
we will suppose that they are).
\begin{example}
\label{exa:toss-ten-coins}Toss ten coins. What is the probability
of observing at least one Head? Answer: Let $A_{i}=\left\{ \mbox{the }i^{\text{th}}\mbox{ coin shows }H\right\} ,\ i=1,2,\ldots,10$.
Supposing that we toss the coins in such a way that they do not interfere
with each other, this is one of the situations where all of the $A_{i}$
may be considered mutually independent due to the nature of the tossing.
Of course, the only way that there will not be at least one Head showing
is if all tosses are Tails. Therefore,\begin{align*}
\P(\mbox{at least one }H) & =1-\P(\mbox{all }T),\\
 & =1-\P(A_{1}^{c}\cap A_{2}^{c}\cap\cdots\cap A_{10}^{c}),\\
 & =1-\P(A_{1}^{c})\P(A_{2}^{c})\cdots\P(A_{10}^{c}),\\
 & =1-\left(\frac{1}{2}\right)^{10},\end{align*}
which is approximately $0.9990234$.
\end{example}

\subsection{How to do it with \textsf{R}}
\begin{example}
Toss ten coins. What is the probability of observing at least one
Head?

<<>>=
S <- tosscoin(10, makespace = TRUE)
A <- subset(S, isrep(S, vals = "T", nrep = 10))
1 - prob(A)
@

Compare this answer to what we got in Example \ref{exa:toss-ten-coins}.

\end{example}

\subsection*{Independent, Repeated Experiments}

Generalizing from above it is common to repeat a certain experiment
multiple times under identical conditions and in an independent manner.
We have seen many examples of this already: tossing a coin repeatedly,
rolling a die or dice, \textit{etc}.

The \inputencoding{latin9}\lstinline[showstringspaces=false]!iidspace!\inputencoding{utf8}
function was designed specifically for this situation. It has three
arguments: \inputencoding{latin9}\lstinline[showstringspaces=false]!x!\inputencoding{utf8},
which is a vector of outcomes, \inputencoding{latin9}\lstinline[showstringspaces=false]!ntrials!\inputencoding{utf8},
which is an integer telling how many times to repeat the experiment,
and \inputencoding{latin9}\lstinline[showstringspaces=false]!probs!\inputencoding{utf8}
to specify the probabilities of the outcomes of \inputencoding{latin9}\lstinline[showstringspaces=false]!x!\inputencoding{utf8}
in a single trial. 


\begin{example}
\textbf{An unbalanced coin} (continued, see Example \ref{exa:unbalanced-coin}).
It was easy enough to set up the probability space for one unbalanced
toss, however, the situation becomes more complicated when there are
many tosses involved. Clearly, the outcome $HHH$ should not have
the same probability as $TTT$, which should again not have the same
probability as $HTH$. At the same time, there is symmetry in the
experiment in that the coin does not remember the face it shows from
toss to toss, and it is easy enough to toss the coin in a similar
way repeatedly.

We may represent tossing our unbalanced coin three times with the
following: 

<<keep.source = TRUE>>= 
iidspace(c("H","T"), ntrials = 3, probs = c(0.7, 0.3)) 
@ 

As expected, the outcome $HHH$ has the largest probability, while
$TTT$ has the smallest. (Since the trials are independent, $\P(HHH)=0.7^{3}$
and $\P(TTT)=0.3^{3}$, \textit{etc}.) Note that the result of the
function call is a probability space, not a sample space (which we
could construct already with the \inputencoding{latin9}\lstinline[showstringspaces=false]!tosscoin!\inputencoding{utf8}
or \inputencoding{latin9}\lstinline[showstringspaces=false]!urnsamples!\inputencoding{utf8}
functions). The same procedure could be used to model an unbalanced
die or any other experiment that may be represented with a vector
of possible outcomes.

\end{example}
Note that \inputencoding{latin9}\lstinline[showstringspaces=false]!iidspace!\inputencoding{utf8}
will assume \inputencoding{latin9}\lstinline[showstringspaces=false]!x!\inputencoding{utf8}
has equally likely outcomes if no \inputencoding{latin9}\lstinline[showstringspaces=false]!probs!\inputencoding{utf8}
argument is specified. Also note that the argument \inputencoding{latin9}\lstinline[showstringspaces=false]!x!\inputencoding{utf8}
is a \emph{vector}, not a data frame. Something like \inputencoding{latin9}\lstinline[showstringspaces=false]!iidspace(tosscoin(1),...)!\inputencoding{utf8}
would give an error.


\section{Bayes' Rule\label{sec:Bayes'-Rule}}

We mentioned the subjective view of probability in Section \ref{sec:Interpreting-Probabilities}.
In this section we introduce a rule that allows us to update our probabilities
when new information becomes available. 
\begin{thm}
\textbf{\emph{(Bayes' Rule).}} Let $B_{1}$, $B_{2}$, \ldots{},
$B_{n}$ be mutually exclusive and exhaustive and let $A$ be an event
with $\P(A)>0$. Then \begin{equation}
\P(B_{k}|A)=\frac{\P(B_{k})\P(A|B_{k})}{\sum_{i=1}^{n}\P(B_{i})\P(A|B_{i})},\quad k=1,2,\ldots,n.\label{eq:bayes-rule}\end{equation}
\end{thm}
\begin{proof}
The proof follows from looking at $\P(B_{k}\cap A)$ in two different
ways. For simplicity, suppose that $P(B_{k})>0$ for all $k$. Then\[
\P(A)\P(B_{k}|A)=\P(B_{k}\cap A)=\P(B_{k})\P(A|B_{k}).\]
Since $\P(A)>0$ we may divide through to obtain \[
\P(B_{k}|A)=\frac{\P(B_{k})\P(A|B_{k})}{\P(A)}.\]
Now remembering that $\left\{ B_{k}\right\} $ is a partition, the
Theorem of Total Probability (Equation \ref{eq:theorem-total-probability})
gives the denominator of the last expression to be\[
\P(A)=\sum_{k=1}^{n}\P(B_{k}\cap A)=\sum_{k=1}^{n}\P(B_{k})\P(A|B_{k}).\]

\end{proof}
What does it mean? Usually in applications we are given (or know)
\emph{a priori} probabilities $\P(B_{k})$. We go out and collect
some data, which we represent by the event $A$. We want to know:
how do we \textbf{update} $\P(B_{k})$ to $\P(B_{k}|A)$? The answer:
Bayes' Rule.
\begin{example}
\textbf{\label{exa:misfiling-assistants}Misfiling Assistants}\textbf{\emph{.}}
In this problem, there are three assistants working at a company:
Moe, Larry, and Curly. Their primary job duty is to file paperwork
in the filing cabinet when papers become available. The three assistants
have different work schedules:

%
\begin{table}[H]
\begin{tabular}{c|ccc}
 & Moe & Larry & Curly\tabularnewline
\hline
Workload & 60\% & 30\% & 10\%\tabularnewline
\end{tabular}
\end{table}


That is, Moe works 60\% of the time, Larry works 30\% of the time,
and Curly does the remaining 10\%, and they file documents at approximately
the same speed. Suppose a person were to select one of the documents
from the cabinet at random. Let $M$ be the event\[
M=\left\{ \mbox{Moe filed the document}\right\} \]
and let $L$ and $C$ be the events that Larry and Curly, respectively,
filed the document. What are these events' respective probabilities?
In the absence of additional information, reasonable prior probabilities
would just be

%
\begin{table}[H]
\begin{tabular}{c|ccc}
 & Moe & Larry & Curly\tabularnewline
\hline
Prior Probability & $\P(M)=0.60$ & $\P(L)=0.30$ & $\P(C)=0.10$\tabularnewline
\end{tabular}
\end{table}


Now, the boss comes in one day, opens up the file cabinet, and selects
a file at random. The boss discovers that the file has been misplaced.
The boss is so angry at the mistake that (s)he threatens to fire the
one who erred. The question is: who misplaced the file?

The boss decides to use probability to decide, and walks straight
to the workload schedule. (S)he reasons that, since the three employees
work at the same speed, the probability that a randomly selected file
would have been filed by each one would be proportional to his workload.
The boss notifies \textbf{Moe} that he has until the end of the day
to empty his desk.

But Moe argues in his defense that the boss has ignored additional
information. Moe's likelihood of having misfiled a document is smaller
than Larry's and Curly's, since he is a diligent worker who pays close
attention to his work. Moe admits that he works longer than the others,
but he doesn't make as many mistakes as they do. Thus, Moe recommends
that -- before making a decision -- the boss should update the probability
(initially based on workload alone) to incorporate the likelihood
of having observed a misfiled document.

And, as it turns out, the boss has information about Moe, Larry, and
Curly's filing accuracy in the past (due to historical performance
evaluations). The performance information may be represented by the
following table:

%
\begin{table}[H]
\begin{tabular}{c|ccc}
 & Moe & Larry & Curly\tabularnewline
\hline
Misfile Rate & 0.003 & 0.007 & 0.010\tabularnewline
\end{tabular}
\end{table}


In other words, on the average, Moe misfiles 0.3\% of the documents
he is supposed to file. Notice that Moe was correct: he is the most
accurate filer, followed by Larry, and lastly Curly. If the boss were
to make a decision based only on the worker's overall accuracy, then
\textbf{Curly} should get the axe. But Curly hears this and interjects
that he only works a short period during the day, and consequently
makes mistakes only very rarely; there is only the tiniest chance
that he misfiled this particular document.

The boss would like to use this updated information to update the
probabilities for the three assistants, that is, (s)he wants to use
the additional likelihood that the document was misfiled to update
his/her beliefs about the likely culprit. Let $A$ be the event that
a document is misfiled. What the boss would like to know are the three
probabilities \[
\P(M|A),\mbox{ }\P(L|A),\mbox{ and }\P(C|A).\]
We will show the calculation for $\P(M|A)$, the other two cases being
similar. We use Bayes' Rule in the form\[
\P(M|A)=\frac{\P(M\cap A)}{\P(A)}.\]
Let's try to find $\P(M\cap A)$, which is just $\P(M)\cdot\P(A|M)$
by the Multiplication Rule. We already know $\P(M)=0.6$ and $\P(A|M)$
is nothing more than Moe's misfile rate, given above to be $\P(A|M)=0.003$.
Thus, we compute\[
\P(M\cap A)=(0.6)(0.003)=0.0018.\]
Using the same procedure we may calculate\[
\P(L|A)=0.0021\mbox{ and }\P(C|A)=0.0010.\]
Now let's find the denominator, $\P(A)$. The key here is the notion
that if a file is misplaced, then either Moe or Larry or Curly must
have filed it; there is no one else around to do the misfiling. Further,
these possibilities are mutually exclusive. We may use the Theorem
of Total Probability \ref{eq:theorem-total-probability} to write\[
\P(A)=\P(A\cap M)+\P(A\cap L)+\P(A\cap C).\]
Luckily, we have computed these above. Thus\[
\P(A)=0.0018+0.0021+0.0010=0.0049.\]
Therefore, Bayes' Rule yields\[
\P(M|A)=\frac{0.0018}{0.0049}\approx0.37.\]
This last quantity is called the posterior probability that Moe misfiled
the document, since it incorporates the observed data that a randomly
selected file was misplaced (which is governed by the misfile rate).
We can use the same argument to calculate

%
\begin{table}[H]
\begin{tabular}{c|ccc}
 & Moe & Larry & Curly\tabularnewline
\hline
Posterior Probability & $\P(M|A)\approx0.37$ & $\P(L|A)\approx0.43$ & $\P(C|A)\approx0.20$\tabularnewline
\end{tabular}
\end{table}


The conclusion: \textbf{Larry} gets the axe. What is happening is
an intricate interplay between the time on the job and the misfile
rate. It is not obvious who the winner (or in this case, loser) will
be, and the statistician needs to consult Bayes' Rule to determine
the best course of action.
\end{example}

\begin{example}
\label{exa:misfiling-assistants-multiple}Suppose the boss gets a
change of heart and does not fire anybody. But the next day (s)he
randomly selects another file and again finds it to be misplaced.
To decide whom to fire now, the boss would use the same procedure,
with one small change. (S)he would not use the prior probabilities
$ $60\%, 30\%, and 10\%; those are old news. Instead, she would replace
the prior probabilities with the posterior probabilities just calculated.
After the math she will have new posterior probabilities, updated
even more from the day before.

In this way, probabilities found by Bayes' rule are always on the
cutting edge, always updated with respect to the best information
available at the time.
\end{example}

\subsection{How to do it with \textsf{R}}

There are not any special functions for Bayes' Rule in the \inputencoding{latin9}\lstinline[showstringspaces=false]!prob!\inputencoding{utf8}
package, but problems like the ones above are easy enough to do by
hand.
\begin{example}
\textbf{Misfiling assistants} (continued from Example \ref{exa:misfiling-assistants}).
We store the prior probabilities and the likelihoods in vectors and
go to town.

<<>>=
prior <- c(0.6, 0.3, 0.1)
like <- c(0.003, 0.007, 0.010)
post <- prior * like
post / sum(post)
@

\end{example}


Compare these answers with what we got in Example \ref{exa:misfiling-assistants}.
We would replace \inputencoding{latin9}\lstinline[showstringspaces=false]!prior!\inputencoding{utf8}
with \inputencoding{latin9}\lstinline[showstringspaces=false]!post!\inputencoding{utf8}
in a future calculation. We could raise \inputencoding{latin9}\lstinline[showstringspaces=false]!like!\inputencoding{utf8}
to a power to see how the posterior is affected by future document
mistakes. (Do you see why? Think back to Section \ref{sec:Independent-Events}.)


\begin{example}
Let us incorporate the posterior probability (\inputencoding{latin9}\lstinline[showstringspaces=false]!post!\inputencoding{utf8})
information from the last example and suppose that the assistants
misfile seven more documents. Using Bayes' Rule, what would the new
posterior probabilities be?

<<>>=
newprior <- post
post <- newprior * like^7
post / sum(post)
@

We see that the individual with the highest probability of having
misfiled all eight documents given the observed data is no longer
Larry, but Curly. 
\end{example}


There are two important points. First, we did not divide \inputencoding{latin9}\lstinline[showstringspaces=false]!post!\inputencoding{utf8}
by the sum of its entries until the very last step; we do not need
to calculate it, and it will save us computing time to postpone normalization
until absolutely necessary, namely, until we finally want to interpret
them as probabilities.

Second, the reader might be wondering what the boss would get if (s)he
skipped the intermediate step of calculating the posterior after only
one misfiled document. What if she started from the \emph{original}
prior, then observed eight misfiled documents, and calculated the
posterior? What would she get? It must be the same answer, of course.

<<>>=
fastpost <- prior * like^8
fastpost / sum(fastpost)
@

Compare this to what we got in Example \ref{exa:misfiling-assistants-multiple}.




\section{Random Variables\label{sec:Random-Variables}}

We already know about experiments, sample spaces, and events. In this
section, we are interested in a \emph{number} that is associated with
the experiment. We conduct a random experiment $E$ and after learning
the outcome $\omega$ in $S$ we calculate a number $X$. That is,
to each outcome $\omega$ in the sample space we associate a number
$X(\omega)=x$. 
\begin{defn}
A \emph{random variable} $X$ is a function $X:S\to\R$ that associates
to each outcome $\omega\in S$ exactly one number $X(\omega)=x$. 
\end{defn}
We usually denote random variables by uppercase letters such as $X$,
$Y$, and $Z$, and we denote their observed values by lowercase letters
$x$, $y$, and $z$. Just as $S$ is the set of all possible outcomes
of $E$, we call the set of all possible values of $X$ the \emph{support}
of $X$ and denote it by $S_{X}$.
\begin{example}
Let $E$ be the experiment of flipping a coin twice. We have seen
that the sample space is $S=\left\{ HH,\ HT,\ TH,\ TT\right\} $.
Now define the random variable $X=\mbox{the number of heads}$. That
is, for example, $X(HH)=2$, while $X(HT)=1$. We may make a table
of the possibilities:

%
\begin{table}[H]
\begin{tabular}{c|cccc}
$\omega\in S$ & $HH$ & $HT$ & $TH$ & $TT$\tabularnewline
\hline
$X(\omega)=x$ & 2 & 1 & 1 & 0\tabularnewline
\end{tabular}
\end{table}


Taking a look at the second row of the table, we see that the support
of $X$ -- the set of all numbers that $X$ assumes -- would be $S_{X}=\left\{ 0,1,2\right\} $.
\end{example}

\begin{example}
Let $E$ be the experiment of flipping a coin repeatedly until observing
a Head. The sample space would be $S=\left\{ H,\ TH,\ TTH,\ TTTH,\ \ldots\right\} $.
Now define the random variable $Y=\mbox{the number of Tails before the first head}$.
Then the support of $Y$ would be $S_{Y}=\left\{ 0,1,2,\ldots\right\} $.
\end{example}

\begin{example}
Let $E$ be the experiment of tossing a coin in the air, and define
the random variable $Z=\mbox{the time (in seconds) until the coin hits the ground}$.
In this case, the sample space is inconvenient to describe. Yet the
support of $Z$ would be $(0,\infty)$. Of course, it is reasonable
to suppose that the coin will return to Earth in a short amount of
time; in practice, the set $(0,\infty)$ is admittedly too large.
However, we will find that in many circumstances it is mathematically
convenient to study the extended set rather than a restricted one.
\end{example}
There are important differences between the supports of $X$, $Y$,
and $Z$. The support of $X$ is a finite collection of elements that
can be inspected all at once. And while the support of $Y$ cannot
be exhaustively written down, its elements can nevertheless be listed
in a naturally ordered sequence. Random variables with supports similar
to those of $X$ and $Y$ are called \emph{discrete random variables}.
We study these in Chapter \ref{cha:Discrete-Distributions}.

In contrast, the support of $Z$ is a continuous interval, containing
all rational and irrational positive real numbers. For this reason%
\footnote{This isn't really the reason, but it serves as an effective litmus
test at the introductory level. See Billingsley or Resnick.%
}, random variables with supports like $Z$ are called \emph{continuous
random variables}, to be studied in Chapter \ref{cha:Continuous-Distributions}.


\subsection{How to do it with \textsf{R}}

The primary vessel for this task is the \inputencoding{latin9}\lstinline[showstringspaces=false]!addrv!\inputencoding{utf8}
function. There are two ways to use it, and we will describe both.


\subsubsection*{Supply a Defining Formula}

The first method is based on the \inputencoding{latin9}\lstinline[showstringspaces=false]!transform!\inputencoding{utf8}
function. See \inputencoding{latin9}\lstinline[showstringspaces=false]!?transform!\inputencoding{utf8}.
The idea is to write a formula defining the random variable inside
the function, and it will be added as a column to the data frame.
As an example, let us roll a 4-sided die three times, and let us define
the random variable $U=X1-X2+X3$. 

<<echo=TRUE,print=FALSE>>= 
S <- rolldie(3, nsides = 4, makespace = TRUE) 
S <- addrv(S, U = X1-X2+X3) 
@ 

Now let's take a look at the values of $U$. In the interest of space,
we will only reproduce the first few rows of $S$ (there are $4^{3}=64$
rows in total). 

<<echo=TRUE,print=TRUE>>= 
head(S)
@ 

We see from the $U$ column it is operating just like it should. We
can now answer questions like

<<echo=TRUE,print=TRUE>>= 
prob(S, U > 6) 
@ 


\subsubsection*{Supply a Function}

Sometimes we have a function laying around that we would like to apply
to some of the outcome variables, but it is unfortunately tedious
to write out the formula defining what the new variable would be.
The \inputencoding{latin9}\lstinline[showstringspaces=false]!addrv!\inputencoding{utf8}
function has an argument \inputencoding{latin9}\lstinline[showstringspaces=false]!FUN!\inputencoding{utf8}
specifically for this case. Its value should be a legitimate function
from \textsf{R}, such as \inputencoding{latin9}\lstinline[showstringspaces=false]!sum!\inputencoding{utf8},
\inputencoding{latin9}\lstinline[showstringspaces=false]!mean!\inputencoding{utf8},
\inputencoding{latin9}\lstinline[showstringspaces=false]!median!\inputencoding{utf8},
\emph{etc}. Or, you can define your own function. Continuing the previous
example, let's define $V=\max(X1,X2,X3)$ and $W=X1+X2+X3$. 

<<echo=TRUE,print=FALSE>>= 
S <- addrv(S, FUN = max, invars = c("X1","X2","X3"), name = "V") 
S <- addrv(S, FUN = sum, invars = c("X1","X2","X3"), name = "W") 
head(S) 
@ 

Notice that \inputencoding{latin9}\lstinline[showstringspaces=false]!addrv!\inputencoding{utf8}
has an \inputencoding{latin9}\lstinline[showstringspaces=false]!invars!\inputencoding{utf8}
argument to specify exactly to which columns one would like to apply
the function \inputencoding{latin9}\lstinline[showstringspaces=false]!FUN!\inputencoding{utf8}.
If no input variables are specified, then \inputencoding{latin9}\lstinline[showstringspaces=false]!addrv!\inputencoding{utf8}
will apply \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!FUN!\inputencoding{utf8}
to all non-\inputencoding{latin9}\lstinline[basicstyle={\ttfamily},showstringspaces=false]!probs!\inputencoding{utf8}
columns. Further, \inputencoding{latin9}\lstinline[showstringspaces=false]!addrv!\inputencoding{utf8}
has an optional argument \inputencoding{latin9}\lstinline[showstringspaces=false]!name!\inputencoding{utf8}
to give the new variable; this can be useful when adding several random
variables to a probability space (as above). If not specified, the
default name is {}``\inputencoding{latin9}\lstinline[showstringspaces=false]!X!\inputencoding{utf8}''.


\subsubsection*{Marginal Distributions}

As we can see above, often after adding a random variable $V$ to
a probability space one will find that $V$ has values that are repeated,
so that it becomes difficult to understand what the ultimate behavior
of $V$ actually is. We can use the \inputencoding{latin9}\lstinline[showstringspaces=false]!marginal!\inputencoding{utf8}
function to aggregate the rows of the sample space by values of $V$,
all the while accumulating the probability associated with $V$'s
distinct values. Continuing our example from above, suppose we would
like to focus entirely on the values and probabilities of $V=\max(X1,X2,X3)$. 

<<echo=TRUE,print=TRUE>>= 
marginal(S, vars = "V") 
@ 

We could save the probability space of $V$ in a data frame and study
it further, if we wish. As a final remark, we can calculate the marginal
distributions of multiple variables desired using the \inputencoding{latin9}\lstinline[showstringspaces=false]!vars!\inputencoding{utf8}
argument. For example, suppose we would like to examine the joint
distribution of $V$ and $W$. 

<<echo=TRUE,print=TRUE>>= 
marginal(S, vars = c("V", "W")) 
@ 

Note that the default value of \inputencoding{latin9}\lstinline[showstringspaces=false]!vars!\inputencoding{utf8}
is the names of all columns except \inputencoding{latin9}\lstinline[showstringspaces=false]!probs!\inputencoding{utf8}.
This can be useful if there are duplicated rows in the probability
space.

\newpage{}


\section*{Chapter Exercises}

\addcontentsline{toc}{section}{Chapter Exercises}
\setcounter{thm}{0}

<<echo = FALSE, results = hide>>=
rnorm(1)
@
\begin{xca}
\label{xca:numb-cond-indep}Prove the assertion given in the text:
the number of conditions that the events $A_{1}$, $A_{2}$, \ldots{},
$A_{n}$ must satisfy in order to be mutually independent is $2^{n}-n-1$.
(\emph{Hint}: think about Pascal's triangle.)
\end{xca}

\paragraph*{Answer:}

The events must satisfy the product equalities two at a time, of which
there are ${n \choose 2}$, then they must satisfy an additional ${n \choose 3}$
conditions three at a time, and so on, until they satisfy the ${n \choose n}=1$
condition including all $n$ events. In total, there are \[
{n \choose 2}+{n \choose 3}+\cdots+{n \choose n}=\sum_{k=0}^{n}{n \choose k}-\left[{n \choose 0}+{n \choose 1}\right]\]
conditions to be satisfied, but the binomial series in the expression
on the right is the sum of the entries of the $n$$^{\text{th}}$
row of Pascal's triangle, which is $2^{n}$.






\chapter{Discrete Distributions\label{cha:Discrete-Distributions}}

\noindent In this chapter we introduce discrete random variables,
those who take values in a finite or countably infinite support set.
We discuss probability mass functions and some special expectations,
namely, the mean, variance and standard deviation. Some of the more
important discrete distributions are explored in detail, and the more
general concept of expectation is defined, which paves the way for
moment generating functions.

We give special attention to the empirical distribution since it plays
such a fundamental role with respect to re sampling and Chapter \ref{cha:resampling-methods};
it will also be needed in Section \ref{sub:Kolmogorov-Smirnov-Goodness-of-Fit-Test}
where we discuss the Kolmogorov-Smirnov test. Following this is a
section in which we introduce a catalogue of discrete random variables
that can be used to model experiments.

There are some comments on simulation, and we mention transformations
of random variables in the discrete case. The interested reader who
would like to learn more about any of the assorted discrete distributions
mentioned here should take a look at \emph{Univariate Discrete Distributions}
by Johnson \emph{et al }\cite{Johnson1993}.


\paragraph*{What do I want them to know?}
\begin{itemize}
\item how to choose a reasonable discrete model under a variety of physical
circumstances
\item the notion of mathematical expectation, how to calculate it, and basic
properties
\item moment generating functions (yes, I want them to hear about those)
\item the general tools of the trade for manipulation of continuous random
variables, integration, \emph{etc}.
\item some details on a couple of discrete models, and exposure to a bunch
of other ones
\item how to make new discrete random variables from old ones
\end{itemize}

\section{Discrete Random Variables\label{sec:discrete-random-variables}}


\subsection{Probability Mass Functions\label{sub:probability-mass-functions}}

Discrete random variables are characterized by their supports which
take the form \begin{equation}
S_{X}=\{u_{1},u_{2},\ldots,u_{k}\}\mbox{ or }S_{X}=\{u_{1},u_{2},u_{3}\ldots\}.\end{equation}
Every discrete random variable $X$ has associated with it a probability
mass function (PMF) $f_{X}:S_{X}\to[0,1]$ defined by \begin{equation}
f_{X}(x)=\P(X=x),\quad x\in S_{X}.\end{equation}
Since values of the PMF represent probabilities, we know from Chapter
\ref{cha:Probability} that PMFs enjoy certain properties. In particular,
all PMFs satisfy 
\begin{enumerate}
\item $f_{X}(x)>0$ for $x\in S$,
\item $\sum_{x\in S}f_{X}(x)=1$, and
\item $\P(X\in A)=\sum_{x\in A}f_{X}(x)$, for any event $A\subset S$.\end{enumerate}
\begin{example}
\label{exa:Toss-a-coin}Toss a coin 3 times. The sample space would
be\[
S=\left\{ HHH,\ HTH,\ THH,\ TTH,\ HHT,\ HTT,\ THT,\ TTT\right\} .\]
Now let $X$ be the number of Heads observed. Then $X$ has support
$S_{X}=\left\{ 0,1,2,3\right\} $. Assuming that the coin is fair
and was tossed in exactly the same way each time, it is not unreasonable
to suppose that the outcomes in the sample space are all equally likely.
What is the PMF of $X$? Notice that $X$ is zero exactly when the
outcome $TTT$ occurs, and this event has probability $1/8$. Therefore,
$f_{X}(0)=1/8$, and the same reasoning shows that $f_{X}(3)=1/8$.
Exactly three outcomes result in $X=1$, thus, $f_{X}(1)=3/8$ and
$f_{X}(3)$ holds the remaining $3/8$ probability (the total is 1).
We can represent the PMF with a table:

%
\begin{table}[H]
\begin{tabular}{c|cccc|c}
$x\in S_{X}$ & 0 & 1 & 2 & 3 & Total\tabularnewline
\hline
$f_{X}(x)=\P(X=x)$ & 1/8 & 3/8 & 3/8 & 1/8 & 1\tabularnewline
\end{tabular}
\end{table}

\end{example}

\subsection{Mean, Variance, and Standard Deviation\label{sub:mean-variance-sd}}

There are numbers associated with PMFs. One important example is the
mean $\mu$, also known as $\E X$ (which we will discuss later):\begin{equation}
\mu=\E X=\sum_{x\in S}xf_{X}(x),\end{equation}
provided the (potentially infinite) series $\sum|x|f_{X}(x)$ is convergent.
Another important number is the variance:\begin{equation}
\sigma^{2}=\sum_{x\in S}(x-\mu)^{2}f_{X}(x),\end{equation}
which can be computed (see Exercise \ref{xca:variance-shortcut})
with the alternate formula $\sigma^{2}=\sum x{}^{2}f_{X}(x)-\mu^{2}$.
Directly defined from the variance is the standard deviation $\sigma=\sqrt{\sigma^{2}}$. 
\begin{example}
\label{exa:disc-pmf-mean}We will calculate the mean of $X$ in Example
\ref{exa:Toss-a-coin}.\[
\mu=\sum_{x=0}^{3}xf_{X}(x)=0\cdot\frac{1}{8}+1\cdot\frac{3}{8}+2\cdot\frac{3}{8}+3\cdot\frac{1}{8}=3.5.\]
We interpret $\mu=3.5$ by reasoning that if we were to repeat the
random experiment many times, independently each time, observe many
corresponding outcomes of the random variable $X$, and take the sample
mean of the observations, then the calculated value would fall close
to 3.5. The approximation would get better as we observe more and
more values of $X$ (another form of the Law of Large Numbers; see
Section \ref{sec:Interpreting-Probabilities}). Another way it is
commonly stated is that $X$ is 3.5 {}``on the average'' or {}``in
the long run''.\end{example}
\begin{rem}
Note that although we say $X$ is 3.5 on the average, we must keep
in mind that our $X$ never actually equals 3.5 (in fact, it is impossible
for $X$ to equal 3.5).
\end{rem}
Related to the probability mass function $f_{X}(x)=\P(X=x)$ is another
important function called the cumulative distribution function (CDF),
$F_{X}$. It is defined by the formula\begin{equation}
F_{X}(t)=\P(X\leq t),\quad-\infty<t<\infty.\end{equation}


We know that all PMFs satisfy certain properties, and a similar statement
may be made for CDFs. In particular, any CDF $F_{X}$ satisfies
\begin{itemize}
\item $F_{X}$ is nondecreasing ($t_{1}\leq t_{2}$ implies $F_{X}(t_{1})\leq F_{X}(t_{2})$).
\item $F_{X}$ is right-continuous ($\lim_{t\to a^{+}}F_{X}(t)=F_{X}(a)$
for all $a\in\R$).
\item $\lim_{t\to-\infty}F_{X}(t)=0$ and $\lim_{t\to\infty}F_{X}(t)=1$.
\end{itemize}
We say that $X$ has the distribution $F_{X}$ and we write $X\sim F_{X}$.
In an abuse of notation we will also write $X\sim f_{X}$ and for
the named distributions the PMF or CDF will be identified by the family
name instead of the defining formula.


\subsection{How to do it with \textsf{R\label{sub:disc-rv-how-r}}}

The mean and variance of a discrete random variable is easy to compute
at the console. Let's return to Example \ref{exa:disc-pmf-mean}.
We will start by defining a vector \inputencoding{latin9}\lstinline[showstringspaces=false]!x!\inputencoding{utf8}
containing the support of $X$, and a vector \inputencoding{latin9}\lstinline[showstringspaces=false]!f!\inputencoding{utf8}
to contain the values of $f_{X}$ at the respective outcomes in \inputencoding{latin9}\lstinline[showstringspaces=false]!x!\inputencoding{utf8}:

<<keep.source = TRUE>>=
x <- c(0,1,2,3)
f <- c(1/8, 3/8, 3/8, 1/8)
@

To calculate the mean $\mu$, we need to multiply the corresponding
values of \inputencoding{latin9}\lstinline[showstringspaces=false]!x!\inputencoding{utf8}
and \inputencoding{latin9}\lstinline[showstringspaces=false]!f!\inputencoding{utf8}
and add them. This is easily accomplished in \textsf{R} since operations
on vectors are performed \emph{element-wise} (see Section \ref{sub:Functions-and-Expressions}): 

<<>>=
mu <- sum(x * f)
mu
@

To compute the variance $\sigma^{2}$, we subtract the value of \inputencoding{latin9}\lstinline[showstringspaces=false]!mu!\inputencoding{utf8}
from each entry in \inputencoding{latin9}\lstinline[showstringspaces=false]!x!\inputencoding{utf8},
square the answers, multiply by \inputencoding{latin9}\lstinline[showstringspaces=false]!f!\inputencoding{utf8},
and \inputencoding{latin9}\lstinline[showstringspaces=false]!sum!\inputencoding{utf8}.
The standard deviation $\sigma$ is simply the square root of $\sigma^{2}$.

<<keep.source = TRUE>>=
sigma2 <- sum((x-mu)^2 * f)
sigma2
sigma <- sqrt(sigma2)
sigma
@

Finally, we may find the values of the CDF $F_{X}$ on the support
by accumulating the probabilities in $f_{X}$ with the \inputencoding{latin9}\lstinline[showstringspaces=false]!cumsum!\inputencoding{utf8}
function. 

<<>>=
F = cumsum(f)
F
@

As easy as this is, it is even easier to do with the \inputencoding{latin9}\lstinline[showstringspaces=false]!distrEx!\inputencoding{utf8}
package \cite{Ruckdescheldistr}. We define a random variable \inputencoding{latin9}\lstinline[showstringspaces=false]!X!\inputencoding{utf8}
as an object, then compute things from the object such as mean, variance,
and standard deviation with the functions \inputencoding{latin9}\lstinline[showstringspaces=false]!E!\inputencoding{utf8},
\inputencoding{latin9}\lstinline[showstringspaces=false]!var!\inputencoding{utf8},
and \inputencoding{latin9}\lstinline[showstringspaces=false]!sd!\inputencoding{utf8}: 

<<keep.source = TRUE>>=
library(distrEx)
X <- DiscreteDistribution(supp = 0:3, prob = c(1,3,3,1)/8)
E(X); var(X); sd(X)
@


\section{The Discrete Uniform Distribution\label{sec:disc-uniform-dist}}

We have seen the basic building blocks of discrete distributions and
we now study particular models that statisticians often encounter
in the field. Perhaps the most fundamental of all is the \emph{discrete
uniform} distribution.

A random variable $X$ with the discrete uniform distribution on the
integers $1,2,\ldots,m$ has PMF\begin{equation}
f_{X}(x)=\frac{1}{m},\quad x=1,2,\ldots,m.\end{equation}
We write $X\sim\mathsf{disunif}(m)$. A random experiment where this
distribution occurs is the choice of an integer at random between
1 and 100, inclusive. Let $X$ be the number chosen. Then $X\sim\mathsf{disunif}(m=100)$
and \[
\P(X=x)=\frac{1}{100},\quad x=1,\ldots,100.\]
We find a direct formula for the mean of $X\sim\mathsf{disunif}(m)$:\begin{equation}
\mu=\sum_{x=1}^{m}xf_{X}(x)=\sum_{x=1}^{m}x\cdot\frac{1}{m}=\frac{1}{m}(1+2+\cdots+m)=\frac{m+1}{2},\end{equation}
where we have used the famous identity $1+2+\cdots+m=m(m+1)/2$. That
is, if we repeatedly choose integers at random from 1 to $m$ then,
on the average, we expect to get $(m+1)/2$. To get the variance we
first calculate\[
\sum_{x=1}^{m}x^{2}f_{X}(x)=\frac{1}{m}\sum_{x=1}^{m}x^{2}=\frac{1}{m}\frac{m(m+1)(2m+1)}{6}=\frac{(m+1)(2m+1)}{6},\]
 and finally,\begin{equation}
\sigma^{2}=\sum_{x=1}^{m}x^{2}f_{X}(x)-\mu^{2}=\frac{(m+1)(2m+1)}{6}-\left(\frac{m+1}{2}\right)^{2}=\cdots=\frac{m^{2}-1}{12}.\end{equation}

\begin{example}
Roll a die and let $X$ be the upward face showing. Then $m=6$, $\mu=7/2=3.5$,
and $\sigma^{2}=(6^{2}-1)/12=35/12$.
\end{example}

\subsection{How to do it with \textsf{R}}


\paragraph*{From the console:}

One can choose an integer at random with the \inputencoding{latin9}\lstinline[showstringspaces=false,tabsize=4]!sample!\inputencoding{utf8}
function. The general syntax to simulate a discrete uniform random
variable is \inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false,tabsize=4]!sample(x, size, replace = TRUE)!\inputencoding{utf8}.

The argument \inputencoding{latin9}\lstinline[showstringspaces=false,tabsize=4]!x!\inputencoding{utf8}
identifies the numbers from which to randomly sample. If \inputencoding{latin9}\lstinline[showstringspaces=false,tabsize=4]!x!\inputencoding{utf8}
is a number, then sampling is done from 1 to \inputencoding{latin9}\lstinline[showstringspaces=false,tabsize=4]!x!\inputencoding{utf8}.
The argument \inputencoding{latin9}\lstinline[showstringspaces=false,tabsize=4]!size!\inputencoding{utf8}
tells how big the sample size should be, and \inputencoding{latin9}\lstinline[showstringspaces=false,tabsize=4]!replace!\inputencoding{utf8}
tells whether or not numbers should be replaced in the urn after having
been sampled. The default option is \inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false,tabsize=2]!replace = FALSE!\inputencoding{utf8}
but for discrete uniforms the sampled values should be replaced. Some
examples follow.


\subsection{Examples}
\begin{itemize}
\item To roll a fair die 3000 times, do \inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false,tabsize=2]!sample(6, size = 3000, replace = TRUE)!\inputencoding{utf8}.
\item To choose 27 random numbers from 30 to 70, do \inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false,tabsize=2]!sample(30:70, size = 27, replace = TRUE)!\inputencoding{utf8}.
\item To flip a fair coin 1000 times, do \inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false,tabsize=2]!sample(c("H","T"), size = 1000, replace = TRUE)!\inputencoding{utf8}.
\end{itemize}

\paragraph*{With the \textsf{R} Commander:}

Follow the sequence \textsf{Probability $\triangleright$ Discrete
Distributions $\triangleright$ Discrete Uniform distribution $\triangleright$
Simulate Discrete uniform variates\ldots{}. }

Suppose we would like to roll a fair die 3000 times. In the \inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false,tabsize=2]!Number of samples!\inputencoding{utf8}
field we enter \inputencoding{latin9}\lstinline[showstringspaces=false,tabsize=2]!1!\inputencoding{utf8}.
Next, we describe what interval of integers to be sampled. Since there
are six faces numbered 1 through 6, we set \inputencoding{latin9}\lstinline[showstringspaces=false,tabsize=2]!from = 1!\inputencoding{utf8},
we set \inputencoding{latin9}\lstinline[showstringspaces=false,tabsize=2]!to = 6!\inputencoding{utf8},
and set \inputencoding{latin9}\lstinline[showstringspaces=false,tabsize=2]!by = 1!\inputencoding{utf8}
(to indicate that we travel from 1 to 6 in increments of 1 unit).
We will generate a list of 3000 numbers selected from among 1, 2,
\ldots{}, 6, and we store the results of the simulation. For the
time being, we select \inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false,tabsize=2]!New Data set!\inputencoding{utf8}.
Click \textsf{OK}.

Since we are defining a new data set, the \textsf{R} Commander requests
a name for the data set. The default name is \inputencoding{latin9}\lstinline[showstringspaces=false,tabsize=2]!Simset1!\inputencoding{utf8},
although in principle you could name it whatever you like (according
to \textsf{R}'s rules for object names). We wish to have a list that
is 3000 long, so we set \inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false,tabsize=2]!Sample Size = 3000!\inputencoding{utf8}
and click \textsf{OK}.

In the \textsf{R} Console window, the \textsf{R} Commander should
tell you that \inputencoding{latin9}\lstinline[showstringspaces=false,tabsize=2]!Simset1!\inputencoding{utf8}
has been initialized, and it should also alert you that \inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false,tabsize=2]!There was 1 discrete uniform variate sample stored in Simset 1.!\inputencoding{utf8}.
To take a look at the rolls of the die, we click \textsf{View data
set} and a window opens. 

The default name for the variable is \inputencoding{latin9}\lstinline[showstringspaces=false,tabsize=2]!disunif.sim1!\inputencoding{utf8}.


\section{The Binomial Distribution\label{sec:binom-dist}}

The binomial distribution is based on a \emph{Bernoulli trial}, which
is a random experiment in which there are only two possible outcomes:
success ($S$) and failure ($F$). We conduct the Bernoulli trial
and let \begin{equation}
X=\begin{cases}
1 & \mbox{if the outcome is \ensuremath{S}},\\
0 & \mbox{if the outcome is \ensuremath{F}}.\end{cases}\end{equation}
If the probability of success is $p$ then the probability of failure
must be $1-p=q$ and the PMF of $X$ is\begin{equation}
f_{X}(x)=p^{x}(1-p)^{1-x},\quad x=0,1.\end{equation}
It is easy to calculate $\mu=\E X=p$ and $\E X^{2}=p$ so that $\sigma^{2}=p-p^{2}=p(1-p)$.


\subsection{The Binomial Model\label{sub:The-Binomial-Model}}

The Binomial model has three defining properties:
\begin{itemize}
\item Bernoulli trials are conducted $n$ times,
\item the trials are independent,
\item the probability of success $p$ does not change between trials.
\end{itemize}
If $X$ counts the number of successes in the $n$ independent trials,
then the PMF of $X$ is 

\begin{equation}
f_{X}(x)={n \choose x}p^{x}(1-p)^{n-x},\quad x=0,1,2,\ldots,n.\end{equation}
We say that $X$ has a \emph{binomial distribution} and we write $X\sim\mathsf{binom}(\mathtt{size}=n,\,\mathtt{prob}=p)$.
It is clear that $f_{X}(x)\geq0$ for all $x$ in the support because
the value is the product of nonnegative numbers. We next check that
$\sum f(x)=1$:\[
\sum_{x=0}^{n}{n \choose x}p^{x}(1-p)^{n-x}=[p+(1-p)]^{n}=1^{n}=1.\]
We next find the mean:

\begin{alignat*}{1}
\mu= & \sum_{x=0}^{n}x\,{n \choose x}p^{x}(1-p)^{n-x},\\
= & \sum_{x=1}^{n}x\,\frac{n!}{x!(n-x)!}p^{x}q^{n-x},\\
= & n\cdot p\sum_{x=1}^{n}\frac{(n-1)!}{(x-1)!(n-x)!}p^{x-1}q^{n-x},\\
= & np\,\sum_{x-1=0}^{n-1}{n-1 \choose x-1}p^{(x-1)}(1-p)^{(n-1)-(x-1)},\\
= & np.\end{alignat*}
 A similar argument shows that $\E X(X-1)=n(n-1)p^{2}$ (see Exercise
\ref{xca:binom-factorial-expectation}). Therefore\begin{alignat*}{1}
\sigma^{2}= & \E X(X-1)+\E X-[\E X]^{2},\\
= & n(n-1)p^{2}+np-(np)^{2},\\
= & n^{2}p^{2}-np^{2}+np-n^{2}p^{2},\\
= & np-np^{2}=np(1-p).\end{alignat*}

\begin{example}
A four-child family. Each child may be either a boy ($B$) or a girl
($G$). For simplicity we suppose that $\P(B)=\P(G)=1/2$ and that
the genders of the children are determined independently. If we let
$X$ count the number of $B$'s, then $X\sim\mathsf{binom}(\mathtt{size}=4,\,\mathtt{prob}=1/2)$.
Further, $\P(X=2)$ is\[
f_{X}(2)={4 \choose 2}(1/2)^{2}(1/2)^{2}=\frac{6}{2^{4}}.\]
The mean number of boys is $4(1/2)=2$ and the variance of $X$ is
$4(1/2)(1/2)=1$.
\end{example}

\subsection{How to do it with \textsf{R}}

The corresponding \textsf{R} function for the PMF and CDF are \inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false]!dbinom!\inputencoding{utf8}
and \inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false]!pbinom!\inputencoding{utf8},
respectively. We demonstrate their use in the following examples. 
\begin{example}
We can calculate it in \textsf{R} Commander under the \textsf{Binomial
Distribution} menu with the \textsf{Binomial probabilities} menu item.

<<echo = FALSE>>=
A <- data.frame(Pr=dbinom(0:4, size = 4, prob = 0.5))
rownames(A) <- 0:4 
A
@

\end{example}
We know that the $\mathsf{binom}(\mathtt{size}=4,\,\mathtt{prob}=1/2)$
distribution is supported on the integers 0, 1, 2, 3, and 4; thus
the table is complete. We can read off the answer to be $\P(X=2)=0.3750$.


\begin{example}
Roll 12 dice simultaneously, and let $X$ denote the number of 6's
that appear. We wish to find the probability of getting seven, eight,
or nine 6's. If we let $S=\left\{ \mbox{get a 6 on one roll}\right\} $,
then $\P(S)=1/6$ and the rolls constitute Bernoulli trials; thus
$X\sim\mathsf{binom}(\mathtt{size=}12,\ \mathtt{prob=}1/6)$ and our
task is to find $\P(7\leq X\leq9)$. This is just\[
\P(7\leq X\leq9)=\sum_{x=7}^{9}{12 \choose x}(1/6)^{x}(5/6)^{12-x}.\]
 Again, one method to solve this problem would be to generate a probability
mass table and add up the relevant rows. However, an alternative method
is to notice that $\P(7\leq X\leq9)=\P(X\leq9)-\P(X\leq6)=F_{X}(9)-F_{X}(6)$,
so we could get the same answer by using the \textsf{Binomial tail
probabilities\ldots{} }menu in the \textsf{R} Commander or the following
from the command line: 

<<keep.source = TRUE>>=
pbinom(9, size=12, prob=1/6) - pbinom(6, size=12, prob=1/6)
diff(pbinom(c(6,9), size = 12, prob = 1/6))  # same thing
@

\end{example}

\begin{example}
\label{exa:toss-coin-3-withR}Toss a coin three times and let $X$
be the number of Heads observed. We know from before that $X\sim\mathsf{binom}(\mathtt{size}=3,\,\mathtt{prob}=1/2)$
which implies the following PMF:

%
\begin{table}[H]
\begin{tabular}{c|cccc}
$x=\mbox{\#of Heads}$ & 0 & 1 & 2 & 3\tabularnewline
\hline
$f(x)=\P(X=x)$ & 1/8 & 3/8 & 3/8 & 1/8\tabularnewline
\end{tabular}
\end{table}


Our next goal is to write down the CDF of $X$ explicitly. The first
case is easy: it is impossible for $X$ to be negative, so if $x<0$
then we should have $\P(X\leq x)=0$. Now choose a value $x$ satisfying
$0\leq x<1$, say, $x=0.3$. The only way that $X\leq x$ could happen
would be if $X=0$, therefore, $\P(X\leq x)$ should equal $\P(X=0)$,
and the same is true for any $0\leq x<1$. Similarly, for any $1\leq x<2$,
say, $x=1.73$, the event $\left\{ X\leq x\right\} $ is exactly the
event $\left\{ X=0\mbox{ or }X=1\right\} $. Consequently, $\P(X\leq x)$
should equal $\P(X=0\mbox{ or }X=1)=\P(X=0)+\P(X=1)$. Continuing
in this fashion, we may figure out the values of $F_{X}(x)$ for all
possible inputs $-\infty<x<\infty$, and we may summarize our observations
with the following piecewise defined function:\[
F_{X}(x)=\P(X\leq x)=\begin{cases}
0, & x<0,\\
\frac{1}{8}, & 0\leq x<1,\\
\frac{1}{8}+\frac{3}{8}=\frac{4}{8}, & 1\leq x<2,\\
\frac{4}{8}+\frac{3}{8}=\frac{7}{8}, & 2\leq x<3,\\
1, & x\geq3.\end{cases}\]
In particular, the CDF of $X$ is defined for the entire real line,
$\R$. The CDF is right continuous and nondecreasing. A graph of the
$\mathsf{binom}(\mathtt{size}=3,\,\mathtt{prob}=1/2)$ CDF is shown
in Figure \ref{fig:binom-cdf-base}.

\end{example}
%
\begin{figure}
\begin{centering}
<<echo = FALSE, fig = true, height = 4.5, width = 6>>=
plot(0, xlim = c(-1.2, 4.2), ylim = c(-0.04, 1.04), type = "n", xlab = "number of successes", ylab = "cumulative probability")
abline(h = c(0,1), lty = 2, col = "grey")
lines(stepfun(0:3, pbinom(-1:3, size = 3, prob = 0.5)), verticals = FALSE, do.p = FALSE)
points(0:3, pbinom(0:3, size = 3, prob = 0.5), pch = 16, cex = 1.2)
points(0:3, pbinom(-1:2, size = 3, prob = 0.5), pch = 1, cex = 1.2)
@
\par\end{centering}

\caption{Graph of the $\mathsf{binom}(\mathtt{size}=3,\,\mathtt{prob}=1/2)$
CDF\label{fig:binom-cdf-base}}

\end{figure}



\begin{example}
Another way to do Example \ref{exa:toss-coin-3-withR} is with the
\inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!distr!\inputencoding{utf8}
family of packages \cite{Ruckdescheldistr}. They use an object oriented
approach to random variables, that is, a random variable is stored
in an object \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!X!\inputencoding{utf8},
and then questions about the random variable translate to functions
on and involving \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!X!\inputencoding{utf8}.
Random variables with distributions from the \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!base!\inputencoding{utf8}
package are specified by capitalizing the name of the distribution.

<<keep.source = TRUE>>=
library(distr)
X <- Binom(size = 3, prob = 1/2)
X
@

The analogue of the \inputencoding{latin9}\lstinline[showstringspaces=false]!dbinom!\inputencoding{utf8}
function for \inputencoding{latin9}\lstinline[showstringspaces=false]!X!\inputencoding{utf8}
is the \inputencoding{latin9}\lstinline[showstringspaces=false]!d(X)!\inputencoding{utf8}
function, and the analogue of the \inputencoding{latin9}\lstinline[showstringspaces=false]!pbinom!\inputencoding{utf8}
function is the \inputencoding{latin9}\lstinline[showstringspaces=false]!p(X)!\inputencoding{utf8}
function. Compare the following:

<<keep.source = TRUE>>=
d(X)(1)   # pmf of X evaluated at x = 1
p(X)(2)   # cdf of X evaluated at x = 2
@

\end{example}
Random variables defined via the \inputencoding{latin9}\lstinline[showstringspaces=false]!distr!\inputencoding{utf8}
package may be \emph{plotted}, which will return graphs of the PMF,
CDF, and quantile function (introduced in Section \ref{sub:Normal-Quantiles-QF}).
See Figure \ref{fig:binom-plot-distr} for an example.

%
\begin{figure}[H]
\begin{centering}
<<echo = FALSE, fig=true, height = 4, width = 6>>=
plot(X, cex = 0.2)
@
\par\end{centering}

\caption{The \textsf{binom}(\texttt{size} = 3, \texttt{prob} = 0.5) distribution
from the \texttt{distr} package\label{fig:binom-plot-distr}}

\end{figure}


%
\begin{table}
\begin{centering}
\begin{tabular}{lllll}
\multicolumn{5}{l}{Given $X\sim\mathsf{dbinom}(\mathtt{size}=n,\,\mathtt{prob}=p)$.}\tabularnewline
 &  &  &  & \tabularnewline
How to do: &  & with $\mathtt{stats}$ (default)  &  & with $\mathtt{distr}$\tabularnewline
\hline
PMF: $\P(X=x)$ &  & $\mathtt{dbinom(x,size=n,prob=p)}$ &  & $\mathtt{d(X)(x)}$\tabularnewline
CDF: $\P(X\leq x)$ &  & $\mathtt{pbinom(x,size=n,prob=p)}$ &  & $\mathtt{p(X)(x)}$\tabularnewline
Simulate $k$ variates &  & $\mathtt{rbinom(k,size=n,prob=p)}$ &  & $\mathtt{r(X)(k)}$\tabularnewline
\hline
 &  &  &  & \tabularnewline
\multicolumn{5}{r}{For $\mathtt{distr}$ need \texttt{X <-} $\mathtt{Binom(size=}n\mathtt{,\ prob=}p\mathtt{)}$}\tabularnewline
\end{tabular}
\par\end{centering}

\caption{Correspondence between \texttt{stats} and \texttt{distr}}

\end{table}



\section{Expectation and Moment Generating Functions\label{sec:expectation-and-mgfs}}


\subsection{The Expectation Operator\label{sub:expectation-operator}}

We next generalize some of the concepts from Section \ref{sub:mean-variance-sd}.
There we saw that every%
\footnote{Not every, only those PMFs for which the (potentially infinite) series
converges.%
} PMF has two important numbers associated with it:\begin{equation}
\mu=\sum_{x\in S}xf_{X}(x),\quad\sigma^{2}=\sum_{x\in S}(x-\mu)^{2}f_{X}(x).\end{equation}
Intuitively, for repeated observations of $X$ we would expect the
sample mean to closely approximate $\mu$ as the sample size increases
without bound. For this reason we call $\mu$ the \emph{expected value}
of $X$ and we write $\mu=\E X$, where $\E$ is an \emph{expectation
operator}.
\begin{defn}
More generally, given a function $g$ we define the \emph{expected
value of} $g(X)$ by\begin{equation}
\E\, g(X)=\sum_{x\in S}g(x)f_{X}(x),\end{equation}
provided the (potentially infinite) series $\sum_{x}|g(x)|f(x)$ is
convergent. We say that $\E g(X)$ \emph{exists}.
\end{defn}
In this notation the variance is $\sigma^{2}=\E(X-\mu)^{2}$ and we
prove the identity \begin{equation}
\E(X-\mu)^{2}=\E X^{2}-(\E X)^{2}\end{equation}
in Exercise \ref{xca:variance-shortcut}. Intuitively, for repeated
observations of $X$ we would expect the sample mean of the $g(X)$
values to closely approximate $\E\, g(X)$ as the sample size increases
without bound.

Let us take the analogy further. If we expect $g(X)$ to be close
to $\E g(X)$ on the average, where would we expect $3g(X)$ to be
on the average? It could only be $3\E g(X)$. The following theorem
makes this idea precise.
\begin{prop}
\label{pro:expectation-properties}For any functions $g$ and $h$,
any random variable $X$, and any constant $c$: 
\begin{enumerate}
\item $\E\: c=c$,
\item $\E[c\cdot g(X)]=c\E g(X)$
\item $\E[g(X)+h(X)]=\E g(X)+\E h(X)$,
\end{enumerate}
provided $\E g(X)$ and $\E h(X)$ exist.

\end{prop}
\begin{proof}
Go directly from the definition. For example,\[
\E[c\cdot g(X)]=\sum_{x\in S}c\cdot g(x)f_{X}(x)=c\cdot\sum_{x\in S}g(x)f_{X}(x)=c\E g(X).\]

\end{proof}

\subsection{Moment Generating Functions\label{sub:MGFs}}
\begin{defn}
Given a random variable $X$, its \emph{moment generating function}
(abbreviated MGF) is defined by the formula\begin{equation}
M_{X}(t)=\E\me^{tX}=\sum_{x\in S}\me^{tx}f_{X}(x),\end{equation}
provided the (potentially infinite) series is convergent for all $t$
in a neighborhood of zero (that is, for all $-\epsilon<t<\epsilon$,
for some $\epsilon>0$). 
\end{defn}
Note that for any MGF $M_{X}$, \begin{equation}
M_{X}(0)=\E\me^{0\cdot X}=\E1=1.\end{equation}
We will calculate the MGF for the two distributions introduced above.
\begin{example}
Find the MGF for $X\sim\mathsf{disunif}(m)$. 

Since $f(x)=1/m$, the MGF takes the form\[
M(t)=\sum_{x=1}^{m}\me^{tx}\frac{1}{m}=\frac{1}{m}(\me^{t}+\me^{2t}+\cdots+\me^{mt}),\quad\mbox{for any \ensuremath{t}.}\]

\end{example}

\begin{example}
Find the MGF for $X\sim\mathsf{binom}(\mathtt{size}=n,\,\mathtt{prob}=p)$.
\end{example}
\begin{alignat*}{1}
M_{X}(t)= & \sum_{x=0}^{n}\me^{tx}\,{n \choose x}\, p^{x}(1-p)^{n-x},\\
= & \sum_{x=0}^{n}{n \choose x}\,(p\me^{t})^{x}q^{n-x},\\
= & (p\me^{t}+q)^{n},\quad\mbox{for any \ensuremath{t}.}\end{alignat*}



\subsection*{Applications}

We will discuss three applications of moment generating functions
in this book. The first is the fact that an MGF may be used to accurately
identify the probability distribution that generated it, which rests
on the following:
\begin{thm}
\label{thm:mgf-unique}The moment generating function, if it exists
in a neighborhood of zero, determines a probability distribution \emph{uniquely}. \end{thm}
\begin{proof}
Unfortunately, the proof of such a theorem is beyond the scope of
a text like this one. Interested readers could consult Billingsley
\cite{Billingsley1995}.
\end{proof}


We will see an example of Theorem \ref{thm:mgf-unique} in action.
\begin{example}
Suppose we encounter a random variable which has MGF \[
M_{X}(t)=(0.3+0.7\me^{t})^{13}.\]
Then $X\sim\mathsf{binom}(\mathtt{size}=13,\,\mathtt{prob}=0.7)$.
\end{example}


An MGF is also known as a {}``Laplace Transform'' and is manipulated
in that context in many branches of science and engineering.


\subsection*{Why is it called a Moment Generating Function?}

This brings us to the second powerful application of MGFs. Many of
the models we study have a simple MGF, indeed, which permits us to
determine the mean, variance, and even higher moments very quickly.
Let us see why. We already know that

\begin{alignat*}{1}
M(t)= & \sum_{x\in S}\me^{tx}f(x).\end{alignat*}
Take the derivative with respect to $t$ to get\begin{equation}
M'(t)=\frac{\diff}{\diff t}\left(\sum_{x\in S}\me^{tx}f(x)\right)=\sum_{x\in S}\ \frac{\diff}{\diff t}\left(\me^{tx}f(x)\right)=\sum_{x\in S}x\me^{tx}f(x),\end{equation}
and so if we plug in zero for $t$ we see \begin{equation}
M'(0)=\sum_{x\in S}x\me^{0}f(x)=\sum_{x\in S}xf(x)=\mu=\E X.\end{equation}
Similarly, $M''(t)=\sum x^{2}\me^{tx}f(x)$ so that $M''(0)=\E X^{2}$.
And in general, we can see%
\footnote{We are glossing over some significant mathematical details in our
derivation. Suffice it to say that when the MGF exists in a neighborhood
of $t=0$, the exchange of differentiation and summation is valid
in that neighborhood, and our remarks hold true.%
} that\begin{equation}
M_{X}^{(r)}(0)=\E X^{r}=\mbox{\ensuremath{r^{\mathrm{th}}}moment of \ensuremath{X}about the origin.}\end{equation}
These are also known as \emph{raw moments} and are sometimes denoted
$\mu_{r}'$. In addition to these are the so called \emph{central
moments} $\mu_{r}$ defined by\begin{equation}
\mu_{r}=\E(X-\mu)^{r},\quad r=1,2,\ldots\end{equation}

\begin{example}
Let $X\sim\mathsf{binom}(\mathtt{size}=n,\,\mathtt{prob}=p)\mbox{ with \ensuremath{M(t)=(q+p\me^{t})^{n}}}$.
We calculated the mean and variance of a binomial random variable
in Section \ref{sec:binom-dist} by means of the binomial series.
But look how quickly we find the mean and variance with the moment
generating function.

\begin{alignat*}{1}
M'(t)= & n(q+p\me^{t})^{n-1}p\me^{t}\left|_{t=0}\right.,\\
= & n\cdot1^{n-1}p,\\
= & np.\end{alignat*}
And\begin{alignat*}{1}
M''(0)= & n(n-1)[q+p\me^{t}]^{n-2}(p\me^{t})^{2}+n[q+p\me^{t}]^{n-1}p\me^{t}\left|_{t=0}\right.,\\
\E X^{2}= & n(n-1)p^{2}+np.\end{alignat*}
Therefore \begin{alignat*}{1}
\sigma^{2}= & \E X^{2}-(\E X)^{2},\\
= & n(n-1)p^{2}+np-n^{2}p^{2},\\
= & np-np^{2}=npq.\end{alignat*}
See how much easier that was?\end{example}
\begin{rem}
We learned in this section that $M^{(r)}(0)=\E X^{r}$. We remember
from Calculus II that certain functions $f$ can be represented by
a Taylor series expansion about a point $a$, which takes the form\begin{equation}
f(x)=\sum_{r=0}^{\infty}\frac{f^{(r)}(a)}{r!}(x-a)^{r},\quad\mbox{for all \ensuremath{|x-a|<R},}\end{equation}
where $R$ is called the \emph{radius of convergence} of the series
(see Appendix \ref{sec:Sequences-and-Series}). We combine the two
to say that if an MGF exists for all $t$ in the interval $(-\epsilon,\epsilon)$,
then we can write\begin{equation}
M_{X}(t)=\sum_{r=0}^{\infty}\frac{\E X^{r}}{r!}t^{r},\quad\mbox{for all \ensuremath{|t|<\epsilon}.}\end{equation}

\end{rem}

\subsection{How to do it with \textsf{R}}

The \inputencoding{latin9}\lstinline[showstringspaces=false]!distrEx!\inputencoding{utf8}
package provides an expectation operator \inputencoding{latin9}\lstinline[showstringspaces=false]!E!\inputencoding{utf8}
which can be used on random variables that have been defined in the
ordinary \inputencoding{latin9}\lstinline[showstringspaces=false]!distr!\inputencoding{utf8}
sense:

<<>>=
X <- Binom(size = 3, prob = 0.45)
library(distrEx)
E(X)
E(3*X + 4)
@

For discrete random variables with finite support, the expectation
is simply computed with direct summation. In the case that the random
variable has infinite support and the function is crazy, then the
expectation is not computed directly, rather, it is estimated by first
generating a random sample from the underlying model and next computing
a sample mean of the function of interest. 

There are methods for other population parameters:

<<>>=
var(X)
sd(X)
@

There are even methods for \inputencoding{latin9}\lstinline[showstringspaces=false]!IQR!\inputencoding{utf8},
\inputencoding{latin9}\lstinline[showstringspaces=false]!mad!\inputencoding{utf8},
\inputencoding{latin9}\lstinline[showstringspaces=false]!skewness!\inputencoding{utf8},
and \inputencoding{latin9}\lstinline[showstringspaces=false]!kurtosis!\inputencoding{utf8}.


\section{The Empirical Distribution\label{sec:empirical-distribution}}

Do an experiment $n$ times and observe $n$ values $x_{1}$, $x_{2}$,
\ldots{}, $x_{n}$ of a random variable $X$. For simplicity in most
of the discussion that follows it will be convenient to imagine that
the observed values are distinct, but the remarks are valid even when
the observed values are repeated. 
\begin{defn}
The \emph{empirical cumulative distribution function} $F_{n}$ (written
ECDF)\index{Empirical distribution} is the probability distribution
that places probability mass $1/n$ on each of the values $x_{1}$,
$x_{2}$, \ldots{}, $x_{n}$. The empirical PMF takes the form\begin{equation}
f_{X}(x)=\frac{1}{n},\quad x\in\left\{ x_{1},x_{2},...,x_{n}\right\} .\end{equation}
If the value $x_{i}$ is repeated $k$ times, the mass at $x_{i}$
is accumulated to $k/n$.
\end{defn}


The mean of the empirical distribution is\begin{equation}
\mu=\sum_{x\in S}xf_{X}(x)=\sum_{i=1}^{n}x_{i}\cdot\frac{1}{n}\end{equation}
and we recognize this last quantity to be the sample mean, $\xbar$.
The variance of the empirical distribution is\begin{equation}
\sigma^{2}=\sum_{x\in S}(x-\mu)^{2}f_{X}(x)=\sum_{i=1}^{n}(x_{i}-\xbar)^{2}\cdot\frac{1}{n}\end{equation}
and this last quantity looks very close to what we already know to
be the sample variance.\begin{equation}
s^{2}=\frac{1}{n-1}\sum_{i=1}^{n}(x_{i}-\xbar)^{2}.\end{equation}
The \emph{empirical quantile function} is the inverse of the ECDF.
See Section \ref{sub:Normal-Quantiles-QF}.


\subsection{How to do it with \textsf{R}}

The empirical distribution is not directly available as a distribution
in the same way that the other base probability distributions are,
but there are plenty of resources available for the determined investigator. 

Given a data vector of observed values \inputencoding{latin9}\lstinline[showstringspaces=false]!x!\inputencoding{utf8},
we can see the empirical CDF with the \inputencoding{latin9}\lstinline[showstringspaces=false]!ecdf!\inputencoding{utf8}\index{ecdf@\texttt{ecdf}}
function:

<<>>=
x <- c(4, 7, 9, 11, 12)
ecdf(x)
@

The above shows that the returned value of \inputencoding{latin9}\lstinline[showstringspaces=false]!ecdf(x)!\inputencoding{utf8}
is not a \emph{number} but rather a \emph{function}. The ECDF is not
usually used by itself in this form, by itself. More commonly it is
used as an intermediate step in a more complicated calculation, for
instance, in hypothesis testing (see Chapter \ref{cha:Hypothesis-Testing})
or resampling (see Chapter \ref{cha:resampling-methods}). It is nevertheless
instructive to see what the \inputencoding{latin9}\lstinline[showstringspaces=false]!ecdf!\inputencoding{utf8}
looks like, and there is a special plot method for \inputencoding{latin9}\lstinline[showstringspaces=false]!ecdf!\inputencoding{utf8}
objects.

<<eval = FALSE>>=
plot(ecdf(x))
@

%
\begin{figure}[H]
\begin{centering}
<<echo = FALSE, fig=true, height = 4.5, width = 6>>=
plot(ecdf(x))
@
\par\end{centering}

\caption{The empirical CDF\label{fig:empirical-CDF}}

\end{figure}


See Figure \ref{fig:empirical-CDF}. The graph is of a right-continuous
function with jumps exactly at the locations stored in \inputencoding{latin9}\lstinline[showstringspaces=false]!x!\inputencoding{utf8}.
There are no repeated values in \inputencoding{latin9}\lstinline[showstringspaces=false]!x!\inputencoding{utf8}
so all of the jumps are equal to $1/5=0.2$.

The empirical PDF is not usually of particular interest in itself,
but if we really wanted we could define a function to serve as the
empirical PDF:

<<keep.source = TRUE>>=
epdf <- function(x) function(t){sum(x %in% t)/length(x)}
x <- c(0,0,1)
epdf(x)(0)       # should be 2/3
@

To simulate from the empirical distribution supported on the vector
\inputencoding{latin9}\lstinline[showstringspaces=false]!x!\inputencoding{utf8},
we use the \inputencoding{latin9}\lstinline[showstringspaces=false]!sample!\inputencoding{utf8}\index{sample@\texttt{sample}}
function.

<<>>=
x <- c(0,0,1)
sample(x, size = 7, replace = TRUE)
@

We can get the empirical quantile function in \textsf{R} with \inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false]!quantile(x, probs = p, type = 1)!\inputencoding{utf8};
see Section \ref{sub:Normal-Quantiles-QF}.

As we hinted above, the empirical distribution is significant more
because of how and where it appears in more sophisticated applications.
We will explore some of these in later chapters -- see, for instance,
Chapter \ref{cha:resampling-methods}.


\section{Other Discrete Distributions\label{sec:other-discrete-distributions}}

The binomial and discrete uniform distributions are popular, and rightly
so; they are simple and form the foundation for many other more complicated
distributions. But the particular uniform and binomial models only
apply to a limited range of problems. In this section we introduce
situations for which we need more than what the uniform and binomial
offer.


\subsection{Dependent Bernoulli Trials\label{sec:non-bernoulli-trials}}


\subsubsection*{The Hypergeometric Distribution\label{sub:hypergeometric-dist}}

Consider an urn with 7 white balls and 5 black balls. Let our random
experiment be to randomly select 4 balls, without replacement, from
the urn. Then the probability of observing 3 white balls (and thus
1 black ball) would be \begin{equation}
\P(3W,1B)=\frac{{7 \choose 3}{5 \choose 1}}{{12 \choose 4}}.\end{equation}
More generally, we sample without replacement $K$ times from an urn
with $M$ white balls and $N$ black balls. Let $X$ be the number
of white balls in the sample. The PMF of $X$ is 

\begin{equation}
f_{X}(x)=\frac{{M \choose x}{N \choose K-x}}{{M+N \choose K}}.\end{equation}
We say that $X$ has a \emph{hypergeometric distribution} and write
$X\sim\mathsf{hyper}(\mathtt{m}=M,\,\mathtt{n}=N,\,\mathtt{k}=K)$. 

The support set for the hypergeometric distribution is a little bit
tricky. It is tempting to say that $x$ should go from 0 (no white
balls in the sample) to $K$ (no black balls in the sample), but that
does not work if $K>M$, because it is impossible to have more white
balls in the sample than there were white balls originally in the
urn. We have the same trouble if $K>N$. The good news is that the
majority of examples we study have $K\leq M$ and $K\leq N$ and we
will happily take the support to be $x=0,\ 1,\ \ldots,\ K$. 

It is shown in Exercise \ref{xca:hyper-mean-variance} that \begin{equation}
\mu=K\frac{M}{M+N},\quad\sigma^{2}=K\frac{MN}{(M+N)^{2}}\frac{M+N-K}{M+N-1}.\end{equation}


The associated \textsf{R} functions for the PMF and CDF are \inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false,tabsize=4]!dhyper(x, m, n, k)!\inputencoding{utf8}
and \inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false,tabsize=4]!phyper!\inputencoding{utf8},
respectively. There are two more functions: \inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false,tabsize=4]!qhyper!\inputencoding{utf8},
which we will discuss in Section \ref{sub:Normal-Quantiles-QF}, and
\inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false,tabsize=4]!rhyper!\inputencoding{utf8},
discussed below.
\begin{example}
Suppose in a certain shipment of 250 Pentium processors there are
17 defective processors. A quality control consultant randomly collects
5 processors for inspection to determine whether or not they are defective.
Let $X$ denote the number of defectives in the sample. \end{example}
\begin{enumerate}
\item Find the probability of exactly 3 defectives in the sample, that is,
find $\P(X=3)$. 


\emph{Solution:} We know that $X\sim\mathsf{hyper}(\mathtt{m}=17,\,\mathtt{n}=233,\,\mathtt{k}=5)$.
So the required probability is just\[
f_{X}(3)=\frac{{17 \choose 3}{233 \choose 2}}{{250 \choose 5}}.\]
To calculate it in \textsf{R} we just type 

<<>>=
dhyper(3, m = 17, n = 233, k = 5)
@

To find it with the \textsf{R} Commander we go \textsf{Probability}
$\triangleright$ \textsf{Discrete Distributions} $\triangleright$
\textsf{Hypergeometric distribution} $\triangleright$ \textsf{Hypergeometric
probabilities}\ldots{}.We fill in the parameters $m=17$, $n=233$,
and $k=5$. Click \textsf{OK}, and the following table is shown in
the window.

<<>>=
A <- data.frame(Pr=dhyper(0:4, m = 17, n = 233, k = 5))
rownames(A) <- 0:4 
A
@

We wanted $\P(X=3)$, and this is found from the table to be approximately
0.0024. The value is rounded to the fourth decimal place.

We know from our above discussion that the sample space should be
$x=0,1,2,3,4,5$, yet, in the table the probabilities are only displayed
for $x=1,2,3$, and 4. What is happening? As it turns out, the \textsf{R}
Commander will only display probabilities that are 0.00005 or greater.
Since $x=5$ is not shown, it suggests that the outcome has a tiny
probability. To find its exact value we use the \inputencoding{latin9}\lstinline[basicstyle={\ttfamily},showstringspaces=false,tabsize=4]!dhyper!\inputencoding{utf8}
function:

<<>>=
dhyper(5, m = 17, n = 233, k = 5)
@

In other words, $\P(X=5)\approx0.0000007916049$, a small number indeed.

\item Find the probability that there are at most 2 defectives in the sample,
that is, compute $\P(X\leq2)$.


\emph{Solution:} Since $\P(X\leq2)=\P(X=0,1,2)$, one way to do this
would be to add the 0, 1, and 2 entries in the above table. this gives
$0.7011+0.2602+0.0362=0.9975$. Our answer should be correct up to
the accuracy of 4 decimal places. However, a more precise method is
provided by the \textsf{R} Commander. Under the \textsf{Hypergeometric
distribution} menu we select \textsf{Hypergeometric tail probabilities}\textsf{\emph{\ldots{}}}.
We fill in the parameters $m$, $n$, and $k$ as before, but in the
\textsf{Variable value(s)} dialog box we enter the value 2. We notice
that the \texttt{Lower tail} option is checked, and we leave that
alone. Click \textsf{OK}.

<<>>=
phyper(2, m = 17, n = 233, k = 5)
@

And thus $\P(X\leq2)\approx0.9975771$. We have confirmed that the
above answer was correct up to four decimal places.

\item Find $\P(X>1)$. 


The table did not give us the explicit probability $\P(X=5)$, so
we can not use the table to give us this probability. We need to use
another method. Since $\P(X>1)=1-\P(X\leq1)=1-F_{X}(1)$, we can find
the probability with \textsf{Hypergeometric tail probabilities}\textsf{\emph{\ldots{}}}.
We enter 1 for \textsf{Variable Value(s)}, we enter the parameters
as before, and in this case we choose the \texttt{Upper tail} option.
This results in the following output.

<<>>=
phyper(1, m = 17, n = 233, k = 5, lower.tail = FALSE)
@

In general, the \texttt{Upper tail} option of a tail probabilities
dialog computes $\P(X>x)$ for all given \textsf{Variable Value(s)}
$x$.

\item Generate $100,000$ observations of the random variable $X$.


We can randomly simulate as many observations of $X$ as we want in
\textsf{R} Commander. Simply choose \textsf{Simulate hypergeometric
variates\ldots{}} in the \textsf{Hypergeometric distribution} dialog.

In the \textsf{Number of samples} dialog, type 1. Enter the parameters
as above. Under the \textsf{Store Values} section, make sure \textsf{New
Data set} is selected. Click \textsf{OK}. 

A new dialog should open, with the default name \texttt{Simset1}.
We could change this if we like, according to the rules for \textsf{R}
object names. In the sample size box, enter 100000. Click \textsf{OK}. 

In the Console Window, \textsf{R} Commander should issue an alert
that \texttt{Simset1} has been initialized, and in a few seconds,
it should also state that 100,000 hypergeometric variates were stored
in \texttt{hyper.sim1}. We can view the sample by clicking the \textsf{View
Data Set}\texttt{ }button on the \textsf{R} Commander interface.

We know from our formulas that $\mu=K\cdot M/(M+N)=5*17/250=0.34$.
We can check our formulas using the fact that with repeated observations
of $X$ we would expect about 0.34 defectives on the average. To see
how our sample reflects the true mean, we can compute the sample mean

\texttt{\textcolor{blue}{Rcmdr> mean(Simset2\$hyper.sim1, na.rm=TRUE)}}~\\
\texttt{\textcolor{blue}{{[}1{]} 0.340344}}

\texttt{\textcolor{blue}{Rcmdr> sd(Simset2\$hyper.sim1, na.rm=TRUE)}}~\\
\texttt{\textcolor{blue}{{[}1{]} 0.5584982}}

\texttt{\textcolor{blue}{$\vdots$}}

We see that when given many independent observations of $X$, the
sample mean is very close to the true mean $\mu$. We can repeat the
same idea and use the sample standard deviation to estimate the true
standard deviation of $X$. From the output above our estimate is
0.5584982, and from our formulas we get\[
\sigma^{2}=K\frac{MN}{(M+N)^{2}}\frac{M+N-K}{M+N-1}\approx0.3117896,\]
 with $\sigma=\sqrt{\sigma^{2}}\approx0.5583811944$. Our estimate
was pretty close.

From the console we can generate random hypergeometric variates with
the \inputencoding{latin9}\lstinline!rhyper!\inputencoding{utf8}
function, as demonstrated below.

<<>>=
rhyper(10, m = 17, n = 233, k = 5)
@

\end{enumerate}

\subsubsection*{Sampling With and Without Replacement\label{sub:Sampling-With-and}}

Suppose that we have a large urn with, say, $M$ white balls and $N$
black balls. We take a sample of size $n$ from the urn, and let $X$
count the number of white balls in the sample. If we sample
\begin{description}
\item [{without~replacement,}] then $X\sim\mathsf{hyper}(\mathtt{m=}M,\,\mathtt{n}=N,\,\mathtt{k}=n)$
and has mean and variance\begin{alignat*}{1}
\mu= & n\frac{M}{M+N},\\
\sigma^{2}= & n\frac{MN}{(M+N)^{2}}\frac{M+N-n}{M+N-1},\\
= & n\frac{M}{M+N}\left(1-\frac{M}{M+N}\right)\frac{M+N-n}{M+N-1}.\end{alignat*}
On the other hand, if we sample
\item [{with~replacement,}] then $X\sim\mathsf{binom}(\mathtt{size}=n,\,\mathtt{prob}=M/(M+N))$
with mean and variance\begin{alignat*}{1}
\mu= & n\frac{M}{M+N},\\
\sigma^{2}= & n\frac{M}{M+N}\left(1-\frac{M}{M+N}\right).\end{alignat*}

\end{description}
We see that both sampling procedures have the same mean, and the method
with the larger variance is the {}``with replacement'' scheme. The
factor by which the variances differ,\begin{equation}
\frac{M+N-n}{M+N-1},\end{equation}
is called a \emph{finite population correction}. For a fixed sample
size $n$, as $M,N\to\infty$ it is clear that the correction goes
to 1, that is, for infinite populations the sampling schemes are essentially
the same with respect to mean and variance.


\subsection{Waiting Time Distributions\label{sec:Waiting-Time-Distributions}}

Another important class of problems is associated with the amount
of time it takes for a specified event of interest to occur. For example,
we could flip a coin repeatedly until we observe Heads. We could toss
a piece of paper repeatedly until we make it in the trash can.


\subsubsection*{The Geometric Distribution\label{sub:The-Geometric-Distribution}}

Suppose that we conduct Bernoulli trials repeatedly, noting the successes
and failures. Let $X$ be the number of failures before a success.
If $\P(S)=p$ then $X$ has PMF

\begin{equation}
f_{X}(x)=p(1-p)^{x},\quad x=0,1,2,\ldots\end{equation}
(Why?) We say that $X$ has a \emph{Geometric distribution} and we
write $X\sim\mathsf{geom}(\mathtt{prob}=p)$. The associated \textsf{R}
functions are \inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false,tabsize=4]!dgeom(x, prob)!\inputencoding{utf8},
\inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false,tabsize=4]!pgeom!\inputencoding{utf8},
\inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false,tabsize=4]!qgeom!\inputencoding{utf8},
and \inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false,tabsize=4]!rhyper!\inputencoding{utf8},
which give the PMF, CDF, quantile function, and simulate random variates,
respectively.

Again it is clear that $f(x)\geq0$ and we check that $\sum f(x)=1$
(see Equation \ref{eq:geom-series} in Appendix \ref{sec:Sequences-and-Series}):\begin{alignat*}{1}
\sum_{x=0}^{\infty}p(1-p)^{x}= & p\sum_{x=0}^{\infty}q^{x}=p\,\frac{1}{1-q}=1.\end{alignat*}
We will find in the next section that the mean and variance are\begin{equation}
\mu=\frac{1-p}{p}=\frac{q}{p}\mbox{ and }\sigma^{2}=\frac{q}{p^{2}}.\end{equation}

\begin{example}
The Pittsburgh Steelers place kicker, Jeff Reed, made 81.2\% of his
attempted field goals in his career up to 2006. Assuming that his
successive field goal attempts are approximately Bernoulli trials,
find the probability that Jeff misses at least 5 field goals before
his first successful goal.\emph{ }

\emph{Solution}: If $X=$ the number of missed goals until Jeff's
first success, then $X\sim\mathsf{geom}(\mathtt{prob}=0.812)$ and
we want $\P(X\geq5)=\P(X>4)$. We can find this in \textsf{R} with 

<<>>=
pgeom(4, prob = 0.812, lower.tail = FALSE)
@

\end{example}
\begin{note}
Some books use a slightly different definition of the geometric distribution.
They consider Bernoulli trials and let $Y$ count instead the number
of trials until a success, so that $Y$ has PMF\begin{equation}
f_{Y}(y)=p(1-p)^{y-1},\quad y=1,2,3,\ldots\end{equation}
When they say {}``geometric distribution'', this is what they mean.
It is not hard to see that the two definitions are related. In fact,
if $X$ denotes our geometric and $Y$ theirs, then $Y=X+1$. Consequently,
they have $\mu_{Y}=\mu_{X}+1$ and $\sigma_{Y}^{2}=\sigma_{X}^{2}$.
\end{note}

\subsubsection*{The Negative Binomial Distribution\label{sub:The-Negative-Binomial}}

We may generalize the problem and consider the case where we wait
for \emph{more} than one success. Suppose that we conduct Bernoulli
trials repeatedly, noting the respective successes and failures. Let
$X$ count the number of failures before $r$ successes. If $\P(S)=p$
then $X$ has PMF

\begin{equation}
f_{X}(x)={r+x-1 \choose r-1}\, p^{r}(1-p)^{x},\quad x=0,1,2,\ldots\end{equation}
We say that $X$ has a \emph{Negative Binomial distribution} and write
$X\sim\mathsf{nbinom}(\mathtt{size}=r,\,\mathtt{prob}=p)$. The associated
\textsf{R} functions are \inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false,tabsize=4]!dnbinom(x, size, prob)!\inputencoding{utf8},
\inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false,tabsize=4]!pnbinom!\inputencoding{utf8},
\inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false,tabsize=4]!qnbinom!\inputencoding{utf8},
and \inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false,tabsize=4]!rnbinom!\inputencoding{utf8},
which give the PMF, CDF, quantile function, and simulate random variates,
respectively.

As usual it should be clear that $f_{X}(x)\geq0$ and the fact that
$\sum f_{X}(x)=1$ follows from a generalization of the geometric
series by means of a Maclaurin's series expansion:\begin{alignat}{1}
\frac{1}{1-t}= & \sum_{k=0}^{\infty}t^{k},\quad\mbox{for \ensuremath{-1<t<1}},\mbox{ and}\\
\frac{1}{(1-t)^{r}}= & \sum_{k=0}^{\infty}{r+k-1 \choose r-1}\, t^{k},\quad\mbox{for \ensuremath{-1<t<1}}.\end{alignat}
Therefore \begin{equation}
\sum_{x=0}^{\infty}f_{X}(x)=p^{r}\sum_{x=0}^{\infty}{r+x-1 \choose r-1}\, q^{x}=p^{r}(1-q)^{-r}=1,\end{equation}
since $|q|=|1-p|<1$. 
\begin{example}
We flip a coin repeatedly and let $X$ count the number of Tails until
we get seven Heads. What is $\P(X=5)?$

\emph{Solution:} We know that $X\sim\mathsf{nbinom}(\mathtt{size}=7,\,\mathtt{prob}=1/2)$.
\[
\P(X=5)=f_{X}(5)={7+5-1 \choose 7-1}(1/2)^{7}(1/2)^{5}={11 \choose 6}2^{-12}\]
and we can get this in \textsf{R} with

<<>>=
dnbinom(5, size = 7, prob = 0.5)
@

Let us next compute the MGF of $X\sim\mathsf{nbinom}(\mathtt{size}=r,\,\mathtt{prob}=p)$.

\begin{alignat*}{1}
M_{X}(t)= & \sum_{x=0}^{\infty}\me^{tx}\ {r+x-1 \choose r-1}p^{r}q^{x}\\
= & p^{r}\sum_{x=0}^{\infty}{r+x-1 \choose r-1}[q\me^{t}]^{x}\\
= & p^{r}(1-qe^{t})^{-r},\quad\mbox{provided \ensuremath{|q\me^{t}|<1,}}\end{alignat*}
and so \begin{equation}
M_{X}(t)=\left(\frac{p}{1-q\me^{t}}\right)^{r},\quad\mbox{for \ensuremath{q\me^{t}<1}}.\end{equation}
We see that $q\me^{t}<1$ when $t<-\ln(1-p)$.

Let $X\sim\mathsf{nbinom}(\mathtt{size}=r,\mathtt{prob}=p)\mbox{ with \ensuremath{M(t)=p^{r}(1-q\me^{t})^{-r}}}$.
We proclaimed above the values of the mean and variance. Now we are
equipped with the tools to find these directly.

\begin{alignat*}{1}
M'(t)= & p^{r}(-r)(1-q\me^{t})^{-r-1}(-q\me^{t}),\\
= & rq\me^{t}p^{r}(1-q\me^{t})^{-r-1},\\
= & \frac{rq\me^{t}}{1-q\me^{t}}M(t),\mbox{ and so }\\
M'(0)= & \frac{rq}{1-q}\cdot1=\frac{rq}{p}.\end{alignat*}
Thus $\mu=rq/p$. We next find $\E X^{2}$.\begin{alignat*}{1}
M''(0)= & \left.\frac{rq\me^{t}(1-q\me^{t})-rq\me^{t}(-q\me^{t})}{(1-q\me^{t})^{2}}M(t)+\frac{rq\me^{t}}{1-q\me^{t}}M'(t)\right|_{t=0},\\
= & \frac{rqp+rq^{2}}{p^{2}}\cdot1+\frac{rq}{p}\left(\frac{rq}{p}\right),\\
= & \frac{rq}{p^{2}}+\left(\frac{rq}{p}\right)^{2}.\end{alignat*}
Finally we may say $\sigma^{2}=M''(0)-[M'(0)]^{2}=rq/p^{2}.$
\end{example}

\begin{example}
A random variable has MGF\[
M_{X}(t)=\left(\frac{0.19}{1-0.81\me^{t}}\right)^{31}.\]
Then $X\sim\mathsf{nbinom}(\mathtt{size}=31,\,\mathtt{prob}=0.19)$.\end{example}
\begin{note}
As with the Geometric distribution, some books use a slightly different
definition of the Negative Binomial distribution. They consider Bernoulli
trials and let $Y$ be the number of trials until $r$ successes,
so that $Y$ has PMF\begin{equation}
f_{Y}(y)={y-1 \choose r-1}p^{r}(1-p)^{y-r},\quad y=r,r+1,r+2,\ldots\end{equation}
It is again not hard to see that if $X$ denotes our Negative Binomial
and $Y$ theirs, then $Y=X+r$. Consequently, they have $\mu_{Y}=\mu_{X}+r$
and $\sigma_{Y}^{2}=\sigma_{X}^{2}$.
\end{note}

\subsection{Arrival Processes\label{sec:Arrival-Processes}}


\subsubsection*{The Poisson Distribution\label{sub:The-Poisson-Distribution}}

This is a distribution associated with {}``rare events'', for reasons
which will become clear in a moment. The events might be:
\begin{itemize}
\item traffic accidents,
\item typing errors, or
\item customers arriving in a bank.
\end{itemize}
Let $\lambda$ be the average number of events in the time interval
$[0,1]$. Let the random variable $X$ count the number of events
occurring in the interval. Then under certain reasonable conditions
it can be shown that

\begin{equation}
f_{X}(x)=\P(X=x)=\me^{-\lambda}\frac{\lambda^{x}}{x!},\quad x=0,1,2,\ldots\end{equation}
We use the notation $X\sim\mathsf{pois}(\mathtt{lambda}=\lambda)$.
The associated \textsf{R} functions are \inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false,tabsize=4]!dpois(x, lambda)!\inputencoding{utf8},
\inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false,tabsize=4]!ppois!\inputencoding{utf8},
\inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false,tabsize=4]!qpois!\inputencoding{utf8},
and \inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false,tabsize=4]!rpois!\inputencoding{utf8},
which give the PMF, CDF, quantile function, and simulate random variates,
respectively.


\paragraph*{What are the reasonable conditions?}

Divide $[0,1]$ into subintervals of length $1/n$. A \emph{Poisson
process}\index{Poisson process} satisfies the following conditions:
\begin{itemize}
\item the probability of an event occurring in a particular subinterval
is $\approx\lambda/n$.
\item the probability of two or more events occurring in any subinterval
is $\approx0$.
\item occurrences in disjoint subintervals are independent.\end{itemize}
\begin{rem}
\label{rem:poisson-process}If $X$ counts the number of events in
the interval $[0,t]$ and $\lambda$ is the average number that occur
in unit time, then $X\sim\mathsf{pois}(\mathtt{lambda}=\lambda t)$,
that is,\begin{equation}
\P(X=x)=\me^{-\lambda t}\frac{(\lambda t)^{x}}{x!},\quad x=0,1,2,3\ldots\end{equation}
\end{rem}
\begin{example}
On the average, five cars arrive at a particular car wash every hour.
Let $X$ count the number of cars that arrive from 10AM to 11AM. Then
$X\sim\mathsf{pois}(\mathtt{lambda}=5)$. Also, $\mu=\sigma^{2}=5$.
What is the probability that no car arrives during this period? 

Solution: The probability that no car arrives is \[
\P(X=0)=\me^{-5}\frac{5^{0}}{0!}=\me^{-5}\approx0.0067.\]


\end{example}

\begin{example}
Suppose the car wash above is in operation from 8AM to 6PM, and we
let $Y$ be the number of customers that appear in this period. Since
this period covers a total of 10 hours, from Remark \ref{rem:poisson-process}
we get that $Y\sim\mathsf{pois}(\mathtt{lambda}=5\ast10=50)$. What
is the probability that there are between 48 and 50 customers, inclusive?

Solution: We want $\P(48\leq Y\leq50)=\P(X\leq50)-\P(X\leq47)$. 

<<>>=
diff(ppois(c(47, 50), lambda = 50))
@

\end{example}

\section{Functions of Discrete Random Variables\label{sec:functions-discrete-rvs}}

We have built a large catalogue of discrete distributions, but the
tools of this section will give us the ability to consider infinitely
many more. Given a random variable $X$ and a given function $h$,
we may consider $Y=h(X)$. Since the values of $X$ are determined
by chance, so are the values of $Y$. The question is, what is the
PMF of the random variable $Y$? The answer, of course, depends on
$h$. In the case that $h$ is one-to-one (see Appendix \ref{sec:Differential-and-Integral}),
the solution can be found by simple substitution.
\begin{example}
Let $X\sim\mathsf{nbinom}(\mathtt{size}=r,\,\mathtt{prob}=p)$. We
saw in \ref{sec:other-discrete-distributions} that $X$ represents
the number of failures until $r$ successes in a sequence of Bernoulli
trials. Suppose now that instead we were interested in counting the
number of trials (successes and failures) until the $r$$^{\text{th}}$
success occurs, which we will denote by $Y$. In a given performance
of the experiment, the number of failures ($X$) and the number of
successes ($r$) together will comprise the total number of trials
($Y$), or in other words, $X+r=Y$. We may let $h$ be defined by
$h(x)=x+r$ so that $Y=h(X)$, and we notice that $h$ is linear and
hence one-to-one. Finally, $X$ takes values $0,\ 1,\ 2,\ldots$ implying
that the support of $Y$ would be $\left\{ r,\ r+1,\ r+2,\ldots\right\} $.
Solving for $X$ we get $X=Y-r$. Examining the PMF of $X$\begin{equation}
f_{X}(x)={r+x-1 \choose r-1}\, p^{r}(1-p)^{x},\end{equation}
we can substitute $x=y-r$ to get\begin{eqnarray*}
f_{Y}(y) & = & f_{X}(y-r),\\
 & = & {r+(y-r)-1 \choose r-1}\, p^{r}(1-p)^{y-r},\\
 & = & {y-1 \choose r-1}\, p^{r}(1-p)^{y-r},\quad y=r,\, r+1,\ldots\end{eqnarray*}

\end{example}


Even when the function $h$ is not one-to-one, we may still find the
PMF of $Y$ simply by accumulating, for each $y$, the probability
of all the $x$'s that are mapped to that $y$.


\begin{prop}
Let $X$ be a discrete random variable with PMF $f_{X}$ supported
on the set $S_{X}$. Let $Y=h(X)$ for some function $h$. Then $Y$
has PMF $f_{Y}$ defined by\begin{equation}
f_{Y}(y)=\sum_{\{x\in S_{X}|\, h(x)=y\}}f_{X}(x)\end{equation}
 \end{prop}
\begin{example}
Let $X\sim\mathsf{binom}(\mathtt{size}=4,\,\mathtt{prob}=1/2)$, and
let $Y=(X-1)^{2}$. Consider the following table:

%
\begin{table}[H]
\begin{tabular}{c|ccccc}
$x$ & 0 & 1 & 2 & 3 & 4\tabularnewline
\hline
$f_{X}(x)$ & $1/16$ & $1/4$ & $6/16$ & $1/4$ & $1/16$\tabularnewline
\hline
$y=(x-2)^{2}$ & 1 & 0 & 1 & 4 & 9\tabularnewline
\end{tabular}
\end{table}


From this we see that $Y$ has support $S_{Y}=\{0,1,4,9\}$. We also
see that $h(x)=(x-1)^{2}$ is not one-to-one on the support of $X$,
because both $x=0$ and $x=2$ are mapped by $h$ to $y=1$. Nevertheless,
we see that $Y=0$ only when $X=1$, which has probability $1/4$;
therefore, $f_{Y}(0)$ should equal $1/4$. A similar approach works
for $y=4$ and $y=9$. And $Y=1$ exactly when $X=0$ or $X=2$, which
has total probability $7/16$. In summary, the PMF of $Y$ may be
written:

%
\begin{table}[H]
\begin{tabular}{c|cccc}
$y$ & 0 & 1 & 4 & 9\tabularnewline
\hline
$f_{X}(x)$ & $1/4$ & $7/16$ & $1/4$ & $1/16$\tabularnewline
\end{tabular}
\end{table}


Note that there is not a special name for the distribution of $Y$,
it is just an example of what to do when the transformation of a random
variable is not one-to-one. The method is the same for more complicated
problems.\end{example}
\begin{prop}
If $X$ is a random variable with $\E X=\mu$ and $\mbox{Var}(X)=\sigma^{2}$,
then the mean and variance of $Y=mX+b$ is \begin{equation}
\mu_{Y}=m\mu+b,\quad\sigma_{Y}^{2}=m^{2}\sigma^{2},\quad\sigma_{Y}=|m|\sigma.\end{equation}

\end{prop}
\newpage{}


\section*{Chapter Exercises}

\addcontentsline{toc}{section}{Chapter Exercises}
\setcounter{thm}{0}
\begin{xca}
A recent national study showed that approximately 44.7\% of college
students have used Wikipedia as a source in at least one of their
term papers. Let $X$ equal the number of students in a random sample
of size $n=31$ who have used Wikipedia as a source. 
\begin{enumerate}
\item How is $X$ distributed? \[
X\sim\mathsf{binom}(\mathtt{size}=31,\,\mathtt{prob}=0.447)\]

\item Sketch the probability mass function (roughly).


\begin{center}
<<echo = FALSE, fig=true, height = 3, width = 6>>=
xmin <- qbinom(.0005, size=31 , prob=0.447) 
xmax <- qbinom(.9995, size=31 , prob=0.447) 
.x <- xmin:xmax 
plot(.x, dbinom(.x, size=31, prob=0.447), xlab="Number of Successes", ylab="Probability Mass",    main="Binomial Dist'n: Trials = 31, Prob of success = 0.447", type="h") 
points(.x, dbinom(.x, size=31, prob=0.447), pch=16) 
abline( h = 0, lty = 2, col = "grey" ) 
remove(.x, xmin, xmax)
@
\par\end{center}

\item Sketch the cumulative distribution function (roughly).


\begin{center}
<<echo = FALSE, fig=true, height = 3, width = 6>>=
xmin <- qbinom(.0005, size=31 , prob=0.447) 
xmax <- qbinom(.9995, size=31 , prob=0.447) 
.x <- xmin:xmax 
plot( stepfun(.x, pbinom((xmin-1):xmax, size=31, prob=0.447)), verticals=FALSE, do.p=FALSE, xlab="Number of Successes", ylab="Cumulative Probability", main="Binomial Dist'n: Trials = 31, Prob of success = 0.447") 
points( .x, pbinom(xmin:xmax, size=31, prob=0.447), pch = 16, cex=1.2 ) 
points( .x, pbinom((xmin-1):(xmax-1), size=31, prob=0.447), pch = 1,    cex=1.2 ) 
abline( h = 1, lty = 2, col = "grey" ) 
abline( h = 0, lty = 2, col = "grey" ) 
remove(.x, xmin, xmax) 
@
\par\end{center}

\item Find the probability that $X$ is equal to 17.


<<>>=
dbinom(17, size = 31, prob = 0.447)
@

\item Find the probability that $X$ is at most 13.


<<>>=
pbinom(13, size = 31, prob = 0.447)
@

\item Find the probability that $X$ is bigger than 11.


<<>>=
pbinom(11, size = 31, prob = 0.447, lower.tail = FALSE)
@

\item Find the probability that $X$ is at least 15.


<<>>=
pbinom(14, size = 31, prob = 0.447, lower.tail = FALSE)
@

\item Find the probability that $X$ is between 16 and 19, inclusive.


<<>>=
sum(dbinom(16:19, size = 31, prob = 0.447))
diff(pbinom(c(19,15), size = 31, prob = 0.447, lower.tail = FALSE))
@

\item Give the mean of $X$, denoted $\E X$.


<<>>=
library(distrEx)
X = Binom(size = 31, prob = 0.447)
E(X)
@

\item Give the variance of $X$.


<<>>=
var(X)
@

\item Give the standard deviation of $X$.


<<>>=
sd(X)
@

\item Find $\E(4X+51.324)$


<<>>=
E(4*X + 51.324)
@

\end{enumerate}
\end{xca}
<<echo = FALSE, results = hide>>=
rnorm(1)
@
\begin{xca}
For the following situations, decide what the distribution of $X$
should be. In nearly every case, there are additional assumptions
that should be made for the distribution to apply; identify those
assumptions (which may or may not hold in practice.)\end{xca}
\begin{enumerate}
\item We shoot basketballs at a basketball hoop, and count the number of
shots until we make a goal. Let $X$ denote the number of missed shots.
On a normal day we would typically make about 37\% of the shots.
\item In a local lottery in which a three digit number is selected randomly,
let $X$ be the number selected.
\item We drop a Styrofoam cup to the floor twenty times, each time recording
whether the cup comes to rest perfectly right side up, or not. Let
$X$ be the number of times the cup lands perfectly right side up.
\item We toss a piece of trash at the garbage can from across the room.
If we miss the trash can, we retrieve the trash and try again, continuing
to toss until we make the shot. Let $X$ denote the number of missed
shots. 
\item Working for the border patrol, we inspect shipping cargo as when it
enters the harbor looking for contraband. A certain ship comes to
port with 557 cargo containers. Standard practice is to select 10
containers randomly and inspect each one very carefully, classifying
it as either having contraband or not. Let $X$ count the number of
containers that illegally contain contraband.
\item At the same time every year, some migratory birds land in a bush outside
for a short rest. On a certain day, we look outside and let $X$ denote
the number of birds in the bush.
\item We count the number of rain drops that fall in a circular area on
a sidewalk during a ten minute period of a thunder storm.
\item We count the number of moth eggs on our window screen.
\item We count the number of blades of grass in a one square foot patch
of land.
\item We count the number of pats on a baby's back until (s)he burps.
\end{enumerate}

\begin{xca}
\label{xca:variance-shortcut}Show that $\E(X-\mu)^{2}=\E X^{2}-\mu^{2}$.
\emph{Hint}: expand the quantity $(X-\mu)^{2}$ and distribute the
expectation over the resulting terms.
\end{xca}

\begin{xca}
\label{xca:binom-factorial-expectation}If $X\sim\mathsf{binom}(\mathtt{size}=n,\,\mathtt{prob}=p)$
show that $\E X(X-1)=n(n-1)p^{2}$.
\end{xca}

\begin{xca}
\label{xca:hyper-mean-variance}Calculate the mean and variance of
the hypergeometric distribution. Show that \begin{equation}
\mu=K\frac{M}{M+N},\quad\sigma^{2}=K\frac{MN}{(M+N)^{2}}\frac{M+N-K}{M+N-1}.\end{equation}

\end{xca}

\chapter{Continuous Distributions\label{cha:Continuous-Distributions}}

\noindent The focus of the last chapter was on random variables whose
support can be written down in a list of values (finite or countably
infinite), such as the number of successes in a sequence of Bernoulli
trials. Now we move to random variables whose support is a whole range
of values, say, an interval $(a,b)$. It is shown in later classes
that it is impossible to write all of the numbers down in a list;
there are simply too many of them. 

This chapter begins with continuous random variables and the associated
PDFs and CDFs The continuous uniform distribution is highlighted,
along with the Gaussian, or normal, distribution. Some mathematical
details pave the way for a catalogue of models.

The interested reader who would like to learn more about any of the
assorted discrete distributions mentioned below should take a look
at \emph{Continuous Univariate Distributions, Volumes 1} and \emph{2}
by Johnson \emph{et al }\cite{Johnson1994,Johnson1995}.


\paragraph*{What do I want them to know?}
\begin{itemize}
\item how to choose a reasonable continuous model under a variety of physical
circumstances
\item basic correspondence between continuous versus discrete random variables
\item the general tools of the trade for manipulation of continuous random
variables, integration, \emph{etc}.
\item some details on a couple of continuous models, and exposure to a bunch
of other ones
\item how to make new continuous random variables from old ones
\end{itemize}

\section{Continuous Random Variables\label{sec:continuous-random-variables}}


\subsection{Probability Density Functions\label{sub:probability-density-functions}}

Continuous random variables have supports that look like\begin{equation}
S_{X}=[a,b]\mbox{ or }(a,b),\end{equation}
or unions of intervals of the above form. Examples of random variables
that are often taken to be continuous are:
\begin{itemize}
\item the height or weight of an individual,
\item other physical measurements such as the length or size of an object,
and
\item durations of time (usually).
\end{itemize}
Every continuous random variable $X$ has a \emph{probability density
function} (PDF) denoted $f_{X}$ associated with it%
\footnote{Not true. There are pathological random variables with no density
function. (This is one of the crazy things that can happen in the
world of measure theory). But in this book we will not get even close
to these anomalous beasts, and regardless it can be proved that the
CDF always exists.%
} that satisfies three basic properties:
\begin{enumerate}
\item $f_{X}(x)>0$ for $x\in S_{X}$,
\item $\int_{x\in S_{X}}f_{X}(x)\,\diff x=1$, and
\item $\P(X\in A)=\int_{x\in A}f_{X}(x)\:\diff x$, for an event $A\subset S_{X}$.\label{enu: contrvcond3}\end{enumerate}
\begin{rem}
We can say the following about continuous random variables:
\begin{itemize}
\item Usually, the set $A$ in \ref{enu: contrvcond3} takes the form of
an interval, for example, $A=[c,d]$, in which case\begin{equation}
\P(X\in A)=\int_{c}^{d}f_{X}(x)\:\diff x.\end{equation}

\item It follows that the probability that $X$ falls in a given interval
is simply the \emph{area under the curve }of $f_{X}$ over the interval.
\item Since the area of a line $x=c$ in the plane is zero, $\P(X=c)=0$
for any value $c$. In other words, the chance that $X$ equals a
particular value $c$ is zero, and this is true for any number $c$.
Moreover, when $a<b$ all of the following probabilities are the same:
\begin{equation}
\P(a\leq X\leq b)=\P(a<X\leq b)=\P(a\leq X<b)=\P(a<X<b).\end{equation}

\item The PDF $f_{X}$ can sometimes be greater than 1. This is in contrast
to the discrete case; every nonzero value of a PMF is a probability
which is restricted to lie in the interval $[0,1]$.
\end{itemize}
\end{rem}
We met the cumulative distribution function, $F_{X}$, in Chapter
\ref{cha:Discrete-Distributions}. Recall that it is defined by $F_{X}(t)=\P(X\leq t)$,
for $-\infty<t<\infty$. While in the discrete case the CDF is unwieldy,
in the continuous case the CDF has a relatively convenient form:\begin{equation}
F_{X}(t)=\P(X\leq t)=\int_{-\infty}^{t}f_{X}(x)\:\diff x,\quad-\infty<t<\infty.\end{equation}



\begin{rem}
For any continuous CDF $F_{X}$ the following are true.
\begin{itemize}
\item $F_{X}$ is nondecreasing , that is, $t_{1}\leq t_{2}$ implies $F_{X}(t_{1})\leq F_{X}(t_{2})$.
\item $F_{X}$ is continuous (see Appendix \ref{sec:Differential-and-Integral}).
Note the distinction from the discrete case: CDFs of discrete random
variables are not continuous, they are only right continuous.
\item $\lim_{t\to-\infty}F_{X}(t)=0$ and $\lim_{t\to\infty}F_{X}(t)=1$.
\end{itemize}
\end{rem}
There is a handy relationship between the CDF and PDF in the continuous
case. Consider the derivative of $F_{X}$:\begin{equation}
F'_{X}(t)=\frac{\diff}{\diff t}F_{X}(t)=\frac{\diff}{\diff t}\,\int_{-\infty}^{t}f_{X}(x)\,\diff x=f_{X}(t),\end{equation}
the last equality being true by the Fundamental Theorem of Calculus,
part (2) (see Appendix \ref{sec:Differential-and-Integral}). In short,
$(F_{X})'=f_{X}$ in the continuous case%
\footnote{In the discrete case, $f_{X}(x)=F_{X}(x)-\lim_{t\to x^{-}}F_{X}(t)$.%
}. 


\subsection{Expectation of Continuous Random Variables\label{sub:Expectation-of-Continuous}}

For a continuous random variable $X$ the expected value of $g(X)$
is \begin{equation}
\E g(X)=\int_{x\in S}g(x)f_{X}(x)\:\diff x,\end{equation}
provided the (potentially improper) integral $\int_{S}|g(x)|\, f(x)\diff x$
is convergent. One important example is the mean $\mu$, also known
as $\E X$:\begin{equation}
\mu=\E X=\int_{x\in S}xf_{X}(x)\:\diff x,\end{equation}
provided $\int_{S}|x|f(x)\diff x$ is finite. Also there is the variance\begin{equation}
\sigma^{2}=\E(X-\mu)^{2}=\int_{x\in S}(x-\mu)^{2}f_{X}(x)\,\diff x,\end{equation}
which can be computed with the alternate formula $\sigma^{2}=\E X^{2}-(\E X)^{2}$.
In addition, there is the standard deviation $\sigma=\sqrt{\sigma^{2}}$.
The moment generating function is given by\begin{equation}
M_{X}(t)=\E\:\me^{tX}=\int_{-\infty}^{\infty}\me^{tx}f_{X}(x)\:\diff x,\end{equation}
provided the integral exists (is finite) for all $t$ in a neighborhood
of $t=0$.
\begin{example}
\label{exa:cont-pdf3x2}Let the continuous random variable $X$ have
PDF \[
f_{X}(x)=3x^{2},\quad0\leq x\leq1.\]
We will see later that $f_{X}$ belongs to the \emph{Beta} family
of distributions. It is easy to see that $\int_{-\infty}^{\infty}f(x)\diff x=1$.\begin{align*}
\int_{-\infty}^{\infty}f_{X}(x)\diff x & =\int_{0}^{1}3x^{2}\:\diff x\\
 & =\left.x^{3}\right|_{x=0}^{1}\\
 & =1^{3}-0^{3}\\
 & =1.\end{align*}
This being said, we may find $\P(0.14\leq X<0.71)$.\begin{align*}
\P(0.14\leq X<0.71) & =\int_{0.14}^{0.71}3x^{2}\diff x,\\
 & =\left.x^{3}\right|_{x=0.14}^{0.71}\\
 & =0.71^{3}-0.14^{3}\\
 & \approx0.355167.\end{align*}
We can find the mean and variance in an identical manner.\begin{align*}
\mu=\int_{-\infty}^{\infty}xf_{X}(x)\diff x & =\int_{0}^{1}x\cdot3x^{2}\:\diff x,\\
 & =\frac{3}{4}x^{4}|_{x=0}^{1},\\
 & =\frac{3}{4}.\end{align*}
It would perhaps be best to calculate the variance with the shortcut
formula $\sigma^{2}=\E X^{2}-\mu^{2}$:
\end{example}
\begin{align*}
\E X^{2}=\int_{-\infty}^{\infty}x^{2}f_{X}(x)\diff x & =\int_{0}^{1}x^{2}\cdot3x^{2}\:\diff x\\
 & =\left.\frac{3}{5}x^{5}\right|_{x=0}^{1}\\
 & =3/5.\end{align*}
which gives $\sigma^{2}=3/5-(3/4)^{2}=3/80$.


\begin{example}
\label{exa:cont-pdf-3x4}We will try one with unbounded support to
brush up on improper integration. Let the random variable $X$ have
PDF \[
f_{X}(x)=\frac{3}{x^{4}},\quad x>1.\]
We can show that $\int_{-\infty}^{\infty}f(x)\diff x=1$:\begin{align*}
\int_{-\infty}^{\infty}f_{X}(x)\diff x & =\int_{1}^{\infty}\frac{3}{x^{4}}\:\diff x\\
 & =\lim_{t\to\infty}\int_{1}^{t}\frac{3}{x^{4}}\:\diff x\\
 & =\lim_{t\to\infty}\ \left.3\,\frac{1}{-3}x^{-3}\right|_{x=1}^{t}\\
 & =-\left(\lim_{t\to\infty}\frac{1}{t^{3}}-1\right)\\
 & =1.\end{align*}
We calculate $\P(3.4\leq X<7.1)$:\begin{align*}
\P(3.4\leq X<7.1) & =\int_{3.4}^{7.1}3x^{-4}\diff x\\
 & =\left.3\,\frac{1}{-3}x^{-3}\right|_{x=3.4}^{7.1}\\
 & =-1(7.1^{-3}-3.4^{-3})\\
 & \approx0.0226487123.\end{align*}
We locate the mean and variance just like before.\begin{align*}
\mu=\int_{-\infty}^{\infty}xf_{X}(x)\diff x & =\int_{1}^{\infty}x\cdot\frac{3}{x^{4}}\:\diff x\\
 & =\left.3\,\frac{1}{-2}x^{-2}\right|_{x=1}^{\infty}\\
 & =-\frac{3}{2}\left(\lim_{t\to\infty}\frac{1}{t^{2}}-1\right)\\
 & =\frac{3}{2}.\end{align*}
Again we use the shortcut $\sigma^{2}=\E X^{2}-\mu^{2}$:\begin{align*}
\E X^{2}=\int_{-\infty}^{\infty}x^{2}f_{X}(x)\diff x & =\int_{1}^{\infty}x^{2}\cdot\frac{3}{x^{4}}\:\diff x\\
 & =\left.3\:\frac{1}{-1}x^{-1}\right|_{x=1}^{\infty}\\
 & =-3\left(\lim_{t\to\infty}\frac{1}{t^{2}}-1\right)\\
 & =3,\end{align*}
which closes the example with $\sigma^{2}=3-(3/2)^{2}=3/4$.
\end{example}

\subsection{How to do it with \textsf{R}}

There exist utilities to calculate probabilities and expectations
for general continuous random variables, but it is better to find
a built-in model, if possible. Sometimes it is not possible. We show
how to do it the long way, and the \inputencoding{latin9}\lstinline[basicstyle={\ttfamily},breaklines=true,showstringspaces=false]!distr!\inputencoding{utf8}\index{R packages@\textsf{R} packages!distr@\texttt{distr}}
package way.
\begin{example}
Let $X$ have PDF $f(x)=3x^{2}$, $0<x<1$ and find $\P(0.14\leq X\leq0.71)$.
(We will ignore that $X$ is a beta random variable for the sake of
argument.)

<<>>=
f <- function(x) 3*x^2
integrate(f, lower = 0.14, upper = 0.71)
@

Compare this to the answer we found in Example \ref{exa:cont-pdf3x2}.
We could integrate the function $xf(x)=$ \inputencoding{latin9}\lstinline[basicstyle={\ttfamily},breaklines=true,showstringspaces=false]!3*x^3!\inputencoding{utf8}
from zero to one to get the mean, and use the shortcut $\sigma^{2}=\E X^{2}-\left(\E X\right)^{2}$
for the variance. 

\end{example}

\begin{example}
Let $X$ have PDF $f(x)=3/x^{4}$, $x>1$. We may integrate the function
$xf(x)=$ \inputencoding{latin9}\lstinline[basicstyle={\ttfamily},breaklines=true,showstringspaces=false]!3/x^3!\inputencoding{utf8}
from zero to infinity to get the mean of $X$.

<<>>=
g <- function(x) 3/x^3
integrate(g, lower = 1, upper = Inf)
@

Compare this to the answer we got in Example \ref{exa:cont-pdf-3x4}.
Use \inputencoding{latin9}\lstinline[basicstyle={\ttfamily},breaklines=true,showstringspaces=false]!-Inf!\inputencoding{utf8}
for $-\infty$.

\end{example}

\begin{example}
Let us redo Example \ref{exa:cont-pdf3x2} with the \inputencoding{latin9}\lstinline[basicstyle={\ttfamily},breaklines=true,showstringspaces=false]!distr!\inputencoding{utf8}
package. The method is similar to that encountered in Section \ref{sub:disc-rv-how-r}
in Chapter \ref{cha:Discrete-Distributions}. We define an absolutely
continuous random variable:

<<>>=
library(distr)
f <- function(x) 3*x^2
X <- AbscontDistribution(d = f, low1 = 0, up1 = 1)
p(X)(0.71) - p(X)(0.14)
@

Compare this answers we found earlier. Now let us try expectation
with the \inputencoding{latin9}\lstinline[showstringspaces=false]!distrEx!\inputencoding{utf8}
package \cite{Ruckdescheldistr}:

<<>>=
library(distrEx)
E(X)
var(X)
3/80
@

Compare these answers to the ones we found in Example \ref{exa:cont-pdf3x2}.
Why are they different? Because the \inputencoding{latin9}\lstinline[showstringspaces=false]!distrEx!\inputencoding{utf8}
package resorts to numerical methods when it encounters a model it
does not recognize. This means that the answers we get for calculations
may not exactly match the theoretical values. Be careful.

\end{example}



\section{The Continuous Uniform Distribution\label{sec:The-Continuous-Uniform}}

A random variable $X$ with the continuous uniform distribution on
the interval $(a,b)$ has PDF\begin{equation}
f_{X}(x)=\frac{1}{b-a},\quad a<x<b.\end{equation}
The associated \textsf{R} function is $\mathsf{dunif}(\mathtt{min}=a,\,\mathtt{max}=b)$.
We write $X\sim\mathsf{unif}(\mathtt{min}=a,\,\mathtt{max}=b)$. Due
to the particularly simple form of this PDF we can also write down
explicitly a formula for the CDF $F_{X}$:\begin{equation}
F_{X}(t)=\begin{cases}
0, & t<0,\\
\frac{t-a}{b-a}, & a\leq t<b,\\
1, & t\geq b.\end{cases}\label{eq:unif-cdf}\end{equation}
The continuous uniform distribution is the continuous analogue of
the discrete uniform distribution; it is used to model experiments
whose outcome is an interval of numbers that are {}``equally likely''
in the sense that any two intervals of equal length in the support
have the same probability associated with them.
\begin{example}
Choose a number in {[}0,1{]} at random, and let $X$ be the number
chosen. Then $X\sim\mathsf{unif}(\mathtt{min}=0,\,\mathtt{max}=1)$. 
\end{example}
The mean of $X\sim\mathsf{unif}(\mathtt{min}=a,\,\mathtt{max}=b)$
is relatively simple to calculate:

\begin{align*}
\mu=\E X & =\int_{-\infty}^{\infty}x\, f_{X}(x)\,\diff x,\\
 & =\int_{a}^{b}x\ \frac{1}{b-a}\ \diff x,\\
 & =\left.\frac{1}{b-a}\ \frac{x^{2}}{2}\ \right|_{x=a}^{b},\\
 & =\frac{1}{b-a}\ \frac{b^{2}-a^{2}}{2},\\
 & =\frac{b+a}{2},\end{align*}
using the popular formula for the difference of squares. The variance
is left to Exercise \ref{xca:variance-dunif}.


\section{The Normal Distribution\label{sec:The-Normal-Distribution}}

We say that $X$ has a \emph{normal distribution} if it has PDF\begin{equation}
f_{X}(x)=\frac{1}{\sigma\sqrt{2\pi}}\exp\left\{ \frac{-(x-\mu)^{2}}{2\sigma^{2}}\right\} ,\quad-\infty<x<\infty.\end{equation}
We write $X\sim\mathsf{norm}(\mathtt{mean}=\mu,\,\mathtt{sd}=\sigma)$,
and the associated \textsf{R} function is \inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false]!dnorm(x, mean = 0, sd = 1)!\inputencoding{utf8}.

The familiar bell-shaped curve, the normal distribution is also known
as the \emph{Gaussian distribution} because the German mathematician
C.~F.~Gauss largely contributed to its mathematical development.
This distribution is by far the most important distribution, continuous
or discrete. The normal model appears in the theory of all sorts of
natural phenomena, from to the way particles of smoke dissipate in
a closed room, to the journey of a bottle in the ocean to the white
noise of cosmic background radiation.

When $\mu=0$ and $\sigma=1$ we say that the random variable has
a \emph{standard normal} distribution and we typically write $Z\sim\mathsf{norm}(\mathtt{mean}=0,\,\mathtt{sd}=1)$.
The lowercase Greek letter phi ($\phi$) is used to denote the standard
normal PDF and the capital Greek letter phi $\Phi$ is used to denote
the standard normal CDF: for $-\infty<z<\infty$,\begin{equation}
\phi(z)=\frac{1}{\sqrt{2\pi}}\,\me^{-z^{2}/2}\mbox{ and }\Phi(t)=\int_{-\infty}^{t}\phi(z)\,\diff z.\end{equation}

\begin{prop}
If $X\sim\mathsf{norm}(\mathtt{mean}=\mu,\,\mathtt{sd}=\sigma)$ then
\begin{equation}
Z=\frac{X-\mu}{\sigma}\sim\mathsf{norm}(\mathtt{mean}=0,\,\mathtt{sd}=1).\end{equation}

\end{prop}


The MGF of $Z\sim\mathsf{norm}(\mathtt{mean}=0,\,\mathtt{sd}=1)$
is relatively easy to derive:\begin{eqnarray*}
M_{Z}(t) & = & \int_{-\infty}^{\infty}\me^{tz}\frac{1}{\sqrt{2\pi}}\me^{-z^{2}/2}\diff z,\\
 & = & \int_{-\infty}^{\infty}\frac{1}{\sqrt{2\pi}}\exp\left\{ -\frac{1}{2}\left(z^{2}+2tz+t^{2}\right)+\frac{t^{2}}{2}\right\} \diff z,\\
 & = & \me^{t^{2}/2}\left(\int_{-\infty}^{\infty}\frac{1}{\sqrt{2\pi}}\me^{-[z-(-t)]^{2}/2}\diff z\right),\end{eqnarray*}
and the quantity in the parentheses is the total area under a $\mathsf{norm}(\mathtt{mean}=-t,\,\mathtt{sd}=1)$
density, which is one. Therefore,\begin{equation}
M_{Z}(t)=\me^{-t^{2}/2},\quad-\infty<t<\infty.\end{equation}



\begin{example}
The MGF of $X\sim\mathsf{norm}(\mathtt{mean}=\mu,\,\mathtt{sd}=\sigma)$
is then not difficult either because \[
Z=\frac{X-\mu}{\sigma},\mbox{ or rewriting, }X=\sigma Z+\mu.\]
Therefore: \[
M_{X}(t)=\E\me^{tX}=\E\me^{t(\sigma Z+\mu)}=\E\me^{\sigma tX}\me^{\mu}=\me^{t\mu}M_{Z}(\sigma t),\]
and we know that $M_{Z}(t)=\me^{t^{2}/2}$, thus substituting we get\[
M_{X}(t)=\me^{t\mu}\me^{(\sigma t)^{2}/2}=\exp\left\{ \mu t+\sigma^{2}t^{2}/2\right\} ,\]
for $-\infty<t<\infty$.
\end{example}

\begin{fact}
The same argument above shows that if $X$ has MGF $M_{X}(t)$ then
the MGF of $Y=a+bX$ is \begin{equation}
M_{Y}(t)=\me^{ta}M_{X}(bt).\end{equation}

\end{fact}

\begin{example}
The 68-95-99.7 Rule. We saw in Section \ref{sub:Measures-of-Spread}
that when an empirical distribution is approximately bell shaped there
are specific proportions of the observations which fall at varying
distances from the (sample) mean. We can see where these come from
-- and obtain more precise proportions -- with the following:
\end{example}
<<>>=
pnorm(1:3)-pnorm(-(1:3))
@


\begin{example}
\label{exa:iq-model}Let the random experiment consist of a person
taking an IQ test, and let $X$ be the score on the test. The scores
on such a test are typically standardized to have a mean of 100 and
a standard deviation of 15. What is $\P(85\leq X\leq115)$?

Solution: this one is easy because the limits 85 and 115 fall exactly
one standard deviation (below and above, respectively) from the mean
of 100. The answer is therefore approximately 68\%.
\end{example}

\subsection{Normal Quantiles and the Quantile Function\label{sub:Normal-Quantiles-QF}}

Until now we have been given two values and our task has been to find
the area under the PDF between those values. In this section, we go
in reverse: we are given an area, and we would like to find the value(s)
that correspond to that area. 
\begin{example}
\label{exa:iq-quantile-state-problem}Assuming the IQ model of Example
\ref{exa:iq-model}, what is the lowest possible IQ score that a person
can have and still be in the top 1\% of all IQ scores?

Solution: If a person is in the top 1\%, then that means that 99\%
of the people have lower IQ scores. So, in other words, we are looking
for a value $x$ such that $F(x)=\P(X\leq x)$ satisfies $F(x)=0.99$,
or yet another way to say it is that we would like to solve the equation
$F(x)-0.99=0$. For the sake of argument, let us see how to do this
the long way. We define the function $g(x)=F(x)-0.99$, and then look
for the root of $g$ with the \inputencoding{latin9}\lstinline[showstringspaces=false]!uniroot!\inputencoding{utf8}
function. It uses numerical procedures to find the root so we need
to give it an interval of $x$ values in which to search for the root.
We can get an educated guess from the Empirical Rule \ref{fac:Empirical-Rule};
the root should be somewhere between two and three standard deviations
(15 each) above the mean (which is 100).
\end{example}
<<>>=
g <- function(x) pnorm(x, mean = 100, sd = 15) - 0.99
uniroot(g, interval = c(130, 145))
@
<<echo = FALSE, results = hide>>=
temp <- round(uniroot(g, interval = c(130, 145))$root, 4)
@

The answer is shown in \inputencoding{latin9}\lstinline[showstringspaces=false]!$root!\inputencoding{utf8}
which is approximately \Sexpr{temp}, that is, a person with this
IQ score or higher falls in the top 1\% of all IQ scores.



The discussion in example \ref{exa:iq-quantile-state-problem} was
centered on the search for a value $x$ that solved an equation $F(x)=p$,
for some given probability $p$, or in mathematical parlance, the
search for $F^{-1}$, the inverse of the CDF of $X$, evaluated at
$p$. This is so important that it merits a definition all its own.
\begin{defn}
The \emph{quantile function}%
\footnote{The precise definition of the quantile function is $Q_{X}(p)=\inf\left\{ x:\ F_{X}(x)\geq p\right\} $,
so at least it is well defined (though perhaps infinite) for the values
$p=0$ and $p=1$.%
} of a random variable $X$ is the inverse of its cumulative distribution
function:\begin{equation}
Q_{X}(p)=\min\left\{ x:\ F_{X}(x)\geq p\right\} ,\quad0<p<1.\end{equation}

\end{defn}

\begin{rem}
Here are some properties of quantile functions:
\begin{enumerate}
\item The quantile function is defined and finite for all $0<p<1$.
\item $Q_{X}$ is left-continuous (see Appendix \ref{sec:Differential-and-Integral}).
For discrete random variables it is a step function, and for continuous
random variables it is a continuous function.
\item In the continuous case the graph of $Q_{X}$ may be obtained by reflecting
the graph of $F_{X}$ about the line $y=x$. In the discrete case,
before reflecting one should: 1) connect the dots to get rid of the
jumps -- this will make the graph look like a set of stairs, 2) erase
the horizontal lines so that only vertical lines remain, and finally
3) swap the open circles with the solid dots. Please see Figure \ref{fig:binom-plot-distr}
for a comparison. 
\item The two limits \[
\lim_{p\to0^{+}}Q_{X}(p)\quad\mbox{and}\quad\lim_{p\to1^{-}}Q_{X}(p)\]
always exist, but may be infinite (that is, sometimes $\lim_{p\to0}Q(p)=-\infty$
and/or $\lim_{p\to1}Q(p)=\infty$).
\end{enumerate}
\end{rem}
As the reader might expect, the standard normal distribution is a
very special case and has its own special notation.
\begin{defn}
For $0<\alpha<1$, the symbol $z_{\alpha}$ denotes the unique solution
of the equation $\P(Z>z_{\alpha})=\alpha$, where $Z\sim\mathsf{norm}(\mathtt{mean}=0,\,\mathtt{sd}=1)$.
It can be calculated in one of two equivalent ways: \inputencoding{latin9}\lstinline[showstringspaces=false]!qnorm(!\inputencoding{utf8}$1-\alpha$\inputencoding{latin9}\lstinline[showstringspaces=false]!)!\inputencoding{utf8}
and \inputencoding{latin9}\lstinline[showstringspaces=false]!qnorm(!\inputencoding{utf8}$\alpha$\inputencoding{latin9}\lstinline[basicstyle={\ttfamily},showstringspaces=false]!, lower.tail = FALSE)!\inputencoding{utf8}.
\end{defn}
There are a few other very important special cases which we will encounter
in later chapters. 


\subsection{How to do it with \textsf{R}}

Quantile functions are defined for all of the base distributions with
the \inputencoding{latin9}\lstinline[showstringspaces=false]!q!\inputencoding{utf8}
prefix to the distribution name, except for the ECDF whose quantile
function is exactly the $Q_{x}(p)=$\inputencoding{latin9}\lstinline[showstringspaces=false]!quantile(x, probs = !\inputencoding{utf8}$p$
\inputencoding{latin9}\lstinline[showstringspaces=false]!, type = 1)!\inputencoding{utf8}
function. 
\begin{example}
Back to Example \ref{exa:iq-quantile-state-problem}, we are looking
for $Q_{X}(0.99)$, where $X\sim\mathsf{norm}(\mathtt{mean}=100,\,\mathtt{sd}=15)$.
It could not be easier to do with \textsf{R}.

<<>>=
qnorm(0.99, mean = 100, sd = 15)
@

Compare this answer to the one obtained earlier with \inputencoding{latin9}\lstinline[showstringspaces=false]!uniroot!\inputencoding{utf8}.
\end{example}

\begin{example}
Find the values $z_{0.025}$, $z_{0.01}$, and $z_{0.005}$ (these
will play an important role from Chapter \ref{cha:Estimation} onward).
\end{example}
<<>>=
qnorm(c(0.025, 0.01, 0.005), lower.tail = FALSE)
@

Note the \inputencoding{latin9}\lstinline[showstringspaces=false]!lower.tail!\inputencoding{utf8}
argument. We would get the same answer with \inputencoding{latin9}
\begin{lstlisting}[showstringspaces=false]
qnorm(c(0.975, 0.99, 0.995))
\end{lstlisting}
\inputencoding{utf8}


\section{Functions of Continuous Random Variables\label{sec:Functions-of-Continuous}}

The goal of this section is to determine the distribution of $U=g(X)$
based on the distribution of $X$. In the discrete case all we needed
to do was back substitute for $x=g^{-1}(u)$ in the PMF of $X$ (sometimes
accumulating probability mass along the way). In the continuous case,
however, we need more sophisticated tools. Now would be a good time
to review Appendix \ref{sec:Differential-and-Integral}.


\subsection{The PDF Method}
\begin{prop}
\label{pro:func-cont-rvs-pdf-formula}Let $X$ have PDF $f_{X}$ and
let $g$ be a function which is one-to-one with a differentiable inverse
$g^{-1}$. Then the PDF of $U=g(X)$ is given by\begin{equation}
f_{U}(u)=f_{X}\left[g^{-1}(u)\right]\ \left|\frac{\diff}{\diff u}g^{-1}(u)\right|.\label{eq:univ-trans-pdf-long}\end{equation}
\end{prop}
\begin{rem}
The formula in Equation \ref{eq:univ-trans-pdf-long} is nice, but
does not really make any sense. It is better to write in the intuitive
form\begin{equation}
f_{U}(u)=f_{X}(x)\left|\frac{\diff x}{\diff u}\right|.\label{eq:univ-trans-pdf-short}\end{equation}
\end{rem}
\begin{example}
\label{exa:lnorm-transformation}Let $X\sim\mathsf{norm}(\mathtt{mean}=\mu,\,\mathtt{sd}=\sigma)$,
and let $Y=\me^{X}$. What is the PDF of $Y$? 

Solution: Notice first that $\me^{x}>0$ for any $x$, so the support
of $Y$ is $(0,\infty)$. Since the transformation is monotone, we
can solve $y=\me^{x}$ for $x$ to get $x=\ln\, y$, giving $\diff x/\diff y=1/y$.
Therefore, for any $y>0$,\[
f_{Y}(y)=f_{X}(\ln y)\cdot\left|\frac{1}{y}\right|=\frac{1}{\sigma\sqrt{2\pi}}\exp\left\{ \frac{(\ln y-\mu)^{2}}{2\sigma^{2}}\right\} \cdot\frac{1}{y},\]
where we have dropped the absolute value bars since $y>0$. The random
variable $Y$ is said to have a \emph{lognormal distribution}; see
Section \ref{sec:Other-Continuous-Distributions}.
\end{example}

\begin{example}
\label{exa:lin-trans-norm}Suppose $X\sim\mathsf{norm}(\mathtt{mean}=0,\,\mathtt{sd}=1)$
and let $Y=4-3X$. What is the PDF of $Y$?
\end{example}
The support of $X$ is $(-\infty,\infty)$, and as $x$ goes from
$-\infty$ to $\infty$, the quantity $y=4-3x$ also traverses $(-\infty,\infty)$.
Solving for $x$ in the equation $y=4-3x$ yields $x=-(y-4)/3$ giving
$\diff x/\diff y=-1/3$. And since\[
f_{X}(x)=\frac{1}{\sqrt{2\pi}}\me^{-x^{2}/2},\quad-\infty<x<\infty,\]
 we have \begin{eqnarray*}
f_{Y}(y) & = & f_{X}\left(\frac{y-4}{3}\right)\cdot\left|-\frac{1}{3}\right|,\quad-\infty<y<\infty,\\
 & = & \frac{1}{3\sqrt{2\pi}}\me^{-(y-4)^{2}/2\cdot3^{2}},\quad-\infty<y<\infty.\end{eqnarray*}
We recognize the PDF of $Y$ to be that of a $\mathsf{norm}(\mathtt{mean}=4,\,\mathtt{sd}=3)$
distribution. Indeed, we may use an identical argument as the above
to prove the following fact:
\begin{fact}
\label{fac:lin-trans-norm-is-norm}If $X\sim\mathsf{norm}(\mathtt{mean}=\mu,\,\mathtt{sd}=\sigma)$
and if $Y=a+bX$ for constants $a$ and $b$, with $b\neq0$, then
$Y\sim\mathsf{norm}(\mathtt{mean}=a+b\mu,\,\mathtt{sd}=|b|\sigma)$. 
\end{fact}
Note that it is sometimes easier to \emph{postpone} solving for the
inverse transformation $x=x(u)$. Instead, leave the transformation
in the form $u=u(x)$ and calculate the derivative of the \emph{original}
transformation\begin{equation}
\diff u/\diff x=g'(x).\end{equation}
Once this is known, we can get the PDF of $U$ with\begin{equation}
f_{U}(u)=f_{X}(x)\left|\frac{1}{\diff u/\diff x}\right|.\end{equation}
In many cases there are cancellations and the work is shorter. Of
course, it is not always true that\begin{equation}
\frac{\diff x}{\diff u}=\frac{1}{\diff u/\diff x},\label{eq:univ-jacob-recip}\end{equation}
but for the well-behaved examples in this book the trick works just
fine.
\begin{rem}
In the case that $g$ is not monotone we cannot apply Proposition
\ref{pro:func-cont-rvs-pdf-formula} directly. However, hope is not
lost. Rather, we break the support of $X$ into pieces such that $g$
is monotone on each one. We apply Proposition \ref{pro:func-cont-rvs-pdf-formula}
on each piece, and finish up by adding the results together.
\end{rem}

\subsection{The CDF method}

We know from Section \ref{sec:continuous-random-variables} that $f_{X}=F_{X}'$
in the continuous case. Starting from the equation $F_{Y}(y)=\P(Y\leq y)$,
we may substitute $g(X)$ for $Y$, then solve for $X$ to obtain
$\P[X\leq g^{-1}(y)]$, which is just another way to write $F_{X}[g^{-1}(y)]$.
Differentiating this last quantity with respect to $y$ will yield
the PDF of $Y$.
\begin{example}
Suppose $X\sim\mathsf{unif}(\mathtt{min}=0,\,\mathtt{max}=1)$ and
suppose that we let $Y=-\ln\, X$. What is the PDF of $Y$?

The support set of $X$ is $(0,1),$ and $y$ traverses $(0,\infty)$
as $x$ ranges from $0$ to $1$, so the support set of $Y$ is $S_{Y}=(0,\infty)$.
For any $y>0$, we consider\[
F_{Y}(y)=\P(Y\leq y)=\P(-\ln\, X\leq y)=\P(X\geq\me^{-y})=1-\P(X<\me^{-y}),\]
where the next to last equality follows because the exponential function
is \emph{monotone} (this point will be revisited later). Now since
$X$ is continuous the two probabilities $\P(X<\me^{-y})$ and $\P(X\leq\me^{-y})$
are equal; thus\[
1-\P(X<\me^{-y})=1-\P(X\leq\me^{-y})=1-F_{X}(\me^{-y}).\]
Now recalling that the CDF of a $\mathsf{unif}(\mathtt{min}=0,\,\mathtt{max}=1)$
random variable satisfies $F(u)=u$ (see Equation \ref{eq:unif-cdf}),
we can say \[
F_{Y}(y)=1-F_{X}(\me^{-y})=1-\me^{-y},\quad\mbox{for }y>0.\]
We have consequently found the formula for the CDF of $Y$; to obtain
the PDF $f_{Y}$ we need only differentiate $F_{Y}$:\[
f_{Y}(y)=\frac{\diff}{\diff y}\left(1-\me^{-y}\right)=0-\me^{-y}(-1),\]
or $f_{Y}(y)=\me^{-y}$ for $y>0$. This turns out to be a member
of the exponential family of distributions, see Section \ref{sec:Other-Continuous-Distributions}. 
\end{example}

\begin{example}
\textbf{\emph{The Probability Integral Transform}}. Given a continuous
random variable $X$ with strictly increasing CDF $F_{X}$, let the
random variable $Y$ be defined by $Y=F_{X}(X)$. Then the distribution
of $Y$ is $\mathsf{unif}(\mathtt{min}=0,\,\mathtt{max}=1)$. \end{example}
\begin{proof}
We employ the CDF method. First note that the support of $Y$ is $(0,1)$.
Then for any $0<y<1$,\[
F_{Y}(y)=\P(Y\leq y)=\P(F_{X}(X)\leq y).\]
Now since $F_{X}$ is strictly increasing, it has a well defined inverse
function $F_{X}^{-1}$. Therefore,\[
\P(F_{X}(X)\leq y)=\P(X\leq F_{X}^{-1}(y))=F_{X}[F_{X}^{-1}(y)]=y.\]
Summarizing, we have seen that $F_{Y}(y)=y$, $0<y<1$. But this is
exactly the CDF of a $\mathsf{unif}(\mathtt{min}=0,\,\mathtt{max}=1)$
random variable. \end{proof}
\begin{fact}
The Probability Integral Transform is true for all continuous random
variables with continuous CDFs, not just for those with strictly increasing
CDFs (but the proof is more complicated). The transform is \textbf{not}
true for discrete random variables, or for continuous random variables
having a discrete component (that is, with jumps in their CDF). \end{fact}
\begin{example}
\label{exa:distn-of-z-squared}Let $Z\sim\mathsf{norm}(\mathtt{mean}=0,\,\mathtt{sd}=1)$
and let $U=Z^{2}$. What is the PDF of $U$? 

Notice first that $Z^{2}\geq0$, and thus the support of $U$ is $[0,\infty)$.
And for any $u\geq0$, \[
F_{U}(u)=\P(U\leq u)=\P(Z^{2}\leq u).\]
But $Z^{2}\leq u$ occurs if and only if $-\sqrt{u}\leq Z\leq\sqrt{u}$.
The last probability above is simply the area under the standard normal
PDF from $-\sqrt{u}$ to $\sqrt{u}$, and since $\phi$ is symmetric
about 0, we have\[
\P(Z^{2}\leq u)=2\P(0\leq Z\leq\sqrt{u})=2\left[F_{Z}(\sqrt{u})-F_{Z}(0)\right]=2\Phi(\sqrt{u})-1,\]
because $\Phi(0)=1/2$. To find the PDF of $U$ we differentiate the
CDF recalling that $\Phi'=\phi$.\[
f_{U}(u)=\left(2\Phi(\sqrt{u})-1\right)'=2\phi(\sqrt{u})\cdot\frac{1}{2\sqrt{u}}=u^{-1/2}\phi(\sqrt{u}).\]
Substituting,\[
f_{U}(u)=u^{-1/2}\frac{1}{\sqrt{2\pi}}\,\me^{-(\sqrt{u})^{2}/2}=(2\pi u)^{-1/2}\me^{-u},\quad u>0.\]
This is what we will later call a \emph{chi-square distribution with
1 degree of freedom}. See Section \ref{sec:Other-Continuous-Distributions}. 
\end{example}

\subsection{How to do it with \textsf{R}}

The \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!distr!\inputencoding{utf8}
package has functionality to investigate transformations of univariate
distributions. There are exact results for ordinary transformations
of the standard distributions, and \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!distr!\inputencoding{utf8}
takes advantage of these in many cases. For instance, the \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!distr!\inputencoding{utf8}
package can handle the transformation in Example \ref{exa:lin-trans-norm}
quite nicely:

<<>>=
library(distr)
X <- Norm(mean = 0, sd = 1)
Y <- 4 - 3*X
Y
@

So \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!distr!\inputencoding{utf8}
{}``knows'' that a linear transformation of a normal random variable
is again normal, and it even knows what the correct \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!mean!\inputencoding{utf8}
and \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!sd!\inputencoding{utf8}
should be. But it is impossible for \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!distr!\inputencoding{utf8}
to know everything, and it is not long before we venture outside of
the transformations that \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!distr!\inputencoding{utf8}
recognizes. Let us try Example \ref{exa:lnorm-transformation}:

<<>>=
Y <- exp(X)
Y
@

The result is an object of class \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!AbscontDistribution!\inputencoding{utf8},
which is one of the classes that \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!distr!\inputencoding{utf8}
uses to denote general distributions that it does not recognize (it
turns out that $Z$ has a \emph{lognormal} distribution; see Section
\ref{sec:Other-Continuous-Distributions}). A simplified description
of the process that \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!distr!\inputencoding{utf8}
undergoes when it encounters a transformation $Y=g(X)$ that it does
not recognize is
\begin{enumerate}
\item Randomly generate many, many copies $X_{1}$, $X_{2}$, \ldots{},
$X_{n}$ from the distribution of $X$,
\item Compute $Y_{1}=g(X_{1})$, $Y_{2}=g(X_{2})$, \ldots{}, $Y_{n}=g(X_{n})$
and store them for use.
\item Calculate the PDF, CDF, quantiles, and random variates using the simulated
values of $Y$.
\end{enumerate}
As long as the transformation is sufficiently nice, such as a linear
transformation, the exponential, absolute value, \emph{etc.}, the
d-p-q functions are calculated analytically based on the d-p-q functions
associated with $X$. But if we try a crazy transformation then we
are greeted by a warning:

<<>>=
W <- sin(exp(X) + 27)
W
@

The warning confirms that the d-p-q functions are not calculated analytically,
but are instead based on the randomly simulated values of $Y$. \emph{We
must be careful to remember this.} The nature of random simulation
means that we can get different answers to the same question: watch
what happens when we compute $\P(W\leq0.5)$ using the $W$ above,
then define $W$ again, and compute the (supposedly) same $\P(W\leq0.5)$
a few moments later.

<<>>=
p(W)(0.5)
W <- sin(exp(X) + 27)
p(W)(0.5)
@

The answers are not the same! Furthermore, if we were to repeat the
process we would get yet another answer for $\P(W\leq0.5)$. 

The answers were close, though. And the underlying randomly generated
$X$'s were not the same so it should hardly be a surprise that the
calculated $W$'s were not the same, either. This serves as a warning
(in concert with the one that \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!distr!\inputencoding{utf8}
provides) that we should be careful to remember that complicated transformations
computed by \textsf{R} are only approximate and may fluctuate slightly
due to the nature of the way the estimates are calculated.


\section{Other Continuous Distributions\label{sec:Other-Continuous-Distributions}}


\subsection{Waiting Time Distributions\label{sub:Waiting-Time-Distributions}}

In some experiments, the random variable being measured is the time
until a certain event occurs. For example, a quality control specialist
may be testing a manufactured product to see how long it takes until
it fails. An efficiency expert may be recording the customer traffic
at a retail store to streamline scheduling of staff. 


\subsection*{The Exponential Distribution\label{sub:The-Exponential-Distribution}}

We say that $X$ has an \emph{exponential distribution} and write
$X\sim\mathsf{exp}(\mathtt{rate}=\lambda)$. 

\begin{equation}
f_{X}(x)=\lambda\me^{-\lambda x},\quad x>0\end{equation}
The associated \textsf{R} functions are \inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false,tabsize=4]!dexp(x, rate = 1)!\inputencoding{utf8},
\inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false,tabsize=4]!pexp!\inputencoding{utf8},
\inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false,tabsize=4]!qexp!\inputencoding{utf8},
and \inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false,tabsize=4]!rexp!\inputencoding{utf8},
which give the PDF, CDF, quantile function, and simulate random variates,
respectively.

The parameter $\lambda$ measures the rate of arrivals (to be described
later) and must be positive. The CDF is given by the formula\begin{equation}
F_{X}(t)=1-\me^{-\lambda t},\quad t>0.\end{equation}
The mean is $\mu=1/\lambda$ and the variance is $\sigma^{2}=1/\lambda^{2}$. 

The exponential distribution is closely related to the Poisson distribution.
If customers arrive at a store according to a Poisson process with
rate $\lambda$ and if $Y$ counts the number of customers that arrive
in the time interval $[0,t)$, then we saw in Section \ref{sec:other-discrete-distributions}
that $Y\sim\mathsf{pois}(\mathtt{lambda}=\lambda t).$ Now consider
a different question: let us start our clock at time 0 and stop the
clock when the first customer arrives. Let $X$ be the length of this
random time interval. Then $X\sim\mathsf{exp}(\mathtt{rate}=\lambda)$.
Observe the following string of equalities:\begin{align*}
\P(X>t) & =\P(\mbox{first arrival after time \emph{t}}),\\
 & =\P(\mbox{no events in [0,\emph{t})}),\\
 & =\P(Y=0),\\
 & =\me^{-\lambda t},\end{align*}
where the last line is the PMF of $Y$ evaluated at $y=0$. In other
words, $\P(X\leq t)=1-\me^{-\lambda t}$, which is exactly the CDF
of an $\mathsf{exp}(\mathtt{rate}=\lambda)$ distribution.

The exponential distribution is said to be \emph{memoryless} because
exponential random variables \textquotedbl{}forget\textquotedbl{}
how old they are at every instant. That is, the probability that we
must wait an additional five hours for a customer to arrive, given
that we have already waited seven hours, is exactly the probability
that we needed to wait five hours for a customer in the first place.
In mathematical symbols, for any $s,\, t>0$,\begin{equation}
\P(X>s+t\,|\, X>t)=\P(X>s).\end{equation}
See Exercise \ref{xca:prove-the-memoryless}.


\subsection*{The Gamma Distribution\label{sub:The-Gamma-Distribution}}

This is a generalization of the exponential distribution. We say that
$X$ has a gamma distribution and write $X\sim\mathsf{gamma}(\mathtt{shape}=\alpha,\,\mathtt{rate}=\lambda)$.
It has PDF\begin{equation}
f_{X}(x)=\frac{\lambda^{\alpha}}{\Gamma(\alpha)}\: x^{\alpha-1}\me^{-\lambda x},\quad x>0.\end{equation}
The associated \textsf{R} functions are \inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false,tabsize=4]!dgamma(x, shape, rate = 1)!\inputencoding{utf8},
\inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false,tabsize=4]!pgamma!\inputencoding{utf8},
\inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false,tabsize=4]!qgamma!\inputencoding{utf8},
and \inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false,tabsize=4]!rgamma!\inputencoding{utf8},
which give the PDF, CDF, quantile function, and simulate random variates,
respectively. If $\alpha=1$ then $X\sim\mathsf{exp}(\mathtt{rate}=\lambda)$.
The mean is $\mu=\alpha/\lambda$ and the variance is $\sigma^{2}=\alpha/\lambda^{2}$.

To motivate the gamma distribution recall that if $X$ measures the
length of time until the first event occurs in a Poisson process with
rate $\lambda$ then $X\sim\mathsf{exp}(\mathtt{rate}=\lambda)$.
If we let $Y$ measure the length of time until the $\alpha^{\text{th}}$
event occurs then $Y\sim\mathsf{gamma}(\mathtt{shape}=\alpha,\,\mathtt{rate}=\lambda)$.
When $\alpha$ is an integer this distribution is also known as the
\emph{Erlang} distribution.
\begin{example}
At a car wash, two customers arrive per hour on the average. We decide
to measure how long it takes until the third customer arrives. If
$Y$ denotes this random time then $Y\sim\mathsf{gamma}(\mathtt{shape}=3,\,\mathtt{rate}=1/2)$.
\end{example}

\subsection{The Chi square, Student's $t$, and Snedecor's $F$ Distributions\label{sub:The-Chi-Square-t-F}}


\subsection*{The Chi square Distribution\label{sub:The-Chi-Square}}

A random variable $X$ with PDF\begin{equation}
f_{X}(x)=\frac{1}{\Gamma(p/2)2^{p/2}}x^{p/2-1}\me^{-x/2},\quad x>0,\end{equation}
is said to have a \emph{chi-square distribution} with $p$ \emph{degrees
of freedom}. We write $X\sim\mathsf{chisq}(\mathtt{df}=p)$. The associated
\textsf{R} functions are \inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false,tabsize=4]!dchisq(x, df)!\inputencoding{utf8},
\inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false,tabsize=4]!pchisq!\inputencoding{utf8},
\inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false,tabsize=4]!qchisq!\inputencoding{utf8},
and \inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false,tabsize=4]!rchisq!\inputencoding{utf8},
which give the PDF, CDF, quantile function, and simulate random variates,
respectively. See Figure \ref{fig:chisq-dist-vary-df}. In an obvious
notation we may define $\chi_{\alpha}^{2}(p)$ as the number on the
$x$-axis such that there is exactly $\alpha$ area under the $\mathsf{chisq}(\mathtt{df}=p)$
curve to its right.

The code to produce Figure \ref{fig:chisq-dist-vary-df} is

<<eval = FALSE>>=
curve(dchisq(x, df = 3), from = 0, to = 20, ylab = "y")
ind <- c(4, 5, 10, 15)
for (i in ind) curve(dchisq(x, df = i), 0, 20, add = TRUE)
@

%
\begin{figure}
\begin{centering}
<<echo = FALSE, fig=true, height = 4.5, width = 6>>=
curve(dchisq(x, df = 3), from = 0, to = 20, ylab = "y")
ind <- c(4, 5, 10, 15)
for (i in ind) curve(dchisq(x, df = i), 0, 20, add = TRUE)
@
\par\end{centering}

\caption{Chi square distribution for various degrees of freedom\label{fig:chisq-dist-vary-df}}

\end{figure}

\begin{rem}
Here are some useful things to know about the chi-square distribution.
\begin{enumerate}
\item If $Z\sim\mathtt{norm}(\mathtt{mean}=0,\,\mathtt{sd}=1)$, then $Z^{2}\sim\mathsf{chisq}(\mathtt{df}=1)$.
We saw this in Example \ref{exa:distn-of-z-squared}, and the fact
is important when it comes time to find the distribution of the sample
variance, $S^{2}$. See Theorem \ref{thm:Xbar-andS} in Section \ref{sub:Samp-Var-Dist}.
\item The chi-square distribution is supported on the positive $x$-axis,
with a right-skewed distribution.
\item The $\mathsf{chisq}(\mathtt{df}=p)$ distribution is the same as a
$\mathsf{gamma}(\mathtt{shape}=p/2,\,\mathtt{rate}=1/2)$ distribution.
\item The MGF of $X\sim\mathsf{chisq}(\mathtt{df}=p)$ is\begin{equation}
M_{X}(t)=\left(1-2t\right)^{-p},\quad t<1/2.\label{eq:mgf-chisq}\end{equation}

\end{enumerate}
\end{rem}

\subsection*{Student's $t$ distribution\label{sub:Student's-t-distribution}}

A random variable $X$ with PDF

\begin{equation}
f_{X}(x)=\frac{\Gamma\left[(r+1)/2\right]}{\sqrt{r\pi}\,\Gamma(r/2)}\left(1+\frac{x^{2}}{r}\right)^{-(r+1)/2},\quad-\infty<x<\infty\end{equation}
is said to have \emph{Student's} $t$ distribution with $r$ \emph{degrees
of freedom}, and we write $X\sim\mathsf{t}(\mathtt{df}=r)$. The associated
\textsf{R} functions are \inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false,tabsize=4]!dt!\inputencoding{utf8},
\inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false,tabsize=4]!pt!\inputencoding{utf8},
\inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false,tabsize=4]!qt!\inputencoding{utf8},
and \inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false,tabsize=4]!rt!\inputencoding{utf8},
which give the PDF, CDF, quantile function, and simulate random variates,
respectively. See Section \ref{sec:sampling-from-normal-dist}.


\subsection*{Snedecor's $F$ distribution\label{sub:snedecor-F-distribution}}

A random variable $X$ with p.d.f.

\begin{equation}
f_{X}(x)=\frac{\Gamma[(m+n)/2]}{\Gamma(m/2)\Gamma(n/2)}\left(\frac{m}{n}\right)^{m/2}x^{m/2-1}\left(1+\frac{m}{n}x\right)^{-(m+n)/2},\quad x>0.\end{equation}
is said to have an $F$ distribution with $(m,n)$ degrees of freedom.
We write $X\sim\mathsf{f}(\mathtt{df1}=m,\,\mathtt{df2}=n)$. The
associated \textsf{R} functions are \inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false,tabsize=4]!df(x, df1, df2)!\inputencoding{utf8},
\inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false,tabsize=4]!pf!\inputencoding{utf8},
\inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false,tabsize=4]!qf!\inputencoding{utf8},
and \inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false,tabsize=4]!rf!\inputencoding{utf8},
which give the PDF, CDF, quantile function, and simulate random variates,
respectively. We define $F_{\alpha}(m,n)$ as the number on the $x$-axis
such that there is exactly $\alpha$ area under the $\mathsf{f}(\mathtt{df1}=m,\,\mathtt{df2}=n)$
curve to its right. 
\begin{rem}
Here are some notes about the $F$ distribution.
\begin{enumerate}
\item If $X\sim\mathsf{f}(\mathtt{df1}=m,\,\mathtt{df2}=n)$ and $Y=1/X$,
then $Y\sim\mathsf{f}(\mathtt{df1}=n,\,\mathtt{df2}=m)$. Historically,
this fact was especially convenient. In the old days, statisticians
used printed tables for their statistical calculations. Since the
$F$ tables were symmetric in $m$ and $n$, it meant that publishers
could cut the size of their printed tables in half. It plays less
of a role today now that personal computers are widespread.
\item If $X\sim\mathsf{t}(\mathtt{df}=r)$, then $X^{2}\sim\mathsf{f}(\mathtt{df1}=1,\,\mathtt{df2}=r)$.
We will see this again in Section \ref{sub:slr-overall-F-statistic}.
\end{enumerate}
\end{rem}

\subsection{Other Popular Distributions\label{sub:Other-Popular-Distributions}}


\subsubsection*{The Cauchy Distribution\label{sub:The-Cauchy-Distribution}}

This is a special case of the Student's $t$ distribution. It has
PDF\begin{equation}
f_{X}(x)=\frac{1}{\beta\pi}\left[1+\left(\frac{x-m}{\beta}\right)^{2}\right]^{-1},\quad-\infty<x<\infty\end{equation}


We write $X\sim\mathsf{cauchy}(\mathtt{location}=m,\,\mathtt{scale}=\beta)$.
The associated \textsf{R} function is \inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false,tabsize=4]!dcauchy(x, location = 0, scale = 1)!\inputencoding{utf8}.

It is easy to see that a $\mathsf{cauchy}(\mathtt{location}=0,\,\mathtt{scale}=1)$
distribution is the same as a $\mathsf{t}(\mathtt{df}=1)$ distribution.
The $\mathsf{cauchy}$ distribution looks like a $\mathsf{norm}$
distribution but with very heavy tails. The mean (and variance) do
not exist, that is, they are infinite. The median is represented by
the $\mathtt{location}$ parameter, and the $\mathtt{scale}$ parameter
influences the spread of the distribution about its median.


\subsubsection*{The Beta Distribution\label{sub:The-Beta-Distribution}}

This is a generalization of the continuous uniform distribution.

\begin{equation}
f_{X}(x)=\frac{\Gamma(\alpha+\beta)}{\Gamma(\alpha)\Gamma(\beta)}\: x^{\alpha-1}(1-x)^{\beta-1},\quad0<x<1\end{equation}
We write $X\sim\mathsf{beta}(\mathtt{shape1}=\alpha,\,\mathtt{shape2}=\beta)$.
The associated \textsf{R} function is \inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false,tabsize=4]!dbeta(x, shape1, shape2)!\inputencoding{utf8}.
The mean and variance are\begin{equation}
\mu=\frac{\alpha}{\alpha+\beta}\mbox{ and }\sigma^{2}=\frac{\alpha\beta}{\left(\alpha+\beta\right)^{2}\left(\alpha+\beta+1\right)}.\end{equation}
See Example \ref{exa:cont-pdf3x2}. This distribution comes up a lot
in Bayesian statistics because it is a good model for one's prior
beliefs about a population proportion $p$, $0\leq p\leq1$.


\subsubsection*{The Logistic Distribution\label{sub:The-Logistic-Distribution}}

\begin{equation}
f_{X}(x)=\frac{1}{\sigma}\exp\left(-\frac{x-\mu}{\sigma}\right)\left[1+\exp\left(-\frac{x-\mu}{\sigma}\right)\right]^{-2},\quad-\infty<x<\infty.\end{equation}
We write $X\sim\mathsf{logis}(\mathtt{location}=\mu,\,\mathtt{scale}=\sigma)$.
The associated \textsf{R} function is \inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false,tabsize=4]!dlogis(x, location = 0, scale = 1)!\inputencoding{utf8}.
The logistic distribution comes up in differential equations as a
model for population growth under certain assumptions. The mean is
$\mu$ and the variance is $\pi^{2}\sigma^{2}/3$.


\subsubsection*{The Lognormal Distribution\label{sub:The-Lognormal-Distribution}}

This is a distribution derived from the normal distribution (hence
the name). If $U\sim\mathtt{norm}(\mathtt{mean}=\mu,\,\mathtt{sd}=\sigma)$,
then $X=\me^{U}$has PDF\begin{equation}
f_{X}(x)=\frac{1}{\sigma x\sqrt{2\pi}}\exp\left[\frac{-(\ln x-\mu)^{2}}{2\sigma^{2}}\right],\quad0<x<\infty.\end{equation}
We write $X\sim\mathsf{lnorm}(\mathtt{meanlog}=\mu,\,\mathtt{sdlog}=\sigma)$.
The associated \textsf{R} function is \inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false,tabsize=4]!dlnorm(x, meanlog = 0, sdlog = 1)!\inputencoding{utf8}.
Notice that the support is concentrated on the positive $x$ axis;
the distribution is right-skewed with a heavy tail. See Example \ref{exa:lnorm-transformation}.


\subsubsection*{The Weibull Distribution\label{sub:The-Weibull-Distribution}}

This has PDF

\begin{equation}
f_{X}(x)=\frac{\alpha}{\beta}\left(\frac{x}{\beta}\right)^{\alpha-1}\exp\left(\frac{x}{\beta}\right)^{\alpha},\quad x>0.\end{equation}
We write $X\sim\mathsf{weibull}(\mathtt{shape}=\alpha,\,\mathtt{scale}=\beta)$.
The associated \textsf{R} function is \inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false,tabsize=4]!dweibull(x, shape, scale = 1)!\inputencoding{utf8}.


\subsection{How to do it with \textsf{R}}

There is some support of moments and moment generating functions for
some continuous probability distributions included in the \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!actuar!\inputencoding{utf8}
package \cite{Dutangactuar}. The convention is \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!m!\inputencoding{utf8}
in front of the distribution name for raw moments, and \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!mgf!\inputencoding{utf8}
in front of the distribution name for the moment generating function.
At the time of this writing, the following distributions are supported:
gamma, inverse Gaussian, (non-central) chi-squared, exponential, and
uniform.
\begin{example}
Calculate the first four raw moments for $X\sim\mathsf{gamma}(\mathtt{shape}=13,\,\mathtt{rate}=1)$
and plot the moment generating function.

We load the \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!actuar!\inputencoding{utf8}
package and use the functions \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!mgamma!\inputencoding{utf8}
and \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!mgfgamma!\inputencoding{utf8}:
\end{example}
<<>>=
library(actuar)
mgamma(1:4, shape = 13, rate = 1)
@

For the plot we can use the function in the following form:

<<>>=
plot(function(x){mgfgamma(x, shape = 13, rate = 1)}, from=-0.1, to=0.1, ylab = "gamma mgf")
@

%
\begin{figure}
\begin{centering}
<<echo = FALSE, fig=true, height = 4.5, width = 6>>=
plot(function(x){mgfgamma(x, shape = 13, rate = 1)}, from=-0.1, to=0.1, ylab = "gamma mgf")
@
\par\end{centering}

\caption{Plot of the \textsf{gamma}(\texttt{shape} = 13, \texttt{rate} = 1)
MGF \label{fig:gamma-mgf}}

\end{figure}


\newpage{}


\section*{Chapter Exercises}

\addcontentsline{toc}{section}{Chapter Exercises}
\setcounter{thm}{0}


\begin{xca}
Find the constant $c$ so that the given function is a valid PDF of
a random variable $X$.\end{xca}
\begin{enumerate}
\item $f(x)=Cx^{n},\quad0<x<1$.
\item $f(x)=Cx\me^{-x},\quad0<x<\infty$.
\item $f(x)=\me^{-(x-C)},\quad7<x<\infty.$
\item $f(x)=Cx^{3}(1-x)^{2},\quad0<x<1.$
\item ${\displaystyle f(x)=C(1+x^{2}/4)^{-1}},\quad-\infty<x<\infty.$
\end{enumerate}


<<echo = FALSE, results = hide>>=
rnorm(1)
@
\begin{xca}
For the following random experiments, decide what the distribution
of $X$ should be. In nearly every case, there are additional assumptions
that should be made for the distribution to apply; identify those
assumptions (which may or may not strictly hold in practice).
\begin{enumerate}
\item We throw a dart at a dart board. Let $X$ denote the squared linear
distance from the bulls-eye to the where the dart landed.
\item We randomly choose a textbook from the shelf at the bookstore and
let $P$ denote the proportion of the total pages of the book devoted
to exercises. 
\item We measure the time it takes for the water to completely drain out
of the kitchen sink.
\item We randomly sample strangers at the grocery store and ask them how
long it will take them to drive home. 
\end{enumerate}
\end{xca}

\begin{xca}
If $Z$ is $\mathsf{norm}(\mathtt{mean}=0,\,\mathtt{sd}=1)$, find 
\begin{enumerate}
\item $\P(Z>2.64)$


<<>>=
pnorm(2.64, lower.tail = FALSE)
@

\item $\P(0\leq Z<0.87)$


<<>>=
pnorm(0.87) - 1/2
@

\item $\P(|Z|>1.39)$ (Hint: draw a picture!)


<<>>=
2 * pnorm(-1.39)
@

\end{enumerate}
\end{xca}

\begin{xca}
\label{xca:variance-dunif}Calculate the variance of $X\sim\mathsf{unif}(\mathtt{min}=a,\,\mathtt{max}=b)$.
Hint: First calculate $\E X^{2}$.

type the exercise here
\end{xca}

\begin{xca}
\label{xca:prove-the-memoryless}Prove the memoryless property for
exponential random variables. That is, for $X\sim\mathsf{exp}(\mathtt{rate}=\lambda)$
show that for any $s,t>0$,\[
\P(X>s+t\,|\, X>t)=\P(X>s).\]

\end{xca}





\chapter{Multivariate Distributions\label{cha:Multivariable-Distributions}}

\noindent We have built up quite a catalogue of distributions, discrete
and continuous. They were all univariate, however, meaning that we
only considered one random variable at a time. We can imagine nevertheless
many random variables associated with a single person: their height,
their weight, their wrist circumference (all continuous), or their
eye/hair color, shoe size, whether they are right handed, left handed,
or ambidextrous (all categorical), and we can even surmise reasonable
probability distributions to associate with each of these variables. 

But there is a difference: for a single person, these variables are
related. For instance, a person's height betrays a lot of information
about that person's weight.

The concept we are hinting at is the notion of \emph{dependence} between
random variables. It is the focus of this chapter to study this concept
in some detail. Along the way, we will pick up additional models to
add to our catalogue. Moreover, we will study certain classes of dependence,
and clarify the special case when there is no dependence, namely,
independence.

The interested reader who would like to learn more about any of the
below mentioned multivariate distributions should take a look at \emph{Discrete
Multivariate Distributions} by Johnson \emph{et al }\cite{Johnson1997}
or \emph{Continuous Multivariate Distributions} \cite{Kotz2000} by
Kotz \emph{et al}.


\paragraph*{What do I want them to know?}
\begin{itemize}
\item the basic notion of dependence and how it is manifested with multiple
variables (two, in particular)
\item joint versus marginal distributions/expectation (discrete and continuous)
\item some numeric measures of dependence
\item conditional distributions, in the context of independence and exchangeability
\item some details of at least one multivariate model (discrete and continuous)
\item what it looks like when there are more than two random variables present
\end{itemize}

\section{Joint and Marginal Probability Distributions\label{sec:Joint-Probability-Distributions}}

Consider two discrete random variables $X$ and $Y$ with PMFs $f_{X}$
and $f_{Y}$ that are supported on the sample spaces $S_{X}$ and
$S_{Y}$, respectively. Let $S_{X,Y}$ denote the set of all possible
observed \emph{pairs} $(x,y)$, called the \emph{joint support set}
of $X$ and $Y$. Then the \emph{joint probability mass function}
of $X$ and $Y$ is the function $f_{X,Y}$ defined by\begin{equation}
f_{X,Y}(x,y)=\P(X=x,\, Y=y),\quad\mbox{for }(x,y)\in S_{X,Y}.\label{eq:joint-pmf}\end{equation}
Every joint PMF satisfies\begin{equation}
f_{X,Y}(x,y)>0\mbox{ for all }(x,y)\in S_{X,Y},\end{equation}
and\begin{equation}
\sum_{(x,y)\in S_{X,Y}}f_{X,Y}(x,y)=1.\end{equation}
It is customary to extend the function $f_{X,Y}$ to be defined on
all of $\R^{2}$ by setting $f_{X,Y}(x,y)=0$ for $(x,y)\not\in S_{X,Y}$. 

In the context of this chapter, the PMFs $f_{X}$ and $f_{Y}$ are
called the \emph{marginal PMFs} of $X$ and $Y$, respectively. If
we are given only the joint PMF then we may recover each of the marginal
PMFs by using the Theorem of Total Probability (see Equation\ref{eq:theorem-total-probability}):
observe\begin{eqnarray}
f_{X}(x) & = & \P(X=x),\\
 & = & \sum_{y\in S_{Y}}\P(X=x,\, Y=y),\\
 & = & \sum_{y\in S_{Y}}f_{X,Y}(x,y).\end{eqnarray}
By interchanging the roles of $X$ and $Y$ it is clear that \begin{equation}
f_{Y}(y)=\sum_{x\in S_{Y}}f_{X,Y}(x,y).\label{eq:marginal-pmf}\end{equation}
Given the joint PMF we may recover the marginal PMFs, but the converse
is not true. Even if we have \emph{both} marginal distributions they
are not sufficient to determine the joint PMF; more information is
needed%
\footnote{We are not at a total loss, however. There are Frechet bounds which
pose limits on how large (and small) the joint distribution must be
at each point.%
}. 

Associated with the joint PMF is the \emph{joint cumulative distribution
function} $F_{X,Y}$ defined by\[
F_{X,Y}(x,y)=\P(X\leq x,\, Y\leq y),\quad\mbox{for }(x,y)\in\R^{2}.\]
The bivariate joint CDF is not quite as tractable as the univariate
CDFs, but in principle we could calculate it by adding up quantities
of the form in Equation \ref{eq:joint-pmf}. The joint CDF is typically
not used in practice due to its inconvenient form; one can usually
get by with the joint PMF alone.

We now introduce some examples of bivariate discrete distributions.
The first we have seen before, and the second is based on the first.
\begin{example}
\label{exa:toss-two-dice-joint-pmf}Roll a fair die twice. Let $X$
be the face shown on the first roll, and let $Y$ be the face shown
on the second roll. We have already seen this example in Chapter \ref{cha:Probability},
Example \ref{exa:Toss-a-six-sided-die-twice}. For this example, it
suffices to define\[
f_{X,Y}(x,y)=\frac{1}{36},\quad x=1,\ldots,6,\ y=1,\ldots,6.\]
The marginal PMFs are given by $f_{X}(x)=1/6$, $x=1,2,\ldots,6$,
and $f_{Y}(y)=1/6$, $y=1,2,\ldots,6$, since\[
f_{X}(x)=\sum_{y=1}^{6}\frac{1}{36}=\frac{1}{6},\quad x=1,\ldots,6,\]
and the same computation with the letters switched works for $Y$. 
\end{example}


In the previous example, and in many other ones, the joint support
can be written as a product set of the support of $X$ {}``times''
the support of $Y$, that is, it may be represented as a cartesian
product set, or rectangle, $S_{X,Y}=S_{X}\times S_{Y}$, where $S_{X}\times S_{Y}=\left\{ (x,y):\ x\in S_{X},\, y\in S_{Y}\right\} $.
As we shall see presently in Section \ref{sec:Independent-Random-Variables},
this form is a necessary condition for $X$ and $Y$ to be \emph{independent}
(or alternatively \emph{exchangeable} when $S_{X}=S_{Y}$). But please
note that in general it is not required for $S_{X,Y}$ to be of rectangle
form. We next investigate just such an example.


\begin{example}
\label{exa:max-sum-two-dice}Let the random experiment again be to
roll a fair die twice, except now let us define the random variables
$U$ and $V$ by\begin{eqnarray*}
U & = & \mbox{the maximum of the two rolls, and }\\
V & = & \mbox{the sum of the two rolls.}\end{eqnarray*}
We see that the support of $U$ is $S_{U}=\left\{ 1,2,\ldots,6\right\} $
and the support of $V$ is $S_{V}=\left\{ 2,3,\ldots,12\right\} $.
We may represent the sample space with a matrix, and for each entry
in the matrix we may calculate the value that $U$ assumes. The result
is in the left half of Table \ref{tab:max-and-sum-two-dice}. 

We can use the table to calculate the marginal PMF of $U$, because
from Example \ref{exa:Toss-a-six-sided-die-twice} we know that each
entry in the matrix has probability $1/36$ associated with it. For
instance, there is only one outcome in the matrix with $U=1$, namely,
the top left corner. This single entry has probability $1/36$, therefore,
it must be that $f_{U}(1)=\P(U=1)=1/36$. Similarly we see that there
are three entries in the matrix with $U=2$, thus $f_{U}(2)=3/36$.
Continuing in this fashion we will find the marginal distribution
of $U$ may be written\begin{equation}
f_{U}(u)=\frac{2u-1}{36},\quad u=1,\,2,\ldots,6.\end{equation}


We may do a similar thing for $V$; see the right half of Table \ref{tab:max-and-sum-two-dice}.
Collecting all of the probability we will find that the marginal PMF
of $V$ is\begin{equation}
f_{V}(v)=\frac{6-|v-7|}{36},\quad v=2,\,3,\ldots,12.\end{equation}


%
\begin{table}
\hfill{}\subfloat[$U=\max(X,Y)$]{\begin{centering}
\begin{tabular}{c|cccccc}
$U$ & 1 & 2 & 3 & 4 & 5 & 6\tabularnewline
\hline
1 & 1 & 2 & 3 & 4 & 5 & 6\tabularnewline
2 & 2 & 2 & 3 & 4 & 5 & 6\tabularnewline
3 & 3 & 3 & 3 & 4 & 5 & 6\tabularnewline
4 & 4 & 4 & 4 & 4 & 5 & 6\tabularnewline
5 & 5 & 5 & 5 & 5 & 5 & 6\tabularnewline
6 & 6 & 6 & 6 & 6 & 6 & 6\tabularnewline
\end{tabular}
\par\end{centering}

}\hfill{}\subfloat[$V=X+Y$]{\begin{centering}
\begin{tabular}{c|cccccc}
$V$ & 1 & 2 & 3 & 4 & 5 & 6\tabularnewline
\hline
1 & 2 & 3 & 4 & 5 & 6 & 7\tabularnewline
2 & 3 & 4 & 5 & 6 & 7 & 8\tabularnewline
3 & 4 & 5 & 6 & 7 & 8 & 9\tabularnewline
4 & 5 & 6 & 7 & 8 & 9 & 10\tabularnewline
5 & 6 & 7 & 8 & 9 & 10 & 11\tabularnewline
6 & 7 & 8 & 9 & 10 & 11 & 12\tabularnewline
\end{tabular}
\par\end{centering}

}\hfill{}

\caption{Maximum $U$ and sum $V$ of a pair of dice rolls $(X,Y)$\label{tab:max-and-sum-two-dice}}

\end{table}


We may collapse the two matrices from Table \ref{tab:max-and-sum-two-dice}
into one, big matrix of pairs of values $(u,v)$. The result is shown
in Table \ref{tab:-max-sum-two-dice-joint}. 

%
\begin{table}
\begin{centering}
\begin{tabular}{c|cccccc}
$(U,V)$ & 1 & 2 & 3 & 4 & 5 & 6\tabularnewline
\hline
1 & (1,2) & (2,3) & (3,4) & (4,5) & (5,6) & (6,7)\tabularnewline
2 & (2,3) & (2,4) & (3,5) & (4,6) & (5,7) & (6,8)\tabularnewline
3 & (3,4) & (3,5) & (3,6) & (4,7) & (5,8) & (6,9)\tabularnewline
4 & (4,5) & (4,6) & (4,7) & (4,8) & (5,9) & (6,10)\tabularnewline
5 & (5,6) & (5,7) & (5,8) & (5,9) & (5,10) & (6,11)\tabularnewline
6 & (6,7) & (6,8) & (6,9) & (6,10) & (6,11) & (6,12)\tabularnewline
\end{tabular}
\par\end{centering}

\caption{Joint values of $U=\max(X,Y)$ and $V=X+Y$\label{tab:-max-sum-two-dice-joint}}

\end{table}


Again, each of these pairs has probability $1/36$ associated with
it and we are looking at the joint PDF of $(U,V)$ albeit in an unusual
form. Many of the pairs are repeated, but some of them are not: $(1,2)$
appears twice, but $(2,3)$ appears only once. We can make more sense
out of this by writing a new table with $U$ on one side and $V$
along the top. We will accumulate the probability just like we did
in Example \ref{exa:toss-two-dice-joint-pmf}. See Table \ref{tab:max-sum-joint-pmf}.

%
\begin{table}
\begin{centering}
\begin{tabular}{|c|ccccccccccc|c|}
\hline 
 & {\small 2} & {\small 3} & {\small 4} & {\small 5} & {\small 6} & {\small 7} & {\small 8} & {\small 9} & {\small 10} & {\small 11} & {\small 12} & {\small Total}\tabularnewline
\hline
{\small 1} & {\small $1/36$} &  &  &  &  &  &  &  &  &  &  & {\small $1/36$}\tabularnewline
{\small 2} &  & {\small $2/36$} & {\small $1/36$} &  &  &  &  &  &  &  &  & {\small $3/36$}\tabularnewline
{\small 3} &  &  & {\small $2/36$} & {\small $2/36$} & {\small $1/36$} &  &  &  &  &  &  & {\small $5/36$}\tabularnewline
{\small 4} &  &  &  & {\small $2/36$} & {\small $2/36$} & {\small $2/36$} & {\small $1/36$} &  &  &  &  & {\small $7/36$}\tabularnewline
{\small 5} &  &  &  &  & {\small $2/36$} & {\small $2/36$} & {\small $2/36$} & {\small $2/36$} & {\small $1/36$} &  &  & {\small $9/36$}\tabularnewline
{\small 6} &  &  &  &  &  & {\small $2/36$} & {\small $2/36$} & {\small $2/36$} & {\small $2/36$} & {\small $2/36$} & {\small $1/36$} & {\small $11/36$}\tabularnewline
\hline 
{\small Total} & {\small $1/36$} & {\small $2/36$} & {\small $3/36$} & {\small $4/36$} & {\small $5/36$} & {\small $6/36$} & {\small $5/36$} & {\small $4/36$} & {\small $3/36$} & {\small $2/36$} & {\small $1/36$} & {\small 1}\tabularnewline
\hline
\end{tabular}
\par\end{centering}

\caption{The joint PMF of $(U,V)$\label{tab:max-sum-joint-pmf} }
{\small The outcomes of $U$ are along the left and the outcomes of
$V$ are along the top. Empty entries in the table have zero probability.
The row totals (on the right) and column totals (on the bottom) correspond
to the marginal distribution of $U$ and $V$, respectively. }
\end{table}


The joint support of $(U,V)$ is concentrated along the main diagonal;
note that the nonzero entries do not form a rectangle. Also notice
that if we form row and column totals we are doing exactly the same
thing as Equation \ref{eq:marginal-pmf}, so that the marginal distribution
of $U$ is the list of totals in the right {}``margin'' of the Table
\ref{tab:max-sum-joint-pmf}, and the marginal distribution of $V$
is the list of totals in the bottom {}``margin''.
\end{example}
Continuing the reasoning for the discrete case, given two continuous
random variables $X$ and $Y$ there similarly exists%
\footnote{Strictly speaking, the joint density function does not necessarily
exist. But the joint CDF always exists.%
} a function $f_{X,Y}(x,y)$ associated with $X$ and $Y$ called the
\emph{joint probability density function} of $X$ and $Y$. Every
joint PDF satisfies\begin{equation}
f_{X,Y}(x,y)\geq0\mbox{ for all }(x,y)\in S_{X,Y},\end{equation}
and\begin{equation}
\iintop_{S_{X,Y}}f_{X,Y}(x,y)\,\diff x\,\diff y=1.\end{equation}


In the continuous case there is not such a simple interpretation for
the joint PDF; however, we do have one for the joint CDF, namely,\[
F_{X,Y}(x,y)=\P(X\leq x,\, Y\leq y)=\int_{-\infty}^{x}\int_{-\infty}^{y}f_{X,Y}(u,v)\,\diff v\,\diff u,\]
for $(x,y)\in\R^{2}$. If $X$ and $Y$ have the joint PDF $f_{X,Y}$,
then the marginal density of $X$ may be recovered by\begin{equation}
f_{X}(x)=\int_{S_{Y}}f_{X,Y}(x,y)\,\diff y,\quad x\in S_{X}\end{equation}
and the marginal PDF of $Y$ may be found with \begin{equation}
f_{Y}(y)=\int_{S_{X}}f_{X,Y}(x,y)\,\diff x,\quad y\in S_{Y}.\end{equation}

\begin{example}
\label{exa:joint-pdf}Let the joint PDF of $(X,Y)$ be given by\[
f_{X,Y}(x,y)=\frac{6}{5}\left(x+y^{2}\right),\quad0<x<1,\ 0<y<1.\]
The marginal PDF of $X$ is\begin{eqnarray*}
f_{X}(x) & = & \int_{0}^{1}\frac{6}{5}\left(x+y^{2}\right)\,\diff y,\\
 & = & \left.\frac{6}{5}\left(xy+\frac{y^{3}}{3}\right)\right|_{y=0}^{1},\\
 & = & \frac{6}{5}\left(x+\frac{1}{3}\right),\end{eqnarray*}
for $0<x<1$, and the marginal PDF of $Y$ is\begin{eqnarray*}
f_{Y}(y) & = & \int_{0}^{1}\frac{6}{5}\left(x+y^{2}\right)\,\diff x,\\
 & = & \left.\frac{6}{5}\left(\frac{x^{2}}{2}+xy^{2}\right)\right|_{x=0}^{1},\\
 & = & \frac{6}{5}\left(\frac{1}{2}+y^{2}\right),\end{eqnarray*}
for $0<y<1$. In this example the joint support set was a rectangle
$[0,1]\times[0,1]$, but it turns out that $X$ and $Y$ are not independent.
See Section \ref{sec:Independent-Random-Variables}.
\end{example}

\subsection{How to do it with \textsf{R}}

We will show how to do Example \ref{exa:max-sum-two-dice} using \textsf{R};
it is much simpler to do it with \textsf{R} than without. First we
set up the sample space with the \inputencoding{latin9}\lstinline[showstringspaces=false]!rolldie!\inputencoding{utf8}
function. Next, we add random variables $U$ and $V$ with the \inputencoding{latin9}\lstinline[showstringspaces=false]!addrv!\inputencoding{utf8}
function. We take a look at the very top of the data frame (probability
space) to make sure that everything is operating according to plan.

<<>>=
S <- rolldie(2, makespace = TRUE)
S <- addrv(S, FUN = max, invars = c("X1","X2"), name = "U")
S <- addrv(S, FUN = sum, invars = c("X1","X2"), name = "V")
head(S)
@

Yes, the $U$ and $V$ columns have been added to the data frame and
have been computed correctly. This result would be fine as it is,
but the data frame has too many rows: there are repeated pairs $(u,v)$
which show up as repeated rows in the data frame. The goal is to aggregate
the rows of $S$ such that the result has exactly one row for each
unique pair $(u,v)$ with positive probability. This sort of thing
is exactly the task for which the \inputencoding{latin9}\lstinline[showstringspaces=false]!marginal!\inputencoding{utf8}
function was designed. We may take a look at the joint distribution
of $U$ and $V$ (we only show the first few rows of the data frame,
but the complete one has 11 rows).

<<>>=
UV <- marginal(S, vars = c("U", "V"))
head(UV)
@

The data frame is difficult to understand. It would be better to have
a tabular display like Table \ref{tab:max-sum-joint-pmf}. We can
do that with the \inputencoding{latin9}\lstinline[showstringspaces=false]!xtabs!\inputencoding{utf8}
function. 

<<>>=
xtabs(round(probs,3) ~ U + V, data = UV)
@

Compare these values to the ones shown in Table \ref{tab:max-sum-joint-pmf}.
We can repeat the process with \inputencoding{latin9}\lstinline[showstringspaces=false]!marginal!\inputencoding{utf8}
to get the univariate marginal distributions of $U$ and $V$ separately.

<<>>=
marginal(UV, vars = "U")
head(marginal(UV, vars = "V"))
@

Another way to do the same thing is with the \inputencoding{latin9}\lstinline[showstringspaces=false]!rowSums!\inputencoding{utf8}
and \inputencoding{latin9}\lstinline[showstringspaces=false]!colSums!\inputencoding{utf8}
of the \inputencoding{latin9}\lstinline[showstringspaces=false]!xtabs!\inputencoding{utf8}
object. Compare

<<>>=
temp <- xtabs(probs ~ U + V, data = UV)
rowSums(temp)
colSums(temp)
@

You should check that the answers that we have obtained exactly match
the same (somewhat laborious) calculations that we completed in Example
\ref{exa:max-sum-two-dice}.


\section{Joint and Marginal Expectation\label{sec:Joint-and-Marginal-Expectation}}

Given a function $g$ with arguments $(x,y)$ we would like to know
the long-run average behavior of $g(X,Y)$ and how to mathematically
calculate it. Expectation in this context is computed in the pedestrian
way. We simply integrate (sum) with respect to the joint probability
density (mass) function.

\begin{equation}
\E\, g(X,Y)=\iintop_{S_{X,Y}}g(x,y)\, f_{X,Y}(x,y)\,\diff x\,\diff y,\end{equation}
or in the discrete case\begin{equation}
\E\, g(X,Y)=\mathop{\sum\sum}\limits _{(x,y)\in S_{X,Y}}g(x,y)\, f_{X,Y}(x,y).\end{equation}



\subsection{Covariance and Correlation}

There are two very special cases of joint expectation: the \emph{covariance}
and the \emph{correlation}. These are measures which help us quantify
the dependence between $X$ and $Y$. 
\begin{defn}
The \emph{covariance} of $X$ and $Y$ is \begin{equation}
\mbox{Cov}(X,Y)=\E(X-\E X)(Y-\E Y).\end{equation}

\end{defn}
By the way, there is a shortcut formula for covariance which is almost
as handy as the shortcut for the variance:\begin{equation}
\mbox{Cov}(X,Y)=\E(XY)-(\E X)(\E Y).\end{equation}
The proof is left to Exercise \ref{xca:Prove-cov-shortcut}.

The Pearson product moment correlation between $X$ and $Y$ is the
covariance between $X$ and $Y$ rescaled to fall in the interval
$[-1,1]$. It is formally defined by \begin{equation}
\mbox{Corr}(X,Y)=\frac{\mbox{Cov}(X,Y)}{\sigma_{X}\sigma_{Y}}\end{equation}
The correlation is usually denoted by $\rho_{X,Y}$ or simply $\rho$
if the random variables are clear from context. There are some important
facts about the correlation coefficient:
\begin{enumerate}
\item The range of correlation is $-1\leq\rho_{X,Y}\leq1$.
\item Equality holds above ($\rho_{X,Y}=\pm1$) if and only if $Y$ is a
linear function of $X$ with probability one.\end{enumerate}
\begin{example}
\label{exa:max-sum-dice-covariance}We will compute the covariance
for the discrete distribution in Example \ref{exa:max-sum-two-dice}.
The expected value of $U$ is\[
\E U=\sum_{u=1}^{6}u\, f_{U}(u)=\sum_{u=1}^{6}u\,\frac{2u-1}{36}=1\left(\frac{1}{36}\right)+2\left(\frac{3}{36}\right)+\cdots+6\left(\frac{11}{36}\right)=\frac{161}{36},\]
and the expected value of $V$ is\[
\E V=\sum_{v=2}^{12}v\,\frac{6-|7-v|}{36}=2\left(\frac{1}{36}\right)+3\left(\frac{2}{36}\right)+\cdots+12\left(\frac{1}{36}\right)=7,\]
and the expected value of $UV$ is\[
\E UV=\sum_{u=1}^{6}\sum_{v=2}^{12}uv\, f_{U,V}(u,v)=1\cdot2\left(\frac{1}{36}\right)+2\cdot3\left(\frac{2}{36}\right)+\cdots+6\cdot12\left(\frac{1}{36}\right)=\frac{308}{9}.\]
Therefore the covariance of $(U,V)$ is\[
\mbox{Cov}(U,V)=\E UV-\left(\E U\right)\left(\E V\right)=\frac{308}{9}-\frac{161}{36}\cdot7=\frac{35}{12}.\]
All we need now are the standard deviations of $U$ and $V$ to calculate
the correlation coefficient (omitted).
\end{example}


We will do a continuous example so that you can see how it works.
\begin{example}
Let us find the covariance of the variables $(X,Y)$ from Example
\ref{exa:joint-pdf}. The expected value of $X$ is\[
\E X=\int_{0}^{1}x\cdot\frac{6}{5}\left(x+\frac{1}{3}\right)\diff x=\left.\frac{2}{5}x^{3}+\frac{1}{5}x^{2}\right|_{x=0}^{1}=\frac{3}{5},\]
and the expected value of $Y$ is\[
\E Y=\int_{0}^{1}y\cdot\frac{6}{5}\left(\frac{1}{2}+y^{2}\right)\diff x=\left.\frac{3}{10}y^{2}+\frac{3}{20}y^{4}\right|_{y=0}^{1}=\frac{9}{20}.\]
Finally, the expected value of $XY$ is\begin{eqnarray*}
\E XY & = & \int_{0}^{1}\int_{0}^{1}xy\,\frac{6}{5}\left(x+y^{2}\right)\diff x\,\diff y,\\
 & = & \int_{0}^{1}\left.\left(\frac{2}{5}x^{3}y+\frac{3}{10}xy^{4}\right)\right|_{x=0}^{1}\diff y,\\
 & = & \int_{0}^{1}\left(\frac{2}{5}y+\frac{3}{10}y^{4}\right)\diff y,\\
 & = & \frac{1}{5}+\frac{3}{50},\end{eqnarray*}
which is 13/50. Therefore the covariance of $(X,Y)$ is\[
\mbox{Cov}(X,Y)=\frac{13}{50}-\left(\frac{3}{5}\right)\left(\frac{9}{20}\right)=-\frac{1}{100}.\]

\end{example}

\subsection{How to do it with \textsf{R}}

There are not any specific functions in the \inputencoding{latin9}\lstinline[showstringspaces=false]!prob!\inputencoding{utf8}
package designed for multivariate expectation. This is not a problem,
though, because it is easy enough to do expectation the long way --
with column operations. We just need to keep the definition in mind.
For instance, we may compute the covariance of $(U,V)$ from Example
\ref{exa:max-sum-dice-covariance}.

<<>>=
Eu <- sum(S$U*S$probs)
Ev <- sum(S$V*S$probs)
Euv <- sum(S$U*S$V*S$probs)
Euv - Eu * Ev
@

Compare this answer to what we got in Example \ref{exa:max-sum-dice-covariance}.

To do the continuous case we could use the computer algebra utilities
of \inputencoding{latin9}\lstinline[showstringspaces=false]!Yacas!\inputencoding{utf8}
and the associated \textsf{R} package \inputencoding{latin9}\lstinline[showstringspaces=false]!Ryacas!\inputencoding{utf8}
\cite{ryacas}. See Section \ref{sub:bivariate-transf-R} for another
example where the \inputencoding{latin9}\lstinline[showstringspaces=false]!Ryacas!\inputencoding{utf8}
package appears.


\section{Conditional Distributions\label{sec:Conditional-Distributions}}

If $x\in S_{X}$ is such that $f_{X}(x)>0$, then we define the \emph{conditional
density of} $Y|\, X=x$, denoted $f_{Y|x}$, by \begin{equation}
f_{Y|x}(y|x)=\frac{f_{X,Y}(x,y)}{f_{X}(x)},\quad y\in S_{Y}.\end{equation}
We define $f_{X|y}$ in a similar fashion.
\begin{example}
Let the joint PMF of $X$ and $Y$ be given by\[
f_{X,Y}(x,y)=\]

\end{example}

\begin{example}
Let the joint PDF of $X$ and $Y$ be given by
\end{example}

\subsection*{Bayesian Connection}

Conditional distributions play a fundamental role in Bayesian probability
and statistics. There is a parameter $\theta$ which is of primary
interest, and about which we would like to learn. But rather than
observing $\theta$ directly, we instead observe a random variable
$X$ whose probability distribution depends on $\theta$. Using the
information we provided by $X,$ we would like to update the information
that we have about $\theta$.

Our initial beliefs about $\theta$ are represented by a probability
distribution, called the \emph{prior distribution}, denoted by $\pi$.
The PDF $f_{X|\theta}$ is called the \emph{likelihood function},
also called the \emph{likelihood of} $X$ \emph{conditional on} $\theta$.
Given an observation $X=x$, we would like to update our beliefs $\pi$
to a new distribution, called the \emph{posterior distribution of}
$\theta$ \emph{given the observation} $X=x$, denoted $\pi_{\theta|x}$.
It may seem a mystery how to obtain $\pi_{\theta|x}$ based only on
the information provided by $\pi$ and $f_{X|\theta}$, but it should
not be. We have already studied this in Section \ref{sec:Bayes'-Rule}
where it was called Bayes' Rule:\begin{equation}
\pi(\theta|x)=\frac{\pi(\theta)\, f(x|\theta)}{\int\pi(u)\, f(x|u)\diff u}.\end{equation}
Compare the above expression to Equation \ref{eq:bayes-rule}.
\begin{example}
Suppose the parameter $\theta$ is the $\P(\mbox{Heads})$ for a biased
coin. It could be any value from 0 to 1. Perhaps we have some prior
information about this coin, for example, maybe we have seen this
coin before and we have reason to believe that it shows Heads less
than half of the time. Suppose that we represent our beliefs about
$\theta$ with a $\mathsf{beta}(\mathtt{shape1}=1,\,\mathtt{shape2}=3)$
prior distribution, that is, we assume \[
\theta\sim\pi(\theta)=3(1-\theta)^{2},\quad0<\theta<1.\]
To learn more about $\theta$, we will do what is natural: flip the
coin. We will observe a random variable $X$ which takes the value
$1$ if the coin shows Heads, and 0 if the coin shows Tails. Under
these circumstances, $X$ will have a Bernoulli distribution, and
in particular, $X|\theta\sim\mathsf{binom}(\mathtt{size}=1,\,\mathtt{prob}=\theta)$:\[
f_{X|\theta}(x|\theta)=\theta^{x}(1-\theta)^{1-x},\quad x=0,1.\]
Based on the observation $X=x$, we will update the prior distribution
to the posterior distribution, and we will do so with Bayes' Rule:
it says\begin{eqnarray*}
\pi(\theta|x) & \propto & \pi(\theta)\, f(x|\theta),\\
 & = & \theta^{x}(1-\theta)^{1-x}\cdot3(1-\theta)^{2},\\
 & = & 3\,\theta^{x}(1-\theta)^{3-x},\quad0<\theta<1,\end{eqnarray*}
where the constant of proportionality is given by \[
\int3\, u^{x}(1-u)^{3-x}\diff u=\int3\, u^{(1+x)-1}(1-u)^{(4-x)-1}\diff u=3\,\frac{\Gamma(1+x)\Gamma(4-x)}{\Gamma[(1+x)+(4-x)]},\]
the integral being calculated by inspection of the formula for a $\mathsf{beta}(\mathtt{shape1}=1+x,\,\mathtt{shape2}=4-x)$
distribution. That is to say, our posterior distribution is precisely\[
\theta|x\sim\mathsf{beta}(\mathtt{shape1}=1+x,\,\mathtt{shape2}=4-x).\]


The Bayesian statistician uses the posterior distribution for all
matters concerning inference about $\theta$.\end{example}
\begin{rem}
We usually do not restrict ourselves to the observation of only one
$X$ conditional on $\theta$. In fact, it is common to observe an
entire sample $X_{1}$, $X_{2}$,\ldots{},$X_{n}$ conditional on
$\theta$ (which itself is often multidimensional). Do not be frightened,
however, because the intuition is the same. There is a prior distribution
$\pi(\theta)$, a likelihood $f(x_{1},x_{2},\ldots,x_{n}|\theta)$,
and a posterior distribution $\pi(\theta|x_{1},x_{2},\ldots,x_{n})$.
Bayes' Rule states that the relationship between the three is\[
\pi(\theta|x_{1},x_{2},\ldots,x_{n})\propto\pi(\theta)\, f(x_{1},x_{2},\ldots,x_{n}|\theta),\]
where the constant of proportionality is $\int\pi(u)\, f(x_{1},x_{2},\ldots,x_{n}|u)\,\diff u$.
Any good textbook on Bayesian Statistics will explain these notions
in detail; to the interested reader I recommend Gelman \cite{Gelman2004}
or Lee \cite{Lee1997}. 
\end{rem}

\section{Independent Random Variables\label{sec:Independent-Random-Variables}}


\subsection{Independent Random Variables\label{sub:Independent-Random-Variables}}

We recall from Chapter \ref{cha:Probability} that the events $A$
and $B$ are said to be independent when\begin{equation}
\P(A\cap B)=\P(A)\P(B).\end{equation}
If it happens that\begin{equation}
\P(X=x,Y=y)=\P(X=x)\P(Y=y),\quad\mbox{for every }x\in S_{X},\ y\in S_{Y},\end{equation}
then we say that $X$ and $Y$ are \emph{independent random variables}.
Otherwise, we say that $X$ and $Y$ are \emph{dependent}. Using the
PMF notation from above, we see that independent discrete random variables
satisfy \begin{equation}
f_{X,Y}(x,y)=f_{X}(x)f_{Y}(y)\quad\mbox{for every }x\in S_{X},\ y\in S_{Y}.\end{equation}
Continuing the reasoning, given two continuous random variables $X$
and $Y$ with joint PDF $f_{X,Y}$ and respective marginal PDFs $f_{X}$
and $f_{Y}$ that are supported on the sets $S_{X}$ and $S_{Y}$,
if it happens that \begin{equation}
f_{X,Y}(x,y)=f_{X}(x)f_{Y}(y)\quad\mbox{for every }x\in S_{X},\ y\in S_{Y},\end{equation}
then we say that $X$ and $Y$ are independent.
\begin{example}
In Example \ref{exa:toss-two-dice-joint-pmf} we considered the random
experiment of rolling a fair die twice. There we found the joint PMF
to be\[
f_{X,Y}(x,y)=\frac{1}{36},\quad x=1,\ldots,6,\ y=1,\ldots,6,\]
and we found the marginal PMFs $f_{X}(x)=1/6$, $x=1,2,\ldots,6$,
and $f_{Y}(y)=1/6$, $y=1,2,\ldots,6$. Therefore in this experiment
$X$ and $Y$ are independent since for every $x$ and $y$ in the
joint support the joint PMF satisfies \[
f_{X,Y}(x,y)=\frac{1}{36}=\left(\frac{1}{6}\right)\left(\frac{1}{6}\right)=f_{X}(x)\, f_{Y}(y).\]

\end{example}

\begin{example}
In Example \ref{exa:max-sum-two-dice} we considered the same experiment
but different random variables $U$ and $V$. We can prove that $U$
and $V$ are not independent if we can find a single pair $(u,v)$
where the independence equality does not hold. There are many such
pairs. One of them is $(6,12)$:\[
f_{U,V}(6,12)=\frac{1}{36}\neq\left(\frac{11}{36}\right)\left(\frac{1}{36}\right)=f_{U}(6)\, f_{V}(12).\]

\end{example}
Independent random variables are very useful to the mathematician.
They have many, many, tractable properties. We mention some of the
more important ones.
\begin{prop}
\label{pro:indep-implies-prodexpect}If $X$ and $Y$ are independent,
then for any functions $u$ and $v$, \begin{equation}
\E\left(u(X)v(Y)\right)=\left(\E u(X)\right)\left(\E v(Y)\right).\end{equation}

\end{prop}
\begin{proof}
This is straightforward from the definition.\begin{eqnarray*}
\E\left(u(X)v(Y)\right) & = & \iint\, u(x)v(y)\, f_{X,Y}(x,y)\,\diff x\diff y\\
 & = & \iint\, u(x)v(y)\, f_{X}(x)\, f_{Y}(y)\,\diff x\diff y\\
 & = & \int u(x)\, f_{X}(x)\,\diff x\ \int v(y)\, f_{Y}(y)\,\diff y\end{eqnarray*}
and this last quantity is exactly $\left(\E u(X)\right)\left(\E v(Y)\right)$.
\end{proof}


Now that we have Proposition \ref{pro:indep-implies-prodexpect} we
mention a corollary that will help us later to quickly identify those
random variables which are \emph{not} independent.
\begin{cor}
If $X$ and $Y$ are independent, then $\mbox{Cov}(X,Y)=0$, and consequently,
$\mbox{Corr}(X,Y)=0$.\label{cor:indep-implies-uncorr}
\end{cor}
\begin{proof}
When $X$ and $Y$ are independent then $\E XY=\E X\,\E Y$. And when
the covariance is zero the numerator of the correlation is 0. 
\end{proof}



\begin{rem}
\label{rem:cov0-not-imply-indep}Unfortunately, the converse of Corollary
\ref{cor:indep-implies-uncorr} is not true. That is, there are many
random variables which are dependent yet their covariance and correlation
is zero. For more details, see Casella and Berger \cite{Casella2002}. 
\end{rem}
Proposition \ref{pro:indep-implies-prodexpect} is useful to us and
we will receive mileage out of it, but there is another fact which
will play an even more important role. Unfortunately, the proof is
beyond the techniques presented here. The inquisitive reader should
consult Casella and Berger \cite{Casella2002}, Resnick \cite{Resnick1999},
\emph{etc}.
\begin{fact}
\label{fac:indep-then-function-indep}If $X$ and $Y$ are independent,
then $u(X)$ and $v(Y)$ are independent for any functions $u$ and
$v$.
\end{fact}

\subsection{Combining Independent Random Variables\label{sub:Combining-Independent-Random}}

Another important corollary of Proposition \ref{pro:indep-implies-prodexpect}
will allow us to find the distribution of sums of random variables.
\begin{cor}
If $X$ and $Y$ are independent, then the moment generating function
of $X+Y$ is \begin{equation}
M_{X+Y}(t)=M_{X}(t)\cdot M_{Y}(t).\end{equation}

\end{cor}
\begin{proof}
Choose $u(x)=\me^{x}$ and $v(y)=\me^{y}$ in Proposition \ref{pro:indep-implies-prodexpect},
and remember the identity $\me^{t(x+y)}=\me^{tx}\,\me^{ty}$.
\end{proof}


Let us take a look at some examples of the corollary in action.
\begin{example}
Let $X\sim\mathsf{binom}(\mathtt{size}=n_{1},\,\mathtt{prob}=p)$
and $Y\sim\mathsf{binom}(\mathtt{size}=n_{2},\,\mathtt{prob}=p)$
be independent. Then $X+Y$ has MGF\[
M_{X+Y}(t)=M_{X}(t)\, M_{Y}(t)=\left(q+p\me^{t}\right)^{n_{1}}\left(q+p\me^{t}\right)^{n_{2}}=\left(q+p\me^{t}\right)^{n_{1}+n_{2}},\]
which is the MGF of a $\mathsf{binom}(\mathtt{size}=n_{1}+n_{2},\,\mathtt{prob}=p)$
distribution. Therefore, $X+Y\sim\mathsf{binom}(\mathtt{size}=n_{1}+n_{2},\,\mathtt{prob}=p)$.
\end{example}

\begin{example}
Let $X\sim\mathsf{norm}(\mathtt{mean}=\mu_{1},\,\mathtt{sd}=\sigma_{1})$
and $Y\sim\mathsf{norm}(\mathtt{mean}=\mu_{2},\,\mathtt{sd}=\sigma_{2})$
be independent. Then $X+Y$ has MGF\[
M_{X}(t)\, M_{Y}(t)=\exp\left\{ \mu_{1}t+t^{2}\sigma_{1}^{2}/2\right\} \exp\left\{ \mu_{2}t+t^{2}\sigma_{2}^{2}/2\right\} =\exp\left\{ \left(\mu_{1}+\mu_{2}\right)t+t^{2}\left(\sigma_{1}^{2}+\sigma_{2}^{2}\right)/2\right\} ,\]
which is the MGF of a $\mathsf{norm}(\mathtt{mean}=\mu_{1}+\mu_{2},\,\mathtt{sd}=\sqrt{\sigma_{1}^{2}+\sigma_{2}^{2}})$
distribution. 
\end{example}
Even when we cannot use the MGF trick to identify the exact distribution
of a linear combination of random variables, we can still say something
about its mean and variance.
\begin{prop}
\label{pro:mean-sd-lin-comb-two}Let $X_{1}$ and $X_{2}$ be independent
with respective population means $\mu_{1}$ and $\mu_{2}$ and population
standard deviations $\sigma_{1}$ and $\sigma_{2}$. For given constants
$a_{1}$ and $a_{2}$, define $Y=a_{1}X_{1}+a_{2}X_{2}$. Then the
mean and standard deviation of $Y$ are given by the formulas\begin{equation}
\mu_{Y}=a_{1}\mu_{1}+a_{2}\mu_{2},\quad\sigma_{Y}=\left(a_{1}^{2}\sigma_{1}^{2}+a_{2}^{2}\sigma_{2}^{2}\right)^{1/2}.\end{equation}

\end{prop}
\begin{proof}
We use Proposition \ref{pro:expectation-properties}:\[
\E Y=\E\left(a_{1}X_{1}+a_{2}X_{2}\right)=a_{1}\E X_{1}+a_{2}\E X_{2}=a_{1}\mu_{1}+a_{2}\mu_{2}.\]
For the standard deviation, we will find the variance and take the
square root at the end. And to calculate the variance we will first
compute $\E Y^{2}$ with an eye toward using the identity $\sigma_{Y}^{2}=\E Y^{2}-\left(\E Y\right)^{2}$
as a final step. \[
\E Y^{2}=\E\left(a_{1}X_{1}+a_{2}X_{2}\right)^{2}=\E\left(a_{1}^{2}X_{1}^{2}+a_{2}^{2}X_{2}^{2}+2a_{1}a_{2}X_{1}X_{2}\right).\]
Using linearity of expectation the $\E$ distributes through the sum.
Now $\E X_{i}^{2}=\sigma_{i}^{2}+\mu_{i}^{2}$, for $i=1$ and 2 and
$\E X_{1}X_{2}=\E X_{1}\E X_{2}=\mu_{1}\mu_{2}$ because of independence.
Thus\begin{eqnarray*}
\E Y^{2} & = & a_{1}^{2}(\sigma_{1}^{2}+\mu_{1}^{2})+a_{2}^{2}(\sigma_{2}^{2}+\mu_{2}^{2})+2a_{1}a_{2}\mu_{1}\mu_{2},\\
 & = & a_{1}^{2}\sigma_{1}^{2}+a_{2}^{2}\sigma_{2}^{2}+\left(a_{1}^{2}\mu_{1}^{2}+a_{2}^{2}\mu_{2}^{2}+2a_{1}a_{2}\mu_{1}\mu_{2}\right).\end{eqnarray*}
But notice that the expression in the parentheses is exactly $\left(a_{1}\mu_{1}+a_{2}\mu_{2}\right)^{2}=\left(\E Y\right)^{2}$,
so the proof is complete.. 
\end{proof}



\section{Exchangeable Random Variables\label{sec:Exchangeable-Random-Variables}}

Two random variables $X$ and $Y$ are said to be \emph{exchangeable}
if their joint CDF is a symmetric function of its arguments:\begin{equation}
F_{X,Y}(x,y)=F_{X,Y}(y,x),\quad\mbox{for all }(x,y)\in\R^{2}.\end{equation}
When the joint density $f$ exists, we may equivalently say that $X$
and $Y$ are exchangeable if $f(x,y)=f(y,x)$ for all $(x,y)$.

Exchangeable random variables exhibit symmetry in the sense that a
person may exchange one variable for the other with no substantive
changes to their joint random behavior. While independence speaks
to a \emph{lack of influence} between the two variables, exchangeability
aims to capture the \emph{symmetry} between them.
\begin{example}
Let $X$ and $Y$ have joint PDF\begin{multline}
f_{X,Y}(x,y)=(1+\alpha)\lambda^{2}\me^{-\lambda(x+y)}+\alpha(2\lambda)^{2}\me^{-2\lambda(x+y)}-2\alpha\lambda^{2}\left(\me^{-\lambda(2x+y)}+\me^{-\lambda(x+2y)}\right).\end{multline}
It is straightforward and tedious to check that $\iint f=1$. We may
see immediately that $f_{X,Y}(x,y)=f_{X,Y}(y,x)$ for all $(x,y)$,
which confirms that $X$ and $Y$ are exchangeable. Here, $\alpha$
is said to be an association parameter. This particular example is
one from the Farlie-Gumbel-Morgenstern family of distributions; see
\cite{Kotz2000}.
\end{example}

\begin{example}
\label{exa:binom-exchangeable}Suppose $X$ and $Y$ are i.i.d.~$\mathsf{binom}(\mathtt{size}=n,\,\mathtt{prob}=p)$.
Then their joint PMF is\begin{eqnarray*}
f_{X,Y}(x,y) & = & f_{X}(x)f_{Y}(y)\\
 & = & {n \choose x}\, p^{x}(1-p)^{n-x}\,{n \choose y}\, p^{y}(1-p)^{n-y},\\
 & = & {n \choose x}{n \choose y}\, p^{x+y}(1-p)^{2n-(x+y)},\end{eqnarray*}
and the value is the same if we exchange $x$ and $y$. Therefore
$(X,Y)$ are exchangeable.
\end{example}


Looking at Example \ref{exa:binom-exchangeable} more closely we see
that the fact that $(X,Y)$ are exchangeable has nothing to do with
the $\mathsf{binom}(\mathtt{size}=n,\,\mathtt{prob}=p)$ distribution;
it only matters that they are independent (so that the joint PDF factors)
and they are identically distributed (in which case we may swap letters
to no effect). We could just have easily used any other marginal distribution.
We will take this as a proof of the following proposition.
\begin{prop}
If $X$ and $Y$ are i.i.d.~(with common marginal distribution $F$)
then $X$ and $Y$ are exchangeable. 
\end{prop}


Exchangeability thus contains i.i.d.~as a special case. 


\section{The Bivariate Normal Distribution\label{sec:The-Bivariate-Normal}}

The bivariate normal PDF is given by the unwieldy formula\begin{multline}
f_{X,Y}(x,y)=\frac{1}{2\pi\,\sigma_{X}\sigma_{Y}\sqrt{1-\rho^{2}}}\exp\left\{ -\frac{1}{2(1-\rho^{2})}\left[\left(\frac{x-\mu_{X}}{\sigma_{X}}\right)^{2}+\cdots\right.\right.\\
\left.\left.\cdots+2\rho\left(\frac{x-\mu_{X}}{\sigma_{X}}\right)\left(\frac{y-\mu_{Y}}{\sigma_{Y}}\right)+\left(\frac{y-\mu_{Y}}{\sigma_{Y}}\right)^{2}\right]\right\} ,\end{multline}
for $(x,y)\in\R^{2}$. We write $(X,Y)\sim\mathsf{mvnorm}(\mathtt{mean}=\upmu,\,\mathtt{sigma}=\Sigma)$,
where\begin{equation}
\upmu=(\mu_{X},\,\mu_{Y})^{T},\quad\sum=\left(\begin{array}{cc}
\sigma_{X}^{2} & \rho\sigma_{X}\sigma_{Y}\\
\rho\sigma_{X}\sigma_{Y} & \sigma_{Y}^{2}\end{array}\right).\end{equation}
See Appendix \ref{cha:Mathematical-Machinery}. The vector notation
allows for a more compact rendering of the joint PDF:\begin{equation}
f_{X,Y}(\mathbf{x})=\frac{1}{2\pi\left|\Sigma\right|^{1/2}}\exp\left\{ -\frac{1}{2}\left(\mathbf{x}-\upmu\right)^{\top}\Sigma^{-1}\left(\mathbf{x}-\upmu\right)\right\} ,\end{equation}
where in an abuse of notation we have written $\mathbf{x}$ for $(x,y)$.
Note that the formula only holds when $\rho\neq\pm1$.
\begin{rem}
In Remark \ref{rem:cov0-not-imply-indep} we noted that just because
random variables are uncorrelated it does not necessarily mean that
they are independent. However, there is an important exception to
this rule: the bivariate normal distribution. Indeed, $(X,Y)\sim\mathsf{mvnorm}(\mathtt{mean}=\upmu,\,\mathtt{sigma}=\Sigma)$
are independent if and only if $\rho=0$. 
\end{rem}

\begin{rem}
Inspection of the joint PDF shows that if $\mu_{X}=\mu_{Y}$ and $\sigma_{X}=\sigma_{Y}$
then $X$ and $Y$ are exchangeable.
\end{rem}
The bivariate normal MGF is\begin{equation}
M_{X,Y}(\mathbf{t})=\exp\left(\upmu^{\top}\mathbf{t}+\frac{1}{2}\mathbf{t}^{\top}\Sigma\mathbf{t}\right),\end{equation}
where $\mathbf{t}=(t_{1},t_{2})$.

The bivariate normal distribution may be intimidating at first but
it turns out to be very tractable compared to other multivariate distributions.
An example of this is the following fact about the marginals.
\begin{fact}
If $(X,Y)\sim\mathsf{mvnorm}(\mathtt{mean}=\upmu,\,\mathtt{sigma}=\Sigma)$
then\begin{equation}
X\sim\mathsf{norm}(\mathtt{mean}=\mu_{X},\,\mathtt{sd}=\sigma_{X})\mbox{ and }Y\sim\mathsf{norm}(\mathtt{mean}=\mu_{Y},\,\mathtt{sd}=\sigma_{Y}).\end{equation}

\end{fact}
From this we immediately get that $\E X=\mu_{X}$ and $\mbox{Var}(X)=\sigma_{X}^{2}$
(and the same is true for $Y$ with the letters switched). And it
should be no surprise that the correlation between $X$ and $Y$ is
exactly $\mbox{Corr}(X,Y)=\rho$.
\begin{prop}
\label{pro:mvnorm-cond-dist}The conditional distribution of $Y|\, X=x$
is $\mathsf{norm}(\mathtt{mean}=\mu_{Y|x},\,\mathtt{sd}=\sigma_{Y|x})$,
where\begin{equation}
\mu_{Y|x}=\mu_{Y}+\rho\frac{\sigma_{Y}}{\sigma_{X}}\left(x-\mu_{X}\right),\mbox{ and }\sigma_{Y|x}=\sigma_{Y}\sqrt{1-\rho^{2}}.\end{equation}

\end{prop}
There are a few things to note about Proposition \ref{pro:mvnorm-cond-dist}
which will be important in Chapter \ref{cha:simple-linear-regression}.
First, the conditional mean of $Y|x$ is linear in $x$, with slope\begin{equation}
\rho\,\frac{\sigma_{Y}}{\sigma_{X}}.\label{eq:population-slope-slr}\end{equation}
Second, the conditional variance of $Y|x$ is independent of $x$. 


\subsection{How to do it with \textsf{R}}

The multivariate normal distribution is implemented in both the \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!mvtnorm!\inputencoding{utf8}
package \cite{Genzmvtnorm} and the \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!mnormt!\inputencoding{utf8}
package \cite{mnormt}. We use the \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!mvtnorm!\inputencoding{utf8}
package in this book simply because it is a dependency of another
package used in the book. 

The \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!mvtnorm!\inputencoding{utf8}
package has functions \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!dmvnorm!\inputencoding{utf8}
and \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!rmvnorm!\inputencoding{utf8}
for the PDF and to generate random vectors, respectively. Let us get
started with a graph of the bivariate normal PDF. We can make the
plot with the following code%
\footnote{Another way to do this is with the \texttt{curve3d} function in the
\texttt{emdbook} package \cite{emdbook}. It looks like this:
\begin{lyxcode}
library(emdbook);~library(mvtnorm)~~~\#~note:~the~order~matters

mu~<-~c(0,0);~~sigma~<-~diag(2)

f~<-~function(x,y)~dmvnorm(c(x,y),~mean~=~mu,~sigma~=~sigma)

curve3d(f(x,y),~from~=~c(-3,-3),~to~=~c(3,3),~theta~=~-30,~phi~=~30)
\end{lyxcode}
The code above is slightly shorter than that using \texttt{persp}
and is easier to understand. One must be careful, however. If the
\texttt{library} calls are swapped then the code will not work because
both packages \texttt{emdbook} and \texttt{mvtnorm} have a function
called {}``\texttt{dmvnorm}''; one must load them to the search
path in the correct order or \textsf{R} will use the wrong one (the
arguments are named differently and the underlying algorithms are
different). %
}, where the workhorse is the \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!persp!\inputencoding{utf8}
function in base \textsf{R}.

<<eval = FALSE>>=
library(mvtnorm)
x <- y <- seq(from = -3, to = 3, length.out = 30)
f <- function(x,y) dmvnorm(cbind(x,y), mean = c(0,0), sigma = diag(2))
z <- outer(x, y, FUN = f)
persp(x, y, z, theta = -30, phi = 30, ticktype = "detailed")
@

We chose the standard bivariate normal, $\mathsf{mvnorm}(\mathtt{mean}=\mathbf{0},\,\mathtt{sigma}=\mathbf{I})$,
to display.

%
\begin{figure}
\begin{centering}
<<echo = FALSE, fig=true, height = 8, width = 6>>=
library(mvtnorm)
x <- y <- seq(from = -3, to = 3, length.out = 30)
f <- function(x,y) dmvnorm(cbind(x,y), mean = c(0,0), sigma = diag(2))
z <- outer(x, y, FUN = f)
persp(x, y, z, theta = -30, phi = 30, ticktype = "detailed")
@
\par\end{centering}

\caption{Graph of a bivariate normal PDF\label{fig:mvnorm-pdf}}

\end{figure}



\section{Bivariate Transformations of Random Variables\label{sec:Transformations-Multivariate}}

We studied in Section \ref{sec:Functions-of-Continuous} how to find
the PDF of $Y=g(X)$ given the PDF of $X$. But now we have two random
variables $X$ and Y, with joint PDF $f_{X,Y}$, and we would like
to consider the joint PDF of two new random variables\begin{equation}
U=g(X,Y)\quad\mbox{and}\quad V=h(X,Y),\end{equation}
where $g$ and $h$ are two given functions, typically {}``nice''
in the sense of Appendix \ref{sec:Multivariable-Calculus}. 

Suppose that the transformation $(x,y)\longmapsto(u,v)$ is one-to-one.
Then an inverse transformation $x=x(u,v)$ and $y=y(u,v)$ exists,
so let $\partial(x,y)/\partial(u,v)$ denote the Jacobian of the inverse
transformation. Then the joint PDF of $(U,V)$ is given by

\begin{equation}
f_{U,V}(u,v)=f_{X,Y}\left[x(u,v),\, y(u,v)\right]\left|\frac{\partial(x,y)}{\partial(u,v)}\right|,\end{equation}
or we can rewrite more shortly as\begin{equation}
f_{U,V}(u,v)=f_{X,Y}(x,y)\left|\frac{\partial(x,y)}{\partial(u,v)}\right|.\label{eq:biv-trans-pdf-short}\end{equation}
Take a moment and compare Equation \ref{eq:biv-trans-pdf-short} to
Equation \ref{eq:univ-trans-pdf-short}. Do you see the connection?
\begin{rem}
It is sometimes easier to \emph{postpone} solving for the inverse
transformation $x=x(u,v)$ and $y=y(u,v)$. Instead, leave the transformation
in the form $u=u(x,y)$ and $v=v(x,y)$ and calculate the Jacobian
of the \emph{original} transformation\begin{equation}
\frac{\partial(u,v)}{\partial(x,y)}=\left|\begin{array}{cc}
\frac{\partial u}{\partial x} & \frac{\partial u}{\partial y}\\
\frac{\partial v}{\partial x} & \frac{\partial v}{\partial y}\end{array}\right|=\frac{\partial u}{\partial x}\frac{\partial v}{\partial y}-\frac{\partial u}{\partial y}\frac{\partial v}{\partial x}.\end{equation}
Once this is known, we can get the PDF of $(U,V)$ by\begin{equation}
f_{U,V}(u,v)=f_{X,Y}(x,y)\left|\frac{1}{\frac{\partial(u,v)}{\partial(x,y)}}\right|.\end{equation}
In some cases there will be a cancellation and the work will be a
lot shorter. Of course, it is not always true that\begin{equation}
\frac{\partial(x,y)}{\partial(u,v)}=\frac{1}{\frac{\partial(u,v)}{\partial(x,y)}},\label{eq:biv-jacob-recip}\end{equation}
but for the well-behaved examples that we will see in this book it
works just fine\ldots{} do you see the connection between Equations
\ref{eq:biv-jacob-recip} and \ref{eq:univ-jacob-recip}?\end{rem}
\begin{example}
Let $(X,Y)\sim\mathsf{mvnorm}(\mathtt{mean}=\mathbf{0}_{2\times1},\,\mathtt{sigma}=\mathbf{I}_{2\times2})$
and consider the transformation\begin{align*}
U= & \ 3X+4Y,\\
V= & \ 5X+6Y.\end{align*}
We can solve the system of equations to find the inverse transformations;
they are\begin{align*}
X= & -3U+2V,\\
Y= & \ \frac{5}{2}U-\frac{3}{2}V,\end{align*}
in which case the Jacobian of the inverse transformation is\[
\begin{vmatrix}-3 & 2\\
\frac{5}{2} & -\frac{3}{2}\end{vmatrix}=3\left(-\frac{3}{2}\right)-2\left(\frac{5}{2}\right)=-\frac{1}{2}.\]
As $(x,y)$ traverses $\R^{2}$, so too does $(u,v)$. Since the joint
PDF of $(X,Y)$ is\[
f_{X,Y}(x,y)=\frac{1}{2\pi}\exp\left\{ -\frac{1}{2}\left(x^{2}+y^{2}\right)\right\} ,\quad(x,y)\in\R^{2},\]
we get that the joint PDF of $(U,V)$ is

\begin{equation}
f_{U,V}(u,v)=\frac{1}{2\pi}\exp\left\{ -\frac{1}{2}\left[\left(-3u+2v\right)^{2}+\left(\frac{5u-3v}{2}\right)^{2}\right]\right\} \cdot\frac{1}{2},\quad(u,v)\in\R^{2}.\label{eq:biv-norm-hidden}\end{equation}
\end{example}
\begin{rem}
It may not be obvious, but Equation \ref{eq:biv-norm-hidden} is the
PDF of a $\mathsf{mvnorm}$ distribution. For a more general result
see Theorem \ref{thm:mvnorm-dist-matrix-prod}.
\end{rem}

\subsection{How to do it with \textsf{R\label{sub:bivariate-transf-R}}}

It is possible to do the computations above in \textsf{R} with the
\inputencoding{latin9}\lstinline[showstringspaces=false]!Ryacas!\inputencoding{utf8}
package. The package is an interface to the open-source computer algebra
system, {}``Yacas''. The user installs Yacas, then employs \inputencoding{latin9}\lstinline[showstringspaces=false]!Ryacas!\inputencoding{utf8}
to submit commands to Yacas, after which the output is displayed in
the \textsf{R} console.

There are not yet any examples of Yacas in this book, but there are
online materials to help the interested reader: see \url{http://code.google.com/p/ryacas/}
to get started.


\section{Remarks for the Multivariate Case\label{sec:Remarks-for-the-Multivariate}}

There is nothing spooky about $n\geq3$ random variables. We just
have a whole bunch of them: $X_{1}$, $X_{2}$,\ldots{}, $X_{n}$,
which we can shorten to $\mathbf{X}=(X_{1},X_{2},\ldots,X_{n})^{\mathrm{T}}$
to make the formulas prettier (now may be a good time to check out
Appendix \ref{sec:Linear-Algebra}). For $\mathbf{X}$ supported on
the set $S_{\mathbf{X}}$, the joint PDF $f_{\mathbf{X}}$ (if it
exists) satisfies\begin{equation}
f_{\mathbf{X}}(\mathbf{x})>0,\quad\mbox{for }\mathbf{x}\in S_{\mathbf{X}},\end{equation}
and\begin{equation}
\int\!\!\!\int\cdots\int f_{\mathbf{X}}(\mathbf{x})\,\diff x_{1}\diff x_{2}\cdots\diff x_{n}=1,\end{equation}
or even shorter: $\int f_{\mathbf{X}}(\mathbf{x})\,\diff\mathbf{x}=1$.
The joint CDF $F_{\mathbf{X}}$ is defined by \begin{equation}
F_{\mathbf{X}}(\mathbf{x})=\P(X_{1}\leq x_{1},\, X_{2}\leq x_{2},\ldots,\, X_{n}\leq x_{n}),\end{equation}
for $\mathbf{x}\in\R^{n}$. The expectation of a function $g(\mathbf{X})$
is defined just as we would imagine:\begin{equation}
\E g(\mathbf{X})=\int g(\mathbf{x})\, f_{\mathbf{X}}(\mathbf{x})\,\diff\mathbf{x}.\end{equation}
provided the integral exists and is finite. And the moment generating
function in the multivariate case is defined by\begin{eqnarray}
M_{\mathbf{X}}(\mathbf{t}) & = & \E\exp\left\{ \mathbf{t}^{\mathrm{T}}\mathbf{X}\right\} ,\end{eqnarray}
whenever the integral exists and is finite for all $\mathbf{t}$ in
a neighborhood of $\mathbf{0}_{\mathrm{n}\times1}$ (note that $\mathbf{t}^{\mathrm{T}}\mathbf{X}$
is shorthand for $t_{1}X_{1}+t_{2}X_{2}+\cdots+t_{n}X_{n}$). The
only difference in any of the above for the discrete case is that
integrals are replaced by sums. 

Marginal distributions are obtained by integrating out remaining variables
from the joint distribution. And even if we are given all of the univariate
marginals it is not enough to determine the joint distribution uniquely.

We say that $X_{1}$, $X_{2}$, \ldots{}, $X_{n}$ are \emph{mutually
independent} if their joint PDF factors into the product of the marginals\begin{equation}
f_{\mathbf{X}}(\mathbf{x})=f_{X_{1}}(x_{1})\, f_{X_{2}}(x_{2})\,\cdots\, f_{X_{n}}(x_{n}),\end{equation}
for every $\mathbf{x}$ in their joint support $S_{\mathbf{X}}$,
and we say that $X_{1}$, $X_{2}$, \ldots{}, $X_{n}$ are \emph{exchangeable}
if their joint PDF (or CDF) is a symmetric function of its $n$ arguments,
that is, if \begin{equation}
f_{\mathbf{X}}(\mathbf{x^{\ast}})=f_{\mathbf{X}}(\mathbf{x}),\end{equation}
for any reordering $\mathbf{x^{\ast}}$ of the elements of $\mathbf{x}=(x_{1},x_{2},\ldots,x_{n})$
in the joint support.
\begin{prop}
\label{pro:mean-sd-lin-comb}Let $X_{1}$, $X_{2}$, \ldots{}, $X_{n}$
be independent with respective population means $\mu_{1}$, $\mu_{2}$,
\ldots{}, $\mu_{n}$ and standard deviations $\sigma_{1}$, $\sigma_{2}$,
\ldots{}, $\sigma_{n}$. For given constants $a_{1}$, $a_{2}$,
\ldots{},$a_{n}$ define $Y=\sum_{i=1}^{n}a_{i}X_{i}$. Then the
mean and standard deviation of $Y$ are given by the formulas\begin{equation}
\mu_{Y}=\sum_{i=1}^{n}a_{i}\mu_{i},\quad\sigma_{Y}=\left(\sum_{i=1}^{n}a_{i}^{2}\sigma_{i}^{2}\right)^{1/2}.\end{equation}

\end{prop}
\begin{proof}
The mean is easy:\[
\E Y=\E\left(\sum_{i=1}^{n}a_{i}X_{i}\right)=\sum_{i=1}^{n}a_{i}\E X_{i}=\sum_{i=1}^{n}a_{i}\mu_{i}.\]
The variance is not too difficult to compute either. As an intermediate
step, we calculate $\E Y^{2}$. \[
\E Y^{2}=\E\left(\sum_{i=1}^{n}a_{i}X_{i}\right)^{2}=\E\left(\sum_{i=1}^{n}a_{i}^{2}X_{i}^{2}+2\sum_{i=1}^{n-1}\sum_{j=i+1}^{n}a_{i}a_{j}X_{i}X_{j}\right).\]
Using linearity of expectation the $\E$ distributes through the sums.
Now $\E X_{i}^{2}=\sigma_{i}^{2}+\mu_{i}^{2}$ and $\E X_{i}X_{j}=\E X_{i}\E X_{j}=\mu_{i}\mu_{j}$
when $i\neq j$ because of independence. Thus\begin{eqnarray*}
\E Y^{2} & = & \sum_{i=1}^{n}a_{i}^{2}(\sigma_{i}^{2}+\mu_{i}^{2})+2\sum_{i=1}^{n-1}\sum_{j=i+1}^{n}a_{i}a_{j}\mu_{i}\mu_{j}\\
 & = & \sum_{i=1}^{n}a_{i}^{2}\sigma_{i}^{2}+\left(\sum_{i=1}^{n}a_{i}^{2}\mu_{i}^{2}+2\sum_{i=1}^{n-1}\sum_{j=i+1}^{n}a_{i}a_{j}\mu_{i}\mu_{j}\right)\end{eqnarray*}
To complete the proof, note that the expression in the parentheses
is exactly $\left(\E Y\right)^{2}$, and recall the identity $\sigma_{Y}^{2}=\E Y^{2}-\left(\E Y\right)^{2}$. 
\end{proof}




There is a corresponding statement of Fact \ref{fac:indep-then-function-indep}
for the multivariate case. The proof is also omitted here.
\begin{fact}
If $\mathbf{X}$ and $\mathbf{Y}$ are mutually independent random
vectors, then $u(\mathbf{X})$ and $v(\mathbf{Y})$ are independent
for any functions $u$ and $v$.
\end{fact}


Bruno de Finetti was a strong proponent of the subjective approach
to probability. He proved an important theorem in 1931 which illuminates
the link between exchangeable random variables and independent random
variables. Here it is in one of its simplest forms.
\begin{thm}
De Finetti's Theorem. Let $X_{1}$, $X_{2}$, \ldots{} be a sequence
of $\mathsf{binom}(\mathtt{size}=1,\,\mathtt{prob}=p)$ random variables
such that $(X_{1},\ldots,X_{k})$ are exchangeable for every $k$.
Then there exists a random variable $\Theta$ with support $[0,1]$
and PDF $f_{\Theta}(\theta)$ such that\begin{equation}
\P(X_{1}=x_{1},\ldots,\, X_{k}=x_{k})=\int_{0}^{1}\theta^{\sum x_{i}}(1-\theta)^{k-\sum x_{i}}\, f_{\Theta}(\theta)\,\diff\theta,\label{eq:definetti-binary}\end{equation}
for all $x_{i}=0,\,1$, $i=1,\,2,\ldots,k$.
\end{thm}


To get a handle on the intuitive content de Finetti's theorem, imagine
that we have a \emph{bunch} of coins in our pocket with each having
its own unique value of $\theta=\P(\mbox{Heads})$. We reach into
our pocket and select a coin at random according to some probability
-- say, $f_{\Theta}(\theta)$. We take the randomly selected coin
and flip it $k$ times. 

Think carefully: the conditional probability of observing a sequence
$X_{1}=x_{1},\ldots,\, X_{k}=x_{k}$, given a specific coin $\theta$
would just be $\theta^{\sum x_{i}}(1-\theta)^{k-\sum x_{i}}$, because
the coin flips are an independent sequence of Bernoulli trials. But
the coin is random, so the Theorem of Total Probability says we can
get the \emph{unconditional} probability $\P(X_{1}=x_{1},\ldots,\, X_{k}=x_{k})$
by adding up terms that look like\begin{equation}
\theta^{\sum x_{i}}(1-\theta)^{k-\sum x_{i}}\, f_{\Theta}(\theta),\end{equation}
where we sum over all possible coins. The right-hand side of Equation
\ref{eq:definetti-binary} is a sophisticated way to denote this process.

Of course, the integral's value does not change if we jumble the $x_{i}$'s,
so $(X_{1},\ldots,X_{k})$ are clearly exchangeable. The power of
de Finetti's Theorem is that \emph{every} infinite binary exchangeable
sequence can be written in the above form.

The connection to subjective probability: our prior information about
$\theta$ corresponds to $f_{\Theta}(\theta)$ and the likelihood
of the sequence $X_{1}=x_{1},\ldots,\, X_{k}=x_{k}$ (conditional
on $\theta$) corresponds to $\theta^{\sum x_{i}}(1-\theta)^{k-\sum x_{i}}$.
Compare Equation \ref{eq:definetti-binary} to Section \ref{sec:Bayes'-Rule}
and Section \ref{sec:Conditional-Distributions}.



The multivariate normal distribution immediately generalizes from
the bivariate case. If the matrix $\Sigma$ is nonsingular then the
joint PDF of $\mathbf{X}\sim\mathsf{mvnorm}(\mathtt{mean}=\upmu,\,\mathtt{sigma}=\Sigma)$
is\begin{equation}
f_{\mathbf{X}}(\mathbf{x})=\frac{1}{(2\pi)^{n/2}\left|\Sigma\right|^{1/2}}\exp\left\{ -\frac{1}{2}\left(\mathbf{x}-\upmu\right)^{\top}\Sigma^{-1}\left(\mathbf{x}-\upmu\right)\right\} ,\end{equation}
and the MGF is\begin{equation}
M_{\mathbf{X}}(\mathbf{t})=\exp\left\{ \upmu^{\top}\mathbf{t}+\frac{1}{2}\mathbf{t}^{\top}\Sigma\mathbf{t}\right\} .\end{equation}
We will need the following in Chapter \ref{cha:multiple-linear-regression}.
\begin{thm}
\label{thm:mvnorm-dist-matrix-prod}If $\mathbf{X}\sim\mathsf{mvnorm}(\mathtt{mean}=\upmu,\,\mathtt{sigma}=\Sigma)$
and $\mathbf{A}$ is any matrix, then the random vector $\mathbf{Y}=\mathbf{AX}$
is distributed\begin{equation}
\mathbf{Y}\sim\mathsf{mvnorm}(\mathtt{mean}=\mathbf{A}\upmu,\,\mathtt{sigma}=\mathbf{A}\Sigma\mathbf{A}^{\mathrm{T}}).\end{equation}
\end{thm}
\begin{proof}
Look at the MGF of $\mathbf{Y}$:\begin{eqnarray*}
M_{\mathbf{Y}}(\mathbf{t}) & = & \E\,\exp\left\{ \mathbf{t}^{\mathrm{T}}(\mathbf{AX})\right\} ,\\
 & = & \E\,\exp\left\{ (\mathbf{A}^{\mathrm{T}}\mathbf{t})^{\mathrm{T}}\mathbf{X}\right\} ,\\
 & = & \exp\left\{ \upmu^{\mathrm{T}}(\mathbf{A}^{\top}\mathbf{t})+\frac{1}{2}(\mathbf{A}^{\mathrm{T}}\mathbf{t})^{\mathrm{T}}\Sigma(\mathbf{A}^{\mathrm{T}}\mathbf{t})\right\} ,\\
 & = & \exp\left\{ \left(\mathbf{A}\upmu\right)^{\mathrm{T}}\mathbf{t}+\frac{1}{2}\mathbf{t}^{\mathrm{T}}\left(\mathbf{A}\Sigma\mathbf{A}^{\mathrm{T}}\right)\mathbf{t}\right\} ,\end{eqnarray*}
and the last expression is the MGF of an $\mathsf{mvnorm}(\mathtt{mean}=\mathbf{A}\upmu,\,\mathtt{sigma}=\mathbf{A}\Sigma\mathbf{A}^{\mathrm{T}})$
distribution.
\end{proof}



\section{The Multinomial Distribution\label{sec:Multinomial}}

We sample $n$ times, with replacement, from an urn that contains
balls of $k$ different types. Let $X_{1}$ denote the number of balls
in our sample of type 1, let $X_{2}$ denote the number of balls of
type 2, \ldots{} , and let $X_{k}$ denote the number of balls of
type $k$. Suppose the urn has proportion $p_{1}$ of balls of type
1, proportion $p_{2}$ of balls of type 2, \ldots{}, and proportion
$p_{k}$ of balls of type $k$. Then the joint PMF of $(X_{1},\ldots,X_{k})$
is\begin{eqnarray}
f_{X_{1},\ldots,X_{k}}(x_{1},\ldots,x_{k}) & = & {n \choose x_{1}\, x_{2}\,\cdots\, x_{k}}\, p_{1}^{x_{1}}p_{2}^{x_{2}}\cdots p_{k}^{x_{k}},\end{eqnarray}
for $(x_{1},\ldots,x_{k})$ in the joint support $S_{X_{1},\ldots X_{K}}$.
We write\begin{equation}
(X_{1},\ldots,X_{k})\sim\mathsf{multinom}(\mathtt{size}=n,\,\mathtt{prob}=\mathbf{p}_{\mathrm{k}\times1}).\end{equation}


Several comments are in order. First, the joint support set $S_{X_{1},\ldots X_{K}}$
contains all nonnegative integer $k$-tuples $(x_{1},\ldots,x_{k})$
such that $x_{1}+x_{2}+\cdots+x_{k}=n$. A support set like this is
called a \emph{simplex}. Second, the proportions $p_{1}$, $p_{2}$,
\ldots{}, $p_{k}$ satisfy $p_{i}\geq0$ for all $i$ and $p_{1}+p_{2}+\cdots+p_{k}=1$.
Finally, the symbol\begin{equation}
{n \choose x_{1}\, x_{2}\,\cdots\, x_{k}}=\frac{n!}{x_{1}!\, x_{2}!\,\cdots x_{k}!}\end{equation}
is called a \emph{multinomial coefficient} which generalizes the notion
of a binomial coefficient we saw in Equation \ref{eq:binomial-coefficient}. 

The form and notation we have just described matches the \textsf{R}
usage but is not standard among other texts. Most other books use
the above for a $k-1$ dimension multinomial distribution, because
the linear constraint $x_{1}+x_{2}+\cdots+x_{k}=n$ means that once
the values of $X_{1}$, $X_{2}$, \ldots{}, $X_{k-1}$ are known
the final value $X_{k}$ is determined, not random. Another term used
for this is a \emph{singular} distribution.

For the most part we will ignore these difficulties, but the careful
reader should keep them in mind. There is not much of a difference
in practice, except that below we will use a two-dimensional support
set for a three-dimension multinomial distribution. See Figure \ref{fig:multinom-pmf2}. 

When $k=2$, we have $x_{1}=x$ and $x_{2}=n-x$, we have $p_{1}=p$
and $p_{2}=1-p$, and the multinomial coefficient is literally a binomial
coefficient. In the previous notation we have thus shown that the
$\mathsf{multinom}(\mathtt{size}=n,\,\mathtt{prob}=\mathbf{p}_{2\times1})$
distribution is the same as a $\mathsf{binom}(\mathtt{size}=n,\,\mathtt{prob}=p)$
distribution.
\begin{example}
Dinner with Barack Obama. During the 2008 U.S.~presidential primary,
Barack Obama offered to have dinner with three randomly selected monetary
contributors to his campaign. Imagine the thousands of people in the
contributor database. For the sake of argument, Suppose that the database
was approximately representative of the U.S.~population as a whole,
Suppose Barack Obama wants to have dinner \url{http://pewresearch.org/pubs/773/fewer-voters-identify-as-republicans}
36 democrat, 27 republican , 37 independent.\end{example}
\begin{rem}
Here are some facts about the multinomial distribution.
\begin{enumerate}
\item The expected value of $(X_{1},\, X_{2},\,\ldots,\, X_{k})$ is $n\mathbf{p}_{k\times1}$.
\item The variance-covariance matrix $\Sigma$ is symmetric with diagonal
entries $\sigma_{i}^{2}=np_{i}(1-p_{i})$, $i=1,\,2,\,\ldots,\, k$
and off-diagonal entries $\mbox{Cov}(X_{i},\, X_{j})=-np_{i}p_{j}$,
for $i\neq j$. The correlation between $X_{i}$ and $X_{j}$ is therefore
$\mbox{Corr}(X_{i},\, X_{j})=-\sqrt{p_{i}p_{j}/(1-p_{i})(1-p_{j})}$.
\item The marginal distribution of $(X_{1},\, X_{2},\,\ldots,\, X_{k-1})$
is $\mathsf{multinom}(\mathtt{size}=n,\,\mathtt{prob}=\mathbf{p}_{(k-1)\times1})$
with\begin{equation}
\mathbf{p}_{(k-1)\times1}=\left(p_{1},\, p_{2},\,\ldots,\, p_{k-2},\, p_{k-1}+p_{k}\right),\end{equation}
 and in particular, $X_{i}\sim\mathsf{binom}(\mathtt{size}=n,\,\mathtt{prob}=p_{i})$.
\end{enumerate}
\end{rem}

\subsection{How to do it with \textsf{R}}

There is support for the multinomial distribution in base \textsf{R},
namely in the \inputencoding{latin9}\lstinline[showstringspaces=false]!stats!\inputencoding{utf8}
package. The \inputencoding{latin9}\lstinline[showstringspaces=false]!dmultinom!\inputencoding{utf8}
function represents the PMF and the \inputencoding{latin9}\lstinline[showstringspaces=false]!rmultinom!\inputencoding{utf8}
function generates random variates.

<<>>=
library(combinat)
tmp <- t(xsimplex(3, 6))
p <- apply(tmp, MARGIN = 1, FUN = dmultinom, prob = c(36,27,37))
library(prob)
S <- probspace(tmp, probs = p)
ProbTable <- xtabs(probs ~ X1 + X2, data = S)
round(ProbTable, 3)
@

Do some examples of \inputencoding{latin9}\lstinline[showstringspaces=false]!rmultinom!\inputencoding{utf8}.

Here is another way to do it%
\footnote{Another way to do the plot is with the \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!scatterplot3d!\inputencoding{utf8}
function in the \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!scatterplot3d!\inputencoding{utf8}
package \cite{Liggesscatterplot3d}. It looks like this:
\begin{lyxcode}
library(scatterplot3d)

X~<-~t(as.matrix(expand.grid(0:6,~0:6)))

X~<-~X{[}~,~colSums(X)~<=~6{]};~X~<-~rbind(X,~6~-~colSums(X))

Z~<-~round(apply(X,~2,~function(x)~dmultinom(x,~prob~=~1:3)),~3)

A~<-~data.frame(x~=~X{[}1,~{]},~y~=~X{[}2,~{]},~probability~=~Z)

scatterplot3d(A,~type~=~{}``h'',~lwd~=~3,~box~=~FALSE)
\end{lyxcode}
The \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!scatterplot3d!\inputencoding{utf8}
graph looks better in this example, but the code is clearly more difficult
to understand. And with \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!cloud!\inputencoding{utf8}
one can easily do conditional plots of the form \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!cloud(z ~ x + y|f)!\inputencoding{utf8},
where \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!f!\inputencoding{utf8}
is a factor.%
}.

%
\begin{figure}
\begin{centering}
<<echo = FALSE, fig=true, height = 4.5, width = 6>>=
library(lattice)
print(cloud(probs ~ X1 + X2, data = S, type = c("p","h"), lwd = 2, pch = 16, cex = 1.5), screen = list(z = 15, x = -70))
@
\par\end{centering}

\caption{Plot of a multinomial PMF\label{fig:multinom-pmf2}}

\end{figure}


\newpage{}


\section*{Chapter Exercises}

\addcontentsline{toc}{section}{Chapter Exercises}
\setcounter{thm}{0}


\begin{xca}
Prove that $\mbox{Cov}(X,Y)=\E(XY)-(\E X)(\E Y).$\label{xca:Prove-cov-shortcut}
\end{xca}

\begin{xca}
\label{xca:sum-indep-chisq}Suppose $X\sim\mathsf{chisq}(\mathtt{df}=p_{1})$
and $Y\sim\mathsf{chisq}(\mathtt{df}=p_{2})$ are independent. Find
the distribution of $X+Y$ (you may want to refer to Equation \ref{eq:mgf-chisq}).
\end{xca}

\begin{xca}
\label{xca:diff-indep-norm}Show that when $X$ and $Y$ are independent
the MGF of $X-Y$ is $M_{X}(t)M_{Y}(-t)$. Use this to find the distribution
of $X-Y$ when $X\sim\mathsf{norm}(\mathtt{mean}=\mu_{1},\,\mathtt{sd}=\sigma_{1})$
and $Y\sim\mathsf{norm}(\mathtt{mean}=\mu_{2},\,\mathtt{sd}=\sigma_{2})$
are independent. 
\end{xca}



\chapter{Sampling Distributions\label{cha:Sampling-Distributions}}

\noindent This is an important chapter; it is the bridge from probability
and descriptive statistics that we studied in Chapters \ref{cha:Describing-Data-Distributions}
through \ref{cha:Multivariable-Distributions} to inferential statistics
which forms the latter part of this book.

Here is the link: we are presented with a \emph{population} about
which we would like to learn. And while it would be desirable to examine
every single member of the population, we find that it is either impossible
or infeasible to for us to do so, thus, we resort to collecting a
\emph{sample} instead. We do not lose heart. Our method will suffice,
provided the sample is \emph{representative} of the population. A
good way to achieve this is to sample \emph{randomly} from the population.

Supposing for the sake of argument that we have collected a random
sample, the next task is to make some \emph{sense} out of the data
because the complete list of sample information is usually cumbersome,
unwieldy. We summarize the data set with a descriptive \emph{statistic},
a quantity calculated from the data (we saw many examples of these
in Chapter \ref{cha:Describing-Data-Distributions}). But our sample
was random\ldots{} therefore, it stands to reason that our statistic
will be random, too. How is the statistic distributed?

The probability distribution associated with the population (from
which we sample) is called the \emph{population distribution}, and
the probability distribution associated with our statistic is called
its \emph{sampling distribution}; clearly, the two are interrelated.
To learn about the population distribution, it is imperative to know
everything we can about the sampling distribution. Such is the goal
of this chapter.

We begin by introducing the notion of simple random samples and cataloguing
some of their more convenient mathematical properties. Next we focus
on what happens in the special case of sampling from the normal distribution
(which, again, has several convenient mathematical properties), and
in particular, we meet the sampling distribution of $\Xbar$ and $S^{2}$.
Then we explore what happens to $\Xbar$'s sampling distribution when
the population is not normal and prove one of the most remarkable
theorems in statistics, the Central Limit Theorem (CLT).

With the CLT in hand, we then investigate the sampling distributions
of several other popular statistics, taking full advantage of those
with a tractable form. We finish the chapter with an exploration of
statistics whose sampling distributions are not quite so tractable,
and to accomplish this goal we will use simulation methods that are
grounded in all of our work in the previous four chapters.


\paragraph*{What do I want them to know?}
\begin{itemize}
\item the notion of population versus simple random sample, parameter versus
statistic, and population distribution versus sampling distribution
\item the classical sampling distributions of the standard one and two sample
statistics
\item how to generate a simulated sampling distribution when the statistic
is crazy
\item the Central Limit Theorem, period.
\item some basic concepts related to sampling distribution utility, such
as bias and variance
\end{itemize}

\section{Simple Random Samples\label{sec:simple-random-samples}}


\subsection{Simple Random Samples\label{sub:simple-random-samples}}
\begin{defn}
If $X_{1}$, $X_{2}$, \ldots{}, $X_{n}$ are independent with $X_{i}\sim f$
for $i=1,2,\ldots,n$, then we say that $X_{1}$, $X_{2}$, \ldots{},
$X_{n}$ are \emph{independent and identically distributed} (i.i.d.)
from the population $f$ or alternatively we say that $X_{1}$, $X_{2}$,
\ldots{}, $X_{n}$ are a \emph{simple random sample of size} $n$,
denoted $SRS(n)$, from the population $f$. \end{defn}
\begin{prop}
\label{pro:mean-sd-xbar}Let $X_{1}$, $X_{2}$, \ldots{}, $X_{n}$
be a $SRS(n)$ from a population distribution with mean $\mu$ and
finite standard deviation $\sigma$. Then the mean and standard deviation
of $\Xbar$ are given by the formulas $\mu_{\Xbar}=\mu$ and $\sigma_{\Xbar}=\sigma/\sqrt{n}$.
\end{prop}
\begin{proof}
Plug in $a_{1}=a_{2}=\cdots=a_{n}=1/n$ in Proposition \ref{pro:mean-sd-lin-comb}. 
\end{proof}


The next fact will be useful to us when it comes time to prove the
Central Limit Theorem in Section \ref{sec:Central-Limit-Theorem}.
\begin{prop}
\label{pro:mgf-xbar}Let $X_{1}$, $X_{2}$, \ldots{}, $X_{n}$ be
a $SRS(n)$ from a population distribution with MGF $M(t)$. Then
the MGF of $\Xbar$ is given by\begin{equation}
M_{\Xbar}(t)=\left[M\left(\frac{t}{n}\right)\right]^{n}.\end{equation}

\end{prop}
\begin{proof}
Go from the definition:\begin{eqnarray*}
M_{\Xbar}(t) & = & \E\,\me^{t\Xbar},\\
 & = & \E\,\me^{t(X_{1}+\cdots+X_{n})/n},\\
 & = & \E\,\me^{tX_{1}/n}\me^{tX_{2}/n}\cdots\me^{tX_{n}/n}.\end{eqnarray*}
And because $X_{1}$, $X_{2}$, \ldots{}, $X_{n}$ are independent,
Proposition \ref{pro:indep-implies-prodexpect} allows us to distribute
the expectation among each term in the product, which is\[
\E\me^{tX_{1}/n}\,\E\me^{tX_{2}/n}\cdots\E\me^{tX_{n}/n}.\]
The last step is to recognize that each term in last product above
is exactly $M(t/n)$.
\end{proof}



\section{Sampling from a Normal Distribution\label{sec:sampling-from-normal-dist}}


\subsection{The Distribution of the Sample Mean\label{sub:samp-mean-dist-of}}
\begin{prop}
Let $X_{1}$, $X_{2}$, \ldots{}, $X_{n}$ be a $SRS(n)$ from a
$\mathsf{norm}(\mathtt{mean}=\mu,\,\mathtt{sd}=\sigma)$ distribution.
Then the sample mean $\Xbar$ has a $\mathsf{norm}(\mathtt{mean}=\mu,\,\mathtt{sd}=\sigma/\sqrt{n})$
sampling distribution.
\end{prop}
\begin{proof}
The mean and standard deviation of $\Xbar$ follow directly from Proposition
\ref{pro:mean-sd-xbar}. To address the shape, first remember from
Section \ref{sec:The-Normal-Distribution} that the $\mathsf{norm}(\mathtt{mean}=\mu,\,\mathtt{sd}=\sigma)$
MGF is of the form\[
M(t)=\exp\left\{ \mu t+\sigma^{2}t^{2}/2\right\} .\]
Now use Proposition \ref{pro:mgf-xbar} to find\begin{eqnarray*}
M_{\Xbar}(t) & = & \left[M\left(\frac{t}{n}\right)\right]^{n},\\
 & = & \left[\exp\left\{ \mu(t/n)+\sigma^{2}(t/n)^{2}/2\right\} \right]^{n},\\
 & = & \exp\left\{ \, n\cdot\left[\mu(t/n)+\sigma^{2}(t/n)^{2}/2\right]\right\} ,\\
 & = & \exp\left\{ \mu t+(\sigma/\sqrt{n})^{2}t^{2}/2\right\} ,\end{eqnarray*}
and we recognize this last quantity as the MGF of a $\mathsf{norm}(\mathtt{mean}=\mu,\,\mathtt{sd}=\sigma/\sqrt{n})$
distribution.
\end{proof}



\subsection{The Distribution of the Sample Variance\label{sub:Samp-Var-Dist}}
\begin{thm}
\label{thm:Xbar-andS}Let $X_{1}$, $X_{2}$, \ldots{}, $X_{n}$
be a $SRS(n)$ from a $\mathsf{norm}(\mathtt{mean}=\mu,\,\mathtt{sd}=\sigma)$
distribution, and let\begin{equation}
\Xbar=\sum_{i=1}^{n}X_{i}\quad\mbox{and}\quad S^{2}=\frac{1}{n-1}\sum_{i=1}^{n}(X_{i}-\Xbar)^{2}.\end{equation}
Then
\begin{enumerate}
\item $\Xbar$ and $S^{2}$ are independent, and
\item The rescaled sample variance \begin{equation}
\frac{(n-1)}{\sigma^{2}}S^{2}=\frac{\sum_{i=1}^{n}(X_{i}-\Xbar)^{2}}{\sigma^{2}}\end{equation}
has \textup{a $\mathsf{chisq}(\mathtt{df}=n-1)$ sampling distribution.}
\end{enumerate}
\end{thm}
\begin{proof}
The proof is beyond the scope of the present book, but the theorem
is simply too important to be omitted. The interested reader could
consult Casella and Berger \cite{Casella2002}, or Hogg \emph{et al}
\cite{Hogg2005}. 
\end{proof}



\subsection{The Distribution of Student's $T$ Statistic\label{sub:Student's-t-Distribution}}
\begin{prop}
Let $X_{1}$, $X_{2}$, \ldots{}, $X_{n}$ be a $SRS(n)$ from a
$\mathsf{norm}(\mathtt{mean}=\mu,\,\mathtt{sd}=\sigma)$ distribution.
Then the quantity \begin{equation}
T=\frac{\Xbar-\mu}{S/\sqrt{n}}\end{equation}
 has a $\mathsf{t}(\mathtt{df}=n-1)$ sampling distribution.
\end{prop}
\begin{proof}
Divide the numerator and denominator by $\sigma$ and rewrite\[
T=\frac{\frac{\Xbar-\mu}{\sigma/\sqrt{n}}}{S/\sigma}=\frac{\frac{\Xbar-\mu}{\sigma/\sqrt{n}}}{\sqrt{\left.\frac{(n-1)S^{2}}{\sigma^{2}}\right\slash (n-1)}}.\]
Now let \[
Z=\frac{\Xbar-\mu}{\sigma/\sqrt{n}}\quad\mbox{and}\quad V=\frac{(n-1)S^{2}}{\sigma^{2}},\]
so that\begin{equation}
T=\frac{Z}{\sqrt{V/r}},\end{equation}
where $r=n-1$.

We know from Section \ref{sub:samp-mean-dist-of} that $Z\sim\mathsf{norm}(\mathtt{mean}=0,\,\mathtt{sd}=1)$
and we know from Section \ref{sub:Samp-Var-Dist} that $V\sim\mathsf{chisq}(\mathtt{df}=n-1)$.
Further, since we are sampling from a normal distribution, Theorem
\ref{thm:Xbar-andS} gives that $\Xbar$ and $S^{2}$ are independent
and by Fact \ref{fac:indep-then-function-indep} so are $Z$ and $V$.
In summary, the distribution of $T$ is the same as the distribution
of the quantity $Z/\sqrt{V/r}$, where $Z\sim\mathsf{norm}(\mathtt{mean}=0,\,\mathtt{sd}=1)$
and $V\sim\mathsf{chisq}(\mathtt{df}=r)$ are independent. This is
in fact the definition of Student's $t$ distribution.
\end{proof}


This distribution was first published by W.~S.~Gosset (1900) under
the pseudonym Student, and the distribution has consequently come
to be known as Student's $t$ distribution. The PDF of $T$ can be
derived explicitly using the techniques of Section \ref{sec:Functions-of-Continuous};
it takes the form 

\begin{equation}
f_{X}(x)=\frac{\Gamma[(r+1)/2]}{\sqrt{r\pi}\ \Gamma(r/2)}\left(1+\frac{x^{2}}{r}\right)^{-(r+1)/2},\quad-\infty<x<\infty\end{equation}


Any random variable $X$ with the preceding PDF is said to have Student's
$t$ distribution with $r$ \emph{degrees of freedom}, and we write
$X\sim\mathsf{t}(\mathtt{df}=r)$. The shape of the PDF is similar
to the normal, but the tails are considerably heavier. See Figure
\ref{fig:Student's-t-dist-vary-df}. As with the normal distribution,
there are four functions in \textsf{R} associated with the $t$ distribution,
namely \inputencoding{latin9}\lstinline[showstringspaces=false]!dt!\inputencoding{utf8},
\inputencoding{latin9}\lstinline[showstringspaces=false]!pt!\inputencoding{utf8},
\inputencoding{latin9}\lstinline[showstringspaces=false]!qt!\inputencoding{utf8},
and \inputencoding{latin9}\lstinline[showstringspaces=false]!rt!\inputencoding{utf8},
which compute the PDF, CDF, quantile function, and generate random
variates, respectively.

The code to produce Figure \ref{fig:Student's-t-dist-vary-df} is

<<eval = FALSE>>=
curve(dt(x, df = 30), from = -3, to = 3, lwd = 3, ylab = "y")
ind <- c(1, 2, 3, 5, 10)
for (i in ind) curve(dt(x, df = i), -3, 3, add = TRUE)
@

%
\begin{figure}
\begin{centering}
<<echo = FALSE, fig=true, height = 4.5, width = 6>>=
curve(dt(x, df = 30), from = -3, to = 3, lwd = 3, ylab = "y")
ind <- c(1, 2, 3, 5, 10)
for (i in ind) curve(dt(x, df = i), -3, 3, add = TRUE)
@
\par\end{centering}

\caption{Student's $t$ distribution for various degrees of freedom\label{fig:Student's-t-dist-vary-df}}

\end{figure}


Similar to that done for the normal we may define $\mathsf{t}_{\alpha}(\mathtt{df}=n-1)$
as the number on the $x$-axis such that there is exactly $\alpha$
area under the $\mathsf{t}(\mathtt{df}=n-1)$ curve to its right.
\begin{example}
Find $\mathsf{t}{}_{0.01}(\mathtt{df}=23)$ with the quantile function.
\end{example}
<<>>=
qt(0.01, df = 23, lower.tail = FALSE)
@
\begin{rem}
There are a few things to note about the $\mathtt{t}(\mathtt{df}=r)$
distribution.
\begin{enumerate}
\item The $\mathtt{t}(\mathtt{df}=1)$ distribution is the same as the $\mathsf{cauchy}(\mathtt{location}=0,\,\mathtt{scale}=1)$
distribution. The Cauchy distribution is rather pathological and is
a counterexample to many famous results. 
\item The standard deviation of $\mathsf{t}(\mathtt{df}=r)$ is undefined
(that is, infinite) unless $r>2$. When $r$ is more than 2, the standard
deviation is always bigger than one, but decreases to 1 as $r\to\infty$.
\item As $r\to\infty$, the $\mathtt{t}(\mathtt{df}=r)$ distribution approaches
the $\mathsf{norm}(\mathtt{mean}=0,\,\mathtt{sd}=1)$ distribution.
\end{enumerate}
\end{rem}

\section{The Central Limit Theorem\label{sec:Central-Limit-Theorem}}

In this section we study the distribution of the sample mean when
the underlying distribution is \emph{not} normal. We saw in Section
\ref{sec:sampling-from-normal-dist} that when $X_{1}$, $X_{2}$,
\ldots{}, $X_{n}$ is a $SRS(n)$ from a $\mathsf{norm}(\mathtt{mean}=\mu,\,\mathtt{sd}=\sigma)$
distribution then $\Xbar\sim\mathsf{norm}(\mathtt{mean}=\mu,\,\mathtt{sd}=\sigma/\sqrt{n})$.
In other words, we may say (owing to Fact \ref{fac:lin-trans-norm-is-norm})
when the underlying population is normal that the sampling distribution
of $Z$ defined by \begin{equation}
Z=\frac{\Xbar-\mu}{\sigma/\sqrt{n}}\end{equation}
is $\mathsf{norm}(\mathtt{mean}=0,\,\mathtt{sd}=1)$. 

However, there are many populations that are \emph{not} normal\ldots{}
and the statistician often finds herself sampling from such populations.
What can be said in this case? The surprising answer is contained
in the following theorem.
\begin{thm}
\textbf{\label{thm:central-limit-thrm}The Central Limit Theorem.}
Let $X_{1}$, $X_{2}$, \ldots{}, $X_{n}$ be a $SRS(n)$ from a
population distribution with mean $\mu$ and finite standard deviation
$\sigma$. Then the sampling distribution of \begin{equation}
Z=\frac{\Xbar-\mu}{\sigma/\sqrt{n}}\end{equation}
approaches a $\mathsf{norm}(\mathtt{mean}=0,\,\mathtt{sd}=1)$ distribution
as $n\to\infty$. \end{thm}
\begin{rem}
We suppose that $X_{1}$, $X_{2}$, \ldots{}, $X_{n}$ are i.i.d.,
and we learned in Section \ref{sub:simple-random-samples} that $\Xbar$
has mean $\mu$ and standard deviation $\sigma/\sqrt{n}$, so we already
knew that $Z$ has mean 0 and standard deviation 1. The beauty of
the CLT is that it addresses the \emph{shape} of $Z$'s distribution
when the sample size is large.
\end{rem}

\begin{rem}
Notice that the shape of the underlying population's distribution
is not mentioned in Theorem \ref{thm:central-limit-thrm}; indeed,
the result is true for any population that is well-behaved enough
to have a finite standard deviation. In particular, if the population
is normally distributed then we know from Section \ref{sub:samp-mean-dist-of}
that the distribution of $\Xbar$ (and $Z$ by extension) is \emph{exactly}
normal, for \emph{every} $n$.
\end{rem}

\begin{rem}
How large is {}``sufficiently large''? It is here that the shape
of the underlying population distribution plays a role. For populations
with distributions that are approximately symmetric and mound-shaped,
the samples may need to be only of size four or five, while for highly
skewed or heavy-tailed populations the samples may need to be much
larger for the distribution of the sample means to begin to show a
bell-shape. Regardless, for a given population distribution (with
finite standard deviation) the approximation tends to be better for
larger sample sizes. 
\end{rem}

\subsection{How to do it with \textsf{R}}

The \inputencoding{latin9}\lstinline[showstringspaces=false]!TeachingDemos!\inputencoding{utf8}
package \cite{Snowteachingdemos} has \inputencoding{latin9}\lstinline[showstringspaces=false]!clt.examp!\inputencoding{utf8}
and the \inputencoding{latin9}\lstinline[showstringspaces=false]!distrTeach!\inputencoding{utf8}
\cite{Ruckdescheldistr} package has \inputencoding{latin9}\lstinline[showstringspaces=false]!illustrateCLT!\inputencoding{utf8}.
Try the following at the command line (output omitted):

<<eval = FALSE>>=
library(TeachingDemos)
example(clt.examp)
@

and

<<eval = FALSE>>=
library(distrTeach)
example(illustrateCLT)
@

The \inputencoding{latin9}\lstinline[showstringspaces=false]!IPSUR!\inputencoding{utf8}
package has the functions \inputencoding{latin9}\lstinline[showstringspaces=false]!clt1!\inputencoding{utf8},
\inputencoding{latin9}\lstinline[showstringspaces=false]!clt2!\inputencoding{utf8},
and \inputencoding{latin9}\lstinline[showstringspaces=false]!clt3!\inputencoding{utf8}
(see Exercise \ref{xca:clt123} at the end of this chapter). Its purpose
is to investigate what happens to the sampling distribution of $\Xbar$
when the population distribution is mound shaped, finite support,
and skewed, namely $\mathsf{t}(\mathtt{df}=3)$, $\mathsf{unif}(\mathtt{a}=0,\,\mathtt{b}=10)$
and $\mathsf{gamma}(\mathtt{shape}=1.21,\,\mathtt{scale}=1/2.37)$,
respectively. 

For example, when the command \inputencoding{latin9}\lstinline[showstringspaces=false]!clt1()!\inputencoding{utf8}
is issued a plot window opens to show a graph of the PDF of a $\mathsf{t}(\mathtt{df}=3)$
distribution. On the display are shown numerical values of the population
mean and variance. While the students examine the graph the computer
is simulating random samples of size \inputencoding{latin9}\lstinline[showstringspaces=false]!sample.size = 2!\inputencoding{utf8}
from the \inputencoding{latin9}\lstinline[showstringspaces=false]!population = "rt"!\inputencoding{utf8}
distribution a total of \inputencoding{latin9}\lstinline[showstringspaces=false]!N.iter = 100000!\inputencoding{utf8}
times, and sample means are calculated of each sample. Next follows
a histogram of the simulated sample means, which closely approximates
the sampling distribution of $\Xbar$, see Section \ref{sec:Simulated-Sampling-Distributions}.
Also show are the sample mean and sample variance of all of the simulated
$\Xbar$s. As a final step, when the student clicks the second plot,
a normal curve with the same mean and variance as the simulated $\Xbar$s
is superimposed over the histogram. Students should compare the population
theoretical mean and variance to the simulated mean and variance of
the sampling distribution. They should also compare the shape of the
simulated sampling distribution to the shape of the normal distribution.

The three separate \inputencoding{latin9}\lstinline[showstringspaces=false]!clt1!\inputencoding{utf8},
\inputencoding{latin9}\lstinline[showstringspaces=false]!clt2!\inputencoding{utf8},
and \inputencoding{latin9}\lstinline[showstringspaces=false]!clt3!\inputencoding{utf8}
functions were written so that students could compare what happens
overall when the shape of the population distribution changes. It
would be possible to combine all three into one big function, \inputencoding{latin9}\lstinline[showstringspaces=false]!clt!\inputencoding{utf8}
which covers all three cases (and more). 


\section{Sampling Distributions of Two-Sample Statistics\label{sec:Samp-Dist-Two-Samp}}

There are often two populations under consideration, and it sometimes
of interest to compare properties between groups. To do so we take
independent samples from each population and calculate respective
sample statistics for comparison. In some simple cases the sampling
distribution of the comparison is known and easy to derive; such cases
are the subject of the present section.


\subsection{Difference of Independent Sample Means}
\begin{prop}
Let $X_{1}$, $X_{2}$, \ldots{}, $X_{n_{1}}$ be an $SRS(n_{1})$
from a $\mathsf{norm}(\mathtt{mean}=\mu_{X},\,\mathtt{sd}=\sigma_{X})$
distribution and let $Y_{1}$, $Y_{2}$, \ldots{}, $Y_{n_{2}}$ be
an $SRS(n_{2})$ from a $\mathsf{norm}(\mathtt{mean}=\mu_{Y},\,\mathtt{sd}=\sigma_{Y})$
distribution. Suppose that $X_{1}$, $X_{2}$, \ldots{}, $X_{n_{1}}$
and $Y_{1}$, $Y_{2}$, \ldots{}, $Y_{n_{2}}$ are independent samples.
Then the quantity \begin{equation}
\frac{\Xbar-\Ybar-(\mu_{X}-\mu_{Y})}{\sqrt{\left.\sigma_{X}^{2}\right\slash n_{1}+\left.\sigma_{Y}^{2}\right\slash n_{2}}}\label{eq:diff-indep-sample-means}\end{equation}
has a $\mathsf{norm}(\mathtt{mean}=0,\,\mathtt{sd}=1)$ sampling distribution.
Equivalently, $\Xbar-\Ybar$ has a $\mathsf{norm}(\mathtt{mean}=\mu_{X}-\mu_{Y},\,\mathtt{sd}=\sqrt{\left.\sigma_{X}^{2}\right\slash n_{1}+\left.\sigma_{Y}^{2}\right\slash n_{2}})$
sampling distribution. \end{prop}
\begin{proof}
We know that $\Xbar$ is $\mathsf{norm}(\mathtt{mean}=\mu_{X},\,\mathtt{sd}=\sigma_{X}/\sqrt{n_{1}})$
and we also know that $\Ybar$ is $\mathsf{norm}(\mathtt{mean}=\mu_{Y},\,\mathtt{sd}=\sigma_{Y}/\sqrt{n_{2}})$.
And since the samples $X_{1}$, $X_{2}$, \ldots{}, $X_{n_{1}}$
and $Y_{1}$, $Y_{2}$, \ldots{}, $Y_{n_{2}}$ are independent, so
too are $\Xbar$ and $\Ybar$. The distribution of their difference
is thus normal as well, and the mean and standard deviation are given
by Proposition \ref{pro:mean-sd-lin-comb-two}.\end{proof}
\begin{rem}
Even if the distribution of one or both of the samples is not normal,
the quantity in Equation \ref{eq:diff-indep-sample-means} will be
approximately normal provided both sample sizes are large. 
\end{rem}

\begin{rem}
For the special case of $\mu_{X}=\mu_{Y}$ we have shown that \begin{equation}
\frac{\Xbar-\Ybar}{\sqrt{\sigma_{X}^{2}/n_{1}+\sigma_{Y}^{2}/n_{2}}}\end{equation}
has a $\mathsf{norm}(\mathtt{mean}=0,\,\mathtt{sd}=1)$ sampling distribution,
or in other words, $\Xbar-\Ybar$ has a $\mathsf{norm}(\mathtt{mean}=0,\,\mathtt{sd}=\sqrt{\sigma_{X}^{2}/n_{1}+\sigma_{Y}^{2}/n_{2}})$
sampling distribution. This will be important when it comes time to
do hypothesis tests; see Section \ref{sec:Conf-Interv-for-Diff-Means}.
\end{rem}

\subsection{Difference of Independent Sample Proportions}
\begin{prop}
Let $X_{1}$, $X_{2}$, \ldots{}, $X_{n_{1}}$ be an $SRS(n_{1})$
from a $\mathsf{binom}(\mathtt{size}=1,\,\mathtt{prob}=p_{1})$ distribution
and let $Y_{1}$, $Y_{2}$, \ldots{}, $Y_{n_{2}}$ be an $SRS(n_{2})$
from a $\mathsf{binom}(\mathtt{size}=1,\,\mathtt{prob}=p_{2})$ distribution.
Suppose that $X_{1}$, $X_{2}$, \ldots{}, $X_{n_{1}}$ and $Y_{1}$,
$Y_{2}$, \ldots{}, $Y_{n_{2}}$ are independent samples. Define
\begin{equation}
\hat{p}_{1}=\frac{1}{n_{1}}\sum_{i=1}^{n_{1}}X_{i}\quad\mbox{and}\quad\hat{p}_{2}=\frac{1}{n_{2}}\sum_{j=1}^{n_{2}}Y_{j}.\end{equation}
Then the sampling distribution of\begin{equation}
\frac{\hat{p}_{1}-\hat{p}_{2}-(p_{1}-p_{2})}{\sqrt{\frac{p_{1}(1-p_{1})}{n_{1}}+\frac{p_{2}(1-p_{2})}{n_{2}}}}\end{equation}
approaches a $\mathsf{norm}(\mathtt{mean}=0,\,\mathtt{sd}=1)$ distribution
as both $n_{1},\, n_{2}\to\infty$. In other words, the sampling distribution
of $\hat{p}_{1}-\hat{p}_{2}$ is approximately\begin{equation}
\mathsf{norm}\left(\mathtt{mean}=p_{1}-p_{2},\,\mathtt{sd}=\sqrt{\frac{p_{1}(1-p_{1})}{n_{1}}+\frac{p_{2}(1-p_{2})}{n_{2}}}\right),\end{equation}
provided both $n_{1}$ and $n_{2}$ are sufficiently large.
\end{prop}
\begin{proof}
We know that $\hat{p}_{1}$ is approximately normal for $n_{1}$ sufficiently
large by the CLT, and we know that $\hat{p}_{2}$ is approximately
normal for $n_{2}$ sufficiently large, also by the CLT. Further,
$\hat{p}_{1}$ and $\hat{p}_{2}$ are independent since they are derived
from independent samples. And a difference of independent (approximately)
normal distributions is (approximately) normal, by Exercise \ref{xca:diff-indep-norm}%
\footnote{\begin{rem}
This does not explicitly follow, because of our cavalier use of {}``approximately''
in too many places. To be more thorough, however, would require more
concepts than we can afford at the moment. The interested reader may
consult a more advanced text, specifically the topic of weak convergence,
that is, convergence in distribution.
\end{rem}
%
}. The expressions for the mean and standard deviation follow immediately
from Proposition \ref{pro:mean-sd-lin-comb-two} combined with the
formulas for the $\mathsf{binom}(\mathtt{size}=1,\,\mathtt{prob}=p)$
distribution from Chapter \ref{cha:Discrete-Distributions}.
\end{proof}



\subsection{Ratio of Independent Sample Variances}
\begin{prop}
Let $X_{1}$, $X_{2}$, \ldots{}, $X_{n_{1}}$ be an $SRS(n_{1})$
from a $\mathsf{norm}(\mathtt{mean}=\mu_{X},\,\mathtt{sd}=\sigma_{X})$
distribution and let $Y_{1}$, $Y_{2}$, \ldots{}, $Y_{n_{2}}$ be
an $SRS(n_{2})$ from a $\mathsf{norm}(\mathtt{mean}=\mu_{Y},\,\mathtt{sd}=\sigma_{Y})$
distribution. Suppose that $X_{1}$, $X_{2}$, \ldots{}, $X_{n_{1}}$
and $Y_{1}$, $Y_{2}$, \ldots{}, $Y_{n_{2}}$ are independent samples.
Then the ratio \begin{equation}
F=\frac{\sigma_{Y}^{2}S_{X}^{2}}{\sigma_{X}^{2}S_{Y}^{2}}\end{equation}
has an $\mathsf{f}(\mathtt{df1}=n_{1}-1,\,\mathtt{df2}=n_{2}-1)$
sampling distribution.
\end{prop}
\begin{proof}
We know from Theorem \ref{thm:Xbar-andS} that $(n_{1}-1)S_{X}^{2}/\sigma_{X}^{2}$
is distributed $\mathsf{chisq}(\mathtt{df}=n_{1}-1)$ and $(n_{2}-1)S_{Y}^{2}/\sigma_{Y}^{2}$
is distributed $\mathsf{chisq}(\mathtt{df}=n_{2}-1)$. Now write \[
F=\frac{\sigma_{Y}^{2}S_{X}^{2}}{\sigma_{X}^{2}S_{Y}^{2}}=\frac{\left.(n_{1}-1)S_{Y}^{2}\right\slash (n_{1}-1)}{\left.(n_{2}-1)S_{Y}^{2}\right\slash (n_{2}-1)}\cdot\frac{\left.1\right\slash \sigma_{X}^{2}}{\left.1\right\slash \sigma_{Y}^{2}},\]
by multiplying and dividing the numerator with $n_{1}-1$ and doing
likewise for the denominator with $n_{2}-1$. Now we may regroup the
terms into \[
F=\frac{\left.\frac{(n_{1}-1)S_{X}^{2}}{\sigma_{X}^{2}}\right\slash (n_{1}-1)}{\left.\frac{(n_{2}-1)S_{Y}^{2}}{\sigma_{Y}^{2}}\right\slash (n_{2}-1)},\]
and we recognize $F$ to be the ratio of independent $\mathsf{chisq}$
distributions, each divided by its respective numerator $\mathtt{df}=n_{1}-1$
and denominator $\mathtt{df}=n_{1}-1$ degrees of freedom. This is,
indeed, the definition of Snedecor's $F$ distribution. 
\end{proof}

\begin{rem}
For the special case of $\sigma_{X}=\sigma_{Y}$ we have shown that
\begin{equation}
F=\frac{S_{X}^{2}}{S_{Y}^{2}}\end{equation}
has an $\mathsf{f}(\mathtt{df1}=n_{1}-1,\,\mathtt{df2}=n_{2}-1)$
sampling distribution. This will be important in Chapters \ref{cha:Estimation}
onward.
\end{rem}

\section{Simulated Sampling Distributions\label{sec:Simulated-Sampling-Distributions}}

Some comparisons are meaningful, but their sampling distribution is
not quite so tidy to describe analytically. What do we do then?

As it turns out, we do not need to know the exact analytical form
of the sampling distribution; sometimes it is enough to approximate
it with a simulated distribution. In this section we will show you
how. Note that \textsf{R} is particularly well suited to compute simulated
sampling distributions, much more so than, say, SPSS or SAS.


\subsection{The Interquartile Range}

<<>>=
iqrs <- replicate(100, IQR(rnorm(100)))
@

We can look at the mean of the simulated values

<<keep.source = TRUE>>=
mean(iqrs)    # close to 1
@

and we can see the standard deviation

<<>>=
sd(iqrs)
@

Now let's take a look at a plot of the simulated values

%
\begin{figure}
\begin{centering}
<<echo = FALSE, fig=true, height = 4, width = 6>>=
hist(iqrs, breaks = 20)
@
\par\end{centering}

\caption{Plot of simulated IQRs\label{fig:simulated-IQR}}

\end{figure}



\subsection{The Median Absolute Deviation}

<<>>=
mads <- replicate(100, mad(rnorm(100)))
@

We can look at the mean of the simulated values

<<keep.source = TRUE>>=
mean(mads)    # close to 1.349
@

and we can see the standard deviation

<<>>=
sd(mads)
@

Now let's take a look at a plot of the simulated values

%
\begin{figure}[H]
\begin{centering}
<<echo = FALSE, fig=TRUE, height = 4, width = 6>>=
hist(mads, breaks = 20)
@
\par\end{centering}

\caption{Plot of simulated MADs\label{fig:simulated-MAD}}

\end{figure}


\newpage{}


\section*{Chapter Exercises}

\addcontentsline{toc}{section}{Chapter Exercises}
\setcounter{thm}{0}

<<echo = FALSE, results = hide>>=
k = 1
n = sample(10:30, size=10, replace = TRUE)
mu = round(rnorm(10, mean = 20))
@
\begin{xca}
Suppose that we observe a random sample $X_{1}$, $X_{2}$, \ldots{},
$X_{n}$ of size $SRS(n=$\Sexpr{n[k]}) from a $\mathsf{norm}(\mathtt{mean}=$\Sexpr{mu[k]})
distribution. \end{xca}
\begin{enumerate}
\item What is the mean of $\Xbar$?
\item What is the standard deviation of $\Xbar$?
\item What is the distribution of $\Xbar$? (approximately)
\item Find $\P(a<\Xbar\leq b)$
\item Find $\P(\Xbar>c)$.\end{enumerate}
\begin{xca}
\label{xca:clt123}In this exercise we will investigate how the shape
of the population distribution affects the time until the distribution
of $\Xbar$ is acceptably normal.
\end{xca}
Answer the questions and write a report about what you have learned.
Use plots and histograms to support your conclusions. See Appendix
\ref{cha:Writing-Reports-with} for instructions about writing reports
with \textsf{R}. For these problems, the discussion/interpretation
parts are the most important, so be sure to ANSWER THE WHOLE QUESTION.
\vspace{0.02in}



\subsection*{The Central Limit Theorem}

For Questions 1-3, we assume that we have observed random variables
$X_{1}$, $X_{2}$, \ldots{},$X_{n}$ that are an $SRS(n)$ from
a given population (depending on the problem) and we want to investigate
the distribution of $\Xbar$ as the sample size $n$ increases. 
\begin{enumerate}
\item The population of interest in this problem has a Student's $t$ distribution
with $r=3$ degrees of freedom. We begin our investigation with a
sample size of $n=2$. Open an \textsf{R} session, make sure to type
\inputencoding{latin9}\lstinline[showstringspaces=false]!library(IPSUR)!\inputencoding{utf8}
and then follow that with \inputencoding{latin9}\lstinline[showstringspaces=false]!clt1()!\inputencoding{utf8}. 

\begin{enumerate}
\item Look closely and thoughtfully at the first graph. How would you describe
the population distribution? Think back to the different properties
of distributions in Chapter \ref{cha:Describing-Data-Distributions}.
Is the graph symmetric? Skewed? Does it have heavy tails or thin tails?
What else can you say?
\item What is the population mean $\mu$ and the population variance $\sigma^{2}$?
(Read these from the first graph.)
\item The second graph shows (after a few seconds) a relative frequency
histogram which closely approximates the distribution of $\Xbar$.
Record the values of \inputencoding{latin9}\lstinline[showstringspaces=false]!mean(xbar)!\inputencoding{utf8}
and \inputencoding{latin9}\lstinline[showstringspaces=false]!var(xbar)!\inputencoding{utf8},
where \inputencoding{latin9}\lstinline[showstringspaces=false]!xbar!\inputencoding{utf8}
denotes the vector that contains the simulated sample means. Use the
answers from part (b) to calculate what these estimates \emph{should}
be, based on what you know about the theoretical mean and variance
of $\Xbar$. How well do your answers to parts (b) and (c) agree?
\item Click on the histogram to superimpose a red normal curve, which is
the theoretical limit of the distribution of $\Xbar$ as $n\to\infty$.
How well do the histogram and the normal curve match? Describe the
differences between the two distributions. When judging between the
two, do not worry so much about the scale (the graphs are being rescaled
automatically, anyway). Rather, look at the peak: does the histogram
poke through the top of the normal curve? How about on the sides:
are there patches of white space between the histogram and line on
either side (or both)? How do the curvature of the histogram and the
line compare? Check down by the tails: does the red line drop off
visibly below the level of the histogram, or do they taper off at
the same height? 
\item We can increase our sample size from 2 to 11 with the command \inputencoding{latin9}\lstinline[showstringspaces=false]!clt1(sample.size = 11)!\inputencoding{utf8}.
Return to the command prompt to do this. Answer parts (b) and (c)
for this new sample size.
\item Go back to \inputencoding{latin9}\lstinline[showstringspaces=false]!clt1!\inputencoding{utf8}
and increase the \inputencoding{latin9}\lstinline[showstringspaces=false]!sample.size!\inputencoding{utf8}
from 11 to 31. Answer parts (b) and (c) for this new sample size.
\item Comment on whether it appears that the histogram and the red curve
are {}``noticeably different'' or whether they are {}``essentially
the same'' for the largest sample size $n=31$. If they are still
{}``noticeably different'' at $n=31$, how large does $n$ need
to be until they are {}``essentially the same''? (Experiment with
different values of $n$).
\end{enumerate}
\item Repeat Question 1 for the function \inputencoding{latin9}\lstinline[showstringspaces=false]!clt2!\inputencoding{utf8}.
In this problem, the population of interest has a $\mathsf{unif}(\mathtt{min}=0,\,\mathtt{max}=10)$
distribution.
\item Repeat Question 1 for the function \inputencoding{latin9}\lstinline[showstringspaces=false]!clt3!\inputencoding{utf8}.
In this problem, the population of interest has a $\mathsf{gamma}(\mathtt{shape}=1.21,\,\mathtt{rate}=1/2.37)$
distribution.
\item Summarize what you have learned. In your own words, what is the general
trend that is being displayed in these histograms, as the sample size
$n$ increases from 2 to 11, on to 31, and onward?
\item How would you describe the relationship between the \textbf{\emph{shape}}
of the population distribution and the \textbf{\emph{speed}} at which
$\Xbar$'s distribution converges to normal? In particular, consider
a population which is highly \textbf{skewed}. Will we need a relatively
\emph{large} sample size or a relatively \emph{small} sample size
in order for $\Xbar$'s distribution to be approximately bell shaped?
\end{enumerate}

\begin{xca}
Let $X_{1}$,\ldots{}, $X_{25}$ be a random sample from a $\mathsf{norm}(\mathtt{mean}=37,\,\mathtt{sd}=45)$
distribution, and let $\Xbar$ be the sample mean of these $n=25$
observations. Find the following probabilities.
\begin{enumerate}
\item How is $\Xbar$ distributed? 


$\mathsf{norm}(\mathtt{mean}=37,\,\mathtt{sd}=45/\sqrt{25})$ 

\item Find $\P(\Xbar>43.1)$.


<<>>=
pnorm(43.1, mean = 37, sd = 9, lower.tail = FALSE)
@

\end{enumerate}
\end{xca}



\chapter{Estimation\label{cha:Estimation}}

\noindent We will discuss two branches of estimation procedures: point
estimation and interval estimation. We briefly discuss point estimation
first and then spend the rest of the chapter on interval estimation.

We find an estimator with the methods of Section \ref{sec:Point-Estimation-1}.
We make some assumptions about the underlying population distribution
and use what we know from Chapter \ref{cha:Sampling-Distributions}
about sampling distributions both to study how the estimator will
perform, and to find intervals of confidence for underlying parameters
associated with the population distribution. Once we have confidence
intervals we can do inference in the form of hypothesis tests in the
next chapter.


\paragraph*{What do I want them to know?}
\begin{itemize}
\item how to look at a problem, identify a reasonable model, and estimate
a parameter associated with the model
\item about maximum likelihood, and in particular, how to

\begin{itemize}
\item eyeball a likelihood to get a maximum
\item use calculus to find an MLE for one-parameter families
\end{itemize}
\item about properties of the estimators they find, such as bias, minimum
variance, MSE
\item point versus interval estimation, and how to find and interpret confidence
intervals for basic experimental designs
\item the concept of margin of error and its relationship to sample size
\end{itemize}

\section{Point Estimation\label{sec:Point-Estimation-1}}

The following example is how I was introduced to maximum likelihood.
\begin{example}
\label{exa:how-many-fish}Suppose we have a small pond in our backyard,
and in the pond there live some fish. We would like to know how many
fish live in the pond. How can we estimate this? One procedure developed
by researchers is the capture-recapture method. Here is how it works.

We will fish from the pond and suppose that we capture $M=7$ fish.
On each caught fish we attach an unobtrusive tag to the fish's tail,
and release it back into the water. 

Next, we wait a few days for the fish to remix and become accustomed
to their new tag. Then we go fishing again. On the second trip some
of the fish we catch may be tagged; some may not be. Let $X$ denote
the number of caught fish which are tagged%
\footnote{It is theoretically possible that we could catch the same tagged fish
more than once, which would inflate our count of tagged fish. To avoid
this difficulty, suppose that on the second trip we use a tank on
the boat to hold the caught fish until data collection is completed.%
}, and suppose for the sake of argument that we catch $K=4$ fish and
we find that 3 of them are tagged.

Now let $F$ denote the (unknown) total number of fish in the pond.
We know that $F\geq7$, because we tagged that many on the first trip.
In fact, if we let $N$ denote the number of untagged fish in the
pond, then $F=M+N$. We have sampled $K=4$ times, without replacement,
from an urn which has $M=7$ white balls and $N=F-M$ black balls,
and we have observed $x=3$ of them to be white. What is the probability
of this?

Looking back to Section \ref{sec:other-discrete-distributions}, we
see that the random variable $X$ has a $\mathsf{hyper}(\mathtt{m}=M,\,\mathtt{n}=F-M,\,\mathtt{k}=K)$
distribution. Therefore, for an observed value $X=x$ the probability
would be\[
\P(X=x)=\frac{{M \choose x}{F-M \choose K-x}}{{F \choose K}}.\]


First we notice that $F$ must be at least 7. Could $F$ be equal
to seven? If $F=7$ then all of the fish would have been tagged on
the first run, and there would be no untagged fish in the pond, thus,
$\P(\mbox{3 successes in 4 trials})=0$.

What about $F=8$; what would be the probability of observing $X=3$
tagged fish?\[
\P(\mbox{3 successes in 4 trials})=\frac{{7 \choose 3}{1 \choose 1}}{{8 \choose 4}}=\frac{35}{70}=0.5.\]


Similarly, if $F=9$ then the probability of observing $X=3$ tagged
fish would be\[
\P(\mbox{3 successes in 4 trials})=\frac{{7 \choose 3}{2 \choose 1}}{{9 \choose 4}}=\frac{70}{126}\approx0.556.\]
We can see already that the observed data $X=3$ is more likely when
$F=9$ than it is when $F=8$. And here lies the genius of Sir Ronald
Aylmer Fisher: he asks, {}``What is the value of $F$ which has the
highest likelihood?'' In other words, for all of the different possible
values of $F$, which one makes the above probability the biggest?
We can answer this question with a plot of $\P(X=x)$ versus $F$.
See Figure \ref{fig:capture-recapture}.
\end{example}
%
\begin{figure}
\begin{centering}
<<echo = FALSE, fig=true, height = 4.5, width = 6>>=
heights = rep(0, 16)
for (j in 7:15) heights[j] <- dhyper(3, m = 7, n = j - 7, k = 4)
plot(6:15, heights[6:15], pch = 16, cex = 1.5, xlab = "number of fish in pond", ylab = "Likelihood")
abline(h = 0)
lines(6:15, heights[6:15], type = "h", lwd = 2, lty = 3)
text(9, heights[9]/6, bquote(hat(F)==.(9)), cex = 2, pos = 4)
lines(9, heights[9], type = "h", lwd = 2)
points(9, 0, pch = 4, lwd = 3, cex = 2) 
@
\par\end{centering}

\caption{Capture-recapture experiment\label{fig:capture-recapture}}

\end{figure}



\begin{example}
\label{exa:bass-bluegill}In the last example we were only concerned
with how many fish were in the pond, but now, we will ask a different
question. Suppose it is known that there are only two species of fish
in the pond: smallmouth bass (\emph{Micropterus dolomieu}) and bluegill
(\emph{Lepomis macrochirus}); perhaps we built the pond some years
ago and stocked it with only these two species. We would like to estimate
the proportion of fish in the pond which are bass.

Let $p=\mbox{the proportion of bass}$. Without any other information,
it is conceivable for $p$ to be any value in the interval $[0,1]$,
but for the sake of argument we will suppose that $p$ falls strictly
between 0 and 1. How can we learn about the true value of $p$? Go
fishing! As before, we will use catch-and-release, but unlike before,
we will not tag the fish. We will simply note the species of any caught
fish before returning it to the pond.

Suppose we catch $n$ fish. Let\[
X_{i}=\begin{cases}
1, & \mbox{if the \mbox{i}\mbox{th} fish is a bass,}\\
0, & \mbox{if the \mbox{i}\mbox{th} fish is a bluegill.}\end{cases}\]


Since we are returning the fish to the pond once caught, we may think
of this as a sampling scheme with replacement where the proportion
of bass $p$ does not change. Given that we allow the fish sufficient
time to {}``mix'' once returned, it is not completely unreasonable
to model our fishing experiment as a sequence of Bernoulli trials,
so that the $X_{i}$'s would be i.i.d.~$\mathsf{binom(\mathtt{size}}=1,\,\mathtt{prob}=p)$.
Under those assumptions we would have \begin{eqnarray*}
\P(X_{1}=x_{1},\, X_{2}=x_{2},\,\ldots,\, X_{n}=x_{n}) & = & \P(X_{1}=x_{1})\,\P(X_{2}=x_{2})\,\cdots\P(X_{n}=x_{n}),\\
 & = & p^{x_{1}}(1-p)^{x_{1}}\, p^{x_{2}}(1-p)^{x_{2}}\cdots\, p^{x_{n}}(1-p)^{x_{n}},\\
 & = & p^{\sum x_{i}}(1-p)^{n-\sum x_{i}}.\end{eqnarray*}
That is, \[
\P(X_{1}=x_{1},\, X_{2}=x_{2},\,\ldots,\, X_{n}=x_{n})=p^{\sum x_{i}}(1-p)^{n-\sum x_{i}}.\]
This last quantity is a function of $p$, called the \emph{likelihood
function} $L(p)$:\[
L(p)=p^{\sum x_{i}}(1-p)^{n-\sum x_{i}}.\]
A graph of $L$ for values of $\sum x_{i}=3,\ 4$, and 5 when $n=7$
is shown in Figure \ref{fig:fishing-part-two}. 

<<eval = FALSE>>=
curve(x^5*(1-x)^2, from = 0, to = 1, xlab = "p", ylab = "L(p)")
curve(x^4*(1-x)^3, from = 0, to = 1, add = TRUE)
curve(x^3*(1-x)^4, 0, 1, add = TRUE)
@

%
\begin{figure}
\begin{centering}
<<echo = FALSE, fig=true, height = 4.5, width = 6>>=
curve(x^5*(1-x)^2, 0, 1, xlab = "p", ylab = "L(p)")
curve(x^4*(1-x)^3, 0, 1, add = TRUE)
curve(x^3*(1-x)^4, 0, 1, add = TRUE)
@
\par\end{centering}

\caption{Assorted likelihood functions for fishing, part two\label{fig:fishing-part-two}}


{\small Three graphs are shown of $L$ when $\sum x_{i}$ equals 3,
4, and 5, respectively, from left to right. We pick an $L$ that matches
the observed data and then maximize $L$ as a function of $p$. If
$\sum x_{i}=4$, then the maximum appears to occur somewhere around
$p\approx0.6$.}
\end{figure}


We want the value of $p$ which has the highest likelihood, that is,
we again wish to maximize the likelihood. We know from calculus (see
Appendix \ref{sec:Differential-and-Integral}) to differentiate $L$
and set $L'=0$ to find a maximum.\[
L'(p)=\left(\sum x_{i}\right)p^{\sum x_{i}-1}(1-p)^{n-\sum x_{i}}+p^{\sum x_{i}}\left(n-\sum x_{i}\right)(1-p)^{n-\sum x_{i}-1}(-1).\]
 The derivative vanishes ($L'=0$) when\begin{eqnarray*}
\left(\sum x_{i}\right)p^{\sum x_{i}-1}(1-p)^{n-\sum x_{i}} & = & p^{\sum x_{i}}\left(n-\sum x_{i}\right)(1-p)^{n-\sum x_{i}-1},\\
\sum x_{i}(1-p) & = & \left(n-\sum x_{i}\right)p,\\
\sum x_{i}-p\sum x_{i} & = & np-p\sum x_{i},\\
\frac{1}{n}\sum_{i=1}^{n}x_{i} & = & p.\end{eqnarray*}
This {}``best'' $p$, the one which maximizes the likelihood, is
called the maximum likelihood estimator (MLE) of $p$ and is denoted
$\hat{p}$. That is, \begin{equation}
\hat{p}=\frac{\sum_{i=1}^{n}x_{i}}{n}=\xbar.\end{equation}

\begin{rem}
Strictly speaking we have only shown that the derivative equals zero
at $\hat{p}$, so it is theoretically possible that the critical value
$\hat{p}=\xbar$ is located at a minimum%
\footnote{We can tell from the graph that our value of $\hat{p}$ is a maximum
instead of a minimum so we do not really need to worry for this example.
Other examples are not so easy, however, and we should be careful
to be cognizant of this extra step.%
} instead of a maximum! We should be thorough and check that $L'>0$
when $p<\xbar$ and $L'<0$ when $p>\xbar$. Then by the First Derivative
Test (Theorem \ref{thm:First-Derivative-Test}) we could be certain
that $\hat{p}=\xbar$ is indeed a maximum likelihood estimator, and
not a minimum likelihood estimator.
\end{rem}
The result is shown in Figure \ref{fig:species-mle}.
\end{example}
%
\begin{figure}
\begin{centering}
<<echo = FALSE, fig=true, height = 4.5, width = 6>>=
dat <- rbinom(27, size = 1, prob = 0.3)
like <- function(x){
r <- 1
for (k in 1:27){ r <- r*dbinom(dat[k], size = 1, prob = x)}
return(r)
}
curve(like, from = 0, to = 1, xlab = "parameter space", ylab = "Likelihood", lwd = 3, col = "blue")
abline(h = 0, lwd = 1, lty = 3, col = "grey")
mle <- mean(dat)
mleobj <- like(mle)
lines(mle, mleobj, type = "h", lwd = 2, lty = 3, col = "red")
points(mle, 0, pch = 4, lwd = 2, cex = 2, col = "red")
text(mle, mleobj/6, substitute(hat(theta)==a, list(a=round(mle, 4))), cex = 2, pos = 4)
@
\par\end{centering}

\caption{Species maximum likelihood\label{fig:species-mle}}

\end{figure}


In general, we have a family of PDFs $f(x|\theta)$ indexed by a parameter
$\theta$ in some parameter space $\Theta$. We want to learn about
$\theta$. We take a $SRS(n)$:\begin{equation}
X_{1},\, X_{2},\,\ldots,X_{n}\mbox{ which are i.i.d.\,\mbox{f(x|\ensuremath{\theta})}.}\end{equation}

\begin{defn}
Given the observed data $x_{1}$, $x_{2}$, \ldots{}, $x_{n}$, the
\emph{likelihood function} $L$ is defined by \[
L(\theta)=\prod_{i=1}^{n}f(x_{i}|\theta),\quad\theta\in\Theta.\]

\end{defn}


The next step is to maximize $L$. The method we will use in this
book is to find the derivative $L'$ and solve the equation $L'(\theta)=0$.
Call a solution $\hat{\theta}$. We will check that $L$ is maximized
at $\hat{\theta}$ using the First Derivative Test or the Second Derivative
Test $\left(L''(\hat{\theta})<0\right)$.
\begin{defn}
A value $\theta$ that maximizes $L$ is called a \emph{maximum likelihood
estimator} (MLE) and is denoted $\hat{\theta}$. It is a function
of the sample, $\hat{\theta}=\hat{\theta}\left(X_{1},\, X_{2},\,\ldots,X_{n}\right)$,
and is called a \emph{point estimator} of $\theta$.\end{defn}
\begin{rem}
Some comments about maximum likelihood estimators:
\begin{itemize}
\item Often it is easier to maximize the \emph{log-likelihood} $l(\theta)=\ln L(\theta)$
instead of the likelihood $L$. Since the logarithmic function $y=\ln x$
is a monotone transformation, the solutions to both problems are the
same.
\item MLEs do not always exist (for instance, sometimes the likelihood has
a vertical asymptote), and even when they do exist, they are not always
unique (imagine a function with a bunch of humps of equal height).
For any given problem, there could be zero, one, or any number of
values of $\theta$ for which $L(\theta)$ is a maximum.
\item The problems we encounter in this book are all very nice with likelihood
functions that have closed form representations and which are optimized
by some calculus acrobatics. In practice, however, likelihood functions
are sometimes nasty in which case we are obliged to use numerical
methods to find maxima (if there are any).
\item MLEs are just one of \underbar{many} possible estimators. One of the
more popular alternatives are the \emph{method of moments estimators};
see Casella and Berger \cite{Casella2002} for more.
\end{itemize}
\end{rem}
Notice, in Example \ref{exa:bass-bluegill} we had $X_{i}$ i.i.d.~$\mathsf{binom}(\mathtt{size}=1,\,\mathtt{prob}=p)$,
and we saw that the MLE was $\hat{p}=\Xbar$. But further\begin{eqnarray*}
\E\Xbar & = & \E\frac{X_{1}+X_{2}+\cdots+X_{n}}{n},\\
 & = & \frac{1}{n}\left(\E X_{1}+\E X_{2}+\cdots+\E X_{n}\right),\\
 & = & \frac{1}{n}\left(np\right),\\
 & = & p,\end{eqnarray*}
which is exactly the same as the parameter which we estimated. More
concisely, $\E\hat{p}=p$, that is, on the average, the estimator
is exactly right.
\begin{defn}
Let $s(X_{1},X_{2},\ldots,X_{n})$ be a statistic which estimates
$\theta$. If \[
\E s(X_{1},X_{2},\ldots,X_{n})=\theta,\]
then the statistic $s(X_{1},X_{2},\ldots,X_{n})$ is said to be an
\emph{unbiased estimator} of $\theta$. Otherwise, it is \emph{biased}.\end{defn}
\begin{example}
\label{exa:normal-MLE-both}Let $X_{1}$, $X_{2}$, \ldots{}, $X_{n}$
be an $SRS(n)$ from a $\mathsf{norm}(\mathtt{mean}=\mu,\,\mathtt{sd}=\sigma)$
distribution. It can be shown (in Exercise \ref{xca:norm-mu-sig-MLE})
that if $\mbox{\ensuremath{\theta}}=(\mu,\sigma^{2})$ then the MLE
of $\theta$ is\begin{equation}
\hat{\theta}=(\hat{\mu},\hat{\sigma}^{2}),\end{equation}
 where $\hat{\mu}=\Xbar$ and \begin{equation}
\hat{\sigma^{2}}=\frac{1}{n}\sum_{i=1}^{n}\left(X_{i}-\Xbar\right)^{2}=\frac{n-1}{n}S^{2}.\end{equation}
We of course know from \ref{pro:mean-sd-xbar} that $\hat{\mu}$ is
unbiased. What about $\hat{\sigma^{2}}$? Let us check: \begin{eqnarray*}
\E\,\hat{\sigma^{2}} & = & \E\,\frac{n-1}{n}S^{2}\\
 & = & \E\left(\frac{\sigma^{2}}{n}\frac{(n-1)S^{2}}{\sigma^{2}}\right)\\
 & = & \frac{\sigma^{2}}{n}\E\ \mathsf{chisq}(\mathtt{df}=n-1)\\
 & = & \frac{\sigma^{2}}{n}(n-1),\end{eqnarray*}
from which we may conclude two things:
\begin{enumerate}
\item $\hat{\sigma^{2}}$ is a biased estimator of $\sigma^{2}$, and 
\item $S^{2}=n\hat{\sigma^{2}}/(n-1)$ is an unbiased estimator of $\sigma^{2}$.
\end{enumerate}
\end{example}
One of the most common questions in an introductory statistics class
is, {}``Why do we divide by $n-1$ when we compute the sample variance?
Why do we not divide by $n$?'' We see now that division by $n$
amounts to the use of a \emph{biased} estimator for $\sigma^{2}$,
that is, if we divided by $n$ then on the average we would \emph{underestimate}
the true value of $\sigma^{2}$. We use $n-1$ so that, on the average,
our estimator of $\sigma^{2}$ will be exactly right. 


\subsection{How to do it with \textsf{R}}

\textsf{R} can be used to find maximum likelihood estimators in a
lot of diverse settings. We will discuss only the most basic here
and will leave the rest to more sophisticated texts.

For one parameter estimation problems we may use the \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!optimize!\inputencoding{utf8}
function to find MLEs. The arguments are the function to be maximized
(the likelihood function), the range over which the optimization is
to take place, and optionally any other arguments to be passed to
the likelihood if needed.

Let us see how to do Example \ref{exa:bass-bluegill}. Recall that
our likelihood function was given by\begin{equation}
L(p)=p^{\sum x_{i}}(1-p)^{n-\sum x_{i}}.\end{equation}
Notice that the likelihood is just a product of $\mathsf{binom}(\mathtt{size}=1,\,\mathtt{prob}=p)$
PMFs. We first give some sample data (in the vector \inputencoding{latin9}\lstinline[basicstyle={\ttfamily},showstringspaces=false]!datavals!\inputencoding{utf8}),
next we define the likelihood function \inputencoding{latin9}\lstinline[basicstyle={\ttfamily},showstringspaces=false]!L!\inputencoding{utf8},
and finally we \inputencoding{latin9}\lstinline[basicstyle={\ttfamily},showstringspaces=false]!optimize!\inputencoding{utf8}
\inputencoding{latin9}\lstinline[basicstyle={\ttfamily},showstringspaces=false]!L!\inputencoding{utf8}
over the range \inputencoding{latin9}\lstinline[basicstyle={\ttfamily},showstringspaces=false]!c(0,1)!\inputencoding{utf8}.

<<>>=
x <- mtcars$am
L <- function(p,x) prod(dbinom(x, size = 1, prob = p))
optimize(L, interval = c(0,1), x = x, maximum = TRUE)
@

<<echo = FALSE>>=
A <- optimize(L, interval = c(0,1), x = x, maximum = TRUE)
@

Note that the \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!optimize!\inputencoding{utf8}
function by default minimizes the function \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!L!\inputencoding{utf8},
so we have to set \inputencoding{latin9}\lstinline[basicstyle={\ttfamily},showstringspaces=false]!maximum = TRUE!\inputencoding{utf8}
to get an MLE. The returned value of \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!$maximum!\inputencoding{utf8}
gives an approximate value of the MLE to be \Sexpr{round(A$maximum, 3)}
and \inputencoding{latin9}\lstinline[basicstyle={\ttfamily},showstringspaces=false]!$objective!\inputencoding{utf8}
gives \inputencoding{latin9}\lstinline[basicstyle={\ttfamily},showstringspaces=false]!L!\inputencoding{utf8}
evaluated at the MLE which is approximately \Sexpr{round(A$objective, 3)}.

We previously remarked that it is usually more numerically convenient
to maximize the log-likelihood (or minimize the negative log-likelihood),
and we can just as easily do this with \textsf{R}. We just need to
calculate the log-likelihood beforehand which (for this example) is\[
-l(p)=-\sum x_{i}\ln\, p-\left(n-\sum x_{i}\right)\ln(1-p).\]
It is done in \textsf{R} with

<<>>=
minuslogL <- function(p,x) -sum(dbinom(x, size = 1, prob = p, log = TRUE))
optimize(minuslogL, interval = c(0,1), x = x)
@

Note that we did not need \inputencoding{latin9}\lstinline[basicstyle={\ttfamily},showstringspaces=false]!maximum = TRUE!\inputencoding{utf8}
because we minimized the negative log-likelihood. The answer for the
MLE is essentially the same as before, but the \inputencoding{latin9}\lstinline[basicstyle={\ttfamily},showstringspaces=false]!$objective!\inputencoding{utf8}
value was different, of course.

For multiparameter problems we may use a similar approach by way of
the \inputencoding{latin9}\lstinline[basicstyle={\ttfamily},showstringspaces=false]!mle!\inputencoding{utf8}
function in the \inputencoding{latin9}\lstinline[basicstyle={\ttfamily},showstringspaces=false]!stats4!\inputencoding{utf8}
package. 
\begin{example}
\textbf{Plant Growth.} We will investigate the \inputencoding{latin9}\lstinline[basicstyle={\ttfamily},showstringspaces=false]!weight!\inputencoding{utf8}
variable of the \inputencoding{latin9}\lstinline[basicstyle={\ttfamily},showstringspaces=false]!PlantGrowth!\inputencoding{utf8}
data. We will suppose that the weights constitute a random observations
$X_{1}$, $X_{2}$,\ldots{}, $X_{n}$ that are i.i.d.~$\mathsf{norm}(\mathtt{mean}=\mu,\,\mathtt{sd}=\sigma)$
which is not unreasonable based on a histogram and other exploratory
measures. We will find the MLE of $\theta=(\mu,\sigma^{2})$. We claimed
in Example \ref{exa:normal-MLE-both} that $\hat{\theta}=(\hat{\mu},\hat{\sigma}^{2})$
had the form given above. Let us check whether this is plausible numerically.
The negative log-likelihood function is

<<keep.source = TRUE>>=
minuslogL <- function(mu, sigma2){
  -sum(dnorm(x, mean = mu, sd = sqrt(sigma2), log = TRUE))
}
@

Note that we omitted the data as an argument to the log-likelihood
function; the only arguments were the parameters over which the maximization
is to take place. Now we will simulate some data and find the MLE.
The optimization algorithm requires starting values (intelligent guesses)
for the parameters. We choose values close to the sample mean and
variance (which turn out to be approximately 5 and 0.5, respectively)
to illustrate the procedure.

<<>>=
x <- PlantGrowth$weight
library(stats4)
MaxLikeEst <- mle(minuslogL, start = list(mu = 5, sigma2 = 0.5))
summary(MaxLikeEst)
@

The outputted MLEs are shown above, and \inputencoding{latin9}\lstinline[basicstyle={\ttfamily},showstringspaces=false]!mle!\inputencoding{utf8}
even gives us estimates for the standard errors of $\hat{\mu}$ and
$\hat{\sigma}^{2}$ (which were obtained by inverting the numerical
Hessian matrix at the optima; see Appendix \ref{sec:Multivariable-Calculus}).
Let us check how close the numerical MLEs came to the theoretical
MLEs:

<<>>=
mean(x)
var(x)*29/30
sd(x)/sqrt(30)
@

The numerical MLEs were very close to the theoretical MLEs. We already
knew that the standard error of $\hat{\mu}=\Xbar$ is $\sigma/\sqrt{n}$,
and the numerical estimate of this was very close too.

\end{example}
There is functionality in the \inputencoding{latin9}\lstinline[basicstyle={\ttfamily},showstringspaces=false]!distrTest!\inputencoding{utf8}
package \cite{Ruckdescheldistr} to calculate theoretical MLEs; we
will skip examples of these for the time being.


\section{Confidence Intervals for Means\label{sec:Confidence-Intervals-for-Means}}

We are given $X_{1}$, $X_{2}$, \ldots{}, $X_{n}$ that are an $SRS(n)$
from a $\mathsf{norm}(\mathtt{mean}=\mu,\,\mathtt{sd}=\sigma)$ distribution,
where $\mu$ is unknown. We know that we may estimate $\mu$ with
$\Xbar$, and we have seen that this estimator is the MLE. But how
good is our estimate? We know that \begin{equation}
\frac{\Xbar-\mu}{\sigma/\sqrt{n}}\sim\mathsf{norm}(\mathtt{mean}=0,\,\mathtt{sd}=1).\end{equation}
For a big probability $1-\alpha$, for instance, 95\%, we can calculate
the quantile $z_{\alpha/2}$. Then \begin{equation}
\P\left(-z_{\alpha/2}\leq\frac{\Xbar-\mu}{\sigma/\sqrt{n}}\leq z_{\alpha/2}\right)=1-\alpha.\end{equation}
But now consider the following string of equivalent inequalities:\[
-z_{\alpha/2}\leq\frac{\Xbar-\mu}{\sigma/\sqrt{n}}\leq z_{\alpha/2},\]
\[
-z_{\alpha/2}\left(\frac{\sigma}{\sqrt{n}}\right)\leq\Xbar-\mu\leq z_{\alpha/2}\left(\frac{\sigma}{\sqrt{n}}\right),\]
\[
-\Xbar-z_{\alpha/2}\left(\frac{\sigma}{\sqrt{n}}\right)\leq-\mu\leq-\Xbar+z_{\alpha/2}\left(\frac{\sigma}{\sqrt{n}}\right),\]
\[
\Xbar-z_{\alpha/2}\left(\frac{\sigma}{\sqrt{n}}\right)\leq\mu\leq\Xbar+z_{\alpha/2}\left(\frac{\sigma}{\sqrt{n}}\right).\]
That is, \begin{equation}
\P\left(\Xbar-z_{\alpha/2}\frac{\sigma}{\sqrt{n}}\leq\mu\leq\Xbar+z_{\alpha/2}\frac{\sigma}{\sqrt{n}}\right)=1-\alpha.\end{equation}



\begin{defn}
The interval\begin{equation}
\left[\Xbar-z_{\alpha/2}\frac{\sigma}{\sqrt{n}},\ \Xbar+z_{\alpha/2}\frac{\sigma}{\sqrt{n}}\right]\end{equation}
 is a $100(1-\alpha)\%$ \emph{confidence interval for} $\mu$. The
quantity $1-\alpha$ is called the \emph{confidence coefficient}.\end{defn}
\begin{rem}
The interval is also sometimes written more compactly as\begin{equation}
\Xbar\pm z_{\alpha/2}\frac{\sigma}{\sqrt{n}}.\label{eq:z-interval}\end{equation}

\end{rem}
The interpretation of confidence intervals is tricky and often mistaken
by novices. When I am teaching the concept {}``live'' during class,
I usually ask the students to imagine that my piece of chalk represents
the {}``unknown'' parameter, and I lay it down on the desk in front
of me. Once the chalk has been lain, it is \emph{fixed}; it does not
move. Our goal is to estimate the parameter. For the estimator I pick
up a sheet of loose paper lying nearby. The estimation procedure is
to randomly drop the piece of paper from above, and observe where
it lands. If the piece of paper covers the piece of chalk, then we
are successful -- our estimator covers the parameter. If it falls
off to one side or the other, then we are unsuccessful; our interval
fails to cover the parameter.

Then I ask them: suppose we were to repeat this procedure hundreds,
thousands, millions of times. Suppose we kept track of how many times
we covered and how many times we did not. What percentage of the time
would we be successful?

In the demonstration, the parameter corresponds to the chalk, the
sheet of paper corresponds to the confidence interval, and the random
experiment corresponds to dropping the sheet of paper. The percentage
of the time that we are successful \emph{exactly} corresponds to the
\emph{confidence coefficient}. That is, if we use a 95\% confidence
interval, then we can say that, in the long run, approximately 95\%
of our intervals will cover the true parameter (which is fixed, but
unknown). 

See Figure \ref{fig:ci-examp}, which is a graphical display of these
ideas.%
\begin{figure}
\begin{centering}
<<echo = FALSE, fig=true, height = 6, width = 6>>=
set.seed(seed + 1)
library(TeachingDemos)
ci.examp()
@
\par\end{centering}

\caption{Simulated confidence intervals\label{fig:ci-examp}}


~

{\small The graph was generated by the }\texttt{\small ci.examp}{\small{}
function from the }\texttt{\small TeachingDemos}{\small{} package.
Fifty (50) samples of size twenty five (25) were generated from a
$\mathsf{norm}(\mathtt{mean}=100,\,\mathtt{sd}=10)$ distribution,
and each sample was used to find a 95\% confidence interval for the
population mean using Equation \ref{eq:z-interval}. The 50 confidence
intervals are represented above by horizontal lines, and the respective
sample means are denoted by vertical slashes. Confidence intervals
that {}``cover'' the true mean value of 100 are plotted in black;
those that fail to cover are plotted in a lighter color. In the plot
we see that only one (1) of the simulated intervals out of the 50
failed to cover $\mu=100$, which is a success rate of 98\%. If the
number of generated samples were to increase from 50 to 500 to 50000,
\ldots{}, then we would expect our success rate to approach the exact
value of 95\%.}
\end{figure}


Under the above framework, we can reason that an {}``interval''
with a \emph{larger} confidence coefficient corresponds to a \emph{wider}
sheet of paper. Furthermore, the width of the confidence interval
(sheet of paper) should be \emph{somehow} related to the amount of
information contained in the random sample, $X_{1}$, $X_{2}$, \ldots{},
$X_{n}$. The following remarks makes these notions precise. 
\begin{rem}
For a fixed confidence coefficient $1-\alpha$,\begin{equation}
\mbox{if }n\mbox{ increases, then the confidence interval gets \emph{SHORTER}.}\end{equation}

\end{rem}

\begin{rem}
For a fixed sample size $n$,\begin{equation}
\mbox{if }1-\alpha\mbox{ increases, then the confidence interval gets \emph{WIDER}.}\end{equation}
\end{rem}
\begin{example}
\textbf{Results from an Experiment on Plant Growth.\label{exa:plant-one-samp-z-int}}
The \inputencoding{latin9}\lstinline[basicstyle={\ttfamily},showstringspaces=false]!PlantGrowth!\inputencoding{utf8}
data frame gives the results of an experiment to measure plant yield
(as measured by the weight of the plant). We would like to a 95\%
confidence interval for the mean weight of the plants. Suppose that
we know from prior research that the true population standard deviation
of the plant weights is $0.7$~g.

The parameter of interest is $\mu$, which represents the true mean
weight of the population of all plants of the particular species in
the study. We will first take a look at a stemplot of the data:
\end{example}
<<>>=
library(aplpack)
with(PlantGrowth, stem.leaf(weight))
@

The data appear to be approximately normal with no extreme values.
The data come from a designed experiment, so it is reasonable to suppose
that the observations constitute a simple random sample of weights%
\footnote{Actually we will see later that there is reason to believe that the
observations are simple random samples from three distinct populations.
See Section \ref{sec:Analysis-of-Variance}.%
}. We know the population standard deviation $\sigma=0.70$ from prior
research. We are going to use the one-sample $z$-interval.

<<keep.source = TRUE>>=
dim(PlantGrowth)   # sample size is first entry
with(PlantGrowth, mean(weight))
qnorm(0.975)
@

We find the sample mean of the data to be $\xbar=5.073$ and $z_{\alpha/2}=z_{0.025}\approx1.96$.
Our interval is therefore\[
\xbar\pm z_{\alpha/2}\frac{\sigma}{\sqrt{n}}=5.073\pm1.96\cdot\frac{0.70}{\sqrt{30}},\]
which comes out to approximately $[4.823,\,5.323]$. In conclusion,
we are 95\% confident that the true mean weight $\mu$ of all plants
of this species lies somewhere between 4.823~g and 5.323~g, that
is, we are 95\% confident that the interval $[4.823,\,5.323]$ covers
$\mu$. See Figure 

%
\begin{figure}
\begin{centering}
<<echo = FALSE, fig=true, height = 6.5, width = 6.5>>=
library(TeachingDemos)
plot(z.test(PlantGrowth$weight, stdev = 0.70), "Conf")
@
\par\end{centering}

\caption{Confidence interval plot for the \texttt{PlantGrowth} data\label{fig:plant-z-int-plot}}


~

{\small The shaded portion represents 95\% of the total area under
the curve, and the upper and lower bounds are the limits of the one-sample
95\% confidence interval. The graph is centered at the observed sample
mean. It was generated by computing a }\texttt{\small z.test}{\small{}
from the }\texttt{\small TeachingDemos}{\small{} package, storing the
resulting }\texttt{\small htest}{\small{} object, and plotting it with
the }\texttt{\small normal.and.t.dist}{\small{} function from the }\texttt{\small HH}{\small{}
package. See the remarks in the {}``How to do it with }\textsf{\small R}{\small ''
discussion later in this section.}
\end{figure}



\begin{example}
Give some data with $X_{1}$, $X_{2}$, \ldots{}, $X_{n}$ an $SRS(n)$
from a $\mathsf{norm}(\mathtt{mean}=\mu,\,\mathtt{sd}=\sigma)$ distribution.
Maybe small sample?\end{example}
\begin{enumerate}
\item What is the parameter of interest? in the context of the problem.
Give a point estimate for $\mu$.
\item What are the assumptions being made in the problem? Do they meet the
conditions of the interval?
\item Calculate the interval.
\item Draw the conclusion.\end{enumerate}
\begin{rem}
What if $\sigma$ is unknown? We instead use the interval\begin{equation}
\Xbar\pm z_{\alpha/2}\frac{S}{\sqrt{n}},\end{equation}
where $S$ is the sample standard deviation.
\begin{itemize}
\item If $n$ is large, then $\Xbar$ will have an approximately normal
distribution regardless of the underlying population (by the CLT)
and $S$ will be very close to the parameter $\sigma$ (by the SLLN);
thus the above interval will have approximately $100(1-\alpha)\%$
confidence of covering $\mu$.
\item If $n$ is small, then 

\begin{itemize}
\item If the underlying population is normal then we may replace $z_{\alpha/2}$
with $t_{\alpha/2}(\mathtt{df}=n-1)$. The resulting $100(1-\alpha)\%$
confidence interval is\begin{equation}
\Xbar\pm t_{\alpha/2}(\mathtt{df}=n-1)\frac{S}{\sqrt{n}}\label{eq:one-samp-t-int}\end{equation}

\item if the underlying population is not normal, but approximately normal,
then we may use the $t$ interval, Equation \ref{eq:one-samp-t-int}.
The interval will have approximately $100(1-\alpha)\%$ confidence
of covering $\mu$. However, if the population is highly skewed or
the data have outliers, then we should ask a professional statistician
for advice.
\end{itemize}
\end{itemize}
\end{rem}
The author learned of a handy acronym from AP Statistics Exam graders
that summarizes the important parts of confidence interval estimation,
which is PANIC: \emph{P}arameter, \emph{A}ssumptions, \emph{N}ame,
\emph{I}nterval, and \emph{C}onclusion.
\begin{description}
\item [{Parameter:}] identify the parameter of interest with the proper
symbols. Write down what the parameter means in the context of the
problem. 
\item [{Assumptions:}] list any assumptions made in the experiment. If
there are any other assumptions needed or that were not checked, state
what they are and why they are important.
\item [{Name:}] choose a statistical procedure from your bag of tricks
based on the answers to the previous two parts. The assumptions of
the procedure you choose should match those of the problem; if they
do not match then either pick a different procedure or openly admit
that the results may not be reliable. Write down any underlying formulas
used.
\item [{Interval:}] calculate the interval from the sample data. This can
be done by hand but will more often be done with the aid of a computer.
Regardless of the method, all calculations or code should be shown
so that the entire process is repeatable by a subsequent reader.
\item [{Conclusion:}] state the final results, using language in the context
of the problem. Include the appropriate interpretation of the interval,
making reference to the confidence coefficient.\end{description}
\begin{rem}
All of the above intervals for $\mu$ were two-sided, but there are
also one-sided intervals for $\mu$. They look like \begin{equation}
\left[\Xbar-z_{\alpha}\frac{\sigma}{\sqrt{n}},\ \infty\right)\quad\mbox{or}\quad\left(-\infty,\ \Xbar+z_{\alpha}\frac{\sigma}{\sqrt{n}}\right]\end{equation}
and satisfy\begin{equation}
\P\left(\Xbar-z_{\alpha}\frac{\sigma}{\sqrt{n}}\leq\mu\right)=1-\alpha\quad\mbox{and}\quad\P\left(\Xbar+z_{\alpha}\frac{\sigma}{\sqrt{n}}\geq\mu\right)=1-\alpha.\end{equation}
\end{rem}
\begin{example}
Small sample, some data with $X_{1}$, $X_{2}$, \ldots{}, $X_{n}$
an $SRS(n)$ from a $\mathsf{norm}(\mathtt{mean}=\mu,\,\mathtt{sd}=\sigma)$
distribution.
\begin{enumerate}
\item PANIC
\end{enumerate}
\end{example}

\subsection{How to do it with \textsf{R}}

We can do Example \ref{exa:plant-one-samp-z-int} with the following
code.

<<>>=
library(TeachingDemos)
temp <- with(PlantGrowth, z.test(weight, stdev = 0.7))
temp
@

The confidence interval bounds are shown in the sixth line down of
the output (please disregard all of the additional output information
for now -- we will use it in Chapter \ref{cha:Hypothesis-Testing}).
We can make the plot for Figure \ref{fig:plant-z-int-plot} with

<<eval = FALSE>>=
library(IPSUR)
plot(temp, "Conf")
@


\section{Confidence Intervals for Differences of Means\label{sec:Conf-Interv-for-Diff-Means}}

Let $X_{1}$, $X_{2}$, \ldots{}, $X_{n}$ be a $SRS(n)$ from a
$\mathsf{norm}(\mathtt{mean}=\mu_{X},\,\mathtt{sd}=\sigma_{X})$ distribution
and let $Y_{1}$, $Y_{2}$, \ldots{}, $Y_{m}$ be a $SRS(m)$ from
a $\mathsf{norm}(\mathtt{mean}=\mu_{Y},\,\mathtt{sd}=\sigma_{Y})$
distribution. Further, assume that the $X_{1}$, $X_{2}$, \ldots{},
$X_{n}$ sample is independent of the $Y_{1}$, $Y_{2}$, \ldots{},
$Y_{m}$ sample.

Suppose that $\sigma_{X}$ and $\sigma_{Y}$ are known. We would like
a confidence interval for $\mu_{X}-\mu_{Y}$. We know that \begin{equation}
\Xbar-\Ybar\sim\mathsf{norm}\left(\mathtt{mean}=\mu_{X}-\mu_{Y},\,\mathtt{sd}=\sqrt{\frac{\sigma_{X}^{2}}{n}+\frac{\sigma_{Y}^{2}}{m}}\right).\end{equation}
Therefore, a $100(1-\alpha)$\% confidence interval for $\mu_{X}-\mu_{Y}$
is given by\begin{equation}
\left(\Xbar-\Ybar\right)\pm z_{\alpha/2}\sqrt{\frac{\sigma_{X}^{2}}{n}+\frac{\sigma_{Y}^{2}}{m}}.\label{eq:two-samp-mean-CI}\end{equation}
Unfortunately, most of the time the values of $\sigma_{X}$ and $\sigma_{Y}$
are unknown. This leads us to the following:
\begin{itemize}
\item If both sample sizes are large, then we may appeal to the CLT/SLLN
(see \ref{sec:Central-Limit-Theorem}) and substitute $S_{X}^{2}$
and $S_{Y}^{2}$ for $\sigma_{X}^{2}$ and $\sigma_{Y}^{2}$ in the
interval \ref{eq:two-samp-mean-CI}. The resulting confidence interval
will have approximately $100(1-\alpha)\%$ confidence.
\item If one or more of the sample sizes is small then we are in trouble,
unless

\begin{itemize}
\item the underlying populations are both normal and $\sigma_{X}=\sigma_{Y}$.
In this case (setting $\sigma=\sigma_{X}=\sigma_{Y}$), \begin{equation}
\Xbar-\Ybar\sim\mathsf{norm}\left(\mathtt{mean}=\mu_{X}-\mu_{Y},\,\mathtt{sd}=\sigma\sqrt{\frac{1}{n}+\frac{1}{m}}\right).\end{equation}
Now let \begin{equation}
U=\frac{n-1}{\sigma^{2}}S_{X}^{2}+\frac{m-1}{\sigma^{2}}S_{Y}^{2}.\end{equation}
Then by Exercise \ref{xca:sum-indep-chisq} we know that $U\sim\mathsf{chisq}(\mathtt{df}=n+m-2)$
and is not a large leap to believe that $U$ is independent of $\Xbar-\Ybar$;
thus \begin{equation}
T=\frac{Z}{\sqrt{\left.U\right\slash (n+m-2)}}\sim\mathsf{t}(\mathtt{df}=n+m-2).\end{equation}
But\begin{align*}
T & =\frac{\frac{\Xbar-\Ybar-(\mu_{X}-\mu_{Y})}{\sigma\sqrt{\frac{1}{n}+\frac{1}{m}}}}{\sqrt{\left.\frac{n-1}{\sigma^{2}}S_{X}^{2}+\frac{m-1}{\sigma^{2}}S_{Y}^{2}\right\slash (n+m-2)}},\\
 & =\frac{\Xbar-\Ybar-(\mu_{X}-\mu_{Y})}{\sqrt{\left(\frac{1}{n}+\frac{1}{m}\right)\left(\frac{(n-1)S_{X}^{2}+(m-1)S_{Y}^{2}}{n+m-2}\right)}},\\
 & \sim\mathsf{t}(\mathtt{df}=n+m-2).\end{align*}
Therefore a $100(1-\alpha)\%$ confidence interval for $\mu_{X}-\mu_{Y}$
is given by\begin{equation}
\left(\Xbar-\Ybar\right)\pm t_{\alpha/2}(\mathtt{df}=n+m-2)\, S_{p}\sqrt{\frac{1}{n}+\frac{1}{m}},\end{equation}
where \begin{equation}
S_{p}=\sqrt{\frac{(n-1)S_{X}^{2}+(m-1)S_{Y}^{2}}{n+m-2}}\end{equation}
is called the {}``pooled'' estimator of $\sigma$.
\item if one of the samples is small, and both underlying populations are
normal, but $\sigma_{X}\neq\sigma_{Y}$, then we may use a Welch (or
Satterthwaite) approximation to the degrees of freedom. See Welch
\cite{Welch1947}, Satterthwaite \cite{Satterthwaite1946}, or Neter
\emph{et al} \cite{Neter1996}. The idea is to use an interval of
the form\begin{equation}
\left(\Xbar-\Ybar\right)\pm\mathsf{t}_{\alpha/2}(\mathtt{df}=r)\,\sqrt{\frac{S_{X}^{2}}{n}+\frac{S_{Y}^{2}}{m}},\end{equation}
where the degrees of freedom $r$ is chosen so that the interval has
nice statistical properties. It turns out that a good choice for $r$
is given by\begin{equation}
r=\frac{\left(S_{X}^{2}/n+S_{Y}^{2}/m\right)^{2}}{\frac{1}{n-1}\left(S_{X}^{2}/n\right)^{2}+\frac{1}{m-1}\left(S_{Y}^{2}/m\right)^{2}},\end{equation}
where we understand that $r$ is rounded down to the nearest integer.
The resulting interval has approximately $100(1-\alpha)\%$ confidence.
\end{itemize}
\end{itemize}

\subsection{How to do it with \textsf{R}}

The basic function is \inputencoding{latin9}\lstinline!t.test!\inputencoding{utf8}
which has a \inputencoding{latin9}\lstinline!var.equal!\inputencoding{utf8}
argument that may be set to \inputencoding{latin9}\lstinline!TRUE!\inputencoding{utf8}
or \inputencoding{latin9}\lstinline!FALSE!\inputencoding{utf8}. The
confidence interval is shown as part of the output, although there
is a lot of additional information that is not needed until Chapter
\ref{cha:Hypothesis-Testing}.

There is not any specific functionality to handle the $z$-interval
for small samples, but if the samples are large then \inputencoding{latin9}\lstinline!t.test!\inputencoding{utf8}
with \inputencoding{latin9}\lstinline!var.equal = FALSE!\inputencoding{utf8}
will be essentially the same thing. The standard deviations are never
(?) known in advance anyway so it does not really matter in practice. 


\section{Confidence Intervals for Proportions\label{sec:Confidence-Intervals-Proportions}}

We would like to know $p$ which is the {}``proportion of successes''.
For instance, $p$ could be:
\begin{itemize}
\item the proportion of U.S.~citizens that support Obama,
\item the proportion of smokers among adults age 18 or over,
\item the proportion of people worldwide infected by the H1N1 virus.
\end{itemize}
We are given an $SRS(n)$ $X_{1}$, $X_{2}$, \ldots{}, $X_{n}$
distributed $\mathsf{binom}(\mathtt{size}=1,\,\mathtt{prob}=p)$.
Recall from Section \ref{sec:binom-dist} that the common mean of
these variables is $\E X=p$ and the variance is $\E(X-p)^{2}=p(1-p)$.
If we let $Y=\sum X_{i}$, then from Section \ref{sec:binom-dist}
we know that $Y\sim\mathsf{binom}(\mathtt{size}=n,\,\mathtt{prob}=p)$
and that \[
\Xbar=\frac{Y}{n}\mbox{ has }\E\Xbar=p\mbox{ and }\mathrm{Var}(\Xbar)=\frac{p(1-p)}{n}.\]
Thus if $n$ is large (here is the CLT) then an approximate $100(1-\alpha)\%$
confidence interval for $p$ would be given by\begin{equation}
\Xbar\pm z_{\alpha/2}\sqrt{\frac{p(1-p)}{n}}.\label{eq:ci-p-no-good}\end{equation}
OOPS\ldots{}! Equation \ref{eq:ci-p-no-good} is of no use to us
because the \underbar{unknown} parameter $p$ is in the formula! (If
we knew what $p$ was to plug in the formula then we would not need
a confidence interval in the first place.) There are two solutions
to this problem.
\begin{enumerate}
\item Replace $p$ with $\hat{p}=\Xbar$. Then an approximate $100(1-\alpha)\%$
confidence interval for $p$ is given by \begin{equation}
\hat{p}\pm z_{\alpha/2}\sqrt{\frac{\hat{p}(1-\hat{p})}{n}}.\end{equation}
This approach is called the \emph{Wald interval} and is also known
as the \emph{asymptotic interval} because it appeals to the CLT for
large sample sizes.
\item Go back to first principles. Note that\[
-z_{\alpha/2}\leq\frac{Y/n-p}{\sqrt{p(1-p)/n}}\leq z_{\alpha/2}\]
exactly when the function $f$ defined by\[
f(p)=\left(Y/n-p\right)^{2}-z_{\alpha/2}^{2}\frac{p(1-p)}{n}\]
satisfies $f(p)\leq0$. But $f$ is quadratic in $p$ so its graph
is a parabola; it has two roots, and these roots form the limits of
the confidence interval. We can find them with the quadratic formula
(see Exercise \ref{xca:CI-quad-form}):\begin{equation}
\left.\left[\left(\hat{p}+\frac{z_{\alpha/2}^{2}}{2n}\right)\pm z_{\alpha/2}\sqrt{\frac{\hat{p}(1-\hat{p})}{n}+\frac{z_{\alpha/2}^{2}}{(2n)^{2}}}\right]\right\slash \left(1+\frac{z_{\alpha/2}^{2}}{n}\right)\end{equation}
 This approach is called the \emph{score interval} because it is based
on the inversion of the {}``Score test''. See Chapter \ref{cha:Categorical-Data-Analysis}.
It is also known as the \emph{Wilson interval}; see Agresti \cite{Agresti2002}.
\end{enumerate}
For two proportions $p_{1}$ and $p_{2}$, we may collect independent
$\mathsf{binom}(\mathtt{size}=1,\,\mathtt{prob}=p)$ samples of size
$n_{1}$ and $n_{2}$, respectively. Let $Y_{1}$ and $Y_{2}$ denote
the number of successes in the respective samples. 

We know that \[
\frac{Y_{1}}{n_{1}}\approx\mathsf{norm}\left(\mathtt{mean}=p_{1},\,\mathtt{sd}=\sqrt{\frac{p_{1}(1-p_{1})}{n_{1}}}\right)\]
and\[
\frac{Y_{2}}{n_{2}}\approx\mathsf{norm}\left(\mathtt{mean}=p_{2},\,\mathtt{sd}=\sqrt{\frac{p_{2}(1-p_{2})}{n_{2}}}\right)\]
so it stands to reason that an approximate $100(1-\alpha)\%$ confidence
interval for $p_{1}-p_{2}$ is given by\begin{equation}
\left(\hat{p}_{1}-\hat{p}_{2}\right)\pm z_{\alpha/2}\sqrt{\frac{\hat{p}_{1}(1-\hat{p}_{1})}{n_{1}}+\frac{\hat{p}_{2}(1-\hat{p}_{2})}{n_{2}}},\end{equation}
where $\hat{p}_{1}=Y_{1}/n_{1}$ and $\hat{p}_{2}=Y_{2}/n_{2}$.
\begin{rem}
When estimating a single proportion, one-sided intervals are sometimes
needed. They take the form\begin{equation}
\left[0,\ \hat{p}+z_{\alpha/2}\sqrt{\frac{\hat{p}(1-\hat{p})}{n}}\right]\end{equation}
or \begin{equation}
\left[\hat{p}-z_{\alpha/2}\sqrt{\frac{\hat{p}(1-\hat{p})}{n}},\ 1\right]\end{equation}
or in other words, we know in advance that the true proportion is
restricted to the interval $[0,1]$, so we can truncate our confidence
interval to those values on either side.
\end{rem}

\subsection{How to do it with \textsf{R}}

<<>>=
library(Hmisc)
binconf(x = 7, n = 25, method = "asymptotic")
binconf(x = 7, n = 25, method = "wilson")
@

The default value of the \inputencoding{latin9}\lstinline!method!\inputencoding{utf8}
argument is \inputencoding{latin9}\lstinline!wilson!\inputencoding{utf8}.

An alternate way is 

<<>>=
tab <- xtabs(~gender, data = RcmdrTestDrive)
prop.test(rbind(tab), conf.level = 0.95, correct = FALSE)
@

<<>>=
A <- as.data.frame(Titanic)
library(reshape)
B <- with(A, untable(A, Freq))
@


\section{Confidence Intervals for Variances\label{sec:Confidence-Intervals-for-Variances}}

I am thinking one and two sample problems here.


\subsection{How to do it with \textsf{R}}

I am thinking about sigma.test in the TeachingDemos package and var.test
in base R here.


\section{Fitting Distributions\label{sec:Fitting-Distributions}}


\subsection{How to do it with \textsf{R}}

I am thinking about fitdistr from the MASS package \cite{Venables2002}.


\section{Sample Size and Margin of Error\label{sec:Sample-Size-and-MOE}}

Sections \ref{sec:Confidence-Intervals-for-Means} through \ref{sec:Confidence-Intervals-for-Variances}
all began the same way: we were given the sample size $n$ and the
confidence coefficient $1-\alpha$, and our task was to find a margin
of error $E$ so that \[
\hat{\theta}\pm E\mbox{ is a }100(1-\alpha)\%\mbox{ confidence interval for }\theta.\]
Some examples we saw were:
\begin{itemize}
\item $E=z_{\alpha/2}\sigma/\sqrt{n}$, in the one-sample $z$-interval,
\item $E=t_{\alpha/2}(\mathtt{df}=n+m-2)S_{p}\sqrt{n^{-1}+m^{-1}}$, in
the two-sample pooled $t$-interval. 
\end{itemize}
We already know (we can see in the formulas above) that $E$ decreases
as $n$ increases. Now we would like to use this information to our
advantage: suppose that we have a fixed margin of error $E,$ say
$E=3$, and we want a $100(1-\alpha)\%$ confidence interval for $\mu$.
The question is: how big does $n$ have to be?

For the case of a population mean the answer is easy: we set up an
equation and solve for $n$.
\begin{example}
Given a situation, given $\sigma$, given $E$, we would like to know
how big $n$ has to be to ensure that $\Xbar\pm5$ is a 95\% confidence
interval for $\mu$. \end{example}
\begin{rem}
~
\begin{enumerate}
\item Always round up any decimal values of $n$, no matter how small the
decimal is.
\item Another name for $E$ is the {}``maximum error of the estimate''.
\end{enumerate}
\end{rem}
For proportions, recall that the asymptotic formula to estimate $p$
was\[
\hat{p}\pm z_{\alpha/2}\sqrt{\frac{\hat{p}(1-\hat{p})}{n}}.\]
Reasoning as above we would want\begin{align}
E & =z_{\alpha/2}\sqrt{\frac{\hat{p}(1-\hat{p})}{n}},\mbox{ or}\\
n & =z_{\alpha/2}^{2}\frac{\hat{p}(1-\hat{p})}{E^{2}}.\label{eq:samp-size-prop-ME}\end{align}
OOPS! Recall that $\hat{p}=Y/n$, which would put the variable $n$
on both sides of Equation \ref{eq:samp-size-prop-ME}. Again, there
are two solutions to the problem.
\begin{enumerate}
\item If we have a good idea of what $p$ is, say $p^{\ast}$ then we can
plug it in to get\begin{equation}
n=z_{\alpha/2}^{2}\frac{p^{\ast}(1-p^{\ast})}{E^{2}}.\end{equation}

\item Even if we have no idea what $p$ is, we do know from calculus that
$p(1-p)\leq1/4$ because the function $f(x)=x(1-x)$ is quadratic
(so its graph is a parabola which opens downward) with maximum value
attained at $x=1/2$. Therefore, regardless of our choice for $p^{\ast}$
the sample size must satisfy\begin{equation}
n=z_{\alpha/2}^{2}\frac{p^{\ast}(1-p^{\ast})}{E^{2}}\leq\frac{z_{\alpha/2}^{2}}{4E^{2}}.\end{equation}
The quantity $z_{\alpha/2}^{2}/4E^{2}$ is large enough to guarantee
$100(1-\alpha)\%$ confidence.\end{enumerate}
\begin{example}
Proportion example\end{example}
\begin{rem}
For very small populations sometimes the value of $n$ obtained from
the formula is too big. In this case we should use the hypergeometric
distribution for a sampling model rather than the binomial model.
With this modification the formulas change to the following: if $N$
denotes the population size then let\begin{equation}
m=z_{\alpha/2}^{2}\frac{p^{\ast}(1-p^{\ast})}{E^{2}}\end{equation}
 and the sample size needed to ensure $100(1-\alpha)\%$ confidence
is achieved is\begin{equation}
n=\frac{m}{1+\frac{m-1}{N}}.\end{equation}
If we do not have a good value for the estimate $p^{\ast}$ then we
may use $p^{\ast}=1/2$. 
\end{rem}

\subsection{How to do it with \textsf{R}}

I am thinking about power.t.test, power.prop.test, power.anova.test,
and I am also thinking about replicate.


\section{Other Topics\label{sec:Other-Topics}}

Mention mle from the stats4 package.

\newpage{}


\section*{Chapter Exercises}

\addcontentsline{toc}{section}{Chapter Exercises}
\setcounter{thm}{0}
\begin{xca}
Let $X_{1}$, $X_{2}$, \ldots{}, $X_{n}$ be an $SRS(n)$ from a
$\mathsf{norm}(\mathtt{mean}=\mu,\,\mathtt{sd}=\sigma)$ distribution.
Find a two-dimensional MLE for $\theta=(\mu,\sigma)$.\label{xca:norm-mu-sig-MLE}
\end{xca}

\begin{xca}
\label{xca:CI-quad-form}Find the upper and lower limits for the confidence
interval procedure by finding the roots of $f$ defined by \[
f(p)=\left(Y/n-p\right)^{2}-z_{\alpha/2}^{2}\frac{p(1-p)}{n}.\]
You are going to need the quadratic formula.
\end{xca}

\chapter{Hypothesis Testing\label{cha:Hypothesis-Testing}}


\paragraph*{What do I want them to know?}
\begin{itemize}
\item basic terminology and philosophy of the Neyman-Pearson paradigm
\item classical hypothesis tests for the standard one and two sample problems
with means, variances, and proportions
\item the notion of between versus within group variation and how it plays
out with one-way ANOVA
\item the concept of statistical power and its relation to sample size
\end{itemize}

\section{Introduction\label{sec:Introduction-Hypothesis}}

I spent a week during the summer of 2005 at the University of Nebraska
at Lincoln grading Advanced Placement Statistics exams, and while
I was there I attended a presentation by Dr.~Roxy Peck. At the end
of her talk she described an activity she had used with students to
introduce the basic concepts of hypothesis testing. I was impressed
by the activity and have used it in my own classes several times since.

The instructor (with a box of cookies in hand) enters a class of fifteen
or more students and produces a brand-new, sealed deck of ordinary
playing cards. The instructor asks for a student volunteer to break
the seal, and then the instructor prominently shuffles the deck%
\footnote{The jokers are removed before shuffling.%
} several times in front of the class, after which time the students
are asked to line up in a row. They are going to play a game. Each
student will draw a card from the top of the deck, in turn. If the
card is black, then the lucky student will get a cookie. If the card
is red, then the unlucky student will sit down empty-handed. Let the
game begin.

The first student draws a card: red. There are jeers and outbursts,
and the student slinks off to his/her chair. (S)he is disappointed,
of course, but not really. After all, (s)he had a 50-50 chance of
getting black, and it did not happen. Oh well.

The second student draws a card: red, again. There are more jeers,
and the second student slips away. This student is also disappointed,
but again, not so much, because it is probably his/her unlucky day.
On to the next student.

The student draws: red again! There are a few wiseguys who yell (happy
to make noise, more than anything else), but there are a few other
students who are not yelling any more -- they are thinking. This is
the third red in a row, which is possible, of course, but what is
going on, here? They are not quite sure. They are now concentrating
on the next card\ldots{} it is bound to be black, right?

The fourth student draws: red. Hmmm\ldots{} now there are groans
instead of outbursts. A few of the students at the end of the line
shrug their shoulders and start to make their way back to their desk,
complaining that the teacher does not want to give away any cookies.
There are still some students in line though, salivating, waiting
for the inevitable black to appear.

The fifth student draws red. Now it isn't funny any more. As the remaining
students make their way back to their seats an uproar ensues, from
an entire classroom demanding cookies.

\bigskip{}


Keep the preceding experiment in the back of your mind as you read
the following sections. When you have finished the entire chapter,
come back and read this introduction again. All of the mathematical
jargon that follows is connected to the above paragraphs. In the meantime,
I will get you started:
\begin{description}
\item [{Null~hypothesis:}] it is an ordinary deck of playing cards, shuffled
thoroughly.
\item [{Alternative~hypothesis:}] either it is a trick deck of cards,
or the instructor did some fancy shufflework.
\item [{Observed~data:}] a sequence of draws from the deck, five reds
in a row.
\end{description}
If it were truly an ordinary, well-shuffled deck of cards, the probability
of observing zero blacks out of a sample of size five (without replacement)
from a deck with 26 black cards and 26 red cards would be

<<>>=
dhyper(0, m = 26, n = 26, k = 5)
@

There are two very important final thoughts. First, everybody gets
a cookie in the end. Second, the students invariably (and aggressively)
attempt to get me to open up the deck and reveal the true nature of
the cards. I never do.


\section{Tests for Proportions\label{sec:Tests-for-Proportions}}
\begin{example}
\label{exa:widget-machine}We have a machine that makes widgets. 
\begin{itemize}
\item Under normal operation, about 0.10 of the widgets produced are defective.
\item Go out and purchase a torque converter.
\item Install the torque converter, and observe $n=100$ widgets from the
machine.
\item Let $Y=\mbox{number of defective widgets observed}$.
\end{itemize}
If
\begin{itemize}
\item $Y=0$, then the torque converter is great!
\item $Y=4$, then the torque converter seems to be helping. 
\item $Y=9$, then there is not much evidence that the torque converter
helps.
\item $Y=17$, then throw away the torque converter.
\end{itemize}
Let $p$ denote the proportion of defectives produced by the machine.
Before the installation of the torque converter $p$ was $0.10$.
Then we installed the torque converter. Did $p$ change? Did it go
up or down? We use statistics to decide. Our method is to observe
data and construct a 95\% confidence interval for $p$,\begin{equation}
\hat{p}\pm z_{\alpha/2}\sqrt{\frac{\hat{p}(1-\hat{p})}{n}}.\end{equation}
If the confidence interval is
\begin{itemize}
\item $[0.01,\,0.05]$, then we are 95\% confident that $0.01\leq p\leq0.05$,
so there is evidence that the torque converter is helping.
\item $[0.15,\,0.19]$, then we are 95\% confident that $0.15\leq p\leq0.19$,
so there is evidence that the torque converter is hurting.
\item $[0.07,\,0.11]$, then there is not enough evidence to conclude that
the torque converter is doing anything at all, positive or negative.
\end{itemize}
\end{example}

\subsection{Terminology}

The \emph{null hypothesis} $H_{0}$ is a {}``nothing'' hypothesis,
whose interpretation could be that nothing has changed, there is no
difference, there is nothing special taking place, \emph{etc}.. In
Example \ref{exa:widget-machine} the null hypothesis would be $H_{0}:\ p=0.10.$
The \emph{alternative hypothesis} $H_{1}$ is the hypothesis that
something has changed, in this case, $H_{1}:\ p\neq0.10$. Our goal
is to statistically \emph{test} the hypothesis $H_{0}:\ p=0.10$ versus
the alternative $H_{1}:\ p\neq0.10$. Our procedure will be:
\begin{enumerate}
\item Go out and collect some data, in particular, a simple random sample
of observations from the machine.
\item Suppose that $H_{0}$ is true and construct a $100(1-\alpha)\%$ confidence
interval for $p$.
\item If the confidence interval does not cover $p=0.10$, then we \emph{reject}$H_{0}$.
Otherwise, we \emph{fail to reject}$H_{0}$.\end{enumerate}
\begin{rem}
Every time we make a decision it is possible to be wrong, and there
are two possible mistakes that we could make. We have committed a
\begin{description}
\item [{Type~I~Error}] if we reject $H_{0}$ when in fact $H_{0}$ is
true. This would be akin to convicting an innocent person for a crime
(s)he did not commit.
\item [{Type~II~Error}] if we fail to reject $H_{0}$ when in fact $H_{1}$
is true. This is analogous to a guilty person escaping conviction.
\end{description}
\end{rem}
Type I Errors are usually considered worse%
\footnote{There is no mathematical difference between the errors, however. The
bottom line is that we choose one type of error to control with an
iron fist, and we try to minimize the probability of making the other
type. That being said, null hypotheses are often by design to correspond
to the {}``simpler'' model, so it is often easier to analyze (and
thereby control) the probabilities associated with Type I Errors.%
}, and we design our statistical procedures to control the probability
of making such a mistake. We define the\begin{equation}
\mbox{significance level of the test}=\P(\mbox{Type I Error})=\alpha.\end{equation}
We want $\alpha$ to be small which conventionally means, say, $\alpha=0.05$,
$\alpha=0.01$, or $\alpha=0.005$ (but could mean anything, in principle). 
\begin{itemize}
\item The \emph{rejection region} (also known as the \emph{critical region})
for the test is the set of sample values which would result in the
rejection of $H_{0}$. For Example \ref{exa:widget-machine}, the
rejection region would be all possible samples that result in a 95\%
confidence interval that does not cover $p=0.10$.
\item The above example with $H_{1}:p\neq0.10$ is called a \emph{two-sided}
test. Many times we are interested in a \emph{one-sided} test, which
would look like $H_{1}:p<0.10$ or $H_{1}:p>0.10$.
\end{itemize}
We are ready for tests of hypotheses for one proportion.

Table here.

Don't forget the assumptions.
\begin{example}
Find
\begin{enumerate}
\item The null and alternative hypotheses
\item Check your assumptions.
\item Define a critical region with an $\alpha=0.05$ significance level.
\item Calculate the value of the test statistic and state your conclusion.
\end{enumerate}
\end{example}

\begin{example}
\label{exa:prop-test-pvalue-A}Suppose $p=\mbox{the proportion of students}$
who are admitted to the graduate school of the University of California
at Berkeley, and suppose that a public relations officer boasts that
UCB has historically had a 40\% acceptance rate for its graduate school.
Consider the data stored in the table \inputencoding{latin9}\lstinline!UCBAdmissions!\inputencoding{utf8}
from 1973. Assuming these observations constituted a simple random
sample, are they consistent with the officer's claim, or do they provide
evidence that the acceptance rate was significantly less than 40\%?
Use an $\alpha=0.01$ significance level.

Our null hypothesis in this problem is $H_{0}:\, p=0.4$ and the alternative
hypothesis is $H_{1}:\, p<0.4$. We reject the null hypothesis if
$\hat{p}$ is too small, that is, if\begin{equation}
\frac{\hat{p}-0.4}{\sqrt{0.4(1-0.4)/n}}<-z_{\alpha},\end{equation}
where $\alpha=0.01$ and $-z_{0.01}$ is 

<<>>=
- qnorm(0.99)
@

Our only remaining task is to find the value of the test statistic
and see where it falls relative to the critical value. We can find
the number of people admitted and not admitted to the UCB graduate
school with the following. 

<<>>=
A <- as.data.frame(UCBAdmissions)
head(A)
xtabs(Freq ~ Admit, data = A)
@

Now we calculate the value of the test statistic.

<<>>=
phat <- 1755/(1755 + 2771)
(phat - 0.4)/sqrt(0.4 * 0.6/(1755 + 2771)) 
@

Our test statistic is not less than $-2.32$, so it does not fall
into the critical region. Therefore, we \emph{fail} to reject the
null hypothesis that the true proportion of students admitted to graduate
school is less than 40\% and say that the observed data are consistent
with the officer's claim at the $\alpha=0.01$ significance level. 

\end{example}

\begin{example}
\label{exa:prop-test-pvalue-B}We are going to do Example \ref{exa:prop-test-pvalue-A}
all over again. Everything will be exactly the same except for one
change. Suppose we choose significance level $\alpha=0.05$ instead
of $\alpha=0.01$. Are the 1973 data consistent with the officer's
claim?

Our null and alternative hypotheses are the same. Our observed test
statistic is the same: it was approximately $-1.68$. But notice that
our critical value has changed: $\alpha=0.05$ and $-z_{0.05}$ is
\end{example}
<<>>=
-qnorm(0.95)
@

Our test statistic is less than $-1.64$ so it now falls into the
critical region! We now \emph{reject} the null hypothesis and conclude
that the 1973 data provide evidence that the true proportion of students
admitted to the graduate school of UCB in 1973 was significantly less
than 40\%. The data are \emph{not} consistent with the officer's claim
at the $\alpha=0.05$ significance level.



What is going on, here? If we choose $\alpha=0.05$ then we reject
the null hypothesis, but if we choose $\alpha=0.01$ then we fail
to reject the null hypothesis. Our final conclusion seems to depend
on our selection of the significance level. This is bad; for a particular
test, we never know whether our conclusion would have been different
if we had chosen a different significance level. 

Or do we?

Clearly, for some significance levels we reject, and for some significance
levels we do not. Where is the boundary? That is, what is the significance
level for which we would \emph{reject} at any significance level \emph{bigger},
and we would \emph{fail to reject} at any significance level \emph{smaller}?
This boundary value has a special name: it is called the $p$\emph{-value}
of the test.
\begin{defn}
The $p$-\emph{value}, or \emph{observed significance level}, of a
hypothesis test is the probability when the null hypothesis is true
of obtaining the observed value of the test statistic (such as $\hat{p}$)
or values more extreme -- meaning, in the direction of the alternative
hypothesis%
\footnote{Bickel and Doksum \cite{Bickel2001} state the definition particularly
well: the $p$-value is {}``the smallest level of significance $\alpha$
at which an experimenter using {[}the test statistic{]} $T$ would
reject {[}$H_{0}${]} on the basis of the observed {[}sample{]} outcome
$x$''.%
}. \end{defn}
\begin{example}
Calculate the $p$-value for the test in Examples \ref{exa:prop-test-pvalue-A}
and \ref{exa:prop-test-pvalue-B}.

The $p$-value for this test is the probability of obtaining a $z$-score
equal to our observed test statistic (which had $z$-score $\approx-1.680919$)
or more extreme, which in this example is less than the observed test
statistic. In other words, we want to know the area under a standard
normal curve on the interval $(-\infty,\,-1.680919]$. We can get
this easily with
\end{example}
<<>>=
pnorm(-1.680919)
@

We see that the $p$-value is strictly between the significance levels
$\alpha=0.01$ and $\alpha=0.05$. This makes sense: it has to be
bigger than $\alpha=0.01$ (otherwise we would have rejected $H_{0}$
in Example \ref{exa:prop-test-pvalue-A}) and it must also be smaller
than $\alpha=0.05$ (otherwise we would not have rejected $H_{0}$
in Example \ref{exa:prop-test-pvalue-B}). Indeed, $p$-values are
a characteristic indicator of whether or not we would have rejected
at assorted significance levels, and for this reason a statistician
will often skip the calculation of critical regions and critical values
entirely. If (s)he knows the $p$-value, then (s)he knows immediately
whether or not (s)he would have rejected at \emph{any} given significance
level.

Thus, another way to phrase our significance test procedure is: we
will reject $H_{0}$ at the $\alpha$-level of significance if the
$p$-value is less than $\alpha$.
\begin{rem}
If we have two populations with proportions $p_{1}$ and $p_{2}$
then we can test the null hypothesis $H_{0}:p_{1}=p_{2}$.
\end{rem}
Table Here.
\begin{example}
Example.
\end{example}

\subsection{How to do it with \textsf{R}}

The following does the test.

<<>>=
prop.test(1755, 1755 + 2771, p = 0.4, alternative = "less", conf.level = 0.99, correct = FALSE)
@

Do the following to make the plot.

<<eval = FALSE>>=
library(IPSUR)
library(HH)
temp <- prop.test(1755, 1755 + 2771, p = 0.4, alternative = "less", conf.level = 0.99, correct = FALSE)
plot(temp, 'Hypoth')
@

%
\begin{figure}
\begin{centering}
<<echo = FALSE, fig=true, height = 6, width = 6.5>>=
library(HH)
plot(prop.test(1755, 1755 + 2771, p = 0.4, alternative = "less", conf.level = 0.99, correct = FALSE), 'Hypoth')
@
\par\end{centering}

\caption{Hypothesis test plot based on \texttt{normal.and.t.dist} from the
\texttt{HH} package\label{fig:Hypothesis-test-plot-1}}


{\small ~}{\small \par}

{\small This plot shows all of the important features of hypothesis
tests in one magnificent display. The (asymptotic) distribution of
the test statistic (under the null hypothesis) is standard normal,
represented by the bell curve, above. We see the critical region to
the left, and the blue shaded area is the significance level, which
for this example is $\alpha=0.05$. The area outlined in green is
the $p$-value, and the observed test statistic determines the upper
bound of this region. We can see clearly that the $p$-value is larger
than the significance level, thus, we would not reject the null hypothesis.
There are all sorts of tick marks shown below the graph which detail
how the different pieces are measured on different scales (the original
data scale, the standardized scale, }\emph{\small etc}{\small .).
The workhorse behind the plot is the }\texttt{normal.and.t.dist }{\small function
from the }\texttt{HH}{\small{} package. See the discussion in {}``How
to do it with }\textsf{\small R}{\small '' for the exact sequence
of commands to generate the plot.} 
\end{figure}


Use Yates' continuity correction when the expected frequency of successes
is less than 10. You can use it all of the time, but you will have
a decrease in power. For large samples the correction does not matter.


\paragraph*{With the \textsf{R} Commander}

If you already know the number of successes and failures, then you
can use the menu \textsf{Statistics $\triangleright$ Proportions
$\triangleright$ IPSUR Enter table for single sample\ldots{}}

Otherwise, your data -- the raw successes and failures -- should be
in a column of the Active Data Set. Furthermore, the data must be
stored as a {}``factor'' internally. If the data are not a factor
but are numeric then you can use the menu \textsf{Data $\triangleright$
Manage variables in active data set $\triangleright$ Convert numeric
variables to factors\ldots{}} to convert the variable to a factor.
Or, you can always use the \inputencoding{latin9}\lstinline!factor!\inputencoding{utf8}
function.

Once your unsummarized data is a column, then you can use the menu
\textsf{Statistics $\triangleright$ Proportions $\triangleright$
Single-sample proportion test\ldots{}}


\section{One Sample Tests for Means and Variances\label{sec:One-Sample-Tests}}


\subsection{For Means}

Here, $X_{1}$, $X_{2}$, \ldots{}, $X_{n}$ are a $SRS(n)$ from
a $\mathsf{norm}(\mathtt{mean}=\mu,\,\mathtt{sd}=\sigma)$ distribution.
We would like to test $H_{0}:\mu=\mu_{0}$.
\begin{description}
\item [{Case~A:}] Suppose $\sigma$ is known. Then under $H_{0}$,\[
Z=\frac{\Xbar-\mu_{0}}{\sigma/\sqrt{n}}\sim\mathsf{norm}(\mathtt{mean}=0,\,\mathtt{sd}=1).\]
Table here.
\item [{Case~B:}] When $\sigma$ is unknown, under $H_{0}$\[
T=\frac{\Xbar-\mu_{0}}{S/\sqrt{n}}\sim\mathsf{t}(\mathtt{df}=n-1).\]
Table here. \end{description}
\begin{rem}
If $\sigma$ is unknown but $n$ is large then we can use the $z$-test.\end{rem}
\begin{example}
In this example we
\begin{enumerate}
\item Find the null and alternative hypotheses.
\item Choose a test and find the critical region.
\item Calculate the value of the test statistic and state the conclusion.
\item Find the $p$-value.
\end{enumerate}
\end{example}

\begin{rem}
Remarks
\begin{itemize}
\item $p$-values are also known as tail end probabilities. We reject $H_{0}$
when the $p$-value is small.
\item $\sigma/\sqrt{n}$ when $\sigma$ is known, is called the standard
error of the sample mean. In general, if we have an estimator $\hat{\theta}$
then $\sigma_{\hat{\theta}}$ is called the standard error of $\hat{\theta}$.
We usually need to estimate $\sigma_{\hat{\theta}}$ with $\hat{\sigma_{\hat{\theta}}}$.
\end{itemize}
\end{rem}

\subsection{How to do it with \textsf{R}}

I am thinking \inputencoding{latin9}\lstinline!z.test!\inputencoding{utf8}\index{z.test@\texttt{z.test}}
in \inputencoding{latin9}\lstinline!TeachingDemos!\inputencoding{utf8},
\inputencoding{latin9}\lstinline!t.test!\inputencoding{utf8}\index{t.test@\texttt{t.test}}
in base \textsf{\small R}.

<<>>=
x <- rnorm(37, mean = 2, sd = 3)
library(TeachingDemos)
z.test(x, mu = 1, sd = 3, conf.level = 0.90)
@

%
\begin{figure}
\begin{centering}
<<echo = FALSE, fig=true, height = 6, width = 6.5>>=
library(HH)
plot(prop.test(1755, 1755 + 2771, p = 0.4, alternative = "less", conf.level = 0.99, correct = FALSE), 'Hypoth')
@
\par\end{centering}

\caption{Hypothesis test plot based on \texttt{normal.and.t.dist} from the
\texttt{HH} package\label{fig:Hypothesis-test-plot-2}}


{\small ~}{\small \par}

{\small This plot shows the important features of hypothesis tests.}
\end{figure}


The \inputencoding{latin9}\lstinline!RcmdrPlugin.IPSUR!\inputencoding{utf8}
package does not have a menu for \inputencoding{latin9}\lstinline!z.test!\inputencoding{utf8}
yet. 

<<>>=
x <- rnorm(13, mean = 2, sd = 3)
t.test(x, mu = 0, conf.level = 0.90, alternative = "greater")
@


\paragraph*{With the \textsf{R} Commander}

Your data should be in a single numeric column (a variable) of the
Active Data Set. Use the menu \textsf{Statistics $\triangleright$
Means $\triangleright$ Single-sample t-test\ldots{}}


\subsection{Tests for a Variance}

Here, $X_{1}$, $X_{2}$, \ldots{}, $X_{n}$ are a $SRS(n)$ from
a $\mathsf{norm}(\mathtt{mean}=\mu,\,\mathtt{sd}=\sigma)$ distribution.
We would like to test $H_{0}:\sigma^{2}=\sigma_{0}$. We know that
under $H_{0}$,\[
X^{2}=\frac{(n-1)S^{2}}{\sigma^{2}}\sim\mathsf{chisq}(\mathtt{df}=n-1).\]
Table here.
\begin{example}
Give some data and a hypothesis.
\begin{enumerate}
\item Give an $\alpha$-level and test the critical region way.
\item Find the $p$-value for the test.
\end{enumerate}
\end{example}

\subsection{How to do it with \textsf{R}}

I am thinking about \inputencoding{latin9}\lstinline!sigma.test!\inputencoding{utf8}\index{sigma.test@\texttt{sigma.test}}
in the \inputencoding{latin9}\lstinline!TeachingDemos!\inputencoding{utf8}
package.

<<>>=
library(TeachingDemos)
sigma.test(women$height, sigma = 8)
@


\section{Two-Sample Tests for Means and Variances\label{sec:Two-Sample-Tests-for-Means}}

The basic idea for this section is the following. We have $X\sim\mathsf{norm}(\mathtt{mean}=\mu_{X},\,\mathtt{sd}=\sigma_{X})$
and $Y\sim\mathsf{norm}(\mathtt{mean}=\mu_{Y},\,\mathtt{sd}=\sigma_{Y})$.
distributed independently. We would like to know whether $X$ and
$Y$ come from the same population distribution, that is, we would
like to know:\begin{equation}
\mbox{Does }X\overset{\mathrm{d}}{=}Y?\end{equation}
where the symbol $\overset{\mathrm{d}}{=}$ means equality of probability
distributions.

Since both $X$ and $Y$ are normal, we may rephrase the question:\begin{equation}
\mbox{Does }\mu_{X}=\mu_{Y}\mbox{ and }\sigma_{X}=\sigma_{Y}?\end{equation}
Suppose first that we do not know the values of $\sigma_{X}$ and
$\sigma_{Y}$, but we know that they are equal, $\sigma_{X}=\sigma_{Y}$.
Our test would then simplify to $H_{0}:\mu_{X}=\mu_{Y}$. We collect
data $X_{1}$, $X_{2}$, \ldots{}, $X_{n}$ and $Y_{1}$, $Y_{2}$,
\ldots{}, $Y_{m}$, both simple random samples of size $n$ and $m$
from their respective normal distributions. Then under $H_{0}$ (that
is, assuming $H_{0}$ is true) we have $\mu_{X}=\mu_{Y}$ or rewriting,
$\mu_{X}-\mu_{Y}=0$, so \begin{equation}
T=\frac{\Xbar-\Ybar}{S_{p}\sqrt{\frac{1}{n}+\frac{1}{m}}}=\frac{\Xbar-\Ybar-(\mu_{X}-\mu_{Y})}{S_{p}\sqrt{\frac{1}{n}+\frac{1}{m}}}\sim\mathsf{t}(\mathtt{df}=n+m-2).\end{equation}



\subsection{Independent Samples}
\begin{rem}
If the values of $\sigma_{X}$ and $\sigma_{Y}$ are known, then we
can plug them in to our statistic:\begin{equation}
Z=\frac{\Xbar-\Ybar}{\sqrt{\sigma_{X}^{2}/n+\sigma_{Y}^{2}/m}};\end{equation}
the result will have a $\mathsf{norm}(\mathtt{mean}=0,\,\mathtt{sd}=1)$
distribution when $H_{0}:\mu_{X}=\mu_{Y}$ is true.
\end{rem}

\begin{rem}
Even if the values of $\sigma_{X}$ and $\sigma_{Y}$ are not known,
if both $n$ and $m$ are large then we can plug in the sample estimates
and the result will have approximately a $\mathsf{norm}(\mathtt{mean}=0,\,\mathtt{sd}=1)$
distribution when $H_{0}:\mu_{X}=\mu_{Y}$ is true.\begin{equation}
Z=\frac{\Xbar-\Ybar}{\sqrt{S_{X}^{2}/n+S_{Y}^{2}/m}}.\end{equation}

\end{rem}

\begin{rem}
It is usually important to construct side-by-side boxplots and other
visual displays in concert with the hypothesis test. This gives a
visual comparison of the samples and helps to identify departures
from the test's assumptions -- such as outliers.
\end{rem}

\begin{rem}
WATCH YOUR ASSUMPTIONS.
\begin{itemize}
\item The normality assumption can be relaxed as long as the population
distributions are not highly skewed.
\item The equal variance assumption can be relaxed as long as both sample
sizes $n$ and $m$ are large. However, if one (or both) samples is
small, then the test does not perform well; we should instead use
the methods of Chapter \ref{cha:resampling-methods}.
\end{itemize}
\end{rem}
For a nonparametric alternative to the two-sample $F$ test see Chapter
\ref{cha:Nonparametric-Statistics}.


\subsection{Paired Samples}


\subsection{How to do it with \textsf{R}}

<<>>=
t.test(extra ~ group, data = sleep, paired = TRUE)
@


\section{Other Hypothesis Tests\label{sec:Other-Hypothesis-Tests}}


\subsection{Kolmogorov-Smirnov Goodness-of-Fit Test\label{sub:Kolmogorov-Smirnov-Goodness-of-Fit-Test}}


\subsection{How to do it with \textsf{R}}

<<>>=
ks.test(randu$x, "punif")
@


\subsection{Shapiro-Wilk Normality Test\label{sub:Shapiro-Wilk-Normality-Test}}


\subsection{How to do it with \textsf{R}}

<<>>=
shapiro.test(women$height)
@


\section{Analysis of Variance\label{sec:Analysis-of-Variance}}


\subsection{How to do it with \textsf{R}}

I am thinking 

<<>>=
with(chickwts, by(weight, feed, shapiro.test))
@

and

<<>>=
temp <- lm(weight ~ feed, data = chickwts)
@

and 

<<>>=
anova(temp)
@

Plot for the intuition of between versus within group variation.

%
\begin{figure}
\begin{centering}
<<echo = FALSE, fig = true, height = 4.5, width = 6>>=
y1 <- rnorm(300, mean = c(2,8,22))
plot(y1, xlim = c(-1,25), ylim = c(0,0.45) , type = "n")
f <- function(x){dnorm(x, mean = 2)}
curve(f, from = -1, to = 5, add = TRUE, lwd = 2)
f <- function(x){dnorm(x, mean = 8)}
curve(f, from = 5, to = 11, add = TRUE, lwd = 2)
f <- function(x){dnorm(x, mean = 22)}
curve(f, from = 19, to = 25, add = TRUE, lwd = 2)
rug(y1)
@
\par\end{centering}

\caption{Between group versus within group variation\label{fig:Between-versus-within}}

\end{figure}




Plots for the hypothesis tests:

%
\begin{figure}
\begin{centering}
<<echo = FALSE, fig = true, height = 4.5, width = 6>>=
y2 <- rnorm(300, mean = c(4,4.1,4.3))
hist(y2, 30, prob = TRUE)
f <- function(x){dnorm(x, mean = 4)/3}
curve(f, add = TRUE, lwd = 2)
f <- function(x){dnorm(x, mean = 4.1)/3}
curve(f, add = TRUE, lwd = 2)
f <- function(x){dnorm(x, mean = 4.3)/3}
curve(f, add = TRUE, lwd = 2)
@
\par\end{centering}

\caption{Between group versus within group variation\label{fig:Between-versus-within-2}}

\end{figure}
%
\begin{figure}
\begin{centering}
<<echo = FALSE, fig = true, height = 4.5, width = 6>>=
library(HH)
old.omd <- par(omd = c(.05,.88, .05,1))
F.setup(df1 = 5, df2 = 30)
F.curve(df1 = 5, df2 = 30, col='blue')
F.observed(3, df1 = 5, df2 = 30)
par(old.omd)
@
\par\end{centering}

\caption{Some \emph{F} plots from the \texttt{HH} package\label{fig:Some-F-plots-HH}}

\end{figure}



\section{Sample Size and Power\label{sec:Sample-Size-and-Power}}

The power function of a test for a parameter $\theta$ is\[
\beta(\theta)=\P_{\theta}(\mbox{Reject }H_{0}),\quad-\infty<\theta<\infty.\]
Here are some properties of power functions:
\begin{enumerate}
\item $\beta(\theta)\leq\alpha$ for any $\theta\in\Theta_{0}$, and $\beta(\theta_{0})=\alpha$.
We interpret this by saying that no matter what value $\theta$ takes
inside the null parameter space, there is never more than a chance
of $\alpha$ of rejecting the null hypothesis. We have controlled
the Type I error rate to be no greater than $\alpha$.
\item $\lim_{n\to\infty}\beta(\theta)=1$ for any fixed $\theta\in\Theta_{1}$.
In other words, as the sample size grows without bound we are able
to detect a nonnull value of $\theta$ with increasing accuracy, no
matter how close it lies to the null parameter space. This may appear
to be a good thing at first glance, but it often turns out to be a
curse. For another interpretation is that our Type II error rate grows
as the sample size increases. 
\end{enumerate}

\subsection{How to do it with \textsf{R}}

I am thinking about \inputencoding{latin9}\lstinline!replicate!\inputencoding{utf8}\index{replicate@\texttt{replicate}}
here, and also \inputencoding{latin9}\lstinline!power.examp!\inputencoding{utf8}\index{power.examp@\texttt{power.examp}}
from the \inputencoding{latin9}\lstinline!TeachingDemos!\inputencoding{utf8}
package. There is an even better plot in upcoming work from the HH
package.

%
\begin{figure}
\begin{centering}
<<echo = FALSE, fig=true, height = 6, width = 6>>=
library(TeachingDemos)
power.examp()
@
\par\end{centering}

\caption{Plot of significance level and power\label{fig:power-examp}}


~

{\small This graph was generated by the }\texttt{\small power.examp}{\small{}
function from the }\texttt{\small TeachingDemos}{\small{} package.
The plot corresponds to the hypothesis test $H_{0}:\,\mu=\mu_{0}$
versus $H_{1}:\,\mu=\mu_{1}$ (where $\mu_{0}=0$ and $\mu_{1}=1$,
by default) based on a single observation $X\sim\mathsf{norm}(\mathtt{mean}=\mu,\,\mathtt{sd}=\sigma)$.
The top graph is of the $H_{0}$ density while the bottom is of the
$H_{1}$ density. The significance level is set at $\alpha=0.05$,
the sample size is $n=1$, and the standard deviation is $\sigma=1$.
The pink area is the significance level, and the critical value $z_{0.05}\approx1.645$
is marked at the left boundary -- this defines the rejection region.
When $H_{0}$ is true, the probability of falling in the rejection
region is exactly $\alpha=0.05$. The same rejection region is marked
on the bottom graph, and the probability of falling in it (when $H_{1}$
is true) is the blue area shown at the top of the display to be approximately
$0.26$. This probability represents the }\emph{\small power}{\small{}
to detect a non-null mean value of $\mu=1$. With the command the
}\texttt{\small run.power.examp()}{\small{} at the command line the
same plot opens, but in addition, there are sliders available that
allow the user to interactively change the sample size $n$, the standard
deviation $\sigma$, the true difference between the means $\mu_{1}-\mu_{0}$,
and the significance level $\alpha$. By playing around the student
can investigate the effect each of the aforementioned parameters has
on the statistical power. Note that you need the }\texttt{\small tkrplot}{\small{}
package for }\texttt{\small run.power.examp}{\small . }
\end{figure}


\newpage{}


\section*{Chapter Exercises}

\addcontentsline{toc}{section}{Chapter Exercises}
\setcounter{thm}{0}


\chapter{Simple Linear Regression\label{cha:simple-linear-regression}}


\paragraph*{What do I want them to know?}
\begin{itemize}
\item basic philosophy of SLR and the regression assumptions
\item point and interval estimation of the model parameters, and how to
use it to make predictions
\item point and interval estimation of future observations from the model
\item regression diagnostics, including $R^{2}$ and basic residual analysis
\item the concept of influential versus outlying observations, and how to
tell the difference
\end{itemize}

\section{Basic Philosophy\label{sec:Basic-Philosophy}}

Here we have two variables $X$ and $Y$. For our purposes, $X$ is
not random (so we will write $x$), but $Y$ is random. We believe
that $Y$ depends in \emph{some} way on $x$. Some typical examples
of $(x,Y)$ pairs are
\begin{itemize}
\item $x=$ study time and $Y=$ score on a test.
\item $x=$ height and $Y=$ weight.
\item $x=$ smoking frequency and $Y=$ age of first heart attack.
\end{itemize}
Given information about the relationship between $x$ and $Y$, we
would like to \emph{predict} future values of $Y$ for particular
values of $x$. This turns out to be a difficult problem%
\footnote{Yogi Berra once said, {}``It is always difficult to make predictions,
especially about the future.''%
}, so instead we first tackle an easier problem: we estimate $\E Y$.
How can we accomplish this? Well, we know that $Y$ depends somehow
on $x,$ so it stands to reason that\begin{equation}
\E Y=\mu(x),\ \mbox{a function of }x.\end{equation}
But we should be able to say more than that. To focus our efforts
we impose some structure on the functional form of $\mu$. For instance,
\begin{itemize}
\item if $\mu(x)=\beta_{0}+\beta_{1}x$, we try to estimate $\beta_{0}$
and $\beta_{1}$.
\item if $\mu(x)=\beta_{0}+\beta_{1}x+\beta_{2}x^{2}$ , we try to estimate
$\beta_{0}$, $\beta_{1}$, and $\beta_{2}$.
\item if $\mu(x)=\beta_{0}\me^{\beta_{1}x}$, we try to estimate $\beta_{0}$
and $\beta_{1}$.
\end{itemize}
This helps us in the sense that we concentrate on the estimation of
just a few parameters, $\beta_{0}$ and $\beta_{1}$, say, rather
than some nebulous function. Our \emph{modus operandi} is simply to
perform the random experiment $n$ times and observe the $n$ ordered
pairs of data $(x_{1},Y_{1}),\ (x_{2},Y_{2}),\ \ldots,(x_{n},Y_{n})$.
We use these $n$ data points to estimate the parameters.

More to the point, there are \emph{three simple linear regression
(SLR) assumptions}\index{regression assumptions} that will form the
basis for the rest of this chapter:
\begin{assumption}
We assume that $\mu$ is a linear function of $x$, that is, \begin{equation}
\mu(x)=\beta_{0}+\beta_{1}x,\end{equation}
\textup{\emph{where $\beta_{0}$ and $\beta_{1}$ are unknown constants
to be estimated}}\textup{.}
\end{assumption}

\begin{assumption}
We further assume that $Y_{i}$ is $\mu(x_{i})$ -- the {}``signal''
-- plus some {}``error'' (represented by the symbol $\epsilon_{i}$):\begin{equation}
Y_{i}=\beta_{0}+\beta_{1}x_{i}+\epsilon_{i},\quad i=1,2,\ldots,n.\end{equation}

\end{assumption}

\begin{assumption}
We lastly assume that the errors are i.i.d.~normal with mean 0 and
variance $\sigma^{2}$:\begin{equation}
\epsilon_{1},\epsilon_{2},\ldots,\epsilon_{n}\sim\mathsf{norm}(\mathtt{mean}=0,\,\mathtt{sd}=\sigma).\end{equation}
\end{assumption}
\begin{rem}
We assume both the normality of the errors $\epsilon$ and the linearity
of the mean function $\mu$. Recall from Proposition \ref{pro:mvnorm-cond-dist}
of Chapter \ref{cha:Multivariable-Distributions} that if $(X,Y)\sim\mathsf{mvnorm}$
then the mean of $Y|x$ is a linear function of $x$. This is not
a coincidence. In more advanced classes we study the case that both
$X$ and $Y$ are random, and in particular, when they are jointly
normally distributed.
\end{rem}

\subsection*{What does it all mean?}

See Figure \ref{fig:philosophy}. Shown in the figure is a solid line,
the regression line\index{regression line} $\mu$, which in this
display has slope $0.5$ and $y$-intercept 2.5, that is, $\mu(x)=2.5+0.5x$.
The intuition is that for each given value of $x$, we observe a random
value of $Y$ which is normally distributed with a mean equal to the
height of the regression line at that $x$ value. Normal densities
are superimposed on the plot to drive this point home; in principle,
the densities stand outside of the page, perpendicular to the plane
of the paper. The figure shows three such values of $x$, namely,
$x=1$, $x=2.5$, and $x=4$. Not only do we assume that the observations
at the three locations are independent, but we also assume that their
distributions have the same spread. In mathematical terms this means
that the normal densities all along the line have identical standard
deviations -- there is no {}``fanning out'' or {}``scrunching in''
of the normal densities as $x$ increases%
\footnote{In practical terms, this constant variance assumption is often violated,
in that we often observe scatterplots that fan out from the line as
$x$ gets large or small. We say under those circumstances that the
data show \emph{heteroscedasticity}. There are methods to address
it, but they fall outside the realm of SLR. %
}. 
\begin{quotation}
%
\begin{figure}
\begin{centering}
<<echo = FALSE, fig=true, height = 5, width = 6>>=
 # open window
plot(c(0,5), c(0,6.5), type = "n", xlab="x", ylab="y")
## the x- and y-axes
abline(h=0, v=0, col = "gray60")
# regression line
abline(a = 2.5, b = 0.5, lwd = 2)
# normal curves
x <- 600:3000/600
y <- dnorm(x, mean = 3, sd = 0.5)
lines(y + 1.0, x)
lines(y + 2.5, x + 0.75)
lines(y + 4.0, x + 1.5)
# pretty it up
abline(v = c(1, 2.5, 4), lty = 2, col = "grey")
segments(1,3, 1+dnorm(0,0,0.5),3, lty = 2, col = "gray")
segments(2.5, 3.75, 2.5+dnorm(0,0,0.5), 3.75, lty = 2, col = "gray")
segments(4,4.5, 4+dnorm(0,0,0.5),4.5, lty = 2, col = "gray")
@
\par\end{centering}

\caption{Philosophical foundations of SLR\label{fig:philosophy}}

\end{figure}
\end{quotation}
\begin{example}
\label{exa:Speed-and-Stopping}\textbf{Speed and stopping distance
of cars.} We will use the data frame \inputencoding{latin9}\lstinline[showstringspaces=false]!cars!\inputencoding{utf8}\index{Data sets!cars@\texttt{cars}}
from the \inputencoding{latin9}\lstinline[showstringspaces=false]!datasets!\inputencoding{utf8}
package. It has two variables: \inputencoding{latin9}\lstinline[showstringspaces=false]!speed!\inputencoding{utf8}
and \inputencoding{latin9}\lstinline[showstringspaces=false]!dist!\inputencoding{utf8}.
We can take a look at some of the values in the data frame: 

<<>>=
head(cars)
@

The \inputencoding{latin9}\lstinline[showstringspaces=false]!speed!\inputencoding{utf8}
represents how fast the car was going ($x$) in miles per hour and
\inputencoding{latin9}\lstinline[showstringspaces=false]!dist!\inputencoding{utf8}
($Y$) measures how far it took the car to stop, in feet. We can make
a simple scatterplot of the data with the command \inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false,tabsize=2]!plot(dist ~ speed, data = cars)!\inputencoding{utf8}. 
\begin{quotation}
%
\begin{figure}
\begin{centering}
<<echo = FALSE, fig=true, height = 4.5, width = 6>>=
plot(dist ~ speed, data = cars)
@
\par\end{centering}

\caption{Scatterplot of \texttt{dist} versus \texttt{speed} for the \texttt{cars}
data\label{fig:Scatter-cars}}

\end{figure}

\end{quotation}
You can see the output in Figure \ref{fig:Scatter-cars}, which was
produced by the following code.

\end{example}
<<eval = FALSE>>=
plot(dist ~ speed, data = cars)
@

There is a pronounced upward trend to the data points, and the pattern
looks approximately linear. There does not appear to be substantial
fanning out of the points or extreme values. 


\section{Estimation\label{sec:SLR-Estimation}}


\subsection{\label{sub:point-estimate-mle-slr}Point Estimates of the Parameters}

Where is $\mu(x)$? In essence, we would like to {}``fit'' a line
to the points. But how do we determine a {}``good'' line? Is there
a \emph{best} line? We will use maximum likelihood\index{maximum likelihood}
to find it. We know:\begin{equation}
Y_{i}=\beta_{0}+\beta_{1}x_{i}+\epsilon_{i},\quad i=1,\ldots,n,\end{equation}
where the $\epsilon_{i}$'s are i.i.d.~$\mathsf{norm}(\mathtt{mean}=0,\,\mathtt{sd}=\sigma)$.
Thus $Y_{i}\sim\mathsf{norm}(\mathtt{mean}=\beta_{0}+\beta_{1}x_{i},\,\mathtt{sd}=\sigma),\ i=1,\ldots,n$.
Furthermore, $Y_{1},\ldots,Y_{n}$ are independent -- but not identically
distributed. The likelihood function\index{likelihood function} is:\begin{alignat}{1}
L(\beta_{0},\beta_{1},\sigma)= & \prod_{i=1}^{n}f_{Y_{i}}(y_{i}),\\
= & \prod_{i=1}^{n}(2\pi\sigma^{2})^{-1/2}\exp\left\{ \frac{-(y_{i}-\beta_{0}-\beta_{1}x_{i})^{2}}{2\sigma^{2}}\right\} ,\\
= & (2\pi\sigma^{2})^{-n/2}\exp\left\{ \frac{-\sum_{i=1}^{n}(y_{i}-\beta_{0}-\beta_{1}x_{i})^{2}}{2\sigma^{2}}\right\} .\end{alignat}
We take the natural logarithm to get\begin{equation}
\ln L(\beta_{0},\beta_{1},\sigma)=-\frac{n}{2}\ln(2\pi\sigma^{2})-\frac{\sum_{i=1}^{n}(y_{i}-\beta_{0}-\beta_{1}x_{i})^{2}}{2\sigma^{2}}.\label{eq:regML-lnL}\end{equation}
 We would like to maximize this function of $\beta_{0}$ and $\beta_{1}$.
See Appendix \ref{sec:Multivariable-Calculus} which tells us that
we should find critical points by means of the partial derivatives.
Let us start by differentiating with respect to $\beta_{0}$: \begin{equation}
\frac{\partial}{\partial\beta_{0}}\ln L=0-\frac{1}{2\sigma^{2}}\sum_{i=1}^{n}2(y_{i}-\beta_{0}-\beta_{1}x_{i})(-1),\end{equation}
and the partial derivative equals zero when $\sum_{i=1}^{n}(y_{i}-\beta_{0}-\beta_{1}x_{i})=0$,
that is, when\begin{equation}
n\beta_{0}+\beta_{1}\sum_{i=1}^{n}x_{i}=\sum_{i=1}^{n}y_{i}.\label{eq:regML-a}\end{equation}
Moving on, we next take the partial derivative of $\ln L$ (Equation
\ref{eq:regML-lnL}) with respect to $\beta_{1}$ to get \begin{alignat}{1}
\frac{\partial}{\partial\beta_{1}}\ln L=\  & 0-\frac{1}{2\sigma^{2}}\sum_{i=1}^{n}2(y_{i}-\beta_{0}-\beta_{1}x_{i})(-x_{i}),\\
= & \frac{1}{\sigma^{2}}\sum_{i=1}^{n}\left(x_{i}y_{i}-\beta_{0}x_{i}-\beta_{1}x_{i}^{2}\right),\end{alignat}
and this equals zero when the last sum equals zero, that is, when
\begin{equation}
\beta_{0}\sum_{i=1}^{n}x_{i}+\beta_{1}\sum_{i=1}^{n}x_{i}^{2}=\sum_{i=1}^{n}x_{i}y_{i}.\label{eq:regML-b}\end{equation}
Solving the system of equations \ref{eq:regML-a} and \ref{eq:regML-b}
\begin{eqnarray}
n\beta_{0}+\beta_{1}\sum_{i=1}^{n}x_{i} & = & \sum_{i=1}^{n}y_{i}\\
\beta_{0}\sum_{i=1}^{n}x_{i}+\beta_{1}\sum_{i=1}^{n}x_{i}^{2} & = & \sum_{i=1}^{n}x_{i}y_{i}\end{eqnarray}
for $\beta_{0}$ and $\beta_{1}$ (in Exercise \ref{xca:find-mles-SLR})
gives \begin{equation}
\hat{\beta}_{1}=\frac{\sum_{i=1}^{n}x_{i}y_{i}-\left.\left(\sum_{i=1}^{n}x_{i}\right)\left(\sum_{i=1}^{n}y_{i}\right)\right\slash n}{\sum_{i=1}^{n}x_{i}^{2}-\left.\left(\sum_{i=1}^{n}x_{i}\right)^{2}\right\slash n}\label{eq:regline-slope-formula}\end{equation}
and\begin{equation}
\hat{\beta}_{0}=\ybar-\hat{\beta}_{1}\xbar.\end{equation}
The conclusion? To estimate the mean line \begin{equation}
\mu(x)=\beta_{0}+\beta_{1}x,\end{equation}
 we use the {}``line of best fit''\begin{equation}
\hat{\mu}(x)=\hat{\beta}_{0}+\hat{\beta}_{1}x,\end{equation}
where $\hat{\beta}_{0}$ and $\hat{\beta}_{1}$ are given as above.
For notation we will usually write $b_{0}=\hat{\beta_{0}}$ and $b_{1}=\hat{\beta_{1}}$
so that $\hat{\mu}(x)=b_{0}+b_{1}x$.
\begin{rem}
The formula for $b_{1}$ in Equation \ref{eq:regline-slope-formula}
gets the job done but does not really make any sense. There are many
equivalent formulas for $b_{1}$ that are more intuitive, or at the
least are easier to remember. One of the author's favorites is\begin{equation}
b_{1}=r\frac{s_{y}}{s_{x}},\label{eq:sample-correlation-formula}\end{equation}
where $r$, $s_{y}$, and $s_{x}$ are the sample correlation coefficient
and the sample standard deviations of the $Y$ and $x$ data, respectively.
See Exercise \ref{xca:show-alternate-slope-formula}. Also, notice
the similarity between Equation \ref{eq:sample-correlation-formula}
and Equation \ref{eq:population-slope-slr}.
\end{rem}

\subsection*{How to do it with \textsf{R}}

Here we go. \textsf{R} will calculate the linear regression line with
the \inputencoding{latin9}\lstinline[showstringspaces=false]!lm!\inputencoding{utf8}
function. We will store the result in an object which we will call
\inputencoding{latin9}\lstinline[showstringspaces=false]!cars.lm!\inputencoding{utf8}.
Here is how it works:

<<>>=
cars.lm <- lm(dist ~ speed, data = cars)
@

The first part of the input to the \inputencoding{latin9}\lstinline[showstringspaces=false]!lm!\inputencoding{utf8}
function, \inputencoding{latin9}\lstinline[showstringspaces=false]!dist~speed!\inputencoding{utf8},
is a \emph{model formula}, read as {}``\inputencoding{latin9}\lstinline[showstringspaces=false]!dist!\inputencoding{utf8}
is described by \inputencoding{latin9}\lstinline[showstringspaces=false]!speed!\inputencoding{utf8}''.
The \inputencoding{latin9}\lstinline[showstringspaces=false]!data = cars!\inputencoding{utf8}
argument tells \textsf{R} where to look for the variables quoted in
the model formula. The output object \inputencoding{latin9}\lstinline[showstringspaces=false]!cars.lm!\inputencoding{utf8}
contains a multitude of information. Let's first take a look at the
coefficients of the fitted regression line, which are extracted by
the \inputencoding{latin9}\lstinline[showstringspaces=false]!coef!\inputencoding{utf8}
function%
\footnote{Alternatively, we could just type \inputencoding{latin9}\lstinline[showstringspaces=false]!cars.lm!\inputencoding{utf8}
to see the same thing.%
}:

<<>>=
coef(cars.lm)
@

The parameter estimates $b_{0}$ and $b_{1}$ for the intercept and
slope, respectively, are shown above. The regression line is thus
given by $\hat{\mu}(\mathtt{speed})=$ \Sexpr{round(coef(cars.lm)[1],2)}
$+$ \Sexpr{round(coef(cars.lm)[2],2)}$\mathtt{speed}$.

It is good practice to visually inspect the data with the regression
line added to the plot. To do this we first scatterplot the original
data and then follow with a call to the \inputencoding{latin9}\lstinline[showstringspaces=false]!abline!\inputencoding{utf8}
function. The inputs to \inputencoding{latin9}\lstinline[showstringspaces=false]!abline!\inputencoding{utf8}
are the coefficients of \inputencoding{latin9}\lstinline[showstringspaces=false]!cars.lm!\inputencoding{utf8}
(see Figure \ref{fig:Scatter-cars-regline}): %
\begin{figure}
\begin{centering}
<<echo = FALSE, fig=true, height = 4.5, width = 6>>=
plot(dist ~ speed, data = cars, pch = 16)
abline(coef(cars.lm))
@
\par\end{centering}

\caption{Scatterplot with added regression line for the \texttt{cars} data\label{fig:Scatter-cars-regline}}

\end{figure}


<<eval = FALSE>>=
plot(dist ~ speed, data = cars, pch = 16)
abline(coef(cars))
@

To calculate points on the regression line we may simply plug the
desired $x$ value(s) into $\hat{\mu}$, either by hand, or with the
\inputencoding{latin9}\lstinline[showstringspaces=false]!predict!\inputencoding{utf8}
function. The inputs to \inputencoding{latin9}\lstinline[showstringspaces=false]!predict!\inputencoding{utf8}
are the fitted linear model object, \inputencoding{latin9}\lstinline[showstringspaces=false]!cars.lm!\inputencoding{utf8},
and the desired $x$ value(s) represented by a data frame. See the
example below.
\begin{example}
\label{exa:regline-cars-interpret}Using the regression line for the
\inputencoding{latin9}\lstinline[showstringspaces=false]!cars!\inputencoding{utf8}
data:
\begin{enumerate}
\item What is the meaning of $\mu(60)=\beta_{0}+\beta_{1}(8$)? 


This represents the average stopping distance (in feet) for a car
going 8\,mph. 

\item Interpret the slope $\beta_{1}$. 


The true slope $\beta_{1}$ represents the increase in average stopping
distance for each mile per hour faster that the car drives. In this
case, we estimate the car to take approximately \Sexpr{round(coef(cars.lm)[2],2)}
additional feet to stop for each additional mph increase in speed.

\item Interpret the intercept $\beta_{0}$. 


This would represent the mean stopping distance for a car traveling
0\,mph (which our regression line estimates to be \Sexpr{round(coef(cars.lm)[1],2)}).
Of course, this interpretation does not make any sense for this example,
because a car travelling 0\,mph takes 0\,ft to stop (it was not
moving in the first place)! What went wrong? Looking at the data,
we notice that the smallest speed for which we have measured data
is 4\,mph. Therefore, if we predict what would happen for slower
speeds then we would be \emph{extrapolating}, a dangerous practice
which often gives nonsensical results.

\end{enumerate}
\end{example}

\subsection{Point Estimates of the Regression Line\label{sub:slr-point-est-regline}}

We said at the beginning of the chapter that our goal was to estimate
$\mu=\E Y$, and the arguments in Section \ref{sub:point-estimate-mle-slr}
showed how to obtain an estimate $\hat{\mu}$ of $\mu$ when the regression
assumptions hold. Now we will reap the benefits of our work in more
ways than we previously disclosed. Given a particular value $x_{0}$,
there are two values we would like to estimate:
\begin{enumerate}
\item the mean value of $Y$ at $x_{0}$, and
\item a future value of $Y$ at $x_{0}$.
\end{enumerate}
The first is a number, $\mu(x_{0})$, and the second is a random variable,
$Y(x_{0})$, but our point estimate is the same for both: $\hat{\mu}(x_{0})$.
\begin{example}
\label{exa:regline-cars-pe-8mph}We may use the regression line to
obtain a point estimate of the mean stopping distance for a car traveling
8\,mph: $\hat{\mu}(15)=b_{0}+8b_{1}\approx$ \Sexpr{round(coef(cars.lm)[1],2)}
$+(8)$ (\Sexpr{round(coef(cars.lm)[2],2)})$\approx13.88$. We would
also use 13.88 as a point estimate for the stopping distance of a
future car traveling 8\,mph.
\end{example}
Note that we actually have observed data for a car traveling 8\,mph;
its stopping distance was 16\,ft as listed in the fifth row of the
\inputencoding{latin9}\lstinline[showstringspaces=false]!cars!\inputencoding{utf8}
data:

<<>>=
cars[5, ]
@

There is a special name for estimates $\hat{\mu}(x_{0})$ when $x_{0}$
matches an observed value $x_{i}$ from the data set. They are called
\emph{fitted values}, they are denoted by $\hat{Y}_{1}$, $\hat{Y}_{2}$,
\ldots{}, $\hat{Y}_{n}$ (ignoring repetition), and they play an
important role in the sections that follow. 

In an abuse of notation we will sometimes write $\hat{Y}$ or $\hat{Y}(x_{0})$
to denote a point on the regression line even when $x_{0}$ does not
belong to the original data if the context of the statement obviates
any danger of confusion.

We saw in Example \ref{exa:regline-cars-interpret} that spooky things
can happen when we are cavalier about point estimation. While it is
usually acceptable to predict/estimate at values of $x_{0}$ that
fall within the range of the original $x$ data, it is reckless to
use $\hat{\mu}$ for point estimates at locations outside that range.
Such estimates are usually worthless. \emph{Do not extrapolate} unless
there are compelling external reasons, and even then, temper it with
a good deal of caution.


\subsection*{How to do it with \textsf{R}}

The fitted values are automatically computed as a byproduct of the
model fitting procedure and are already stored as a component of the
\inputencoding{latin9}\lstinline[showstringspaces=false]!cars.lm!\inputencoding{utf8}
object. We may access them with the \inputencoding{latin9}\lstinline[showstringspaces=false]!fitted!\inputencoding{utf8}
function (we only show the first five entries):

<<>>=
fitted(cars.lm)[1:5]
@

Predictions at $x$ values that are not necessarily part of the original
data are done with the \inputencoding{latin9}\lstinline[showstringspaces=false]!predict!\inputencoding{utf8}
function. The first argument is the original \inputencoding{latin9}\lstinline[showstringspaces=false]!cars.lm!\inputencoding{utf8}
object and the second argument \inputencoding{latin9}\lstinline[showstringspaces=false]!newdata!\inputencoding{utf8}
accepts a dataframe (in the same form that was used to fit \inputencoding{latin9}\lstinline[showstringspaces=false]!cars.lm!\inputencoding{utf8})
that contains the locations at which we are seeking predictions.

Let us predict the average stopping distances of cars traveling 6\,mph,
8\,mph, and 21\,mph:

<<>>=
predict(cars.lm, newdata = data.frame(speed = c(6, 8, 21)))
@

Note that there were no observed cars that traveled 6\,mph or 21\,mph.
Also note that our estimate for a car traveling 8\,mph matches the
value we computed by hand in Example \ref{exa:regline-cars-pe-8mph}.


\subsection{Mean Square Error and Standard Error}

To find the MLE of $\sigma^{2}$ we consider the partial derivative\begin{equation}
\frac{\partial}{\partial\sigma^{2}}\ln L=\frac{n}{2\sigma^{2}}-\frac{1}{2(\sigma^{2})^{2}}\sum_{i=1}^{n}(y_{i}-\beta_{0}-\beta_{1}x_{i})^{2},\end{equation}
and after plugging in $\hat{\beta}_{0}$ and $\hat{\beta}_{1}$ and
setting equal to zero we get\begin{equation}
\hat{\sigma^{2}}=\frac{1}{n}\sum_{i=1}^{n}(y_{i}-\hat{\beta}_{0}-\hat{\beta}_{1}x_{i})^{2}=\frac{1}{n}\sum_{i=1}^{n}[y_{i}-\hat{\mu}(x_{i})]^{2}.\end{equation}
We write $\hat{Yi}=\hat{\mu}(x_{i})$, and we let $E_{i}=Y_{i}-\hat{Y_{i}}$
be the $i^{\mathrm{th}}$ \emph{residual}. We see \begin{equation}
n\hat{\sigma^{2}}=\sum_{i=1}^{n}E_{i}^{2}=SSE=\mbox{ the sum of squared errors.}\end{equation}
For a point estimate of $\sigma^{2}$ we use the \emph{mean square
error} $S^{2}$ defined by \begin{equation}
S^{2}=\frac{SSE}{n-2},\end{equation}
and we estimate $\sigma$ with the \emph{standard error} $S=\sqrt{S^{2}}$.
%
\footnote{Be careful not to confuse the mean square error $S^{2}$ with the
sample variance $S^{2}$ in Chapter \ref{cha:Describing-Data-Distributions}.
Other notation the reader may encounter is the lowercase $s^{2}$
or the bulky $MSE$.%
}


\subsection*{How to do it with \textsf{R}}

The residuals for the model may be obtained with the \inputencoding{latin9}\lstinline[showstringspaces=false]!residuals!\inputencoding{utf8}
function; we only show the first few entries in the interest of space:

<<>>=
residuals(cars.lm)[1:5]
@

In the last section, we calculated the fitted value for $x=8$ and
found it to be approximately $\hat{\mu}(8)\approx$\Sexpr{round(predict(cars.lm, newdata = data.frame(speed = 8)), 2)}.
Now, it turns out that there was only one recorded observation at
$x=8$, and we have seen this value in the output of \inputencoding{latin9}\lstinline[showstringspaces=false]!head(cars)!\inputencoding{utf8}
in Example \ref{exa:Speed-and-Stopping}; it was $\mathtt{dist}=16$\,ft
for a car with $\mathtt{speed}=8$\,mph. Therefore, the residual
should be $E=Y-\hat{Y}$ which is $E\approx16-$\Sexpr{round(predict(cars.lm, newdata = data.frame(speed = 8)), 2)}.
Now take a look at the last entry of \inputencoding{latin9}\lstinline[showstringspaces=false]!residuals(cars.lm)!\inputencoding{utf8},
above. It is not a coincidence.

The estimate $S$ for $\sigma$ is called the \inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false]!Residual standard error!\inputencoding{utf8}
and for the \inputencoding{latin9}\lstinline[showstringspaces=false]!cars!\inputencoding{utf8}
data is shown a few lines up on the \inputencoding{latin9}\lstinline[showstringspaces=false]!summary(cars.lm)!\inputencoding{utf8}
output (see How to do it with \textsf{R} in Section \ref{sub:slr-interval-est-params}).
We may read it from there to be $S\approx\,$\Sexpr{round(summary(cars.lm)$sigma, 2)},
or we can access it directly from the \inputencoding{latin9}\lstinline[showstringspaces=false]!summary!\inputencoding{utf8}
object.

<<>>=
carsumry <- summary(cars.lm)
carsumry$sigma
@


\subsection{Interval Estimates of the Parameters\label{sub:slr-interval-est-params}}

We discussed general interval estimation in Chapter \ref{cha:Estimation}.
There we found that we could use what we know about the sampling distribution
of certain statistics to construct confidence intervals for the parameter
being estimated. We will continue in that vein, and to get started
we will determine the sampling distributions of the parameter estimates,
$b_{1}$ and $b_{0}$.

To that end, we can see from Equation \ref{eq:regline-slope-formula}
(and it is made clear in Chapter \ref{cha:multiple-linear-regression})
that $b_{1}$ is just a linear combination of normally distributed
random variables, so $b_{1}$ is normally distributed too. Further,
it can be shown that\begin{equation}
b_{1}\sim\mathsf{norm}\left(\mathtt{mean}=\beta_{1},\,\mathtt{sd}=\sigma_{b_{1}}\right)\end{equation}
 where \begin{equation}
\sigma_{b_{1}}=\frac{\sigma}{\sqrt{\sum_{i=1}^{n}(x_{i}-\xbar)^{2}}}\end{equation}
is called \emph{the standard error of} $b_{1}$ which unfortunately
depends on the unknown value of $\sigma$. We do not lose heart, though,
because we can estimate $\sigma$ with the standard error $S$ from
the last section. This gives us an estimate $S_{b_{1}}$ for $\sigma_{b_{1}}$
defined by\begin{equation}
S_{b_{1}}=\frac{S}{\sqrt{\sum_{i=1}^{n}(x_{i}-\xbar)^{2}}}.\end{equation}
Now, it turns out that $b_{0}$, $b_{1}$, and $S$ are mutually independent
(see the footnote in Section \ref{sub:mlr-interval-est-params}).
Therefore, the quantity\begin{equation}
T=\frac{b_{1}-\beta_{1}}{S_{b_{1}}}\end{equation}
has a $\mathsf{t}(\mathtt{df}=n-2)$ distribution. Therefore, a $100(1-\alpha)\%$
confidence interval for $\beta_{1}$ is given by \begin{equation}
b_{1}\pm\mathsf{t}_{\alpha/2}(\mathtt{df}=n-1)\, S_{b_{1}}\end{equation}


It is also sometimes of interest to construct a confidence interval
for $\beta_{0}$ in which case we will need the sampling distribution
of $b_{0}$. It is shown in Chapter \ref{cha:multiple-linear-regression}
that\begin{equation}
b_{0}\sim\mathsf{norm}\left(\mathtt{mean}=\beta_{0},\,\mathtt{sd}=\sigma_{b_{0}}\right),\end{equation}
where $\sigma_{b_{0}}$ is given by\begin{equation}
\sigma_{b_{0}}=\sigma\sqrt{\frac{1}{n}+\frac{\xbar^{2}}{\sum_{i=1}^{n}(x_{i}-\xbar)^{2}}},\end{equation}
and which we estimate with the $S_{b_{0}}$ defined by\begin{equation}
S_{b_{0}}=S\sqrt{\frac{1}{n}+\frac{\xbar^{2}}{\sum_{i=1}^{n}(x_{i}-\xbar)^{2}}}.\end{equation}
Thus the quantity\begin{equation}
T=\frac{b_{0}-\beta_{0}}{S_{b_{0}}}\end{equation}
has a $\mathsf{t}(\mathtt{df}=n-2)$ distribution and a $100(1-\alpha)\%$
confidence interval for $\beta_{0}$ is given by\begin{equation}
b_{0}\pm\mathsf{t}_{\alpha/2}(\mathtt{df}=n-1)\, S_{b_{0}}\end{equation}



\subsection*{How to do it with \textsf{R}}

Let us take a look at the output from \inputencoding{latin9}\lstinline[showstringspaces=false]!summary(cars.lm)!\inputencoding{utf8}:

<<>>=
summary(cars.lm)
@
<<echo = FALSE>>=
A <- round(summary(cars.lm)$coef, 3)
B <- round(confint(cars.lm), 3)
@

In the \inputencoding{latin9}\lstinline[showstringspaces=false]!Coefficients!\inputencoding{utf8}
section we find the parameter estimates and their respective standard
errors in the second and third columns; the other columns are discussed
in Section \ref{sec:Model-Utility-SLR}. If we wanted, say, a 95\%
confidence interval for $\beta_{1}$ we could use $b_{1}=\ $\Sexpr{A[2,1]}
and $S_{b_{1}}=\ $\Sexpr{A[2,2]} together with a $\mathsf{t}_{0.025}(\mathtt{df}=23)$
critical value to calculate $b_{1}\pm\mathsf{t}_{0.025}(\mathtt{df}=23)S_{b_{1}}$.

Or, we could use the \inputencoding{latin9}\lstinline[showstringspaces=false]!confint!\inputencoding{utf8}
function.

<<>>=
confint(cars.lm)
@

With 95\% confidence, the random interval {[}\Sexpr{B[2,1]}, \Sexpr{B[2,2]}{]}
covers the parameter $\beta_{1}$.


\subsection{Interval Estimates of the Regression Line\label{sub:slr-interval-est-regline}}

We have seen how to estimate the coefficients of regression line with
both point estimates and confidence intervals. We even saw how to
estimate a value $\hat{\mu}(x)$ on the regression line for a given
value of $x$, such as $x=15$. 

But how good is our estimate $\hat{\mu}(15)$? How much confidence
do we have in \emph{this} estimate? Furthermore, suppose we were going
to observe another value of $Y$ at $x=15$. What could we say?

Intuitively, it should be easier to get bounds on the mean (average)
value of $Y$ at $x_{0}$ (called a \emph{confidence interval for
the mean value of $Y$ at $x_{0}$}) than it is to get bounds on a
future observation of $Y$ (called a \emph{prediction interval for
$Y$ at $x_{0}$}). As we shall see, the intuition serves us well
and confidence intervals are shorter for the mean value, longer for
the individual value.

Our point estimate of $\mu(x_{0})$ is of course $\hat{Y}=\hat{Y}(x_{0})$,
so for a confidence interval we will need to know $\hat{Y}$'s sampling
distribution. It turns out (see Section ) that $\hat{Y}=\hat{\mu}(x_{0})$
is distributed\begin{equation}
\hat{Y}\sim\mathsf{norm}\left(\mathtt{mean}=\mu(x_{0}),\:\mathtt{sd}=\sigma\sqrt{\frac{1}{n}+\frac{(x_{0}-\xbar)^{2}}{\sum_{i=1}^{n}(x_{i}-\xbar)^{2}}}\right).\end{equation}
Since $\sigma$ is unknown we estimate it with $S$ (we should expect
the appearance of a $\mathsf{t}(\mathtt{df}=n-2)$ distribution in
the near future). 

A $100(1-\alpha)\%$ \emph{confidence interval (CI) for} $\mu(x_{0})$
is given by\begin{equation}
\hat{Y}\pm\mathsf{t}_{\alpha/2}(\mathtt{df}=n-2)\, S\sqrt{\frac{1}{n}+\frac{(x_{0}-\xbar^{2})}{\sum_{i=1}^{n}(x_{i}-\xbar)^{2}}}.\label{eq:SLR-conf-int-formula}\end{equation}


It is time for prediction intervals, which are slightly different.
In order to find confidence bounds for a new observation of $Y$ (we
will denote it $Y_{\mbox{new}}$) we use the fact that\begin{equation}
Y_{\mbox{new}}\sim\mathtt{norm}\left(\mathtt{mean}=\mu(x_{0}),\,\mathtt{sd}=\sigma\sqrt{1+\frac{1}{n}+\frac{(x_{0}-\xbar)^{2}}{\sum_{i=1}^{n}(x_{i}-\xbar)^{2}}}\right).\end{equation}
Of course $\sigma$ is unknown and we estimate it with $S$. Thus,
a $100(1-\alpha)\%$ prediction interval (PI) for a future value of
$Y$ at $x_{0}$ is given by \begin{equation}
\hat{Y}(x_{0})\pm\mathsf{t}_{\alpha/2}(\mathtt{df}=n-1)\: S\,\sqrt{1+\frac{1}{n}+\frac{(x_{0}-\xbar)^{2}}{\sum_{i=1}^{n}(x_{i}-\xbar)^{2}}}.\label{eq:SLR-pred-int-formula}\end{equation}
We notice that the prediction interval in Equation \ref{eq:SLR-pred-int-formula}
is wider than the confidence interval in Equation \ref{eq:SLR-conf-int-formula},
as we expected at the beginning of the section.


\subsection*{How to do it with \textsf{R}}

Confidence and prediction intervals are calculated in \textsf{R} with
the \inputencoding{latin9}\lstinline[showstringspaces=false]!predict!\inputencoding{utf8}\index{predict@\texttt{predict}}
function, which we encountered in Section \ref{sub:slr-point-est-regline}.
There we neglected to take advantage of its additional \inputencoding{latin9}\lstinline[showstringspaces=false]!interval!\inputencoding{utf8}
argument. The general syntax follows.
\begin{example}
We will find confidence and prediction intervals for the stopping
distance of a car travelling 5, 6, and 21\,mph (note from the graph
that there are no collected data for these speeds). We have computed
\inputencoding{latin9}\lstinline[showstringspaces=false]!cars.lm!\inputencoding{utf8}
earlier, and we will use this for input to the \inputencoding{latin9}\lstinline[showstringspaces=false]!predict!\inputencoding{utf8}
function. Also, we need to tell \textsf{R} the values of $x_{0}$
at which we want the predictions made, and store the $x_{0}$ values
in a data frame whose variable is labeled with the correct name. \emph{This
is important}. 

<<>>=
new <- data.frame(speed = c(5, 6, 21))
@

Next we instruct \textsf{R} to calculate the intervals. Confidence
intervals are given by

<<>>=
predict(cars.lm, newdata = new, interval = "confidence")
@

<<echo = FALSE, results = hide>>=
carsCI <- round(predict(cars.lm, newdata = new, interval = "confidence"), 2)
@

Prediction intervals are given by

<<>>=
predict(cars.lm, newdata = new, interval = "prediction")
@

<<echo = FALSE, results = hide>>=
carsPI <- round(predict(cars.lm, newdata = new, interval = "prediction"), 2)
@

\end{example}
The type of interval is dictated by the \inputencoding{latin9}\lstinline[showstringspaces=false]!interval!\inputencoding{utf8}
argument (which is \inputencoding{latin9}\lstinline[showstringspaces=false]!none!\inputencoding{utf8}
by default), and the default confidence level is 95\% (which can be
changed with the \inputencoding{latin9}\lstinline[showstringspaces=false]!level!\inputencoding{utf8}
argument). 
\begin{example}
Using the \inputencoding{latin9}\lstinline[showstringspaces=false]!cars!\inputencoding{utf8}
data,
\begin{enumerate}
\item Report a point estimate of and a 95\% confidence interval for the
mean stopping distance for a car travelling 5\,mph.


The fitted value for $x=5$ is \Sexpr{carsCI[1, 1]}, so a point estimate
would be \Sexpr{carsCI[1, 1]}\,ft. The 95\% CI is given by {[}\Sexpr{carsCI[1, 2]},
\Sexpr{carsCI[1, 3]}{]}, so with 95\% confidence the mean stopping
distance lies somewhere between \Sexpr{carsCI[1, 2]}\,ft and \Sexpr{carsCI[1, 3]}\,ft.

\item Report a point prediction for and a 95\% prediction interval for the
stopping distance of a hypothetical car travelling 21\,mph.


The fitted value for $x=21$ is \Sexpr{carsPI[3, 1]}, so a point
prediction for the stopping distance is \Sexpr{carsPI[3, 1]}\,ft.
The 95\% PI is given by {[}\Sexpr{carsPI[3, 2]}, \Sexpr{carsPI[3, 3]}{]},
so with 95\% confidence we may assert that the hypothetical stopping
distance for a car travelling 21\,mph would lie somewhere between
\Sexpr{carsPI[3, 2]}\,ft and \Sexpr{carsPI[3, 3]}\,ft.

\end{enumerate}
\end{example}

\subsection*{Graphing the Confidence and Prediction Bands}

We earlier guessed that a bound on the value of a single new observation
would be inherently less certain than a bound for an average (mean)
value; therefore, we expect the CIs for the mean to be tighter than
the PIs for a new observation. A close look at the standard deviations
in Equations \ref{eq:SLR-conf-int-formula} and \ref{eq:SLR-pred-int-formula}
confirms our guess, but we would like to see a picture to drive the
point home.

We may plot the confidence and prediction intervals with one fell
swoop using the \inputencoding{latin9}\lstinline[showstringspaces=false]!ci.plot!\inputencoding{utf8}
function from the \inputencoding{latin9}\lstinline[showstringspaces=false]!HH!\inputencoding{utf8}
package \cite{Heibergerhh}. The graph is displayed in Figure \ref{fig:Scatter-cars-CIPI}.
%
\begin{figure}
\begin{centering}
<<echo = FALSE, fig=true, height = 4.5, width = 6>>=
library(HH)
print(ci.plot(cars.lm))
@
\par\end{centering}

\caption{Scatterplot with confidence/prediction bands for the \texttt{cars}
data\label{fig:Scatter-cars-CIPI}}

\end{figure}


<<eval = FALSE>>=
library(HH)
ci.plot(cars.lm)
@

Notice that the bands curve outward away from the regression line
as the $x$ values move away from the center. This is expected once
we notice the $(x_{0}-\xbar)^{2}$ term in the standard deviation
formulas in Equations \ref{eq:SLR-conf-int-formula} and \ref{eq:SLR-pred-int-formula}.


\section{Model Utility and Inference\label{sec:Model-Utility-SLR}}


\subsection{Hypothesis Tests for the Parameters\label{sub:slr-hypoth-test-params}}

Much of the attention of SLR is directed toward $\beta_{1}$ because
when $\beta_{1}\neq0$ the mean value of $Y$ increases (or decreases)
as $x$ increases. Further, if $\beta_{1}=0$ then the mean value
of $Y$ remains the same, regardless of the value of $x$ (when the
regression assumptions hold, of course). It is thus very important
to decide whether or not $\beta_{1}=0$. We address the question with
a statistical test of the null hypothesis $H_{0}:\beta_{1}=0$ versus
the alternative hypothesis $H_{1}:\beta_{1}\neq0$, and to do that
we need to know the sampling distribution of $b_{1}$ when the null
hypothesis is true.

To this end we already know from Section \ref{sub:slr-interval-est-params}
that the quantity\begin{equation}
T=\frac{b_{1}-\beta_{1}}{S_{b_{1}}}\end{equation}
has a $\mathsf{t}(\mathtt{df}=n-2)$ distribution; therefore, when
$\beta_{1}=0$ the quantity $b_{1}/S_{b_{1}}$ has a $\mathsf{t}(\mathtt{df}=n-2)$
distribution and we can compute a $p$-value by comparing the observed
value of $b_{1}/S{}_{b_{1}}$ with values under a $\mathsf{t}(\mathtt{df}=n-2)$
curve. 

Similarly, we may test the hypothesis $H_{0}:\beta_{0}=0$ versus
the alternative $H_{1}:\beta_{0}\neq0$ with the statistic $T=b_{0}/S_{b_{0}}$,
where $S_{b_{0}}$ is given in Section \ref{sub:slr-interval-est-params}.
The test is conducted the same way as for $\beta_{1}$. 


\subsection*{How to do it with \textsf{R}}

Let us take another look at the output from \inputencoding{latin9}\lstinline[showstringspaces=false]!summary(cars.lm)!\inputencoding{utf8}:

<<>>=
summary(cars.lm)
@
<<echo = FALSE>>=
A <- round(summary(cars.lm)$coef, 3)
B <- round(confint(cars.lm), 3)
@

In the \inputencoding{latin9}\lstinline[showstringspaces=false]!Coefficients!\inputencoding{utf8}
section we find the $t$ statistics and the $p$-values associated
with the tests that the respective parameters are zero in the fourth
and fifth columns. Since the $p$-values are (much) less than 0.05,
we conclude that there is strong evidence that the parameters $\beta_{1}\neq0$
and $\beta_{0}\neq0$, and as such, we say that there is a statistically
significant linear relationship between \inputencoding{latin9}\lstinline[showstringspaces=false]!dist!\inputencoding{utf8}
and \inputencoding{latin9}\lstinline[showstringspaces=false]!speed!\inputencoding{utf8}. 


\subsection{Simple Coefficient of Determination}

It would be nice to have a single number that indicates how well our
linear regression model is doing, and the \emph{simple coefficient
of determination} is designed for that purpose. In what follows, we
observe the values $Y_{1}$, $Y_{2}$, \ldots{},$Y_{n}$, and the
goal is to estimate $\mu(x_{0})$, the mean value of $Y$ at the location
$x_{0}$. 

If we disregard the dependence of $Y$ and $x$ and base our estimate
only on the $Y$ values then a reasonable choice for an estimator
is just the MLE of $\mu$, which is $\Ybar$. Then the errors incurred
by the estimate are just $Y_{i}-\Ybar$ and the variation about the
estimate as measured by the sample variance is proportional to \begin{equation}
SSTO=\sum_{i=1}^{n}(Y_{i}-\Ybar)^{2}.\end{equation}
Here, $SSTO$ is an acronym for the \emph{total sum of squares}.

But we do have additional information, namely, we have values $x_{i}$
associated with each value of $Y_{i}$. We have seen that this information
leads us to the estimate $\hat{Y_{i}}$ and the errors incurred are
just the residuals, $E_{i}=Y_{i}-\hat{Y_{i}}$. The variation associated
with these errors can be measured with \begin{equation}
SSE=\sum_{i=1}^{n}(Y_{i}-\hat{Y_{i}})^{2}.\end{equation}
We have seen the $SSE$ before, which stands for the \emph{sum of
squared errors} or \emph{error sum of squares}. Of course, we would
expect the error to be less in the latter case, since we have used
more information. The improvement in our estimation as a result of
the linear regression model can be measured with the difference\[
(Y_{i}-\Ybar)-(Y_{i}-\hat{Y_{i}})=\hat{Y_{i}}-\Ybar,\]
and we measure the variation in these errors with\begin{equation}
SSR=\sum_{i=1}^{n}(\hat{Y_{i}}-\Ybar)^{2},\end{equation}
also known as the \emph{regression sum of squares}. It is not obvious,
but some algebra proved a famous result known as the \textbf{ANOVA
Equality}:\begin{equation}
\sum_{i=1}^{n}(Y_{i}-\Ybar)^{2}=\sum_{i=1}^{n}(\hat{Y_{i}}-\Ybar)^{2}+\sum_{i=1}^{n}(Y_{i}-\hat{Y_{i}})^{2}\label{eq:anovaeq}\end{equation}
or in other words,\begin{equation}
SSTO=SSR+SSE.\end{equation}
This equality has a nice interpretation. Consider $SSTO$ to be the
\emph{total variation} of the errors. Think of a decomposition of
the total variation into pieces: one piece measuring the reduction
of error from using the linear regression model, or \emph{explained
variation} ($SSR$), while the other represents what is left over,
that is, the errors that the linear regression model doesn't explain,
or \emph{unexplained variation} ($SSE$). In this way we see that
the ANOVA equality merely partitions the variation into \[
\mbox{total variation}=\mbox{explained variation}+\mbox{unexplained variation}.\]
For a single number to summarize how well our model is doing we use
the simple coefficient of determination $r^{2}$, defined by\begin{equation}
r^{2}=1-\frac{SSE}{SSTO}.\end{equation}
We interpret $r^{2}$ as the proportion of total variation that is
explained by the simple linear regression model. When $r^{2}$ is
large, the model is doing a good job; when $r^{2}$ is small, the
model is not doing a good job.

Related to the simple coefficient of determination is the sample correlation
coefficient, $r$. As you can guess, the way we get $r$ is by the
formula $|r|=\sqrt{r^{2}}$. But how do we get the sign? It is equal
the sign of the slope estimate $b_{1}$. That is, if the regression
line $\hat{\mu}(x)$ has positive slope, then $r=\sqrt{r^{2}}$. Likewise,
if the slope of $\hat{\mu}(x)$ is negative, then $r=-\sqrt{r^{2}}$.


\subsection*{How to do it with \textsf{R}}

The primary method to display partitioned sums of squared errors is
with an \emph{ANOVA table}. The command in \textsf{R} to produce such
a table is \inputencoding{latin9}\lstinline[showstringspaces=false]!anova!\inputencoding{utf8}.
The input to \inputencoding{latin9}\lstinline[showstringspaces=false]!anova!\inputencoding{utf8}
is the result of an \inputencoding{latin9}\lstinline[showstringspaces=false]!lm!\inputencoding{utf8}
call which for the \inputencoding{latin9}\lstinline[showstringspaces=false]!cars!\inputencoding{utf8}
data is \inputencoding{latin9}\lstinline[showstringspaces=false]!cars.lm!\inputencoding{utf8}.

<<>>=
anova(cars.lm)
@

The output gives\[
r^{2}=1-\frac{SSE}{SSR+SSE}=1-\frac{11353.5}{21185.5+11353.5}\approx0.65.\]
 The interpretation should be: {}``The linear regression line accounts
for approximately 65\% of the variation of \inputencoding{latin9}\lstinline[showstringspaces=false]!dist!\inputencoding{utf8}
as explained by \inputencoding{latin9}\lstinline[showstringspaces=false]!speed!\inputencoding{utf8}''.

The value of $r^{2}$ is stored in the \inputencoding{latin9}\lstinline[showstringspaces=false]!r.squared!\inputencoding{utf8}
component of \inputencoding{latin9}\lstinline[showstringspaces=false]!summary(cars.lm)!\inputencoding{utf8},
which we called \inputencoding{latin9}\lstinline[showstringspaces=false]!carsumry!\inputencoding{utf8}.

<<>>=
carsumry$r.squared
@

We already knew this. We saw it in the next to the last line of the
\inputencoding{latin9}\lstinline[showstringspaces=false]!summary(cars.lm)!\inputencoding{utf8}
output where it was called {}``\inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false]!Multiple R-squared!\inputencoding{utf8}''.
Listed right beside it is the \inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false]!Adjusted R-squared!\inputencoding{utf8}
which we will discuss in Chapter \ref{cha:multiple-linear-regression}.

For the \inputencoding{latin9}\lstinline[showstringspaces=false]!cars!\inputencoding{utf8}
data, we find $r$ to be

<<>>=
sqrt(carsumry$r.squared)
@

We choose the principal square root because the slope of the regression
line is positive.


\subsection{Overall \emph{F} statistic\label{sub:slr-overall-F-statistic}}

There is another way to test the significance of the linear regression
model. In SLR, the new way also tests the hypothesis $H_{0}:\beta_{1}=0$
versus $H_{1}:\beta_{1}\neq0$, but it is done with a new test statistic
called the \emph{overall} $F$ \emph{statistic}. It is defined by
\begin{equation}
F=\frac{SSR}{SSE/(n-2)}.\label{eq:slr-overall-F-statistic}\end{equation}
Under the regression assumptions and when $H_{0}$ is true, the $F$
statistic has an $\mathtt{f}(\mathtt{df1}=1,\,\mathtt{df2}=n-2)$
distribution. We reject $H_{0}$ when $F$ is large -- that is, when
the explained variation is large relative to the unexplained variation.

All this being said, we have not yet gained much from the overall
$F$ statistic because we already knew from Section \ref{sub:slr-hypoth-test-params}
how to test $H_{0}:\beta_{1}=0$\ldots{} we use the Student's $t$
statistic. What is worse is that (in the simple linear regression
model) it can be proved that the $F$ in Equation \ref{eq:slr-overall-F-statistic}
is exactly the Student's $t$ statistic for $\beta_{1}$ squared,\begin{equation}
F=\left(\frac{b_{1}}{S_{b_{1}}}\right)^{2}.\end{equation}
So why bother to define the $F$ statistic? Why not just square the
$t$ statistic and be done with it? The answer is that the $F$ statistic
has a more complicated interpretation and plays a more important role
in the multiple linear regression model which we will study in Chapter
\ref{cha:multiple-linear-regression}. See Section \ref{sub:mlr-Overall-F-Test}
for details.


\subsection{How to do it with \textsf{R}}

The overall $F$ statistic and $p$-value are displayed in the bottom
line of the \inputencoding{latin9}\lstinline[showstringspaces=false]!summary(cars.lm)!\inputencoding{utf8}
output. It is also shown in the final columns of \inputencoding{latin9}\lstinline[showstringspaces=false]!anova(cars.lm)!\inputencoding{utf8}:

<<>>=
anova(cars.lm)
@

Here we see that the $F$ statistic is \Sexpr{round(carsumry$fstatistic, 2)}
with a $p$-value very close to zero. The conclusion: there is very
strong evidence that $H_{0}:\beta_{1}=0$ is false, that is, there
is strong evidence that $\beta_{1}\neq0$. Moreover, we conclude that
the regression relationship between \inputencoding{latin9}\lstinline[showstringspaces=false]!dist!\inputencoding{utf8}
and \inputencoding{latin9}\lstinline[showstringspaces=false]!speed!\inputencoding{utf8}
is significant.

Note that the value of the $F$ statistic is the same as the Student's
$t$ statistic for \inputencoding{latin9}\lstinline[showstringspaces=false]!speed!\inputencoding{utf8}
squared.


\section{Residual Analysis\label{sec:Residual-Analysis-SLR}}

We know from our model that $Y=\mu(x)+\epsilon$, or in other words,
$\epsilon=Y-\mu(x)$. Further, we know that $\epsilon\sim\mathsf{norm}(\mathtt{mean}=0,\,\mathtt{sd}=\sigma)$.
We may estimate $\epsilon_{i}$ with the \emph{residual} $E_{i}=Y_{i}-\hat{Y_{i}}$,
where $\hat{Y_{i}}=\hat{\mu}(x_{i})$. If the regression assumptions
hold, then $ $the residuals should be normally distributed. We check
this in Section \ref{sub:Normality-Assumption}. Further, the residuals
should have mean zero with constant variance $\sigma^{2}$, and we
check this in Section \ref{sub:Constant-Variance-Assumption}. Last,
the residuals should be independent, and we check this in Section
\ref{sub:Independence-Assumption}.

In every case, we will begin by looking at residual plots -- that
is, scatterplots of the residuals $E_{i}$ versus index or predicted
values $\hat{Y_{i}}$ -- and follow up with hypothesis tests.


\subsection{Normality Assumption\label{sub:Normality-Assumption}}

We can assess the normality of the residuals with graphical methods
and hypothesis tests. To check graphically whether the residuals are
normally distributed we may look at histograms or \emph{q}-\emph{q}
plots. We first examine a histogram in Figure \ref{fig:Normal-q-q-plot-cars}.
There we see that the distribution of the residuals appears to be
mound shaped, for the most part. We can plot the order statistics
of the sample versus quantiles from a $\mathsf{norm}(\mathtt{mean}=0,\,\mathtt{sd}=1)$
distribution with the command \inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false]!plot(cars.lm, which = 2)!\inputencoding{utf8},
and the results are in Figure \ref{fig:Normal-q-q-plot-cars}. If
the assumption of normality were true, then we would expect points
randomly scattered about the dotted straight line displayed in the
figure. In this case, we see a slight departure from normality in
that the dots show systematic clustering on one side or the other
of the line. The points on the upper end of the plot also appear begin
to stray from the line. We would say there is some evidence that the
residuals are not perfectly normal.

%
\begin{figure}
\begin{centering}
<<echo = FALSE, fig=true, height = 4.5, width = 6>>=
plot(cars.lm, which = 2)
@
\par\end{centering}

\caption{Normal q-q plot of the residuals for the \texttt{cars} data\label{fig:Normal-q-q-plot-cars}}
 {\small Used for checking the normality assumption. Look out for
any curvature or substantial departures from the straight line; hopefully
the dots hug the line closely.}
\end{figure}



\subsubsection*{Testing the Normality Assumption}

Even though we may be concerned about the plots, we can use tests
to determine if the evidence present is statistically significant,
or if it could have happened merely by chance. There are many statistical
tests of normality. We will use the Shapiro-Wilk test, since it is
known to be a good test and to be quite powerful. However, there are
many other fine tests of normality including the Anderson-Darling
test and the Lillefors test, just to mention two of them. 

The Shapiro-Wilk test is based on the statistic\begin{equation}
W=\frac{\left(\sum_{i=1}^{n}a_{i}E_{(i)}\right)^{2}}{\sum_{j=1}^{n}E_{j}^{2}},\end{equation}
where the $E_{(i)}$ are the ordered residuals and the $a_{i}$ are
constants derived from the order statistics of a sample of size $n$
from a normal distribution. See Section \ref{sub:Shapiro-Wilk-Normality-Test}.

We perform the Shapiro-Wilk test below, using the \inputencoding{latin9}\lstinline[showstringspaces=false]!shapiro.test!\inputencoding{utf8}
function from the \inputencoding{latin9}\lstinline[showstringspaces=false]!stats!\inputencoding{utf8}
package. The hypotheses are\[
H_{0}:\mbox{ the residuals are normally distributed }\]
versus\[
H_{1}:\mbox{ the residuals are not normally distributed.}\]
The results from \textsf{R} are

<<>>=
shapiro.test(residuals(cars.lm))
@

For these data we would reject the assumption of normality of the
residuals at the $\alpha=0.05$ significance level, but do not lose
heart, because the regression model is reasonably robust to departures
from the normality assumption. As long as the residual distribution
is not highly skewed, then the regression estimators will perform
reasonably well. Moreover, departures from constant variance and independence
will sometimes affect the quantile plots and histograms, therefore
it is wise to delay final decisions regarding normality until all
diagnostic measures have been investigated.


\subsection{Constant Variance Assumption\label{sub:Constant-Variance-Assumption}}

We will again go to residual plots to try and determine if the spread
of the residuals is changing over time (or index). However, it is
unfortunately not that easy because the residuals do not have constant
variance! In fact, it can be shown that the variance of the residual
$E_{i}$ is \begin{equation}
\mbox{Var\ensuremath{(E_{i})}}=\sigma^{2}(1-h_{ii}),\quad i=1,2,\ldots,n,\end{equation}
where $h_{ii}$ is a quantity called the \emph{leverage} which is
defined below. Consequently, in order to check the constant variance
assumption we must standardize the residuals before plotting. We estimate
the standard error of $E_{i}$ with $s_{E_{i}}=s\sqrt{(1-h_{ii})}$
and define the \emph{standardized residuals} $R_{i}$, $i=1,2,\ldots,n$,
by \begin{equation}
R_{i}=\frac{E_{i}}{s\,\sqrt{1-h_{ii}}},\quad i=1,2,\ldots,n.\end{equation}


For the constant variance assumption we do not need the sign of the
residual so we will plot $\sqrt{|R_{i}|}$ versus the fitted values.
As we look at a scatterplot of $\sqrt{|R_{i}|}$ versus $\hat{Y}_{i}$
we would expect under the regression assumptions to see a constant
band of observations, indicating no change in the magnitude of the
observed distance from the line. We want to watch out for a fanning-out
of the residuals, or a less common funneling-in of the residuals.
Both patterns indicate a change in the residual variance and a consequent
departure from the regression assumptions, the first an increase,
the second a decrease.

In this case, we plot the standardized residuals versus the fitted
values. The graph may be seen in Figure \ref{fig:std-resids-fitted-cars}.
For these data there does appear to be somewhat of a slight fanning-out
of the residuals.

%
\begin{figure}
\begin{centering}
<<echo = FALSE, fig=true, height = 4.5, width = 6>>=
plot(cars.lm, which = 3)
@
\par\end{centering}

\caption{Plot of standardized residuals against the fitted values for the \texttt{cars}
data\label{fig:std-resids-fitted-cars}}
 {\small Used for checking the constant variance assumption. Watch
out for any fanning out (or in) of the dots; hopefully they fall in
a constant band.}
\end{figure}



\subsubsection*{Testing the Constant Variance Assumption}

We will use the Breusch-Pagan test to decide whether the variance
of the residuals is nonconstant. The null hypothesis is that the variance
is the same for all observations, and the alternative hypothesis is
that the variance is not the same for all observations. The test statistic
is found by fitting a linear model to the centered squared residuals\begin{equation}
W_{i}=E_{i}^{2}-\frac{SSE}{n},\quad i=1,2,\ldots,n.\end{equation}
By default the same explanatory variables are used in the new model
which produces fitted values $\hat{W}_{i}$, $i=1,2,\ldots,n$. The
Breusch-Pagan test statistic in \textsf{R} is then calculated with
\begin{equation}
BP=n\sum_{i=1}^{n}\hat{W}_{i}^{2}\div\sum_{i=1}^{n}W_{i}^{2}.\end{equation}
We reject the null hypothesis if $BP$ is too large, which happens
when the explained variation in the new model is large relative to
the unexplained variation in the original model.

We do it in \textsf{R} with the \inputencoding{latin9}\lstinline[showstringspaces=false]!bptest!\inputencoding{utf8}
function from the \inputencoding{latin9}\lstinline[showstringspaces=false]!lmtest!\inputencoding{utf8}
package \cite{Zeileislmtest}. 

<<>>=
library(lmtest)
bptest(cars.lm)
@

For these data we would not reject the null hypothesis at the $\alpha=0.05$
level. There is relatively weak evidence against the assumption of
constant variance. 


\subsection{Independence Assumption\label{sub:Independence-Assumption}}

One of the strongest of the regression assumptions is the one regarding
independence. Departures from the independence assumption are often
exhibited by correlation (or autocorrelation, literally, self-correlation)
present in the residuals. There can be positive or negative correlation.

Positive correlation is displayed by positive residuals followed by
positive residuals, and negative residuals followed by negative residuals.
Looking from left to right, this is exhibited by a cyclical feature
in the residual plots, with long sequences of positive residuals being
followed by long sequences of negative ones.

On the other hand, negative correlation implies positive residuals
followed by negative residuals, which are then followed by positive
residuals, \emph{etc}. Consequently, negatively correlated residuals
are often associated with an alternating pattern in the residual plots.
We examine the residual plot in Figure \ref{fig:resids-fitted-cars}.
There is no obvious cyclical wave pattern or structure to the residual
plot. 

%
\begin{figure}
\begin{centering}
<<echo = FALSE, fig=true, height = 4.5, width = 6>>=
plot(cars.lm, which = 1)
@
\par\end{centering}

\caption{Plot of the residuals versus the fitted values for the \texttt{cars}
data\label{fig:resids-fitted-cars}}


{\small Used for checking the independence assumption. Watch out for
any patterns or structure; hopefully the points are randomly scattered
on the plot.}
\end{figure}



\subsubsection*{Testing the Independence Assumption}

We may statistically test whether there is evidence of autocorrelation
in the residuals with the Durbin-Watson test. The test is based on
the statistic\begin{equation}
D=\frac{\sum_{i=2}^{n}(E_{i}-E_{i-1})^{2}}{\sum_{j=1}^{n}E_{j}^{2}}.\end{equation}
Exact critical values are difficult to obtain, but \textsf{R} will
calculate the \emph{p-}value to great accuracy. It is performed with
the \inputencoding{latin9}\lstinline[showstringspaces=false]!dwtest!\inputencoding{utf8}
function from the \inputencoding{latin9}\lstinline[showstringspaces=false]!lmtest!\inputencoding{utf8}
package. We will conduct a two sided test that the correlation is
not zero, which is not the default (the default is to test that the
autocorrelation is positive).

<<>>=
library(lmtest)
dwtest(cars.lm, alternative = "two.sided")
@

In this case we do not reject the null hypothesis at the $\alpha=0.05$
significance level; there is very little evidence of nonzero autocorrelation
in the residuals.


\subsection{Remedial Measures}

We often find problems with our model that suggest that at least one
of the three regression assumptions is violated. What do we do then?
There are many measures at the statistician's disposal, and we mention
specific steps one can take to improve the model under certain types
of violation.
\begin{description}
\item [{Mean~response~is~not~linear.}] We can directly modify the model
to better approximate the mean response. In particular, perhaps a
polynomial regression function of the form \[
\mu(x)=\beta_{0}+\beta_{1}x_{1}+\beta_{2}x_{1}^{2}\]
would be appropriate. Alternatively, we could have a function of the
form\[
\mu(x)=\beta_{0}\me^{\beta_{1}x}.\]
Models like these are studied in nonlinear regression courses.
\item [{Error~variance~is~not~constant.}] Sometimes a transformation
of the dependent variable will take care of the problem. There is
a large class of them called \emph{Box-Cox transformations}. They
take the form \begin{equation}
Y^{\ast}=Y^{\lambda},\end{equation}
 where $\lambda$ is a constant. (The method proposed by Box and Cox
will determine a suitable value of $\lambda$ automatically by maximum
likelihood). The class contains the transformations \begin{alignat*}{1}
\lambda=2,\quad & Y^{\ast}=Y^{2}\\
\lambda=0.5,\quad & Y^{\ast}=\sqrt{Y}\\
\lambda=0,\quad & Y^{\ast}=\ln\: Y\\
\lambda=-1,\quad & Y^{\ast}=1/Y\end{alignat*}
Alternatively, we can use the method of \emph{weighted least squares}.
This is studied in more detail in later classes. 
\item [{Error~distribution~is~not~normal.}] The same transformations
for stabilizing the variance are equally appropriate for smoothing
the residuals to a more Gaussian form. In fact, often we will kill
two birds with one stone.
\item [{Errors~are~not~independent.}] There is a large class of autoregressive
models to be used in this situation which occupy the latter part of
Chapter \ref{cha:Time-Series}.
\end{description}

\section{Other Diagnostic Tools\label{sec:Other-Diagnostic-Tools-SLR}}

There are two types of observations with which we must be especially
careful:
\begin{description}
\item [{\emph{Influential~observations}}] are those that have a substantial
effect on our estimates, predictions, or inferences. A small change
in an influential observation is followed by a large change in the
parameter estimates or inferences. 
\item [{\emph{Outlying~observations}}] are those that fall fall far from
the rest of the data. They may be indicating a lack of fit for our
regression model, or they may just be a mistake or typographical error
that should be corrected. Regardless, special attention should be
given to these observations. An outlying observation may or may not
be influential.
\end{description}
We will discuss outliers first because the notation builds sequentially
in that order.


\subsection{Outliers}

There are three ways that an observation $(x_{i},y_{i})$ may be an
outlier: it can have an $x_{i}$ value which falls far from the other
$x$ values, it can have a $y_{i}$ value which falls far from the
other $y$ values, or it can have both its $x_{i}$ and $y_{i}$ values
falling far from the other $x$ and $y$ values.


\subsection*{Leverage}

Leverage statistics are designed to identify observations which have
$x$ values that are far away from the rest of the data. In the simple
linear regression model the leverage of $x_{i}$ is denoted by $h_{ii}$
and defined by \begin{equation}
h_{ii}=\frac{1}{n}+\frac{(x_{i}-\xbar)^{2}}{\sum_{k=1}^{n}(x_{k}-\xbar)^{2}},\quad i=1,2,\ldots,n.\end{equation}
The formula has a nice interpretation in the SLR model: if the distance
from $x_{i}$ to $\xbar$ is large relative to the other $x$'s then
$h_{ii}$ will be close to 1. 

Leverages have nice mathematical properties; for example, they satisfy\begin{equation}
0\leq h_{ii}\leq1,\label{eq:slr-leverage-between}\end{equation}
and their sum is \begin{eqnarray}
\sum_{i=1}^{n}h_{ii} & = & \sum_{i=1}^{n}\left[\frac{1}{n}+\frac{(x_{i}-\xbar)^{2}}{\sum_{k=1}^{n}(x_{k}-\xbar)^{2}}\right],\\
 & = & \frac{n}{n}+\frac{\sum_{i}(x_{i}-\xbar)^{2}}{\sum_{k}(x_{k}-\xbar)^{2}},\\
 & = & 2.\label{eq:slr-average-leverage}\end{eqnarray}
A rule of thumb is to consider leverage values to be large if they
are more than double their average size (which is $2/n$ according
to Equation \ref{eq:slr-average-leverage}). So leverages larger than
$4/n$ are suspect. Another rule of thumb is to say that values bigger
than 0.5 indicate high leverage, while values between 0.3 and 0.5
indicate moderate leverage.


\subsection*{Standardized and Studentized Deleted Residuals }

We have already encountered the \emph{standardized residuals} $r_{i}$
in Section \ref{sub:Constant-Variance-Assumption}; they are merely
residuals that have been divided by their respective standard deviations:
\begin{equation}
R_{i}=\frac{E_{i}}{S\sqrt{1-h_{ii}}},\quad i=1,2,\ldots,n.\end{equation}
Values of $|R_{i}|>2$ are extreme and suggest that the observation
has an outlying $y$-value. 

Now delete the $i$$^{\textrm{th}}$ case and fit the regression function
to the remaining $n-1$ cases, producing a fitted value $\hat{Y}_{(i)}$
with \emph{deleted residual} $D_{i}=Y_{i}-\hat{Y}_{(i)}$. It is shown
in later classes that \begin{equation}
\mbox{Var\ensuremath{(D_{i})}}=\frac{S_{(i)}^{2}}{1-h_{ii}},\quad i=1,2,\ldots,n,\end{equation}
so that the \emph{studentized deleted residuals} $t_{i}$ defined
by\begin{equation}
t_{i}=\frac{D_{i}}{S_{(i)}/(1-h_{ii})},\quad i=1,2,\ldots,n,\label{eq:slr-studentized-deleted-resids}\end{equation}
have a $\mathsf{t}(\mathtt{df}=n-3)$ distribution and we compare
observed values of $t_{i}$ to this distribution to decide whether
or not an observation is extreme. 

The folklore in regression classes is that a test based on the statistic
in Equation \ref{eq:slr-studentized-deleted-resids} can be too liberal.
A rule of thumb is if we suspect an observation to be an outlier \emph{before}
seeing the data then we say it is significantly outlying if its two-tailed
$p$-value is less than $\alpha$, but if we suspect an observation
to be an outlier \emph{after} seeing the data then we should only
say it is significantly outlying if its two-tailed $p$-value is less
than $\alpha/n$. The latter rule of thumb is called the \emph{Bonferroni
approach} and can be overly conservative for large data sets. The
responsible statistician should look at the data and use his/her best
judgement, in every case.


\subsection{How to do it with \textsf{R}}

We can calculate the standardized residuals with the \inputencoding{latin9}\lstinline[showstringspaces=false]!rstandard!\inputencoding{utf8}
function. The input is the \inputencoding{latin9}\lstinline[showstringspaces=false]!lm!\inputencoding{utf8}
object, which is \inputencoding{latin9}\lstinline[showstringspaces=false]!cars.lm!\inputencoding{utf8}.

<<>>=
sres <- rstandard(cars.lm)
sres[1:5]
@

We can find out which observations have studentized residuals larger
than two with the command

<<>>=
sres[which(abs(sres) > 2)]
@

In this case, we see that observations 23, 35, and 49 are potential
outliers with respect to their $y$-value.

We can compute the studentized deleted residuals with \inputencoding{latin9}\lstinline[showstringspaces=false]!rstudent!\inputencoding{utf8}:

<<>>=
sdelres <- rstudent(cars.lm)
sdelres[1:5]
@

We should compare these values with critical values from a $\mathsf{t}(\mathtt{df}=n-3)$
distribution, which in this case is $\mathsf{t}(\mathtt{df}=50-3=47)$.
We can calculate a 0.005 quantile and check with

<<>>=
t0.005 <- qt(0.005, df = 47, lower.tail = FALSE)
sdelres[which(abs(sdelres) > t0.005)]
@

This means that observations 23 and 49 have a large studentized deleted
residual. The leverages can be found with the \inputencoding{latin9}\lstinline[showstringspaces=false]!hatvalues!\inputencoding{utf8}
function:

<<>>=
leverage <- hatvalues(cars.lm)
leverage[1:5]
leverage[which(leverage > 4/50)]
@

Here we see that observations 1, 2, and 50 have leverages bigger than
double their mean value. These observations would be considered outlying
with respect to their $x$ value (although they may or may not be
influential).


\subsection{Influential Observations}


\subsection*{$DFBETAS$ and $DFFITS$}

Anytime we do a statistical analysis, we are confronted with the variability
of data. It is always a concern when an observation plays too large
a role in our regression model, and we would not like or procedures
to be overly influenced by the value of a single observation. Hence,
it becomes desirable to check to see how much our estimates and predictions
would change if one of the observations were not included in the analysis.
If an observation changes the estimates/predictions a large amount,
then the observation is influential and should be subjected to a higher
level of scrutiny.

We measure the change in the parameter estimates as a result of deleting
an observation with $DFBETAS$. The $DFBETAS$ for the intercept $b_{0}$
are given by\begin{equation}
(DFBETAS)_{0(i)}=\frac{b_{0}-b_{0(i)}}{S_{(i)}\sqrt{\frac{1}{n}+\frac{\xbar^{2}}{\sum_{i=1}^{n}(x_{i}-\xbar)^{2}}}},\quad i=1,2,\ldots,n.\end{equation}
and the $DFBETAS$ for the slope $b_{1}$ are given by\begin{equation}
(DFBETAS)_{1(i)}=\frac{b_{1}-b_{1(i)}}{S_{(i)}\left[\sum_{i=1}^{n}(x_{i}-\xbar)^{2}\right]^{-1/2}},\quad i=1,2,\ldots,n.\end{equation}
See Section \ref{sec:Residual-Analysis-MLR} for a better way to write
these. The signs of the $DFBETAS$ indicate whether the coefficients
would increase or decrease as a result of including the observation.
If the $DFBETAS$ are large, then the observation has a large impact
on those regression coefficients. We label observations as suspicious
if their $DFBETAS$ have magnitude greater 1 for small data or $2/\sqrt{n}$
for large data sets.

We can calculate the $DFBETAS$ with the \inputencoding{latin9}\lstinline[showstringspaces=false]!dfbetas!\inputencoding{utf8}
function (some output has been omitted):

<<>>=
dfb <- dfbetas(cars.lm)
head(dfb)
@

We see that the inclusion of the first observation slightly increases
the \inputencoding{latin9}\lstinline[showstringspaces=false]!Intercept!\inputencoding{utf8}
and slightly decreases the coefficient on \inputencoding{latin9}\lstinline[showstringspaces=false]!speed!\inputencoding{utf8}.

We can measure the influence that an observation has on its fitted
value with $DFFITS$. These are calculated by deleting an observation,
refitting the model, recalculating the fit, then standardizing. The
formula is 

\begin{equation}
(DFFITS)_{i}=\frac{\hat{Y_{i}}-\hat{Y}_{(i)}}{S_{(i)}\sqrt{h_{ii}}},\quad i=1,2,\ldots,n.\end{equation}
The value represents the number of standard deviations of $\hat{Y_{i}}$
that the fitted value $\hat{Y_{i}}$ increases or decreases with the
inclusion of the $i$$^{\textrm{th}}$ observation. We can compute
them with the \inputencoding{latin9}\lstinline[showstringspaces=false]!dffits!\inputencoding{utf8}
function.

<<>>=
dff <- dffits(cars.lm)
dff[1:5]
@

A rule of thumb is to flag observations whose $DFFIT$ exceeds one
in absolute value, but there are none of those in this data set.


\subsection*{Cook's Distance}

The $DFFITS$ are good for measuring the influence on a single fitted
value, but we may want to measure the influence an observation has
on all of the fitted values simultaneously. The statistics used for
measuring this are Cook's distances which may be calculated%
\footnote{Cook's distances are actually defined by a different formula than
the one shown. The formula in Equation \ref{eq:slr-cooks-distance}
is algebraically equivalent to the defining formula and is, in the
author's opinion, more transparent.%
} by the formula

\begin{equation}
D_{i}=\frac{E_{i}^{2}}{(p+1)S^{2}}\cdot\frac{h_{ii}}{(1-h_{ii})^{2}},\quad i=1,2,\ldots,n.\label{eq:slr-cooks-distance}\end{equation}
It shows that Cook's distance depends both on the residual $E_{i}$
and the leverage $h_{ii}$ and in this way $D_{i}$ contains information
about outlying $x$ and $y$ values. 

To assess the significance of $D$, we compare to quantiles of an
$\mathsf{f}(\mathtt{df1}=2,\,\mathtt{df2}=n-2)$ distribution. A rule
of thumb is to classify observations falling higher than the 50$^{\textrm{th}}$
percentile as being extreme. 


\subsection{How to do it with \textsf{R}}

We can calculate the Cook's Distances with the \inputencoding{latin9}\lstinline[showstringspaces=false]!cooks.distance!\inputencoding{utf8}
function.

<<>>=
cooksD <- cooks.distance(cars.lm)
cooksD[1:5]
@

We can look at a plot of the Cook's distances with the command \inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false]!plot(cars.lm, which = 4)!\inputencoding{utf8}.

%
\begin{figure}
\begin{centering}
<<echo = FALSE, fig=true, height = 4.5, width = 6>>=
plot(cars.lm, which = 4)
@
\par\end{centering}

\caption{Cook's distances for the \texttt{cars} data\label{fig:Cooks-distance-cars}}


{\small Used for checking for influential and/our outlying observations.
Values with large Cook's distance merit further investigation.}
\end{figure}


Observations with the largest Cook's D values are labeled, hence we
see that observations 23, 39, and 49 are suspicious. However, we need
to compare to the quantiles of an $\mathsf{f}(\mathtt{df1}=2,\,\mathtt{df2}=48)$
distribution:

<<>>=
F0.50 <- qf(0.5, df1 = 2, df2 = 48)
cooksD[which(cooksD > F0.50)]
@

We see that with this data set there are no observations with extreme
Cook's distance, after all.


\subsection{All Influence Measures Simultaneously}

We can display the result of diagnostic checking all at once in one
table, with potentially influential points displayed. We do it with
the command \inputencoding{latin9}\lstinline[showstringspaces=false]!influence.measures(cars.lm)!\inputencoding{utf8}:

<<eval = FALSE>>=
influence.measures(cars.lm)
@

The output is a huge matrix display, which we have omitted in the
interest of brevity. A point is identified if it is classified to
be influential with respect to any of the diagnostic measures. Here
we see that observations 2, 11, 15, and 18 merit further investigation. 

We can also look at all diagnostic plots at once with the commands

<<eval = FALSE>>=
par(mfrow = c(2,2))
plot(cars.lm)
par(mfrow = c(1,1))
@

The \inputencoding{latin9}\lstinline[showstringspaces=false]!par!\inputencoding{utf8}
command is used so that $2\times2=4$ plots will be shown on the same
display. The diagnostic plots for the \inputencoding{latin9}\lstinline[showstringspaces=false]!cars!\inputencoding{utf8}
data are shown in Figure \ref{fig:Diagnostic-plots-cars}:

%
\begin{figure}
\begin{centering}
<<echo = FALSE, fig=true>>=
par(mfrow = c(2,2))
plot(cars.lm)
par(mfrow = c(1,1))
@
\par\end{centering}

\caption{Diagnostic plots for the \texttt{cars} data\label{fig:Diagnostic-plots-cars}}

\end{figure}


We have discussed all of the plots except the last, which is possibly
the most interesting. It shows Residuals vs. Leverage, which will
identify outlying $y$ values versus outlying $x$ values. Here we
see that observation 23 has a high residual, but low leverage, and
it turns out that observations 1 and 2 have relatively high leverage
but low/moderate leverage (they are on the right side of the plot,
just above the horizontal line). Observation 49 has a large residual
with a comparatively large leverage. 

We can identify the observations with the \inputencoding{latin9}\lstinline[showstringspaces=false]!identify!\inputencoding{utf8}
command; it allows us to display the observation number of dots on
the plot. First, we plot the graph, then we call \inputencoding{latin9}\lstinline[showstringspaces=false]!identify!\inputencoding{utf8}:

<<eval = FALSE, keep.source = TRUE>>=
plot(cars.lm, which = 5)   # std'd resids vs lev plot
identify(leverage, sres, n = 4)   # identify 4 points
@

The graph with the identified points is omitted (but the plain plot
is shown in the bottom right corner of Figure \ref{fig:Diagnostic-plots-cars}).
Observations 1 and 2 fall on the far right side of the plot, near
the horizontal axis.

\newpage{}


\section*{Chapter Exercises}

\addcontentsline{toc}{section}{Chapter Exercises}
\setcounter{thm}{0}
\begin{xca}
Prove the ANOVA equality, Equation \ref{eq:anovaeq}. \emph{Hint}:
show that\[
\sum_{i=1}^{n}(Y_{i}-\hat{Y_{i}})(\hat{Y_{i}}-\Ybar)=0.\]

\end{xca}

\begin{xca}
\label{xca:find-mles-SLR}Solve the following system of equations
for $\beta_{1}$ and $\beta_{0}$ to find the MLEs for slope and intercept
in the simple linear regression model.\begin{eqnarray*}
n\beta_{0}+\beta_{1}\sum_{i=1}^{n}x_{i} & = & \sum_{i=1}^{n}y_{i}\\
\beta_{0}\sum_{i=1}^{n}x_{i}+\beta_{1}\sum_{i=1}^{n}x_{i}^{2} & = & \sum_{i=1}^{n}x_{i}y_{i}\end{eqnarray*}

\end{xca}

\begin{xca}
\label{xca:show-alternate-slope-formula}Show that the formula given
in Equation \ref{eq:sample-correlation-formula} is equivalent to
\[
\hat{\beta}_{1}=\frac{\sum_{i=1}^{n}x_{i}y_{i}-\left.\left(\sum_{i=1}^{n}x_{i}\right)\left(\sum_{i=1}^{n}y_{i}\right)\right\slash n}{\sum_{i=1}^{n}x_{i}^{2}-\left.\left(\sum_{i=1}^{n}x_{i}\right)^{2}\right\slash n}.\]

\end{xca}

\chapter{Multiple Linear Regression\label{cha:multiple-linear-regression}}

\noindent We know a lot about simple linear regression models, and
a next step is to study multiple regression models that have more
than one independent (explanatory) variable. In the discussion that
follows we will assume that we have $p$ explanatory variables, where
$p>1$.

The language is phrased in matrix terms -- for two reasons. First,
it is quicker to write and (arguably) more pleasant to read. Second,
the matrix approach will be required for later study of the subject;
the reader might as well be introduced to it now.

Most of the results are stated without proof or with only a cursory
justification. Those yearning for more should consult an advanced
text in linear regression for details, such as \emph{Applied Linear
Regression Models} \cite{Neter1996}or \emph{Linear Models: Least
Squares and Alternatives} \cite{Rao1999}.


\paragraph*{What do I want them to know?}
\begin{itemize}
\item the basic MLR model, and how it relates to the SLR
\item how to estimate the parameters and use those estimates to make predictions
\item basic strategies to determine whether or not the model is doing a
good job
\item a few thoughts about selected applications of the MLR, such as polynomial,
interaction, and dummy variable models
\item some of the uses of residuals to diagnose problems
\item hints about what will be coming later
\end{itemize}

\section{The Multiple Linear Regression Model\label{sec:The-MLR-Model}}

The first thing to do is get some better notation. We will write \begin{equation}
\mathbf{Y}_{\mathrm{n}\times1}=\begin{bmatrix}y_{1}\\
y_{2}\\
\vdots\\
y_{n}\end{bmatrix},\quad\mbox{and}\quad\mathbf{X}_{\mathrm{n}\times(\mathrm{p}+1)}=\begin{bmatrix}1 & x_{11} & x_{21} & \cdots & x_{p1}\\
1 & x_{12} & x_{22} & \cdots & x_{p2}\\
\vdots & \vdots & \vdots & \ddots & \vdots\\
1 & x_{1n} & x_{2n} & \cdots & x_{pn}\end{bmatrix}.\end{equation}
The vector $\mathbf{Y}$ is called the \emph{response vector\index{response vector}}
and the matrix $\mathbf{X}$ is called the \emph{model matrix}\index{model matrix}.
As in Chapter \ref{cha:simple-linear-regression}, the most general
assumption that relates $\mathbf{Y}$ to $\mathbf{X}$ is\begin{equation}
\mathbf{Y}=\mu(\mathbf{X})+\upepsilon,\end{equation}
where $\mu$ is some function (the \emph{signal}) and $\upepsilon$
is the \emph{noise} (everything else). We usually impose some structure
on $\mu$ and $\upepsilon$. In particular, the standard multiple
linear regression model\index{model!multiple linear regression} assumes
\begin{equation}
\mathbf{Y}=\mathbf{X}\upbeta+\upepsilon,\end{equation}
where the parameter vector $\upbeta$ looks like \begin{equation}
\upbeta_{(\mathrm{p}+1)\times1}=\begin{bmatrix}\beta_{0} & \beta_{1} & \cdots & \beta_{p}\end{bmatrix}^{\mathrm{T}},\end{equation}
and the random vector $\upepsilon_{\mathrm{n}\times1}=\begin{bmatrix}\epsilon_{1} & \epsilon_{2} & \cdots & \epsilon_{n}\end{bmatrix}^{\mathrm{T}}$
is assumed to be distributed\begin{equation}
\upepsilon\sim\mathsf{mvnorm}\left(\mathtt{mean}=\mathbf{0}_{\mathrm{n}\times1},\,\mathtt{sigma}=\sigma^{2}\mathbf{I}_{\mathrm{n}\times\mathrm{n}}\right).\end{equation}
The assumption on $\upepsilon$ is equivalent to the assumption that
$\epsilon_{1}$, $\epsilon_{2}$, \ldots{}, $\epsilon_{n}$ are i.i.d.~$\mathsf{norm}(\mathtt{mean}=0,\,\mathtt{sd}=\sigma)$.
It is a linear model because the quantity $\mu(\mathbf{X})=\mathbf{X}\upbeta$
is linear in the parameters $\beta_{0}$, $\beta_{1}$,\ldots{},
$\beta_{p}$. It may be helpful to see the model in expanded form;
the above matrix formulation is equivalent to the more lengthy\begin{equation}
Y_{i}=\beta_{0}+\beta_{1}x_{1i}+\beta_{2}x_{2i}+\cdots+\beta_{p}x_{pi}+\epsilon_{i},\quad i=1,2,\ldots,n.\end{equation}

\begin{example}
\textbf{Girth, Height, and Volume for Black Cherry trees.}\index{Data sets!trees@\texttt{trees}}Measurements
were made of the girth, height, and volume of timber in 31 felled
black cherry trees. Note that girth is the diameter of the tree (in
inches) measured at 4\,ft 6\,in above the ground. The variables
are
\begin{enumerate}
\item \inputencoding{latin9}\lstinline[showstringspaces=false]!Girth!\inputencoding{utf8}:
tree diameter in inches (denoted $x_{1}$)
\item \inputencoding{latin9}\lstinline[showstringspaces=false]!Height!\inputencoding{utf8}:
tree height in feet ($x_{2}$).
\item \inputencoding{latin9}\lstinline[showstringspaces=false]!Volume!\inputencoding{utf8}:
volume of the tree in cubic feet. ($y$)
\end{enumerate}
The data are in the \inputencoding{latin9}\lstinline[showstringspaces=false]!datasets!\inputencoding{utf8}
package and are already on the search path; they can be viewed with

<<>>=
head(trees)
@

Let us take a look at a visual display of the data. For multiple variables,
instead of a simple scatterplot we use a scatterplot matrix which
is made with the \inputencoding{latin9}\lstinline[showstringspaces=false]!splom!\inputencoding{utf8}
function in the \inputencoding{latin9}\lstinline[showstringspaces=false]!lattice!\inputencoding{utf8}
package \cite{Sarkarlattice} as shown below. The plot is shown in
Figure \ref{fig:splom-trees}.

%
\begin{figure}
\begin{centering}
<<echo = FALSE, fig=true, height = 4, width = 6>>=
library(lattice)
print(splom(trees))
@
\par\end{centering}

\caption{Scatterplot matrix of \texttt{trees} data\label{fig:splom-trees}}

\end{figure}


<<eval = FALSE>>=
library(lattice)
splom(trees)
@

The dependent (response) variable \inputencoding{latin9}\lstinline[showstringspaces=false]!Volume!\inputencoding{utf8}
is listed in the first row of the scatterplot matrix. Moving from
left to right, we see an approximately linear relationship between
\inputencoding{latin9}\lstinline[showstringspaces=false]!Volume!\inputencoding{utf8}
and the independent (explanatory) variables \inputencoding{latin9}\lstinline[showstringspaces=false]!Height!\inputencoding{utf8}
and \inputencoding{latin9}\lstinline[showstringspaces=false]!Girth!\inputencoding{utf8}.
A first guess at a model for these data might be\begin{equation}
Y=\beta_{0}+\beta_{1}x_{1}+\beta_{2}x_{2}+\epsilon,\end{equation}
in which case the quantity $\mu(x_{1},x_{2})=\beta_{0}+\beta_{1}x_{1}+\beta_{2}x_{2}$
would represent the mean value of $Y$ at the point $(x_{1},x_{2})$.

\end{example}

\subsection*{What does it mean?}

The interpretation is simple. The intercept $\beta_{0}$ represents
the mean \inputencoding{latin9}\lstinline[showstringspaces=false]!Volume!\inputencoding{utf8}
when all other independent variables are zero. The parameter $\beta_{i}$
represents the change in mean \inputencoding{latin9}\lstinline[showstringspaces=false]!Volume!\inputencoding{utf8}
when there is a unit increase in $x_{i}$, while the other independent
variable is held constant. For the \inputencoding{latin9}\lstinline[showstringspaces=false]!trees!\inputencoding{utf8}
data, $\beta_{1}$ represents the change in average \inputencoding{latin9}\lstinline[showstringspaces=false]!Volume!\inputencoding{utf8}
as \inputencoding{latin9}\lstinline[showstringspaces=false]!Girth!\inputencoding{utf8}
increases by one unit when the \inputencoding{latin9}\lstinline[showstringspaces=false]!Height!\inputencoding{utf8}
is held constant, and $\beta_{2}$ represents the change in average
\inputencoding{latin9}\lstinline[showstringspaces=false]!Volume!\inputencoding{utf8}
as \inputencoding{latin9}\lstinline[showstringspaces=false]!Height!\inputencoding{utf8}
increases by one unit when the \inputencoding{latin9}\lstinline[showstringspaces=false]!Girth!\inputencoding{utf8}
is held constant.

In simple linear regression, we had one independent variable and our
linear regression surface was 1D, simply a line. In multiple regression
there are many independent variables and so our linear regression
surface will be many-D\ldots{} in general, a hyperplane. But when
there are only two explanatory variables the hyperplane is just an
ordinary plane and we can look at it with a 3D scatterplot. 

One way to do this is with the \textsf{R} Commander in the \inputencoding{latin9}\lstinline[showstringspaces=false]!Rcmdr!\inputencoding{utf8}
package \cite{Foxrcmdr}. It has a 3D scatterplot option under the
\textsf{Graphs} menu. It is especially great because the resulting
graph is dynamic; it can be moved around with the mouse, zoomed, \emph{etc}.
But that particular display does not translate well to a printed book.

Another way to do it is with the \inputencoding{latin9}\lstinline[showstringspaces=false]!scatterplot3d!\inputencoding{utf8}
function in the \inputencoding{latin9}\lstinline[showstringspaces=false]!scatterplot3d!\inputencoding{utf8}
package. The code follows, and the result is shown in Figure \ref{fig:3D-scatterplot-trees}.

<<eval = FALSE>>=
library(scatterplot3d)
s3d <- with(trees, scatterplot3d(Girth, Height, Volume, pch = 16, highlight.3d = TRUE, angle = 60))
fit <- lm(Volume ~ Girth + Height, data = trees)
s3d$plane3d(fit)
@

%
\begin{figure}
\begin{centering}
<<echo = FALSE, fig=true, height = 4, width = 6>>=
library(scatterplot3d)
s3d <- with(trees, scatterplot3d(Girth, Height, Volume, pch = 16, highlight.3d = TRUE, angle = 60))
fit <- lm(Volume ~ Girth + Height, data = trees)
s3d$plane3d(fit)
@
\par\end{centering}

\caption{3D scatterplot with regression plane for the \texttt{trees} data\label{fig:3D-scatterplot-trees}}

\end{figure}


Looking at the graph we see that the data points fall close to a plane
in three dimensional space. (The plot looks remarkably good. In the
author's experience it is rare to see points fit so well to the plane
without some additional work.)


\section{Estimation and Prediction\label{sec:Estimation-and-Prediction-MLR}}


\subsection{Parameter estimates\label{sub:mlr-parameter-estimates}}

We will proceed exactly like we did in Section \ref{sec:SLR-Estimation}.
We know \begin{equation}
\upepsilon\sim\mathsf{mvnorm}\left(\mathtt{mean}=\mathbf{0}_{\mathrm{n}\times1},\,\mathtt{sigma}=\sigma^{2}\mathbf{I}_{\mathrm{n}\times\mathrm{n}}\right),\end{equation}
which means that $\mathbf{Y}=\mathbf{X}\upbeta+\upepsilon$ has an
$\mathsf{mvnorm}\left(\mathtt{mean}=\mathbf{X}\upbeta,\,\mathtt{sigma}=\sigma^{2}\mathbf{I}_{\mathrm{n}\times\mathrm{n}}\right)$
distribution. Therefore, the likelihood function\index{likelihood function}
is\begin{equation}
L(\upbeta,\sigma)=\frac{1}{2\pi^{n/2}\sigma}\exp\left\{ -\frac{1}{2\sigma^{2}}\left(\mathbf{Y}-\mathbf{X}\upbeta\right)^{\mathrm{T}}\left(\mathbf{Y}-\mathbf{X}\upbeta\right)\right\} .\end{equation}
To \emph{maximize} the likelihood\index{maximum likelihood} in $\upbeta$,
we need to \emph{minimize} the quantity $g(\upbeta)=\left(\mathbf{Y}-\mathbf{X}\upbeta\right)^{\mathrm{T}}\left(\mathbf{Y}-\mathbf{X}\upbeta\right)$.
We do this by differentiating $g$ with respect to $\upbeta$. (It
may be a good idea to brush up on the material in Appendices \ref{sec:Linear-Algebra}
and \ref{sec:Multivariable-Calculus}.) First we will rewrite $g$:\begin{equation}
g(\upbeta)=\mathbf{Y}^{\mathrm{T}}\mathbf{Y}-\mathbf{Y}^{\mathrm{T}}\mathbf{X}\upbeta-\upbeta^{\mathrm{T}}\mathbf{X}^{\mathrm{T}}\mathbf{Y}+\upbeta^{\mathrm{T}}\mathbf{X}^{\mathrm{T}}\mathbf{X}\upbeta,\end{equation}
which can be further simplified to $g(\upbeta)=\mathbf{Y}^{\mathrm{T}}\mathbf{Y}-2\upbeta^{\mathrm{T}}\mathbf{X}^{\mathrm{T}}\mathbf{Y}+\upbeta^{\mathrm{T}}\mathbf{X}^{\mathrm{T}}\mathbf{X}\upbeta$
since $\upbeta^{\mathrm{T}}\mathbf{X}^{\mathrm{T}}\mathbf{Y}$ is
$1\times1$ and thus equal to its transpose. Now we differentiate
to get \begin{equation}
\frac{\partial g}{\partial\upbeta}=\mathbf{0}-2\mathbf{X}^{\mathrm{T}}\mathbf{Y}+2\mathbf{X}^{\mathrm{T}}\mathbf{X}\upbeta,\end{equation}
since $\mathbf{X}^{\mathrm{T}}\mathbf{X}$ is symmetric. Setting the
derivative equal to the zero vector yields the so called {}``normal
equations''\index{normal equations}\begin{equation}
\mathbf{X}^{\mathrm{T}}\mathbf{X}\upbeta=\mathbf{X}^{\mathrm{T}}\mathbf{Y}.\end{equation}
In the case that $\mathbf{X}^{\mathrm{T}}\mathbf{X}$ is invertible%
\footnote{We can find solutions of the normal equations even when $\mathbf{X}^{\mathrm{T}}\mathbf{X}$
is not of full rank, but the topic falls outside the scope of this
book. The interested reader can consult an advanced text such as Rao
\cite{Rao1999}.%
}, we may solve the equation for $\upbeta$ to get the maximum likelihood
estimator of $\upbeta$ which we denote by $\mathbf{b}$:\begin{equation}
\mathbf{b}=\left(\mathbf{X}^{\mathrm{T}}\mathbf{X}\right)^{-1}\mathbf{X}^{\mathrm{T}}\mathbf{Y}.\label{eq:b-formula-matrix}\end{equation}

\begin{rem}
The formula in Equation \ref{eq:b-formula-matrix} is convenient for
mathematical study but is inconvenient for numerical computation.
Researchers have devised much more efficient algorithms for the actual
calculation of the parameter estimates, and we do not explore them
here.
\end{rem}

\begin{rem}
We have only found a critical value, and have not actually shown that
the critical value is a minimum. We omit the details and refer the
interested reader to \cite{Rao1999}.
\end{rem}

\subsection{How to do it with \textsf{R}}

We do all of the above just as we would in simple linear regression.
The powerhouse is the \inputencoding{latin9}\lstinline[showstringspaces=false]!lm!\inputencoding{utf8}\index{lm@\texttt{lm}}
function. Everything else is based on it. We separate explanatory
variables in the model formula by a plus sign.

<<>>=
trees.lm <- lm(Volume ~ Girth + Height, data = trees)
trees.lm
@

We see from the output that for the \inputencoding{latin9}\lstinline[showstringspaces=false]!trees!\inputencoding{utf8}
data our parameter estimates are $\mathbf{b}=\begin{bmatrix}-58.0 & 4.7 & 0.3\end{bmatrix}$,
and consequently our estimate of the mean response is $\hat{\mu}$
given by \begin{alignat}{1}
\hat{\mu}(x_{1},x_{2})= & \ b_{0}+b_{1}x_{1}+b_{2}x_{2},\\
\approx & -58.0+4.7x_{1}+0.3x_{2}.\end{alignat}
We could see the entire model matrix $\mathbf{X}$ with the \inputencoding{latin9}\lstinline[showstringspaces=false]!model.matrix!\inputencoding{utf8}\index{model.matrix@\texttt{model.matrix}}
function, but in the interest of brevity we only show the first few
rows.

<<>>=
head(model.matrix(trees.lm))
@


\subsection{Point Estimates of the Regression Surface\label{sub:mlr-point-est-regsurface}}

The parameter estimates $\mathbf{b}$ make it easy to find the fitted
values\index{fitted values}, $\hat{\mathbf{Y}}$. We write them individually
as $\hat{Y}_{i}$, $i=1,2,\ldots,n$, and recall that they are defined
by\begin{eqnarray}
\hat{Y}_{i} & = & \hat{\mu}(x_{1i},x_{2i}),\\
 & = & b_{0}+b_{1}x_{1i}+b_{2}x_{2i},\quad i=1,2,\ldots,n.\end{eqnarray}
They are expressed more compactly by the matrix equation\begin{equation}
\hat{\mathbf{Y}}=\mathbf{X}\mathbf{b}.\end{equation}
From Equation \ref{eq:b-formula-matrix} we know that $\mathbf{b}=\left(\mathbf{X}^{\mathrm{T}}\mathbf{X}\right)^{-1}\mathbf{X}^{\mathrm{T}}\mathbf{Y}$,
so we can rewrite\begin{eqnarray}
\hat{\mathbf{Y}} & = & \mathbf{X}\left[\left(\mathbf{X}^{\mathrm{T}}\mathbf{X}\right)^{-1}\mathbf{X}^{\mathrm{T}}\mathbf{Y}\right],\\
 & = & \mathbf{H}\mathbf{Y},\end{eqnarray}
where $\mathbf{H}=\mathbf{X}\left(\mathbf{X}^{\mathrm{T}}\mathbf{X}\right)^{-1}\mathbf{X}^{\mathrm{T}}$
is appropriately named \emph{the hat matrix}\index{hat matrix} because
it {}``puts the hat on $\mathbf{Y}$''. The hat matrix is very important
in later courses. Some facts about $\mathbf{H}$ are
\begin{itemize}
\item $\mathbf{H}$ is a symmetric square matrix, of dimension $\mathrm{n}\times\mathrm{n}$.
\item The diagonal entries $h_{ii}$ satisfy $0\leq h_{ii}\leq1$ (compare
to Equation \ref{eq:slr-leverage-between}).
\item The trace is $\mathrm{tr}(\mathbf{H})=p$.
\item $\mathbf{H}$ is \emph{idempotent} (also known as a \emph{projection
matrix}) which means that $\mathbf{H}^{2}=\mathbf{H}$. The same is
true of $\mathbf{I}-\mathbf{H}$.
\end{itemize}
Now let us write a column vector $\mathbf{x}_{0}=(x_{10},x_{20})^{\mathrm{T}}$
to denote given values of the explanatory variables \inputencoding{latin9}\lstinline[showstringspaces=false]!Girth =!\inputencoding{utf8}
$x_{10}$ and \inputencoding{latin9}\lstinline[showstringspaces=false]!Height =!\inputencoding{utf8}
$x_{20}$. These values may match those of the collected data, or
they may be completely new values not observed in the original data
set. We may use the parameter estimates to find $\hat{Y}(\mathbf{x}_{0})$,
which will give us
\begin{enumerate}
\item an estimate of $\mu(\mathbf{x}_{0})$, the mean value of a future
observation at $\mathbf{x}_{0}$, and
\item a prediction for $Y(\mathbf{x}_{0})$, the actual value of a future
observation at $\mathbf{x}_{0}$.
\end{enumerate}
We can represent $\hat{Y}(\mathbf{x}_{0})$ by the matrix equation\begin{equation}
\hat{Y}(\mathbf{x}_{0})=\mathbf{x}_{0}^{\mathrm{T}}\mathbf{b},\label{eq:mlr-single-yhat-matrix}\end{equation}
which is just a fancy way to write\begin{equation}
\hat{Y}(x_{10},x_{20})=b_{0}+b_{1}x_{10}+b_{2}x_{20}.\end{equation}
 
\begin{example}
If we wanted to predict the average volume of black cherry trees that
have \inputencoding{latin9}\lstinline[showstringspaces=false]!Girth = 15!\inputencoding{utf8}\,in
and are \inputencoding{latin9}\lstinline[showstringspaces=false]!Height = 77!\inputencoding{utf8}\,ft
tall then we would use the estimate \begin{alignat*}{1}
\hat{\mu}(15,\,77)= & -58+4.7(15)+0.3(77),\\
\approx & 35.6\mbox{\,\ ft}^{3}.\end{alignat*}
We would use the same estimate $\hat{Y}=35.6$ to predict the measured
\inputencoding{latin9}\lstinline[showstringspaces=false]!Volume!\inputencoding{utf8}
of another black cherry tree -- yet to be observed -- that has \inputencoding{latin9}\lstinline[showstringspaces=false]!Girth = 15!\inputencoding{utf8}\,in
and is \inputencoding{latin9}\lstinline[showstringspaces=false]!Height = 77!\inputencoding{utf8}\,ft
tall.
\end{example}

\subsection{How to do it with \textsf{R}}

The fitted values are stored inside \inputencoding{latin9}\lstinline[showstringspaces=false]!trees.lm!\inputencoding{utf8}
and may be accessed with the \inputencoding{latin9}\lstinline[showstringspaces=false]!fitted!\inputencoding{utf8}
function. We only show the first five fitted values.

<<>>=
fitted(trees.lm)[1:5]
@

The syntax for general prediction does not change much from simple
linear regression. The computations are done with the \inputencoding{latin9}\lstinline[showstringspaces=false]!predict!\inputencoding{utf8}
function as described below.

The only difference from SLR is in the way we tell \textsf{R} the
values of the explanatory variables for which we want predictions.
In SLR we had only one independent variable but in MLR we have many
(for the \inputencoding{latin9}\lstinline[showstringspaces=false]!trees!\inputencoding{utf8}
data we have two). We will store values for the independent variables
in the data frame \inputencoding{latin9}\lstinline[showstringspaces=false]!new!\inputencoding{utf8},
which has two columns (one for each independent variable) and three
rows (we shall make predictions at three different locations).

<<>>=
new <- data.frame(Girth = c(9.1, 11.6, 12.5), Height = c(69, 74, 87))
@

We can view the locations at which we will predict:

<<>>=
new
@

We continue just like we would have done in SLR.

<<>>=
predict(trees.lm, newdata = new)
@

<<echo = FALSE>>=
treesFIT <- round(predict(trees.lm, newdata = new), 1)
@
\begin{example}
Using the \inputencoding{latin9}\lstinline[showstringspaces=false]!trees!\inputencoding{utf8}
data,
\begin{enumerate}
\item Report a point estimate of the mean \inputencoding{latin9}\lstinline[showstringspaces=false]!Volume!\inputencoding{utf8}
of a tree of \inputencoding{latin9}\lstinline!Girth!\inputencoding{utf8}
9.1\,in and \inputencoding{latin9}\lstinline[showstringspaces=false]!Height!\inputencoding{utf8}
69\,ft.


The fitted value for $x_{1}=9.1$ and $x_{2}=69$ is \Sexpr{treesFIT[1]},
so a point estimate would be \Sexpr{treesFIT[1]} cubic feet. 

\item Report a point prediction for and a 95\% prediction interval for the
\inputencoding{latin9}\lstinline[showstringspaces=false]!Volume!\inputencoding{utf8}
of a hypothetical tree of \inputencoding{latin9}\lstinline[showstringspaces=false]!Girth!\inputencoding{utf8}
12.5\,in and \inputencoding{latin9}\lstinline[showstringspaces=false]!Height!\inputencoding{utf8}
87\,ft.


The fitted value for $x_{1}=12.5$ and $x_{2}=87$ is \Sexpr{treesFIT[3]},
so a point prediction for the \inputencoding{latin9}\lstinline[showstringspaces=false]!Volume!\inputencoding{utf8}
is \Sexpr{treesFIT[3]} cubic feet. 

\end{enumerate}
\end{example}

\subsection{Mean Square Error and Standard Error\label{sub:mlr-mse-se}}

The residuals are given by\begin{equation}
\mathbf{E}=\mathbf{Y}-\hat{\mathbf{Y}}=\mathbf{Y}-\mathbf{H}\mathbf{Y}=(\mathbf{I}-\mathbf{H})\mathbf{Y}.\end{equation}
Now we can use Theorem \ref{thm:mvnorm-dist-matrix-prod} to see that
the residuals are distributed\begin{equation}
\mathbf{E}\sim\mathsf{mvnorm}(\mathtt{mean}=\mathbf{0},\,\mathtt{sigma}=\sigma^{2}(\mathbf{I}-\mathbf{H})),\end{equation}
since $(\mathbf{I}-\mathbf{H})\mathbf{X}\upbeta=\mathbf{X}\upbeta-\mathbf{X}\upbeta=\mathbf{0}$
and $(\mathbf{I}-\mathbf{H})\,(\sigma^{2}\mathbf{I})\,(\mathbf{I}-\mathbf{H})^{\mathrm{T}}=\sigma^{2}(\mathbf{I}-\mathbf{H})^{2}=\sigma^{2}(\mathbf{I}-\mathbf{H})$.
The sum of squared errors $SSE$ is just\begin{equation}
SSE=\mathbf{E}^{\mathrm{T}}\mathbf{E}=\mathbf{Y}^{\mathrm{T}}(\mathbf{I}-\mathbf{H})(\mathbf{I}-\mathbf{H})\mathbf{Y}=\mathbf{Y}^{\mathrm{T}}(\mathbf{I}-\mathbf{H})\mathbf{Y}.\end{equation}
Recall that in SLR we had two parameters ($\beta_{0}$ and $\beta_{1}$)
in our regression model and we estimated $\sigma^{2}$ with $s^{2}=SSE/(n-2)$.
In MLR, we have $p+1$ parameters in our regression model and we might
guess that to estimate $\sigma^{2}$ we would use the \emph{mean square
error} $S^{2}$ defined by \begin{equation}
S^{2}=\frac{SSE}{n-(p+1)}.\end{equation}
That would be a good guess. The \emph{residual standard error} is
$S=\sqrt{S^{2}}$.


\subsection{How to do it with \textsf{R}}

The residuals are also stored with \inputencoding{latin9}\lstinline[showstringspaces=false]!trees.lm!\inputencoding{utf8}
and may be accessed with the \inputencoding{latin9}\lstinline[showstringspaces=false]!residuals!\inputencoding{utf8}
function. We only show the first five residuals.

<<>>=
residuals(trees.lm)[1:5]
@

The \inputencoding{latin9}\lstinline[showstringspaces=false]!summary!\inputencoding{utf8}
function output (shown later) lists the \inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false]!Residual Standard Error!\inputencoding{utf8}
which is just $S=\sqrt{S^{2}}$. It is stored in the \inputencoding{latin9}\lstinline[showstringspaces=false]!sigma!\inputencoding{utf8}
component of the \inputencoding{latin9}\lstinline[showstringspaces=false]!summary!\inputencoding{utf8}
object.

<<>>=
treesumry <- summary(trees.lm)
treesumry$sigma
@

For the \inputencoding{latin9}\lstinline[showstringspaces=false]!trees!\inputencoding{utf8}
data we find $s\approx$ \Sexpr{round(treesumry$sigma, 3)}.


\subsection{Interval Estimates of the Parameters\label{sub:mlr-interval-est-params}}

We showed in Section \ref{sub:mlr-parameter-estimates} that $\mathbf{b}=\left(\mathbf{X}^{\mathrm{T}}\mathbf{X}\right)^{-1}\mathbf{X}^{\mathrm{T}}\mathbf{Y}$,
which is really just a big matrix -- namely $\left(\mathbf{X}^{\mathrm{T}}\mathbf{X}\right)^{-1}\mathbf{X}^{\mathrm{T}}$
-- multiplied by $\mathbf{Y}$. It stands to reason that the sampling
distribution of $\mathbf{b}$ would be intimately related to the distribution
of $\mathbf{Y}$, which we assumed to be

\begin{equation}
\mathbf{Y}\sim\mathsf{mvnorm}\left(\mathtt{mean}=\mathbf{X}\upbeta,\,\mathtt{sigma}=\sigma^{2}\mathbf{I}\right).\end{equation}
Now recall Theorem \ref{thm:mvnorm-dist-matrix-prod} that we said
we were going to need eventually (the time is now). That proposition
guarantees that

\begin{equation}
\mathbf{b}\sim\mathsf{mvnorm}\left(\mathtt{mean}=\upbeta,\,\mathtt{sigma}=\sigma^{2}\left(\mathbf{X}^{\mathrm{T}}\mathbf{X}\right)^{-1}\right),\label{eq:distn-b-mlr}\end{equation}
since\begin{equation}
\E\mathbf{b}=\left(\mathbf{X}^{\mathrm{T}}\mathbf{X}\right)^{-1}\mathbf{X}^{\mathrm{T}}(\mathbf{X}\upbeta)=\upbeta,\end{equation}
and\begin{equation}
\mbox{Var}(\mathbf{b})=\left(\mathbf{X}^{\mathrm{T}}\mathbf{X}\right)^{-1}\mathbf{X}^{\mathrm{T}}(\sigma^{2}\mathbf{I})\mathbf{X}\left(\mathbf{X}^{\mathrm{T}}\mathbf{X}\right)^{-1}=\sigma^{2}\left(\mathbf{X}^{\mathrm{T}}\mathbf{X}\right)^{-1},\end{equation}
the first equality following because the matrix $\left(\mathbf{X}^{\mathrm{T}}\mathbf{X}\right)^{-1}$
is symmetric.

There is a lot that we can glean from Equation \ref{eq:distn-b-mlr}.
First, it follows that the estimator $\mathbf{b}$ is unbiased (see
Section \ref{sec:Point-Estimation-1}). Second, the variances of $b_{0}$,
$b_{1}$, \ldots{}, $b_{n}$ are exactly the diagonal elements of
$\sigma^{2}\left(\mathbf{X}^{\mathrm{T}}\mathbf{X}\right)^{-1}$,
which is completely known except for that pesky parameter $\sigma^{2}$.
Third, we can estimate the standard error of $b_{i}$ (denoted $S_{b_{i}}$)
with the mean square error $S$ (defined in the previous section)
multiplied by the corresponding diagonal element of $\left(\mathbf{X}^{\mathrm{T}}\mathbf{X}\right)^{-1}$.
Finally, given estimates of the standard errors we may construct confidence
intervals for $\beta_{i}$ with an interval that looks like\begin{equation}
b_{i}\pm\mathsf{t}_{\alpha/2}(\mathtt{df}=n-p-1)S_{b_{i}}.\end{equation}
The degrees of freedom for the Student's $t$ distribution%
\footnote{We are taking great leaps over the mathematical details. In particular,
we have yet to show that $s^{2}$ has a chi-square distribution and
we have not even come close to showing that $b_{i}$ and $s_{b_{i}}$
are independent. But these are entirely outside the scope of the present
book and the reader may rest assured that the proofs await in later
classes. See C.R. Rao for more.%
} are the same as the denominator of $S^{2}$. 


\subsection{How to do it with \textsf{R}}

To get confidence intervals for the parameters we need only use \inputencoding{latin9}\lstinline[showstringspaces=false]!confint!\inputencoding{utf8}\index{confint@\texttt{confint}}:

<<>>=
confint(trees.lm)
@

<<echo = FALSE, results = hide>>=
treesPAR <- round(confint(trees.lm), 1)
@

For example, using the calculations above we say that for the regression
model \inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false]!Volume ~ Girth + Height!\inputencoding{utf8}
we are 95\% confident that the parameter $\beta_{1}$ lies somewhere
in the interval {[}\Sexpr{treesPAR[2, 1]}, \Sexpr{treesPAR[2, 2]}{]}.


\subsection{Confidence and Prediction Intervals}

We saw in Section \ref{sub:mlr-point-est-regsurface} how to make
point estimates of the mean value of additional observations and predict
values of future observations, but how good are our estimates? We
need confidence and prediction intervals to gauge their accuracy,
and lucky for us the formulas look similar to the ones we saw in SLR.

In Equation \ref{eq:mlr-single-yhat-matrix} we wrote $\hat{Y}(\mathbf{x}_{0})=\mathbf{x}_{0}^{\mathrm{T}}\mathbf{b},$
and in Equation \ref{eq:distn-b-mlr} we saw that \begin{equation}
\mathbf{b}\sim\mathsf{mvnorm}\left(\mathtt{mean}=\upbeta,\,\mathtt{sigma}=\sigma^{2}\left(\mathbf{X}^{\mathrm{T}}\mathbf{X}\right)^{-1}\right),\end{equation}
The following is therefore immediate from Theorem \ref{thm:mvnorm-dist-matrix-prod}:\begin{equation}
\hat{Y}(\mathbf{x}_{0})\sim\mathsf{mvnorm}\left(\mathtt{mean}=\mathbf{x}_{0}^{\mathrm{T}}\upbeta,\,\mathtt{sigma}=\sigma^{2}\mathbf{x}_{0}^{\mathrm{T}}\left(\mathbf{X}^{\mathrm{T}}\mathbf{X}\right)^{-1}\mathbf{x}_{0}\right).\end{equation}
It should be no surprise that confidence intervals for the mean value
of a future observation at the location $\mathbf{x}_{0}=\begin{bmatrix}x_{10} & x_{20} & \ldots & x_{p0}\end{bmatrix}^{\mathrm{T}}$
are given by\begin{equation}
\hat{Y}(\mathbf{x}_{0})\pm\mathsf{t}_{\alpha/2}(\mathtt{df}=n-p-1)\, S\sqrt{\mathbf{x}_{0}^{\mathrm{T}}\left(\mathbf{X}^{\mathrm{T}}\mathbf{X}\right)^{-1}\mathbf{x}_{0}}.\end{equation}
Intuitively, $\mathbf{x}_{0}^{\mathrm{T}}\left(\mathbf{X}^{\mathrm{T}}\mathbf{X}\right)^{-1}\mathbf{x}_{0}$
measures the distance of $\mathbf{x}_{0}$ from the center of the
data. The degrees of freedom in the Student's $t$ critical value
are $n-(p+1)$ because we need to estimate $p+1$ parameters.

Prediction intervals for a new observation at $\mathbf{x}_{0}$ are
given by\begin{equation}
\hat{Y}(\mathbf{x}_{0})\pm\mathsf{t}_{\alpha/2}(\mathtt{df}=n-p-1)\, S\sqrt{1+\mathbf{x}_{0}^{\mathrm{T}}\left(\mathbf{X}^{\mathrm{T}}\mathbf{X}\right)^{-1}\mathbf{x}_{0}}.\end{equation}
The prediction intervals are wider than the confidence intervals,
just as in Section \ref{sub:slr-interval-est-regline}.


\subsection{How to do it with \textsf{R}}

The syntax is identical to that used in SLR, with the proviso that
we need to specify values of the independent variables in the data
frame \inputencoding{latin9}\lstinline[showstringspaces=false]!new!\inputencoding{utf8}
as we did in Section \ref{sub:slr-interval-est-regline} (which we
repeat here for illustration).

<<>>=
new <- data.frame(Girth = c(9.1, 11.6, 12.5), Height = c(69, 74, 87))
@

Confidence intervals are given by

<<>>=
predict(trees.lm, newdata = new, interval = "confidence")
@

<<echo = FALSE, results = hide>>=
treesCI <- round(predict(trees.lm, newdata = new, interval = "confidence"), 1)
@

Prediction intervals are given by

<<>>=
predict(trees.lm, newdata = new, interval = "prediction")
@

<<echo = FALSE, results = hide>>=
treesPI <- round(predict(trees.lm, newdata = new, interval = "prediction"), 1)
@

As before, the interval type is decided by the \inputencoding{latin9}\lstinline[showstringspaces=false]!interval!\inputencoding{utf8}
argument and the default confidence level is 95\% (which can be changed
with the \inputencoding{latin9}\lstinline[showstringspaces=false]!level!\inputencoding{utf8}
argument). 
\begin{example}
Using the \inputencoding{latin9}\lstinline[showstringspaces=false]!trees!\inputencoding{utf8}
data,
\begin{enumerate}
\item Report a 95\% confidence interval for the mean \inputencoding{latin9}\lstinline[showstringspaces=false]!Volume!\inputencoding{utf8}
of a tree of \inputencoding{latin9}\lstinline[showstringspaces=false]!Girth!\inputencoding{utf8}
9.1\,in and \inputencoding{latin9}\lstinline[showstringspaces=false]!Height!\inputencoding{utf8}
69\,ft.


The 95\% CI is given by {[}\Sexpr{treesCI[1, 2]}, \Sexpr{treesCI[1, 3]}{]},
so with 95\% confidence the mean \inputencoding{latin9}\lstinline[showstringspaces=false]!Volume!\inputencoding{utf8}
lies somewhere between \Sexpr{treesCI[1, 2]} cubic feet and \Sexpr{treesCI[1, 3]}
cubic feet.

\item Report a 95\% prediction interval for the \inputencoding{latin9}\lstinline[showstringspaces=false]!Volume!\inputencoding{utf8}
of a hypothetical tree of \inputencoding{latin9}\lstinline[showstringspaces=false]!Girth!\inputencoding{utf8}
12.5\,in and \inputencoding{latin9}\lstinline[showstringspaces=false]!Height!\inputencoding{utf8}
87\,ft.


The 95\% prediction interval is given by {[}\Sexpr{treesCI[3, 2]},
\Sexpr{treesCI[3, 3]}{]}, so with 95\% confidence we may assert that
the hypothetical \inputencoding{latin9}\lstinline[showstringspaces=false]!Volume!\inputencoding{utf8}
of a tree of \inputencoding{latin9}\lstinline[showstringspaces=false]!Girth!\inputencoding{utf8}
12.5\,in and \inputencoding{latin9}\lstinline[showstringspaces=false]!Height!\inputencoding{utf8}
87\,ft would lie somewhere between \Sexpr{treesCI[3, 2]} cubic feet
and \Sexpr{treesCI[3, 3]} feet.

\end{enumerate}
\end{example}

\section{Model Utility and Inference\label{sec:Model-Utility-and-MLR}}


\subsection{Multiple Coefficient of Determination}

We saw in Section \ref{sub:mlr-mse-se} that the error sum of squares
$SSE$ can be conveniently written in MLR as \begin{equation}
SSE=\mathbf{Y}^{\mathrm{T}}(\mathbf{I}-\mathbf{H})\mathbf{Y}.\label{eq:mlr-sse-matrix}\end{equation}
It turns out that there are equally convenient formulas for the total
sum of squares $SSTO$ and the regression sum of squares $SSR$. They
are :\begin{alignat}{1}
SSTO= & \mathbf{Y}^{\mathrm{T}}\left(\mathbf{I}-\frac{1}{n}\mathbf{J}\right)\mathbf{Y}\label{eq:mlr-ssto-matrix}\end{alignat}
and\begin{alignat}{1}
SSR= & \mathbf{Y}^{\mathrm{T}}\left(\mathbf{H}-\frac{1}{n}\mathbf{J}\right)\mathbf{Y}.\label{eq:mlr-ssr-matrix}\end{alignat}
(The matrix $\mathbf{J}$ is defined in Appendix \ref{sec:Linear-Algebra}.)
Immediately from Equations \ref{eq:mlr-sse-matrix}, \ref{eq:mlr-ssto-matrix},
and \ref{eq:mlr-ssr-matrix} we get the \emph{Anova Equality}\begin{equation}
SSTO=SSE+SSR.\end{equation}
(See Exercise \ref{xca:anova-equality}.) We define the \emph{multiple
coefficient of determination} by the formula\begin{equation}
R^{2}=1-\frac{SSE}{SSTO}.\end{equation}


We interpret $R^{2}$ as the proportion of total variation that is
explained by the multiple regression model. In MLR we must be careful,
however, because the value of $R^{2}$ can be artificially inflated
by the addition of explanatory variables to the model, regardless
of whether or not the added variables are useful with respect to prediction
of the response variable. In fact, it can be proved that the addition
of a single explanatory variable to a regression model will increase
the value of $R^{2}$, \emph{no matter how worthless} the explanatory
variable is. We could model the height of the ocean tides, then add
a variable for the length of cheetah tongues on the Serengeti plain,
and our $R^{2}$ would inevitably increase.

This is a problem, because as the philosopher, Occam, once said: {}``causes
should not be multiplied beyond necessity''. We address the problem
by penalizing $R^{2}$ when parameters are added to the model. The
result is an \emph{adjusted $R^{2}$} which we denote by $\overline{R}^{2}$.\begin{equation}
\overline{R}^{2}=\left(R^{2}-\frac{p}{n-1}\right)\left(\frac{n-1}{n-p-1}\right).\end{equation}
It is good practice for the statistician to weigh both $R^{2}$ and
$\overline{R}^{2}$ during assessment of model utility. In many cases
their values will be very close to each other. If their values differ
substantially, or if one changes dramatically when an explanatory
variable is added, then (s)he should take a closer look at the explanatory
variables in the model.


\subsection{How to do it with \textsf{R}}

For the \inputencoding{latin9}\lstinline[showstringspaces=false]!trees!\inputencoding{utf8}
data, we can get $R^{2}$ and $\overline{R}^{2}$ from the \inputencoding{latin9}\lstinline[showstringspaces=false]!summary!\inputencoding{utf8}
output or access the values directly by name as shown (recall that
we stored the \inputencoding{latin9}\lstinline[showstringspaces=false]!summary!\inputencoding{utf8}
object in \inputencoding{latin9}\lstinline[showstringspaces=false]!treesumry!\inputencoding{utf8}).

<<>>=
treesumry$r.squared
treesumry$adj.r.squared
@

High values of $R^{2}$ and $\overline{R}^{2}$ such as these indicate
that the model fits very well, which agrees with what we saw in Figure
\ref{fig:3D-scatterplot-trees}.


\subsection{Overall \emph{F}-Test\label{sub:mlr-Overall-F-Test}}

Another way to assess the model's utility is to to test the hypothesis\[
H_{0}:\beta_{1}=\beta_{2}=\cdots=\beta_{p}=0\mbox{ versus }H_{1}:\mbox{ at least one \ensuremath{\beta_{i}\neq0}}.\]
The idea is that if all $\beta_{i}$'s were zero, then the explanatory
variables $X_{1},\ldots,X_{p}$ would be worthless predictors for
the response variable $Y$. We can test the above hypothesis with
the overall $F$ statistic, which in MLR is defined by\begin{equation}
F=\frac{SSR/p}{SSE/(n-p-1)}.\end{equation}
When the regression assumptions hold and under $H_{0}$, it can be
shown that $F\sim\mathsf{f}(\mathtt{df1}=p,\,\mathtt{df2}=n-p-1)$.
We reject $H_{0}$ when $F$ is large, that is, when the explained
variation is large relative to the unexplained variation.


\subsection{How to do it with \textsf{R}}

The overall $F$ statistic and its associated \emph{p}-value is listed
at the bottom of the \inputencoding{latin9}\lstinline[showstringspaces=false]!summary!\inputencoding{utf8}
output, or we can access it directly by name; it is stored in the
\inputencoding{latin9}\lstinline[showstringspaces=false]!fstatistic!\inputencoding{utf8}
component of the \inputencoding{latin9}\lstinline[showstringspaces=false]!summary!\inputencoding{utf8}
object. 

<<>>=
treesumry$fstatistic
@

For the \inputencoding{latin9}\lstinline[showstringspaces=false]!trees!\inputencoding{utf8}
data, we see that $F=$ \Sexpr{treesumry$fstatistic[1]} with a \emph{p}-value
\inputencoding{latin9}\lstinline[showstringspaces=false]!< 2.2e-16!\inputencoding{utf8}.
Consequently we reject $H_{0}$, that is, the data provide strong
evidence that not all $\beta_{i}$'s are zero.


\subsection{Student's \emph{t} Tests\label{sub:mlr-Student's-t-Tests}}

We know that \begin{equation}
\mathbf{b}\sim\mathsf{mvnorm}\left(\mathtt{mean}=\upbeta,\,\mathtt{sigma}=\sigma^{2}\left(\mathbf{X}^{\mathrm{T}}\mathbf{X}\right)^{-1}\right)\end{equation}
and we have seen how to test the hypothesis $H_{0}:\beta_{1}=\beta_{2}=\cdots=\beta_{p}=0$,
but let us now consider the test\begin{equation}
H_{0}:\beta_{i}=0\mbox{ versus }H_{1}:\beta_{i}\neq0,\end{equation}
where $\beta_{i}$ is the coefficient for the $i$$^{\textrm{th}}$
independent variable. We test the hypothesis by calculating a statistic,
examining it's null distribution, and rejecting $H_{0}$ if the \emph{p}-value
is small. If $H_{0}$ is rejected, then we conclude that there is
a significant relationship between $Y$ and $x_{i}$ \emph{in the
regression model} $Y\sim(x_{1},\ldots,x_{p})$. This last part of
the sentence is very important because the significance of the variable
$x_{i}$ sometimes depends on the presence of other independent variables
in the model%
\footnote{In other words, a variable might be highly significant one moment
but then fail to be significant when another variable is added to
the model. When this happens it often indicates a problem with the
explanatory variables, such as \emph{multicollinearity}. See Section
\ref{sub:Multicollinearity}.%
}. 

To test the hypothesis we go to find the sampling distribution of
$b_{i},$ the estimator of the corresponding parameter $\beta_{i}$,
when the null hypothesis is true. We saw in Section \ref{sub:mlr-interval-est-params}
that \begin{equation}
T_{i}=\frac{b_{i}-\beta_{i}}{S_{b_{i}}}\end{equation}
has a Student's $t$ distribution with $n-(p+1)$ degrees of freedom.
(Remember, we are estimating $p+1$ parameters.) Consequently, under
the null hypothesis $H_{0}:\beta_{i}=0$ the statistic $t_{i}=b_{i}/S_{b_{i}}$
has a $\mathsf{t}(\mathtt{df}=n-p-1)$ distribution.


\subsection{How to do it with \textsf{R}}

The Student's $t$ tests for significance of the individual explanatory
variables are shown in the \inputencoding{latin9}\lstinline[showstringspaces=false]!summary!\inputencoding{utf8}
output.

<<>>=
treesumry
@

We see from the \emph{p}-values that there is a significant linear
relationship between \inputencoding{latin9}\lstinline[showstringspaces=false]!Volume!\inputencoding{utf8}
and \inputencoding{latin9}\lstinline[showstringspaces=false]!Girth!\inputencoding{utf8}
and between \inputencoding{latin9}\lstinline[showstringspaces=false]!Volume!\inputencoding{utf8}
and \inputencoding{latin9}\lstinline[showstringspaces=false]!Height!\inputencoding{utf8}
in the regression model \inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false]!Volume ~ Girth + Height!\inputencoding{utf8}.
Further, it appears that the \inputencoding{latin9}\lstinline[showstringspaces=false]!Intercept!\inputencoding{utf8}
is significant in the aforementioned model.


\section{Polynomial Regression\label{sec:Polynomial-Regression}}


\subsection{Quadratic Regression Model}

In each of the previous sections we assumed that $\mu$ was a linear
function of the explanatory variables. For example, in SLR we assumed
that $\mu(x)=\beta_{0}+\beta_{1}x$, and in our previous MLR examples
we assumed $\mu(x_{1},x_{2})=\beta_{0}+\beta_{1}x_{1}+\beta_{2}x_{2}$.
In every case the scatterplots indicated that our assumption was reasonable.
Sometimes, however, plots of the data suggest that the linear model
is incomplete and should be modified. 

%
\begin{figure}
\begin{centering}
<<echo = FALSE, fig=true, height = 4, width = 6>>=
plot(Volume ~ Girth, data = trees)
@
\par\end{centering}

\caption{Scatterplot of \texttt{Volume} versus \texttt{Girth} for the \texttt{trees}
data\label{fig:Scatterplot-Volume-Girth-trees}}

\end{figure}


For example, let us examine a scatterplot of \inputencoding{latin9}\lstinline[showstringspaces=false]!Volume!\inputencoding{utf8}
versus \inputencoding{latin9}\lstinline[showstringspaces=false]!Girth!\inputencoding{utf8}
a little more closely. See Figure \ref{fig:Scatterplot-Volume-Girth-trees}.
There might be a slight curvature to the data; the volume curves ever
so slightly upward as the girth increases. After looking at the plot
we might try to capture the curvature with a mean response such as
\begin{equation}
\mu(x_{1})=\beta_{0}+\beta_{1}x_{1}+\beta_{2}x_{1}^{2}.\end{equation}
The model associated with this choice of $\mu$ is\begin{equation}
Y=\beta_{0}+\beta_{1}x_{1}+\beta_{2}x_{1}^{2}+\epsilon.\end{equation}
The regression assumptions are the same. Almost everything indeed
is the same. In fact, it is still called a {}``linear regression
model'', since the mean response $\mu$ is linear \emph{in the parameters}
$\beta_{0}$, $\beta_{1}$, and $\beta_{2}$. 

\textbf{\textsc{However, there is one important difference.}} When
we introduce the squared variable in the model we inadvertently also
introduce strong dependence between the terms which can cause significant
numerical problems when it comes time to calculate the parameter estimates.
Therefore, we should usually rescale the independent variable to have
mean zero (and even variance one if we wish) \textbf{\textsc{before}}
fitting the model. That is, we replace the $x_{i}$'s with $x_{i}-\xbar$
(or $(x_{i}-\xbar)/s$) before fitting the model%
\footnote{Rescaling the data gets the job done but a better way to avoid the
multicollinearity introduced by the higher order terms is with \emph{orthogonal
polynomials}, whose coefficients are chosen just right so that the
polynomials are not correlated with each other. This is beginning
to linger outside the scope of this book, however, so we will content
ourselves with a brief mention and then stick with the rescaling approach
in the discussion that follows. A nice example of orthogonal polynomials
in action can be run with \texttt{example(cars)}.%
}.


\subsection*{How to do it with \textsf{R}}

There are multiple ways to fit a quadratic model to the variables
\inputencoding{latin9}\lstinline[showstringspaces=false]!Volume!\inputencoding{utf8}
and \inputencoding{latin9}\lstinline[showstringspaces=false]!Girth!\inputencoding{utf8}
using \textsf{R}.
\begin{enumerate}
\item One way would be to square the values for \inputencoding{latin9}\lstinline[showstringspaces=false]!Girth!\inputencoding{utf8}
and save them in a vector \inputencoding{latin9}\lstinline[showstringspaces=false]!Girthsq!\inputencoding{utf8}.
Next, fit the linear model \inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false]!Volume ~ Girth + Girthsq!\inputencoding{utf8}.
\item A second way would be to use the \emph{insulate} function in \textsf{R},
denoted by \inputencoding{latin9}\lstinline[basicstyle={\ttfamily}]!I!\inputencoding{utf8}:


\inputencoding{latin9}
\begin{lstlisting}[showstringspaces=false]
Volume ~ Girth + I(Girth^2)
\end{lstlisting}
\inputencoding{utf8}

The second method is shorter than the first but the end result is
the same. And once we calculate and store the fitted model (in, say,
\inputencoding{latin9}\lstinline[showstringspaces=false]!treesquad.lm!\inputencoding{utf8})
all of the previous comments regarding \textsf{R} apply.

\item A third and {}``right'' way to do it is with orthogonal polynomials:
\inputencoding{latin9}
\begin{lstlisting}[showstringspaces=false]
Volume ~ poly(Girth, degree = 2)
\end{lstlisting}
\inputencoding{utf8}See \inputencoding{latin9}\lstinline[showstringspaces=false]!?poly!\inputencoding{utf8}
and \inputencoding{latin9}\lstinline[showstringspaces=false]!?cars!\inputencoding{utf8}
for more information. Note that we can recover the approach in 2 with
\inputencoding{latin9}\lstinline[showstringspaces=false]!poly(Girth, degree = 2, raw = TRUE)!\inputencoding{utf8}. \end{enumerate}
\begin{example}
We will fit the quadratic model to the \inputencoding{latin9}\lstinline[showstringspaces=false]!trees!\inputencoding{utf8}
data and display the results with \inputencoding{latin9}\lstinline[showstringspaces=false]!summary!\inputencoding{utf8},
being careful to rescale the data before fitting the model. We may
rescale the \inputencoding{latin9}\lstinline[showstringspaces=false]!Girth!\inputencoding{utf8}
variable to have zero mean and unit variance on-the-fly with the \inputencoding{latin9}\lstinline[showstringspaces=false]!scale!\inputencoding{utf8}
function.

<<>>=
treesquad.lm <- lm(Volume ~ scale(Girth) + I(scale(Girth)^2), data = trees)
summary(treesquad.lm)
@

We see that the $F$ statistic indicates the overall model including
\inputencoding{latin9}\lstinline[showstringspaces=false]!Girth!\inputencoding{utf8}
and \inputencoding{latin9}\lstinline[showstringspaces=false]!Girth^2!\inputencoding{utf8}
is significant. Further, there is strong evidence that both \inputencoding{latin9}\lstinline[showstringspaces=false]!Girth!\inputencoding{utf8}
and \inputencoding{latin9}\lstinline[showstringspaces=false]!Girth^2!\inputencoding{utf8}
are significantly related to \inputencoding{latin9}\lstinline[showstringspaces=false]!Volume!\inputencoding{utf8}.
We may examine a scatterplot together with the fitted quadratic function
using the \inputencoding{latin9}\lstinline[showstringspaces=false]!lines!\inputencoding{utf8}
function, which adds a line to the plot tracing the estimated mean
response.

<<eval = FALSE>>=
plot(Volume ~ scale(Girth), data = trees)
lines(fitted(treesquad.lm) ~ scale(Girth), data = trees)
@

The plot is shown in Figure \ref{cap:Fitting-the-Quadratic}. Pay
attention to the scale on the $x$-axis: it is on the scale of the
transformed \inputencoding{latin9}\lstinline[showstringspaces=false]!Girth!\inputencoding{utf8}
data and not on the original scale.

\end{example}
%
\begin{figure}
\begin{centering}
<<echo = FALSE, fig=true, height = 4, width = 6>>=
plot(Volume ~ scale(Girth), data = trees)
lines(fitted(treesquad.lm) ~ scale(Girth), data = trees)
@
\par\end{centering}

\caption{A quadratic model for the \texttt{trees} data\label{cap:Fitting-the-Quadratic}}

\end{figure}

\begin{rem}
When a model includes a quadratic term for an independent variable,
it is customary to also include the linear term in the model. The
principle is called \emph{parsimony}. More generally, if the researcher
decides to include $x^{m}$ as a term in the model, then (s)he should
also include all lower order terms $x$, $x^{2}$, \ldots{},$x^{m-1}$
in the model. 
\end{rem}
We do estimation/prediction the same way that we did in Section \ref{sub:mlr-point-est-regsurface},
except we do not need a \inputencoding{latin9}\lstinline[showstringspaces=false]!Height!\inputencoding{utf8}
column in the dataframe \inputencoding{latin9}\lstinline[showstringspaces=false]!new!\inputencoding{utf8}
since the variable is not included in the quadratic model.

<<>>=
new <- data.frame(Girth = c(9.1, 11.6, 12.5))
predict(treesquad.lm, newdata = new, interval = "prediction")
@

The predictions and intervals are slightly different from what they
were previously. Notice that it was not necessary to rescale the \inputencoding{latin9}\lstinline[showstringspaces=false]!Girth!\inputencoding{utf8}
prediction data before input to the \inputencoding{latin9}\lstinline[showstringspaces=false]!predict!\inputencoding{utf8}
function; the model did the rescaling for us automatically.
\begin{rem}
We have mentioned on several occasions that it is important to rescale
the explanatory variables for polynomial regression. Watch what happens
if we ignore this advice:

<<>>=
summary(lm(Volume ~ Girth + I(Girth^2), data = trees))
@

Now nothing is significant in the model except \inputencoding{latin9}\lstinline[showstringspaces=false]!Girth^2!\inputencoding{utf8}.
We could delete the \inputencoding{latin9}\lstinline[showstringspaces=false]!Intercept!\inputencoding{utf8}
and \inputencoding{latin9}\lstinline[showstringspaces=false]!Girth!\inputencoding{utf8}
from the model, but the model would no longer be \emph{parsimonious}.
A novice may see the output and be confused about how to proceed,
while the seasoned statistician recognizes immediately that \inputencoding{latin9}\lstinline[showstringspaces=false]!Girth!\inputencoding{utf8}
and \inputencoding{latin9}\lstinline[showstringspaces=false]!Girth^2!\inputencoding{utf8}
are highly correlated (see Section \ref{sub:Multicollinearity}).
The only remedy to this ailment is to rescale \inputencoding{latin9}\lstinline[showstringspaces=false]!Girth!\inputencoding{utf8},
which we should have done in the first place.

In Example \ref{exa:mlr-trees-poly-no-rescale} of Section \ref{sec:Partial-F-Statistic}
we investigate this issue further.
\end{rem}

\section{Interaction\label{sec:Interaction}}

In our model for tree volume there have been two independent variables:
\inputencoding{latin9}\lstinline[showstringspaces=false]!Girth!\inputencoding{utf8}
and \inputencoding{latin9}\lstinline[showstringspaces=false]!Height!\inputencoding{utf8}.
We may suspect that the independent variables are related, that is,
values of one variable may tend to influence values of the other.
It may be desirable to include an additional term in our model to
try and capture the dependence between the variables. Interaction
terms are formed by multiplying one (or more) explanatory variable(s)
by another. 
\begin{example}
Perhaps the \inputencoding{latin9}\lstinline[showstringspaces=false]!Girth!\inputencoding{utf8}
and \inputencoding{latin9}\lstinline[showstringspaces=false]!Height!\inputencoding{utf8}
of the tree interact to influence the its \inputencoding{latin9}\lstinline[showstringspaces=false]!Volume!\inputencoding{utf8};
we would like to investigate whether the model (\inputencoding{latin9}\lstinline[showstringspaces=false]!Girth!\inputencoding{utf8}
= $x_{1}$ and \inputencoding{latin9}\lstinline[showstringspaces=false]!Height!\inputencoding{utf8}
= $x_{2}$) \begin{equation}
Y=\beta_{0}+\beta_{1}x_{1}+\beta_{2}x_{2}+\epsilon\end{equation}
would be significantly improved by the model
\end{example}
\begin{equation}
Y=\beta_{0}+\beta_{1}x_{1}+\beta_{2}x_{2}+\beta_{1:2}x_{1}x_{2}+\epsilon,\end{equation}
where the subscript $1:2$ denotes that $\beta_{1:2}$ is a coefficient
of an interaction term between $x{}_{1}$ and $x_{2}$. 


\paragraph*{What does it mean?}

Consider the mean response $\mu(x_{1},x_{2})$ as a function of $x_{2}$:\begin{equation}
\mu(x_{2})=(\beta_{0}+\beta_{1}x_{1})+\beta_{2}x_{2}.\end{equation}
This is a linear function of $x_{2}$ with slope $\beta_{2}$. As
$x_{1}$ changes, the $y$-intercept of the mean response in $x_{2}$
changes, but the slope remains the same. Therefore, the mean response
in $x_{2}$ is represented by a collection of parallel lines all with
common slope $\beta_{2}$.

Now think about what happens when the interaction term $\beta_{1:2}x_{1}x_{2}$
is included. The mean response in $x_{2}$ now looks like\begin{equation}
\mu(x_{2})=(\beta_{0}+\beta_{1}x_{1})+(\beta_{2}+\beta_{1:2}x_{1})x_{2}.\end{equation}
In this case we see that not only the $y$-intercept changes when
$x_{1}$ varies, but the slope also changes in $x_{1}$. Thus, the
interaction term allows the slope of the mean response in $x_{2}$
to increase and decrease as $x_{1}$ varies.


\subsection*{How to do it with \textsf{R}}

There are several ways to introduce an interaction term into the model.
\begin{enumerate}
\item Make a new variable \inputencoding{latin9}\lstinline[showstringspaces=false]!prod <- Girth * Height!\inputencoding{utf8},
then include \inputencoding{latin9}\lstinline[showstringspaces=false]!prod!\inputencoding{utf8}
in the model formula \inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false]!Volume ~ Girth + Height + prod!\inputencoding{utf8}.
This method is perhaps the most transparent, but it also reserves
memory space unnecessarily.
\item Once can construct an interaction term directly in \textsf{R} with
a colon {}``\inputencoding{latin9}\lstinline[showstringspaces=false]!:!\inputencoding{utf8}''.
For this example, the model formula would look like \inputencoding{latin9}\lstinline[showstringspaces=false]!Volume ~ Girth + Height + Girth:Height!\inputencoding{utf8}.
\end{enumerate}
For the \inputencoding{latin9}\lstinline[showstringspaces=false]!trees!\inputencoding{utf8}
data, we fit the model with the interaction using method two and see
if it is significant:

<<>>=
treesint.lm <- lm(Volume ~ Girth + Height + Girth:Height, data = trees)
summary(treesint.lm)
@

We can see from the output that the interaction term is highly significant.
Further, the estimate $b_{1:2}$ is positive. This means that the
slope of $\mu(x_{2})$ is steeper for bigger values of \inputencoding{latin9}\lstinline[showstringspaces=false]!Girth!\inputencoding{utf8}.
Keep in mind: the same interpretation holds for $\mu(x_{1})$; that
is, the slope of $\mu(x_{1})$ is steeper for bigger values of \inputencoding{latin9}\lstinline[showstringspaces=false]!Height!\inputencoding{utf8}.

For the sake of completeness we calculate confidence intervals for
the parameters and do prediction as before.

<<>>=
confint(treesint.lm)
new <- data.frame(Girth = c(9.1, 11.6, 12.5), Height = c(69, 74, 87))
predict(treesint.lm, newdata = new, interval = "prediction")
@
\begin{rem}
There are two other ways to include interaction terms in model formulas.
For example, we could have written \inputencoding{latin9}\lstinline[showstringspaces=false]!Girth * Height!\inputencoding{utf8}
or even \inputencoding{latin9}\lstinline[showstringspaces=false]!(Girth + Height)^2!\inputencoding{utf8}
and both would be the same as \inputencoding{latin9}\lstinline[showstringspaces=false]!Girth + Height + Girth:Height!\inputencoding{utf8}.
\end{rem}
These examples can be generalized to more than two independent variables,
say three, four, or even more. We may be interested in seeing whether
any pairwise interactions are significant. We do this with a model
formula that looks something like\texttt{ }\inputencoding{latin9}\lstinline[showstringspaces=false]!y ~ (x1 + x2 + x3 + x4)^2!\inputencoding{utf8}. 


\section{Qualitative Explanatory Variables\label{sec:Qualitative-Explanatory-Variables}}

We have so far been concerned with numerical independent variables
taking values in a subset of real numbers. In this section, we extend
our treatment to include the case in which one of the explanatory
variables is qualitative, that is, a \emph{factor}. Qualitative variables
take values in a set of \emph{levels}, which may or may not be ordered.
See Section \ref{sub:Qualitative-Data}.
\begin{note*}
The \inputencoding{latin9}\lstinline[showstringspaces=false]!trees!\inputencoding{utf8}
data do not have any qualitative explanatory variables, so we will
construct one for illustrative purposes%
\footnote{This procedure of replacing a continuous variable by a discrete/qualitative
one is called \emph{binning}, and is almost \emph{never} the right
thing to do. We are in a bind at this point, however, because we have
invested this chapter in the \texttt{trees} data and I do not want
to switch mid-discussion. I am currently searching for a data set
with pre-existing qualitative variables that also conveys the same
points present in the trees data, and when I find it I will update
this chapter accordingly.%
}. We will leave the \inputencoding{latin9}\lstinline[showstringspaces=false]!Girth!\inputencoding{utf8}
variable alone, but we will replace the variable \inputencoding{latin9}\lstinline[showstringspaces=false]!Height!\inputencoding{utf8}
by a new variable \inputencoding{latin9}\lstinline[showstringspaces=false]!Tall!\inputencoding{utf8}
which indicates whether or not the cherry tree is taller than a certain
threshold (which for the sake of argument will be the sample median
height of 76\,ft). That is, \inputencoding{latin9}\lstinline[showstringspaces=false]!Tall!\inputencoding{utf8}
will be defined by\begin{equation}
\mathtt{Tall}=\begin{cases}
\mathtt{yes}, & \mbox{if }\mathtt{Height}>76,\\
\mathtt{no}, & \mbox{if }\mathtt{Height}\leq76.\end{cases}\end{equation}
We can construct \inputencoding{latin9}\lstinline[showstringspaces=false]!Tall!\inputencoding{utf8}
very quickly in \textsf{R} with the \inputencoding{latin9}\lstinline[showstringspaces=false]!cut!\inputencoding{utf8}
function:

<<>>=
trees$Tall <- cut(trees$Height, breaks = c(-Inf, 76, Inf), labels = c("no","yes"))
trees$Tall[1:5]
@

Note that \inputencoding{latin9}\lstinline[showstringspaces=false]!Tall!\inputencoding{utf8}
is automatically generated to be a factor with the labels in the correct
order. See \inputencoding{latin9}\lstinline[showstringspaces=false]!?cut!\inputencoding{utf8}
for more. 

\end{note*}
Once we have \inputencoding{latin9}\lstinline[showstringspaces=false]!Tall!\inputencoding{utf8},
we include it in the regression model just like we would any other
variable. It is handled internally in a special way. Define a {}``dummy
variable'' \inputencoding{latin9}\lstinline[showstringspaces=false]!Tallyes!\inputencoding{utf8}
that takes values\begin{equation}
\mathtt{Tallyes}=\begin{cases}
1, & \mbox{if }\mathtt{Tall}=\mathtt{yes},\\
0, & \mbox{otherwise.}\end{cases}\end{equation}
That is, \inputencoding{latin9}\lstinline[showstringspaces=false]!Tallyes!\inputencoding{utf8}
is an \emph{indicator} \emph{variable} which indicates when a respective
tree is tall. The model may now be written as \begin{equation}
\mathtt{Volume}=\beta_{0}+\beta_{1}\mathtt{Girth}+\beta_{2}\mathtt{Tallyes}+\epsilon.\end{equation}
Let us take a look at what this definition does to the mean response.
Trees with \inputencoding{latin9}\lstinline[showstringspaces=false]!Tall = yes!\inputencoding{utf8}
will have the mean response\begin{equation}
\mu(\mathtt{Girth})=(\beta_{0}+\beta_{2})+\beta_{1}\mathtt{Girth},\end{equation}
while trees with \inputencoding{latin9}\lstinline[showstringspaces=false]!Tall = no!\inputencoding{utf8}
will have the mean response\begin{equation}
\mu(\mathtt{Girth})=\beta_{0}+\beta_{1}\mathtt{Girth}.\end{equation}
In essence, we are fitting two regression lines: one for tall trees,
and one for short trees. The regression lines have the same slope
but they have different $y$ intercepts (which are exactly $|\beta_{2}|$
far apart).


\subsection*{How to do it with \textsf{R}}

The important thing is to double check that the qualitative variable
in question is stored as a factor. The way to check is with the \inputencoding{latin9}\lstinline[showstringspaces=false]!class!\inputencoding{utf8}
command. For example,

<<>>=
class(trees$Tall)
@

If the qualitative variable is not yet stored as a factor then we
may convert it to one with the \inputencoding{latin9}\lstinline[showstringspaces=false]!factor!\inputencoding{utf8}
command. See Section \ref{sub:Qualitative-Data}. Other than this
we perform MLR as we normally would.

<<>>=
treesdummy.lm <- lm(Volume ~ Girth + Tall, data = trees)
summary(treesdummy.lm)
@

From the output we see that all parameter estimates are statistically
significant and we conclude that the mean response differs for trees
with \inputencoding{latin9}\lstinline[showstringspaces=false]!Tall = yes!\inputencoding{utf8}
and trees with \inputencoding{latin9}\lstinline[showstringspaces=false]!Tall = no!\inputencoding{utf8}. 
\begin{rem}
We were somewhat disingenuous when we defined the dummy variable \inputencoding{latin9}\lstinline[showstringspaces=false]!Tallyes!\inputencoding{utf8}
because, in truth, \textsf{R} defines \inputencoding{latin9}\lstinline[showstringspaces=false]!Tallyes!\inputencoding{utf8}
automatically without input from the user%
\footnote{That is, \textsf{R} by default handles contrasts according to its
internal settings which may be customized by the user for fine control.
Given that we will not investigate contrasts further in this book
it does not serve the discussion to delve into those settings, either.
The interested reader should check ?contrasts for details.%
}. Indeed, the author fit the model beforehand and wrote the discussion
afterward with the knowledge of what \textsf{R} would do so that the
output the reader saw would match what (s)he had previously read.
The way that \textsf{R} handles factors internally is part of a much
larger topic concerning \emph{contrasts}, which falls outside the
scope of this book. The interested reader should see Neter et al \cite{Neter1996}
or Fox \cite{Fox1997} for more.
\end{rem}

\begin{rem}
In general, if an explanatory variable \inputencoding{latin9}\lstinline[showstringspaces=false]!foo!\inputencoding{utf8}
is qualitative with $n$ levels \inputencoding{latin9}\lstinline[showstringspaces=false]!bar1!\inputencoding{utf8},
\inputencoding{latin9}\lstinline[showstringspaces=false]!bar2!\inputencoding{utf8},
\ldots{}, \inputencoding{latin9}\lstinline[showstringspaces=false]!barn!\inputencoding{utf8}
then \textsf{R} will by default automatically define $n-1$ indicator
variables in the following way:\begin{eqnarray*}
\mathtt{foobar2} & = & \begin{cases}
1, & \mbox{if }\mathtt{foo}=\mathtt{"bar2"},\\
0, & \mbox{otherwise.}\end{cases},\,\ldots,\,\mathtt{foobarn}=\begin{cases}
1, & \mbox{if }\mathtt{foo}=\mathtt{"barn"},\\
0, & \mbox{otherwise.}\end{cases}\end{eqnarray*}
The level \inputencoding{latin9}\lstinline[showstringspaces=false]!bar1!\inputencoding{utf8}
is represented by $\mathtt{foobar2}=\cdots=\mathtt{foobarn}=0$. We
just need to make sure that \inputencoding{latin9}\lstinline[showstringspaces=false]!foo!\inputencoding{utf8}
is stored as a factor and \textsf{R} will take care of the rest.
\end{rem}

\subsection*{Graphing the Regression Lines}

We can see a plot of the two regression lines with the following mouthful
of code.

<<eval = FALSE>>=
treesTall <- split(trees, trees$Tall)
treesTall[["yes"]]$Fit <- predict(treesdummy.lm, treesTall[["yes"]])
treesTall[["no"]]$Fit <- predict(treesdummy.lm, treesTall[["no"]])
plot(Volume ~ Girth, data = trees, type = "n")
points(Volume ~ Girth, data = treesTall[["yes"]], pch = 1)
points(Volume ~ Girth, data = treesTall[["no"]], pch = 2)
lines(Fit ~ Girth, data = treesTall[["yes"]])
lines(Fit ~ Girth, data = treesTall[["no"]])
@

%
\begin{figure}
\begin{centering}
<<echo = FALSE, fig=true, height = 4, width = 6>>=
treesTall <- split(trees, trees$Tall)
treesTall[["yes"]]$Fit <- predict(treesdummy.lm, treesTall[["yes"]])
treesTall[["no"]]$Fit <- predict(treesdummy.lm, treesTall[["no"]])
plot(Volume ~ Girth, data = trees, type = "n")
points(Volume ~ Girth, data = treesTall[["yes"]], pch = 1)
points(Volume ~ Girth, data = treesTall[["no"]], pch = 2)
lines(Fit ~ Girth, data = treesTall[["yes"]])
lines(Fit ~ Girth, data = treesTall[["no"]])
@
\par\end{centering}

\caption{A dummy variable model for the \texttt{trees} data\label{fig:dummy-variable-trees}}

\end{figure}


It may look intimidating but there is reason to the madness. First
we \inputencoding{latin9}\lstinline[showstringspaces=false]!split!\inputencoding{utf8}
the \inputencoding{latin9}\lstinline[showstringspaces=false]!trees!\inputencoding{utf8}
data into two pieces, with groups determined by the \inputencoding{latin9}\lstinline[showstringspaces=false]!Tall!\inputencoding{utf8}
variable. Next we add the \inputencoding{latin9}\lstinline[showstringspaces=false]!Fit!\inputencoding{utf8}ted
values to each piece via \inputencoding{latin9}\lstinline[showstringspaces=false]!predict!\inputencoding{utf8}.
Then we set up a \inputencoding{latin9}\lstinline[showstringspaces=false]!plot!\inputencoding{utf8}
for the variables \inputencoding{latin9}\lstinline[showstringspaces=false]!Volume!\inputencoding{utf8}
versus \inputencoding{latin9}\lstinline[showstringspaces=false]!Girth!\inputencoding{utf8},
but we do not plot anything yet (\inputencoding{latin9}\lstinline[showstringspaces=false]!type = n!\inputencoding{utf8})
because we want to use different symbols for the two groups. Next
we add \inputencoding{latin9}\lstinline[showstringspaces=false]!points!\inputencoding{utf8}
to the plot for the \inputencoding{latin9}\lstinline[showstringspaces=false]!Tall = yes!\inputencoding{utf8}
trees and use an open circle for a plot character (\inputencoding{latin9}\lstinline[showstringspaces=false]!pch = 1!\inputencoding{utf8}),
followed by \inputencoding{latin9}\lstinline[showstringspaces=false]!points!\inputencoding{utf8}
for the \inputencoding{latin9}\lstinline[showstringspaces=false]!Tall = no!\inputencoding{utf8}
trees with a triangle character (\inputencoding{latin9}\lstinline[showstringspaces=false]!pch = 2!\inputencoding{utf8}).
Finally, we add regression \inputencoding{latin9}\lstinline[showstringspaces=false]!lines!\inputencoding{utf8}
to the plot, one for each group.

There are other -- shorter -- ways to plot regression lines by groups,
namely the \inputencoding{latin9}\lstinline[showstringspaces=false]!scatterplot!\inputencoding{utf8}
function in the \inputencoding{latin9}\lstinline[showstringspaces=false]!car!\inputencoding{utf8}
\cite{car} package and the \inputencoding{latin9}\lstinline[showstringspaces=false]!xyplot!\inputencoding{utf8}
function in the \inputencoding{latin9}\lstinline[showstringspaces=false]!lattice!\inputencoding{utf8}
package. We elected to introduce the reader to the above approach
since many advanced plots in \textsf{R} are done in a similar, consecutive
fashion.


\section{Partial \emph{F} Statistic\label{sec:Partial-F-Statistic}}

We saw in Section \ref{sub:mlr-Overall-F-Test} how to test $H_{0}:\beta_{0}=\beta_{1}=\cdots=\beta_{p}=0$
with the overall $F$ statistic and we saw in Section \ref{sub:mlr-Student's-t-Tests}
how to test $H_{0}:\beta_{i}=0$ that a particular coefficient $\beta_{i}$
is zero. Sometimes, however, we would like to test whether a certain
part of the model is significant. Consider the regression model\begin{equation}
Y=\beta_{0}+\beta_{1}x_{1}+\cdots+\beta_{j}x_{j}+\beta_{j+1}x_{j+1}+\cdots+\beta_{p}x_{p}+\epsilon,\end{equation}
 where $j\geq1$ and $p\geq2$. Now we wish to test the hypothesis\begin{equation}
H_{0}:\beta_{j+1}=\beta_{j+2}=\cdots=\beta_{p}=0\end{equation}
versus the alternative \begin{equation}
H_{1}:\mbox{at least one of \ensuremath{\beta_{j+1},\ \beta_{j+2},\ ,\ldots,\beta_{p}\neq0}}.\end{equation}
The interpretation of $H_{0}$ is that none of the variables $x_{j+1}$,
\ldots{},$x_{p}$ is significantly related to $Y$ and the interpretation
of $H_{1}$ is that at least one of $x_{j+1}$, \ldots{},$x_{p}$
is significantly related to $Y$. In essence, for this hypothesis
test there are two competing models under consideration:\begin{align}
\mbox{the full model:} & \quad y=\beta_{0}+\beta_{1}x_{1}+\cdots+\beta_{p}x_{p}+\epsilon,\\
\mbox{the reduced model:} & \quad y=\beta_{0}+\beta_{1}x_{1}+\cdots+\beta_{j}x_{j}+\epsilon,\end{align}
Of course, the full model will always explain the data \emph{better}
than the reduced model, but does the full model explain the data \emph{significantly
better} than the reduced model? This question is exactly what the
partial $F$ statistic is designed to answer.

We first calculate $SSE_{f}$, the unexplained variation in the full
model, and $SSE_{r}$, the unexplained variation in the reduced model.
We base our test on the difference $SSE_{r}-SSE_{f}$ which measures
the reduction in unexplained variation attributable to the variables
$x_{j+1}$, \ldots{},$x_{p}$. In the full model there are $p+1$
parameters and in the reduced model there are $j+1$ parameters, which
gives a difference of $p-j$ parameters (hence degrees of freedom).
The partial \emph{F} statistic is \begin{equation}
F=\frac{(SSE_{r}-SSE_{f})/(p-j)}{SSE_{f}/(n-p-1)}.\end{equation}
It can be shown when the regression assumptions hold under $H_{0}$
that the partial $F$ statistic has an $\mathsf{f}(\mathtt{df1}=p-j,\,\mathtt{df2}=n-p-1)$
distribution. We calculate the $p$-value of the observed partial
$F$ statistic and reject $H_{0}$ if the $p$-value is small.


\subsection*{How to do it with \textsf{R}}

The key ingredient above is that the two competing models are \emph{nested}
in the sense that the reduced model is entirely contained within the
complete model. The way to test whether the improvement is significant
is to compute \inputencoding{latin9}\lstinline[showstringspaces=false]!lm!\inputencoding{utf8}
objects both for the complete model and the reduced model then compare
the answers with the \inputencoding{latin9}\lstinline[showstringspaces=false]!anova!\inputencoding{utf8}
function.
\begin{example}
\label{exa:mlr-trees-poly-no-rescale}For the \inputencoding{latin9}\lstinline[showstringspaces=false]!trees!\inputencoding{utf8}
data, let us fit a polynomial regression model and for the sake of
argument we will ignore our own good advice and fail to rescale the
explanatory variables. 

<<>>=
treesfull.lm <- lm(Volume ~ Girth + I(Girth^2) + Height + I(Height^2), data = trees)
summary(treesfull.lm)
@

In this ill-formed model nothing is significant except \inputencoding{latin9}\lstinline[showstringspaces=false]!Girth!\inputencoding{utf8}
and \inputencoding{latin9}\lstinline[showstringspaces=false]!Girth^2!\inputencoding{utf8}.
Let us continue down this path and suppose that we would like to try
a reduced model which contains nothing but \inputencoding{latin9}\lstinline[showstringspaces=false]!Girth!\inputencoding{utf8}
and \inputencoding{latin9}\lstinline[showstringspaces=false]!Girth^2!\inputencoding{utf8}
(not even an \inputencoding{latin9}\lstinline[showstringspaces=false]!Intercept!\inputencoding{utf8}).
Our two models are now\begin{align*}
\mbox{the full model:} & \quad Y=\beta_{0}+\beta_{1}x_{1}+\beta_{2}x_{1}^{2}+\beta_{3}x_{2}+\beta_{4}x_{2}^{2}+\epsilon,\\
\mbox{the reduced model:} & \quad Y=\beta_{1}x_{1}+\beta_{2}x_{1}^{2}+\epsilon,\end{align*}
We fit the reduced model with \inputencoding{latin9}\lstinline[showstringspaces=false]!lm!\inputencoding{utf8}
and store the results:

<<>>=
treesreduced.lm <- lm(Volume ~ -1 + Girth + I(Girth^2), data = trees)
@

To delete the intercept from the model we used \inputencoding{latin9}\lstinline[showstringspaces=false]!-1!\inputencoding{utf8}
in the model formula. Next we compare the two models with the \inputencoding{latin9}\lstinline[showstringspaces=false]!anova!\inputencoding{utf8}
function. The convention is to list the models from smallest to largest.

<<>>=
anova(treesreduced.lm, treesfull.lm)
@

We see from the output that the complete model is highly significant
compared to the model that does not incorporate \inputencoding{latin9}\lstinline[showstringspaces=false]!Height!\inputencoding{utf8}
or the \inputencoding{latin9}\lstinline[showstringspaces=false]!Intercept!\inputencoding{utf8}.
We wonder (with our tongue in our cheek) if the \inputencoding{latin9}\lstinline[showstringspaces=false]!Height^2!\inputencoding{utf8}
term in the full model is causing all of the trouble. We will fit
an alternative reduced model that only deletes \inputencoding{latin9}\lstinline[showstringspaces=false]!Height^2!\inputencoding{utf8}.

<<>>=
treesreduced2.lm <- lm(Volume ~ Girth + I(Girth^2) + Height, data = trees)
anova(treesreduced2.lm, treesfull.lm)
@

In this case, the improvement to the reduced model that is attributable
to \inputencoding{latin9}\lstinline[showstringspaces=false]!Height^2!\inputencoding{utf8}
is not significant, so we can delete \inputencoding{latin9}\lstinline[showstringspaces=false]!Height^2!\inputencoding{utf8}
from the model with a clear conscience. We notice that the \emph{p}-value
for this latest partial $F$ test is 0.8865, which seems to be remarkably
close to the \emph{p}-value we saw for the univariate \emph{t} test
of \inputencoding{latin9}\lstinline[showstringspaces=false]!Height^2!\inputencoding{utf8}
at the beginning of this example. In fact, the \emph{p}-values are
\emph{exactly} the same. Perhaps now we gain some insight into the
true meaning of the univariate tests.
\end{example}

\section{Residual Analysis and Diagnostic Tools\label{sec:Residual-Analysis-MLR}}

We encountered many, many diagnostic measures for simple linear regression
in Sections \ref{sec:Residual-Analysis-SLR} and \ref{sec:Other-Diagnostic-Tools-SLR}.
All of these are valid in multiple linear regression, too, but there
are some slight changes that we need to make for the multivariate
case. We list these below, and apply them to the trees example. 
\begin{description}
\item [{Shapiro-Wilk,~Breusch-Pagan,~Durbin-Watson:}] unchanged from
SLR, but we are now equipped to talk about the Shapiro-Wilk test statistic
for the residuals. It is defined by the formula \begin{equation}
W=\frac{\mathbf{a}^{\mathrm{T}}\mathbf{E}^{\ast}}{\mathbf{E}^{\mathrm{T}}\mathbf{E}},\end{equation}
where $\mathbf{E}^{\ast}$ is the sorted residuals and $\mathbf{a}_{1\times\mathrm{n}}$
is defined by \begin{equation}
\mathbf{a}=\frac{\mathbf{m}^{\mathrm{T}}\mathbf{V}^{-1}}{\sqrt{\mathbf{m}^{\mathrm{T}}\mathbf{V}^{-1}\mathbf{V}^{-1}\mathbf{m}}},\end{equation}
where $\mathbf{m}_{\mathrm{n}\times1}$ and $\mathbf{V}_{\mathrm{n}\times\mathrm{n}}$
are the mean and covariance matrix, respectively, of the order statistics
from an $\mathsf{mvnorm}\left(\mathtt{mean}=\mathbf{0},\,\mathtt{sigma}=\mathbf{I}\right)$
distribution. 
\item [{Leverages:}] are defined to be the diagonal entries of the hat
matrix $\mathbf{H}$ (which is why we called them $h_{ii}$ in Section
\ref{sub:mlr-point-est-regsurface}). The sum of the leverages is
$\mbox{tr}(\mathbf{H})=p+1$. One rule of thumb considers a leverage
extreme if it is larger than double the mean leverage value, which
is $2(p+1)/n$, and another rule of thumb considers leverages bigger
than 0.5 to indicate high leverage, while values between 0.3 and 0.5
indicate moderate leverage.
\item [{Standardized~residuals:}] unchanged. Considered extreme if $|R_{i}|>2$.
\item [{Studentized~residuals:}] compared to a $\mathsf{t}(\mathtt{df}=n-p-2)$
distribution. 
\item [{$DFBETAS$:}] The formula is generalized to\begin{equation}
(DFBETAS)_{j(i)}=\frac{b_{j}-b_{j(i)}}{S_{(i)}\sqrt{c_{jj}}},\quad j=0,\ldots p,\ i=1,\ldots,n,\end{equation}
where $c_{jj}$ is the $j^{\mathrm{th}}$ diagonal entry of $(\mathbf{X}^{\mathrm{T}}\mathbf{X})^{-1}$.
Values larger than one for small data sets or $2/\sqrt{n}$ for large
data sets should be investigated.
\item [{$DFFITS$:}] unchanged. Larger than one in absolute value is considered
extreme.
\item [{Cook's~$D$:}] compared to an $\mathsf{f}(\mathtt{df1}=p+1,\,\mathtt{df2}=n-p-1)$
distribution. Observations falling higher than the 50$^{\textrm{th}}$
percentile are extreme. 
\end{description}
Note that plugging the value $p=1$ into the formulas will recover
all of the ones we saw in Chapter \ref{cha:simple-linear-regression}.


\section{Additional Topics\label{sec:Additional-Topics-MLR}}


\subsection{Nonlinear Regression}

We spent the entire chapter talking about the \inputencoding{latin9}\lstinline[showstringspaces=false]!trees!\inputencoding{utf8}
data, and all of our models looked like \inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false]!Volume ~ Girth + Height!\inputencoding{utf8}
or a variant of this model. But let us think again: we know from elementary
school that the volume of a rectangle is $V=lwh$ and the volume of
a cylinder (which is closer to what a black cherry tree looks like)
is\begin{equation}
V=\pi r^{2}h\quad\mbox{or}\quad V=4\pi dh,\end{equation}
where $r$ and $d$ represent the radius and diameter of the tree,
respectively. With this in mind, it would seem that a more appropriate
model for $\mu$ might be\begin{equation}
\mu(x_{1},x_{2})=\beta_{0}x_{1}^{\beta_{1}}x_{2}^{\beta_{2}},\label{eq:trees-nonlin-reg}\end{equation}
where $\beta_{1}$ and $\beta_{2}$ are parameters to adjust for the
fact that a black cherry tree is not a perfect cylinder.

How can we fit this model? The model is not linear in the parameters
any more, so our linear regression methods will not work\ldots{}or
will they? In the \inputencoding{latin9}\lstinline[showstringspaces=false]!trees!\inputencoding{utf8}
example we may take the logarithm of both sides of Equation \ref{eq:trees-nonlin-reg}
to get\begin{equation}
\mu^{\ast}(x_{1},x_{2})=\ln\left[\mu(x_{1},x_{2})\right]=\ln\beta_{0}+\beta_{1}\ln x_{1}+\beta_{2}\ln x_{2},\end{equation}
and this new model $\mu^{\ast}$ is linear in the parameters $\beta_{0}^{\ast}=\ln\beta_{0}$,
$\beta_{1}^{\ast}=\beta_{1}$ and $\beta_{2}^{\ast}=\beta_{2}$. We
can use what we have learned to fit a linear model \inputencoding{latin9}\lstinline[breaklines=true,showstringspaces=false]!log(Volume) ~ log(Girth) + log(Height)!\inputencoding{utf8},
and everything will proceed as before, with one exception: we will
need to be mindful when it comes time to make predictions because
the model will have been fit on the log scale, and we will need to
transform our predictions back to the original scale (by exponentiating
with \inputencoding{latin9}\lstinline[showstringspaces=false]!exp!\inputencoding{utf8})
to make sense.

<<>>=
treesNonlin.lm <- lm(log(Volume) ~ log(Girth) + log(Height), data = trees)
summary(treesNonlin.lm)
@

This is our best model yet (judging by $R^{2}$ and $\overline{R}^{2}$),
all of the parameters are significant, it is simpler than the quadratic
or interaction models, and it even makes theoretical sense. It rarely
gets any better than that.

We may get confidence intervals for the parameters, but remember that
it is usually better to transform back to the original scale for interpretation
purposes :

<<>>=
exp(confint(treesNonlin.lm))
@

(Note that we did not update the row labels of the matrix to show
that we exponentiated and so they are misleading as written.) We do
predictions just as before. Remember to transform the response variable
back to the original scale after prediction.

<<>>=
new <- data.frame(Girth = c(9.1, 11.6, 12.5), Height = c(69, 74, 87))
exp(predict(treesNonlin.lm, newdata = new, interval = "confidence"))
@

The predictions and intervals are slightly different from those calculated
earlier, but they are close. Note that we did not need to transform
the \inputencoding{latin9}\lstinline[showstringspaces=false]!Girth!\inputencoding{utf8}
and \inputencoding{latin9}\lstinline[showstringspaces=false]!Height!\inputencoding{utf8}
arguments in the dataframe \inputencoding{latin9}\lstinline[showstringspaces=false]!new!\inputencoding{utf8}.
All transformations are done for us automatically.


\subsection{Real Nonlinear Regression}

We saw with the \inputencoding{latin9}\lstinline[showstringspaces=false]!trees!\inputencoding{utf8}
data that a nonlinear model might be more appropriate for the data
based on theoretical considerations, and we were lucky because the
functional form of $\mu$ allowed us to take logarithms to transform
the nonlinear model to a linear one. The same trick will not work
in other circumstances, however. We need techniques to fit general
models of the form\begin{equation}
\mathbf{Y}=\mu(\mathbf{X})+\epsilon,\end{equation}
where $\mu$ is some crazy function that does not lend itself to linear
transformations.

There are a host of methods to address problems like these which are
studied in advanced regression classes. The interested reader should
see Neter \emph{et al} \cite{Neter1996} or Tabachnick and Fidell
\cite{Tabachnick2006}. 

It turns out that John Fox has posted an Appendix to his book \cite{Fox2002}
which discusses some of the methods and issues associated with nonlinear
regression; see 

\begin{center}
\url{http://cran.r-project.org/doc/contrib/Fox-Companion/appendix.html}
\par\end{center}

Here is an example of how it works, based on a question from R-help.

<<>>=
# fake data 
set.seed(1) 
x <- seq(from = 0, to = 1000, length.out = 200) 
y <- 1 + 2*(sin((2*pi*x/360) - 3))^2 + rnorm(200, sd = 2)
plot(x, y)
acc.nls <- nls(y ~ a + b*(sin((2*pi*x/360) - c))^2, start = list(a = 0.9, b = 2.3, c = 2.9))
summary(acc.nls)
#plot(x, fitted(acc.nls))
@


\subsection{Multicollinearity\label{sub:Multicollinearity}}

A multiple regression model exhibits \emph{multicollinearity} when
two or more of the explanatory variables are substantially correlated
with each other. We can measure multicollinearity by having one of
the explanatory play the role of {}``dependent variable'' and regress
it on the remaining explanatory variables. The the $R^{2}$ of the
resulting model is near one, then we say that the model is multicollinear
or shows multicollinearity.

Multicollinearity is a problem because it causes instability in the
regression model. The instability is a consequence of redundancy in
the explanatory variables: a high $R^{2}$ indicates a strong dependence
between the selected independent variable and the others. The redundant
information inflates the variance of the parameter estimates which
can cause them to be statistically insignificant when they would have
been significant otherwise. To wit, multicollinearity is usually measured
by what are called \emph{variance inflation factors}.

Once multicollinearity has been diagnosed there are several approaches
to remediate it. Here are a couple of important ones.
\begin{description}
\item [{Principal~Components~Analysis.}] This approach casts out two
or more of the original explanatory variables and replaces them with
new variables, derived from the original ones, that are by design
uncorrelated with one another. The redundancy is thus eliminated and
we may proceed as usual with the new variables in hand. Principal
Components Analysis is important for other reasons, too, not just
for fixing multicollinearity problems.
\item [{Ridge~Regression.}] The idea of this approach is to replace the
original parameter estimates with a different type of parameter estimate
which is more stable under multicollinearity. The estimators are not
found by ordinary least squares but rather a different optimization
procedure which incorporates the variance inflation factor information.
\end{description}
We decided to omit a thorough discussion of multicollinearity because
we are not equipped to handle the mathematical details. Perhaps the
topic will receive more attention in a later edition.
\begin{itemize}
\item What to do when data are not normal

\begin{itemize}
\item Bootstrap (see Chapter \ref{cha:resampling-methods}).
\end{itemize}
\end{itemize}

\subsection{Akaike's Information Criterion}

\[
AIC=-2\ln L+2(p+1)\]


\newpage{}


\section*{Chapter Exercises}

\addcontentsline{toc}{section}{Chapter Exercises}
\setcounter{thm}{0}
\begin{xca}
\label{xca:anova-equality}Use Equations \ref{eq:mlr-sse-matrix},
\ref{eq:mlr-ssto-matrix}, and \ref{eq:mlr-ssr-matrix} to prove the
Anova Equality:\[
SSTO=SSE+SSR.\]

\end{xca}

\chapter{Resampling Methods\label{cha:resampling-methods}}

\noindent Computers have changed the face of statistics. Their quick
computational speed and flawless accuracy, coupled with large data
sets acquired by the researcher, make them indispensable for many
modern analyses. In particular, resampling methods (due in large part
to Bradley Efron) have gained prominence in the modern statistician's
repertoire. We first look at a classical problem to get some insight
why. 

I have seen \emph{Statistical Computing with }\textsf{\emph{R}} by
Rizzo \cite{Rizzo2008} and I recommend it to those looking for a
more advanced treatment with additional topics. I believe that \emph{Monte
Carlo Statistical Methods} by Robert and Casella \cite{Robert2004}
has a new edition that integrates \textsf{R} into the narrative.


\paragraph*{What do I want them to know?}
\begin{itemize}
\item basic philosophy of resampling and why it is important
\item resampling for standard errors and confidence intervals
\item resampling for hypothesis tests (permutation tests)
\end{itemize}

\section{Introduction\label{sec:Introduction-Resampling}}
\begin{description}
\item [{Classical~question}] Given a population of interest, how may we
effectively learn some of its salient features, \emph{e.g.}, the population's
mean? One way is through representative random sampling. Given a random
sample, we summarize the information contained therein by calculating
a reasonable statistic, \emph{e.g.}, the sample mean. Given a value
of a statistic, how do we know whether that value is significantly
different from that which was expected? We don't; we look at the \emph{sampling
distribution} of the statistic, and we try to make probabilistic assertions
based on a confidence level or other consideration. For example, we
may find ourselves saying things like, \textquotedbl{}With 95\% confidence,
the true population mean is greater than zero.\textquotedbl{}
\item [{Problem}] Unfortunately, in most cases the sampling distribution
is \emph{unknown}. Thus, in the past, in efforts to say something
useful, statisticians have been obligated to place some restrictive
assumptions on the underlying population. For example, if we suppose
that the population has a normal distribution, then we can say that
the distribution of $\Xbar$ is normal, too, with the same mean (and
a smaller standard deviation). It is then easy to draw conclusions,
make inferences, and go on about our business.
\item [{Alternative}] We don't know what the underlying population distributions
is, so let us \emph{estimate} it, just like we would with any other
parameter. The statistic we use is the \emph{empirical CDF}, that
is, the function that places mass $1/n$ at each of the observed data
points $x_{1},\ldots,x_{n}$ (see Section \ref{sec:empirical-distribution}).
As the sample size increases, we would expect the approximation to
get better and better (with i.i.d.~observations, it does, and there
is a wonderful theorem by Glivenko and Cantelli that proves it). And
now that we have an (estimated) population distribution, it is easy
to find the sampling distribution of any statistic we like: just \textbf{sample}
from the empirical CDF many, many times, calculate the statistic each
time, and make a histogram. Done! Of course, the number of samples
needed to get a representative histogram is prohibitively large\ldots{}human
beings are simply too slow (and clumsy) to do this tedious procedure.\\
Fortunately, computers are very skilled at doing simple, repetitive
tasks very quickly and accurately. So we employ them to give us a
reasonable idea about the sampling distribution of our statistic,
and we use the generated sampling distribution to guide our inferences
and draw our conclusions. If we would like to have a better approximation
for the sampling distribution (within the confines of the information
contained in the original sample), we merely tell the computer to
sample more. In this (restricted) sense, we are limited only by our
current computational speed and pocket book.
\end{description}
In short, here are some of the benefits that the advent of resampling
methods has given us:
\begin{description}
\item [{Fewer~assumptions.}] We are no longer required to assume the population
is normal or the sample size is large (though, as before, the larger
the sample the better).
\item [{Greater~accuracy.}] Many classical methods are based on rough
upper bounds or Taylor expansions. The bootstrap procedures can be
iterated long enough to give results accurate to several decimal places,
often beating classical approximations. 
\item [{Generality.}] Resampling methods are easy to understand and apply
to a large class of seemingly unrelated procedures. One no longer
needs to memorize long complicated formulas and algorithms.\end{description}
\begin{rem}
Due to the special structure of the empirical CDF, to get an i.i.d.~sample
we just need to take a random sample of size $n$, with replacement,
from the observed data $x_{1},\ldots,x_{n}$. Repeats are expected
and acceptable. Since we already sampled to get the original data,
the term \emph{resampling} is used to describe the procedure.
\end{rem}

\paragraph*{General bootstrap procedure.}

The above discussion leads us to the following general procedure to
approximate the sampling distribution of a statistic $S=S(x_{1},x_{2},\ldots,x_{n})$
based on an observed simple random sample $\mathbf{x}=(x_{1},x_{2},\ldots,x_{n})$
of size $n$:
\begin{enumerate}
\item Create many many samples $\mathbf{x}_{1}^{\ast},\ldots,\mathbf{x}_{M}^{\ast}$,
called \emph{resamples}, by sampling with replacement from the data. 
\item Calculate the statistic of interest $S(\mathbf{x}_{1}^{\ast}),\ldots,S(\mathbf{x}_{M}^{\ast})$
for each resample. The distribution of the resample statistics is
called a \emph{bootstrap distribution}. 
\item The bootstrap distribution gives information about the sampling distribution
of the original statistic $S$. In particular, the bootstrap distribution
gives us some idea about the center, spread, and shape of the sampling
distribution of $S$.
\end{enumerate}

\section{Bootstrap Standard Errors\label{sec:Bootstrap-Standard-Errors}}

Since the bootstrap distribution gives us information about a statistic's
sampling distribution, we can use the bootstrap distribution to estimate
properties of the statistic. We will illustrate the bootstrap procedure
in the special case that the statistic $S$ is a standard error.
\begin{example}
\textbf{Standard error of the mean.\label{exa:Bootstrap-se-mean}}
In this example we illustrate the bootstrap by estimating the standard
error of the sample meanand we will do it in the special case that
the underlying population is $\mathsf{norm}(\mathtt{mean}=3,\,\mathtt{sd}=1)$. 

Of course, we do not really need a bootstrap distribution here because
from Section \ref{sec:sampling-from-normal-dist} we know that $\Xbar\sim\mathsf{norm}(\mathtt{mean}=3,\,\mathtt{sd}=1/\sqrt{n})$,
but we proceed anyway to investigate how the bootstrap performs when
we know what the answer should be ahead of time.

We will take a random sample of size $n=25$ from the population.
Then we will \emph{resample} the data 1000 times to get 1000 resamples
of size 25. We will calculate the sample mean of each of the resamples,
and will study the data distribution of the 1000 values of $\xbar$.

<<>>=
srs <- rnorm(25, mean = 3)
resamps <- replicate(1000, sample(srs, 25, TRUE), simplify = FALSE)
xbarstar <- sapply(resamps, mean, simplify = TRUE)
@

%
\begin{figure}
\begin{centering}
<<echo = FALSE, fig=true, height = 4, width = 6>>=
hist(xbarstar, breaks = 40, prob = TRUE)
curve(dnorm(x, 3, 0.2), add = TRUE)
@
\par\end{centering}

\caption{Bootstrapping the standard error of the mean, simulated data\label{fig:Bootstrap-se-mean}}


{\small ~}{\small \par}

{\small The original data were 25 observations generated from a $\mathsf{norm}(\mathtt{mean}=3,\,\mathtt{sd}=1)$
distribution. We next resampled to get 1000 resamples, each of size
25, and calculated the sample mean for each resample. A histogram
of the 1000 values of $\xbar$ is shown above. Also shown (with a
solid line) is the true sampling distribution of $\Xbar$, which is
a $\mathsf{norm}(\mathtt{mean}=3,\,\mathtt{sd}=0.2)$ distribution.
Note that the histogram is centered at the sample mean of the original
data, while the true sampling distribution is centered at the true
value of $\mu=3$. The shape and spread of the histogram is similar
to the shape and spread of the true sampling distribution.}
\end{figure}
A histogram of the 1000 values of $\xbar$ is shown in Figure \ref{fig:Bootstrap-se-mean},
and was produced by the following code.

<<eval = FALSE, keep.source = TRUE>>=
hist(xbarstar, breaks = 40, prob = TRUE)
curve(dnorm(x, 3, 0.2), add = TRUE)  # overlay true normal density
@

We have overlain what we know to be the true sampling distribution
of $\Xbar$, namely, a $\mathsf{norm}(\mathtt{mean}=3,\,\mathtt{sd}=1/\sqrt{25})$
distribution. The histogram matches the true sampling distribution
pretty well with respect to shape and spread\ldots{}but notice how
the histogram is off-center a little bit. This is not a coincidence
-- in fact, it can be shown that the mean of the bootstrap distribution
is exactly the mean of the original sample, that is, the value of
the statistic that we originally observed. Let us calculate the mean
of the bootstrap distribution and compare it to the mean of the original
sample:

<<>>=
mean(xbarstar)
mean(srs)
mean(xbarstar) - mean(srs)
@

\end{example}
Notice how close the two values are. The difference between them is
an estimate of how biased the original statistic is, the so-called
\emph{bootstrap estimate of bias}. Since the estimate is so small
we would expect our original statistic ($\Xbar$) to have small bias,
but this is no surprise to us because we already knew from Section
\ref{sub:simple-random-samples} that $\Xbar$ is an unbiased estimator
of the population mean.

Now back to our original problem, we would like to estimate the standard
error of $\Xbar$. Looking at the histogram, we see that the spread
of the bootstrap distribution is similar to the spread of the sampling
distribution. Therefore, it stands to reason that we could estimate
the standard error of $\Xbar$ with the sample standard deviation
of the resample statistics. Let us try and see.

<<>>=
sd(xbarstar)
@

We know from theory that the true standard error is $1/\sqrt{25}=0.20$.
Our bootstrap estimate is not very far from the theoretical value.


\begin{rem}
What would happen if we take more resamples? Instead of 1000 resamples,
we could increase to, say, 2000, 3000, or even 4000\ldots{}would
it help? The answer is both yes and no. Keep in mind that with resampling
methods there are two sources of randomness: that from the original
sample, and that from the subsequent resampling procedure. An increased
number of resamples would reduce the variation due to the second part,
but would do nothing to reduce the variation due to the first part.
We only took an original sample of size $n=25$, and resampling more
and more would never generate more information about the population
than was already there. In this sense, the statistician is limited
by the information contained in the original sample. \end{rem}
\begin{example}
\textbf{Standard error of the median.\label{exa:Bootstrap-se-median}}
We look at one where we do not know the answer ahead of time. This
example uses the \inputencoding{latin9}\lstinline[showstringspaces=false]!rivers!\inputencoding{utf8}\index{Data sets!rivers@\texttt{rivers}}
data set. Recall the stemplot on page \vpageref{ite:stemplot-rivers}
that we made for these data which shows them to be markedly right-skewed,
so a natural estimate of center would be the sample median. Unfortunately,
its sampling distribution falls out of our reach. We use the bootstrap
to help us with this problem, and the modifications to the last example
are trivial.

<<>>=
resamps <- replicate(1000, sample(rivers, 141, TRUE), simplify = FALSE)
medstar <- sapply(resamps, median, simplify = TRUE)
sd(medstar)
@

%
\begin{figure}
\begin{centering}
<<echo = FALSE, fig=true, height = 4, width = 6>>=
hist(medstar, breaks = 40, prob = TRUE)
@
\par\end{centering}

\caption{Bootstrapping the standard error of the median for the \texttt{rivers}
data\label{fig:Bootstrapping-se-median}}

\end{figure}


The graph is shown in Figure \ref{fig:Bootstrapping-se-median}, and
was produced by the following code.

<<eval = FALSE, keep.source = TRUE>>=
hist(medstar, breaks = 40, prob = TRUE)
@

<<>>=
median(rivers)
mean(medstar)
mean(medstar) - median(rivers)
@

\end{example}

\begin{example}
\textbf{The boot package in }\texttt{\textbf{R}}. It turns out that
there are many bootstrap procedures and commands already built into
base \texttt{R}, in the \inputencoding{latin9}\lstinline[showstringspaces=false]!boot!\inputencoding{utf8}
package. Further, inside the \inputencoding{latin9}\lstinline[showstringspaces=false]!boot!\inputencoding{utf8}
package there is even a function called \inputencoding{latin9}\lstinline[showstringspaces=false]!boot!\inputencoding{utf8}\index{boot@\texttt{boot}}.
The basic syntax is of the form:\inputencoding{latin9}
\begin{lstlisting}[showstringspaces=false]
boot(data, statistic, R)
\end{lstlisting}
\inputencoding{utf8}
\end{example}
Here, \inputencoding{latin9}\lstinline[showstringspaces=false]!data!\inputencoding{utf8}
is a vector (or matrix) containing the data to be resampled, \inputencoding{latin9}\lstinline[showstringspaces=false]!statistic!\inputencoding{utf8}
is a defined function, \emph{of two arguments}, that tells which statistic
should be computed, and the parameter \inputencoding{latin9}\lstinline[showstringspaces=false]!R!\inputencoding{utf8}
specifies how many resamples should be taken.

For the standard error of the mean (Example \ref{exa:Bootstrap-se-mean}):

<<>>=
library(boot)
mean_fun <- function(x, indices) mean(x[indices])
boot(data = srs, statistic = mean_fun, R = 1000)
@

For the standard error of the median (Example \ref{exa:Bootstrap-se-median}):

<<>>=
median_fun <- function(x, indices) median(x[indices])
boot(data = rivers, statistic = median_fun, R = 1000)
@

We notice that the output from both methods of estimating the standard
errors produced similar results. In fact, the \inputencoding{latin9}\lstinline[showstringspaces=false]!boot!\inputencoding{utf8}
procedure is to be preferred since it invisibly returns much more
information (which we will use later) than our naive script and it
is much quicker in its computations.
\begin{rem}
Some things to keep in mind about the bootstrap:
\begin{itemize}
\item For many statistics, the bootstrap distribution closely resembles
the sampling distribution with respect to spread and shape. However,
the bootstrap will not have the same center as the true sampling distribution.
While the sampling distribution is centered at the population mean
(plus any bias), the bootstrap distribution is centered at the original
value of the statistic (plus any bias). The \inputencoding{latin9}\lstinline[showstringspaces=false]!boot!\inputencoding{utf8}
function gives an empirical estimate of the bias of the statistic
as part of its output. 
\item We tried to estimate the standard error, but we could have (in principle)
tried to estimate something else. Note from the previous remark, however,
that it would be useless to estimate the population mean $\mu$ using
the bootstrap since the mean of the bootstrap distribution is the
observed $\xbar$. 
\item You don't get something from nothing. We have seen that we can take
a random sample from a population and use bootstrap methods to get
a very good idea about standard errors, bias, and the like. However,
one must not get lured into believing that by doing some random resampling
somehow one gets more information about the parameters than that which
was contained in the original sample. Indeed, there is some uncertainty
about the parameter due to the randomness of the original sample,
and there is even more uncertainty introduced by resampling. One should
think of the bootstrap as just another estimation method, nothing
more, nothing less.
\end{itemize}
\end{rem}

\section{Bootstrap Confidence Intervals\label{sec:Bootstrap-Confidence-Intervals}}


\subsection{Percentile Confidence Intervals}

As a first try, we want to obtain a 95\% confidence interval for a
parameter. Typically the statistic we use to estimate the parameter
is centered at (or at least close by) the parameter; in such cases
a 95\% confidence interval for the parameter is nothing more than
a 95\% confidence interval for the statistic. And to find a 95\% confidence
interval for the statistic we need only go to its sampling distribution
to find an interval that contains 95\% of the area. (The most popular
choice is the equal-tailed interval with 2.5\% in each tail.)

This is incredibly easy to accomplish with the bootstrap. We need
only to take a bunch of bootstrap resamples, order them, and choose
the $\alpha/2$th and $(1-\alpha)$th percentiles. There is a function
\inputencoding{latin9}\lstinline[showstringspaces=false]!boot.ci!\inputencoding{utf8}\index{boot.ci@\texttt{boot.ci}}
in \textsf{R} already created to do just this. Note that in order
to use the function \inputencoding{latin9}\lstinline[showstringspaces=false]!boot.ci!\inputencoding{utf8}
we must first run the \inputencoding{latin9}\lstinline[showstringspaces=false]!boot!\inputencoding{utf8}
function and save the output in a variable, for example, \inputencoding{latin9}\lstinline[showstringspaces=false]!data.boot!\inputencoding{utf8}.
We then plug \inputencoding{latin9}\lstinline[showstringspaces=false]!data.boot!\inputencoding{utf8}
into the function \inputencoding{latin9}\lstinline[showstringspaces=false]!boot.ci!\inputencoding{utf8}.
\begin{example}
\label{exa:percentile-interval-median-first}\textbf{Percentile interval
for the expected value of the median.} Wee will try the naive approach
where we generate the resamples and calculate the percentile interval
by hand.

<<>>=
btsamps <- replicate(2000, sample(stack.loss, 21, TRUE), simplify = FALSE)
thetast <- sapply(btsamps, median, simplify = TRUE)
mean(thetast)
median(stack.loss)
quantile(thetast, c(0.025, 0.975))
@

\end{example}

\begin{example}
\textbf{Confidence interval for expected value of the median, $2^{\mathrm{nd}}$
try.} Now we will do it the right way with the \inputencoding{latin9}\lstinline[showstringspaces=false]!boot!\inputencoding{utf8}
function.

<<>>=
library(boot)
med_fun <- function(x, ind) median(x[ind])
med_boot <- boot(stack.loss, med_fun, R = 2000)
boot.ci(med_boot, type = c("perc", "norm", "bca"))
@

\end{example}

\subsection{Student's $t$ intervals ({}``normal intervals'')}

The idea is to use confidence intervals that we already know and let
the bootstrap help us when we get into trouble. We know that a $100(1-\alpha)\%$
confidence interval for the mean of a $SRS(n)$ from a normal distribution
is \begin{equation}
\Xbar\pm\mathsf{t}_{\alpha/2}(\mathtt{df}=n-1)\frac{S}{\sqrt{n}},\end{equation}
where $\mathsf{t}_{\alpha/2}(\mathtt{df}=n-1)$ is the appropriate
critical value from Student's $t$ distribution, and we remember that
an estimate for the standard error of $\Xbar$ is $S/\sqrt{n}$. Of
course, the estimate for the standard error will change when the underlying
population distribution is not normal, or when we use a statistic
more complicated than $\Xbar$. In those situations the bootstrap
will give us quite reasonable estimates for the standard error. And
as long as the sampling distribution of our statistic is approximately
bell-shaped with small bias, the interval \begin{equation}
\mbox{statistic}\pm\mathsf{t}_{\alpha/2}(\mathtt{df}=n-1)*\mathrm{SE}(\mbox{statistic})\end{equation}
 will have approximately $100(1-\alpha)\%$ confidence of containing
$\E(\mathrm{statistic})$.
\begin{example}
We will use the t-interval method to find the bootstrap CI for the
median. We have looked at the bootstrap distribution; it appears to
be symmetric and approximately mound shaped. Further, we may check
that the bias is approximately 40, which on the scale of these data
is practically negligible. Thus, we may consider looking at the $t$-intervals.
Note that, since our sample is so large, instead of $t$-intervals
we will essentially be using $z$-intervals.
\end{example}
Please see the handout, {}``Bootstrapping Confidence Intervals for
the Median, $3^{\mathrm{rd}}$ try.''

We see that, considering the scale of the data, the confidence intervals
compare with each other quite well.
\begin{rem}
We have seen two methods for bootstrapping confidence intervals for
a statistic. Which method should we use? If the bias of the bootstrap
distribution is small and if the distribution is close to normal,
then the percentile and $t$-intervals will closely agree. If the
intervals are noticeably different, then it should be considered evidence
that the normality and bias conditions are not met. In this case,
\emph{neither} interval should be used.\end{rem}
\begin{itemize}
\item $BC_{a}$: bias-corrected and accelerated

\begin{itemize}
\item transformation invariant
\item more correct and accurate
\item not monotone in coverage level?
\end{itemize}
\item t - intervals

\begin{itemize}
\item more natural
\item numerically unstable
\end{itemize}
\item Can do things like transform scales, compute confidence intervals,
and then transform back.
\item Studentized bootstrap confidence intervals where is the Studentized
version of is the  order statistic of the simulation
\end{itemize}

\section{Resampling in Hypothesis Tests\label{sec:Resampling-in-Hypothesis}}

The classical two-sample problem can be stated as follows: given two
groups of interest, we would like to know whether these two groups
are significantly different from one another or whether the groups
are reasonably similar. The standard way to decide is to
\begin{enumerate}
\item Go collect some information from the two groups and calculate an associated
statistic, for example, $\Xbar_{1}-\Xbar_{2}$. 
\item Suppose that there is no difference in the groups, and find the distribution
of the statistic in 1. 
\item Locate the observed value of the statistic with respect to the distribution
found in 2. A value in the main body of the distribution is not spectacular,
it could reasonably have occurred by chance. A value in the tail of
the distribution is unlikely, and hence provides evidence \emph{against}
the null hypothesis that the population distributions are the same. 
\end{enumerate}
Of course, we usually compute a $p$-value, defined to be the probability
of the observed value of the statistic or more extreme when the null
hypothesis is true. Small $p$-values are evidence against the null
hypothesis. It is not immediately obvious how to use resampling methods
here, so we discuss an example.
\begin{example}
A study concerned differing dosages of the antiretroviral drug AZT.
The common dosage is 300mg daily. Higher doses cause more side affects,
but are they significantly higher? We examine for a 600mg dose. The
data are as follows: We compare the scores from the two groups by
computing the difference in their sample means. The 300mg data were
entered in x1 and the 600mg data were entered into x2. The observed
difference was

\begin{center}
\begin{tabular}{l|cccccccccc}
300 mg  & 284  & 279  & 289  & 292  & 287  & 295  & 285  & 279  & 306  & 298 \tabularnewline
600 mg  & 298  & 307  & 297  & 279  & 291  & 335  & 299  & 300  & 306  & 291 \tabularnewline
\end{tabular}
\par\end{center}

The average amounts can be found:

\texttt{> mean(x1)}

\texttt{{[}1{]} 289.4}

\texttt{> mean(x2)}

\texttt{{[}1{]} 300.3}

with an observed difference of \texttt{mean(x2) - mean(x1) = 10.9.}
As expected, the 600 mg measurements seem to have a higher average,
and we might be interested in trying to decide if the average amounts
are \emph{significantly} different. The null hypothesis should be
that there is no difference in the amounts, that is, the groups are
more or less the same. If the null hypothesis were true, then the
two groups would indeed be the same, or just one big group. In that
case, the observed difference in the sample means just reflects the
random assignment into the arbitrary \texttt{x1} and \texttt{x2} categories.
It is now clear how we may resample, consistent with the null hypothesis.


\paragraph*{Procedure:}
\begin{enumerate}
\item Randomly resample 10 scores from the combined scores of \texttt{x1}
and \texttt{x2}, and assign then to the {}``\texttt{x1}'' group.
The rest will then be in the {}``\texttt{x2}'' group. Calculate
the difference in (re)sampled means, and store that value. 
\item Repeat this procedure many, many times and draw a histogram of the
resampled statistics, called the \emph{permutation distribution}.
Locate the observed difference 10.9 on the histogram to get the $p$-value.
If the $p$-value is small, then we consider that evidence against
the hypothesis that the groups are the same. 
\end{enumerate}
\end{example}
\begin{rem}
In calculating the permutation test $p$-value, the formula is essentially
the proportion of resample statistics that are greater than or equal
to the observed value. Of course, this is merely an \emph{estimate}
of the true $p$-value. As it turns out, an adjustment of $+1$ to
both the numerator and denominator of the proportion improves the
performance of the estimated $p$-value, and this adjustment is implemented
in the \inputencoding{latin9}\lstinline[showstringspaces=false]!ts.perm!\inputencoding{utf8}
function.
\end{rem}
<<>>=
library(coin)
oneway_test(len ~ supp, data = ToothGrowth)
@


\subsection{Comparison with the Two Sample \emph{t} test}

We know from Chapter \ref{cha:Hypothesis-Testing} to use the two-sample
$t$-test to tell whether there is an improvement as a result of taking
the intervention class. Note that the $t$-test assumes normal underlying
populations, with unknown variance, and small sample $n=10$. What
does the $t$-test say? Below is the output.

<<>>=
t.test(len ~ supp, data = ToothGrowth, alt = "greater", var.equal = TRUE)
@

<<echo = FALSE, results = hide>>=
A <- show(oneway_test(len ~ supp, data = ToothGrowth))
B <- t.test(len ~ supp, data = ToothGrowth, alt = "greater", var.equal = TRUE)
@

The $p$-value for the $t$-test was \Sexpr{round(B$p.value, 3)},
while the permutation test $p$-value was \Sexpr{round(A$p.value, 3)}.
Note that there is an underlying normality assumption for the $t$-test,
which isn't present in the permutation test. If the normality assumption
may be questionable, then the permutation test would be more reasonable.
We see what can happen when using a test in a situation where the
assumptions are not met: smaller $p$-values. In situations where
the normality assumptions are not met, for example, small sample scenarios,
the permutation test is to be preferred. In particular, if accuracy
is very important then we should use the permutation test.
\begin{rem}
Here are some things about permutation tests to keep in mind.
\begin{itemize}
\item While the permutation test does not require normality of the populations
(as contrasted with the $t$-test), nevertheless it still requires
that the two groups are exchangeable; see Section \ref{sec:Exchangeable-Random-Variables}.
In particular, this means that they must be identically distributed
under the null hypothesis. They must have not only the same means,
but they must also have the same spread, shape, and everything else.
This assumption may or may not be true in a given example, but it
will rarely cause the $t$-test to outperform the permutation test,
because even if the sample standard deviations are markedly different
it does not mean that the population standard deviations are different.
In many situations the permutation test will also carry over to the
$t$-test. 
\item If the distribution of the groups is close to normal, then the $t$-test
$p$-value and the bootstrap $p$-value will be approximately equal.
If they differ markedly, then this should be considered evidence that
the normality assumptions do not hold. 
\item The generality of the permutation test is such that one can use all
kinds of statistics to compare the two groups. One could compare the
difference in variances or the difference in (just about anything).
Alternatively, one could compare the ratio of sample means, $\Xbar_{1}/\Xbar_{2}$.
Of course, under the null hypothesis this last quantity should be
near 1. 
\item Just as with the bootstrap, the answer we get is subject to variability
due to the inherent randomness of resampling from the data. We can
make the variability as small as we like by taking sufficiently many
resamples. How many? If the conclusion is very important (that is,
if lots of money is at stake), then take thousands. For point estimation
problems typically, $R=1000$ resamples, or so, is enough. In general,
if the true $p$-value is $p$ then the standard error of the estimated
$p$-value is $\sqrt{p(1-p)/R}$. You can choose $R$ to get whatever
accuracy desired. 
\end{itemize}
\end{rem}
\begin{itemize}
\item Other possible testing designs:

\begin{itemize}
\item Matched Pairs Designs. 
\item Relationship between two variables. 
\end{itemize}
\end{itemize}
\newpage{}


\section*{Chapter Exercises}

\addcontentsline{toc}{section}{Chapter Exercises}
\setcounter{thm}{0}


\chapter{Categorical Data Analysis\label{cha:Categorical-Data-Analysis}}

This chapter is still under substantial revision. At any time you
can preview any released drafts with the development version of the
\inputencoding{latin9}\lstinline[showstringspaces=false]!IPSUR!\inputencoding{utf8}
package which is available from \textsf{R}-Forge:

<<eval = FALSE>>=
install.packages("IPSUR", repos="http://R-Forge.R-project.org")
library(IPSUR)
read(IPSUR)
@


\chapter{Nonparametric Statistics\label{cha:Nonparametric-Statistics}}

This chapter is still under substantial revision. At any time you
can preview any released drafts with the development version of the
\inputencoding{latin9}\lstinline[showstringspaces=false]!IPSUR!\inputencoding{utf8}
package which is available from \textsf{R}-Forge:

<<eval = FALSE>>=
install.packages("IPSUR", repos="http://R-Forge.R-project.org")
library(IPSUR)
read(IPSUR)
@


\chapter{Time Series\label{cha:Time-Series}}

This chapter is still under substantial revision. At any time you
can preview any released drafts with the development version of the
\inputencoding{latin9}\lstinline[showstringspaces=false]!IPSUR!\inputencoding{utf8}
package which is available from \textsf{R}-Forge:

<<eval = FALSE>>=
install.packages("IPSUR", repos="http://R-Forge.R-project.org")
library(IPSUR)
read(IPSUR)
@

\appendix

\chapter{\textsf{R} Session Information\label{cha:R-Session-Information}}

If you ever write the \textsf{R} help mailing list with a question,
then you should include your session information in the email; it
makes the reader's job easier and is requested by the Posting Guide.
Here is how to do that, and below is what the output looks like. 

<<keep.source = TRUE>>=
sessionInfo()
@

\vfill{}



\chapter{GNU Free Documentation License\label{cha:GNU-Free-Documentation}}

\begin{center}
\textbf{\large Version 1.3, 3 November 2008}\bigskip{}

\par\end{center}

\noindent Copyright (C) 2000, 2001, 2002, 2007, 2008 Free Software
Foundation, Inc.

\begin{center}
\url{http://fsf.org/}
\par\end{center}

\noindent Everyone is permitted to copy and distribute verbatim copies
of this license document, but changing it is not allowed.


\section*{0. PREAMBLE}

The purpose of this License is to make a manual, textbook, or other
functional and useful document \textquotedbl{}free\textquotedbl{}
in the sense of freedom: to assure everyone the effective freedom
to copy and redistribute it, with or without modifying it, either
commercially or noncommercially. Secondarily, this License preserves
for the author and publisher a way to get credit for their work, while
not being considered responsible for modifications made by others.

This License is a kind of \textquotedbl{}copyleft\textquotedbl{},
which means that derivative works of the document must themselves
be free in the same sense. It complements the GNU General Public License,
which is a copyleft license designed for free software.

We have designed this License in order to use it for manuals for free
software, because free software needs free documentation: a free program
should come with manuals providing the same freedoms that the software
does. But this License is not limited to software manuals; it can
be used for any textual work, regardless of subject matter or whether
it is published as a printed book. We recommend this License principally
for works whose purpose is instruction or reference.


\section*{1. APPLICABILITY AND DEFINITIONS}

This License applies to any manual or other work, in any medium, that
contains a notice placed by the copyright holder saying it can be
distributed under the terms of this License. Such a notice grants
a world-wide, royalty-free license, unlimited in duration, to use
that work under the conditions stated herein. The \textquotedbl{}Document\textquotedbl{},
below, refers to any such manual or work. Any member of the public
is a licensee, and is addressed as \textquotedbl{}you\textquotedbl{}.
You accept the license if you copy, modify or distribute the work
in a way requiring permission under copyright law.

A \textquotedbl{}Modified Version\textquotedbl{} of the Document means
any work containing the Document or a portion of it, either copied
verbatim, or with modifications and/or translated into another language.

A \textquotedbl{}Secondary Section\textquotedbl{} is a named appendix
or a front-matter section of the Document that deals exclusively with
the relationship of the publishers or authors of the Document to the
Document's overall subject (or to related matters) and contains nothing
that could fall directly within that overall subject. (Thus, if the
Document is in part a textbook of mathematics, a Secondary Section
may not explain any mathematics.) The relationship could be a matter
of historical connection with the subject or with related matters,
or of legal, commercial, philosophical, ethical or political position
regarding them.

The \textquotedbl{}Invariant Sections\textquotedbl{} are certain Secondary
Sections whose titles are designated, as being those of Invariant
Sections, in the notice that says that the Document is released under
this License. If a section does not fit the above definition of Secondary
then it is not allowed to be designated as Invariant. The Document
may contain zero Invariant Sections. If the Document does not identify
any Invariant Sections then there are none.

The \textquotedbl{}Cover Texts\textquotedbl{} are certain short passages
of text that are listed, as Front-Cover Texts or Back-Cover Texts,
in the notice that says that the Document is released under this License.
A Front-Cover Text may be at most 5 words, and a Back-Cover Text may
be at most 25 words.

A \textquotedbl{}Transparent\textquotedbl{} copy of the Document means
a machine-readable copy, represented in a format whose specification
is available to the general public, that is suitable for revising
the document straightforwardly with generic text editors or (for images
composed of pixels) generic paint programs or (for drawings) some
widely available drawing editor, and that is suitable for input to
text formatters or for automatic translation to a variety of formats
suitable for input to text formatters. A copy made in an otherwise
Transparent file format whose markup, or absence of markup, has been
arranged to thwart or discourage subsequent modification by readers
is not Transparent. An image format is not Transparent if used for
any substantial amount of text. A copy that is not \textquotedbl{}Transparent\textquotedbl{}
is called \textquotedbl{}Opaque\textquotedbl{}.

Examples of suitable formats for Transparent copies include plain
ASCII without markup, Texinfo input format, \LaTeX{} input format,
SGML or XML using a publicly available DTD, and standard-conforming
simple HTML, PostScript or PDF designed for human modification. Examples
of transparent image formats include PNG, XCF and JPG. Opaque formats
include proprietary formats that can be read and edited only by proprietary
word processors, SGML or XML for which the DTD and/or processing tools
are not generally available, and the machine-generated HTML, PostScript
or PDF produced by some word processors for output purposes only.

The \textquotedbl{}Title Page\textquotedbl{} means, for a printed
book, the title page itself, plus such following pages as are needed
to hold, legibly, the material this License requires to appear in
the title page. For works in formats which do not have any title page
as such, \textquotedbl{}Title Page\textquotedbl{} means the text near
the most prominent appearance of the work's title, preceding the beginning
of the body of the text.

The \textquotedbl{}publisher\textquotedbl{} means any person or entity
that distributes copies of the Document to the public.

A section \textquotedbl{}Entitled XYZ\textquotedbl{} means a named
subunit of the Document whose title either is precisely XYZ or contains
XYZ in parentheses following text that translates XYZ in another language.
(Here XYZ stands for a specific section name mentioned below, such
as \textquotedbl{}Acknowledgements\textquotedbl{}, \textquotedbl{}Dedications\textquotedbl{},
\textquotedbl{}Endorsements\textquotedbl{}, or \textquotedbl{}History\textquotedbl{}.)
To \textquotedbl{}Preserve the Title\textquotedbl{} of such a section
when you modify the Document means that it remains a section \textquotedbl{}Entitled
XYZ\textquotedbl{} according to this definition.

The Document may include Warranty Disclaimers next to the notice which
states that this License applies to the Document. These Warranty Disclaimers
are considered to be included by reference in this License, but only
as regards disclaiming warranties: any other implication that these
Warranty Disclaimers may have is void and has no effect on the meaning
of this License.


\section*{2. VERBATIM COPYING}

You may copy and distribute the Document in any medium, either commercially
or noncommercially, provided that this License, the copyright notices,
and the license notice saying this License applies to the Document
are reproduced in all copies, and that you add no other conditions
whatsoever to those of this License. You may not use technical measures
to obstruct or control the reading or further copying of the copies
you make or distribute. However, you may accept compensation in exchange
for copies. If you distribute a large enough number of copies you
must also follow the conditions in section 3.

You may also lend copies, under the same conditions stated above,
and you may publicly display copies.


\section*{3. COPYING IN QUANTITY}

If you publish printed copies (or copies in media that commonly have
printed covers) of the Document, numbering more than 100, and the
Document's license notice requires Cover Texts, you must enclose the
copies in covers that carry, clearly and legibly, all these Cover
Texts: Front-Cover Texts on the front cover, and Back-Cover Texts
on the back cover. Both covers must also clearly and legibly identify
you as the publisher of these copies. The front cover must present
the full title with all words of the title equally prominent and visible.
You may add other material on the covers in addition. Copying with
changes limited to the covers, as long as they preserve the title
of the Document and satisfy these conditions, can be treated as verbatim
copying in other respects.

If the required texts for either cover are too voluminous to fit legibly,
you should put the first ones listed (as many as fit reasonably) on
the actual cover, and continue the rest onto adjacent pages.

If you publish or distribute Opaque copies of the Document numbering
more than 100, you must either include a machine-readable Transparent
copy along with each Opaque copy, or state in or with each Opaque
copy a computer-network location from which the general network-using
public has access to download using public-standard network protocols
a complete Transparent copy of the Document, free of added material.
If you use the latter option, you must take reasonably prudent steps,
when you begin distribution of Opaque copies in quantity, to ensure
that this Transparent copy will remain thus accessible at the stated
location until at least one year after the last time you distribute
an Opaque copy (directly or through your agents or retailers) of that
edition to the public.

It is requested, but not required, that you contact the authors of
the Document well before redistributing any large number of copies,
to give them a chance to provide you with an updated version of the
Document.


\section*{4. MODIFICATIONS}

You may copy and distribute a Modified Version of the Document under
the conditions of sections 2 and 3 above, provided that you release
the Modified Version under precisely this License, with the Modified
Version filling the role of the Document, thus licensing distribution
and modification of the Modified Version to whoever possesses a copy
of it. In addition, you must do these things in the Modified Version:

A. Use in the Title Page (and on the covers, if any) a title distinct
from that of the Document, and from those of previous versions (which
should, if there were any, be listed in the History section of the
Document). You may use the same title as a previous version if the
original publisher of that version gives permission. 

B. List on the Title Page, as authors, one or more persons or entities
responsible for authorship of the modifications in the Modified Version,
together with at least five of the principal authors of the Document
(all of its principal authors, if it has fewer than five), unless
they release you from this requirement. 

C. State on the Title page the name of the publisher of the Modified
Version, as the publisher. 

D. Preserve all the copyright notices of the Document. 

E. Add an appropriate copyright notice for your modifications adjacent
to the other copyright notices. 

F. Include, immediately after the copyright notices, a license notice
giving the public permission to use the Modified Version under the
terms of this License, in the form shown in the Addendum below. 

G. Preserve in that license notice the full lists of Invariant Sections
and required Cover Texts given in the Document's license notice. 

H. Include an unaltered copy of this License. 

I. Preserve the section Entitled \textquotedbl{}History\textquotedbl{},
Preserve its Title, and add to it an item stating at least the title,
year, new authors, and publisher of the Modified Version as given
on the Title Page. If there is no section Entitled \textquotedbl{}History\textquotedbl{}
in the Document, create one stating the title, year, authors, and
publisher of the Document as given on its Title Page, then add an
item describing the Modified Version as stated in the previous sentence. 

J. Preserve the network location, if any, given in the Document for
public access to a Transparent copy of the Document, and likewise
the network locations given in the Document for previous versions
it was based on. These may be placed in the \textquotedbl{}History\textquotedbl{}
section. You may omit a network location for a work that was published
at least four years before the Document itself, or if the original
publisher of the version it refers to gives permission. 

K. For any section Entitled \textquotedbl{}Acknowledgements\textquotedbl{}
or \textquotedbl{}Dedications\textquotedbl{}, Preserve the Title of
the section, and preserve in the section all the substance and tone
of each of the contributor acknowledgements and/or dedications given
therein. 

L. Preserve all the Invariant Sections of the Document, unaltered
in their text and in their titles. Section numbers or the equivalent
are not considered part of the section titles. 

M. Delete any section Entitled \textquotedbl{}Endorsements\textquotedbl{}.
Such a section may not be included in the Modified Version. 

N. Do not retitle any existing section to be Entitled \textquotedbl{}Endorsements\textquotedbl{}
or to conflict in title with any Invariant Section. 

O. Preserve any Warranty Disclaimers.

If the Modified Version includes new front-matter sections or appendices
that qualify as Secondary Sections and contain no material copied
from the Document, you may at your option designate some or all of
these sections as invariant. To do this, add their titles to the list
of Invariant Sections in the Modified Version's license notice. These
titles must be distinct from any other section titles.

You may add a section Entitled \textquotedbl{}Endorsements\textquotedbl{},
provided it contains nothing but endorsements of your Modified Version
by various parties--for example, statements of peer review or that
the text has been approved by an organization as the authoritative
definition of a standard.

You may add a passage of up to five words as a Front-Cover Text, and
a passage of up to 25 words as a Back-Cover Text, to the end of the
list of Cover Texts in the Modified Version. Only one passage of Front-Cover
Text and one of Back-Cover Text may be added by (or through arrangements
made by) any one entity. If the Document already includes a cover
text for the same cover, previously added by you or by arrangement
made by the same entity you are acting on behalf of, you may not add
another; but you may replace the old one, on explicit permission from
the previous publisher that added the old one.

The author(s) and publisher(s) of the Document do not by this License
give permission to use their names for publicity for or to assert
or imply endorsement of any Modified Version.


\section*{5. COMBINING DOCUMENTS}

You may combine the Document with other documents released under this
License, under the terms defined in section 4 above for modified versions,
provided that you include in the combination all of the Invariant
Sections of all of the original documents, unmodified, and list them
all as Invariant Sections of your combined work in its license notice,
and that you preserve all their Warranty Disclaimers.

The combined work need only contain one copy of this License, and
multiple identical Invariant Sections may be replaced with a single
copy. If there are multiple Invariant Sections with the same name
but different contents, make the title of each such section unique
by adding at the end of it, in parentheses, the name of the original
author or publisher of that section if known, or else a unique number.
Make the same adjustment to the section titles in the list of Invariant
Sections in the license notice of the combined work.

In the combination, you must combine any sections Entitled \textquotedbl{}History\textquotedbl{}
in the various original documents, forming one section Entitled \textquotedbl{}History\textquotedbl{};
likewise combine any sections Entitled \textquotedbl{}Acknowledgements\textquotedbl{},
and any sections Entitled \textquotedbl{}Dedications\textquotedbl{}.
You must delete all sections Entitled \textquotedbl{}Endorsements\textquotedbl{}.


\section*{6. COLLECTIONS OF DOCUMENTS}

You may make a collection consisting of the Document and other documents
released under this License, and replace the individual copies of
this License in the various documents with a single copy that is included
in the collection, provided that you follow the rules of this License
for verbatim copying of each of the documents in all other respects.

You may extract a single document from such a collection, and distribute
it individually under this License, provided you insert a copy of
this License into the extracted document, and follow this License
in all other respects regarding verbatim copying of that document.


\section*{7. AGGREGATION WITH INDEPENDENT WORKS}

A compilation of the Document or its derivatives with other separate
and independent documents or works, in or on a volume of a storage
or distribution medium, is called an \textquotedbl{}aggregate\textquotedbl{}
if the copyright resulting from the compilation is not used to limit
the legal rights of the compilation's users beyond what the individual
works permit. When the Document is included in an aggregate, this
License does not apply to the other works in the aggregate which are
not themselves derivative works of the Document.

If the Cover Text requirement of section 3 is applicable to these
copies of the Document, then if the Document is less than one half
of the entire aggregate, the Document's Cover Texts may be placed
on covers that bracket the Document within the aggregate, or the electronic
equivalent of covers if the Document is in electronic form. Otherwise
they must appear on printed covers that bracket the whole aggregate.


\section*{8. TRANSLATION}

Translation is considered a kind of modification, so you may distribute
translations of the Document under the terms of section 4. Replacing
Invariant Sections with translations requires special permission from
their copyright holders, but you may include translations of some
or all Invariant Sections in addition to the original versions of
these Invariant Sections. You may include a translation of this License,
and all the license notices in the Document, and any Warranty Disclaimers,
provided that you also include the original English version of this
License and the original versions of those notices and disclaimers.
In case of a disagreement between the translation and the original
version of this License or a notice or disclaimer, the original version
will prevail.

If a section in the Document is Entitled \textquotedbl{}Acknowledgements\textquotedbl{},
\textquotedbl{}Dedications\textquotedbl{}, or \textquotedbl{}History\textquotedbl{},
the requirement (section 4) to Preserve its Title (section 1) will
typically require changing the actual title.


\section*{9. TERMINATION}

You may not copy, modify, sublicense, or distribute the Document except
as expressly provided under this License. Any attempt otherwise to
copy, modify, sublicense, or distribute it is void, and will automatically
terminate your rights under this License.

However, if you cease all violation of this License, then your license
from a particular copyright holder is reinstated (a) provisionally,
unless and until the copyright holder explicitly and finally terminates
your license, and (b) permanently, if the copyright holder fails to
notify you of the violation by some reasonable means prior to 60 days
after the cessation.

Moreover, your license from a particular copyright holder is reinstated
permanently if the copyright holder notifies you of the violation
by some reasonable means, this is the first time you have received
notice of violation of this License (for any work) from that copyright
holder, and you cure the violation prior to 30 days after your receipt
of the notice.

Termination of your rights under this section does not terminate the
licenses of parties who have received copies or rights from you under
this License. If your rights have been terminated and not permanently
reinstated, receipt of a copy of some or all of the same material
does not give you any rights to use it.


\section*{10. FUTURE REVISIONS OF THIS LICENSE}

The Free Software Foundation may publish new, revised versions of
the GNU Free Documentation License from time to time. Such new versions
will be similar in spirit to the present version, but may differ in
detail to address new problems or concerns. See http://www.gnu.org/copyleft/.

Each version of the License is given a distinguishing version number.
If the Document specifies that a particular numbered version of this
License \textquotedbl{}or any later version\textquotedbl{} applies
to it, you have the option of following the terms and conditions either
of that specified version or of any later version that has been published
(not as a draft) by the Free Software Foundation. If the Document
does not specify a version number of this License, you may choose
any version ever published (not as a draft) by the Free Software Foundation.
If the Document specifies that a proxy can decide which future versions
of this License can be used, that proxy's public statement of acceptance
of a version permanently authorizes you to choose that version for
the Document.


\section*{11. RELICENSING}

\textquotedbl{}Massive Multiauthor Collaboration Site\textquotedbl{}
(or \textquotedbl{}MMC Site\textquotedbl{}) means any World Wide Web
server that publishes copyrightable works and also provides prominent
facilities for anybody to edit those works. A public wiki that anybody
can edit is an example of such a server. A \textquotedbl{}Massive
Multiauthor Collaboration\textquotedbl{} (or \textquotedbl{}MMC\textquotedbl{})
contained in the site means any set of copyrightable works thus published
on the MMC site.

\textquotedbl{}CC-BY-SA\textquotedbl{} means the Creative Commons
Attribution-Share Alike 3.0 license published by Creative Commons
Corporation, a not-for-profit corporation with a principal place of
business in San Francisco, California, as well as future copyleft
versions of that license published by that same organization.

\textquotedbl{}Incorporate\textquotedbl{} means to publish or republish
a Document, in whole or in part, as part of another Document.

An MMC is \textquotedbl{}eligible for relicensing\textquotedbl{} if
it is licensed under this License, and if all works that were first
published under this License somewhere other than this MMC, and subsequently
incorporated in whole or in part into the MMC, (1) had no cover texts
or invariant sections, and (2) were thus incorporated prior to November
1, 2008.

The operator of an MMC Site may republish an MMC contained in the
site under CC-BY-SA on the same site at any time before August 1,
2009, provided the MMC is eligible for relicensing.


\section*{ADDENDUM: How to use this License for your documents}

To use this License in a document you have written, include a copy
of the License in the document and put the following copyright and
license notices just after the title page:
\begin{quotation}
\noindent Copyright (c) YEAR YOUR NAME. Permission is granted to copy,
distribute and/or modify this document under the terms of the GNU
Free Documentation License, Version 1.3 or any later version published
by the Free Software Foundation; with no Invariant Sections, no Front-Cover
Texts, and no Back-Cover Texts. A copy of the license is included
in the section entitled \textquotedbl{}GNU Free Documentation License\textquotedbl{}.
\end{quotation}
If you have Invariant Sections, Front-Cover Texts and Back-Cover Texts,
replace the \textquotedbl{}with...Texts.\textquotedbl{} line with
this:
\begin{quotation}
\noindent with the Invariant Sections being LIST THEIR TITLES, with
the Front-Cover Texts being LIST, and with the Back-Cover Texts being
LIST.
\end{quotation}
If you have Invariant Sections without Cover Texts, or some other
combination of the three, merge those two alternatives to suit the
situation.

If your document contains nontrivial examples of program code, we
recommend releasing these examples in parallel under your choice of
free software license, such as the GNU General Public License, to
permit their use in free software. 


\chapter{History\label{cha:History}}

\begin{tabular}{ll}
\textbf{Title:} & Introduction to Probability and Statistics Using \textsf{R}\tabularnewline
\textbf{Year:} & 2010\tabularnewline
\textbf{Authors:} & G.~Jay Kerns\tabularnewline
\textbf{Publisher:} & G.~Jay Kerns\tabularnewline
\end{tabular}

\vfill{}



\chapter{Data\label{cha:data}}

This appendix is a reference of sorts regarding some of the data structures
a statistician is likely to encounter. We discuss their salient features
and idiosyncrasies.


\section{Data Structures\label{sec:Data-Structures}}


\subsection{Vectors}

See the {}``Vectors and Assignment'' section of \emph{An Introduction
to }\textsf{\emph{R}}. A vector is an ordered sequence of elements,
such as numbers, characters, or logical values, and there may be \inputencoding{latin9}\lstinline[showstringspaces=false]!NA!\inputencoding{utf8}'s
present. We usually make vectors with the assignment operator \inputencoding{latin9}\lstinline[showstringspaces=false]!<-!\inputencoding{utf8}.

<<>>=
x <- c(3, 5, 9)
@

Vectors are atomic in the sense that if you try to mix and match elements
of different modes then all elements will be coerced to the most convenient
common mode.

<<>>=
y <- c(3, "5", TRUE)
@

In the example all elements were coerced to \emph{character} mode.
We can test whether a given object is a vector with \inputencoding{latin9}\lstinline[showstringspaces=false]!is.vector!\inputencoding{utf8}
and can coerce an object (if possible) to a vector with \inputencoding{latin9}\lstinline[showstringspaces=false]!as.vector!\inputencoding{utf8}.


\subsection{Matrices and Arrays}

See the {}``Arrays and Matrices'' section of \emph{An Introduction
to }\textsf{\emph{R}}. Loosely speaking, a matrix is a vector that
has been reshaped into rectangular form, and an array is a multidimensional
matrix. Strictly speaking, it is the other way around: an array is
a data vector with a dimension attribute (\inputencoding{latin9}\lstinline[showstringspaces=false]!dim!\inputencoding{utf8}),
and a matrix is the special case of an array with only two dimensions.
We can construct a matrix with the \inputencoding{latin9}\lstinline[showstringspaces=false]!matrix!\inputencoding{utf8}
function. 

<<>>=
matrix(letters[1:6], nrow = 2, ncol = 3)
@

Notice the order of the matrix entries, which shows how the matrix
is populated by default. We can change this with the \inputencoding{latin9}\lstinline[showstringspaces=false]!byrow!\inputencoding{utf8}
argument:

<<>>=
matrix(letters[1:6], nrow = 2, ncol = 3, byrow = TRUE)
@

We can test whether a given object is a matrix with \inputencoding{latin9}\lstinline[showstringspaces=false]!is.matrix!\inputencoding{utf8}
and can coerce an object (if possible) to a matrix with \inputencoding{latin9}\lstinline[showstringspaces=false]!as.matrix!\inputencoding{utf8}.
As a final example watch what happens when we mix and match types
in the first argument:

<<>>=
matrix(c(1,"2",NA, FALSE), nrow = 2, ncol = 3)
@

Notice how all of the entries were coerced to character for the final
result (except \inputencoding{latin9}\lstinline[showstringspaces=false]!NA!\inputencoding{utf8}).
Also notice how the four values were \emph{recycled} to fill up the
six entries of the matrix.

The standard arithmetic operations work element-wise with matrices.

<<>>=
A <- matrix(1:6, 2, 3)
B <- matrix(2:7, 2, 3)
A + B
A * B
@

If you want the standard definition of matrix multiplication then
use the \inputencoding{latin9}\lstinline[showstringspaces=false]!%*%!\inputencoding{utf8}
function. If we were to try \inputencoding{latin9}\lstinline[showstringspaces=false]!A %*% B!\inputencoding{utf8}
we would get an error because the dimensions do not match correctly,
but for fun, we could transpose \inputencoding{latin9}\lstinline[showstringspaces=false]!B!\inputencoding{utf8}
to get conformable matrices. The transpose function \inputencoding{latin9}\lstinline[showstringspaces=false]!t!\inputencoding{utf8}
only works for matrices (and data frames).

<<keep.source = TRUE>>=
try(A * B)     # an error
A %*% t(B)     # this is alright
@

To get the ordinary matrix inverse use the \inputencoding{latin9}\lstinline[showstringspaces=false]!solve!\inputencoding{utf8}
function:

<<keep.source = TRUE>>=
solve(A %*% t(B))     # input matrix must be square
@

Arrays more general than matrices, and some functions (like transpose)
do not work for the more general array. Here is what an array looks
like: 

<<keep.source = TRUE>>=
array(LETTERS[1:24], dim = c(3,4,2))
@

We can test with \inputencoding{latin9}\lstinline[showstringspaces=false]!is.array!\inputencoding{utf8}
and may coerce with \inputencoding{latin9}\lstinline[showstringspaces=false]!as.array!\inputencoding{utf8}.


\subsection{Data Frames}

A data frame is a rectangular array of information with a special
status in \textsf{R}. It is used as the fundamental data structure
by many of the modeling functions. It is like a matrix in that all
of the columns must be the same length, but it is more general than
a matrix in that columns are allowed to have different modes.

<<>>=
x <- c(1.3, 5.2, 6)
y <- letters[1:3]
z <- c(TRUE, FALSE, TRUE)
A <- data.frame(x, y, z)
A
@

Notice the \inputencoding{latin9}\lstinline[showstringspaces=false]!names!\inputencoding{utf8}
on the columns of \inputencoding{latin9}\lstinline[showstringspaces=false]!A!\inputencoding{utf8}.
We can change those with the \inputencoding{latin9}\lstinline[showstringspaces=false]!names!\inputencoding{utf8}
function.

<<>>=
names(A) <- c("Fred","Mary","Sue")
A
@

Basic command is \inputencoding{latin9}\lstinline[showstringspaces=false]!data.frame!\inputencoding{utf8}.
You can test with \inputencoding{latin9}\lstinline[showstringspaces=false]!is.data.frame!\inputencoding{utf8}
and you can coerce with \inputencoding{latin9}\lstinline[showstringspaces=false]!as.data.frame!\inputencoding{utf8}.


\subsection{Lists}

A list is more general than a data frame.


\subsection{Tables}

The word {}``table'' has a special meaning in \textsf{R}. More precisely,
a contingency table is an object of class {}``\inputencoding{latin9}\lstinline[showstringspaces=false]!table!\inputencoding{utf8}''
which is an array

Suppose you have a contingency table and would like to do descriptive
or inferential statistics on it. The default form of the table is
usually inconvenient to use unless we are working with a function
specially tailored for tables. Here is how to transform your data
to a more manageable form, namely, the raw data used to make the table.

First, we coerce the table to a data frame with : 

<<>>=
A <- as.data.frame(Titanic)
head(A)
@

Note that there are as many preliminary columns of \inputencoding{latin9}\lstinline[showstringspaces=false]!A!\inputencoding{utf8}
as there are dimensions to the table. The rows of \inputencoding{latin9}\lstinline[showstringspaces=false]!A!\inputencoding{utf8}
contain every possible combination of levels from each of the dimensions.
There is also a \inputencoding{latin9}\lstinline[showstringspaces=false]!Freq!\inputencoding{utf8}
column, which shows how many observations there were at that particular
combination of levels.

The form of \inputencoding{latin9}\lstinline[showstringspaces=false]!A!\inputencoding{utf8}
is often sufficient for our purposes, but more often we need to do
more work: we would usually like to repeat each row of \inputencoding{latin9}\lstinline[showstringspaces=false]!A!\inputencoding{utf8}
exactly the number of times shown in the \inputencoding{latin9}\lstinline[showstringspaces=false]!Freq!\inputencoding{utf8}
column. The \inputencoding{latin9}\lstinline[showstringspaces=false]!reshape!\inputencoding{utf8}
package \cite{Wickhamreshape} has the function \inputencoding{latin9}\lstinline[showstringspaces=false]!untable!\inputencoding{utf8}
designed for that very purpose: 

<<>>=
library(reshape)
B <- with(A, untable(A, Freq))
head(B)
@

Now, this is more like it. Note that we slipped in a call to the \inputencoding{latin9}\lstinline[showstringspaces=false]!with!\inputencoding{utf8}
function, which was done to make the call to \inputencoding{latin9}\lstinline[showstringspaces=false]!untable!\inputencoding{utf8}
more pretty; we could just as easily have done

\inputencoding{latin9}
\begin{lstlisting}[showstringspaces=false]
untable(TitanicDF, A$Freq)
\end{lstlisting}
\inputencoding{utf8}

The only fly in the ointment is the lingering \inputencoding{latin9}\lstinline[showstringspaces=false]!Freq!\inputencoding{utf8}
column which has repeated values that do not have any meaning any
more. We could just ignore it, but it would be better to get rid of
the meaningless column so that it does not cause trouble later. While
we are at it, we could clean up the \inputencoding{latin9}\lstinline[showstringspaces=false]!rownames!\inputencoding{utf8},
too.

<<>>=
C <- B[, -5]
rownames(C) <- 1:dim(C)[1]
head(C)
@


\subsection{More about Tables}

Suppose you want to make a table that looks like this:

There are at least two ways to do it.
\begin{itemize}
\item Using a matrix:


<<>>=
tab <- matrix(1:6, nrow = 2, ncol = 3)
rownames(tab) <- c('first', 'second')
colnames(tab) <- c('A', 'B', 'C')
tab  # Counts
@
\begin{itemize}
\item note that the columns are filled in consecutively by default. If you
want to fill the data in by rows then do byrow = TRUE in the matrix
command.
\item the object is a matrix
\end{itemize}
\item Using a dataframe


<<>>=
p <- c("milk","tea")
g <- c("milk","tea")
catgs <- expand.grid(poured = p, guessed = g)
cnts <- c(3, 1, 1, 3)
D <- cbind(catgs, count = cnts)
xtabs(count ~ poured + guessed, data = D)
@
\begin{itemize}
\item again, the data are filled in column-wise.
\item the object is a dataframe
\item if you want to store it as a table then do A <- xtabs(count \textasciitilde{}
poured + guessed, data = D)
\end{itemize}
\end{itemize}

\section{Importing Data\label{sec:Importing-A-Data}}

Statistics is the study of data, so the statistician's first step
is usually to obtain data from somewhere or another and read them
into \textsf{R}. In this section we describe some of the most common
sources of data and how to get data from those sources into a running
\textsf{R} session.

For more information please refer to the \textsf{R} \emph{Data Import/Export
Manual}, \cite{rstatenv} and \emph{An Introduction to }\textsf{\emph{R}},
\cite{Venables2010}.


\subsection{Data in Packages}

There are many data sets stored in the \inputencoding{latin9}\lstinline[showstringspaces=false]!datasets!\inputencoding{utf8}
package of base \textsf{R}. To see a list of them all issue the command
\inputencoding{latin9}\lstinline[showstringspaces=false]!data(package = "datasets")!\inputencoding{utf8}.
The output is omitted here because the list is so long. The names
of the data sets are listed in the left column. Any data set in that
list is already on the search path by default, which means that a
user can use it immediately without any additional work.

There are many other data sets available in the thousands of contributed
packages. To see the data sets available in those packages that are
currently loaded into memory issue the single command \inputencoding{latin9}\lstinline[showstringspaces=false]!data()!\inputencoding{utf8}.
If you would like to see all of the data sets that are available in
all packages that are installed on your computer (but not necessarily
loaded), issue the command 

\inputencoding{latin9}
\begin{lstlisting}[breaklines=true,showstringspaces=false,tabsize=2]
data(package = .packages(all.available = TRUE))
\end{lstlisting}
\inputencoding{utf8}

To load the data set \inputencoding{latin9}\lstinline[showstringspaces=false]!foo!\inputencoding{utf8}
in the contributed package \inputencoding{latin9}\lstinline[showstringspaces=false]!bar!\inputencoding{utf8}
issue the commands \inputencoding{latin9}\lstinline[showstringspaces=false]!library(bar)!\inputencoding{utf8}
followed by \inputencoding{latin9}\lstinline[showstringspaces=false]!data(foo)!\inputencoding{utf8},
or just the single command \inputencoding{latin9}
\begin{lstlisting}[breaklines=true,showstringspaces=false,tabsize=2]
data(foo, package = "bar")
\end{lstlisting}
\inputencoding{utf8}


\subsection{Text Files}

Many sources of data are simple text files. The entries in the file
are separated by delimeters such as TABS (tab-delimeted), commas (comma
separated values, or \inputencoding{latin9}\lstinline[showstringspaces=false]!.csv!\inputencoding{utf8},
for short) or even just white space (no special name). A lot of data
on the Internet are stored with text files, and even if they are not,
a person can copy-paste information from a web page to a text file,
save it on the computer, and read it into \textsf{R}. 


\subsection{Other Software Files}

Often the data set of interest is stored in some other, proprietary,
format by third-party software such as Minitab, SAS, or SPSS. The
\inputencoding{latin9}\lstinline[showstringspaces=false]!foreign!\inputencoding{utf8}
package supports import/conversion from many of these formats. Please
note, however, that data sets from other software sometimes have properties
with no direct analogue in \textsf{R}. In those cases the conversion
process may lose some information which will need to be reentered
manually from within \textsf{R}. See the \emph{Data Import/Export
Manual}.

As an example, suppose the data are stored in the SPSS file \inputencoding{latin9}\lstinline[showstringspaces=false]!foo.sav!\inputencoding{utf8}
which the user has copied to the working directory; it can be imported
with the commands

<<eval = FALSE>>=
library(foreign)
read.spss("foo.sav")
@

See \inputencoding{latin9}\lstinline[showstringspaces=false]!?read.spss!\inputencoding{utf8}
for the available options to customize the file import. Note that
the \textsf{R} Commander will import many of the common file types
with a menu driven interface.


\subsection{Importing a Data Frame}

The basic command is \inputencoding{latin9}\lstinline[showstringspaces=false,tabsize=2]!read.table!\inputencoding{utf8}.


\section{Creating New Data Sets\label{sec:Creating-New-Data}}

Using \inputencoding{latin9}\lstinline[showstringspaces=false,tabsize=2]!c!\inputencoding{utf8}

Using \inputencoding{latin9}\lstinline[showstringspaces=false,tabsize=2]!scan!\inputencoding{utf8}

Using the \textsf{R} Commander.


\section{Editing Data\label{sec:Editing-Data-Sets}}


\subsection{Editing Data Values}


\subsection{Inserting Rows and Columns}


\subsection{Deleting Rows and Columns}


\subsection{Sorting Data}

We can sort a vector with the \inputencoding{latin9}\lstinline[showstringspaces=false,tabsize=2]!sort!\inputencoding{utf8}
function. Normally we have a data frame of several columns (variables)
and many, many rows (observations). The goal is to shuffle the rows
so that they are ordered by the values of one or more columns. This
is done with the \inputencoding{latin9}\lstinline[showstringspaces=false,tabsize=2]!order!\inputencoding{utf8}
function. 

For example, we may sort all of the rows of the \inputencoding{latin9}\lstinline[showstringspaces=false,tabsize=2]!Puromycin!\inputencoding{utf8}
data (in ascending order) by the variable \inputencoding{latin9}\lstinline[showstringspaces=false,tabsize=2]!conc!\inputencoding{utf8}
with the following: 

<<>>=
Tmp <- Puromycin[order(Puromycin$conc), ]
head(Tmp)
@ 

We can accomplish the same thing with the command 

<<eval = FALSE>>=
with(Puromycin, Puromycin[order(conc), ])
@

We can sort by more than one variable. To sort first by \inputencoding{latin9}\lstinline[showstringspaces=false,tabsize=2]!state!\inputencoding{utf8}
and next by \inputencoding{latin9}\lstinline[showstringspaces=false,tabsize=2]!conc!\inputencoding{utf8}
do 

<<eval = FALSE>>=
with(Puromycin, Puromycin[order(state, conc), ])
@

If we would like to sort a numeric variable in descending order then
we put a minus sign in front of it. 

<<>>=
Tmp <- with(Puromycin, Puromycin[order(-conc), ])
head(Tmp)
@

If we would like to sort by a character (or factor) in decreasing
order then we can use the \inputencoding{latin9}\lstinline[showstringspaces=false,tabsize=2]!xtfrm!\inputencoding{utf8}
function which produces a numeric vector in the same order as the
character vector.

<<>>=
Tmp <- with(Puromycin, Puromycin[order(-xtfrm(state)), ])
head(Tmp)
@


\section{Exporting Data\label{sec:Exporting-a-Data}}

The basic function is \inputencoding{latin9}\lstinline[showstringspaces=false,tabsize=2]!write.table!\inputencoding{utf8}.
The \inputencoding{latin9}\lstinline[showstringspaces=false,tabsize=2]!MASS!\inputencoding{utf8}
package also has a \inputencoding{latin9}\lstinline[showstringspaces=false,tabsize=2]!write.matrix!\inputencoding{utf8}
function.


\section{Reshaping Data\label{sec:Reshaping-a-Data}}
\begin{itemize}
\item Aggregation
\item Convert Tables to data frames and back
\end{itemize}
\inputencoding{latin9}\lstinline[showstringspaces=false,tabsize=2]!rbind!\inputencoding{utf8},
\inputencoding{latin9}\lstinline[showstringspaces=false,tabsize=2]!cbind!\inputencoding{utf8}

\inputencoding{latin9}\lstinline[showstringspaces=false,tabsize=2]!ab[order(ab[,1]),]!\inputencoding{utf8}

\inputencoding{latin9}\lstinline[showstringspaces=false,tabsize=2]!complete.cases!\inputencoding{utf8}

\inputencoding{latin9}\lstinline[showstringspaces=false,tabsize=2]!aggregate!\inputencoding{utf8}

\inputencoding{latin9}\lstinline[showstringspaces=false,tabsize=2]!stack!\inputencoding{utf8}


\chapter{Mathematical Machinery\label{cha:Mathematical-Machinery}}

This appendix houses many of the standard definitions and theorems
that are used at some point during the narrative. It is targeted for
someone reading the book who forgets the precise definition of something
and would like a quick reminder of an exact statement. No proofs are
given, and the interested reader should consult a good text on Calculus
(say, Stewart \cite{Stewart2008} or Apostol \cite{Apostol1967,ApostolI1967}),
Linear Algebra (say, Strang \cite{Strang1988} and Magnus \cite{Magnus1999}),
Real Analysis (say, Folland \cite{Folland1999}, or Carothers \cite{Carothers2000}),
or Measure Theory (Billingsley \cite{Billingsley1995}, Ash \cite{Ash2000},
Resnick \cite{Resnick1999}) for details. 


\section{Set Algebra\label{sec:The-Algebra-of}}

We denote sets by capital letters, $A$, $B$, $C$, \emph{etc}. The
letter $S$ is reserved for the sample space, also known as the universe
or universal set, the set which contains all possible elements. The
symbol $\emptyset$ represents the empty set, the set with no elements. 


\subsection*{Set Union, Intersection, and Difference}

Given subsets $A$ and $B$, we may manipulate them in an algebraic
fashion. To this end, we have three set operations at our disposal:
union, intersection, and difference. Below is a table summarizing
the pertinent information about these operations.%
\begin{table}
\begin{centering}
\begin{tabular}{|l|c|l|l|}
\hline 
Name  & Denoted  & Defined by elements  & \textsf{R} syntax\tabularnewline
\hline 
Union  & $A\cup B$  & in $A$ or $B$ or both  & \inputencoding{latin9}\lstinline[showstringspaces=false,tabsize=2]!union(A, B)!\inputencoding{utf8}\tabularnewline
Intersection  & $A\cap B$  & in both $A$ and $B$  & \inputencoding{latin9}\lstinline[showstringspaces=false,tabsize=2]!intersect(A, B)!\inputencoding{utf8} \tabularnewline
Difference & $A\textbackslash B$  & in $A$ but not in $B$  & \inputencoding{latin9}\lstinline[showstringspaces=false,tabsize=2]!setdiff(A, B)!\inputencoding{utf8} \tabularnewline
Complement & $A^{c}$ & in $S$ but not in $A$ & \inputencoding{latin9}\lstinline[showstringspaces=false,tabsize=2]!setdiff(S, A)!\inputencoding{utf8}\tabularnewline
\hline
\end{tabular}
\par\end{centering}

\caption{Set operations\label{tab:Set-Operations}}

\end{table}



\subsection*{Identities and Properties}
\begin{enumerate}
\item $A\cup\emptyset=A,\quad A\cap\emptyset=\emptyset$
\item $A\cup S=S,\quad A\cap S=A$
\item $A\cup A^{c}=S$, $A\cap A^{c}=\emptyset$
\item $(A{}^{c})^{c}=A$
\item The Commutative Property: \begin{equation}
A\cup B=B\cup A,\quad A\cap B=B\cap A\end{equation}

\item The Associative Property: \begin{equation}
(A\cup B)\cup C=A\cup(B\cup C),\quad(A\cap B)\cap C=A\cap(B\cap C)\end{equation}

\item The Distributive Property: \begin{equation}
A\cup(B\cap C)=(A\cup B)\cap(A\cup B),\quad A\cap(B\cup C)=(A\cap B)\cup(A\cap B)\end{equation}

\item DeMorgan's Laws\begin{equation}
(A\cup B)^{c}=A^{c}\cap B^{c}\quad\mbox{and}\quad(A\cap B)^{c}=A^{c}\cup B^{c},\end{equation}
or more generally,\begin{equation}
\left(\bigcup_{\alpha}A_{\alpha}\right)^{c}=\bigcap_{\alpha}A_{\alpha}^{c},\quad\mbox{and}\quad\left(\bigcap_{\alpha}A_{\alpha}\right)^{c}=\bigcup_{\alpha}A_{\alpha}^{c}\end{equation}

\end{enumerate}

\section{Differential and Integral Calculus\label{sec:Differential-and-Integral}}

A function $f$ of one variable is said to be one-to-one if no two
distinct $x$ values are mapped to the same $y=f(x)$ value. To show
that a function is one-to-one we can either use the horizontal line
test or we may start with the equation $f(x_{1})=f(x_{2})$ and use
algebra to show that it implies $x_{1}=x_{2}$.


\subsection*{Limits and Continuity}
\begin{defn}
Let $f$ be a function defined on some open interval that contains
the number $a$, except possibly at $a$ itself. Then we say the \emph{limit
of} $f(x)$ \emph{as} $x$ \emph{approaches} $a$ \emph{is} $L$,
and we write \begin{equation}
\lim_{x\to a}f(x)=L,\end{equation}
if for every $\epsilon>0$ there exists a number $\delta>0$ such
that $0<|x-a|<\delta$ implies $|f(x)-L|<\epsilon$.
\end{defn}

\begin{defn}
A function $f$ is \emph{continuous at a number} $a$ if \begin{equation}
\lim_{x\to a}f(x)=f(a).\end{equation}
The function $f$ is \emph{right-continuous at the number} $a$ if
$\lim_{x\to a^{+}}f(x)=f(a)$, and \emph{left-continuous} at $a$
if $\lim_{x\to a^{-}}f(x)=f(a)$. Finally, the function $f$ is \emph{continuous
on an interval} $I$ if it is continuous at every number in the interval. 
\end{defn}

\subsection*{Differentiation}
\begin{defn}
The \emph{derivative of a function} $f$ \emph{at a number} $a$,
denoted by $f'(a)$, is\begin{equation}
f'(a)=\lim_{h\to0}\frac{f(a+h)-f(a)}{h},\end{equation}
provided this limit exists.

A function is \emph{differentiable at} $a$ if $f'(a)$ exists. It
is \emph{differentiable on an open interval} $(a,b)$ if it is differentiable
at every number in the interval.
\end{defn}

\subsubsection*{Differentiation Rules}

In the table that follows, $f$ and $g$ are differentiable functions
and $c$ is a constant.

%
\begin{table}[H]
\begin{centering}
\begin{tabular}{|c|c|c|}
\hline 
 &  & \tabularnewline
$\frac{\diff}{\diff x}c=0$ & $\frac{\diff}{\diff x}x^{n}=nx^{n-1}$ & $(cf)'=cf'$\tabularnewline
 &  & \tabularnewline
$(f\pm g)'=f'\pm g'$ & $(fg)'=f'g+fg'$ & $\left(\frac{f}{g}\right)'=\frac{f'g-fg'}{g^{2}}$\tabularnewline
 &  & \tabularnewline
\hline
\end{tabular}
\par\end{centering}

\caption{Differentiation rules\textbf{\label{tab:Differentiation-Rules}}}

\end{table}

\begin{thm}
Chain Rule: If $f$ and $g$ are both differentiable and $F=f\circ g$
is the composite function defined by $F(x)=f[g(x)]$, then $F$ is
differentiable and $F'(x)=f'[g(x)]\cdot g'(x)$. 
\end{thm}

\subsubsection*{Useful Derivatives}

%
\begin{table}[H]
\begin{centering}
\begin{tabular}{|c|c|c|}
\hline 
 &  & \tabularnewline
$\frac{\diff}{\diff x}\me^{x}=\me^{x}$ & $\frac{\diff}{\diff x}\ln x=x^{-1}$ & $\frac{\diff}{\diff x}\sin x=\cos x$\tabularnewline
 &  & \tabularnewline
$\frac{\diff}{\diff x}\cos x=-\sin x$ & $\frac{\diff}{\diff x}\tan x=\sec^{2}x$ & $\frac{\diff}{\diff x}\tan^{-1}x=(1+x^{2})^{-1}$$ $\tabularnewline
 &  & \tabularnewline
\hline
\end{tabular}
\par\end{centering}

\caption{Some derivatives\textbf{\label{tab:Useful-Derivatives}}}

\end{table}



\subsection*{Optimization}
\begin{defn}
A \emph{critical number} of the function $f$ is a value $x^{\ast}$
for which $f'(x^{\ast})=0$ or for which $f'(x^{\ast})$ does not
exist.\end{defn}
\begin{thm}
\label{thm:First-Derivative-Test}First Derivative Test. If $f$ is
differentiable and if $x^{\ast}$ is a critical number of $f$ and
if $f'(x)\geq0$ for $x\leq x^{\ast}$ and $f'(x)\leq0$ for $x\geq x^{\ast}$
, then $x^{\ast}$ is a local maximum of $f$. If $f'(x)\leq0$ for
$x\leq x^{\ast}$ and $f'(x)\geq0$ for $x\geq x^{\ast}$ , then $x^{\ast}$
is a local minimum of $f$. 
\end{thm}

\begin{thm}
Second Derivative Test. If $f$ is twice differentiable and if $x^{\ast}$
is a critical number of $f$, then $x^{\ast}$ is a local maximum
of $f$ if $f''(x^{\ast})<0$ and $x^{\ast}$ is a local minimum of
$f$ if $f''(x^{\ast})>0$.
\end{thm}

\subsection*{Integration}

As it turns out, there are all sorts of things called {}``integrals'',
each defined in its own idiosyncratic way. There are \emph{Riemann}
integrals, \emph{Lebesgue} integrals, variants of these called \emph{Stieltjes}
integrals, \emph{Daniell} integrals, \emph{Ito} integrals, and the
list continues. Given that this is an introductory book, we will use
the Riemannian integral with the caveat that the Riemann integral
is \emph{not} the integral that will be used in more advanced study.
\begin{defn}
Let $f$ be defined on $[a,b]$, a closed interval of the real line.
For each $n$, divide $[a,b]$ into subintervals $[x_{i},x_{i+1}]$,
$i=0,1,\ldots,n-1$, of length $\Delta x_{i}=(b-a)/n$ where $x_{0}=a$
and $x_{n}=b$, and let $x_{i}^{\ast}$ be any points chosen from
the respective subintervals. Then the \emph{definite integral} of
$f$ from $a$ to $b$ is defined by\begin{equation}
\int_{a}^{b}f(x)\,\diff x=\lim_{n\to\infty}\sum_{i=0}^{n-1}f(x_{i}^{\ast})\,\Delta x_{i},\end{equation}
provided the limit exists, and in that case, we say that $f$ is \emph{integrable}
from $a$ to $b$. \end{defn}
\begin{thm}
The Fundamental Theorem of Calculus. Suppose $f$ is continuous on
$[a,b]$. Then
\begin{enumerate}
\item the function $g$ defined by $g(x)=\int_{a}^{x}f(t)\:\diff t$, $a\leq x\leq b$,
is continuous on $[a,b]$ and differentiable on $(a,b)$ with $g'(x)=f(x)$.
\item $\int_{a}^{b}f(x)\,\diff x=F(b)-F(a)$, where $F$ is any \emph{antiderivative}
of $f$, that is, any function $F$ satisfying $F'=f$.
\end{enumerate}
\end{thm}

\subsubsection*{Change of Variables}
\begin{thm}
If $g$ is a differentiable function whose range is the interval $[a,b]$
and if both $f$ and $g'$ are continuous on the range of $u=g(x)$,
then\begin{equation}
\int_{g(a)}^{g(b)}f(u)\:\diff u=\int_{a}^{b}f[g(x)]\: g'(x)\:\diff x.\end{equation}

\end{thm}

\subsubsection*{Useful Integrals}

%
\begin{table}[H]
\begin{centering}
\begin{tabular}{|c|c|c|}
\hline 
 &  & \tabularnewline
$\int x^{n}\,\diff x=x^{n+1}/(n+1),\ n\neq-1$ & $\int\me^{x}\,\diff x=\me^{x}$ & $\int x^{-1}\,\diff x=\ln|x|$\tabularnewline
 &  & \tabularnewline
$\int\tan x\:\diff x=\ln|\sec x|$ & $\int a^{x}\,\diff x=a^{x}/\ln a$ & $\int(x^{2}+1)^{-1}\,\diff x=\tan^{-1}x$\tabularnewline
 &  & \tabularnewline
\hline
\end{tabular}
\par\end{centering}

\caption{Some integrals (constants of integration omitted)\textbf{\label{tab:Useful-Integrals}}}

\end{table}



\subsubsection*{Integration by Parts}

\begin{equation}
\int u\:\diff v=uv-\int v\:\diff u\end{equation}

\begin{thm}
L'H\^ opital's Rule. Suppose $f$ and $g$ are differentiable and
$g'(x)\neq0$ near $a$, except possibly at $a$. Suppose that the
limit\begin{equation}
\lim_{x\to a}\frac{f(x)}{g(x)}\end{equation}
is an indeterminate form of type $\frac{0}{0}$ or $\infty/\infty$.
Then\begin{equation}
\lim_{x\to a}\frac{f(x)}{g(x)}=\lim_{x\to a}\frac{f'(x)}{g'(x)},\end{equation}
provided the limit on the right-hand side exists or is infinite.
\end{thm}

\subsubsection*{Improper Integrals}

If $\int_{a}^{t}f(x)\diff x$ exists for every number $t\geq a$,
then we define \begin{equation}
\int_{a}^{\infty}f(x)\,\diff x=\lim_{t\to\infty}\int_{a}^{t}f(x)\,\diff x,\end{equation}
provided this limit exists as a finite number, and in that case we
say that $\int_{a}^{\infty}f(x)\,\diff x$ is \emph{convergent}. Otherwise,
we say that the improper integral is \emph{divergent}.

If $\int_{t}^{b}f(x)\,\diff x$ exists for every number $t\leq b$,
then we define \begin{equation}
\int_{-\infty}^{b}f(x)\,\diff x=\lim_{t\to-\infty}\int_{t}^{b}f(x)\,\diff x,\end{equation}
provided this limit exists as a finite number, and in that case we
say that $\int_{-\infty}^{b}f(x)\,\diff x$ is \emph{convergent}.
Otherwise, we say that the improper integral is \emph{divergent}.

If both $\int_{a}^{\infty}f(x)\,\diff x$ and $\int_{-\infty}^{a}f(x)\,\diff x$
are convergent, then we define\begin{equation}
\int_{-\infty}^{\infty}f(x)\,\diff x=\int_{-\infty}^{a}f(x)\,\diff x+\int_{a}^{\infty}f(x)\diff x,\end{equation}
and we say that $\int_{-\infty}^{\infty}f(x)\,\diff x$ is \emph{convergent}.
Otherwise, we say that the improper integral is \emph{divergent}.


\section{Sequences and Series\label{sec:Sequences-and-Series}}

A \emph{sequence} is an ordered list of numbers, $a_{1}$, $a_{2}$,
$a_{3}$, \ldots{}, $a_{n}$ $=\left(a_{k}\right)_{k=1}^{n}$. A
sequence may be finite or infinite. In the latter case we write $a_{1}$,
$a_{2}$, $a_{3}$, \ldots{}$=\left(a_{k}\right)_{k=1}^{\infty}$.
We say that \emph{the infinite sequence} $\left(a_{k}\right)_{k=1}^{\infty}$
\emph{converges to the finite limit} $L$, and we write \begin{equation}
\lim_{k\to\infty}a_{k}=L,\end{equation}
if for every $\epsilon>0$ there exists an integer $N\geq1$ such
that $|a_{k}-L|<\epsilon$ for all $k\geq N$. We say that \emph{the
infinite sequence} $\left(a_{k}\right)_{k=1}^{\infty}$ \emph{diverges
to} $+\infty$ (or -$\infty$) if for every $M\geq0$ there exists
an integer $N\geq1$ such that $a_{k}\geq M$ for all $k\geq N$ (or
$a_{k}\leq-M$ for all $k\geq N$).


\subsection*{Finite Series}

\begin{equation}
\sum_{k=1}^{n}k=1+2+\cdots+n=\frac{n(n+1)}{2}\label{eq:gauss-series}\end{equation}


\begin{equation}
\sum_{k=1}^{n}k^{2}=1^{2}+2^{2}+\cdots+n^{2}=\frac{n(n+1)(2n+3)}{6}\label{eq:gauss-series-sq}\end{equation}



\subsubsection*{The Binomial Series}


\paragraph*{\begin{equation}
\sum_{k=0}^{n}{n \choose k}\, a^{n-k}b^{k}=(a+b)^{n}\label{eq:binom-series}\end{equation}
}


\subsection*{Infinite Series}

Given an infinite sequence of numbers $a_{1}$, $a_{2}$, $a_{3}$,
\ldots{}$=\left(a_{k}\right)_{k=1}^{\infty}$, let $s_{n}$ denote
the \emph{partial sum} of the first $n$ terms:\begin{equation}
s_{n}=\sum_{k=1}^{n}a_{k}=a_{1}+a_{2}+\cdots+a_{n}.\end{equation}
If the sequence $\left(s_{n}\right)_{n=1}^{\infty}$ converges to
a finite number $S$ then we say that the infinite series $\sum_{k}a_{k}$
is \emph{convergent} and write\begin{equation}
\sum_{k=1}^{\infty}a_{k}=S.\end{equation}
Otherwise we say the infinite series is \emph{divergent}.


\subsection*{Rules for Series}

Let $\left(a_{k}\right)_{k=1}^{\infty}$ and $\left(b_{k}\right)_{k=1}^{\infty}$
be infinite sequences and let $c$ be a constant.


\paragraph*{\begin{equation}
\sum_{k=1}^{\infty}ca_{k}=c\sum_{k=1}^{\infty}a_{k}\end{equation}
\begin{equation}
\sum_{k=1}^{\infty}(a_{k}\pm b_{k})=\sum_{k=1}^{\infty}a_{k}\pm\sum_{k=1}^{\infty}b_{k}\end{equation}
}

In both of the above the series on the left is convergent if the series
on the right is (are) convergent.


\subsubsection*{The Geometric Series\begin{equation}
\sum_{k=0}^{\infty}x^{k}=\frac{1}{1-x},\quad|x|<1.\label{eq:geom-series}\end{equation}
}


\subsubsection*{The Exponential Series\begin{equation}
\sum_{k=0}^{\infty}\frac{x^{k}}{k!}=\me^{x},\quad-\infty<x<\infty.\label{eq:exp-series}\end{equation}
}


\paragraph*{Other Series\begin{equation}
\sum_{k=0}^{\infty}{m+k-1 \choose m-1}x^{k}=\frac{1}{(1-x)^{m}},\quad|x|<1.\label{eq:negbin-series}\end{equation}
}


\paragraph*{\begin{equation}
-\sum_{k=1}^{\infty}\frac{x^{n}}{n}=\ln(1-x),\quad|x|<1.\label{eq:log-series}\end{equation}
}

\begin{equation}
\sum_{k=0}^{\infty}{n \choose k}x^{k}=(1+x)^{n},\quad|x|<1.\label{eq:binom-series-infinite}\end{equation}



\subsection*{Taylor Series}

If the function $f$ has a \emph{power series} representation at the
point $a$ with radius of convergence $R>0$, that is, if\begin{equation}
f(x)=\sum_{k=0}^{\infty}c_{k}(x-a)^{k},\quad|x-a|<R,\end{equation}
for some constants $\left(c_{k}\right)_{k=0}^{\infty}$, then $c_{k}$
must be \begin{equation}
c_{k}=\frac{f^{(k)}(a)}{k!},\quad k=0,1,2,\ldots\end{equation}
Furthermore, the function $f$ is differentiable on the open interval
$(a-R,\, a+R)$ with\begin{equation}
f'(x)=\sum_{k=1}^{\infty}kc_{k}(x-a)^{k-1},\quad|x-a|<R,\end{equation}
\begin{equation}
\int f(x)\,\diff x=C+\sum_{k=0}^{\infty}c_{k}\frac{(x-a)^{k+1}}{k+1},\quad|x-a|<R,\end{equation}
in which case both of the above series have radius of convergence
$R$.


\section{The Gamma Function\label{sec:The-Gamma-Function}}

The \emph{Gamma function} $\Gamma$ will be defined in this book according
to the formula

\begin{equation}
\Gamma(\alpha)=\int_{0}^{\infty}x^{\alpha-1}\me^{-x}\:\diff x,\quad\mbox{for }\alpha>0.\end{equation}

\begin{fact}
Properties of the Gamma Function:
\begin{itemize}
\item $\Gamma(\alpha)=(\alpha-1)\Gamma(\alpha-1)$ for any $\alpha>1$,
and so $\Gamma(n)=(n-1)!$ for any positive integer $n$.
\item $\Gamma(1/2)=\sqrt{\pi}$.
\end{itemize}
\end{fact}

\section{Linear Algebra\label{sec:Linear-Algebra}}


\subsection*{Matrices}

A \emph{matrix} is an ordered array of numbers or expressions; typically
we write $\mathbf{A}=\begin{pmatrix}a_{ij}\end{pmatrix}$ or $\mathbf{A}=\begin{bmatrix}a_{ij}\end{bmatrix}$.
If $\mathbf{A}$ has $m$ rows and $n$ columns then we write\begin{equation}
\mathbf{A}_{\mathrm{m}\times\mathrm{n}}=\begin{bmatrix}a_{11} & a_{12} & \cdots & a_{1n}\\
a_{21} & a_{22} & \cdots & a_{2n}\\
\vdots & \vdots & \ddots & \vdots\\
a_{m1} & a_{m2} & \cdots & a_{mn}\end{bmatrix}.\end{equation}
The \emph{identity matrix} $\mathbf{I}_{\mathrm{n}\times\mathrm{n}}$
is an $\mathrm{n}\times\mathrm{n}$ matrix with zeros everywhere except
for 1's along the main diagonal: \begin{equation}
\mathbf{I}_{\mathrm{n}\times\mathrm{n}}=\begin{bmatrix}1 & 0 & \cdots & 0\\
0 & 1 & \cdots & 0\\
\vdots & \vdots & \ddots & \vdots\\
0 & 0 & \cdots & 1\end{bmatrix}.\end{equation}
and the matrix with ones everywhere is denoted $\mathbf{J}_{\mathrm{n}\times\mathrm{n}}$:\begin{equation}
\mathbf{J}_{\mathrm{n}\times\mathrm{n}}=\begin{bmatrix}1 & 1 & \cdots & 1\\
1 & 1 & \cdots & 1\\
\vdots & \vdots & \ddots & \vdots\\
1 & 1 & \cdots & 1\end{bmatrix}.\end{equation}


A \emph{vector} is a matrix with one of the dimensions equal to one,
such as $\mathbf{A}_{\mathrm{m}\times1}$ (a column vector) or $\mathbf{A}_{\mathrm{1}\times\mathrm{n}}$
(a row vector). The \emph{zero vector} $\mathbf{0}_{\mathrm{n}\times1}$
is an $\mathrm{n}\times1$ matrix of zeros:\begin{equation}
\mathbf{0}_{\mathrm{n}\times1}=\begin{bmatrix}0 & 0 & \cdots & 0\end{bmatrix}^{\mathrm{T}}.\end{equation}


The \emph{transpose} of a matrix $\mathbf{A}=\begin{pmatrix}a_{ij}\end{pmatrix}$
is the matrix $\mathbf{A}^{\mathrm{T}}=\begin{pmatrix}a_{ji}\end{pmatrix}$,
which is just like $\mathbf{A}$ except the rows are columns and the
columns are rows. The matrix $\mathbf{A}$ is said to be \emph{symmetric}
if $\mathbf{A}^{\mathrm{T}}=\mathbf{A}$. Note that $\left(\mathbf{A}\mathbf{B}\right)^{\mathrm{T}}=\mathbf{B}^{\mathrm{T}}\mathbf{A}^{\mathrm{T}}$.

The \emph{trace} of a square matrix $\mathbf{A}$ is the sum of its
diagonal elements: $\mathrm{tr}(\mathbf{A})=\sum_{i}a_{ii}$.

The \emph{inverse} of a square matrix $\mathbf{A}_{\mathrm{n}\times\mathrm{n}}$
(when it exists) is the unique matrix denoted $\mathbf{A}^{-1}$ which
satisfies $\mathbf{A}\mathbf{A}^{-1}=\mathbf{A}^{-1}\mathbf{A}=\mathbf{I}_{\mathrm{n}\times\mathrm{n}}$.
If $\mathbf{A}^{-1}$ exists then we say $\mathbf{A}$ is \emph{invertible},
or alternatively \emph{nonsingular}. Note that $\left(\mathbf{A}^{\mathrm{T}}\right)^{-1}=\left(\mathbf{A}^{\mathrm{-1}}\right)^{\mathrm{T}}$.
\begin{fact}
The inverse of the $2\times2$ matrix\begin{equation}
\mathbf{A}=\begin{bmatrix}a & b\\
c & d\end{bmatrix}\quad\mbox{is}\quad\mathbf{A}^{-1}=\frac{1}{ad-bc}\begin{bmatrix}d & -b\\
-c & a\end{bmatrix},\end{equation}
provided $ad-bc\neq0$.
\end{fact}

\subsection*{Determinants}
\begin{defn}
The \emph{determinant} of a square matrix $\mathbf{A}_{\mathrm{n}\times n}$
is denoted $\mathrm{det}(\mathbf{A})$ or $|\mathbf{A}|$ and is defined
recursively by\begin{equation}
\mathrm{det}(\mathbf{A})=\sum_{i=1}^{n}(-1)^{i+j}a_{ij}\,\mathrm{det}(\mathbf{M}_{ij}),\end{equation}
where $\mathbf{M}_{ij}$ is the submatrix formed by deleting the $i^{\mathrm{th}}$
row and $j^{\mathrm{th}}$ column of $\mathbf{A}$. We may choose
any fixed $1\leq j\leq n$ we wish to compute the determinant; the
final result is independent of the $j$ chosen.\end{defn}
\begin{fact}
The determinant of the $2\times2$ matrix\begin{equation}
\mathbf{A}=\begin{bmatrix}a & b\\
c & d\end{bmatrix}\quad\mbox{is}\quad|\mathbf{A}|=ad-bc.\end{equation}

\end{fact}

\begin{fact}
A square matrix $\mathbf{A}$ is nonsingular if and only if $\mathrm{det}(\mathbf{A})\neq0$.
\end{fact}

\subsection*{Positive (Semi)Definite}

If the matrix $\mathbf{A}$ satisfies $\mathbf{x^{\mathrm{T}}}\mathbf{A}\mathbf{x}\geq0$
for all vectors $\mathbf{x}\neq\mathbf{0}$, then we say that $\mathbf{A}$
is \emph{positive semidefinite}. If strict inequality holds for all
$\mathbf{x}\neq\mathbf{0}$, then $\mathbf{A}$ is \emph{positive
definite}. The connection to statistics is that covariance matrices
(see Chapter \ref{cha:Multivariable-Distributions}) are always positive
semidefinite, and many of them are even positive definite.


\section{Multivariable Calculus\label{sec:Multivariable-Calculus}}


\subsection*{Partial Derivatives}

If $f$ is a function of two variables, its \emph{first-order partial
derivatives} are defined by\begin{equation}
\frac{\partial f}{\partial x}=\frac{\partial}{\partial x}f(x,y)=\lim_{h\to0}\frac{f(x+h,\, y)-f(x,y)}{h}\end{equation}
and\begin{equation}
\frac{\partial f}{\partial y}=\frac{\partial}{\partial y}f(x,y)=\lim_{h\to0}\frac{f(x,\, y+h)-f(x,y)}{h},\end{equation}
provided these limits exist. The \emph{second-order partial derivatives}
of $f$ are defined by\begin{equation}
\frac{\partial^{2}f}{\partial x^{2}}=\frac{\partial}{\partial x}\left(\frac{\partial f}{\partial x}\right),\quad\frac{\partial^{2}f}{\partial y^{2}}=\frac{\partial}{\partial y}\left(\frac{\partial f}{\partial y}\right),\quad\frac{\partial^{2}f}{\partial x\partial y}=\frac{\partial}{\partial x}\left(\frac{\partial f}{\partial y}\right),\quad\frac{\partial^{2}f}{\partial y\partial x}=\frac{\partial}{\partial y}\left(\frac{\partial f}{\partial x}\right).\end{equation}
In many cases (and for all cases in this book) it is true that\begin{equation}
\frac{\partial^{2}f}{\partial x\partial y}=\frac{\partial^{2}f}{\partial y\partial x}.\end{equation}



\subsection*{Optimization}

An function $f$ of two variables has a \emph{local maximum} at $(a,b)$
if $f(x,y)\geq f(a,b)$ for all points $(x,y)$ near $(a,b)$, that
is, for all points in an open disk centered at $(a,b)$. The number
$f(a,b)$ is then called a \emph{local maximum value} of $f$. The
function $f$ has a \emph{local minimum} if the same thing happens
with the inequality reversed.

Suppose the point $(a,b)$ is a \emph{critical point} of $f$, that
is, suppose $(a,b)$ satisfies \begin{equation}
\frac{\partial f}{\partial x}(a,b)=\frac{\partial f}{\partial y}(a,b)=0.\end{equation}
Further suppose $\frac{\partial^{2}f}{\partial x^{2}}$ and $\frac{\partial^{2}f}{\partial y^{2}}$
are continuous near $(a,b)$. Let the \emph{Hessian matrix} $H$ (not
to be confused with the \emph{hat matrix} $\mathbf{H}$ of Chapter
\ref{cha:multiple-linear-regression}) be defined by\begin{equation}
H=\begin{bmatrix}\frac{\partial^{2}f}{\partial x^{2}} & \frac{\partial^{2}f}{\partial x\partial y}\\
\frac{\partial^{2}f}{\partial y\partial x} & \frac{\partial^{2}f}{\partial y^{2}}\end{bmatrix}.\end{equation}
We use the following rules to decide whether $(a,b)$ is an \emph{extremum}
(that is, a local minimum or local maximum) of $f$.
\begin{itemize}
\item If $\mbox{det}(H)>0$ and $\frac{\partial^{2}f}{\partial x^{2}}(a,b)>0$,
then $(a,b)$ is a local minimum of $f$.
\item If $\mbox{det}(H)>0$ and $\frac{\partial^{2}f}{\partial x^{2}}(a,b)<0$,
then $(a,b)$ is a local maximum of $f$.
\item If $\mbox{det}(H)<0$, then $(a,b)$ is a \emph{saddle point} of $f$
and so is not an extremum of $f$.
\item If $\mbox{det}(H)=0$, then we do not know the status of $(a,b)$;
it might be an extremum or it might not be.
\end{itemize}

\subsection*{Double and Multiple Integrals}

Let $f$ be defined on a rectangle $R=[a,b]\times[c,d]$, and for
each $m$ and $n$ divide $[a,b]$ (respectively $[c,d]$) into subintervals
$[x_{j},x_{j+1}]$, $i=0,1,\ldots,m-1$ (respectively $[y_{i},y_{i+1}]$)
of length $\Delta x_{j}=(b-a)/m$ (respectively $\Delta y_{i}=(d-c)/n$)
where $x_{0}=a$ and $x_{m}=b$ (and $y_{0}=c$ and $y_{n}=d$ ),
and let $x_{j}^{\ast}$ ($y_{i}^{\ast}$) be any points chosen from
their respective subintervals. Then the \emph{double integral} of
$f$ over the rectangle $R$ is\begin{equation}
\iintop_{R}f(x,y)\,\diff A=\intop_{c}^{d}\!\!\!\intop_{a}^{b}f(x,y)\,\diff x\diff y=\lim_{m,n\to\infty}\sum_{i=1}^{n}\sum_{j=1}^{m}f(x_{j}^{\ast},y_{i}^{\ast})\Delta x_{j}\Delta y_{i},\end{equation}
provided this limit exists. Multiple integrals are defined in the
same way just with more letters and sums.


\subsection*{Bivariate and Multivariate Change of Variables}

Suppose we have a transformation%
\footnote{For our purposes $T$ is in fact the \emph{inverse} of a one-to-one
transformation that we are initially given. We usually start with
functions that map $(x,y)\longmapsto(u,v)$, and one of our first
tasks is to solve for the inverse transformation that maps $(u,v)\longmapsto(x,y)$.
It is this inverse transformation which we are calling $T$.%
} $T$ that maps points $(u,v)$ in a set $A$ to points $(x,y)$ in
a set $B$. We typically write $x=x(u,v)$ and $y=y(u,v)$, and we
assume that $x$ and $y$ have continuous first-order partial derivatives.
We say that $T$ is \emph{one-to-one} if no two distinct $(u,v)$
pairs get mapped to the same $(x,y)$ pair; in this book, all of our
multivariate transformations $T$ are one-to-one.

The \emph{Jacobian} (pronounced {}``yah-KOH-bee-uhn'') of $T$ is
denoted by $\partial(x,y)/\partial(u,v)$ and is defined by the determinant
of the following matrix of partial derivatives:\begin{equation}
\frac{\partial(x,y)}{\partial(u,v)}=\left|\begin{array}{cc}
\frac{\partial x}{\partial u} & \frac{\partial x}{\partial v}\\
\frac{\partial y}{\partial u} & \frac{\partial y}{\partial v}\end{array}\right|=\frac{\partial x}{\partial u}\frac{\partial y}{\partial v}-\frac{\partial x}{\partial v}\frac{\partial y}{\partial u}.\end{equation}
If the function $f$ is continuous on $A$ and if the Jacobian of
$T$ is nonzero except perhaps on the boundary of $A$, then \begin{equation}
\iint_{B}f(x,y)\,\diff x\,\diff y=\iint_{A}f\left[x(u,v),\, y(u,v)\right]\ \left|\frac{\partial(x,y)}{\partial(u,v)}\right|\diff u\,\diff v.\end{equation}
A multivariate change of variables is defined in an analogous way:
the one-to-one transformation $T$ maps points $(u_{1},u_{2},\ldots,u_{n})$
to points $(x_{1},x_{2},\ldots,x_{n})$, the Jacobian is the determinant
of the $\mathrm{n}\times\mathrm{n}$ matrix of first-order partial
derivatives of $T$ (lined up in the natural manner), and instead
of a double integral we have a multiple integral over multidimensional
sets $A$ and $B$.


\chapter{Writing Reports with \textsf{R\label{cha:Writing-Reports-with}}}

Perhaps the most important part of a statistician's job once the analysis
is complete is to communicate the results to others. This is usually
done with some type of report that is delivered to the client, manager,
or administrator. Other situations that call for reports include term
papers, final projects, thesis work, \emph{etc}. This chapter is designed
to pass along some tips about writing reports once the work is completed
with \textsf{R}.


\section{What to Write\label{sec:What-to-Write}}

It is possible to summarize this entire appendix with only one sentence:
\emph{the statistician's goal is to communicate with others}. To this
end, there are some general guidelines that I give to students which
are based on an outline originally written and shared with me by Dr.~G.~Andy
Chang.


\subsection*{Basic Outline for a Statistical Report}
\begin{enumerate}
\item Executive Summary (a one page description of the study and conclusion) 
\item Introduction 

\begin{enumerate}
\item What is the question, and why is it important?
\item Is the study observational or experimental?
\item What are the hypotheses of interest to the researcher?
\item What are the types of analyses employed? (one sample $t$-test, paired-sample
$t$-test, ANOVA, chi-square test, regression, \ldots{}) 
\end{enumerate}
\item Data Collection 

\begin{enumerate}
\item Describe how the data were collected in detail.
\item Identify all variable types: quantitative, qualitative, ordered or
nominal (with levels), discrete, continuous.
\item Discuss any limitations of the data collection procedure. Look carefully
for any sources of bias.
\end{enumerate}
\item Summary Information 

\begin{enumerate}
\item Give numeric summaries of all variables of interest.

\begin{enumerate}
\item Discrete: (relative) frequencies, contingency tables, odds ratios,
\emph{etc}. 
\item Continuous: measures of center, spread, shape.
\end{enumerate}
\item Give visual summaries of all variables of interest.

\begin{enumerate}
\item Side-by-side boxplots, scatterplots, histograms, \emph{etc}.
\end{enumerate}
\item Discuss any unusual features of the data (outliers, clusters, granularity,
\emph{etc}.)
\item Report any missing data and identify any potential problems or bias.
\end{enumerate}
\item Analysis 

\begin{enumerate}
\item State any hypotheses employed, and check the assumptions. 
\item Report test statistics, \emph{p}-values, and confidence intervals. 
\item Interpret the results in the context of the study.
\item Attach (labeled) tables and/or graphs and make reference to them in
the report as needed. 
\end{enumerate}
\item Conclusion

\begin{enumerate}
\item Summarize the results of the study. What did you learn? 
\item Discuss any limitations of the study or inferences.
\item Discuss avenues of future research suggested by the study. 
\end{enumerate}
\end{enumerate}

\section{How to Write It with \textsf{R\label{sec:How-to-Write}}}

Once the decision has been made what to write, the next task is to
typeset the information to be shared. To do this the author will need
to select software to use to write the documents. There are many options
available, and choosing one over another is sometimes a matter of
taste. But not all software were created equal, and \textsf{R} plays
better with some applications than it does with others.

In short, \textsf{R} does great with \LaTeX{} and there are many resources
available to make writing a document with \textsf{R} and \LaTeX{}
easier. But \LaTeX{} is not for the beginner, and there are other
word processors which may be acceptable depending on the circumstances.


\subsection{Microsoft$\circledR$ Word}

It is a fact of life that Microsoft$\circledR$ Windows is currently
the most prevalent desktop operating system on the planet. Those who
own Windows also typically own some version of Microsoft Office, thus
Microsoft Word is the default word processor for many, many people. 

The standard way to write an \textsf{R} report with Microsoft$\circledR$
Word is to generate material with \textsf{R} and then copy-paste the
material at selected places in a Word document. An advantage to this
approach is that Word is nicely designed to make it easy to copy-and-paste
from \textsf{RGui} to the Word document.

A disadvantage to this approach is that the R input/output needs to
be edited manually by the author to make it readable for others. Another
disadvantage is that the approach does not work on all operating systems
(not on Linux, in particular). Yet another disadvantage is that Microsoft$\circledR$
Word is proprietary, and as a result, \textsf{R} does not communicate
with Microsoft$\circledR$ Word as well as it does with other software
as we shall soon see.

Nevertheless, if you are going to write a report with Word there are
some steps that you can take to make the report more amenable to the
reader.
\begin{enumerate}
\item Copy and paste graphs into the document. You can do this by right
clicking on the graph and selecting \textsf{Copy as bitmap,} or \textsf{Copy
as metafile}, or one of the other options. Then move the cursor to
the document where you want the picture, right-click, and select \textsf{Paste}.
\item Resize (most) pictures so that they take up no more than 1/2 page.
You may want to put graphs side by side; do this by inserting a table
and placing the graphs inside the cells.
\item Copy selected \textsf{R} input and output to the Word document. All
code should be separated from the rest of the writing, except when
specifically mentioning a function or object in a sentence.
\item The font of \textsf{R} input/output should be Courier New, or some
other monowidth font (not Times New Roman or Calibri); the default
font size of \inputencoding{latin9}\lstinline[showstringspaces=false]!12!\inputencoding{utf8}
is usually too big for \textsf{R} code and should be reduced to, for
example, \inputencoding{latin9}\lstinline[showstringspaces=false]!10pt!\inputencoding{utf8}.
\end{enumerate}
It is also possible to communicate with \textsf{R} through OpenOffice.org,
which can export to the proprietary (\inputencoding{latin9}\lstinline[showstringspaces=false]!.doc!\inputencoding{utf8})
format.


\subsection{OpenOffice.org and \texttt{odfWeave}}

OpenOffice.org (OO.o) is an open source desktop productivity suite
which mirrors Microsoft$\circledR$ Office. It is especially nice
because it works on all operating systems. OO.o can read most document
formats, and in particular, it will read \inputencoding{latin9}\lstinline[showstringspaces=false]!.doc!\inputencoding{utf8}
files. The standard OO.o file extension for documents is \inputencoding{latin9}\lstinline[showstringspaces=false]!.odt!\inputencoding{utf8},
which stands for {}``open document text''.

The \inputencoding{latin9}\lstinline[showstringspaces=false]!odfWeave!\inputencoding{utf8}
package \cite{odfweave} provides a way to generate an \inputencoding{latin9}\lstinline[showstringspaces=false]!.odt!\inputencoding{utf8}
file with \textsf{R} input and output code formatted correctly and
inserted in the correct places, without any additional work. In this
way, one does not need to worry about all of the trouble of typesetting
\textsf{R} output. Another advantage of \inputencoding{latin9}\lstinline[showstringspaces=false]!odfWeave!\inputencoding{utf8}
is that it allows you to generate the report dynamically; if the data
underlying the report change or are updated, then a few clicks (or
commands) will generate a brand new report.

One disadvantage is that the source \inputencoding{latin9}\lstinline[showstringspaces=false]!.odt!\inputencoding{utf8}
file is not easy to read, because it is difficult to visually distinguish
the noweb parts (where the \textsf{R} code is) from the non-noweb
parts. This can be fixed by manually changing the font of the noweb
sections to, for instance, Courier font, size \inputencoding{latin9}\lstinline[showstringspaces=false]!10pt!\inputencoding{utf8}.
But it is extra work. It would be nice if a program would discriminate
between the two different sections and automatically typeset the respective
parts in their correct fonts. This is one of the advantages to \LyX{}.

Another advantage of OO.o is that even after you have generated the
outfile, it is fully editable just like any other \inputencoding{latin9}\lstinline[showstringspaces=false]!.odt!\inputencoding{utf8}
document. If there are errors or formatting problems, they can be
fixed at any time.

Here are the basic steps to typeset a statistical report with OO.o.
\begin{enumerate}
\item Write your report as an \inputencoding{latin9}\lstinline[showstringspaces=false]!.odt!\inputencoding{utf8}
document in OO.o just as you would any other document. Call this document
\inputencoding{latin9}\lstinline[showstringspaces=false]!infile.odt!\inputencoding{utf8},
and make sure that it is saved in your working directory.
\item At the places you would like to insert \textsf{R} code in the document,
write the code chunks in the following format:


\texttt{<\textcompwordmark{}<>\textcompwordmark{}>=}~\\
\texttt{x <- rnorm(10)}~\\
\texttt{mean(x)}~\\
\texttt{@}

or write whatever code you want between the symbols \texttt{<\textcompwordmark{}<>\textcompwordmark{}>=}
and \texttt{@}.

\item Open \textsf{R} and type the following:


<<eval = FALSE>>=
library(odfWeave)
odfWeave(file = "infile.odt", dest = "outfile.odt")
@ 

\item The compiled (\inputencoding{latin9}\lstinline[showstringspaces=false]!.odt!\inputencoding{utf8})
file, complete with all of the \textsf{R} output automatically inserted
in the correct places, will now be the file \inputencoding{latin9}\lstinline[showstringspaces=false]!outfile.odt!\inputencoding{utf8}
located in the working directory. Open \inputencoding{latin9}\lstinline[showstringspaces=false]!outfile.odt!\inputencoding{utf8},
examine it, modify it, and repeat if desired.
\end{enumerate}
There are all sorts of extra things that can be done. For example,
the \textsf{R} commands can be suppressed with the tag \texttt{<\textcompwordmark{}<echo
= FALSE>\textcompwordmark{}>=}, and the \textsf{R} output may be hidden
with \texttt{<\textcompwordmark{}<results = hide>\textcompwordmark{}>=}.
See the \inputencoding{latin9}\lstinline[showstringspaces=false]!odfWeave!\inputencoding{utf8}
package documentation for details.


\subsection{Sweave and \protect\LaTeX{}}

This approach is nice because it works for all operating systems.
One can quite literally typeset \emph{anything} with \LaTeX{}. All
of this power comes at a price, however. The writer must learn the
\LaTeX{} language which is a nontrivial enterprise. Even given the
language, if there is a single syntax error, or a single delimeter
missing in the entire document, then the whole thing breaks.

\LaTeX{} can do anything, but it is relatively difficult to learn
and very grumpy about syntax errors and delimiter matching. there
are however programs useful for formatting \LaTeX{}.

A disadvantage is that you cannot see the mathematical formulas until
you run the whole file with \LaTeX{}.

A disadvantage is that figures and tables are relatively difficult.

There are programs to make the process easier AUC\TeX{}

dev.copy2eps, also dev.copy2pdf

\url{http://www.stat.uni-muenchen.de/~leisch/Sweave/}


\subsection{Sweave and \protect\LyX{}}

This approach is nice because it works for all operating systems.
It gives you everything from the last section and makes it easier
to use \LaTeX{}. That being said, it is better to know \LaTeX{} already
when migrating to \LyX{}, because you understand all of the machinery
going on under the hood.

Program Listings and the \textsf{R} language

This book was written with \LyX{}.

\url{http://gregor.gorjanc.googlepages.com/lyx-sweave}


\section{Formatting Tables\label{sec:Formatting-Tables}}

The prettyR package 

the Hmisc package

<<>>=
library(Hmisc)
summary(cbind(Sepal.Length, Sepal.Width) ~ Species, data = iris)
@

There is a method argument to summary, which is set to \inputencoding{latin9}\lstinline[showstringspaces=false]!method = "response"!\inputencoding{utf8}
by default. There are two other methods for summarizing data: reverse
and cross. See \inputencoding{latin9}\lstinline[showstringspaces=false]!?summary.formula!\inputencoding{utf8}
or the following document from Frank Harrell for more details \url{http://biostat.mc.vanderbilt.edu/twiki/bin/view/Main/StatReport}.


\section{Other Formats\label{sec:Other-Formats}}

HTML and prettyR

R2HTML


\chapter{Instructions for Instructors\textsf{\label{cha:Instructions-for-Instructors}}}

\noun{WARNING: this appendix is not applicable until the exercises
have been written.} 

<<echo = FALSE>>=
set.seed(095259)
@

Probably this \emph{book} could more accurately be described as \emph{software}.
The reason is that the document is one big random variable, one observation
realized out of millions. It is electronically distributed under the
GNU FDL, and {}``free'' in both senses: speech and beer. 

There are four components to \IPSUR: the Document, the Program used
to generate it, the \textsf{R} package that holds the Program, and
the Ancillaries that accompany it.

The majority of the data and exercises have been designed to be randomly
generated. Different realizations of this book will have different
graphs and exercises throughout. The advantage of this approach is
that a teacher, say, can generate a unique version to be used in his/her
class. Students can do the exercises and the teacher will have the
answers to all of the problems in their own, unique solutions manual.
Students may download a different solutions manual online somewhere
else, but none of the answers will match the teacher's copy. 

Then next semester, the teacher can generate a \emph{new} book and
the problems will be more or less identical, except the numbers will
be changed. This means that students from different sections of the
same class will not be able to copy from one another quite so easily.
The same will be true for similar classes at different institutions.
Indeed, as long as the instructor protects his/her \emph{key} used
to generate the book, it will be difficult for students to crack the
code. And if they are industrious enough at this level to find a way
to (a) download and decipher my version's source code, (b) hack the
teacher's password somehow, and (c) generate the teacher's book with
all of the answers, then they probably should be testing out of an
{}``Introduction to Probability and Statistics'' course, anyway. 

The book that you are reading was created with a random seed which
was set at the beginning. The original seed is 42. You can choose
your own seed, and generate a new book with brand new data for the
text and exercises, complete with updated manuals. A method I recommend
for finding a seed is to look down at your watch at this very moment
and record the 6 digit hour, minute, and second (say, 9:52:59am):
choose that for a seed%
\footnote{In fact, this is essentially the method used by \textsf{R} to select
an initial random seed (see ?set.seed). However, the instructor should
set the seed manually so that the book can be regenerated at a later
time, if necessary.%
}. This method already provides for over 43,000 books, without taking
military time into account. An alternative would be to go to \textsf{R}
and type 

<<>>=
options(digits = 16)
runif(1)
@

Now choose \Sexpr{set.seed(095259); options(digits = 16); runif(1)*10^16}
as your secret seed\ldots{} write it down in a safe place and do
not share it with anyone. Next generate the book with your seed using
\LyX{}-Sweave or Sweave-\LaTeX{}. You may wish to also generate Student
and Instructor Solution Manuals. Guidance regarding this is given
below in the How to Use This Document section.


\section{Generating This Document\label{sec:Generating-This-Document}}

You will need three (3) things to generate this document for yourself,
in addition to a current \textsf{R} distribution which at the time
of this writing is \texttt{\Sexpr{version$version.string}:}
\begin{enumerate}
\item a \LaTeX{} distribution,
\item Sweave (which comes with \textsf{R} automatically), and 
\item \LyX{} (optional, but recommended).
\end{enumerate}
We will discuss each of these in turn.
\begin{description}
\item [{\LaTeX{}:}] The distribution used by the present author was \TeX{}
Live (\url{http://www.tug.org/texlive/}). There are plenty of other
perfectly suitable \LaTeX{} distributions depending on your operating
system, one such alternative being Mik\TeX{} (\url{http://miktex.org/})
for Microsoft Windows.
\item [{Sweave:}] If you have \textsf{R} installed, then the required Sweave
files are already on your system\ldots{} somewhere. The only problems
that you may have are likely associated with making sure that your
\LaTeX{} distribution knows where to find the \inputencoding{latin9}\lstinline[showstringspaces=false]!Sweave.sty!\inputencoding{utf8}
file. See the Sweave Homepage (\url{http://www.statistik.lmu.de/~leisch/Sweave/})
for guidance on how to get it working on your particular operating
system.
\item [{\LyX{}:}] Strictly speaking, \LyX{} is not needed to generate this
document. But this document was written stem to stern with \LyX{},
taking full advantage of all of the bells and whistles that \LyX{}
has to offer over plain \LaTeX{} editors. And it's free. See the \LyX{}
homepage (\url{http://www.lyx.org/}) for additional information.\\
If you decide to give \LyX{} a try, then you will need to complete
some extra steps to coordinate Sweave and \LyX{} with each other.
Luckily, Gregor Gorjanc has a website and an \textsf{R} News article
\cite{Gorjanc2008} to help you do exactly that. See the \LyX{}-Sweave
homepage (\url{http://gregor.gorjanc.googlepages.com/lyx-sweave})
for details.
\end{description}
An attempt was made to not be extravagant with fonts or packages so
that a person would not need the entire \inputencoding{latin9}\lstinline[showstringspaces=false]!CTAN!\inputencoding{utf8}
(or \inputencoding{latin9}\lstinline[showstringspaces=false]!CRAN!\inputencoding{utf8})
installed on their personal computer to generate the book. Nevertheless,
there are a few extra packages required. These packages are listed
in the \inputencoding{latin9}\lstinline[showstringspaces=false]!preamble!\inputencoding{utf8}
of \inputencoding{latin9}\lstinline[showstringspaces=false]!IPSUR.Rnw!\inputencoding{utf8},
\inputencoding{latin9}\lstinline[showstringspaces=false]!IPSUR.tex!\inputencoding{utf8},
and \inputencoding{latin9}\lstinline[showstringspaces=false]!IPSUR.lyx!\inputencoding{utf8}.


\section{How to Use This Document\label{sec:How-to-Use-Document}}

The easiest way to use this document is to install the \inputencoding{latin9}\lstinline[showstringspaces=false]!IPSUR!\inputencoding{utf8}
package from \inputencoding{latin9}\lstinline[showstringspaces=false]!CRAN!\inputencoding{utf8}
and be all done. This way would be acceptable if there is another,
primary, text being used for the course and \IPSUR\ is only meant
to play a supplementary role.

If you plan for \IPSUR\ to serve as the primary text for your course,
then it would be wise to generate your own version of the document.
You will need the source code for the Program which can be downloaded
from \inputencoding{latin9}\lstinline[showstringspaces=false]!CRAN!\inputencoding{utf8}
or the \IPSUR\ website. Once the source is obtained there are four
(4) basic steps to generating your own copy.
\begin{enumerate}
\item Randomly select a secret {}``seed'' of integers and replace my seed
of 42 with your own seed.
\item Make sure that the \textsf{\textbf{maintext}} branch is turned \textsf{ON}
and also make sure that both the \textsf{\textbf{solutions}} branch
and the \textsf{\textbf{answers}} branch are turned \textsf{OFF}.
Use \LyX{} or your \LaTeX{} editor with Sweave to generate your unique
PDF copy of the book and distribute this copy to your students. (See
the \LyX{} User's Guide to learn more about branches; the ones referenced
above can be found under \textsf{Document $\triangleright$ Settings
$\triangleright$ Branches}.)
\item Turn the \textsf{\textbf{maintext}} branch\textsf{}%
\footnote{You can leave the \textsf{\textbf{maintext}} branch \textsf{ON} when
generating the solutions manuals, but (1) all of the page numbers
will be different, and (2) the typeset solutions will generate and
take up a lot of space between exercises.%
} \textsf{OFF} and the \textsf{\textbf{solutions}} branch \textsf{ON}.
Generate a {}``Student Solutions Manual'' which has complete solutions
to selected exercises and distribute the PDF to the students.
\item Leave the \textsf{\textbf{solutions}} branch \textsf{ON} and also
turn the \textsf{\textbf{answers}} branch \textsf{ON} and generate
an {}``Instructor Solutions and Answers Manual'' with full solutions
to some of the exercises and just answers to the remaining exercises.
Do NOT distribute this to the students -- unless of course you want
them to have the answers to all of the problems. 
\end{enumerate}
To make it easier for those people who do not want to use \LyX{} (or
for whatever reason cannot get it working), I have included three
(3) Sweave files corresponding to the main text, student solutions,
and instructor answers, that are included in the \IPSUR\ source package
in the \texttt{/tex} subdirectory. In principle it is possible to
change the seed and generate the three parts separately with only
Sweave and \LaTeX{}. This method is not recommended by me, but is
perhaps desirable for some people.


\paragraph*{Generating Quizzes and Exams}
\begin{itemize}
\item you can copy paste selected exercises from the text, put them together,
and you have a quiz. Since the numbers are randomly generated you
do not need to worry about different semesters. And you will have
answer keys already for all of your QUIZZES and EXAMS, too.
\end{itemize}

\section{Ancillary Materials\label{sec:Ancillary-Materials}}

In addition to the main text, student manual, and instructor manual,
there is IPSUR.R which is all of the parsed R code used in the document.


\section{Modifying This Document\label{sec:Modifying-This-Document}}

Since this document is released under the GNU-FDL, you are free to
modify this document however you wish (in accordance with the license
-- see Appendix \ref{cha:GNU-Free-Documentation}). The immediate
benefit of this is that you can generate the book, with brand new
problem sets, and distribute it to your students simply as a PDF (in
an email, for instance). As long as you distribute less than 100 such
\emph{Opaque} copies, you are not even required by the GNU-FDL to
share your \emph{Transparent} copy (the source code with the secret
key) that you used to generate them. Next semester, choose a new key
and generate a new copy to be distributed to the new class.
\begin{quotation}
But more generally, if you are not keen on the way I explained (or
failed to explain) something, then you are \underbar{free} to rewrite
it. If you would like to cover more (or less) material, then you are
\underbar{free} to add (or delete) whatever Chapters/Sections/Paragraphs
that you wish. And since you have the source code, you do not need
to retype the wheel.
\end{quotation}
Some individuals will argue that the nature of a statistics textbook
like this one, many of the exercises being randomly generated \emph{by
design}, does a disservice to the students because the exercises do
not use real-world data. That is a valid criticism\ldots{} but in
my case the benefits outweighed the detriments and I moved forward
to incorporate static data sets whenever it was feasible and effective.
Frankly, and most humbly, the only response I have for those individuals
is: {}``Please refer to the preceding paragraph.''


\chapter{\texttt{RcmdrTestDrive} Story\textsf{\label{cha:RcmdrTestDrive-Story}}}

The goal of \inputencoding{latin9}\lstinline[showstringspaces=false]!RcmdrTestDrive!\inputencoding{utf8}
was to have a data set sufficiently rich in the types of data represented
such that a person could load it into the \textsf{R} Commander and
be able to explore all of \inputencoding{latin9}\lstinline[showstringspaces=false]!Rcmdr!\inputencoding{utf8}'s
menu options at once. I decided early-on that an efficient way to
do this would be to generate the data set randomly, and later add
to the list of variables as more \inputencoding{latin9}\lstinline[showstringspaces=false]!Rcmdr!\inputencoding{utf8}
menu options became available. Generating the data was easy, but generating
a \emph{story} that related all of the respective variables proved
to be less so.

In the Summer of 2006 I gave a version of the raw data and variable
names to my STAT 3743 Probability and Statistics class and invited
each of them to write a short story linking all of the variables together
in a coherent narrative. No further direction was given.

The most colorful of those I received was written by Jeffery Cornfield,
submitted July 12, 2006, and is included below with his permission.
It was edited slightly by the present author and updated to respond
dynamically to the random generation of \inputencoding{latin9}\lstinline[showstringspaces=false]!RcmdrTestDrive!\inputencoding{utf8};
otherwise, the story has been unchanged. 


\section*{Case File: ALU-179 {}``Murder Madness in Toon Town”}
\begin{quotation}
\noindent \begin{center}
{*}{*}{*}WARNING{*}{*}{*} 
\par\end{center}

\noindent {*}{*}{*}This file is not for the faint of heart, dear reader,
because it is filled with horrible images that will haunt your nightmares.
If you are weak of stomach, have irritable bowel syndrome, or are
simply paranoid, DO NOT READ FURTHER! Otherwise, read at your own
risk.{*}{*}{*}
\end{quotation}
One fine sunny day, Police Chief R.~Runner called up the forensics
department at Acme-Looney University. There had been \Sexpr{dim(RcmdrTestDrive)[1]-2}
murders in the past \Sexpr{round((dim(RcmdrTestDrive)[1]-2)/24)}
days, approximately one murder every hour, of many of the local Human
workers, shop keepers, and residents of Toon Town. These alarming
rates threatened to destroy the fragile balance of Toon and Human
camaraderie that had developed in Toon Town. 

Professor Twee T.~Bird, a world-renowned forensics specialist and
a Czechoslovakian native, received the call. “Professor, we need your
expertise in this field to identify the pattern of the killer or killers,”
Chief Runner exclaimed. “We need to establish a link between these
people to stop this massacre.”

“Yes, Chief Runner, please give me the details of the case,” Professor
Bird declared with a heavy native accent, (though, for the sake of
the case file, reader, I have decided to leave out the accent due
to the fact that it would obviously drive you -- if you will forgive
the pun -- looney!)

“All prints are wiped clean and there are no identifiable marks on
the bodies of the victims. All we are able to come up with is the
possibility that perhaps there is some kind of alternative method
of which we are unaware. We have sent a secure e-mail with a listing
of all of the victims’ \textbf{races}, \textbf{genders}, locations
of the bodies, and the sequential \textbf{order} in which they were
killed. We have also included other information that might be helpful,”
said Chief Runner.

“Thank you very much. Perhaps I will contact my colleague in the Statistics
Department here, Dr.~Elmer Fudd-Einstein,” exclaimed Professor Bird.
“He might be able to identify a pattern of attack with mathematics
and statistics.” 

“Good luck trying to find him, Professor. Last I heard, he had a bottle
of scotch and was in the Hundred Acre Woods hunting rabbits,” Chief
Runner declared in a manner that questioned the beloved doctor’s credibility.

“Perhaps I will take a drive to find him. The fresh air will do me
good.”
\begin{quotation}
\noindent {*}{*}{*}I will skip ahead, dear reader, for much occurred
during this time. Needless to say, after a fierce battle with a mountain
cat that the Toon-ology Department tagged earlier in the year as “Sylvester,”
Professor Bird found Dr.~Fudd-Einstein and brought him back, with
much bribery of alcohol and the promise of the future slaying of those
“wascally wabbits” (it would help to explain that Dr.~Fudd-Einstein
had a speech impediment which was only worsened during the consumption
of alcohol.){*}{*}{*}
\end{quotation}
Once our two heroes returned to the beautiful Acme-Looney University,
and once Dr.~Fudd-Einstein became sober and coherent, they set off
to examine the case and begin solving these mysterious murders. 

“First off,” Dr.~Fudd-Einstein explained, “these people all worked
at the University at some point or another. Also, there also seems
to be a trend in the fact that they all had a \textbf{salary} between
\$\Sexpr{round(min(RcmdrTestDrive$salary))} and \$\Sexpr{round(max(RcmdrTestDrive$salary))}
when they retired.” 

“That’s not really a lot to live off of,” explained Professor Bird. 

“Yes, but you forget that the Looney Currency System works differently
than the rest of the American Currency System. One Looney is equivalent
to Ten American Dollars. Also, these faculty members are the ones
who faced a cut in their salary, as denoted by ‘\textbf{reduction}’.
Some of them dropped quite substantially when the University had to
fix that little \emph{faux pas} in the Chemistry Department. You remember:
when Dr.~D.~Duck tried to create that ‘Everlasting Elixir?’ As a
result, these faculty left the university. Speaking of which, when
is his memorial service?” inquired Dr.~Fudd-Einstein. 

“This coming Monday. But if there were all of these killings, how
in the world could one person do it? It just doesn’t seem to be possible;
stay up \Sexpr{round((dim(RcmdrTestDrive)[1]-2)/24)} days straight
and be able to kill all of these people and have the energy to continue
on,” Professor Bird exclaimed, doubting the guilt of only one person.

“Perhaps then, it was a group of people, perhaps there was more than
one killer placed throughout Toon Town to commit these crimes. If
I feed in these variables, along with any others that might have a
pattern, the Acme Computer will give us an accurate reading of suspects,
with a scant probability of error. As you know, the Acme Computer
was developed entirely in house here at Acme-Looney University,” Dr.~Fudd-Einstein
said as he began feeding the numbers into the massive server. 

“Hey, look at this,” Professor Bird exclaimed, “What’s with this \textbf{before}/\textbf{after}
information?” 

“Scroll down; it shows it as a note from the coroner’s office. Apparently
Toon Town Coroner Marvin -- that strange fellow from Mars, Pennsylvania
-- feels, in his opinion, that given the fact that the cadavers were
either \textbf{smoke}rs or non-smokers, and given their personal health,
and family medical history, that this was their life expectancy before
contact with cigarettes or second-hand smoke and after,” Dr.~Fudd-Einstein
declared matter-of-factly. 

“Well, would race or gender have something to do with it, Elmer?”
inquired Professor Bird.

“Maybe, but I would bet my money on somebody was trying to quiet these
faculty before they made a big ruckus about the secret money-laundering
of Old Man Acme. You know, most people think that is how the University
receives most of its funds, through the mob families out of Chicago.
And I would be willing to bet that these faculty figured out the connection
and were ready to tell the Looney Police.” Dr.~Fudd-Einstein spoke
lower, fearing that somebody would overhear their conversation.

Dr.~Fudd-Einstein then pressed \textsf{Enter} on the keyboard and
waited for the results. The massive computer roared to life\ldots{}
and when I say roared, I mean it literally \emph{roared}. All the
hidden bells, whistles, and alarm clocks in its secret compartments
came out and created such a loud racket that classes across the university
had to come to a stand-still until it finished computing. 

Once it was completed, the computer listed 4 names:
\begin{quotation}
{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}SUSPECTS{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*} 
\begin{description}
\item [{Yosemite}] Sam (“Looney” Insane Asylum) 
\item [{Wile~E.~Coyote}] (deceased) 
\item [{Foghorn~Leghorn}] (whereabouts unknown) 
\item [{Granny}] (1313 Mockingbird Lane, Toon Town USA)
\end{description}
\end{quotation}
Dr.~Fudd-Einstein and Professor Bird looked on in silence. They could
not believe their eyes. The greatest computer on the Gulf of Mexico
seaboard just released the most obscure results imaginable.

“There seems to be a mistake. Perhaps something is off,” Professor
Bird asked, still unable to believe the results.

“Not possible; the Acme Computer takes into account every kind of
connection available. It considers affiliations to groups, and affiliations
those groups have to other groups. It checks the FBI, CIA, British
intelligence, NAACP, AARP, NSA, JAG, TWA, EPA, FDA, USWA, \textsf{R},
MAPLE, SPSS, SAS, and Ben \& Jerry’s files to identify possible links,
creating the most powerful computer in the world\ldots{} with a tweak
of Toon fanaticism,” Dr.~Fudd-Einstein proclaimed, being a proud
co-founder of the Acme Computer Technology.

“Wait a minute, Ben \& Jerry? What would eating ice cream have to
do with anything?” Professor Bird inquired.

“It is in the works now, but a few of my fellow statistician colleagues
are trying to find a mathematical model to link the type of ice cream
consumed to the type of person they might become. Assassins always
ate vanilla with chocolate sprinkles, a little known fact they would
tell you about Oswald and Booth,” Dr.~Fudd-Einstein declared.

“I’ve heard about this. My forensics graduate students are trying
to identify car thieves with either rocky road or mint chocolate chip…
so far, the pattern is showing a clear trend with chocolate chip,”
Professor Bird declared. 

“Well, what do we know about these suspects, Twee?” Dr.~Fudd-Einstein
asked.

“Yosemite Sam was locked up after trying to rob that bank in the West
Borough. Apparently his guns were switched and he was sent the Acme
Kids Joke Gun and they blew up in his face. The containers of peroxide
they contained turned all of his facial hair red. Some little child
is running around Toon Town with a pair of .38’s to this day.

“Wile E.~Coyote was that psychopath working for the Yahtzee - the
fanatics who believed that Toons were superior to Humans. He strapped
sticks of Acme Dynamite to his chest to be a martyr for the cause,
but before he got to the middle of Toon Town, this defective TNT blew
him up. Not a single other person -- Toon or Human -- was even close.

“Foghorn Leghorn is the most infamous Dog Kidnapper of all times.
He goes to the homes of prominent Dog citizens and holds one of their
relatives for ransom. If they refuse to pay, he sends them to the
pound. Either way, they’re sure stuck in the dog house,” Professor
Bird laughed. Dr.~Fudd-Einstein didn’t seem amused, so Professor
Bird continued. 

“Granny is the most beloved alumnus of Acme-Looney University. She
was in the first graduating class and gives graciously each year to
the university. Without her continued financial support, we wouldn’t
have the jobs we do. She worked as a parking attendant at the University
lots\ldots{} wait a minute, take a look at this,” Professor Bird
said as he scrolled down in the police information. “Granny’s signature
is on each of these faculty members’ \textbf{parking} tickets. Kind
of odd, considering the Chief-of-Parking signed each personally. The
deceased had from as few as \Sexpr{min(RcmdrTestDrive$parking)} ticket
to as many as \Sexpr{max(RcmdrTestDrive$parking)}. All tickets were
unpaid.

“And look at this, Granny married Old Man Acme after graduation. He
was a resident of Chicago and rumored to be a consigliere to one of
the most prominent crime families in Chicago, the Chuck Jones/Warner
Crime Family,” Professor Bird read from the screen as a cold feeling
of terror rose from the pit of his stomach.

“Say, don’t you live at her house? Wow, you’re living under the same
roof as one of the greatest criminals/murderers of all time!” Dr.~Fudd-Einstein
said in awe and sarcasm.

“I would never have suspected her, but I guess it makes sense. She
is older, so she doesn’t need near the amount of sleep as a younger
person. She has access to all of the vehicles so she can copy license
plate numbers and follow them to their houses. She has the finances
to pay for this kind of massive campaign on behalf of the Mob, and
she hates anyone that even remotely smells like smoke,” Professor
Bird explained, wishing to have his hit of nicotine at this time.

“Well, I guess there is nothing left to do but to call Police Chief
Runner and have him arrest her,” Dr.~Fudd-Einstein explained as he
began dialing. “What I can’t understand is how in the world the Police
Chief sent me all of this information and somehow seemed to screw
it up.”

“What do you mean?” inquired Professor Bird.

“Well, look here. The data file from the Chief's email shows \Sexpr{dim(RcmdrTestDrive)[1]}
murders, but there have only been \Sexpr{dim(RcmdrTestDrive)[1]-2}.
This doesn’t make any sense. I’ll have to straighten it out. Hey,
wait a minute. Look at this, Person \#\Sexpr{dim(RcmdrTestDrive)[1]-1}
and Person \#\Sexpr{dim(RcmdrTestDrive)[1]} seem to match our stats.
But how can that be?”

It was at this moment that our two heroes were shot from behind and
fell over the computer, dead. The killer hit \textsf{Delete} on the
computer and walked out slowly (considering they had arthritis) and
cackling loudly in the now quiet computer lab. 

And so, I guess my question to you the reader is, did Granny murder
\Sexpr{dim(RcmdrTestDrive)[1]} people, or did the murderer slip through
the cracks of justice? You be the statistician and come to your own
conclusion. 

Detective Pyork E.~Pig 

{*}{*}{*}End File{*}{*}{*}

\vfill{}


%\bibliographystyle{plainurl}
\cleardoublepage
\phantomsection
\addcontentsline{toc}{chapter}{\bibname}
%\nocite{*} 
%\bibliography{IPSUR}

\bibliographystyle{plainurl}
\nocite{*}
\bibliography{IPSUR}


\vfill{}


\cleardoublepage
\phantomsection
\addcontentsline{toc}{chapter}{\indexname} 
%\printindex

\printindex{}

<<echo=FALSE, results=hide, split=FALSE>>= 
rm(.Random.seed)
# try(dir.create("../../data"), silent = TRUE)
# save.image(file = "../../data/IPSUR.RData")
# tools::resaveRdaFiles('../../data', compress = 'xz')
Stangle(file="IPSUR.Rnw", output="../IPSUR.R", annotate=TRUE)
@
\end{document}
