%last updated: 01/17/2008
\documentclass{seminar}
\usepackage{times,mathptm}
%\usepackage[dvips]{pstcol}
\usepackage{graphics}
\usepackage{epsfig}
\usepackage{color}
\usepackage{graphicx}
\usepackage{comment}
\addtolength{\slidewidth}{2cm} \addtolength{\slideheight}{2cm}

% Makes the slides come out right side up
\special{! TeXDict begin /landplus90{true}store end }

\slideframe{none}

\definecolor{Blue}{rgb}{0.,0.,1.}
\definecolor{Red}{rgb}{1.,0.,0.}
\definecolor{Green}{rgb}{0.,1.,0.}

\def\~{{\tt\char'176}}

%\pagestyle{empty}

\usepackage{graphicx}

\renewcommand{\baselinestretch}{1.48}

\begin{document}

\newcommand{\heading}[1]{%
  \begin{center}
    \Large\bf
    \textcolor{Blue}{#1}
  \end{center}
  \vspace{1ex minus 1ex}}

\newcommand{\bc}{\begin{center}}
\newcommand{\ec}{\end{center}}
\newcommand{\bi}{\begin{itemize}}
\newcommand{\ei}{\end{itemize}}
\newcommand{\bs}{\begin{slide}}
\newcommand{\es}{\end{slide}}

%---------------------------------------------------------------------
% 1 -------------------------------------------------------------------


\begin{slide}
  \heading{Notes for ISyE 6413\\
           Design and Analysis of Experiments}

\vspace{.3 in}

  \begin{center}
{\bf Instructor : C. F. Jeff Wu} \\
School of Industrial and Systems Engineering\\
Georgia Institute of Technology

\vspace{.1 in} \begin{center}{\bf Text book : } \emph{Experiments
: Planning, Analysis, and Optimization}\\ (by Wu and Hamada;
Second Edition, Wiley, 2009)\end{center}
  \end{center}

\vspace{.1 in} {\bf Notes for course instructors:} feel free to
adapt the materials here to suit the needs of your course (latex
files also available).
\end{slide}


%---------------------------------------------------------------------
% 2 -------------------------------------------------------------------


\begin{slide}
  \heading{Unit 1 : Basic Concepts and \\
  Introductory Regresssion Analysis}

\bc Sources : Chapter 1. \ec

\vspace{.1 in}

\bi
 \item Historical perspectives and basic definitions (Section 1.1).
 \item Planning and implementation of experiments (Section 1.2).
 \item Fisher's fundamental principles (Section 1.3).
 \item Simple linear regression (Sections 1.4-1.5).
 \item Multiple regression, variable selection (Sections 1.6-1.7).
 \item Example: Air Pollution Data (Section 1.8).
\ei

\end{slide}

%---------------------------------------------------------------------
% 3 -------------------------------------------------------------------

\begin{slide}
  \heading{Historical perspectives}
\vspace{.2 in}

\bi
 \item {\bf Agricultural Experiments :} Comparisons and
 selection of varieties (and/or treatments) in the presence of
 uncontrollable field conditions, Fisher's pioneering work on
 design of experiments and analysis of variance (ANOVA).

 \item {\bf Industrial Era :} Process modeling and optimization,
 Large batch of materials, large equipments, Box's work motivated in
 chemical industries and applicable to other processing industries, regression
 modeling and response surface methodology.

\ei

\end{slide}

%---------------------------------------------------------------------
% 4 -------------------------------------------------------------------


 \begin{slide}
  \heading{Historical perspectives (Contd.)}
\vspace{.2 in}

\bi
\item {\bf Quality Revolution :} Quality and productivity
 improvement, variation reduction, total quality management,
 Taguchi's work on robust parameter design, Six-sigma movement.

 \item A lot of successful applications in manufacturing (cars, electronics, home
 appliances, etc.)

 \item {\bf Current Trends and Potential New Areas :} Computer
 modelling and experiments, large and complex systems,
 applications to biotechnology, nanotechnology, material
 development, etc.

\ei

\end{slide}

%---------------------------------------------------------------------
% 5 -------------------------------------------------------------------


 \begin{slide}
  \heading{Types of Experiments}
\vspace{.2 in}

\bi

 \item {\bf Treatment Comparisons :} Purpose is to compare several
 treatments of a factor (have 4 rice varieties and would like to see if they
 are different in terms of yield and draught resistence).

 \item {\bf Variable Screening :}  Have a large number of factors,
 but only a few are important.  Experiment should identify the
 important few.

 \item {\bf Response Surface Exploration :}  After important
 factors have been identified, their impact on the system is
 explored; regression model building.

\ei

\end{slide}

%---------------------------------------------------------------------
% 6 -------------------------------------------------------------------


 \begin{slide}
  \heading{Types of Experiments (Contd.)}
\vspace{.2 in}

\bi

\item {\bf System Optimization :}  Interested in determining the
 optimum conditions (e.g., maximize yield of semiconductor
 manufacturing or minimize defects).

\item {\bf System Robustness :} Wish to optimize a system and also
reduce the impact of uncontrollable (noise) factors. (e.g., would
like cars to  run well in different road conditions and different
driving habits; an IC fabrication process to work well in
different conditions of humidity and dust levels).

\ei
\end{slide}

%---------------------------------------------------------------------
% 7 -------------------------------------------------------------------


 \begin{slide}
  \heading{Some Definitions }
\vspace{.2 in}

\bi

 \item {\bf Factor :} variable whose influence upon a response
 variable is being studied in the experiment.
 \item {\bf Factor Level :} numerical values or settings for a factor.
 \item {\bf Trial} (or {\bf run }) :   application of a treatment to an experimental unit.
 \item {\bf Treatment or level combination :} set of values for all factors in a trial.
 \item {\bf Experimental unit :} object to which a treatment is applied.
 \item {\bf Randomization :}  using a chance mechanism to assign
treatments to experimental units or run order.

\ei

\end{slide}

%---------------------------------------------------------------------
% 8 -------------------------------------------------------------------


 \begin{slide}
  \heading{Systematic Approach to Experimentation}
\vspace{.2 in}

\bi
 \item State the objective of the study.
 \item Choose the response variable $\ldots$ should correspond to the
 purpose of the study.
  \bi
  \item {\small Nominal-the-best, larger-the-better or smaller-the-better}.
  \ei

 \item Choose factors and levels.
    \bi \item {\small Use flow chart or cause-and-effect diagram (see Figure 1).}\ei

 \item Choose experimental design (i.e., plan).
 \item Perform the experiment (use a planning matrix to determine the set of
treatments and the order to be run).
 \item Analyze data (design should be selected to meet objective so that the analysis is efficient and
 easy).
 \item Draw conclusions.
\ei
\end{slide}

%---------------------------------------------------------------------
% 9 -------------------------------------------------------------------

 \begin{slide}
  \heading{Cause-and Effect Diagram}
\vspace{0.3 in}
\begin{figure}[htbp]
\begin{center}
{
% Control the size of graph
\unitlength .15mm \tiny
\begin{picture}(400,400) (-25,-200)
%CENTERLINE
\put(350,0){\vector(-1,0){325}} \put(-25,5){COSMETIC}
\put(-25,-11){DEFECTS}
%EQUIPMENT
\put(60,0){\line(1,3){75}} \put(135,250){MACHINE}
\put(135,225){\line(1,0){40}} \put(140,230){injection pressure}
\put(125,195){\line(1,0){40}} \put(130,200){injection speed}
\put(115,165){\line(1,0){40}} \put(120,170){mold temperature}
\put(105,135){\line(1,0){40}} \put(110,140){nozzle temperature}
\put(95,105){\line(1,0){40}} \put(100,110){holding time}
\put(85,75){\line(1,0){40}} \put(90,80){barrel temperature}
\put(75,45){\line(1,0){40}} \put(80,50){material shot length}
%REAGENT
\put(200,0){\line(1,3){35}} \put(235,130){MATERIAL}
\put(235,105){\line(1,0){40}} \put(240,110){pre-blend
pigmentation} \put(225,75){\line(1,0){40}} \put(230,80){humidity}
\put(215,45){\line(1,0){40}} \put(220,50){melt flow}
%SAMPLE
\put(60,0){\line(1,-3){35}} \put(100,-125){METHOD}
\put(95,-105){\line(1,0){40}} \put(100,-100){purge}
\put(85,-75){\line(1,0){40}} \put(90,-70){hopper cleaning}
\put(75,-45){\line(1,0){40}} \put(80,-40){screw \& barrel
cleaning}
%REAGENT
\put(200,0){\line(1,-3){35}} \put(240,-125){MAN}
\put(235,-105){\line(1,0){40}} \put(240,-100){operators on shifts}
\put(225,-75){\line(1,0){40}} \put(230,-70){operator replacements}
\put(215,-45){\line(1,0){40}} \put(220,-40){lack of training}
\end{picture}
}
\end{center}
\caption{Cause-and-Effect Diagram, Injection Molding Experiment}
\label{fig:fishbone}
\end{figure}

\end{slide}
%---------------------------------------------------------------------
% 10 -------------------------------------------------------------------


 \begin{slide}
  \heading{Choice of Response : An Example}
\vspace{.2 in}

\bi
 \item To improve a process that often produces underweight soap
 bars. \\Obvious choice of response, $y$ = soap bar weight.
 \item There are two sub-processes : (i) mixing, which affects soap
 bar density (=$y_1$), (ii) forming, which affects soap bar
 dimensions (=$y_2$).
 \item Even though $y$ is a function of $y_1$ and $y_2$, better to
 study $y_1$ and $y_2$ separately and identify factors important
 for each of the two sub-processes.
\ei
\end{slide}

%---------------------------------------------------------------------
% 11 -------------------------------------------------------------------


 \begin{slide}
  \heading{Fundamental Principles : Replication, randomization, and blocking}
\vspace{.2 in}

{\large \textcolor[rgb]{0.00,0.00,1.00}{Replication}} \bi
 \item Each treatment is applied to units that are
 representative of the population (example : measurements of 3
 units vs. 3 repeated measurements of 1 unit).
 \item Replication vs Repetition (i.e., repeated measurements).
 \item Enable the estimation of experimental error. Use sample
 standard deviation.
 \item Decrease variance of estimates and increase the power
 to detect significant differences : for independent $y_i$'s,
 \begin{eqnarray*}
 Var(\frac{1}{N}\sum_{i=1}^N y_i)= \frac{1}{N} Var(y_1).
 \end{eqnarray*} \ei
\end{slide}

%---------------------------------------------------------------------
% 12 -------------------------------------------------------------------


 \begin{slide}
  \heading{Randomization}
\vspace{.2 in}

Use of a chance mechanism (e.g., random number generators) to
assign treatments to units or to run order. It has the following
advantages.

\bi
 \item Protect against latent variables or ``lurking" variables
 (give an example).
 \item Reduce influence of subjective bias in treatment
 assignments (e.g., clinical trials).
 \item Ensure validity of statistical inference (This is more
 technical; will not be discussed in the book. See Chapter 4 of
 ``{\em Statistics for Experimenters}'' by Box, Hunter, Hunter for discussion
 on randomization distribution.)
 \ei
\end{slide}

%---------------------------------------------------------------------
% 13 -------------------------------------------------------------------


 \begin{slide}
  \heading{Blocking}
\vspace{.2 in}

A {\bf block} refers to a collection of homogeneous units.
Effective blocking : larger between-block variations than
within-block variations. \\(Examples: hours, batches, lots, street
blocks, pairs of twins.)

\bi
 \item Run and compare treatments within the same blocks. (Use
 randomization within blocks.) It can eliminate block-block
 variation and reduce variability of treatment effects estimates.
 \item {\bf Block what you can and randomize what you cannot.}
 \item Discuss {\bf typing experiment} to demonstrate possible elaboration of the blocking
 idea. See next page.
 \ei

\end{slide}


%---------------------------------------------------------------------
% 14 -------------------------------------------------------------------

 \begin{slide}
  \heading{Illustration: Typing Experiment}

 \bi
  \item To compare two keyboards $A$ and $B$ in terms of typing
  efficiency. Six manuscripts 1-6 are given to the same typist.
  \item Several designs (i.e., orders of test sequence) are
  considered:
   \begin{enumerate}
    \item \begin{eqnarray*}
1.\; A, B,\;\; 2.\; A, B,\;\; 3.\; A, B,\;\; 4.\; A, B,\;\; 5.\;
A, B,\;\; 6.\; A, B.
\end{eqnarray*}
    ($A$ always followed by $B$, why bad ?)
    \item Randomizing the order leads to a new sequence like this
    \begin{eqnarray*}
1.\; A, B,\;\; 2.\; B, A,\;\; 3.\; A, B,\;\; 4.\; B, A,\;\; 5.\;
A, B,\;\; 6.\; A , B.
\end{eqnarray*}
 (an improvement, but there are four with $A,B$ and two with
 $B,A$. Why is this not desirable? Impact of \emph{learning effect}.)
 \item \emph{Balanced randomization}: To mitigate the learning effect,
 randomly choose three with $A,B$ and three with $B,A$. (Produce
 one such plan on your own).
 \item Other improved plans?
   \end{enumerate}




 \ei




 \end{slide}
%---------------------------------------------------------------------
% 15 -------------------------------------------------------------------

\bs
  \heading{Simple Linear Regression : Mortality Data}

\vspace{.1 in}

The data, taken from certain regions of Great Britain, Norway, and
Sweden contains the mean annual temperature (in degrees F) and
mortality index for neoplasms of the female breast.
\begin{table}[h]
\small \centering
\begin{tabular}{lrrrrrrrr}
\hline Mortality rate ($M$) & 102.5 & 104.5 & 100.4 & 95.9 & 87.0
& 95.0 & 88.6 & 89.2 \\
Temperature ($T$) & 51.3 & 49.9 & 50.0 & 49.2 & 48.5 & 47.8 & 47.3
& 45.1 \\ \hline Mortality rate ($M$) & 78.9 & 84.6 & 81.7 & 72.2
& 65.1 & 68.1 & 67.3 & 52.5 \\
Temperature ($T$) & 46.3 & 42.1 & 44.2 & 43.5 & 42.3 & 40.2 & 31.8 & 34.0 \\
 \hline
\end{tabular}\label{tabsimplereg1}
\end{table}

 {\bf Objective :}  Obtaining the relationship between mean annual
temperature and the mortality rate for a type of breast cancer in
women.

 \es

%---------------------------------------------------------------------
% 16 -------------------------------------------------------------------


\bs
  \heading{Getting Started}

\bc

\begin{figure}[tb]
\includegraphics[height=2.7in,width=2.7in,angle=0]{ch1_Mortalitydata.ps}\\
\caption{Scatter Plot of Temperature versus Mortality Rate, Breast
Cancer Data.}
\end{figure}

\ec

\es


%---------------------------------------------------------------------
% 17 -------------------------------------------------------------------


\bs
  \heading{Fitting the Regression Line}

\bi \item Underlying Model : $$y = \beta_0 + \beta_1x+\epsilon,
\hspace{.1 in} \epsilon\sim N(0,\sigma^2).$$
 \item Coefficients are estimated by minimizing $$\sum_{i=1}^N \Big(y_i-(\beta_0+\beta_1x_i)\Big)^2.$$
 \item {\bf Least Squares Estimates} \\
 Estimated Coefficients :
   \begin{eqnarray*}
   \hat{\beta}_1 = \frac{\sum(x_i-\bar{x})(y_i-\bar{y})}{\sum(x_i-\bar{x})^2} &,& var\big(\hat{\beta}_1\big) = \frac{\sigma^2}{\sum(x_i - \bar{x})^2},\\
   \hat{\beta}_0 = \bar{y}-\hat{\beta}_1\bar{x} &,&
   var{\big(\hat{\beta}_0\big)}= \sigma^2\bigg(\frac{1}{N}+\frac{\bar{x}^2}{\sum(x_i -
   \bar{x})^2)}\bigg),\\
   \bar{x}=\frac{1}{N}\sum{x_i} &,&    \bar{y}=\frac{1}{N}\sum{y_i}. \\
   \end{eqnarray*}
 \ei

 \es

%---------------------------------------------------------------------
% 18 -------------------------------------------------------------------

\bs
  \heading{Explanatory Power of the Model}
\bi
 \item The total variation in $y$ can be measured by corrected total sum of squares $CTSS= \sum_{i=1}^N (y_i - \bar{y})^2.$
 \item This can be decomposed into two parts (Analysis of Variance
 (ANOVA)):
         $$ CTSS = RegrSS + RSS, $$
where
$$
 RegrSS = \mbox{Regression sum of squares} = \sum_{i=1}^N (\hat{y}_i -
 \bar{y})^2,$$
 $$RSS =  \mbox{Residual sum of squares} = \sum_{i=1}^N (y_i - \hat{y}_i)^2.$$
  $\hat{y}_i= \hat{\beta}_0  + \hat{\beta}_1x_i$ is
called the predicted value of $y_i$ at $x_i$.
 \item $R^2 = \frac{RegrSS}{CTSS} = 1 - \frac{RSS}{CTSS}$ measures the proportion of
 variation in $y$ explained by the fitted model.
 \ei \es


%---------------------------------------------------------------------
% 19 -------------------------------------------------------------------
\bs
  \heading{ANOVA Table for Simple Linear Regression}
%\bi

\bc {ANOVA Table for Simple Linear Regression} \ec
\begin{table}[htbp]
%\caption{ANOVA Table for General Linear Model}
$ $\\
\large\small \centering
\begin{tabular} {|llll|}
\hline \multicolumn{1}{|c}{} & \multicolumn{1}{c}{Degrees of} &
\multicolumn{1}{c}{Sum of} &
\multicolumn{1}{c|}{Mean}  \\
\multicolumn{1}{|c}{Source} & \multicolumn{1}{c}{Freedom} &
\multicolumn{1}{c}{Squares} & \multicolumn{1}{c|}{Squares}  \\
\hline regression & $1$ & $\hat{\beta}_1\sum(x_i - \bar{x})^2$ &
 $\hat{\beta}_1\sum(x_i - \bar{x})^2$ \\
residual & $N-2$ & $\sum_{i=1}^N (y_i - \hat{y}_i)^2$ &
$\frac{\sum_{i=1}^N (y_i - \hat{y}_i)^2}{(N-2)}$ \\
total (corrected) & $N-1$& $\sum_{i =1}^{N}(y_{i} - \bar{y})^{2}$ & \\
\hline
\end{tabular}
\label{tab:ANOVA} \large\normalsize
\end{table}

\bc {ANOVA Table for Breast Cancer Example} \ec
\begin{table}[htbp]
%\caption{ANOVA Table for General Linear Model}
$ $\\
\large\small \centering
\begin{tabular} {|llll|}
\hline \multicolumn{1}{|c}{} & \multicolumn{1}{c}{Degrees of} &
\multicolumn{1}{c}{Sum of} &
\multicolumn{1}{c|}{Mean}  \\
\multicolumn{1}{|c}{Source} & \multicolumn{1}{c}{Freedom} &
\multicolumn{1}{c}{Squares} & \multicolumn{1}{c|}{Squares}  \\
\hline regression & $1$ & $2599.53$ & $2599.53$\\
residual & $14$ & $796.91$ & $56.92$ \\
total (corrected) & $15$& $3396.44$ & \\
\hline
\end{tabular}
\label{tab:ANOVA} \large\normalsize
\end{table}

% \ei
\es
%---------------------------------------------------------------------
% 20 -------------------------------------------------------------------

\bs
 \heading{$t$-Statistic}
 \bi
 \item To test the null hypothesis $H_0 : \beta_j = 0$
against the alternative hypothesis $H_0 : \beta_j \ne 0$, use the
test statistic
$$t_j=\frac{\hat{\beta}_j}{s.d.(\hat{\beta}_j)}.$$
 \item The higher the value of $t$, the more significant is the coefficient.
 \item For 2-sided alternatives, $p$-value $=$ Prob$\Big(|t_{df}| > |t_{obs}|\Big)$,
 df = degrees of freedom for the $t$-statistic, $t_{obs}$ = observed value of the $t$-statistic.
 If $p$-value is very small, then either we have observed something
 which rarely happens, or $H_0$ is not true.
 In practice, if $p$-value is less then $\alpha = 0.05$ or $0.01$, $H_0$ is
 rejected at level $\alpha$.
\ei

 \es


%---------------------------------------------------------------------
% 21 -------------------------------------------------------------------

\bs
 \heading{Confidence Interval}
\bc
    $100(1-\alpha)\%$ confidence interval for $\beta_j$ is given
    by $$\hat{\beta}_j \pm t_{N-2,\frac{\alpha}{2}} \times s.d.(\hat{\beta}_j), $$
 \ec

 where $t_{N-2,\frac{\alpha}{2}}$ is the upper $\alpha/2$
 point of the $t$ distribution with $N-2$ degrees of freedom.

If the confidence interval for $\beta_j$ does not contain 0, then
$H_0$ is rejected. \es


%---------------------------------------------------------------------
% 22 -------------------------------------------------------------------

\bs
 \heading{Predicted Values and Residuals}
 \vspace{0.2 in}
 \bi
   \item $\hat{y}_i = \hat{\beta}_0  + \hat{\beta}_1x_i $ is the
   predicted value of $y_i$ at $x_i$.
   \item $r_i = y_i - \hat{y}_i$ is the corresponding residual.
   \item Standardized residuals are defined as
   $\frac{r_i}{s.d.(r_i)}$.
   \item Plots of residuals are extremely useful to judge the
   ``goodness'' of fitted model.
   \bi
     \item Normal probability plot (will be explained in Unit 2).
     \item Residuals versus predicted values.
     \item Residuals versus covariate $x$.
   \ei
 \ei

\es


%---------------------------------------------------------------------
% 23 -------------------------------------------------------------------

\bs
 \heading{Analysis of Breast Cancer Data}

{\scriptsize
\begin{verbatim}
 The regression equation is
 M = - 21.79 + 2.36 T

 Predictor        Coef     SE Coef          T        P
 Constant       -21.79       15.67      -1.39    0.186
 T              2.3577      0.3489       6.76    0.000

 S = 7.54466     R-Sq = 76.5%      R-Sq(adj) = 74.9%

 Analysis of Variance

 Source            DF          SS          MS         F        P
 Regression         1      2599.5      2599.5     45.67    0.000
 Residual Error    14       796.9        56.9
 Total             15      3396.4

 Unusual Observations
 Obs          T          M         Fit      SE Fit    Residual    St Resid
  15       31.8      67.30       53.18        4.85       14.12        2.44RX

 R denotes an observation with a large standardized residual.
 X denotes an observation whose X value gives it large leverage.
\end{verbatim}
}

\es


%---------------------------------------------------------------------
% 24 -----------------------------------------------------------------
\bs
 \heading{Outlier Detection}
 \vspace{0.1 in}
 \bi
\item Minitab identifies two types of outliers denoted by R and
X:\\
          R: its standardized residual $(y_i-\hat{y}_i)/se(\hat{y}_i)$ is
    large.\\
          X: its X value gives large leverage (i.e., far away from majority
    of the X values).

\item For the mortality data, the observation with T = 31.8, M =
67.3 (i.e., left most point in plot on p. 16) is identified as
both R and X.

\item After removing this outlier and refitting the remaining
data, the output is given on p. 27. There is still an �outlier�
identified as X but not R. This one (second left most point on
p.16) should not be removed (why?)

\item Residual plots on p. 28 show no systematic pattern. \ei

Notes: Outliers are not discussed in the book, see standard
regression texts. Residual plots will be discussed in unit 2.


\es

%---------------------------------------------------------------------
% 25 -----------------------------------------------------------------

\bs
 \heading{Prediction from the Breast Cancer Data}

  \bi
    \item The fitted regression model is $Y = -21.79 + 2.36X$,
    where $Y$ denotes the mortality rate and $X$ denotes the
    temperature.
    \item The predicted mean of $Y$ at $X = x_0$ can be obtained from the above model.
    For example, prediction for the temperature of 49 is obtained by substituting $x_{0}=49$, which gives
    $y_{x_{0}}=93.85$.
    \item The standard error of $y_{x_{0}}$ is given by
$$ S.E.(y_{x_{0}}) = \hat{\sigma} \sqrt{\frac{1}{N}+ \frac{(\bar{x} - x_0)^{2}}{\sum_{i = 1}^{N}(x_{i} - \bar{x})^{2} }}.$$

    \item Here $x_{0} = 49$, $1/N+ (\bar{x} - x_0)^{2}/ \sum_{i = 1}^{N}(x_{i} - \bar{x})^{2} = 0.1041$, and $\hat{\sigma} = \sqrt{MSE} = 7.54$.
Consequently, $S.E.(y_{x_{0}}) = 2.432$.
  \ei

\normalsize \es

%\item Note that in the above formula, $\bf{X}$ is a $16 \times
%2$ matrix, whose first column consists of 1, and the second
%column consists of the $X$ observations, i.e. $51.3, 49.9, \ldots,
%34.0$.
%---------------------------------------------------------------------
% 26-------------------------------------------------------------------

\bs \heading{Confidence interval for mean and prediction interval
for individual observation}

\small

 \bi
 \item A 95\% confidence interval for the mean response $\beta_0  + \beta_1 x_0$ of $y$ at $x = x_0$
 is
 $$
\hat{\beta}_0  + \hat{\beta}_1x_0  \pm t_{N-2,0.025} \times
S.E.(y_{x_{0}}).$$

\item Here the 95\% confidence interval for the mean mortality
corresponding to a temperature of 49 is [88.63, 99.07].

\item A 95\%  prediction interval for an individual observation
$y_{x_{0}}$ corresponding to $x = x_0$
 is
 $$
\hat{\beta}_0  + \hat{\beta}_1x_0  \pm t_{N-2,0.025} \hat{\sigma}
\sqrt{1+\frac{1}{N}+ \frac{(\bar{x} - x_0)^{2}}{\sum_{i =
1}^{N}(x_{i} - \bar{x})^{2} }}, $$ where 1 under the square root
represents $\sigma^2$, variance of the \emph{new} observation
$y_{x_{0}}$.

 \item The 95\% prediction interval for the predicted mortality of an individual corresponding to the
temperature of 49 is [76.85, 110.85].

\ei
 \es


%---------------------------------------------------------------------
% 27 -------------------------------------------------------------------
\bs
 \heading{Regression Results after Removing the Outlier}

{\scriptsize
\begin{verbatim}
 The regression equation is
 M = - 52.62 + 3.02 T

 Predictor        Coef     SE Coef          T        P
 Constant       -52.62       15.82      -3.33    0.005
 T              3.0152      0.3466       8.70    0.000

 S = 5.93258     R-Sq = 85.3%      R-Sq(adj) = 84.2%

 Analysis of Variance

 Source            DF          SS          MS         F        P
 Regression         1      2664.3      2664.3     75.70    0.000
 Residual Error    13       457.5        35.2
 Total             14      3121.9

 Unusual Observations
 Obs          T          M         Fit      SE Fit    Residual    St Resid
  15       34.0      52.50       49.90        4.25        2.60        0.63 X

 X denotes an observation whose X value gives it large leverage.

\end{verbatim}
}

\es

%---------------------------------------------------------------------
% 28 -----------------------------------------------------------------


\bs
  \heading{Residual Plots After Outlier Removal}

\bc
\begin{tabular}{cc}
\psfig{figure=pic2_fitted_res.ps,height=2in,width=2in,angle=0} & \psfig{figure=pic2_temp_res.ps,height=2.in,width=2in,angle=0} \\
\end{tabular}
 Figure 3: Residual Plots\\
\ec
 {\bf Comments :} No systematic pattern is discerned.

\es


%---------------------------------------------------------------------
% 29 -------------------------------------------------------------------


\bs
  \heading{Multiple Linear Regression : Air Pollution Data}

\vspace{.1 in}

{\small\begin{verbatim}
http://lib.stat.cmu.edu/DASL/Stories/AirPollutionandMortality.html
\end{verbatim}}

\bi
 \item Data collected by General Motors.
 \item Response is age-adjusted mortality.
 \item Predictors : \bi
                    {\small
                    \item Variables measuring demographic
                    characteristics.
                    \item Variables measuring climatic
                    characteristics.
                    \item Variables recording pollution potential of 3 air pollutants.
                    }
                    \ei
 \item Objective : To determine whether air pollution is
 significantly related to mortality.
\ei

 \es





%---------------------------------------------------------------------
% 30 -------------------------------------------------------------------


\bs
  \heading{Predictors}

{\scriptsize
 \begin{enumerate}
 \item {\bf JanTemp :} Mean January temperature (degrees Farenheit)
 \item {\bf JulyTemp :} Mean July temperature (degrees Farenheit)
 \item {\bf RelHum :} Relative Humidity
 \item {\bf Rain :} Annual rainfall (inches)
 \item {\bf Education :} Median education
 \item {\bf PopDensity :} Population density
 \item {\bf \%NonWhite :} Percentage of non whites
 \item {\bf \%WC :} Percentage of white collar workers
 \item {\bf pop :} Population
 \item {\bf pop/house :} Population per household
 \item {\bf income :} Median income
 \item {\bf HCPot :} HC pollution potential
 \item {\bf NOxPot :} Nitrous Oxide pollution potential
 \item {\bf SO2Pot :} Sulphur Dioxide pollution potential
 \end{enumerate}
}

\es




%---------------------------------------------------------------------
% 31 -------------------------------------------------------------------


\bs
  \heading{Getting Started}

\vspace{0.2 in}
 \bi
  \item There are 60 data points.
  \item Pollution variables are highly skewed, log transformation makes them nearly symmetric.
  The variables HCPot, NOxPot and SO2Pot are replaced by log(HCPot), log(NOxPot) and
  log(SO2Pot).
  \item Observation~21 (Fort Worth, TX) has two missing values, so this data point
  will be discarded from the analysis.
\ei

 \es



%---------------------------------------------------------------------
% 32 -------------------------------------------------------------------


\bs
  \heading{Scatter Plots}

\bc \small Figure 4: Scatter Plots of mortality against selected
predictors
\begin{tabular}{cc}
{\footnotesize (a) JanTemp} &  {\footnotesize (b) Education}\\
\psfig{figure=pic4.ps,height=1.25in,width=1.5in,angle=0} & \psfig{figure=pic5.ps,height=1.25in,width=1.5in,angle=0} \\
{\footnotesize (c) NonWhite} & (d) {\footnotesize Log(NOxPot)} \\
\psfig{figure=pic6.ps,height=1.25in,width=1.5in,angle=0} & \psfig{figure=pic7.ps,height=1.25in,width=1.5in,angle=0} \\
\end{tabular}
 \ec


\es





%---------------------------------------------------------------------
% 33 -------------------------------------------------------------------


\bs
  \heading{Fitting the Multiple Regression Equation}


\bi

\item Underlying Model : $$y = \beta_0 +
\beta_1x_1+\beta_2x_2+\ldots + \beta_kx_k+ \epsilon, \hspace{.1
in} \epsilon\sim N(0,\sigma^2).$$


 \item Coefficients are estimated by minimizing $$\sum_{i=1}^N \Big(y_i-(\beta_0 +
\beta_1x_{i1}+\beta_2x_{i2}+\ldots + \beta_kx_{ik})\Big)^2 = ({\bf
y - X\beta)^\prime  (y - X\beta)}.$$

\item {\bf Least Squares estimates :} $${\bf \hat{\beta} =
(X^\prime X)^{-1}X^\prime y}.$$

 \item Variance-Covariance matrix of $\hat{\beta}$ : $\Sigma_{\hat{\beta}} = \sigma^2({\bf X^\prime
 X})^{-1}.$
 \ei

\es




%---------------------------------------------------------------------
% 34 -------------------------------------------------------------------


\bs
  \heading{Analysis of Variance}
 \bi
 \item The total variation in $y$, i.e., corrected total sum of squares, $CTSS= \sum_{i=1}^N (y_i - \bar{y})^2 = {\bf y}^{T}{\bf y} -N {\bar{y}}^2 $, can be decomposed  into two parts (Analysis of Variance (ANOVA)):
         $$ CTSS = RegrSS + RSS, $$
 where $RSS = \mbox{Residual sum of squares} = \sum(y_i - \hat{y}_i)^2 = ({\bf
 y-X\hat{\beta})^T(y-X\hat{\beta}})$,
       $RegrSS = \mbox{Regression sum of squares} = \sum_{i=1}^N (\hat{y}_i - \bar{y})^2 = \hat{\mbox{\boldmath $\beta$ }}^{T}
{\bf X}^{T} {\bf X} \hat{\mbox{\boldmath $\beta$ }} - N
{\bar{y}}^2$.

\bc {ANOVA Table} \ec
\begin{table}[htbp]
%\caption{ANOVA Table for General Linear Model}
$ $\\
\large\small \centering
\begin{tabular} {|llll|}
\hline \multicolumn{1}{|c}{} & \multicolumn{1}{c}{Degrees of} &
\multicolumn{1}{c}{Sum of} &
\multicolumn{1}{c|}{Mean}  \\
\multicolumn{1}{|c}{Source} & \multicolumn{1}{c}{Freedom} &
\multicolumn{1}{c}{Squares} & \multicolumn{1}{c|}{Squares}  \\
\hline regression & $k$ & $\hat{\mbox{\boldmath $\beta$ }}^{T}
{\bf X}^{T} {\bf X} \hat{\mbox{\boldmath $\beta$ }} - N
{\bar{y}}^2$ &
 (${\hat{\mbox{\boldmath $\beta$ }}^{T} {\bf X}^{T} {\bf X} \hat{\mbox{\boldmath $\beta$ }} - N {\bar{y}}^2})/{k}$\\
residual & $N-k-1$ & $({\bf y} - {\bf X} \hat{\mbox{\boldmath
$\beta$ }})^{T} ({\bf y} - {\bf X} \hat{\mbox{\boldmath $\beta$
}})$ &
${{({\bf y} - {\bf X} \hat{\mbox{\boldmath $\beta$ }})}^{T} ({\bf y} - {\bf X} \hat{\mbox{\boldmath $\beta$ }})}/{(N-k-1)}$ \\
total & $N-1$& ${\bf y}^{T}{\bf y} -N {\bar{y}}^2$ & \\
(corrected) &&&   \\
\hline
\end{tabular}
\label{tab:ANOVA} \large\normalsize
\end{table}

\ei \es




%---------------------------------------------------------------------
% 35 -------------------------------------------------------------------


\bs
  \heading{Explanatory Power of the Model}
 \bi
 \item $R^2 = \frac{RegrSS}{CTSS} = 1 - \frac{RSS}{CTSS}$ measures of the proportion of variation in $y$ explained by the fitted
 model. $R$ is called the multiple correlation coefficient.
 \item {\bf Adjusted $R^2$ : } $$R^2_a = 1 -\frac{\frac{RSS}{N-(k+1)}}{\frac{CTSS}{N-1}}= 1 -\left(\frac{N-1}{N-k-1}\right)\frac{RSS}{CTSS}.$$
 \item When an additional predictor is included in the regression model, $R^2$ always increases. This is not a desirable property for model selection. However,  $R^2_a$ may decrease if the included variable is not an informative
 predictor. Usually $R^2_a$ \emph{is a better measure of model fit}.
 \ei
\es



%---------------------------------------------------------------------
% 36 -------------------------------------------------------------------




\bs
 \heading{Testing significance of coefficients : $t$-Statistic}
 \bi
 \item To test the null hypothesis $H_0 : \beta_j = 0$
against the alternative hypothesis $H_0 : \beta_j \ne 0$, use the
test statistic
$$t_j=\frac{\hat{\beta}_j}{s.d.(\hat{\beta}_j)}.$$
 \item The higher the value of $t$, the more significant is the coefficient.
 \item %For 2-sided alternatives, $p$-value $=$ Prob$\Big(|t_{df}| > |t_{obs}|\Big)$,
% df = degrees of freedom for the $t$-statistic, $t_{obs}$ = observed value of the $t$-statistic.
 %If $p$-value is very small, then either we have observed something
 %which rarely happens, or $H_0$ is not true.
 In practice, if $p$-value is less then $\alpha = 0.05$ or $0.01$, $H_0$ is
 rejected.
\item  {\bf Confidence Interval :}    $100(1-\alpha)\%$ confidence
interval for $\beta_j$ is given
    by $$\hat{\beta}_j \pm t_{N-(k+1),\frac{\alpha}{2}} \times s.d.(\hat{\beta}_j), $$
 where $t_{N-k-1,\frac{\alpha}{2}}$ is the upper $\alpha/2$
 point of the $t$ distribution with $N-k-1$ degrees of freedom.

If the confidence interval for $\beta_j$ does not contain 0, then
$H_0$ is rejected.
\ei

\es

%---------------------------------------------------------------------
% 37 -------------------------------------------------------------------

\bs
 \heading{Analysis of Air Pollution Data}

{\scriptsize
\begin{verbatim}
 Predictor        Coef     SE Coef          T        P
 Constant       1332.7       291.7       4.57    0.000
 JanTemp       -2.3052      0.8795      -2.62    0.012
 JulyTemp       -1.657       2.051      -0.81    0.424
 RelHum          0.407       1.070       0.38    0.706
 Rain           1.4436      0.5847       2.47    0.018
 Educatio       -9.458       9.080      -1.04    0.303
 PopDensi     0.004509    0.004311       1.05    0.301
 %NonWhit        5.194       1.005       5.17    0.000
 %WC            -1.852       1.210      -1.53    0.133
 pop        0.00000109  0.00000401       0.27    0.788
 pop/hous       -45.95       39.78      -1.16    0.254
 income      -0.000549    0.001309      -0.42    0.677
 logHC          -53.47       35.39      -1.51    0.138
 logNOx          80.22       32.66       2.46    0.018
 logSO2          -6.91       16.72      -0.41    0.681

 S = 34.58       R-Sq = 76.7%     R-Sq(adj) = 69.3%

 Analysis of Variance

 Source            DF          SS          MS         F        P
 Regression        14      173383       12384     10.36    0.000
 Residual Error    44       52610        1196
 Total             58      225993
\end{verbatim}
} \es


%---------------------------------------------------------------------
% 38 -------------------------------------------------------------------

\bs
 \heading{Variable Selection Methods}
\bi
 \item {\bf Principle of Parsimony (Occam's razor):} Choose fewer variables with
 sufficient explanatory power. This is a desirable modeling
 strategy.
 \item The goal is thus to identify the smallest subset of
 covariates that provides good fit. One way of achieving this is to retain the significant
 predictors in the fitted multiple regression. This may not work
 well if some variables are strongly correlated among themselves or if there are too many variables (e.g., exceeding the sample
 size).
  \item Two other possible strategies are
    \bi
     \item { Best subset regression using Mallows' $C_p$
     statistic.}
     \item { Stepwise regression.}
     \ei
\ei

\es

%---------------------------------------------------------------------
% 39 -------------------------------------------------------------------

\bs
 \heading{Best Subset Regression}
 \bi
 \item For a model with $p$ regression coefficients, (i.e.,
 $p-1$ covariates  plus the intercept $\beta_0$), define its
 $C_p$ value as \\
 $$C_p=\frac{RSS}{s^2}-(N-2p),$$ where $RSS=$ residual sum of
 squares for the given model, $s^2=$ mean square error = $\frac{\mbox {\emph{RSS} (for the complete model)}} {\mbox{df (for the complete
 model)}}$, $N =$ number of observations.
 \item If the model is true, then $E(C_p)\approx p$. Thus one should
 choose $p$ by picking models whose $C_p$ values are low and close to
 $p$. For the same $p$, \emph{choose a model with the smallest $C_p$
 value} (i.e., the smallest RSS value).
 \ei

\es

%---------------------------------------------------------------------
% 40 -------------------------------------------------------------------

\bs
 \heading{AIC and BIC Information Criteria}
 \bi
 \item The Akaike information criterion (AIC) is defined by \\
 $$AIC=Nln(\frac{RSS}{N})+2p$$
\item The Bayes information criterion (BIC) is defined by \\
 $$BIC=Nln(\frac{RSS}{N})+pln(N)$$
 \item In choosing a model with the AIC/ BIC criterion, we choose the
 model that minimizes the criterion value.
 \item Unlike the $C_p$ criterion, the AIC criterion is applicable even if
 the number of observations do not allow the complete model to be
 fitted.
 \item The BIC criterion favors smaller models more than the AIC
 criterion.

 \ei

\es

%---------------------------------------------------------------------
% 41 -------------------------------------------------------------------

\bs
 \heading{Stepwise Regression}
\bi
 \item This method involves adding or dropping one variable at a
 time from a given model based on a {\it partial} $F$-statistic.

Let the smaller and bigger models be Model I and Model II,
respectively. The partial $F$-statistic is defined as
\begin{eqnarray*} \frac{RSS(Model\;I) -
RSS(Model\;II)}{RSS(Model\;II)/\nu}, \label{eqn:PartialFStat}
\end{eqnarray*}
where $\nu$ is the degrees of freedom of the $RSS$ (residual sum
of squares) for Model II.

 \item There are three possible ways
    \begin{enumerate}
    \item {\small {\bf Backward elimination :}  starting with the full model and removing covariates.}
    \item {\small {\bf Forward selection :}   starting with the intercept and adding one variable at a time.}
    \item {\small {\bf Stepwise selection :}  alternate backward elimination and forward selection.}
    \end{enumerate}
   Usually stepwise selection is recommended.
\ei

\es


%---------------------------------------------------------------------
% 42 -------------------------------------------------------------------

\bs
 \heading{Criteria for Inclusion and Exclusion of Variables}

 \bi
\item {\bf $F$-to-remove : } At each step of backward elimination,
compute the partial $F$ value for each covariate being considered
for removal. The one with the lowest partial $F$, provided it is
smaller than a preselected value, is dropped. The procedure
continues until no more covariates can be dropped. The preselected
value is often chosen to be $F_{1,\nu,\alpha}$, the upper $\alpha$
critical value of the $F$ distribution with 1 and $\nu$ degrees of
freedom. Typical $\alpha$ values range from 0.1 to 0.2.

\item {\bf $F$-to-enter : } At each step of forward selection, the
covariate with the largest partial $F$ is added, provided it is
larger than a preselected $F$ critical value, which is referred to
as an {\em $F$-to-enter} value.

\item For stepwise selection, the $F$-to-remove and $F$-to-enter
values should be chosen to be the same.

 \ei

(See Section~1.7)

\es

%---------------------------------------------------------------------
% 43 -------------------------------------------------------------------

\bs
 \heading{Air Pollution Example: Best Subsets Regression}

 {\scriptsize
    \begin{verbatim}
 Vars  R-Sq   R-Sq(adj)    C-p     BIC        S   variables
   4   69.7       67.4     8.3     608     35.6   1,4,7,13
   5   72.9       70.3     4.3     606     34.0   1,4,5,7,13
   6   74.2       71.3     3.7     607     33.5   1,4,6,7,8,13
   7   75.0       71.6     4.3     609     33.3   1,4,6,7,8,12,13
   8   75.4       71.5     5.4     612     33.3   1,4,5,7,8,10,12,13
    \end{verbatim}
 }
\bc

 \ec


\es

%---------------------------------------------------------------------
% 44 -------------------------------------------------------------------

\bs
 \heading{Pollution Data Analysis - Stepwise Regression}
{\scriptsize
    \begin{verbatim}
  Stepwise Regression: Mortality versus JanTemp, JulyTemp, ...


   Alpha-to-Enter: 0.15  Alpha-to-Remove: 0.15

  Response is Mortality on 14 predictors, with N =   59
  N(cases with missing observations) =   1 N(all cases) =   60


     Step          1        2        3        4        5        6        7
 Constant      887.9   1208.5   1112.7   1135.4   1008.7   1029.5   1028.7

 %NonWhit       4.49     3.92     3.92     4.73     4.36     4.15     4.15
 T-Value        6.40     6.26     6.81     7.32     6.73     6.60     6.66
 P-Value       0.000    0.000    0.000    0.000    0.000    0.000    0.000

 Educatio               -28.6    -23.5    -21.1    -14.1    -15.6    -15.5
 T-Value                -4.32    -3.74    -3.47    -2.10    -2.40    -2.49
 P-Value                0.000    0.000    0.001    0.041    0.020    0.016

 logSO2                           28.0     21.0     26.8     -0.4
 T-Value                          3.37     2.48     3.11    -0.02
 P-Value                         0.001    0.016    0.003    0.980
    \end{verbatim}
 }

\es
%---------------------------------------------------------------------
% 45 -------------------------------------------------------------------

\bs
 \heading{Pollution Data Analysis - Stepwise Regression (Contd)}
{\scriptsize
    \begin{verbatim}
   Alpha-to-Enter: 0.15  Alpha-to-Remove: 0.15

  Response is Mortality on 14 predictors, with N =   59
  N(cases with missing observations) =   1 N(all cases) =   60

 JanTemp                                  -1.42    -1.29    -2.15    -2.14
 T-Value                                  -2.41    -2.26    -3.25    -4.17
 P-Value                                  0.019    0.028    0.002    0.000

 Rain                                               1.08     1.66     1.65
 T-Value                                            2.15     3.07     3.16
 P-Value                                           0.036    0.003    0.003

 logNOx                                                        42       42
 T-Value                                                     2.35     4.04
 P-Value                                                    0.023    0.000

 S              48.0     42.0     38.5     37.0     35.8     34.3     34.0
 R-Sq          41.80    56.35    63.84    67.36    69.99    72.86    72.86
 R-Sq(adj)     40.78    54.80    61.86    64.94    67.16    69.73    70.30
 C-p            55.0     29.5     17.4     12.7      9.7      6.3      4.3
    \end{verbatim}
 }

\es

%---------------------------------------------------------------------
% 46 -------------------------------------------------------------------

\bs
 \heading{Final Model}
 {\bf Rival Models}

 \bc
 \begin{tabular}{|c|l|cc|l|}
 \hline
  & Variables & $C_p$ & BIC & Remarks \\
  \hline
 Model~1 & 1,4,6,7,8,13 & 3.7 & 607 & Minimum $C_p$\\
 Model~2 & 1,4,5,7,13   & 4.3 & 606 & Minimum BIC and chosen by stepwise\\
 \hline
 \end{tabular}
 \ec

We shall analyze data with Model~2. (Why? Refer to the rules on
page 38 and use the principle of parsimony.) \es

%---------------------------------------------------------------------
% 47 -------------------------------------------------------------------

\bs
 \heading{Analysis of Model~2}
{\scriptsize
    \begin{verbatim}
Predictor     Coef  SE Coef      T      P
Constant   1028.67    80.96  12.71  0.000
JanTemp    -2.1384   0.5122  -4.17  0.000
Rain        1.6526   0.5225   3.16  0.003
Education  -15.542    6.235  -2.49  0.016
%NonWhite   4.1454   0.6223   6.66  0.000
logNOx       41.67    10.32   4.04  0.000

S = 34.0192   R-Sq = 72.9%   R-Sq(adj) = 70.3%

Analysis of Variance

Source          DF      SS     MS      F      P
Regression       5  164655  32931  28.45  0.000
Residual Error  53   61337   1157
Total           58  225993
    \end{verbatim}
 }

\es

%---------------------------------------------------------------------
% 48 -------------------------------------------------------------------

\bs
 \heading{Residual Plot}
 \bc
\begin{tabular}{c}
\psfig{figure=pic9.ps,height=2.5in,width=3in,angle=0}  \\
{\scriptsize Residual versus Fitted Values}\\
Figure 6 : Plot of Residuals
\end{tabular}
\ec

\es



%---------------------------------------------------------------------
% last ---------------------------------------------------------------

\bs
  \heading{Comments on Board}

 \vspace{3 in} \es




\end{document}
