My DS Coding Bolg

Thursday, November 5, 2015

R Basics 6 - Matrices and Arrays

Context

Matices and arrays are an extension on R's atomic vecotrs.
Atomic vectors contain values (not objects).
They hold a contiguous selt of values, all of which are the same basic type. There are six types of atomic vecotr: logical, integer, numeric, complex, caracter and raw.

Importantly: atomic vectors have no dimension attribute.
Matrices and arrays are effectively vectors with a dimension attribute. 
Matrices are two-dimensional(tabular) objects, containing values all of the same type (unlike data frames).
Arrays are multi-dimensional objects(typically with three plus dimensions), with values all of the same type.

Matrix versus data.frame
In a matrix, every column, and every cell is of the same basic atomic type. 
In a data.frame each column can be of a different type(eg. numeric, character, factor). Data frames are best with messy data, and for variables of mixed models.

Matrix creation
## generalCase <- matrix(data=NA, nrow=1, ncol=1, byrow=FALSE, dimnames=NULL)
> M <- matrix(c(2,1,3,4,5,6), nrow=3, byrow=TRUE); M
     [,1] [,2]
[1,]    2    1
[2,]    3    4
[3,]    5    6
> b <- matrix(c(0, -1, 4)); b
     [,1]
[1,]    0
[2,]   -1
[3,]    4
> I <- diag(3); I
     [,1] [,2] [,3]
[1,]    1    0    0
[2,]    0    1    0
[3,]    0    0    1
> D <- diag(c(1,2,3)); D
     [,1] [,2] [,3]
[1,]    1    0    0
[2,]    0    2    0
[3,]    0    0    3
> d <- diag(M); d
[1] 2 4

Basic information about a matrix
> dim(M)
[1] 3 2
> class(M)
[1] "matrix"
> is.matrix(M)
[1] TRUE
> is.array(M)
[1] TRUE
> is.atomic(M)
[1] TRUE
> is.vector(M)
[1] FALSE
> is.list(M)
[1] FALSE
> is.factor(M)
[1] FALSE
> is.recursive(M)
[1] FALSE
> nrow(M)
[1] 3
> ncol(M)
[1] 2
> length(M)
[1] 6
> rownames(M)
NULL
> colnames(M)
NULL

Matrix manipulation
> M <- matrix(c(2,1,3,4,5,6), nrow=3, byrow=TRUE); M
     [,1] [,2]
[1,]    2    1
[2,]    3    4
[3,]    5    6
> N <- matrix(c(6,5,4,3,2,1), nrow=3, byrow=TRUE); N
     [,1] [,2]
[1,]    6    5
[2,]    4    3
[3,]    2    1
> newM <- cbind(M, N); newM
     [,1] [,2] [,3] [,4]
[1,]    2    1    6    5
[2,]    3    4    4    3
[3,]    5    6    2    1
> newM <- rbind(M, N); newM
     [,1] [,2]
[1,]    2    1
[2,]    3    4
[3,]    5    6
[4,]    6    5
[5,]    4    3
[6,]    2    1
> v <- c(M); v
[1] 2 3 5 1 4 6
> df <- data.frame(M); df
  X1 X2
1  2  1
2  3  4
3  5  6

Matrix multiplication
> M
     [,1] [,2]
[1,]    2    1
[2,]    3    4
[3,]    5    6
> N
     [,1] [,2]
[1,]    6    5
[2,]    4    3
[3,]    2    1
> InnerProduct <- M %*% t(N); InnerProduct
     [,1] [,2] [,3]
[1,]   17   11    5
[2,]   38   24   10
[3,]   60   38   16
> OuterProduct <- M %o% N; OuterProduct
, , 1, 1

     [,1] [,2]
[1,]   12    6
[2,]   18   24
[3,]   30   36

, , 2, 1

     [,1] [,2]
[1,]    8    4
[2,]   12   16
[3,]   20   24

, , 3, 1

     [,1] [,2]
[1,]    4    2
[2,]    6    8
[3,]   10   12

, , 1, 2

     [,1] [,2]
[1,]   10    5
[2,]   15   20
[3,]   25   30

, , 2, 2

     [,1] [,2]
[1,]    6    3
[2,]    9   12
[3,]   15   18

, , 3, 2

     [,1] [,2]
[1,]    2    1
[2,]    3    4
[3,]    5    6

> CrossProduct <- crossprod(M, N); CrossProduct
     [,1] [,2]
[1,]   34   24
[2,]   34   23
> M * N
     [,1] [,2]
[1,]   12    5
[2,]   12   12
[3,]   10    6


Matrix maths
> rowMeans(M)
[1] 1.5 3.5 5.5
> colMeans(M)
[1] 3.333333 3.666667
> rowSums(M)
[1]  3  7 11
> colSums(M)
[1] 10 11
> t <- t(M);t
     [,1] [,2] [,3]
[1,]    2    3    5
[2,]    1    4    6
> inverse <- solve(diag(c(1,2,3))); inverse
     [,1] [,2]      [,3]
[1,]    1  0.0 0.0000000
[2,]    0  0.5 0.0000000
[3,]    0  0.0 0.3333333
> e <- eigen(diag(c(1,2,3))); e
$values
[1] 3 2 1

$vectors
     [,1] [,2] [,3]
[1,]    0    0    1
[2,]    0    1    0
[3,]    1    0    0

> d <- det(diag(c(1,2,3))); d
[1] 6

Matrix indexing [row, col] [[row, col]]
# [[ for single cell selection; [ for multi cell selection
# indexed by positive numbers: these ones
# indexed by negative numbers: not these
# indexed by logical atomic vector: in/out
# named rows/cols can be indexed by name
# M[i] or M[[i]] is vector-like indexing
# $ operator is invalid for atomic vectors
# M[r,]
# M[,c]

Arrays

A three dimensional array created in two steps:

> A <- 1:8;A
[1] 1 2 3 4 5 6 7 8
> dim(A) <- c(2,2,2);A
, , 1

     [,1] [,2]
[1,]    1    3
[2,]    2    4

, , 2

     [,1] [,2]
[1,]    5    7
[2,]    6    8

A matrix is a special case of array. Matrices are arrays with two dimensions.
> M <- array(1:9, dim=c(3,3));M
     [,1] [,2] [,3]
[1,]    1    4    7
[2,]    2    5    8
[3,]    3    6    9

R Basics 5 - Data Frames

Create data frame

- The R way of doing spreadsheets

- Internally, a data.frame is a list of equal length vectors or factors.

- Observations in rows; Variables in cols

empty <-data.frame()

> empty <-data.frame()

> c1 <- 1:10

> c2 <- letters[1:10]

> df <- data.frame(col1=c1, col2=c2)

> df

col1 col2

1 1 a

2 2 b

3 3 c

4 4 d

5 5 e

6 6 f

7 7 g

8 8 h

9 9 i

10 10 j

Import from and export to file

d2 <- read.csv('fileName.csv', header = TRUE)

library(gdata);

d2 <- read.xls('file.xls')

write.csv(df, file='fileName.csv')

print(xtable(df), type='html')

Basic infomrmation about the data frame

> is.data.frame(df)

[1] TRUE

> class(df)

[1] "data.frame"

> nrow(df)

[1] 10

> ncol(df)

[1] 2

> colnames(df);

[1] "col1" "col2"

> rownames(df);

[1] "1" "2" "3" "4" "5" "6" "7" "8" "9" "10"

Referencing cells [row, col] [[r, c]]

## [[ for single cell selection;

# [ for multi cell selection;

> vec <- df[[5,2]]; vec

[1] e

Levels: a b c d e f g h i j

> newDF <- df[1:5, 1:2]; newDF

col1 col2

1 1 a

2 2 b

3 3 c

4 4 d

5 5 e

> df[[2, 'col1']]

[1] 2

> df[3:5, c('col1', 'col2')]

col1 col2

3 3 c

4 4 d

5 5 e

Referencing rows [r, ]

# returns a data frame ( and not a vecotr! )

> row.1 <- df[1,]; row.1

col1 col2

1 1 a

> row.n <- df[nrow(df),]; row.n

col1 col2

10 10 j

> vrow <- as.numeric(as.vector(df[1,])); vrow

[1] 1 1

> vrow <- as.character(as.vector(df[1,])); vrow

[1] "1" "1"

Referencing columns [,c] [d] [[d]] $col

> names(df) <- c('num','cats')

> col.vec <- df$cats; col.vec

[1] a b c d e f g h i j

Levels: a b c d e f g h i j

> # returns vector

> col.vec <- df[, 'cats'] ; col.vec

[1] a b c d e f g h i j

Levels: a b c d e f g h i j

> # a is int or string

> col.vec <- df[ , 2]; col.vec

[1] a b c d e f g h i j

Levels: a b c d e f g h i j

> # returns a vector

> col.vec <- df[['cats']]; col.vec

[1] a b c d e f g h i j

Levels: a b c d e f g h i j

> # returns 1 col df

> frog.df <- df['cats']

> # returns 1 col df

> first.df <- df[1]; first.df

num

1 1

2 2

3 3

4 4

5 5

6 6

7 7

8 8

9 9

10 10

> first.col <- df[,1]; first.col

[1] 1 2 3 4 5 6 7 8 9 10

> # returns a vector

> last.col <- df[,ncol(df)]; last.col

[1] a b c d e f g h i j

Levels: a b c d e f g h i j

Adding rows
# The right way ... (both args are DFs)
df <- rbind(df, data.frame(num=1, cats='A')); df

Adding columns
> df$newCol <- rep(NA, nrow(df)); df
col1 col2 newCol
1 1 a NA
2 2 b NA
3 3 c NA
4 4 d NA
5 5 e NA
6 6 f NA
7 7 g NA
8 8 h NA
9 9 i NA
10 10 j NA
> #Copy a column
> df[, 'copyofCol'] <- 1:nrow(df); df
col1 col2 newCol copyofCol
1 1 a NA 1
2 2 b NA 2
3 3 c NA 3
4 4 d NA 4
5 5 e NA 5
6 6 f NA 6
7 7 g NA 7
8 8 h NA 8
9 9 i NA 9
10 10 j NA 10
> names(df) <- c('x','cats','newCol','y')
> df$y.percent.pf.x <- df$y/sum(df$x)*100; df
x cats newCol y y.percent.pf.x
1 1 a NA 1 1.818182
2 2 b NA 2 3.636364
3 3 c NA 3 5.454545
4 4 d NA 4 7.272727
5 5 e NA 5 9.090909
6 6 f NA 6 10.909091
7 7 g NA 7 12.727273
8 8 h NA 8 14.545455
9 9 i NA 9 16.363636
10 10 j NA 10 18.181818
> df <-cbind(col=rep('a',nrow(df)), df); df
col x cats newCol y y.percent.pf.x
1 a 1 a NA 1 1.818182
2 a 2 b NA 2 3.636364
3 a 3 c NA 3 5.454545
4 a 4 d NA 4 7.272727
5 a 5 e NA 5 9.090909
6 a 6 f NA 6 10.909091
7 a 7 g NA 7 12.727273
8 a 8 h NA 8 14.545455
9 a 9 i NA 9 16.363636
10 a 10 j NA 10 18.181818
> df <- cbind(df,col=rep('b',nrow(df))); df
col x cats newCol y y.percent.pf.x col
1 a 1 a NA 1 1.818182 b
2 a 2 b NA 2 3.636364 b
3 a 3 c NA 3 5.454545 b
4 a 4 d NA 4 7.272727 b
5 a 5 e NA 5 9.090909 b
6 a 6 f NA 6 10.909091 b
7 a 7 g NA 7 12.727273 b
8 a 8 h NA 8 14.545455 b
9 a 9 i NA 9 16.363636 b
10 a 10 j NA 10 18.181818 b
> df$c3 <- with(df, col3 <- x*y); df
col x cats newCol y y.percent.pf.x col c3
1 a 1 a NA 1 1.818182 b 1
2 a 2 b NA 2 3.636364 b 4
3 a 3 c NA 3 5.454545 b 9
4 a 4 d NA 4 7.272727 b 16
5 a 5 e NA 5 9.090909 b 25
6 a 6 f NA 6 10.909091 b 36
7 a 7 g NA 7 12.727273 b 49
8 a 8 h NA 8 14.545455 b 64
9 a 9 i NA 9 16.363636 b 81
10 a 10 j NA 10 18.181818 b 100
> transform(df, col4 <- x+y)
col x cats newCol y y.percent.pf.x col c3
1 a 1 a NA 1 1.818182 b 1
2 a 2 b NA 2 3.636364 b 4
3 a 3 c NA 3 5.454545 b 9
4 a 4 d NA 4 7.272727 b 16
5 a 5 e NA 5 9.090909 b 25
6 a 6 f NA 6 10.909091 b 36
7 a 7 g NA 7 12.727273 b 49
8 a 8 h NA 8 14.545455 b 64
9 a 9 i NA 9 16.363636 b 81
10 a 10 j NA 10 18.181818 b 100

Set column names # same for rownames()
> colnames(df) <- c('date', 'alpha', 'beta'); df
date alpha beta NA NA NA NA NA
1 a 1 a NA 1 1.818182 b 1
2 a 2 b NA 2 3.636364 b 4
3 a 3 c NA 3 5.454545 b 9
4 a 4 d NA 4 7.272727 b 16
5 a 5 e NA 5 9.090909 b 25
6 a 6 f NA 6 10.909091 b 36
7 a 7 g NA 7 12.727273 b 49
8 a 8 h NA 8 14.545455 b 64
9 a 9 i NA 9 16.363636 b 81
10 a 10 j NA 10 18.181818 b 100
> colnames(df)[1] <- 'new.name'; df
new.name alpha beta NA NA NA NA NA
1 a 1 a NA 1 1.818182 b 1
2 a 2 b NA 2 3.636364 b 4
3 a 3 c NA 3 5.454545 b 9
4 a 4 d NA 4 7.272727 b 16
5 a 5 e NA 5 9.090909 b 25
6 a 6 f NA 6 10.909091 b 36
7 a 7 g NA 7 12.727273 b 49
8 a 8 h NA 8 14.545455 b 64
9 a 9 i NA 9 16.363636 b 81
10 a 10 j NA 10 18.181818 b 100
> colnames(df)[colnames(df) %in% c('a', 'b')] <- c('x', 'y'); df
new.name alpha beta NA NA NA NA NA
1 a 1 a NA 1 1.818182 b 1
2 a 2 b NA 2 3.636364 b 4
3 a 3 c NA 3 5.454545 b 9
4 a 4 d NA 4 7.272727 b 16
5 a 5 e NA 5 9.090909 b 25
6 a 6 f NA 6 10.909091 b 36
7 a 7 g NA 7 12.727273 b 49
8 a 8 h NA 8 14.545455 b 64
9 a 9 i NA 9 16.363636 b 81
10 a 10 j NA 10 18.181818 b 100

Selecting Multiple Rows
> firstTenRows <- df[1:10,]; firstTenRows
new.name alpha beta NA NA NA NA NA
1 a 1 a NA 1 1.818182 b 1
2 a 2 b NA 2 3.636364 b 4
3 a 3 c NA 3 5.454545 b 9
4 a 4 d NA 4 7.272727 b 16
5 a 5 e NA 5 9.090909 b 25
6 a 6 f NA 6 10.909091 b 36
7 a 7 g NA 7 12.727273 b 49
8 a 8 h NA 8 14.545455 b 64
9 a 9 i NA 9 16.363636 b 81
10 a 10 j NA 10 18.181818 b 100
> everthingButRowTwo <- df[-2,]; everthingButRowTwo
new.name alpha beta NA NA NA NA NA
1 a 1 a NA 1 1.818182 b 1
3 a 3 c NA 3 5.454545 b 9
4 a 4 d NA 4 7.272727 b 16
5 a 5 e NA 5 9.090909 b 25
6 a 6 f NA 6 10.909091 b 36
7 a 7 g NA 7 12.727273 b 49
8 a 8 h NA 8 14.545455 b 64
9 a 9 i NA 9 16.363636 b 81
10 a 10 j NA 10 18.181818 b 100
> sub <- df[(df$x >5 & y<5), ]; sub
[1] new.name alpha beta <NA> <NA> <NA> <NA> <NA>
<0 rows> (or 0-length row.names)
> sub <- subset(df, x>5 & y<5); sub
[1] new.name alpha beta <NA> NA.1 NA.2 NA.3 NA.4
<0 rows> (or 0-length row.names)
> notLastRow <- head(df, -1); notLastRow
new.name alpha beta NA NA NA NA NA
1 a 1 a NA 1 1.818182 b 1
2 a 2 b NA 2 3.636364 b 4
3 a 3 c NA 3 5.454545 b 9
4 a 4 d NA 4 7.272727 b 16
5 a 5 e NA 5 9.090909 b 25
6 a 6 f NA 6 10.909091 b 36
7 a 7 g NA 7 12.727273 b 49
8 a 8 h NA 8 14.545455 b 64
9 a 9 i NA 9 16.363636 b 81
> df[-nrow(df),]
new.name alpha beta NA NA NA NA NA
1 a 1 a NA 1 1.818182 b 1
2 a 2 b NA 2 3.636364 b 4
3 a 3 c NA 3 5.454545 b 9
4 a 4 d NA 4 7.272727 b 16
5 a 5 e NA 5 9.090909 b 25
6 a 6 f NA 6 10.909091 b 36
7 a 7 g NA 7 12.727273 b 49
8 a 8 h NA 8 14.545455 b 64
9 a 9 i NA 9 16.363636 b 81

Selecting multiple columns
> df <- df[,c(1,2,3,4,5)]; df
col x cats newCol y
1 a 1 a NA 1
2 a 2 b NA 2
3 a 3 c NA 3
4 a 4 d NA 4
5 a 5 e NA 5
6 a 6 f NA 6
7 a 7 g NA 7
8 a 8 h NA 8
9 a 9 i NA 9
10 a 10 j NA 10
> names(df) <- c('col1', 'col2', 'col3')
> df <- df[,c('col1','col2')];df
col1 col2
1 a 1
2 a 2
3 a 3
4 a 4
5 a 5
6 a 6
7 a 7
8 a 8
9 a 9
10 a 10
df <- df[,-1]; df
# drop col1 and col3
df <- df[,-c(1,3)]
could not find function "colnmaes"
> df <- df[,!(colnames(df) %in% c('notThis','norThis'))]
> df
col1 col2
1 a 1
2 a 2
3 a 3
4 a 4
5 a 5
6 a 6
7 a 7
8 a 8
9 a 9
10 a 10

Replace column elements by row selection
> df
col1 col2
1 a 1
2 a 2
3 a 3
4 a 4
5 a 5
6 a 6
7 a 7
8 a 8
9 a 9
10 a 10
> df[df$col31 == 'a', 'col2'] <- 1
> df
col1 col2
1 a 1
2 a 2
3 a 3
4 a 4
5 a 5
6 a 6
7 a 7
8 a 8
9 a 9
10 a 10
> df[df$col1 == 'a', 'col2'] <- 1
> df
col1 col2
1 a 1
2 a 1
3 a 1
4 a 1
5 a 1
6 a 1
7 a 1
8 a 1
9 a 1
10 a 1

Missing data(NA)
# detect anywhere in df
> any(is.na(df))
[1] TRUE
> # anywhere in col
> any(is.na(df$newCol))
[1] FALSE
> # deleting selecting missing row
> df2 <- df[!is.na(df$newCol),]; df2
col1 col2 newCol col
1 a NA 0 0
2 a NA 0 0
3 a NA 0 0
4 a NA 0 0
5 a NA 0 0
6 a NA 0 0
7 a NA 0 0
8 a NA 0 0
9 a NA 0 0
10 a NA 0 0
> # replacing NAs with somthing else
> df[is.na(df)] <- 0; df
col1 col2 newCol col
1 a 0 0 0
2 a 0 0 0
3 a 0 0 0
4 a 0 0 0
5 a 0 0 0
6 a 0 0 0
7 a 0 0 0
8 a 0 0 0
9 a 0 0 0
10 a 0 0 0
> df$col[is.na(df$col2)] <- 0; df
col1 col2 newCol col
1 a 0 0 0
2 a 0 0 0
3 a 0 0 0
4 a 0 0 0
5 a 0 0 0
6 a 0 0 0
7 a 0 0 0
8 a 0 0 0
9 a 0 0 0
10 a 0 0 0
> df$col2 <- ifelse(is.na(df$col2), 0, df$col); df
col1 col2 newCol col
1 a 0 0 0
2 a 0 0 0
3 a 0 0 0
4 a 0 0 0
5 a 0 0 0
6 a 0 0 0
7 a 0 0 0
8 a 0 0 0
9 a 0 0 0
10 a 0 0 0
df <- orig[!is.na(orig$series), c('Date, series')]

Traps
1 for loops on possibly empty df's, use: for( in in seq_len(nrow(df))
2 columns coerced to factors, avoid with the argument stringsAsFactor=FALSE
3 confusing row numbers and rows with numbered names(hint: avoid row names)
4 although rbind() accepts vectors and lists; this can fail with factor cols

Wednesday, November 4, 2015

R Basics 4 - Lists

Context: R has two types of vector

Atomic vectors contain values
These values are all of the same type.
They are arranged contiguously.
Atomic vectors cannot contain objects. 
There are six types of atomic vector: raw, logical, integer, numeic, complex and character.

Recursive vectors contain objects
R has two types of recursive vector: : list, expression.

Lists
- At top level: 1-dim indexed object that contains objects (not values)
- Indexed from 1 to length(list)
- Contents can be of different types
- Lists can contain the NULL object
- Deeply nested listed of lists possible
- Can be arbitrarily extended (not fixed)

List creation: usually using list()
> l1 <- list('cat', 5, 1:10, FALSE);l1
[[1]]
[1] "cat"

[[2]]
[1] 5

[[3]]
 [1]  1  2  3  4  5  6  7  8  9 10

[[4]]
[1] FALSE

> l2 <- list ( x='dog', y=5+2i, z=3:8 );l2
$x
[1] "dog"

$y
[1] 5+2i

$z
[1] 3 4 5 6 7 8

> l3 <- c(l1, l2);l3
[[1]]
[1] "cat"

[[2]]
[1] 5

[[3]]
 [1]  1  2  3  4  5  6  7  8  9 10

[[4]]
[1] FALSE

$x
[1] "dog"

$y
[1] 5+2i

$z
[1] 3 4 5 6 7 8

> l4 <- list(l1, l2);l4
[[1]]
[[1]][[1]]
[1] "cat"

[[1]][[2]]
[1] 5

[[1]][[3]]
 [1]  1  2  3  4  5  6  7  8  9 10

[[1]][[4]]
[1] FALSE


[[2]]
[[2]]$x
[1] "dog"

[[2]]$y
[1] 5+2i

[[2]]$z
[1] 3 4 5 6 7 8


> l5 <- as.list( c(1,2,3));l5
[[1]]
[1] 1

[[2]]
[1] 2

[[3]]
[1] 3

> origL <- l4
> inserVorL <- l5
> position <- 3
> l6 <- append(origL, inserVorL, position);l6
[[1]]
[[1]][[1]]
[1] "cat"

[[1]][[2]]
[1] 5

[[1]][[3]]
 [1]  1  2  3  4  5  6  7  8  9 10

[[1]][[4]]
[1] FALSE


[[2]]
[[2]]$x
[1] "dog"

[[2]]$y
[1] 5+2i

[[2]]$z
[1] 3 4 5 6 7 8


[[3]]
[1] 1

[[4]]
[1] 2

[[5]]
[1] 3

Basic information about lists
> dim(l)
NULL
> is.list(l)
[1] TRUE
> is.vector(l)
[1] TRUE
> is.recursive(l)
[1] TRUE
> is.atomic(l)
[1] FALSE
> is.factor(l)
[1] FALSE
> length(l)
[1] 5
> names(l)
NULL
> mode(l)
[1] "list"
> class(l)
[1] "list"
> typeof(l)
[1] "list"
> attributes(l)
NULL

The contents of a list
> print(l)
[[1]]
[[1]][[1]]
[1] "cat"

[[1]][[2]]
[1] 5

[[1]][[3]]
 [1]  1  2  3  4  5  6  7  8  9 10

[[1]][[4]]
[1] FALSE


[[2]]
[[2]]$x
[1] "dog"

[[2]]$y
[1] 5+2i

[[2]]$z
[1] 3 4 5 6 7 8


[[3]]
[1] 1

[[4]]
[1] 2

[[5]]
[1] 3

> str(l)
List of 5
 $ :List of 4
  ..$ : chr "cat"
  ..$ : num 5
  ..$ : int [1:10] 1 2 3 4 5 6 7 8 9 10
  ..$ : logi FALSE
 $ :List of 3
  ..$ x: chr "dog"
  ..$ y: cplx 5+2i
  ..$ z: int [1:6] 3 4 5 6 7 8
 $ : num 1
 $ : num 2
 $ : num 3
> dput(l)
list(list("cat", 5, 1:10, FALSE), structure(list(x = "dog", y = 5+2i, 
    z = 3:8), .Names = c("x", "y", "z")), 1, 2, 3)
> head(l)
[[1]]
[[1]][[1]]
[1] "cat"

[[1]][[2]]
[1] 5

[[1]][[3]]
 [1]  1  2  3  4  5  6  7  8  9 10

[[1]][[4]]
[1] FALSE


[[2]]
[[2]]$x
[1] "dog"

[[2]]$y
[1] 5+2i

[[2]]$z
[1] 3 4 5 6 7 8


[[3]]
[1] 1

[[4]]
[1] 2

[[5]]
[1] 3

> tail(l)
[[1]]
[[1]][[1]]
[1] "cat"

[[1]][[2]]
[1] 5

[[1]][[3]]
 [1]  1  2  3  4  5  6  7  8  9 10

[[1]][[4]]
[1] FALSE


[[2]]
[[2]]$x
[1] "dog"

[[2]]$y
[1] 5+2i

[[2]]$z
[1] 3 4 5 6 7 8


[[3]]
[1] 1

[[4]]
[1] 2

[[5]]
[1] 3

Trap: cat(x) does not work with lists

Indexing [ versus [[ versus $
- Use [ to get/set multiple items at once
Note: [ always returns a list
- Use [[ and $ to get/set a specific item
- $ only works with named list items
all same: $name $"name" $'name' $`name`
- indexed by positive numbers: these ones
- indexed by negative numbers: not these
- indexed by logical atomic vector: in/out
an empty index l[] returns the list
Tip: When using lists, most of the time you wnat ot index with [[ or $; and avoid [

Indexing examples: one-dimension get
> j <- list(a='cat', b=5, c=FALSE)
> x <- j$a;x
[1] "cat"
> x <- j[['a']];x
[1] "cat"
> x <- j['a'];x
$a
[1] "cat"
> x <- j[[1]];x
[1] "cat"
> x <- j[1];x
$a
[1] "cat"

Indexing examples: set operations
- Start with example data

l <- list(x='a', y='b', z='c', t='d')
- Next use [[ or $ because specific selection
> l[[6]] <- 'new';
> names(l)[5] <- 'w'
> l$w <- 'new-W'
> l[['w']] <- 'dog'
- Change named values: note order ignored
> l[names(l) %in% c('t', 'x')] <- c(1,2)
> l
$x
[1] 1

$y
[1] "b"

$z
[1] "c"

$t
[1] 2

$w
[1] "dog"

[[6]]
[1] "new"

Indexing example: multi-dimension get
- Indexing evaluated from left to right
- Let's start with some example data...
> i <- c('aa', 'bb', 'cc')
> j <- list(a='cat', b=5, c=FALSE)
> k <- list(i, j);k
[[1]]
[1] "aa" "bb" "cc"

[[2]]
[[2]]$a
[1] "cat"

[[2]]$b
[1] 5

[[2]]$c
[1] FALSE


> k[[1]]
[1] "aa" "bb" "cc"
> k[[2]]
$a
[1] "cat"

$b
[1] 5

$c
[1] FALSE

> k[1]
[[1]]
[1] "aa" "bb" "cc"

> k[2]
[[1]]
[[1]]$a
[1] "cat"

[[1]]$b
[1] 5

[[1]]$c
[1] FALSE

> x <- k[[1]][[1]];x
[1] "aa"
> x <- k[[1]][[2]];x
[1] "bb"
> x <- k[1][1][1][1][1];x
[[1]]
[1] "aa" "bb" "cc"

> x <- k[1][2];x
[[1]]
NULL

> x <- k[[2]][1];x
$a
[1] "cat"

List manipulation
1 Arithmetic operators cannot be applied to lists (as content types can vary)
2 Use the apply() functions to apply a function of each element in a list:
> x <- list(a=1, b=month.abb, c=letters)
> lapply(x, FUN=length)
$a
[1] 1

$b
[1] 12

$c
[1] 26

> sapply(x, FUN=length)
 a  b  c 
 1 12 26 
y <- list(a=1, b=3, c=3, c=4)
sapply(y, FUN=function=(x,p) x^p, p=2)
sapply(y, FUN=function=(x,p) x^p, p=2:3)

3 Use unlist to convert list ot vector
> unlist(x)
    a    b1    b2    b3    b4    b5    b6    b7    b8    b9   b10   b11   b12    c1    c2    c3    c4    c5    c6    c7 
  "1" "Jan" "Feb" "Mar" "Apr" "May" "Jun" "Jul" "Aug" "Sep" "Oct" "Nov" "Dec"   "a"   "b"   "c"   "d"   "e"   "f"   "g" 
   c8    c9   c10   c11   c12   c13   c14   c15   c16   c17   c18   c19   c20   c21   c22   c23   c24   c25   c26 
  "h"   "i"   "j"   "k"   "l"   "m"   "n"   "o"   "p"   "q"   "r"   "s"   "t"   "u"   "v"   "w"   "x"   "y"   "z" 

unlist wont unlist non-atomic

4 Remove NULL objects from a list
> z <- list(a=1:9, b=letters, c=NULL)
> zNoNull <- Filter(Negate(is.null), z)
> zNoNull
$a
[1] 1 2 3 4 5 6 7 8 9

$b
 [1] "a" "b" "c" "d" "e" "f" "g" "h" "i" "j" "k" "l" "m" "n" "o" "p" "q" "r" "s" "t" "u" "v" "w" "x" "y" "z"


5 Use named lists to return multiple values


6 Factor index treated as integer
decode with v[as.character(f)]

Tuesday, November 3, 2015

R Basics 3 - Atomic Vectors

Atomic vectors:
- An object with contiguous, indexed values
- Indexed from 1 to length(vector)
- All values of the same basic atomic type
- Vectors do not have a dimension attribute
- Has a fixed length once created

Six basic atomic types:
- logical
- integer
- numeric
- complex
- character
- raw

No scalars
In R, the basic types are always in a vecotr. Scalars are just length=1 vectors.

Creation (length determined at creation)
#Default value vectors of length=4

> u <- vector(mode='logical', length=4)

> print(u)

[1] FALSE FALSE FALSE FALSE

> v <- vector(mode= 'integer', length=4)

> print(v)

[1] 0 0 0 0

#Using the sequence operator
> i <- 1:5; i
[1] 1 2 3 4 5
> j <- 1.4:6.4; j
[1] 1.4 2.4 3.4 4.4 5.4 6.4
> k <- seq(from=0, to =1, by=0.1);k
[1] 0.0 0.1 0.2 0.3 0.4 0.5 0.6 0.7 0.8 0.9 1.0

#Using the c() function
> l <- c(TRUE,FALSE); l
[1] TRUE FALSE
> n <- c(1.3, 7, 7/20); n
[1] 1.30 7.00 0.35
> z <- c(1+2i, 2, -3+4i); z
[1] 1+2i 2+0i -3+4i

#Other things
> v1 <- c(a=1,b=2,c=3); v1
a b c
1 2 3
> v2 <- rep(NA, 3); v2
[1] NA NA NA
> v3 <- c(v1, v2); v3
a b c
1 2 3 NA NA NA
> v4 <- append(1:5, 2:10, after=5); v4
[1] 1 2 3 4 5 2 3 4 5 6 7 8 9 10

Conversion
> as.vector(v4)
[1] 1 2 3 4 5 2 3 4 5 6 7 8 9 10
> as.logical(v4)
[1] TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE
> as.integer(v4)
[1] 1 2 3 4 5 2 3 4 5 6 7 8 9 10
> as.numeric(v4)
[1] 1 2 3 4 5 2 3 4 5 6 7 8 9 10
> as.character(v4)
[1] "1" "2" "3" "4" "5" "2" "3" "4" "5" "6" "7" "8" "9" "10"
> unlist(l)
[1] TRUE FALSE

Basic information about atomic vectors
> dim(v)
NULL
> is.atomic(v)
[1] TRUE
> is.vector(v)
[1] TRUE
> is.list(v)
[1] FALSE
> is.factor(v)
[1] FALSE
> is.recursive(v)
[1] FALSE
> length(v)
[1] 14
> names(v)
NULL
> mode(v)
[1] "numeric"
> class(v)
[1] "integer"
> typeof(v)
[1] "integer"
> attributes(v)
NULL
> is.numeric(v);
[1] TRUE
> is.character(v);
[1] FALSE
Trap: lists are vectors (but not atomic)
Trap: array/matrix are atomic (not vectors)
Tip: use(is.vector(v) && is.atomic(v)

The content of a vector
> cat(v)
1 2 3 4 5 2 3 4 5 6 7 8 9 10
> print(v)
[1] 1 2 3 4 5 2 3 4 5 6 7 8 9 10
> str(v)
int [1:14] 1 2 3 4 5 2 3 4 5 6 ...
> dput(v)
c(1L, 2L, 3L, 4L, 5L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L, 10L)
> head(v)
[1] 1 2 3 4 5 2
> tail(v)
[1] 5 6 7 8 9 10

Indexing: [ and [[ but not $
-[x] selects a vecor for the cell/range x
-[[x]] selects a length=1 vector for the single cell index x (rarely used)
- $ operator invalid for atomic vectors

Index by positive numbers
> v[c(1,2,3)]
[1] 1 2 3
> v[1:2]
[1] 1 2
> v[[7]]
[1] 3
> v[which(v == 'M')]

integer(0)

Index by negative numbers
> v[-1] #get all but the first element
[1] 2 3 4 5 2 3 4 5 6 7 8 9 10
> v[-length(v)]
[1] 1 2 3 4 5 2 3 4 5 6 7 8 9
> v[-c(1,3,5,7,9)]
[1] 2 4 2 4 6 7 8 9 10

Index by name(only with named vectors)
> names(v)[1:3] = c('alpha', 'beta','z')
> v[['alpha']]
[1] 1
> v[['beta']]
[1] 2
> v[c('alpha','beta')]
alpha beta
1 2
> v[!(names(v) %in% c('c', 'b'))]
alpha beta z <NA> <NA> <NA> <NA> <NA> <NA> <NA> <NA> <NA> <NA> <NA>
1 2 3 4 5 2 3 4 5 6 7 8 9 10

Sorting
> upsorted = sort(v); upsorted;
alpha beta <NA> z <NA> <NA> <NA> <NA> <NA> <NA> <NA> <NA> <NA> <NA>
1 2 2 3 3 4 4 5 5 6 7 8 9 10
> v[order(v)]
alpha beta <NA> z <NA> <NA> <NA> <NA> <NA> <NA> <NA> <NA> <NA> <NA>
1 2 2 3 3 4 4 5 5 6 7 8 9 10
> d = sort(v, decreasing = TRUE);d
<NA> <NA> <NA> <NA> <NA> <NA> <NA> <NA> <NA> z <NA> beta <NA> alpha
10 9 8 7 6 5 5 4 4 3 3 2 2 1

Raw vectors
> s <- charToRaw('raw');
> r <- as.raw(c(114,97,119))
> print(r)
[1] 72 61 77

Monday, November 2, 2015

R Basics 2 - Basic List of Useful Functions in R

1 Build-in Constants

> LETTERS

[1] "A" "B" "C" "D" "E" "F" "G" "H" "I" "J" "K" "L" "M" "N" "O" "P" "Q" "R" "S" "T" "U" "V"

[23] "W" "X" "Y" "Z"

> letters

[1] "a" "b" "c" "d" "e" "f" "g" "h" "i" "j" "k" "l" "m" "n" "o" "p" "q" "r" "s" "t" "u" "v"

[23] "w" "x" "y" "z"

> month.abb

[1] "Jan" "Feb" "Mar" "Apr" "May" "Jun" "Jul" "Aug" "Sep" "Oct" "Nov" "Dec"

> month.name

[1] "January" "February" "March" "April" "May" "June" "July"

[8] "August" "September" "October" "November" "December"

> pi

[1] 3.141593

2 Object creation

> sz <- 26

> x <- 4

> # length 1 vector

> t <- 'c'

> a <- letters[ceiling(runif(sz, 0.00001, 26))]; a

[1] "m" "d" "v" "i" "a" "a" "t" "a" "i" "n" "y" "y" "x" "j" "h" "m" "j" "i" "i" "h" "a" "w"

[23] "t" "s" "g" "o"

> i <- 1:sz

> j <- i + rnorm(sz, 0, 2);j

[1] 0.725392 3.795510 4.998845 3.118527 5.323526 7.283787 7.834817 9.087679

[9] 8.830070 9.439148 12.960340 10.442182 12.580495 18.454917 17.299115 18.321839

[17] 15.656129 19.342379 20.866082 16.008389 19.242640 22.343947 25.057231 26.150223

[25] 27.198469 28.704727

> names(a) <- LETTERS[1:sz];a

A B C D E F G H I J K L M N O P Q R S T U V W

"m" "d" "v" "i" "a" "a" "t" "a" "i" "n" "y" "y" "x" "j" "h" "m" "j" "i" "i" "h" "a" "w" "t"

X Y Z

"s" "g" "o"

> #complex number

> l <- exp((0+1i)*pi) + (1+0i);l

[1] 0+1.224647e-16i

> d <- as.Date('2010-01-01') + seq(1,sz);d

[1] "2010-01-02" "2010-01-03" "2010-01-04" "2010-01-05" "2010-01-06" "2010-01-07"

[7] "2010-01-08" "2010-01-09" "2010-01-10" "2010-01-11" "2010-01-12" "2010-01-13"

[13] "2010-01-14" "2010-01-15" "2010-01-16" "2010-01-17" "2010-01-18" "2010-01-19"

[19] "2010-01-20" "2010-01-21" "2010-01-22" "2010-01-23" "2010-01-24" "2010-01-25"

[25] "2010-01-26" "2010-01-27"

> f <- factor(rep(1:x, sz/x), levels=x:1); f

[1] 1 2 3 4 1 2 3 4 1 2 3 4 1 2 3 4 1 2 3 4 1 2 3 4

Levels: 4 3 2 1

> df <- data.frame(a=a, f=f, i=i, j=j); df

Error in data.frame(a = a, f = f, i = i, j = j) :

arguments imply differing number of rows: 26, 24

> m <- matrix(rnorm(x^2), nrow=x, ncol=x); m

[,1] [,2] [,3] [,4]

[1,] 1.1105554 0.6154485 0.6311158 -2.13814813

[2,] 0.9578517 0.8107677 -1.2327885 -0.65443181

[3,] -0.6565348 1.8245487 -1.4426480 -0.45362514

[4,] 0.2563039 1.2790322 1.2530851 0.04705056

> l <- list(1:10,LETTERS); l

[[1]]

[1] 1 2 3 4 5 6 7 8 9 10

[[2]]

[1] "A" "B" "C" "D" "E" "F" "G" "H" "I" "J" "K" "L" "M" "N" "O" "P" "Q" "R" "S" "T" "U" "V"

[23] "W" "X" "Y" "Z"

3 Object Inspection

> names(x)

NULL

> dimnames(x)

NULL

> colnames(x)

NULL

> rownames(x)

NULL

> dim(x)

NULL

> nrow(x)

NULL

> ncol(x)

NULL

> is.list(x)

[1] FALSE

> is.factor(x)

[1] FALSE

> is.complex(x)

[1] FALSE

> is.character(x)

[1] FALSE

> is.matrix(x)

[1] FALSE

> is.numeric(x)

[1] TRUE

> is.integer(x)

[1] FALSE

> is.vector(x)

[1] TRUE

> is.data.frame(x)

[1] FALSE

> is.ordered(x)

[1] FALSE

4 Utility Function
> assign('variablename', 5)
> c(1,2)
[1] 1 2
> rep(NA,10)
[1] NA NA NA NA NA NA NA NA NA NA
> append(l, list(c(1,2,3)))
[[1]]
[1] 0+1.224647e-16i

[[2]]
[1] 1 2 3

> seq(15,100,5)
[1] 15 20 25 30 35 40 45 50 55 60 65 70 75 80 85 90 95 100
>
> a=rnorm(10)
> sort(a)
[1] -1.49263272 -1.34121858 -1.05170988 -0.92195631 -0.61930516 -0.29196316 0.01090951
[8] 0.68823271 1.09200068 1.13104951
> order(a)
[1] 8 3 6 5 9 1 7 4 10 2
> rank(a)
[1] 6 10 2 8 4 3 7 1 5 9
> rev(a)
[1] 1.09200068 -0.61930516 -1.49263272 0.01090951 -1.05170988 -0.92195631 0.68823271
[8] -1.34121858 1.13104951 -0.29196316
>
> i=3
> any(i %in% c(1,3,5));
[1] TRUE
> all(i %in% c(1,3,5));
[1] TRUE
> which(i %in% c(1,3,5));
[1] 1
> match('c', a)
[1] NA

> df <- transform(df, k=j+1)[1:10,]
> df
a f i j k
1 r 1 1 2.2447932 3.2447932
2 v 2 2 -1.5417236 -0.5417236
3 s 3 3 -0.3591784 0.6408216
4 c 4 4 5.4814561 6.4814561
5 f 1 5 7.1022692 8.1022692
6 i 2 6 5.3284810 6.3284810
7 p 3 7 5.0873649 6.0873649
8 n 4 8 8.5368351 9.5368351
9 v 1 9 11.9734547 12.9734547
10 r 2 10 10.0198377 11.0198377
> df <- within(df, s<- i/j)
> df
a f i j k s
1 r 1 1 2.2447932 3.2447932 0.4454753
2 v 2 2 -1.5417236 -0.5417236 -1.2972494
3 s 3 3 -0.3591784 0.6408216 -8.3523948
4 c 4 4 5.4814561 6.4814561 0.7297331
5 f 1 5 7.1022692 8.1022692 0.7040003
6 i 2 6 5.3284810 6.3284810 1.1260245
7 p 3 7 5.0873649 6.0873649 1.3759579
8 n 4 8 8.5368351 9.5368351 0.9371154
9 v 1 9 11.9734547 12.9734547 0.7516628
10 r 2 10 10.0198377 11.0198377 0.9980202
> x <- with(df, j+5)
> x
[1] 7.244793 3.458276 4.640822 10.481456 12.102269 10.328481 10.087365 13.536835
[9] 16.973455 15.019838
> z <- rep(1,length(x))
> df <- cbind(df, z)
> df
a f i j k s z
1 r 1 1 2.2447932 3.2447932 0.4454753 1
2 v 2 2 -1.5417236 -0.5417236 -1.2972494 1
3 s 3 3 -0.3591784 0.6408216 -8.3523948 1
4 c 4 4 5.4814561 6.4814561 0.7297331 1
5 f 1 5 7.1022692 8.1022692 0.7040003 1
6 i 2 6 5.3284810 6.3284810 1.1260245 1
7 p 3 7 5.0873649 6.0873649 1.3759579 1
8 n 4 8 8.5368351 9.5368351 0.9371154 1
9 v 1 9 11.9734547 12.9734547 0.7516628 1
10 r 2 10 10.0198377 11.0198377 0.9980202 1
> row.df <- head(df, 1)
> rbind(df, row.df)
a f i j k s z
1 r 1 1 2.2447932 3.2447932 0.4454753 1
2 v 2 2 -1.5417236 -0.5417236 -1.2972494 1
3 s 3 3 -0.3591784 0.6408216 -8.3523948 1
4 c 4 4 5.4814561 6.4814561 0.7297331 1
5 f 1 5 7.1022692 8.1022692 0.7040003 1
6 i 2 6 5.3284810 6.3284810 1.1260245 1
7 p 3 7 5.0873649 6.0873649 1.3759579 1
8 n 4 8 8.5368351 9.5368351 0.9371154 1
9 v 1 9 11.9734547 12.9734547 0.7516628 1
10 r 2 10 10.0198377 11.0198377 0.9980202 1
11 r 1 1 2.2447932 3.2447932 0.4454753 1
> df$f <- reorder(df$f, df$j, mean)
> df
a f i j k s z
1 r 1 1 2.2447932 3.2447932 0.4454753 1
2 v 2 2 -1.5417236 -0.5417236 -1.2972494 1
3 s 3 3 -0.3591784 0.6408216 -8.3523948 1
4 c 4 4 5.4814561 6.4814561 0.7297331 1
5 f 1 5 7.1022692 8.1022692 0.7040003 1
6 i 2 6 5.3284810 6.3284810 1.1260245 1
7 p 3 7 5.0873649 6.0873649 1.3759579 1
8 n 4 8 8.5368351 9.5368351 0.9371154 1
9 v 1 9 11.9734547 12.9734547 0.7516628 1
10 r 2 10 10.0198377 11.0198377 0.9980202 1

5 Math Function
> is.na(i)
[1] FALSE
> is.nan(j)
[1] FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE
[15] FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE
> is.null(d)
[1] FALSE
> is.finite(j)
[1] TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE
[18] TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE
>
> abs(-1)
[1] 1
> sqrt(10)
[1] 3.162278
> log(1)
[1] 0
> log10(10)
[1] 1
> exp(1)
[1] 2.718282
> ceiling(5.3)
[1] 6
> floor(2.3)
[1] 2
> round(3.22,digits=1)
[1] 3.2
> trunc(4.5)
[1] 4
> sin(pi/2)
[1] 1
> cos(pi)
[1] -1
> tan(pi/2)
[1] 1.633124e+16
> asin(0.5)
[1] 0.5235988
> acos(0.5)
[1] 1.047198
> atan(1)
[1] 0.7853982
> sum(1:10)
[1] 55
> prod(1:10)
[1] 3628800
> cumsum(1:10)
[1] 1 3 6 10 15 21 28 36 45 55
> cumprod(1:10)
[1] 1 2 6 24 120 720 5040 40320 362880 3628800

6 Stat Function
> length(0:10)
[1] 11
> sum(0:10)
[1] 55
> min(0:10)
[1] 0
> max(0:10)
[1] 10
> range(0:10)
[1] 0 10
> cut(0:10,5)
[1] (-0.01,2] (-0.01,2] (-0.01,2] (2,4] (2,4] (4,6] (4,6] (6,8]
[9] (6,8] (8,10] (8,10]
Levels: (-0.01,2] (2,4] (4,6] (6,8] (8,10]
>
> mean(0:10)
[1] 5
> median(0:10)
[1] 5
> sd(0:10)
[1] 3.316625
> var(0:10)
[1] 11
> cov(0:10,1:11)
[1] 11
> cor(0:10,1:11)
[1] 1
> diff(1:10, lag=1, diff=1)
[1] 1 1 1 1 1 1 1 1 1
> rnorm(n=10, mean=0, sd=1)
[1] 0.50660408 0.34007608 -0.07856213 0.87086844 0.68152240 0.80075731 -0.57385601
[8] 0.99361746 1.18958557 -1.87070403
> runif(n=10, min=1, max=100)
[1] 46.977626 55.666696 69.418533 8.874474 15.755576 52.141463 57.587816 85.232902
[9] 69.724438 3.472987
>
> r <- lm(j ~ i, data=as.data.frame(df));
> summary(r)

Call:
lm(formula = j ~ i, data = as.data.frame(df))

Residuals:
Min 1Q Median 3Q Max
-3.3564 -1.7633 -0.2411 2.3067 3.2870

Coefficients:
Estimate Std. Error t value Pr(>|t|)
(Intercept) -1.74123 0.44095 -3.949 0.000145 ***
i 1.08017 0.02855 37.831 < 2e-16 ***
---
Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 2.184 on 102 degrees of freedom
Multiple R-squared: 0.9335, Adjusted R-squared: 0.9328
F-statistic: 1431 on 1 and 102 DF, p-value: < 2.2e-16

> anova(r)
Analysis of Variance Table

Response: j
Df Sum Sq Mean Sq F value Pr(>F)
i 1 6825.6 6825.6 1431.2 < 2.2e-16 ***
Residuals 102 486.5 4.8
---
Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
> residuals(r)
1 2 3 4 5 6 7 8
3.2870286 0.2328002 -3.2515109 -0.7149405 -2.0039734 1.8895235 1.0512190 -3.3564285
9 10 11 12 13 14 15 16
3.1063883 -1.2399755 -1.6117162 -1.0684410 2.3715878 2.3066557 1.3768941 -3.2480162
17 18 19 20 21 22 23 24
-2.9309619 2.7913712 0.7080538 2.8996443 -0.9851891 2.3818433 -1.7632818 -1.9259796
25 26 27 28 29 30 31 32
-1.2913915 0.9887964 3.2870286 0.2328002 -3.2515109 -0.7149405 -2.0039734 1.8895235
33 34 35 36 37 38 39 40
1.0512190 -3.3564285 3.1063883 -1.2399755 -1.6117162 -1.0684410 2.3715878 2.3066557
41 42 43 44 45 46 47 48
1.3768941 -3.2480162 -2.9309619 2.7913712 0.7080538 2.8996443 -0.9851891 2.3818433
49 50 51 52 53 54 55 56
-1.7632818 -1.9259796 -1.2913915 0.9887964 3.2870286 0.2328002 -3.2515109 -0.7149405
57 58 59 60 61 62 63 64
-2.0039734 1.8895235 1.0512190 -3.3564285 3.1063883 -1.2399755 -1.6117162 -1.0684410
65 66 67 68 69 70 71 72
2.3715878 2.3066557 1.3768941 -3.2480162 -2.9309619 2.7913712 0.7080538 2.8996443
73 74 75 76 77 78 79 80
-0.9851891 2.3818433 -1.7632818 -1.9259796 -1.2913915 0.9887964 3.2870286 0.2328002
81 82 83 84 85 86 87 88
-3.2515109 -0.7149405 -2.0039734 1.8895235 1.0512190 -3.3564285 3.1063883 -1.2399755
89 90 91 92 93 94 95 96
-1.6117162 -1.0684410 2.3715878 2.3066557 1.3768941 -3.2480162 -2.9309619 2.7913712
97 98 99 100 101 102 103 104
0.7080538 2.8996443 -0.9851891 2.3818433 -1.7632818 -1.9259796 -1.2913915 0.9887964
> coef(r)
(Intercept) i
-1.741227 1.080172
> plot(r)
Hit <Return> to see next plot: plot(r$fitted.values)
Hit <Return> to see next plot: plot(r$residuals)
Hit <Return> to see next plot:
# Also, glm() gam() lme() lmer() nls()

7 Character Function
> as.character(1)
[1] "1"
> toString(10)
[1] "10"
> nchar('qwe')
[1] 3
> B<-toupper('bbb')
> b<-tolower('BBB')
> s <- 'the cow jumped over the moon'
> sub('the', 'a', s)
[1] "a cow jumped over the moon"
> gsub('the', 'a', s)
[1] "a cow jumped over a moon"
> substr(s,5,7)
[1] "cow"
> substr(s,5,7) <- "dog"
> s
[1] "the dog jumped over the moon"
> substr(s,5,7) <- "monkey"
> s
[1] "the mon jumped over the moon"
> paste('a', 'b', 'c', sep=";")
[1] "a;b;c"
> strsplit(s, ' ')
[[1]]
[1] "the" "mon" "jumped" "over" "the" "moon"

> grep('the', s)
[1] 1
> make.unique(a)
[1] "q" "a" "e" "b" "s" "x" "k" "u" "n" "w" "b.1" "k.1" "e.1" "b.2"
[15] "a.1" "u.1" "r" "y" "e.2" "v" "h" "m" "n.1" "y.1" "x.1" "g"
> format(j, digits=2)
[1] " 2.63" " 0.65" "-1.75" " 1.86" " 1.66" " 6.63" " 6.87" " 3.54" "11.09" " 7.82" " 8.53"
[12] "10.15" "14.67" "15.69" "15.84" "12.29" "13.69" "20.49" "19.49" "22.76" "19.96" "24.40"
[23] "21.34" "22.26" "23.97" "27.33"
> sprintf("%d: %s", i, a)
[1] "1: q" "2: a" "3: e" "4: b" "5: s" "6: x" "7: k" "8: u" "9: n" "10: w" "11: b"
[12] "12: k" "13: e" "14: b" "15: a" "16: u" "17: r" "18: y" "19: e" "20: v" "21: h" "22: m"
[23] "23: n" "24: y" "25: x" "26: g"
> format(d, format="%A %Y-%b-%d")
[1] "Saturday 2010-Jan-02" "Sunday 2010-Jan-03" "Monday 2010-Jan-04"
[4] "Tuesday 2010-Jan-05" "Wednesday 2010-Jan-06" "Thursday 2010-Jan-07"
[7] "Friday 2010-Jan-08" "Saturday 2010-Jan-09" "Sunday 2010-Jan-10"
[10] "Monday 2010-Jan-11" "Tuesday 2010-Jan-12" "Wednesday 2010-Jan-13"
[13] "Thursday 2010-Jan-14" "Friday 2010-Jan-15" "Saturday 2010-Jan-16"
[16] "Sunday 2010-Jan-17" "Monday 2010-Jan-18" "Tuesday 2010-Jan-19"
[19] "Wednesday 2010-Jan-20" "Thursday 2010-Jan-21" "Friday 2010-Jan-22"
[22] "Saturday 2010-Jan-23" "Sunday 2010-Jan-24" "Monday 2010-Jan-25"
[25] "Tuesday 2010-Jan-26" "Wednesday 2010-Jan-27"

8 Dates
> x <- as.Date('03-06-1920', format='%d-%m-%Y')
> Sys.Date()
[1] "2015-10-29"
> days.apart <- d-x
> weekdays(d)
[1] "Wednesday" "Thursday" "Friday" "Saturday" "Sunday" "Monday" "Tuesday"
[8] "Wednesday" "Thursday" "Friday" "Saturday" "Sunday" "Monday" "Tuesday"
[15] "Wednesday" "Thursday" "Friday" "Saturday" "Sunday" "Monday" "Tuesday"
[22] "Wednesday" "Thursday" "Friday" "Saturday" "Sunday"
> months(d)
[1] "January" "January" "January" "January" "January" "January" "January" "January"
[9] "January" "January" "January" "January" "January" "January" "January" "January"
[17] "January" "January" "January" "January" "January" "January" "January" "January"
[25] "January" "January"
> dp <- as.POSIXlt(d);
> dp
[1] "2008-01-02 UTC" "2008-01-03 UTC" "2008-01-04 UTC" "2008-01-05 UTC" "2008-01-06 UTC"
[6] "2008-01-07 UTC" "2008-01-08 UTC" "2008-01-09 UTC" "2008-01-10 UTC" "2008-01-11 UTC"
[11] "2008-01-12 UTC" "2008-01-13 UTC" "2008-01-14 UTC" "2008-01-15 UTC" "2008-01-16 UTC"
[16] "2008-01-17 UTC" "2008-01-18 UTC" "2008-01-19 UTC" "2008-01-20 UTC" "2008-01-21 UTC"
[21] "2008-01-22 UTC" "2008-01-23 UTC" "2008-01-24 UTC" "2008-01-25 UTC" "2008-01-26 UTC"
[26] "2008-01-27 UTC"
> dp$year <- dp$year -1
> d <- as.Date(dp)
> names(unclass(dp))
[1] "sec" "min" "hour" "mday" "mon" "year" "wday" "yday" "isdst"
> Sys.time()
[1] "2015-10-29 01:24:33 EDT"
> date()
[1] "Thu Oct 29 01:24:33 2015"
# Really useful: zoo and lubridate packages

9 I/O and the file system
> cat(i)
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26
> i
[1] 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26
> print(j)
[1] 2.6259729 0.6519162 -1.7522234 1.8645187 1.6556576 6.6293262 6.8711933 3.5437175
[9] 11.0867059 7.8205138 8.5289449 10.1523917 14.6725922 15.6878317 15.8382418 12.2935032
[17] 13.6907292 20.4932340 19.4900883 22.7618505 19.9571887 24.4043928 21.3394394 22.2569133
[25] 23.9716730 27.3320326
> getwd()
[1] "/Users/lingduoduo"
> list.files()
[1] "Applications" "Desktop" "Documents" "Downloads"
[5] "Downloads.rar.dmg" "Eclipse" "Library" "Movies"
[9] "Music" "Pictures" "Public"
> list.dirs()
> dir()
[1] "Applications" "Desktop" "Documents" "Downloads"
[5] "Downloads.rar.dmg" "Eclipse" "Library" "Movies"
[9] "Music" "Pictures" "Public"
> Sys.glob()

save(z, file='z.bin')

load('z.bin')

unlink('z.bin')

con <- file('f.txt', 'rt')

y <- readLines(con, 1)

writeLines(text, con=c, sep="\n")

write.csv(df, file="df.csv")

10 Script and package management

source('program.R')

install.packages(ggplot2)

library('ggplot2')

require('ggplot2')

11 In the Workspace

ls();

rm(z);

help('help');

help.search('help');

q();

12 Debug Functions

browser();

debug();

trace();

stopifnot(i[1]==1);

warning('message');

stop('message');

R Basics 1 - Brief Introduction to Language Elements and Control Structures in R

1 Determine the nature of an object

> library(MASS)
> x=rnorm(1000)

# the R type of x
> typeof(x)
[1] "double"

# the data mode of x
> mode(x)
[1] "numeric"

# the storage mode of x
> storage.mode(x)
[1] "double"

# the class of x
> class(x)
[1] "numeric"

# the attributes of x
> attributes(x)
NULL

# print a summary structure of x
> str(x)
num [1:1000] 0.248 1.893 -0.36 0.921 -0.721 ...

# print full text R code for x
> dput(x)
c(0.248081640772571, 1.89344072095915, -0.360359279529437, 0.921223527734382,
-0.721062915381023, -0.564184655238014, ...

2 NULL vs NA
# NULL is an object, typically used to mean the variable contains no object.
> x <- NULL
> is.null(x)
[1] TRUE
> length(NULL)
[1] 0

# NA is a value that means missing data item here
> y <- NA
> is.na(y)
[1] TRUE
> length(NA)
[1] 1

> # Other non-number numbers
> 1/0
[1] Inf
> 0/0
[1] NaN

3 Flow Control Structure
> #if
> if (checkNum == 0) stop("Count is 0 in lh_DevUserFactMembership")

> #if() else
> x <- 0
> if (x < 0) {
+ print("Negative number")
+ } else if (x > 0) {
+ print("Positive number")
+ } else
+ print("Zero")
[1] "Zero"

> #for ()
> str=""
> for (i in 1 : 10) {
+ str <- paste(str, LETTERS[i], sep = ",")
+ print(str)
+ }
[1] ",A"
[1] ",A,B"
[1] ",A,B,C"
[1] ",A,B,C,D"
[1] ",A,B,C,D,E"
[1] ",A,B,C,D,E,F"
[1] ",A,B,C,D,E,F,G"
[1] ",A,B,C,D,E,F,G,H"
[1] ",A,B,C,D,E,F,G,H,I"
[1] ",A,B,C,D,E,F,G,H,I,J"

> #while () expr
> i <- 1
> while (i < 6) {
+ print(i)
+ i = i+1
+ }
[1] 1
[1] 2
[1] 3
[1] 4
[1] 5

> #repeat
> x <- 1
> repeat {
+ print(x)
+ x = x+1
+ if (x == 6){
+ break
+ }
+ }
[1] 1
[1] 2
[1] 3
[1] 4
[1] 5

4 Flow Control Function
> #result <- ifelse()
> a = c(5,7,2,9)
> ifelse(a %% 2 == 0,"even","odd")
[1] "odd" "odd" "even" "odd"

> #switch()
> switch(2,"red","green","blue")
[1] "green"

Part 2: Frequent Item Sets Using SQL

-- total items n

-- 265,588;

select count(distinct ITEM_ID)

from ANALYTICS_STG..LH_DM_COUPON_7_sales_6_product;

-- basket size t

-- 73;

select max(aa.ITEM_CNT),min(aa.ITEM_CNT),avg(aa.ITEM_CNT) from

(

select trans_id, count(ITEM_ID) ITEM_CNT

from ANALYTICS_STG..LH_DM_COUPON_7_sales_6_product

group by 1

) aa;

-- 42,640,950;

select count(distinct TRANS_ID)

from ANALYTICS_STG..LH_DM_COUPON_7_sales_6_product;

-- Finding frequent itemsets;

drop table ANALYTICS_STG..LH_DM_COUPON_7_sales_6_product1;

-- First Pass Singleton Set;

-- 265,588 rows affected;

create table ANALYTICS_STG..LH_DM_COUPON_7_sales_6_product1 as

select ITEM_ID, item_sku_desc, count(*) ITEM_CNT

from ANALYTICS_STG..LH_DM_COUPON_7_sales_6_product

group by 1,2

order by 3 desc;

-- Count the Items;

drop table ANALYTICS_STG..LH_DM_COUPON_7_sales_6_product2;

-- 265,588 rows affected

create table ANALYTICS_STG..LH_DM_COUPON_7_sales_6_product2 as

select a.*, NTILE(100) OVER(ORDER BY a.ITEM_CNT DESC NULLS LAST) as PERCENTILE

from ANALYTICS_STG..LH_DM_COUPON_7_sales_6_product1 a;

select PERCENTILE, count(*), avg(ITEM_CNT), min(ITEM_CNT), max(ITEM_CNT)

from ANALYTICS_STG..LH_DM_COUPON_7_sales_6_product2

group by 1

order by 1;

-- 209137;

select count(*) from ANALYTICS_STG..LH_DM_COUPON_7_sales_6_product1 where ITEM_CNT>1;

select ITEM_CNT, count(*) from ANALYTICS_STG..LH_DM_COUPON_7_sales_6_product1 group by 1 order by 1 desc, 2 desc;

-- Second Pass Doubleton Set;

-- Filter: support > 120

drop table ANALYTICS_STG..LH_DM_COUPON_7_sales_6_product3;

-- 55,966,448 rows affected;

-- ANALYTICS_STG..LH_DM_COUPON_7_sales_6_product3

create table ANALYTICS_STG..LH_DM_COUPON_7_sales_6_product3 as

select a.TRANS_ID, a.HH_ADDRESS_ID, a.ITEM_ID,

row_number() over(order by a.HH_ADDRESS_ID, a.ITEM_ID) id

from ANALYTICS_STG..LH_DM_COUPON_7_sales_6_product a

inner join

(

select distinct aa.TRANS_ID

from ANALYTICS_STG..LH_DM_COUPON_7_sales_6_product aa

inner join

(select distinct ITEM_ID from ANALYTICS_STG..LH_DM_COUPON_7_sales_6_product2 where ITEM_CNT >= 120) bb

on aa.ITEM_ID = bb.ITEM_ID

) b

on a.TRANS_ID = b.TRANS_ID

;

-- Construct

drop table ANALYTICS_STG..LH_DM_COUPON_7_sales_6_product4;

-- 14,161,756 rows affected;

create table ANALYTICS_STG..LH_DM_COUPON_7_sales_6_product4 as

select distinct aa.id, aa.TRANS_ID, aa.HH_ADDRESS_ID, aa.ITEM_ID,

bb.ITEM_ID as ITEM_ID2

from ANALYTICS_STG..LH_DM_COUPON_7_sales_6_product3 aa

inner join ANALYTICS_STG..LH_DM_COUPON_7_sales_6_product3 bb

on aa.HH_ADDRESS_ID = bb.HH_ADDRESS_ID

and aa.TRANS_ID = bb.TRANS_ID

where aa.id < bb.id

and aa.ITEM_ID < bb.ITEM_ID

;

-- Counting

drop table ANALYTICS_STG..LH_DM_COUPON_7_sales_6_product5;

-- 9,517,664 rows affected;

create table ANALYTICS_STG..LH_DM_COUPON_7_sales_6_product5 as

select ITEM_ID, ITEM_ID2, count(*) PAIR_CNT

from ANALYTICS_STG..LH_DM_COUPON_7_sales_6_product4

group by 1,2

order by 3 desc;

-- Counting

drop table ANALYTICS_STG..LH_DM_COUPON_7_sales_6_product6;

-- 9,517,664 rows affected;

create table ANALYTICS_STG..LH_DM_COUPON_7_sales_6_product6 as

select a.*, NTILE(100) OVER(ORDER BY a.PAIR_CNT DESC NULLS LAST) as PERCENTILE

from ANALYTICS_STG..LH_DM_COUPON_7_sales_6_product5 a;

select PERCENTILE, count(*), avg(PAIR_CNT), min(PAIR_CNT), max(PAIR_CNT)

from ANALYTICS_STG..LH_DM_COUPON_7_sales_6_product6

group by 1

order by 1;

select * from ANALYTICS_STG..LH_DM_COUPON_7_sales_6_product6

where item_id = 1846960 and item_id2 = 1871478

;

select * from ANALYTICS_STG..LH_DM_COUPON_7_sales_6_product6

where item_id = 884680 and item_id2 = 961474

;

-- Third Pass Tripleton Set;

-- Filter: support >= 7

drop table ANALYTICS_STG..LH_DM_COUPON_7_sales_6_product7;

-- 47,509,423 rows affected;

-- ANALYTICS_STG..LH_DM_COUPON_7_sales_6_product3

create table ANALYTICS_STG..LH_DM_COUPON_7_sales_6_product7 as

select a.TRANS_ID, a.HH_ADDRESS_ID, a.ITEM_ID, row_number() over(order by a.HH_ADDRESS_ID, a.ITEM_ID) id

from ANALYTICS_STG..LH_DM_COUPON_7_sales_6_product a

inner join

(

select distinct aa.TRANS_ID

from ANALYTICS_STG..LH_DM_COUPON_7_sales_6_product aa

inner join

(select distinct ITEM_ID from ANALYTICS_STG..LH_DM_COUPON_7_sales_6_product6 where PAIR_CNT >= 7) bb

on aa.ITEM_ID = bb.ITEM_ID

) b

on a.TRANS_ID = b.TRANS_ID

;

-- Coonstruct;

drop table ANALYTICS_STG..LH_DM_COUPON_7_sales_6_product8;

-- 4,513,794 rows affected;

create table ANALYTICS_STG..LH_DM_COUPON_7_sales_6_product8 as

select distinct aa.id, aa.TRANS_ID, aa.HH_ADDRESS_ID, aa.ITEM_ID,

bb.ITEM_ID as ITEM_ID2,

cc.ITEM_ID as ITEM_ID3

from ANALYTICS_STG..LH_DM_COUPON_7_sales_6_product7 aa

inner join ANALYTICS_STG..LH_DM_COUPON_7_sales_6_product7 bb

on aa.HH_ADDRESS_ID = bb.HH_ADDRESS_ID

and aa.TRANS_ID = bb.TRANS_ID

inner join ANALYTICS_STG..LH_DM_COUPON_7_sales_6_product7 cc

on bb.HH_ADDRESS_ID = cc.HH_ADDRESS_ID

and bb.TRANS_ID = cc.TRANS_ID

where aa.id < bb.id

and aa.ITEM_ID < bb.ITEM_ID

and bb.id < cc.id

and bb.ITEM_ID < cc.ITEM_ID

;

-- Counting;

drop table ANALYTICS_STG..LH_DM_COUPON_7_sales_6_product9;

-- 4,218,992 rows affected;

create table ANALYTICS_STG..LH_DM_COUPON_7_sales_6_product9 as

select a.ITEM_ID, a.ITEM_ID2, a.ITEM_ID3, count(*) TRIPLE

from ANALYTICS_STG..LH_DM_COUPON_7_sales_6_product8 a

group by 1,2,3

order by 4 desc;

Thursday, November 5, 2015

Wednesday, November 4, 2015

Tuesday, November 3, 2015

Monday, November 2, 2015

Blog Archive