Basic Markov model in R

library("msm")
result4$user_cumcnt <- as.integer(result4$user_cumcnt)
result4$user_cnt <- as.integer(result4$user_cnt)
basic <- data.frame(result4[result4$user_cnt<=60, c(1,5,15,18)])

raw <- statetable.msm(category, userid, data = basic)
write.csv(raw, file= "raw_state.csv")

tmp <- melt(raw)
tmp$to <- as.integer(tmp$to)

tmp2<-cbind(1:23,levels(basic$category))
write.csv(tmp, "data.csv")
write.csv(tmp2, "junk.csv")

#add censor state;
tmp1 <- ddply(basic, .(userid), function(x) data.frame(user_cumcnt = max(x$user_cumcnt)+1))
tmp2 <- unique(basic[,c(1,4)])
tmp3 <- merge(tmp1,tmp2, by = "userid")
tmp4 <- data.frame(tmp3, rep("End", length(tmp1[,1])))
names(tmp4)[4] <- "category"
tmp5 <- data.frame(rbind(basic, tmp4))
basic <- tmp5[order(tmp5$userid, tmp5$user_cumcnt), ]

raw <- ifelse(raw>0, .09, 0)
diag(raw) <- .8
raw <- cbind(raw, rep(0,23))
raw <- rbind(raw, rep(0,24))
dimnames(raw)[[1]][24] <- "24"
dimnames(raw)[[2]][24] <- "End"
twoway4.q <- raw
twoway4.q <- matrix(rep(1, 24*24), nrow=24)
diag(twoway4.q) <- 0
twoway4.q[,24] <- 1
twoway4.q[24,] <- 0

basic$category <- as.integer(basic$category)

Sys.time()
visits.msm <- msm(category ~ user_cumcnt, subject = userid, data = basic, qmatrix = twoway4.q, death = 24)
Sys.time()

data.table in R

require(plyr)
require(data.table)

# Making a data.table from vectors
dt.data <- data.table(a=1:5,b=1:5)

rm(list=ls(all=T))
setwd("DSP data");
data <- fread("out1000.csv", colClasses=c("character", "integer", "integer", "integer", "integer") );
setnames(data, c("user_id", "date", "hour", "click", "action"));

# regular expression
regexp <- "([[:digit:]]+)"
month <- as.integer(str_extract(data$date, regexp))

# automatically sort by key
setkey(data, user_id, date, hour)

# add cols
data[,id:=1:length(data$user_id)];
data[,bcook:=as.factor(data$user_id)];
hflights_dt <- data.table(hflights)
hflights_dt[, DistanceKMs := Distance / 0.62137]
hflights_dt$DistanceKMs <- hflights_dt$Distance / 0.62137
hflights_dt[, c('DistanceKMs', 'DistanceFeets') := list(Distance/0.62137, Distance * 5280)];

# filter rows
users <- unique(data$user_id[data$date %in% c(16103, 16104, 16105)]);
rows = sample(1:n,n*.0001);
results[,rowid:=1:n];
samples=results[results$rowid %in% rows];
test <- data[user_id == users[1],];

# filter cols
test <- data[,c(2,3,4,5,6), with=FALSE]; #filter cols
train3 <- train3[,ScoreLevel:=NULL] #drop cols

# Basic Stat
system.time(
  stats <- ddply(data, ~ date,
                 summarize, imp=length(imp), click=sum(click), action=sum(action))
)
system.time(
stats <- data[,list(imps=.N,
                    clicks=sum(click),
                    actions=sum(action)), by='date'];
)

#Drop outliers
data2 <- read.csv(file="data_extracted_filtered.txt",head=TRUE,sep=",",row.names=NULL,as.is=TRUE);
for (i in c(2:7)){
  print('----------------------------------------------------------');
  print(i);
  data2 <- data2[data2[,i]<=quantile[i-1],]
}
quantile = rep(0, 6);
for (i in c(2:7)){
  print('----------------------------------------------------------');
  print(i);
  quantile[i-1] <- max(data2[,i], .95)
}

# Apply same function to all cols
func1 <- function(x){
return(log(x+1)/log(max(x)))
}
data3 <- data.table(data2)
data3 <- data3[,id:=NULL]
data3[, 1:6 := lapply(.SD, func1), .SDcols = 1:6]

# Basic Merge
keycols= c("start_dt","variable")
setkeyv(top10thisyearlong, keycols)
setkeyv(top10lastyearlong, keycols)
top10all = merge(top10thisyearlong, top10lastyearlong, by=c("start_dt", "variable"))

# compute freq1, and add a new column with name of 'freq1' to the original table
users <- unique(data$user_id[data$date %in% c(16103, 16104, 16105)]);
setkey(data,bcook);
f_freq <- function(dd) {
  setkey(dd,bcook);
  user.merge <- merge(data, dd, by=c('bcook'));
  user.merge <- user.merge[id.x>id.y, ];
  user.merge$gap <- (user.merge$date.x - user.merge$date.y)*24 + (user.merge$hour.x - user.merge$hour.y);
  freq<- user.merge[, list(freq1d=sum(gap<24)), by=id.x];
  return(c(0, freq$freq1d));
}

sink("myfile", append=FALSE, split=FALSE);
train1[HasClick==-1, HasClick:=0];
idx <- names(train1)
for (i in c(1:36)){
  print('----------------------------------------------------------');
  print(i);
  print(train1[,list(Impressions=.N, Clicks=sum(HasClick), CTR=sum(HasClick)/.N), by=eval(idx[i])][order(-CTR)]);
}
# return output to the terminal
sink()

## Merge multiple data frames
list.of.data.frames = list(dat1, dat2, dat3, dat4, dat5, dat6, dat7)
merged.data.frame = Reduce(function(...) merge(..., all=T), list.of.data.frames)
tail(merged.data.frame)

Show Images in R

setwd("~/Desktop/Trend Reports/TrendAnalytics Reports/TrendPulse/10_16_2016/")
data=fread("keyword socialstream.csv",header = FALSE)
names(data) = c('seq','keywords', 'brands', 'categories')

# Outerwear
list = c(
  10,
  11,
  27,
  29,
  30,
  31,
  32,
  33,
  34,
  35
)
pdf("TopKeyword.pdf", width=15, height=8.5)
par(mfrow=c(2,3))
j=1
for(i in list){
  img <- readPNG(paste(i, ".PNG", sep=""))
 
  plot.new()
  plot.window(xlim=c(0,1), ylim=c(0,1), asp=1)
  rasterImage(img,0,0,1,1)
  label=paste(j, data$categories[data$seq==i],"\n",
              data$brands[data$seq==i], "\n",
              data$keyword[data$seq==i])
  title(main=label, font.main=2, line=0)
 
  print(paste("img", j, ".jpg", sep=""))
  j=j+1
}
dev.off()

Visual of Tree by D3.js

<script src="http://d3js.org/d3.v3.min.js"></script>
<script>

var diameter = 1250;

var tree = d3.layout.tree()
    .size([360, diameter/2 -250])
    .separation(function(a, b) { return (a.parent == b.parent ? 1 : 2) / a.depth; });

var diagonal = d3.svg.diagonal.radial()
    .projection(function(d) { return [d.y, d.x / 180 * Math.PI]; });

var svg = d3.select("body").append("svg")
    .attr("width", diameter)
    .attr("height", diameter - 100)
    .append("g")
    .attr("transform", "translate(" + diameter / 2 + "," + diameter / 2 + ")");

d3.json("flare.json", function(error, root) {
  var nodes = tree.nodes(root),
      links = tree.links(nodes);

  var link = svg.selectAll(".link")
      .data(links)
    .enter().append("path")
      .attr("class", "link")
      .attr("d", diagonal);

  var node = svg.selectAll(".node")
      .data(nodes)
    .enter().append("g")
      .attr("class", "node")
      .attr("transform", function(d) { return "rotate(" + (d.x - 90) + ")translate(" + d.y + ")"; })

  node.append("circle")
      .attr("r", 4.5);

  node.append("text")
      .attr("dy", ".35em")
      .attr("text-anchor", function(d) { return d.x < 180 ? "start" : "end"; })
      .attr("transform", function(d) { return d.x < 180 ? "translate(8)" : "rotate(180)translate(-8)"; })
      .text(function(d) { return d.name; });
});

d3.select(self.frameElement).style("height", diameter + "px");

</script>

Running Mincemeat Example on Windows

Lightweight MapReduce in python: https://github.com/michaelfairley/mincemeatpy.

client

E:\Python27>python example.py

server

E:\Python27>python mincemeat.py -p changeme localhost

Word count example

import glob
import mincemeat

#text_files=glob.glob('hw3data/*')
text_files=glob.glob('hw3data/c0001')
print(text_files)

def file_contents(file_name):
    f=open(file_name,'rb')
    try:
        return f.read()
    finally:
        f.close()

source=dict((file_name,file_contents(file_name))
    for file_name in text_files)

# setup map and reduce functions
def mapfn(key,value):
      for line in value.splitlines():
          for word in line.split():
               yield word.lower(),1

def reducefn(key,value):
       return key,len(value)   
   
# start the server
s =    mincemeat.Server()
s.datasource = source
s.mapfn = mapfn
s.reducefn = reducefn

results = s.run_server(password="changeme")
print results   

Shell awk Scripting

Introduction
awk is good for processing and summarizing structured data; awk is particularly suited for filtering, summarizing and rearranging data.

- cat fruits.txt | awk '{ print }' (write all lines to stdout)
- cat fruits.txt | awk '{ print $1}' (write 1 col of all lines to stdout)
- cat fruits.txt | awk '{ print $3, $1}' (write 3 col and then 1 col of all lines to stdout)
- cat fruits.txt | awk '/apple/{print}' (print all lines starting from apple)
- cat fruits.txt | awk '/^a/{print}' (print all lines starting from a)
- cat fruits.txt | awk '! /apple/{print}' (print all lines not starting from apple)
- cat fruits.txt | awk '$1 == "apple" {print}' (print all lines starting from apple)
- cat numbers.txt | awk '{var1 = $1; var2 = $2; avg = (var1+var2)/2; print avg}'
- cat numbers.txt | awk 'NR %2 == 0 {print}'

login3 15:50:31 ~ $ cat vartot.awk
BEGIN {
 var1 = 0
 var2 = 0
 var3 = 0
}
{
 var1 += $1
 var2 += $2
 var3 += $3
}
END {
 print var1, var2, var3
}
- cat numbers.txt | awk -f vartot.awk (write all lines to stdout)

Patterns and Actions
Patterns can be any expression that returns a value, including regex and range of the form expr1, expr2.
awk has certain variables such as record separators whose values are dynamically set by awk itself.
awk interprets addresses and actions.
NF: number of fields, using whitespace as separators as default.
NR: number of records, using whitespace as separators as default.
RS: set record separators
FS: set field separators
- cat numbers.txt | awk 'NR %2 == 0 {print}' (print row lines are even)
- cat numbers.txt | awk '/^10/, NR %2 == 0 {print}' (/, functions as AND)
- cat numbers.txt | awk 'BEGIN{ FS = ":"} {print $1}' (print first col given the field seperator is :)
- cat numbers.txt | awk 'BEGIN{ FS = ":"} NR %2 == 0 {print $1}' (print first col when row lines are even given the field seperator is :)
- cat numbers.txt | awk 'BEGIN{ RS = "\n\nFrom: " ; FS = "\n"} {print $1}' (print first col given the specified field sepearator and record separator)
- cat months.txt | awk 'BEGIN{ tot = 0} { tot = tot + $2} NR%12 == 0 {print tot/12; tot = 0}' (monthly avg of second col)
- echo $((502/12)) ( round it to integer)
- cat months.txt | awk '{ tot = tot + $2} NR%12 == 0 {print tot/12; tot = 0}' (monthly avg of second col; able to drop initialization of tot)
- cat annual.awk
{
tot = tot + $2
}
NR%12 == 0 {
print tot/12;
tot = 0
}
- cat months.txt | awk -f annual.awk (monthly avg of second col; able to drop initialization of tot)

- match(input_var, regular expression, output_var)
- gsub(regular expression, streams)
...
match($1, /:(.*),(.*)/*, resultmatch)
gsub(/ +/, " ", resultmatch[1]) (squeeze into one space)
gsub(/ +/, " ", resultmatch[2]) (squeeze into one space)
if (tolower(resultmatch[1]) ~ result) (search for result to lower case of resultmatch col 1)
{
  print $1;
}

Shell sed Scripting

Unix command such as uniq, sort, nl read-in from standard input, transform text and export to standard output. sed and awk also transform text like filter, can be part of a pipeline, are programmable, have their own scripting language to specify transformation rules.

sed supports a number of operations,
s command (stand for substitute) is the one of the most commonly used commands,
d command removes empty lines.

Regular Expressions use to identify patterns, similar as glob expressions. While globs patterns match pathnames, regexes match any general text data. For example, [a-zA-Z0-9] match letters and digits and meta-characters include * . + ? $ | ():
* represents zero or more place expression,
+ is a regular expression,
^ anchors the match to the beginning of the record,
$ represents end of the expression.
capture parentheses use \( and /) , and back-reference use \1.
\b is a word boundary,
\w is a single word character,
\b\w\+\b matches a word.

sed 's/a/b/' - s:substitution, a:search string, b:replacement string
Syntax is s/REGEXP/REPLACEMENT/FLAGS.
Examples
echo 'upstream' | sed 's/up/down/' (replace up with down)
echo 'upstream and upward' | sed 's/up/down/' (replace once)
echo 'upstream and upward' | sed 's/up/down/g' (replace globally)
echo 'upstream and upward' | sed 's|up|down|g' (delimiter can be / or | or :)
echo 'Mac OS X/Unix: awesome.' | sed 's|Mac OS X/Unix:|sed is |' (use backslash to escape forward slash)
sed 's/apple/mango/' fruits.txt (replace apple with mango first one each line)
sed 's/apple/mango/g' fruits.txt (replace apple with mango globally)
echo 'During daytime we have sunlight.' | sed 's/day/night/'
echo 'During daytime we have sunlight.' | sed -e 's/day/night/' -e 's/sun/moon/' (add second command by -e, e stands for edits)
echo 'who needs vowels?' |sed 's/[aeiou]/_/g' (substitute aeiou using _)
echo 'who needs vowels?' |sed -E 's/[aeiou]+/_/g' (use extended features)
sed 's/^a/  A/g' fruits.txt (indent apple)
sed 's/^/>  /g' fruits.txt (indent everything)
sed 's/^/>(ctrl+v+Tab)/g' fruits.txt (indent using Tab)
sed -E 's/<[^<>]+>//g' homepage.html (take out html tags)
echo 'daytime' | sed 's/\(...)time/daylight/' (change ...time to daylight)
echo 'daytime' | sed -E 's/(...)time/\1daylight/' (extended replace and reused the matched word)
echo 'Dan Stevens' | sed -E 's/([A-Za-z]+) ([A-Za-z]+)/\2, \1/'(change the order of the first and second words)

- cat fruits.txt | sed 's/Apple/apple/' (replace Apple with apple)
- cat fruits.txt | sed 's/Apple/apple/i' (flag to specify case insensitive using /i)
- cat fruits.txt | sed 's/\(A\)pple/apple' (match partially, either Apple or apple will match)
- cat fruits.txt | sed 's/\(A\)pple/apple/g' (globally occurrence, match more than one match)
- cat fruits.txt | sed 's/\(A\)pple/\1pple/g' (globally occurrence, match using back-reference \1)
- cat fruits.txt | sed 's/\b\(A\)pple\b/\1pple/g' (find word boundary)
- cat fruits.txt | sed 's/\(\b\w\+\b\) \1/\1/g' (find the repeated word and keep the first one)
- cat fruits.txt | sed -e '9s/\(A\)pple/apple/g' (execute the s command for only line 9)
- cat fruits.txt | sed -e '9s/\(A\)pple/apple/g' (execute the s command except for line 9)
- cat fruits.txt | sed 's/^apple/xx/g' (replace for the beginning of the streams)
- cat fruits.txt | sed 's/apple$/xx/g' (replace for the end of the streams)
- cat fruits.txt | sed 's/^$/xx/g' (replace for empty lines)
- cat fruits.txt | sed 's/^ *$/xx/g'(* represents zero or more places, like the + metacharacter)
- cat fruits.txt | sed 's/apple | Apple/xx/g'(| represents regex1 or regex2) ??
- cat fruits.txt | sed {'s/apple/xx/g', 's/Apple/xx/g'} (substitute one by one)  ??
- cat fruits.txt | sed 'd' (delete the lines)
- sed -e 's/^-1/0 |f/' |sed -e 's/^+1/1 |f/' (Converts the labels from "-1" to "0" and from "+1" to "1" and put the features into a namespace called "f")
- sed -e 's/$/ const:.01/' (Adds a constant feature called "const" with value ".01")
- sed -i '18164755d' data.reduced (delete line  18164755)
- sed -n '1,1000p' (export first 1000 lines)