.Rhistory

}
text <- tolower(text)
text <- clean_tweet(text)
data$post_text <- as.vector(text)
write.csv(data, file="data/processed_tweets.csv")
library(tidyverse)
library(tidytext)
library(RColorBrewer)
library(wordcloud2)
library(stopwords)
library(webshot)
library(htmlwidgets)
data <- read_csv("data/processed_tweets.csv")
data("stop_words")
stop_words$word <- gsub("[[:punct:]]", "", stop_words$word)
set.seed(100)
words <- data %>% filter(label==1) %>%
select(post_text) %>%
unnest_tokens(word, post_text)
words <- words %>% anti_join(stop_words, by = "word")
words <- words %>% count(word, sort = TRUE)
negative <- wordcloud2(data=words, size=0.8, color='random-dark')
saveWidget(negative,"figure/negative.html",selfcontained = F)
webshot("figure/negative.html","figure/negative.png",
vwidth = 1080, vheight = 720, delay = 5)
words <- data %>% filter(label==0) %>%
select(post_text) %>%
unnest_tokens(word, post_text)
words <- words %>% anti_join(stop_words, by = "word")
words <- words %>% count(word, sort = TRUE)
positive <- wordcloud2(data=words, size=0.8, color='random-dark')
saveWidget(positive,"figure/positive.html",selfcontained = F)
webshot("figure/positive.html","figure/positive.png",
vwidth = 1080, vheight = 720, delay = 5)
webshot::install_phantomjs()
library(tidyverse)
library(tidytext)
library(RColorBrewer)
library(wordcloud2)
library(stopwords)
library(webshot)
library(htmlwidgets)
data <- read_csv("data/processed_tweets.csv")
data("stop_words")
stop_words$word <- gsub("[[:punct:]]", "", stop_words$word)
set.seed(100)
words <- data %>% filter(label==1) %>%
select(post_text) %>%
unnest_tokens(word, post_text)
words <- words %>% anti_join(stop_words, by = "word")
words <- words %>% count(word, sort = TRUE)
negative <- wordcloud2(data=words, size=0.8, color='random-dark')
saveWidget(negative,"figure/negative.html",selfcontained = F)
webshot("figure/negative.html","figure/negative.png",
vwidth = 1080, vheight = 720, delay = 5)
words <- data %>% filter(label==0) %>%
select(post_text) %>%
unnest_tokens(word, post_text)
words <- words %>% anti_join(stop_words, by = "word")
words <- words %>% count(word, sort = TRUE)
positive <- wordcloud2(data=words, size=0.8, color='random-dark')
saveWidget(positive,"figure/positive.html",selfcontained = F)
webshot("figure/positive.html","figure/positive.png",
vwidth = 1080, vheight = 720, delay = 5)
wordcount <- str_count(data$post_text, "\\w+")
wordcount <- str_count(data %>% filter(label==1) %>%
select(post_text), "\\w+")
data %>% filter(label==1) %>%
select(post_text)
wordcount <- str_count(data %>% filter(label==1)$post_text, "\\w+")
wordcount <- str_count(data %>% filter(label==1) %>%
select(post_text) %>% as.vector(), "\\w+")
data %>% filter(label==1) %>%
select(post_text) %>% as.vector()
wordcount <- str_count(as.vector(data %>% filter(label==1) %>%
select(post_text)), "\\w+")
as.vector(data %>% filter(label==1) %>%
select(post_text))
wordcount <- str_count(data$post_text[data$label==1], "\\w+")
wordcount
mean(wordcount)
library(tidyverse)
data <- read_csv("data/processed_tweets.csv")
wordcount <- str_count(data$post_text[data$label==1], "\\w+")
wordcount <- wordcount[!is.na(wordcount)]
wordcount <- str_count(data$post_text[data$label==1], "\\w+")
wordcount <- wordcount[!is.na(wordcount)]
len <- c("<=10", "11-20", "21-30", "31-40", "41-50", ">50")
find_freq <- function(wc, k){
freq = rep(0, k)
itv = c(1, 10)
for(i in 1:(k-1)){
for(j in 1:length(wc)){
if(wc[j] >= itv[1] & wc[j] <= itv[2]){
freq[i] = freq[i] + 1
}
itv = itv + 10
}
}
for(j in 1:length(wc)){
if(wc[j] >= itv[1]){
freq[k] = freq[k] + 1
}
}
return(freq)
}
freq <- find_freq(wordcount, 6)
mean(wordcount)
wordcount <- str_count(data$post_text[data$label==1], "\\w+")
wordcount <- wordcount[!is.na(wordcount)]
len <- c("<=5", "6-10", "11-15", "16-20", ">20")
find_freq <- function(wc, k){
freq = rep(0, k)
itv = c(1, 5)
for(i in 1:(k-1)){
for(j in 1:length(wc)){
if(wc[j] >= itv[1] & wc[j] <= itv[2]){
freq[i] = freq[i] + 1
}
itv = itv + 5
}
}
for(j in 1:length(wc)){
if(wc[j] >= itv[1]){
freq[k] = freq[k] + 1
}
}
return(freq)
}
freq <- find_freq(wordcount, 5)
wordcount <- str_count(data$post_text[data$label==1], "\\w+")
wordcount <- wordcount[!is.na(wordcount)]
len <- c("<=5", "6-10", "11-15", "16-20", ">20")
find_freq <- function(wc, k){
freq = rep(0, k)
itv = c(1, 5)
for(i in 1:(k-1)){
for(j in 1:length(wc)){
if(wc[j] >= itv[1] & wc[j] <= itv[2]){
freq[i] = freq[i] + 1
}
itv = itv + 5
}
}
for(j in 1:length(wc)){
if(wc[j] >= itv[1]){
freq[k] = freq[k] + 1
}
}
return(freq)
}
freq <- find_freq(wordcount, 5)
wordcount <- str_count(data$post_text[data$label==1], "\\w+")
wordcount <- wordcount[!is.na(wordcount)]
len <- c("<=5", "6-10", "11-15", "16-20", ">20")
find_freq <- function(wc, k){
freq = rep(0, k)
itv = c(1, 5)
for(i in 1:(k-1)){
for(j in 1:length(wc)){
if(wc[j] >= itv[1] && wc[j] <= itv[2]){
freq[i] = freq[i] + 1
}
itv = itv + 5
}
}
for(j in 1:length(wc)){
if(wc[j] >= itv[1]){
freq[k] = freq[k] + 1
}
}
return(freq)
}
freq <- find_freq(wordcount, 5)
wordcount[2] >= 10
wordcount[2] >= 10 && wordcount[2] <= 20
length(wordcount)
wordcount <- str_count(data$post_text[data$label==1], "\\w+")
wordcount <- wordcount[!is.na(wordcount)]
len <- c("<=5", "6-10", "11-15", "16-20", ">20")
find_freq <- function(wc, k){
freq = rep(0, k)
itv = c(1, 5)
for(i in 1:(k-1)){
for(j in 1:length(wc)){
if(wc[j] >= itv[1] && wc[j] <= itv[2]){
print(wc[j])
freq[i] = freq[i] + 1
}
itv = itv + 5
}
}
for(j in 1:length(wc)){
if(wc[j] >= itv[1]){
freq[k] = freq[k] + 1
}
}
return(freq)
}
freq <- find_freq(wordcount, 5)
wordcount <- str_count(data$post_text[data$label==1], "\\w+")
wordcount <- wordcount[!is.na(wordcount)]
len <- c("<=5", "6-10", "11-15", "16-20", ">20")
find_freq <- function(wc, k){
freq <- rep(0, k)
itv <- c(1, 5)
for(i in 1:(k-1)){
for(j in 1:length(wc)){
if(wc[j] >= itv[1] && wc[j] <= itv[2]){
print(wc[j])
freq[i] <- freq[i] + 1
}
itv = itv + 5
}
}
for(j in 1:length(wc)){
if(wc[j] >= itv[1]){
freq[k] <- freq[k] + 1
}
}
return(freq)
}
freq <- find_freq(wordcount, 5)
wordcount <- str_count(data$post_text[data$label==1], "\\w+")
wordcount <- wordcount[!is.na(wordcount)]
len <- c("<=5", "6-10", "11-15", "16-20", ">20")
find_freq <- function(wc, k){
freq <- rep(0, k)
itv <- c(1, 5)
for(i in 1:(k-1)){
for(j in 1:length(wc)){
if(wc[j] >= itv[1] && wc[j] <= itv[2]){
freq[i] <- freq[i] + 1
}
}
itv = itv + 5
}
for(j in 1:length(wc)){
if(wc[j] >= itv[1]){
freq[k] <- freq[k] + 1
}
}
return(freq)
}
freq <- find_freq(wordcount, 5)
wordcount <- str_count(data$post_text[data$label==1], "\\w+")
wordcount <- wordcount[!is.na(wordcount)]
len <- c("<=5", "6-10", "11-15", "16-20", ">20")
find_freq <- function(wc, k){
freq <- rep(0, k)
itv <- c(1, 5)
for(i in 1:(k-1)){
for(j in 1:length(wc)){
if(wc[j] >= itv[1] && wc[j] <= itv[2]){
freq[i] <- freq[i] + 1
}
}
itv = itv + 5
}
for(j in 1:length(wc)){
if(wc[j] >= itv[1]){
freq[k] <- freq[k] + 1
}
}
return(freq)
}
freq <- find_freq(wordcount, 5)
df <- data.frame(length = len, frequency = freq)
ggplot(df, aes(length, frequency)) +
geom_linerange(
aes(x = length, ymin = 0, ymax = prop),
color = "lightgray", size = 1.5
) +
geom_point(aes(color = length), size = 2)+
ggpubr::color_palette("jco")+
theme_pubclean()
install.packages("ggpubr")
library(ggpubr)
wordcount <- str_count(data$post_text[data$label==1], "\\w+")
wordcount <- wordcount[!is.na(wordcount)]
len <- c("<=5", "6-10", "11-15", "16-20", ">20")
find_freq <- function(wc, k){
freq <- rep(0, k)
itv <- c(1, 5)
for(i in 1:(k-1)){
for(j in 1:length(wc)){
if(wc[j] >= itv[1] && wc[j] <= itv[2]){
freq[i] <- freq[i] + 1
}
}
itv = itv + 5
}
for(j in 1:length(wc)){
if(wc[j] >= itv[1]){
freq[k] <- freq[k] + 1
}
}
return(freq)
}
freq <- find_freq(wordcount, 5)
df <- data.frame(length = len, frequency = freq)
ggplot(df, aes(length, frequency)) +
geom_linerange(
aes(x = length, ymin = 0, ymax = prop),
color = "lightgray", size = 1.5
) +
geom_point(aes(color = length), size = 2)+
ggpubr::color_palette("jco")+
theme_pubclean()
library(ggpubr)
wordcount <- str_count(data$post_text[data$label==1], "\\w+")
wordcount <- wordcount[!is.na(wordcount)]
len <- c("<=5", "6-10", "11-15", "16-20", ">20")
find_freq <- function(wc, k){
freq <- rep(0, k)
itv <- c(1, 5)
for(i in 1:(k-1)){
for(j in 1:length(wc)){
if(wc[j] >= itv[1] && wc[j] <= itv[2]){
freq[i] <- freq[i] + 1
}
}
itv = itv + 5
}
for(j in 1:length(wc)){
if(wc[j] >= itv[1]){
freq[k] <- freq[k] + 1
}
}
return(freq)
}
freq <- find_freq(wordcount, 5)
df <- data.frame(length = len, frequency = freq)
ggplot(df, aes(length, frequency)) +
geom_linerange(
aes(x = length, ymin = 0, ymax = frequency),
color = "lightgray", size = 1.5
) +
geom_point(aes(color = length), size = 2)+
ggpubr::color_palette("jco")+
theme_pubclean()
View(df)
set.seed(99)
wordcount <- str_count(data$post_text, "\\w+")
label <- label[!is.na(wordcount)]
set.seed(99)
wordcount <- str_count(data$post_text, "\\w+")
label <- data$label[!is.na(wordcount)]
wordcount <- wordcount[!is.na(wordcount)]
df <- data.frame(wordcount=wordcount, label=label)
ggplot(df, aes(wordcount)) +
geom_density(aes(fill = label), alpha = 0.4) +
geom_vline(aes(xintercept = grp.mean, color = label),
data = mu, linetype = "dashed") +
scale_color_manual(values = c("#868686FF", "#EFC000FF"))+
scale_fill_manual(values = c("#868686FF", "#EFC000FF"))
set.seed(99)
wordcount <- str_count(data$post_text, "\\w+")
label <- data$label[!is.na(wordcount)]
wordcount <- wordcount[!is.na(wordcount)]
mu <- c(mean(wordcount[label==0], mean(wordcount[label==1])))
df <- data.frame(wordcount=wordcount, label=label)
ggplot(df, aes(wordcount)) +
geom_density(aes(fill = label), alpha = 0.4) +
geom_vline(aes(xintercept = grp.mean, color = label),
data = mu, linetype = "dashed") +
scale_color_manual(values = c("#868686FF", "#EFC000FF"))+
scale_fill_manual(values = c("#868686FF", "#EFC000FF"))
set.seed(99)
wordcount <- str_count(data$post_text, "\\w+")
label <- data$label[!is.na(wordcount)]
wordcount <- wordcount[!is.na(wordcount)]
mu <- df(wc.mean = c(mean(wordcount[label==0]), mean(wordcount[label==1])),
label = c(0, 1))
mean(wordcount[label==0])
mean(wordcount[label==1])
set.seed(99)
wordcount <- str_count(data$post_text, "\\w+")
label <- data$label[!is.na(wordcount)]
wordcount <- wordcount[!is.na(wordcount)]
mu <- data.frame(wc.mean = c(mean(wordcount[label==0]),
mean(wordcount[label==1])),
label = c(0, 1))
df <- data.frame(wordcount=wordcount, label=label)
ggplot(df, aes(wordcount)) +
geom_density(aes(fill = label), alpha = 0.4) +
geom_vline(aes(xintercept = grp.mean, color = label),
data = mu, linetype = "dashed") +
scale_color_manual(values = c("#868686FF", "#EFC000FF"))+
scale_fill_manual(values = c("#868686FF", "#EFC000FF"))
set.seed(99)
wordcount <- str_count(data$post_text, "\\w+")
label <- data$label[!is.na(wordcount)]
wordcount <- wordcount[!is.na(wordcount)]
mu <- data.frame(wc.mean = c(mean(wordcount[label==0]),
mean(wordcount[label==1])),
label = c(0, 1))
df <- data.frame(wordcount=wordcount, label=label)
ggplot(df, aes(wordcount)) +
geom_density(aes(fill = label), alpha = 0.4) +
geom_vline(aes(xintercept = wc.mean, color = label),
data = mu, linetype = "dashed") +
scale_color_manual(values = c("#868686FF", "#EFC000FF"))+
scale_fill_manual(values = c("#868686FF", "#EFC000FF"))
set.seed(99)
wordcount <- str_count(data$post_text, "\\w+")
label <- data$label[!is.na(wordcount)]
wordcount <- wordcount[!is.na(wordcount)]
mu <- data.frame(wc.mean = c(mean(wordcount[label==0]),
mean(wordcount[label==1])),
label = c(0, 1))
df <- data.frame(wordcount=wordcount, label=label)
ggplot(df, aes(wordcount)) +
geom_density(aes(fill = label), alpha = 0.4) +
scale_color_manual(values = c("#868686FF", "#EFC000FF"))+
scale_fill_manual(values = c("#868686FF", "#EFC000FF"))
set.seed(99)
wordcount <- str_count(data$post_text, "\\w+")
label <- data$label[!is.na(wordcount)]
wordcount <- wordcount[!is.na(wordcount)]
mu <- data.frame(wc.mean = c(mean(wordcount[label==0]),
mean(wordcount[label==1])),
label = c(0, 1))
df <- data.frame(wordcount=wordcount, label=label)
ggplot(df, aes(wordcount)) + geom_density(aes(color = label)) +
scale_color_manual(values = c("#868686FF", "#EFC000FF"))
set.seed(99)
wordcount <- str_count(data$post_text, "\\w+")
label <- data$label[!is.na(wordcount)]
wordcount <- wordcount[!is.na(wordcount)]
mu <- data.frame(wc.mean = c(mean(wordcount[label==0]),
mean(wordcount[label==1])),
label = c(0, 1))
df <- data.frame(wordcount=wordcount, label=as.factor(label))
ggplot(df, aes(wordcount)) + geom_density(aes(color = label)) +
scale_color_manual(values = c("#868686FF", "#EFC000FF"))
set.seed(99)
wordcount <- str_count(data$post_text, "\\w+")
label <- data$label[!is.na(wordcount)]
wordcount <- wordcount[!is.na(wordcount)]
mu <- data.frame(wc.mean = c(mean(wordcount[label==0]),
mean(wordcount[label==1])),
label = c(0, 1))
df <- data.frame(wordcount=wordcount, label=as.factor(label))
ggplot(df, aes(wordcount)) + geom_density(aes(color = label)) +
scale_color_manual(values = c("#3D85C6", "#EFC000FF"))
library(ggpubr)
set.seed(99)
wordcount <- str_count(data$post_text, "\\w+")
label <- data$label[!is.na(wordcount)]
wordcount <- wordcount[!is.na(wordcount)]
mu <- data.frame(wc.mean = c(mean(wordcount[label==0]),
mean(wordcount[label==1])),
label = c(0, 1))
df <- data.frame(wordcount=wordcount, label=as.factor(label))
ggdensity(df, x = "wordcount",
add = "mean", rug = TRUE,
color = "label", fill = "label",
palette = c("#0073C2FF", "#FC4E07"))
set.seed(99)
wordcount <- str_count(data$post_text, "\\w+")
label <- data$label[!is.na(wordcount)]
wordcount <- wordcount[!is.na(wordcount)]
mu <- data.frame(wc.mean = c(mean(wordcount[label==0]),
mean(wordcount[label==1])),
label = c(0, 1))
df <- data.frame(wordcount=wordcount, label=as.factor(label))
mu <- df %>%
group_by(label) %>%
summarise(wc.mean = mean(wordcount))
ggplot(df, aes(wordcount)) +
geom_density(aes(fill = label), alpha = 0.4) +
geom_vline(aes(xintercept = wc.mean, color = label),
data = mu, linetype = "dashed") +
scale_color_manual(values = c("#3D85C6", "#EFC000FF")) +
scale_fill_manual(values = c("#3D85C6", "#EFC000FF"))
set.seed(99)
wordcount <- str_count(data$post_text, "\\w+")
label <- data$label[!is.na(wordcount)]
wordcount <- wordcount[!is.na(wordcount)]
mu <- data.frame(wc.mean = c(mean(wordcount[label==0]),
mean(wordcount[label==1])),
label = c(0, 1))
df <- data.frame(wordcount=wordcount, label=as.factor(label))
mu <- df %>%
group_by(label) %>%
summarise(wc.mean = mean(wordcount))
dens <- ggplot(df, aes(wordcount)) +
geom_density(aes(fill = label), alpha = 0.4) +
geom_vline(aes(xintercept = wc.mean, color = label),
data = mu, linetype = "dashed") +
scale_color_manual(values = c("#3D85C6", "#EFC000FF")) +
scale_fill_manual(values = c("#3D85C6", "#EFC000FF"))
ggsave("figure/density.png", dens)
set.seed(99)
wordcount <- str_count(data$post_text, "\\w+")
label <- data$label[!is.na(wordcount)]
wordcount <- wordcount[!is.na(wordcount)]
mu <- data.frame(wc.mean = c(mean(wordcount[label==0]),
mean(wordcount[label==1])),
label = c(0, 1))
df <- data.frame(wordcount=wordcount, label=as.factor(label))
mu <- df %>%
group_by(label) %>%
summarise(wc.mean = mean(wordcount))
dens <- ggplot(df, aes(wordcount)) +
geom_density(aes(fill = label), alpha = 0.4) +
geom_vline(aes(xintercept = wc.mean, color = label),
data = mu, linetype = "dashed") +
scale_color_manual(values = c("#3D85C6", "#EFC000FF")) +
scale_fill_manual(values = c("#3D85C6", "#EFC000FF"))
ggsave("figure/density.png", dens, width = 8, height = 5)