-
Notifications
You must be signed in to change notification settings - Fork 3
/
Copy pathKmean and loess.Rmd
118 lines (88 loc) · 3.54 KB
/
Kmean and loess.Rmd
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
---
title: "R Notebook"
output: html_notebook
---
This is an [R Markdown](http://rmarkdown.rstudio.com) Notebook. When you execute code within the notebook, the results appear beneath the code.
Try executing this chunk by clicking the *Run* button within the chunk or by placing your cursor inside it and pressing *Ctrl+Shift+Enter*.
```{r}
library(tidyverse)
overage = read_csv("~/9943478/wiki3.csv")
overage %>% filter(y2010>=0) %>% nrow
```
```{r}
overage %>% ungroup %>% filter(`y2010` > 1) %>% sample_n(10000) %>% ggplot() + geom_point() + aes(y=article_length,x=y2010) + scale_y_log10() + scale_x_log10() + geom_smooth(method="loess",span = .75)
```
```{r}
overage = overage %>% filter(y2010 > 0)
model = loess(log(article_length) ~ log(y2010+1),overage,span=.8)
overage$preds = predict(model)
```
```{r}
overage %>% mutate(error = log(article_length) - preds, scaled_error = scale(error) %>% as.numeric) %>% arrange(-abs(scaled_error)) %>% write_csv("~/9943478/errors.csv")
```
```{r}
mat = overage %>% select(starts_with("y")) %>% as.matrix
mat[mat==0] = NA
change = (mat[,-1])/(mat[,-ncol(mat)])
lchange = log(change)
smoother = sapply(1:ncol(lchange),function(i){
row = sapply(1:ncol(lchange),function(j) {
p = pnorm(i-j,0,2)
min(p,1-p)
})
row/sum(row)
})
lchange[is.na(lchange)] = 0
lchange[is.infinite(lchange)] = 0
lchange2 = lchange %*% smoother
k = 9
a = flexclust::kcca(lchange2,k,flexclust::kccaFamily("kmeans"),simple=T)
#a = kmeans(lchange,k,iter.max = 25,nstart = 5)
overage$cluster = a@cluster
plottable = lapply(1:k,function(n) {
p = lchange2[a@cluster==n,]
#exp(cumsum(apply(p,2,mean))) %>%
apply(p,2,mean) %>%
as.data.frame %>% mutate(year = seq(1800,2010,by=10),k = n)
}) %>% bind_rows
names(plottable)[1] = "population"
ggplot(plottable) + geom_line(aes(x=year,y=population)) + facet_wrap(~k,scales="free_y")
ggplot(overage %>% filter(lon < -25, lat < 50,lat > 23) %>% filter(state %in% c("MA","NY","NJ","PA","ME","VT","NH","RI","CT"))) + geom_point(aes(x=lon,y=lat,color=factor(cluster)),alpha=.05) + facet_wrap(~cluster)
pops
```
```{r}
overage %>% group_by(cluster) %>% arrange(-y2010) %>% filter(!is.na(lat),settlement_type=="City") %>% slice(1) %>% select(lat,lon,state,title)
overage %>% filter(title=="Detroit")
```
```{r}
loc_data = overage %>% filter(state=="ME") %>% filter(!grepl("County",title))
mat = loc_data %>% select(starts_with("y")) %>% as.matrix
mat[mat==0] = NA
change = (mat[,-1])/(mat[,-ncol(mat)])
lchange = log(change)
smoother = sapply(1:ncol(lchange),function(i){
row = sapply(1:ncol(lchange),function(j) {
p = pnorm(i-j,0,2)
min(p,1-p)
})
row/sum(row)
})
lchange[is.na(lchange)] = 0
lchange[is.infinite(lchange)] = 0
lchange2 = lchange %*% smoother
k = 6
a = flexclust::kcca(lchange2,k,flexclust::kccaFamily("kmeans"),simple=T)
#a = kmeans(lchange,k,iter.max = 25,nstart = 5)
loc_data$cluster = a@cluster
plottable = lapply(1:k,function(n) {
p = lchange2[a@cluster==n,]
#apply(p,2,mean))) %>%
apply(p,2,mean) %>%
cumsum %>% exp %>%
as.data.frame %>% mutate(year = seq(1800,2010,by=10),k = n)
}) %>% bind_rows
names(plottable)[1] = "population"
ggplot(plottable) + geom_line(aes(x=year,y=population)) + facet_wrap(~k,scales="free_y")
ggplot(loc_data %>% filter(lon < -25, lat < 50,lat > 23) %>% filter(state %in% c("MA","NY","NJ","PA","ME","VT","NH","RI","CT"))) + aes(x=lon,label=title,y=lat,color=factor(cluster)) + geom_point(alpha=.55,aes(size=y2010)) + facet_wrap(~cluster) + coord_map() + geom_text(data=loc_data %>% group_by(cluster) %>% arrange(-y2010) %>% slice(1:10))
pops
```