.Rhistory

#' @return nothing.
#' @author Jason Bryer <jason@bryer.org>
convertRMarkdown <- function(dir=getwd(), images.dir=dir, images.url='/images/',
out_ext='.markdown', in_ext='.rmd', recursive=FALSE) {
require(knitr, quietly=TRUE, warn.conflicts=FALSE)
files <- list.files(path=dir, pattern=in_ext, ignore.case=TRUE, recursive=recursive)
for(f in files) {
message(paste("Processing ", f, sep=''))
content <- readLines(f)
frontMatter <- which(substr(content, 1, 3) == '---')
if(length(frontMatter) >= 2 & 1 %in% frontMatter) {
statusLine <- which(substr(content, 1, 7) == 'status:')
publishedLine <- which(substr(content, 1, 10) == 'published:')
if(statusLine > frontMatter[1] & statusLine < frontMatter[2]) {
status <- unlist(strsplit(content[statusLine], ':'))[2]
status <- sub('[[:space:]]+$', '', status)
status <- sub('^[[:space:]]+', '', status)
if(tolower(status) == 'process') {
#This is a bit of a hack but if a line has zero length (i.e. a
#black line), it will be removed in the resulting markdown file.
#This will ensure that all line returns are retained.
content[nchar(content) == 0] <- ' '
message(paste('Processing ', f, sep=''))
content[statusLine] <- 'status: publish'
content[publishedLine] <- 'published: true'
outFile <- paste(substr(f, 1, (nchar(f)-(nchar(in_ext)))), out_ext, sep='')
render_markdown(strict=TRUE)
opts_knit$set(out.format='markdown')
opts_knit$set(base.dir=images.dir)
opts_knit$set(base.url=images.url)
try(knit(text=content, output=outFile), silent=FALSE)
} else {
warning(paste("Not processing ", f, ", status is '", status,
"'. Set status to 'process' to convert.", sep=''))
}
} else {
warning("Status not found in front matter.")
}
} else {
warning("No front matter found. Will not process this file.")
}
}
invisible()
}
convertRMarkdown
convertRMarkdown()
rmd.sh
rmd.sh
convertRMarkdown()
convertRMarkdown()
convertRMarkdown()
install.packages("rvest")
library(rvest) # for html parsing
library(dplyr) # for data manipulation
library(tidyr) # for TODO
library(ggplot2)
library(readr)
# the url of the page we scrape from
space_weather <- read_html("https://www.spaceweatherlive.com/en/solar-activity/top-50-solar-flares")
# useful rvest tutorial: https://www.r-bloggers.com/using-rvest-to-scrape-an-html-table/
rank_tables <- space_weather %>%
html_node(xpath = "/html/body/div[3]/div/div/div[1]/table") %>%
html_table() %>%
`colnames<-`(c("rank",
"x_classification",
"date",
"region",
"start_time",
"maximum_time",
"end_time",
"movie"))
# make rank_table a tibble for better printing
rank_tables <- tbl_df(rank_tables)
#take a look...
rank_tables
rank_tables <- rank_tables %>% select(-movie)
# unite the date with each of the three times, seperate by space, do not remove input columns
rank_tables <- rank_tables %>% unite("start_datetime", date, start_time, sep = " ", remove=FALSE)
rank_tables <- rank_tables %>% unite("max_datetime", date, maximum_time, sep = " ", remove=FALSE)
rank_tables <- rank_tables %>% unite("end_datetime", date, end_time, sep = " ", remove=FALSE)
# remove the date, start_time, maximum_time, end_time, now that we don't need them
rank_tables <- rank_tables %>% select(-date, -start_time, -maximum_time, -end_time)
rank_tables <- rank_tables %>%
mutate(region = ifelse(stringr::str_detect("-", region), NA, region))
rank_tables <- rank_tables %>% readr::type_convert(col_types = cols(
start_datetime = col_datetime(format="%Y/%m/%d %H:%M"),
end_datetime = col_datetime(format="%Y/%m/%d %H:%M"),
max_datetime = col_datetime(format="%Y/%m/%d %H:%M")
))
rank_tables
waves <- read_html("http://www.hcbravo.org/IntroDataSci/misc/waves_type2.html")
text <- waves %>% html_node(xpath = "/html/body/pre") %>%
html_text()
# split the text by line and restore it as a vector
text <- stringr::str_split(text, "\n")
# save the long string as a data frame
wind_tab <- tbl_df(data.frame(text))
# rename the single column of the data frame to 'first'
names(wind_tab)[1] <- "first"
# only select the lines of text that start with a date (i.e. the filter for the data we want)
wind_tab <- wind_tab %>% filter(first == stringr::str_detect(first, "\\d{4}\\/\\d{2}\\/\\d{2}"))
# create a column names vector (we ignore the link and the notes columns)
col_names <- c("start_date",
"start_time",
"end_date",
"end_time",
"start_frequency",
"end_frequency",
"flare_location",
"flare_region",
"flare_classification",
"cme_date",
"cme_time",
"cme_angle",
"cme_width",
"cme_speed")
# seperate column by spaces, ignore the warning about too many values: this happens because we are
# not seperating past the cme_speed column and ignoring it (and also the notes column for some of
# the entities)
wind_tab <- wind_tab %>% tidyr::separate(col = "first", into = col_names, sep = "\\s+")
# take a look...
wind_tab
# Frequency
# replace start_frequency
wind_tab <- wind_tab %>%
mutate(start_frequency = ifelse((start_frequency == "????"), NA, start_frequency))
# replace end_frequency
wind_tab <- wind_tab %>%
mutate(end_frequency = ifelse((end_frequency == "????"), NA, end_frequency))
# Flare
# replace source region location (flare_location)
wind_tab <- wind_tab %>%
mutate(flare_location = ifelse((flare_location == "------"), NA, flare_location))
# replace active region location (flare_region)
wind_tab <- wind_tab %>%
mutate(flare_region = ifelse((flare_region == "-----"), NA, flare_region))
# replace flare impotence (flare_classification)
wind_tab <- wind_tab %>%
mutate(flare_classification = ifelse((flare_classification == "----"), NA, flare_classification))
# CME Parameters
# replace cme_date
wind_tab <- wind_tab %>%
mutate(cme_date = ifelse((cme_date == "--/--"), NA, cme_date))
# replace cme_time
wind_tab <- wind_tab %>%
mutate(cme_time = ifelse((cme_time == "--:--"), NA, cme_time))
# replace cme_angle (cme CPA)
wind_tab <- wind_tab %>%
mutate(cme_angle = ifelse((cme_angle == "----"), NA, cme_angle))
# replace cme_width (need to replace both 3-dashes and 4-dashes)
wind_tab <- wind_tab %>%
mutate(cme_width = ifelse((cme_width == "---"), NA, cme_width))
wind_tab <- wind_tab %>%
mutate(cme_width = ifelse((cme_width == "----"), NA, cme_width))
# replace cme_speed
wind_tab <- wind_tab %>%
mutate(cme_speed = ifelse((cme_speed == "----"), NA, cme_speed))
wind_tab <- wind_tab %>% mutate(cme_halo = ((cme_angle) == "Halo"))
wind_tab <- wind_tab %>% mutate(cme_angle = ifelse((cme_angle == "Halo"), NA, cme_angle))
# make logical vector (attribute) cme_width_lb
wind_tab <- wind_tab %>% mutate(cme_width_lb = stringr::str_detect(cme_width, ">\\d+"))
# remove the '>' symbol
wind_tab <- wind_tab %>% mutate(cme_width = sub(">","",cme_width))
# unite the date with each of the three times, seperate by space, do not remove input columns
wind_tab <- wind_tab %>% unite("start_datetime", start_date, start_time, sep = " ", remove=FALSE)
# extract start year for end_datetime and cme_datetime
wind_tab <- wind_tab  %>%
separate("start_date", into = "start_dateyear", sep = "/", remove=FALSE, extra = "drop")
wind_tab <- wind_tab %>%
unite("end_datetime", start_dateyear, end_date, end_time, sep = " ", remove=FALSE)
wind_tab <- wind_tab %>%
unite("cme_datetime", start_dateyear, cme_date, cme_time, sep = " ", remove=FALSE)
# remove the date, start_time, maximum_time, end_time, now that we don't need them
wind_tab <- wind_tab %>%
select(-start_date, -start_time, -end_date, -end_time, -cme_time, -cme_date,-start_dateyear)
# convert the <chr> columns to <datetime>
wind_tab <- wind_tab %>% readr::type_convert(col_types = cols(
start_datetime = col_datetime(format="%Y/%m/%d %H:%M"),
end_datetime = col_datetime(format="%Y %m/%d %H:%M"),
cme_datetime = col_datetime(format="%Y %m/%d %H:%M")
))
# take a look...
wind_tab
# seperate flare_classification into main classification and sub classification
tmp <- wind_tab  %>% separate("flare_classification", into = c("main_classification",
"sub_classification"),
sep = "\\.", remove=FALSE)
# seperate main classification into its letter and number
tmp <- tmp  %>%
separate("main_classification", into = c("main_letter_class",
"main_num_class"),
sep = 1,
remove=FALSE)
# convert the classification <chr> to <int>
tmp <- tmp %>% readr::type_convert(col_types = cols(
main_num_class = col_integer(),
sub_classification = col_integer()
))
# order the classification by letter and number
top_class <- tmp %>%
arrange(desc(main_letter_class), desc(main_num_class), desc(sub_classification)) %>%
select(flare_classification)
# display the top 50 classifications side by side
data.frame(rank_tables$rank, top_class[1:50,], rank_tables$x_classification)
# seperate flare_classification into main classification and sub classification
wind_w_class <- wind_tab  %>%
separate("flare_classification", into = c("main_classification",
"sub_classification"),
sep = "\\.", remove=FALSE)
# seperate x_classification into main classification and sub classification
rank_w_class <- rank_tables %>%
separate("x_classification", into = c("main_classification",
"sub_classification"),
sep = "\\.", remove=FALSE)
# seperate date and time of wind_w_class
wind_w_class <- wind_w_class  %>%
separate("start_datetime", into = c("startdate", "starttime"), sep = " ", remove=FALSE)
# seperate date and time of rank_w_class
rank_w_class <- rank_w_class  %>%
separate("start_datetime", into = c("startdate", "starttime"), sep = " ", remove=FALSE)
# the third rank solar flare (from rank_tables) has a classification of 'X17.2' while on the
#wind_tab, the same corresponding classification is a 'X17.', notice the missing '.2'
# has no sub_classification (notice nothing prints under the sub_classification column)
wind_w_class %>%
filter(stringr::str_detect(main_classification, "X17")) %>%
select(start_datetime, flare_classification, main_classification, sub_classification)
# HAS a sub_classification
rank_w_class %>%
filter(stringr::str_detect(main_classification, "X17")) %>%
select(start_datetime, main_classification, sub_classification, rank)
# only keep primary key (classification and startdate) from the rank table
rank_w_class <- rank_w_class %>%
select(rank, main_classification, startdate)
# left join the ranks to the large nasa table (i.e. wind_w_class)
wind_w_class <- wind_w_class %>%
left_join(rank_w_class, by = c("main_classification", "startdate"))
# remove the splited columns we created
wind_w_class <- wind_w_class %>%
select(-main_classification, -sub_classification, -startdate, -starttime)
# set our original wind_tab to this new one that includes rank information
wind_tab <- wind_w_class
# take a look...
wind_tab
# and also w/rank
wind_tab %>%
select(start_datetime, flare_classification, rank)
# cme_speed changing over the years
wind_tab %>%
mutate(month = (format(start_datetime, "%m"))) %>%
group_by("month") %>%
ggplot(aes(x=start_datetime, y=cme_speed)) +
geom_smooth() +
xlab("Date") +
ylab("CME Speed")
# cme_speed vs cme_width of the top 50
wind_tab %>% filter(!is.na(rank)) %>%
ggplot(aes(x=cme_speed, y =cme_width, label = rank))  +
geom_text() +
geom_smooth()
# plot a bar graph of the number of flares per month and cluster the strong (top 50) from the weak
wind_tab %>%
mutate(month = (format(start_datetime, "%m")), top_50 = !is.na(rank)) %>%
group_by(month,is.na(rank)) %>%
ggplot(aes(x = month,fill=top_50)) + geom_bar()
wind_tab %>% filter(!is.na(rank)) %>%
ggplot(aes(x=cme_speed, y =cme_width, label = rank))  +
geom_text() +
geom_smooth()
warnings()
warning()
warnings()
install.packages(c("knitr", "servr", "devtools"))
install.packages(c("knitr", "servr", "devtools"))
libary(brocks)
library(brocks)
devtools::install_github("hadley/lubridate")
brocks reqs dev version
devtools::install_github("brendan-r/brocks")
library(brocks)
new_post("My new Post!")
setwd("~/Documents/CMSC320")
new_post("My new Post!")
setwd("~/mysite")
setwd("~/mysite")
new_post("My new Post!")
#' This R script will process all R mardown files (those with in_ext file extention,
#' .rmd by default) in the current working directory. Files with a status of
#' 'processed' will be converted to markdown (with out_ext file extention, '.markdown'
#' by default). It will change the published parameter to 'true' and change the
#' status parameter to 'publish'.
#'
#' @param path_site path to the local root storing the site files
#' @param dir_rmd directory containing R Markdown files (inputs)
#' @param dir_md directory containing markdown files (outputs)
#' @param url_images where to store/get images created from plots directory +"/" (relative to path_site)
#' @param out_ext the file extention to use for processed files.
#' @param in_ext the file extention of input files to process.
#' @param recursive should rmd files in subdirectories be processed.
#' @return nothing.
#' @author Jason Bryer <jason@bryer.org> edited by Andy South
rmd2md <- function( path_site = getwd(),
dir_rmd = "_rmd",
dir_md = "_posts",
#dir_images = "figures",
url_images = "figures/",
out_ext='.md',
in_ext='.rmd',
recursive=FALSE) {
require(knitr, quietly=TRUE, warn.conflicts=FALSE)
#andy change to avoid path problems when running without sh on windows
files <- list.files(path=file.path(path_site,dir_rmd), pattern=in_ext, ignore.case=TRUE, recursive=recursive)
for(f in files) {
message(paste("Processing ", f, sep=''))
content <- readLines(file.path(path_site,dir_rmd,f))
frontMatter <- which(substr(content, 1, 3) == '---')
if(length(frontMatter) >= 2 & 1 %in% frontMatter) {
statusLine <- which(substr(content, 1, 7) == 'status:')
publishedLine <- which(substr(content, 1, 10) == 'published:')
if(statusLine > frontMatter[1] & statusLine < frontMatter[2]) {
status <- unlist(strsplit(content[statusLine], ':'))[2]
status <- sub('[[:space:]]+$', '', status)
status <- sub('^[[:space:]]+', '', status)
if(tolower(status) == 'process') {
#This is a bit of a hack but if a line has zero length (i.e. a
#black line), it will be removed in the resulting markdown file.
#This will ensure that all line returns are retained.
content[nchar(content) == 0] <- ' '
message(paste('Processing ', f, sep=''))
content[statusLine] <- 'status: publish'
content[publishedLine] <- 'published: true'
#andy change to path
outFile <- file.path(path_site, dir_md, paste0(substr(f, 1, (nchar(f)-(nchar(in_ext)))), out_ext))
#render_markdown(strict=TRUE)
#render_markdown(strict=FALSE) #code didn't render properly on blog
#andy change to render for jekyll
render_jekyll(highlight = "pygments")
#render_jekyll(highlight = "prettify") #for javascript
opts_knit$set(out.format='markdown')
# andy BEWARE don't set base.dir!! it caused me problems
# "base.dir is never used when composing the URL of the figures; it is
# only used to save the figures to a different directory.
# The URL of an image is always base.url + fig.path"
# https://groups.google.com/forum/#!topic/knitr/18aXpOmsumQ
opts_knit$set(base.url = "/")
opts_chunk$set(fig.path = url_images)
#andy I could try to make figures bigger
#but that might make not work so well on mobile
#opts_chunk$set(fig.width  = 8.5,
#               fig.height = 5.25)
try(knit(text=content, output=outFile), silent=FALSE)
} else {
warning(paste("Not processing ", f, ", status is '", status,
"'. Set status to 'process' to convert.", sep=''))
}
} else {
warning("Status not found in front matter.")
}
} else {
warning("No front matter found. Will not process this file.")
}
}
invisible()
}
rmd2md()
setwd("~/mysite/yusufameri.github.io/_rmd")
rmd2md()
wd()
pwd()
pwd
setwd("~/mysite/yusufameri.github.io/_rmd")
#' @author Jason Bryer <jason@bryer.org> edited by Andy South
#' This R script will process all R mardown files (those with in_ext file extention,
#' .rmd by default) in the current working directory. Files with a status of
#' 'processed' will be converted to markdown (with out_ext file extention, '.markdown'
#' by default). It will change the published parameter to 'true' and change the
#' status parameter to 'publish'.
#'
#' @param path_site path to the local root storing the site files
#' @param dir_rmd directory containing R Markdown files (inputs)
#' @param dir_md directory containing markdown files (outputs)
#' @param url_images where to store/get images created from plots directory +"/" (relative to path_site)
#' @param out_ext the file extention to use for processed files.
#' @param in_ext the file extention of input files to process.
#' @param recursive should rmd files in subdirectories be processed.
#' @return nothing.
#' @author Jason Bryer <jason@bryer.org> edited by Andy South
rmd2md <- function( path_site = getwd(),
dir_rmd = "_rmd",
dir_md = "_posts",
#dir_images = "figures",
url_images = "figures/",
out_ext='.md',
in_ext='.rmd',
recursive=FALSE) {
require(knitr, quietly=TRUE, warn.conflicts=FALSE)
#andy change to avoid path problems when running without sh on windows
files <- list.files(path=file.path(path_site,dir_rmd), pattern=in_ext, ignore.case=TRUE, recursive=recursive)
for(f in files) {
message(paste("Processing ", f, sep=''))
content <- readLines(file.path(path_site,dir_rmd,f))
frontMatter <- which(substr(content, 1, 3) == '---')
if(length(frontMatter) >= 2 & 1 %in% frontMatter) {
statusLine <- which(substr(content, 1, 7) == 'status:')
publishedLine <- which(substr(content, 1, 10) == 'published:')
if(statusLine > frontMatter[1] & statusLine < frontMatter[2]) {
status <- unlist(strsplit(content[statusLine], ':'))[2]
status <- sub('[[:space:]]+$', '', status)
status <- sub('^[[:space:]]+', '', status)
if(tolower(status) == 'process') {
#This is a bit of a hack but if a line has zero length (i.e. a
#black line), it will be removed in the resulting markdown file.
#This will ensure that all line returns are retained.
content[nchar(content) == 0] <- ' '
message(paste('Processing ', f, sep=''))
content[statusLine] <- 'status: publish'
content[publishedLine] <- 'published: true'
#andy change to path
outFile <- file.path(path_site, dir_md, paste0(substr(f, 1, (nchar(f)-(nchar(in_ext)))), out_ext))
#render_markdown(strict=TRUE)
#render_markdown(strict=FALSE) #code didn't render properly on blog
#andy change to render for jekyll
render_jekyll(highlight = "pygments")
#render_jekyll(highlight = "prettify") #for javascript
opts_knit$set(out.format='markdown')
# andy BEWARE don't set base.dir!! it caused me problems
# "base.dir is never used when composing the URL of the figures; it is
# only used to save the figures to a different directory.
# The URL of an image is always base.url + fig.path"
# https://groups.google.com/forum/#!topic/knitr/18aXpOmsumQ
opts_knit$set(base.url = "/")
opts_chunk$set(fig.path = url_images)
#andy I could try to make figures bigger
#but that might make not work so well on mobile
#opts_chunk$set(fig.width  = 8.5,
#               fig.height = 5.25)
try(knit(text=content, output=outFile), silent=FALSE)
} else {
warning(paste("Not processing ", f, ", status is '", status,
"'. Set status to 'process' to convert.", sep=''))
}
} else {
warning("Status not found in front matter.")
}
} else {
warning("No front matter found. Will not process this file.")
}
}
invisible()
}
rmd2md()
install.packages(c('servr', 'knitr'), repos = 'http://cran.rstudio.com')
install.packages(c("servr", "knitr"), repos = "http://cran.rstudio.com")
servr::jekyll()
servr::jekyll()
servr::jekyll()
setwd("~/mysite/yusufameri.github.io")
servr::jekyll()
?servr::jekyll
servr::jekyll()
setwd("~/mysite/yusufameri.github.io/_source")
servr::jekyll()
setwd("~/mysite/yusufameri.github.io")
servr::jekyll()
setwd("~/mysite/yusufameri.github.io")
servr::jekyll()
knitr::render_jekyll()
KnitPost <- function(input, base.url = "/") {
require(knitr)
opts_knit$set(base.url = base.url)
fig.path <- paste0("figs/", sub(".Rmd$", "", basename(input)), "/")
opts_chunk$set(fig.path = fig.path)
opts_chunk$set(fig.cap = "center")
render_jekyll()
knit(input, envir = parent.frame())
}
KnitPost("2012-07-03-knitr-jekyll.Rmd")
KnitPost <- function(input, base.url = "/") {
require(knitr)
opts_knit$set(base.url = base.url)
fig.path <- paste0("figs/", sub(".Rmd$", "", basename(input)), "/")
opts_chunk$set(fig.path = fig.path)
opts_chunk$set(fig.cap = "center")
render_jekyll()
knit(input, envir = parent.frame())
}
KnitPost("_source/2016-11-18-Nasa-R-EDA.Rmd")
KnitPost("_source/2016-11-18-Nasa-R-EDA.Rmd")
install.packages("tibble")
install.packages("rvest")
KnitPost("_source/2016-11-18-Nasa-R-EDA.Rmd")
KnitPost("_source/2016-11-18-Nasa-R-EDA.Rmd")
KnitPost("_source/2016-11-18-Nasa-R-EDA.Rmd")
setwd("~/mysite/yusufameri.github.io")
KnitPost
KnitPost <- function(input, base.url = "/") {
require(knitr)
opts_knit$set(base.url = base.url)
fig.path <- paste0("figs/", sub(".Rmd$", "", basename(input)), "/")
opts_chunk$set(fig.path = fig.path)
opts_chunk$set(fig.cap = "center")
render_jekyll()
knit(input, envir = parent.frame())
}
KnitPost("_source/2016-11-2-Money-Ball-EDA.Rmd")
data("LahmanData")
KnitPost("_source/2016-11-2-Money-Ball-EDA.Rmd")
KnitPost("_source/2016-11-2-Money-Ball-EDA.Rmd")
setwd("~/mysite/yusufameri.github.io/_source")
KnitPost("_source/2016-11-2-Money-Ball-EDA.Rmd")
setwd("~/mysite/yusufameri.github.io")
KnitPost("_source/2016-11-2-Money-Ball-EDA.Rmd")
KnitPost("_source/2016-11-2-Money-Ball-EDA.Rmd")