-
Notifications
You must be signed in to change notification settings - Fork 0
/
Makefile
192 lines (161 loc) · 6.59 KB
/
Makefile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
# R package variables
# h/t to @jimhester and @yihui for this parse block:
# https://github.com/yihui/knitr/blob/dc5ead7bcfc0ebd2789fe99c527c7d91afb3de4a/Makefile#L1-L4
PKGNAME := $(shell sed -n "s/Package: *\([^ ]*\)/\1/p" DESCRIPTION)
PKGVERS := $(shell sed -n "s/Version: *\([^ ]*\)/\1/p" DESCRIPTION)
r_files := $(wildcard R/*.R)
pkgdata_csv := $(wildcard data-raw/*.csv)
pkgdata_rda := $(patsubst data-raw/%.csv,data/%.rda,$(pkgdata_csv))
build_file := $(PKGNAME)_$(PKGVERS).tar.gz
# processing directories
raw_dir := analysis/raw
interm_dir := analysis/intermediate
out_dir := analysis/out
scripts_dir := analysis/scripts
docs_dir := analysis/docs
conf_dir := analysis/config
# processing variables
rscript := Rscript --vanilla
export processing_end := 2022-01-01
export raw_version := $(PKGVERS)
sites := WFMS WFMB PSP QC
download_url := http://atmoschem.asrc.cestm.albany.edu/~aqm/AQM_Products/downloads
routine_zip := $(raw_dir)/routine_chemistry_v0.1.zip
old_routine_out := $(patsubst %,$(interm_dir)/old_%.csv,$(sites))
raw_data := $(raw_dir)/raw_data_v$(raw_version)
raw_zip := $(raw_dir)/raw_data_v$(raw_version).zip
# get <site>_<data_source> for each entry in data_sources.csv
data_sources := $(shell sed "1d;s/^\([^,]*\),\([^,]*\).*/\1_\2/" analysis/config/dataloggers.csv)
hourly_files := $(patsubst %,$(interm_dir)/hourly_%.sqlite,$(data_sources))
export routine_out := $(out_dir)/routine_chemistry_v$(PKGVERS)
# ensure R package availability
check_rpkg = if (!requireNamespace("$(1)")) install.packages("$(1)")
## Options available for make:
##
## help : Display available make commands.
.PHONY : help
help : Makefile
@sed -n 's/^##//p' $<
.PHONY: all
all: routine
##
## Data processing commands
##
## check_data : Run data tests.
.PHONY: check_data
check_data:
$(rscript) \
-e '$(call check_rpkg,tinytest)' \
-e 'tinytest::run_test_dir("analysis/tests")'
## clean_data : Remove intermediate data processing files.
.PHONY: clean_data
clean_data:
@rm -rf $(raw_dir) && rm -rf $(interm_dir) && rm -rf $(out_dir)
# save intermediate sqlite files for the processing viewer
.SECONDARY:
## routine : Generate routine monitoring dataset.
.PHONY: routine
routine: $(routine_out).zip
$(routine_out).zip: $(old_routine_out) $(hourly_files) $(docs_dir)/routine.md \
$(scripts_dir)/routine_package.R
$(rscript) $(scripts_dir)/routine_package.R $(routine_out)
$(docs_dir)/routine.md: $(docs_dir)/routine.Rmd $(docs_dir)/routine.bib
Rscript \
-e '$(call check_rpkg,rmarkdown)' \
-e 'rmarkdown::render("analysis/docs/routine.Rmd")'
$(interm_dir)/hourly_QC_AQS.sqlite: $(scripts_dir)/queens.R
$(rscript) $(scripts_dir)/queens.R
$(interm_dir)/hourly_%.sqlite: $(interm_dir)/processed_%.sqlite $(scripts_dir)/aggregate_hourly.R
$(rscript) $(scripts_dir)/aggregate_hourly.R $(shell echo $* | sed "s/_/ /")
# The way calibration files are created for each site (without data source)
# makes things a bit awkward. Processed site/data source files depend on the
# calibration site file
.SECONDEXPANSION:
$(interm_dir)/processed_%.sqlite: $(interm_dir)/raw_%.sqlite \
$(interm_dir)/processedcals_$$(shell echo $$* | sed "s/_.*//").sqlite \
$(scripts_dir)/process_new_data.R \
$(conf_dir)/manual_flags.csv
$(rscript) $(scripts_dir)/process_new_data.R $(shell echo $* | sed "s/_/ /")
# the WFMB campbell processing also depends on the processed Mesonet data
$(interm_dir)/processed_WFMB_campbell.sqlite: $(interm_dir)/processed_WFMB_mesonet.sqlite
$(interm_dir)/processedcals_%.sqlite: $(scripts_dir)/process_calibrations.R \
$(interm_dir)/cals_%.sqlite $(conf_dir)/cal_flags.csv
$(rscript) $(scripts_dir)/process_calibrations.R $*
# Calibration site file depends on all the raw site/calibration files
make_cal_deps = $(patsubst %,$(interm_dir)/raw_%.sqlite, $(filter $(1)%, $(data_sources)))
$(interm_dir)/cals_%.sqlite: $$(call make_cal_deps,$$*) \
raw_data $(scripts_dir)/load_calibration.R
$(rscript) $(scripts_dir)/load_calibration.R $*
$(interm_dir)/raw_%.sqlite: raw_data $(scripts_dir)/load_raw.R
$(rscript) $(scripts_dir)/load_raw.R $(shell echo $* | sed "s/_/ /")
$(interm_dir)/old_%.csv: $(routine_zip) $(scripts_dir)/clean_old_routine.R
$(rscript) -e 'unzip("$(routine_zip)", overwrite = F, exdir = "$(raw_dir)")' && \
$(rscript) $(scripts_dir)/clean_old_routine.R $*
# the old PSP data also depends on the PSP envidas conversion efficiencies, in
# order to correct some old CE numbers
$(interm_dir)/old_PSP.csv: $(interm_dir)/processedcals_PSP.sqlite
.INTERMEDIATE: raw_data
raw_data: $(raw_zip)
$(rscript) -e 'unzip("$(raw_zip)", overwrite = F, exdir = "$(raw_dir)")'
$(raw_dir)/%.zip:
mkdir -p $(raw_dir) && \
wget --user=$(asrc_user) --password=$(asrc_pass) -O $@ $(download_url)/$(shell echo $* | sed -E s/_v[0-9.]+$$//)/$*.zip
## view : Open the data viewer web app.
.PHONY: view
view:
$(rscript) \
-e '$(call check_rpkg,shiny)' \
-e 'shiny::runApp("analysis/processing_viewer", launch.browser = T)'
## save_raw : Package the raw data.
.PHONY: save_raw
save_raw:
cd $(raw_dir) && \
$(rscript) \
-e 'zip("../../$(out_dir)/raw_data_v$(raw_version).zip", "raw_data_v$(raw_version)")'
##
## R package commands
##
## check : Run `R CMD check` on the package.
.PHONY: check
check: $(build_file)
R CMD check --no-manual $(build_file)
## website : Generate and open package documentation website.
.PHONY: website
website: docs
Rscript \
-e '$(call check_rpkg,pkgdown)' \
-e 'pkgdown::build_site(preview = TRUE)'
## install : Install the package.
.PHONY: install
install: docs
Rscript \
-e '$(call check_rpkg,devtools)' \
-e 'devtools::install(build = FALSE, upgrade = FALSE)'
# this alternative installation method is inconvenient due to the very slow
# build step
# install: install_deps $(build_file)
# R CMD INSTALL $(build_file)
.INTERMEDIATE: install_deps
install_deps: DESCRIPTION
Rscript \
-e '$(call check_rpkg,remotes)' \
-e 'remotes::install_deps(dependencies = TRUE)'
$(build_file): docs
R CMD build .
## docs : Update the package documentation.
.INTERMEDIATE: docs
docs: $(pkgdata_rda) $(r_files) README.md
Rscript \
-e '$(call check_rpkg,roxygen2)' \
-e 'roxygen2::roxygenise()'
$(pkgdata_rda): data-raw/package_data.R data-raw/narsto_flags.csv \
data-raw/aqs_flags.csv
$(rscript) data-raw/package_data.R
README.md: README.Rmd DESCRIPTION
Rscript \
-e '$(call check_rpkg,rmarkdown)' \
-e 'rmarkdown::render("README.Rmd")'
## clean : Remove package build files.
.PHONY: clean
clean:
@rm -rf $(build_file) $(PKGNAME).Rcheck