-
Notifications
You must be signed in to change notification settings - Fork 0
/
cloud-config.yml
126 lines (108 loc) · 4.7 KB
/
cloud-config.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
#cloud-config
ssh_authorized_keys:
- %(LOCAL_SSH_KEY)s
# Launch instance with network interfaces configured to the local IP addresses we references in the config.
# $ aws ec2 run-instances --user-data file://check-files.yml --iam-instance-profile Name="encoded-instance" --image-id ami-5a928a3b --region us-west-2 --security-groups ssh-http-https --instance-type c4.xlarge
output:
all: '| tee -a /var/log/cloud-init-output.log'
packages:
- fuse
## - golang
- git
- curl
- ntp
- python3-dev
- python3-venv
- libcurl4
- libncurses5-dev
- liblzma-dev
- libbz2-dev
- libz-dev
- libcurl4-openssl-dev
- python3-pip
## power_state:
## mode: poweroff
runcmd:
- set -ex
- pip3 install awscli
- sudo -u ubuntu mv /home/ubuntu/.ssh/authorized_keys /home/ubuntu/.ssh/authorized_keys2
- sudo -u ubuntu aws s3 cp --region=us-west-2 %(S3_AUTH_KEYS)s /home/ubuntu/.ssh/authorized_keys
- systemctl daemon-reload # See https://bugs.launchpad.net/cloud-init/+bug/1449318
- mkdir -p /s3/encode-files
- mkdir -p /s3/encoded-files-dev
- mkdir -p /s3/encode-private
- mkdir -p /s3/encode-public
- curl -sS -L -o /usr/local/bin/goofys https://github.com/kahing/goofys/releases/download/v0.24.0/goofys
- chmod +x /usr/local/bin/goofys
## - mkdir -p /opt/goofys/bin
## - chown -R build:build /opt/goofys
## - sudo -u build GOPATH=/opt/goofys go get github.com/kahing/goofys
## - mv /opt/goofys/bin/goofys /usr/local/bin/
- mount -a
- mkdir /opt/encValData
- chown build:build /opt/encValData
- sudo -u build git clone --depth 1 https://github.com/ENCODE-DCC/encValData /opt/encValData
- mkdir /opt/ENCODE_CRISPR_Validation
- chown build:build /opt/ENCODE_CRISPR_Validation
- sudo -u build git clone --depth 1 -b v1.3 https://github.com/oh-jinwoo94/ENCODE /opt/ENCODE_CRISPR_Validation
- touch /opt/GRCh38_no_alt_analysis_set_GCA_000001405.15.fasta
- gunzip --stdout "/s3/encode-public/2015/12/03/a7fea375-057d-4cdc-8ccd-0b0f930823df/GRCh38_no_alt_analysis_set_GCA_000001405.15.fasta.gz" > /opt/GRCh38_no_alt_analysis_set_GCA_000001405.15.fasta
- curl -sS -L -o /usr/local/bin/validateFiles http://hgdownload.cse.ucsc.edu/admin/exe/linux.x86_64/validateFiles
- chmod +x /usr/local/bin/validateFiles
- cd /home/ubuntu
- wget https://github.com/samtools/samtools/releases/download/1.11/samtools-1.11.tar.bz2
- sudo tar -jxvf samtools-1.11.tar.bz2
- cd samtools-1.11
- sudo make
- sudo make install
- cd /home/ubuntu
- mkdir /opt/encoded
- chown build:build /opt/encoded
- mkdir /opt/encoded/checkfiles
- chown build:build /opt/encoded/checkfiles
- sudo -u build git clone --no-checkout https://github.com/ENCODE-DCC/checkfiles.git /opt/encoded/checkfiles
- sudo -u build git -C /opt/encoded/checkfiles checkout %(COMMIT)s
- chmod +x /opt/encoded/checkfiles/script
- cd /opt/encoded/checkfiles
#- sudo -u build pyvenv .
#- sudo -u build bin/pip install -r requirements.txt
- sudo -u build python3 -m venv venv
- sudo -u build /opt/encoded/checkfiles/venv/bin/pip install wheel
- sudo -u build /opt/encoded/checkfiles/venv/bin/pip install -r /opt/encoded/checkfiles/requirements.txt
- cd /home/ubuntu
- nohup /opt/encoded/checkfiles/script $(cat /opt/checkfiles_args.txt) 2> errors_prod.log 1> output_prod.log &
- nohup /opt/encoded/checkfiles/writeTestArgs $(cat /opt/checkfiles_args.txt | rev | cut -d " " -f 2- | rev) > /opt/test_args.txt
- nohup /opt/encoded/checkfiles/script $(cat /opt/test_args.txt) 2> errors_test.log 1> output_test.log &
users:
- default
- name: build
gecos: Build user
inactive: true
system: true
mounts:
- [ "goofys-ulimit#encode-files", "/s3/encode-files", "fuse", "_netdev,allow_other,--file-mode=0444,--dir-mode=0555,--stat-cache-ttl=0", "0", "0" ]
- [ "goofys-ulimit#encoded-files-dev", "/s3/encoded-files-dev", "fuse", "_netdev,allow_other,--file-mode=0444,--dir-mode=0555,--stat-cache-ttl=0", "0", "0" ]
- [ "goofys-ulimit#encode-private", "/s3/encode-private", "fuse", "_netdev,allow_other,--file-mode=0444,--dir-mode=0555,--stat-cache-ttl=0", "0", "0" ]
- [ "goofys-ulimit#encode-public", "/s3/encode-public", "fuse", "_netdev,allow_other,--file-mode=0444,--dir-mode=0555,--stat-cache-ttl=0", "0", "0" ]
write_files:
- path: /opt/checkfiles_args.txt
content: |
%(ARGS)s
- path: /etc/systemd/system/cloud-final.service.d/override.conf
content: |
[Service]
# See https://bugs.launchpad.net/cloud-init/+bug/1449318
KillMode=process
- path: /usr/local/bin/goofys-ulimit
permissions: 0755
content: |
#!/bin/sh
ulimit -n 60000
exec goofys "$@"
- path: /etc/rc.local
permissions: 0755
content: |
#!/bin/sh -e
cd /home/ubuntu; nohup /opt/encoded/checkfiles/script %(ARGS)s 2> errors_prod.log 1> output_prod.log &
nohup /opt/encoded/checkfiles/script $(cat /opt/test_args.txt) 2> errors_test.log 1> output_test.log &
exit 0