-
Notifications
You must be signed in to change notification settings - Fork 2
/
install_cmssw_power_archs.sh
276 lines (255 loc) · 11.9 KB
/
install_cmssw_power_archs.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
#!/bin/sh
updated_list=/cvmfs/cms.cern.ch/cvmfs-cms.cern.ch-updates
#export bootstrap_script=http://cmsrep.cern.ch/cmssw/cms/bootstrap.sh # 20MAY2017
export bootstrap_script=http://cmsrep.cern.ch/cmssw/repos/bootstrap.sh
export VO_CMS_SW_DIR=/cvmfs/cms.cern.ch
export SCRAM_ARCH=fc22_ppc64le_gcc530
export CMSSW_RELEASE=CMSSW_8_0_0
export CMSSW_RELEASE=CMSSW_8_0_2
#[ $# -gt 0 ] && export SCRAM_ARCH=$1
#[ $# -gt 1 ] && export CMSSW_RELEASE=$2
fedora_bzip_file=fedora-$(echo $SCRAM_ARCH | cut -c3-4)-ppc64le-rootfs.tar.bz2
# Release Map
releases_map="https://cmssdt.cern.ch/SDT/releases.map"
releases_map_local=/home/cvcms/releases.map
function add_nested_entry_to_cvmfsdirtab () {
if [ $# -lt 1 ] ; then
echo ERROR add_nested_entry_to_cvmfsdirtab arch
return 1
fi
thearch=$1
for thecmssw in cmssw cmssw-patch ; do
n_a_cmssw=$(ls $VO_CMS_SW_DIR/${thearch}/cms/$thecmssw | wc -l)
if [ $n_a_cmssw -gt 0 ] ; then
#grep -q /${thearch}/cms/$thecmssw $VO_CMS_SW_DIR/.cvmfsdirtab
echo $thearch | grep -e 'fc\|sl'\* | grep -q gcc\*
if [ $? -eq 0 ] ; then
echo INFO the entry /${thearch}/cms/$thecmssw is already in $VO_CMS_SW_DIR/.cvmfsdirtab
else
thesl=$(echo $thearch | cut -c1,2)
thecompil=$(echo $thearch | cut -d_ -f3 | cut -c1-3)
#echo INFO adding the entry /${thearch}/cms/$thecmssw to $VO_CMS_SW_DIR/.cvmfsdirtab
#echo /${thearch}/cms/$thecmssw >> $VO_CMS_SW_DIR/.cvmfsdirtab
echo INFO adding the entry /${thesl}\*${thecompil}\*/cms/${thecmssw}/\* to $VO_CMS_SW_DIR/.cvmfsdirtab
echo /${thesl}\*${thecompil}\*/cms/${thecmssw}/\* >> $VO_CMS_SW_DIR/.cvmfsdirtab
printf "add_nested_entry_to_cvmfsdirtab INFO: added the entry /${thearch}/cms/$thecmssw to $VO_CMS_SW_DIR/.cvmfsdirtab\n" | mail -s "add_nested_entry_to_cvmfsdirtab INFO: Nested CVMFS dir entry added for $thearch" $notifytowhom
fi
fi
done
return 0
}
:
: Main
:
# Run this every two hours
if [ $(expr $(date +%H) % 2) -eq 1 ] ; then
echo INFO $(basename $0) executed every 2 hours.
exit 0
fi
# use cmspkg instead of apt-get
which cmspkg 2>/dev/null 1>/dev/null
[ $? -eq 0 ] || { export PATH=$PATH:/cvmfs/cms.cern.ch/common ; } ;
SCRAM_ARCH=
CMSSW_RELEASE=
what=fc
a_archs=$(grep ${what}[0-9][0-9]_ "$releases_map_local" | cut -d\; -f1 | cut -d= -f2 | sort -u | grep ppc64le)
for a in $a_archs ; do
cmssw_releases=$(grep $a "$releases_map_local" | grep label=CMSSW_ | cut -d\; -f2 | cut -d= -f2)
for cmssw_release in $cmssw_releases ; do
grep -q "$cmssw_release $a " $updated_list
[ $? -eq 0 ] && { echo INFO $cmssw_release $a installed according to the $updated_list ; continue ; } ;
echo INFO $cmssw_release $a needs to be installed
export SCRAM_ARCH=$a
export CMSSW_RELEASE=$cmssw_release
# echo INFO executing $HOME/cvmfs_install_POWER8.sh "$a" "$cmssw_release"
# $HOME/cvmfs_install_POWER8.sh "$a" "$cmssw_release" > $HOME/logs/cvmfs_install_POWER8.${a}.${cmssw_release}.log 2>&1
# status=$(expr $status + $?)
# cat $HOME/logs/cvmfs_install_POWER8.${a}.${cmssw_release}.log
break
done
[ "x$SCRAM_ARCH" == "x" ] && [ "x$CMSSW_RELEASE" == "x" ] && continue
break
done
if [ "x$SCRAM_ARCH" == "x" ] ; then
echo INFO $(basename $0) no arch to install
exit 0
fi
if [ "x$CMSSW_RELEASE" == "x" ] ; then
echo INFO $(basename $0) no release to install
exit 0
fi
#printf "$(basename $0): Starting to try to install $CMSSW_RELEASE ${SCRAM_ARCH}\n\n" | mail -s "$(basename $0) Starting " $notifytowhom
[ -d $HOME/POWER8 ] || mkdir $HOME/POWER8
cd $HOME/POWER8
echo VO_CMS_SW_DIR=/cvmfs/cms.cern.ch
grep -q "$CMSSW_RELEASE $SCRAM_ARCH " $updated_list
if [ $? -eq 0 ] ; then
echo INFO "$CMSSW_RELEASE $SCRAM_ARCH" found in $updated_list
exit 0
fi
files="proot qemu-ppc64le"
[ -d $(echo ${fedora_bzip_file} | sed 's#\.tar\.bz2##g') ] || files="$files ${fedora_bzip_file}"
echo DEBUG $files
for f in $files ; do
[ -f $f ] && continue
wget -q -O $f http://davidlt.web.cern.ch/davidlt/vault/proot/$f
status=$?
echo Download status=$status for $f
if [ $status -ne 0 ] ; then
printf "$(basename $0): FAILED wget -q -O $f http://davidlt.web.cern.ch/davidlt/vault/proot/$f " | mail -s "ERROR FAILED downloading one of PROOT files" $notifytowhom
exit 1
fi
if [ "x$f" == "xproot" ] ; then
chmod a+x proot
elif [ "x$f" == "xqemu-ppc64le" ] ; then
chmod a+x qemu-ppc64le
elif [ "x$f" == "x${fedora_bzip_file}" ] ; then
bzip2 -d ${fedora_bzip_file}
tar xvf $(echo ${fedora_bzip_file} | sed 's#\.bz2##g')
fi
done
echo INFO $(basename $0) going to cvmfs write mode cvmfs_server transaction
cvmfs_server transaction
if [ $? -ne 0 ] ; then
echo ERROR cvmfs_server transaction failed. Exiting $(basename $0)...
printf "$(basename $0): cvfms_server transaction failed to install $CMSSW_RELEASE ${SCRAM_ARCH}\n$(cat $HOME/cvmfs_install_POWER8.log | sed 's#%#%%#g')\n" | mail -s "ERROR installation of $CMSSW_RELEASE ${SCRAM_ARCH} failed " $notifytowhom
exit 1
fi
#Because of cmspkg
#if [ $(ls -al $VO_CMS_SW_DIR/${SCRAM_ARCH}/external/apt/*/etc/profile.d/init.sh 2>/dev/null 1>/dev/null ; echo $? ; ) -eq 0 ] ; then
if [ $(ls -al $VO_CMS_SW_DIR/${SCRAM_ARCH}/external/rpm/*/etc/profile.d/init.sh 2>/dev/null 1>/dev/null ; echo $? ; ) -eq 0 ] ; then
echo INFO arch ${SCRAM_ARCH} seems to be already bootstrapped
else
echo INFO downloading bootstrap.sh for ${SCRAM_ARCH}
wget -q -O $VO_CMS_SW_DIR/$(basename $bootstrap_script) $bootstrap_script
#sh -x $VO_CMS_SW_DIR/bootstrap.sh -repository cms setup -path $VO_CMS_SW_DIR -a ${SCRAM_ARCH} > $HOME/bootstrap_${SCRAM_ARCH}.log 2>&1
#cat $HOME/bootstrap_${SCRAM_ARCH}.log
fi
echo INFO installing $CMSSW_RELEASE ${SCRAM_ARCH} in the proot env
PROOT_ROOT=$(echo ${fedora_bzip_file} | sed 's#\.tar\.bz2##g')
./proot -R $PWD/$PROOT_ROOT -b /cvmfs:/cvmfs -q "$PWD/qemu-ppc64le -cpu POWER8" /bin/sh -c "\
star='*' ; \
init_sh=\`ls $VO_CMS_SW_DIR/${SCRAM_ARCH}/external/rpm/*/etc/profile.d/init.sh -t | head -1\` ; \
if [ -f \"\$init_sh\" ] ; then \
echo INFO sourcing \$init.sh ; \
source \$init_sh ; \
else \
echo INFO bootstrapping ${SCRAM_ARCH} ; \
sh -x $VO_CMS_SW_DIR/bootstrap.sh -repository cms setup -path $VO_CMS_SW_DIR -a ${SCRAM_ARCH} ; \
[ \$? -eq 0 ] || { echo proot_status=1 ; exit 1 ; } ; \
init_sh=\`ls $VO_CMS_SW_DIR/${SCRAM_ARCH}/external/rpm/*/etc/profile.d/init.sh -t | head -1\` ; \
if [ -f \"\$init_sh\" ] ; then \
echo INFO sourcing \$init.sh ; \
source \$init_sh ; \
else \
echo ERROR init.sh not found for apt ; \
fi ; \
fi ; \
grep -q mutex_set_max $VO_CMS_SW_DIR/${SCRAM_ARCH}/var/lib/rpm/DB_CONFIG 2>/dev/null ; \
if [ \$? -eq 0 ] ; then \
echo INFO mutex_set_max 100000 already there ; \
else \
echo INFO adding mutex_set_max 100000 to $VO_CMS_SW_DIR/${SCRAM_ARCH}/var/lib/rpm/DB_CONFIG ; \
echo mutex_set_max 100000 >> $VO_CMS_SW_DIR/${SCRAM_ARCH}/var/lib/rpm/DB_CONFIG ; \
fi ; \
which cmspkg 2>/dev/null 1>/dev/null ; \
[ \$? -eq 0 ] || { export PATH=\$PATH:/cvmfs/cms.cern.ch/common ; } ; \
echo INFO executing cmspkg -a ${SCRAM_ARCH} update ; \
cmspkg -a ${SCRAM_ARCH} -y upgrade ; \
[ \$? -eq 0 ] || { echo cmspkg -a ${SCRAM_ARCH} -y upgrade failed ; echo proot_status=1 ; exit 1 ; } ; \
cmspkg -a ${SCRAM_ARCH} update ; \
[ \$? -eq 0 ] || { echo cmspkg -a ${SCRAM_ARCH} update failed ; echo proot_status=1 ; exit 1 ; } ; \
second_plus= ; \
echo $CMSSW_RELEASE | grep -q patch && second_plus=-patch ; \
echo INFO executing cmspkg -a ${SCRAM_ARCH} -y install cms+cmssw\${second_plus}+$CMSSW_RELEASE ; \
cmspkg -a ${SCRAM_ARCH} -y install cms+cmssw\${second_plus}+$CMSSW_RELEASE > $HOME/logs/cmspkg_install.log 2>&1 ; \
[ \$? -eq 0 ] || { echo cmspkg -a ${SCRAM_ARCH} -y install cms+cmssw\${second_plus}+$CMSSW_RELEASE failed ; cat $HOME/logs/cmspkg_install.log ; echo proot_status=1 ; exit 1 ; } ; \
cat $HOME/logs/cmspkg_install.log ; \
echo proot_status=0" > $HOME/logs/proot.log 2>&1 &
job_pid=$!
second_plus=
echo $CMSSW_RELEASE | grep -q patch && second_plus=-patch
if [ ] ; then
n=10800 # 180 minues
i=0
nkill=0
while [ $i -lt $n ] ;do
status=$(grep proot_status= $HOME/logs/proot.log | cut -d= -f2)
if [ "x$status" != "x" ] ; then
[ $status -eq 0 ] || break
fi
# if we have more than enough memory, don't do this
[ $(/usr/bin/free -g | grep ^Mem | awk '{print $2}') -gt 14 ] && break
i=$(expr $i + 1)
ps auxwww | grep -v grep | grep perl | grep -q projectAreaRename.pl
if [ $? -eq 0 ] ; then
[ $(expr $i % 30) -eq 0 ] && { echo DEBUG grep cms+cmssw${second_plus}+${CMSSW_RELEASE} $HOME/apt_get_install.log ; grep ^cms+cmssw${second_plus}+${CMSSW_RELEASE} $HOME/apt_get_install.log ; } ;
grep -q ^cms+cmssw${second_plus}+${CMSSW_RELEASE} $HOME/apt_get_install.log
if [ $? -eq 0 ] ; then
echo KILLING the process
ps auxwww | grep -v grep | grep perl | grep projectAreaRename.pl
pid=$(ps auxwww | grep -v grep | grep perl | grep projectAreaRename.pl | awk '{print $2}')
kill -KILL $pid
nkill=$(expr $nkill + 1)
echo INFO nkill=$nkill
else
echo Not killing it yet
ps auxwww | grep -v grep | grep perl | grep projectAreaRename.pl
fi
fi
[ $nkill -eq 2 ] && break
[ $(expr $i % 30) -eq 0 ] && echo Sleeping 1sec after $i sec out $n sec limit
sleep 1 # 1m
done
echo INFO doing wait
wait $job_pid
status_job_pid=$?
if [ $nkill -eq 2 ] ; then
#echo INFO after killing projectAreaRename.pl executing it from x86_64
build_hash=$(grep BUILDROOT $HOME/logs/proot.log | head -1 | awk '{print $(NF-2)}' | sed 's#/# #g' | awk '{print $(NF-2)}')
echo INFO after killing projectAreaRename.pl executing it from x86_64 $HOME/cvmfs_postinstall_POWER8.sh $CMSSW_RELEASE $SCRAM_ARCH $build_hash
$HOME/cvmfs_postinstall_POWER8.sh $CMSSW_RELEASE $SCRAM_ARCH $build_hash
fi
fi # if [ ] ; then
echo INFO doing wait
wait $job_pid
status_job_pid=$?
status=$(grep proot_status= $HOME/logs/proot.log | cut -d= -f2)
echo status=$status status_job_pid=$status_job_pid
#grep Killed $HOME/apt_get_install.log | grep -q projectAreaRename
#if [ $? -eq 0 ] ; then
# #status=1
# $HOME/cvmfs_postinstall_POWER8.sh $CMSSW_RELEASE $SCRAM_ARCH
# status=$?
#fi
echo INFO proot install status=$status
#second_plus=
#echo $CMSSW_RELEASE | grep -q patch && second_plus=-patch
if [ $status -eq 0 ] ; then
if [ -d $VO_CMS_SW_DIR/${SCRAM_ARCH}/cms/cmssw${second_plus}/$CMSSW_RELEASE ] ; then
echo INFO updating the cvmfs management stuff
add_nested_entry_to_cvmfsdirtab ${SCRAM_ARCH}
ls -al $VO_CMS_SW_DIR/${SCRAM_ARCH}/.cvmfscatalog 2>/dev/null 1>/dev/null
if [ $? -eq 0 ] ; then
echo INFO $VO_CMS_SW_DIR/${SCRAM_ARCH}/.cvmfscatalog exists
else
echo INFO creating $VO_CMS_SW_DIR/${SCRAM_ARCH}/.cvmfscatalog
touch $VO_CMS_SW_DIR/${SCRAM_ARCH}/.cvmfscatalog
fi
grep -q "$CMSSW_RELEASE ${SCRAM_ARCH} " $updated_list
if [ $? -eq 0 ] ; then
echo INFO "$CMSSW_RELEASE ${SCRAM_ARCH} " found in $updated_list
else
echo $CMSSW_RELEASE ${SCRAM_ARCH} $(/bin/date +%s) $(/bin/date -u) >> $updated_list
printf "$(basename $0): $CMSSW_RELEASE ${SCRAM_ARCH} installed " | mail -s "Installed $CMSSW_RELEASE ${SCRAM_ARCH} " $notifytowhom
fi
else
printf "$(basename $0): $CMSSW_RELEASE ${SCRAM_ARCH} install failed\nNot found: $VO_CMS_SW_DIR/${SCRAM_ARCH}/cms/cmssw${second_plus}/$CMSSW_RELEASE\n$(cat $HOME/cvmfs_install_POWER8.log | sed 's###g')\n" | mail -s "ERROR installation of $CMSSW_RELEASE ${SCRAM_ARCH} failed " $notifytowhom
fi
else
printf "$(basename $0): $CMSSW_RELEASE ${SCRAM_ARCH} install failed\n$(cat $HOME/cvmfs_install_POWER8.log | sed 's###g')\n" | mail -s "ERROR installation of $CMSSW_RELEASE ${SCRAM_ARCH} failed " $notifytowhom
fi
echo INFO executing cvmfs_server publish
cvmfs_server publish
exit 0