Makefile 13 KB
Newer Older
1
SHELL=/bin/bash
2

3
# Include the link to the Makefile.def from the SWMF used
4
-include SWMF/Makefile.def
5

6
MYDIR  = $(shell echo `pwd -P`)
7
8
SIMDIR = Runs
RESDIR = Runs
9
IDLDIR = ${DIR}/share/IDL/Solar
Lulu Zhao's avatar
Lulu Zhao committed
10
PYDIR  = ${MYDIR}/Scripts
11

12
13
MODEL = AWSoM
PFSS  = HARMONICS
14

Lulu Zhao's avatar
Lulu Zhao committed
15
16
TIME = MapTime
MAP  = NoMap
17

18
19
20
EVTIME = none
DTEV   = -7

21
POYNTINGFLUX   = -1.0
Zhenguang Huang's avatar
Zhenguang Huang committed
22

23
24
PARAM = Default

25
26
REALIZATIONS    = 1,2,3,4,5,6,7,8,9,10,11,12
REALIZATIONLIST = $(foreach v, $(shell echo ${REALIZATIONS} | tr , ' '), $(shell printf '%02d' $(v)))
27

28
29
RESTART   = F
DOINSTALL = T
30

31
32
JOBNAME  = amap

33
SimDirList = $(sort $(dir $(wildcard run[0-9]*_*/)))
Zhenguang Huang's avatar
Zhenguang Huang committed
34
ResDirList = $(subst ${MYDIR}/Results/,,${FullResDirList})
35
FullResDirList = $(sort $(dir $(wildcard ${MYDIR}/Results/run[0-9]*_*/)))
Zhenguang Huang's avatar
Zhenguang Huang committed
36
37
38
39

FullResDir        = ${MYDIR}/Results/${RESDIR}
FullRunDirList    = $(sort $(dir $(wildcard ${MYDIR}/${SIMDIR}/run[01][0-9]/)))
FullResRunDirList = $(sort $(dir $(wildcard ${FullResDir}/run[01][0-9]/)))
40

41
help: 
42
	@echo "*******************************************************************************"
Zhenguang Huang's avatar
Zhenguang Huang committed
43
44
	@echo "This makefile uses pyfits (in SWMF/share/Python/. "
	@echo "There is a link in SWMFSOLAR/Scripts to pyfits. "
45
	@echo "Check the python version with python --version. Should be 3.7 or above."
46
	@echo ""
47
	@echo "On Pleiades, add 'module load python3/3.7.0' in the .cashrc or .bashrc file."
48
49
50
51
	@echo "On Frontera, the default python is python2 and it is a link in /bin. "
	@echo "Make a bin folder in your home folder and create a link to python3.7:"
	@echo "  mkdir ~/bin; cd ~/bin/; ln -s `which python3.7` ~/bin/python"
	@echo '  setenv PATH ~/bin/:$${PATH} # add it to the beginning of PATH in .cshrc'
52
	@echo "*******************************************************************************"
53
54
	@echo ""
	@echo "Examples:"
55
56
57
	@echo "  make adapt_run_w_compile MODEL=AWSoM  "
	@echo "       (configure SWMF and run AWSoM with 12 ADAPT realizations with B0 "
	@echo "        from Harmonics)"
58
59
60
61
	@echo "  make adapt_run MODEL=AWSoM  "
	@echo "       (run AWSoM   with 12 ADAPT realizations with B0 from Harmonics)"
	@echo "  make adapt_run MODEL=AWSoMR "
	@echo "       (run AWSoM-R with the 12 ADAPT realizations with B0 from Harmonics)"
62
	@echo "  make adapt_run MODEL=AWSoM REALIZATIONS=1 MAP=hmi.dat PFSS=FDIPS"
63
	@echo "       (run AWSoM with a single map named hmi.dat and use FDIPS)"
64
	@echo ""
65
66
67
	@echo "NOTE: "
	@echo "  adapt_run does NOT re-configure/compile SWMF to save time!"
	@echo "  adapt_run_w_compile will uninstall the SWMF, reinstall and compile the code."
68
	@echo ""
69
	@echo "After all simulations are finished post-process the results into Results/event1:"
Zhenguang Huang's avatar
Zhenguang Huang committed
70
71
72
	@echo ""
	@echo "  make check_postproc RESDIR=event1"
	@echo ""
73
74
	@echo "Comparing the simulations with observations is best to do on a local machine."
	@echo "This requires SSWIDL and share/IDL/Solar has to be properly set up. Use"
Zhenguang Huang's avatar
Zhenguang Huang committed
75
	@echo ""
76
	@echo "  make check_compare RESDIR=event1 MODEL=AWSoMR"
Zhenguang Huang's avatar
Zhenguang Huang committed
77
	@echo ""
78
79
	@echo "to compare Results/event1 with observations saved into Results/obsdata."
	@echo "MODEL is needed for plot legends."
80
	@echo ""
81
	@echo "Options:"
82
83
84
85
86
87
88
89
90
91
92
93
	@echo " MODEL=AWSoM         - select model: 'AWSoM' (default) or 'AWSoMR' (case sensitive)"
	@echo " SIMDIR=run01_test   - set name of simulation directory. Default is 'Runs'"
	@echo " RESDIR=run01_test   - set name of result directory in Results/. Default is 'Runs'"
	@echo " PFSS=HARMONICS      - set potential field solver: HARMONICS (default) or FDIPS"
	@echo " TIME=2012-1-1T1:1:1 - set the start time of the simulation, format is "
	@echo "                       YYYY-MM-DDTHH:MM:SC, default is MapTime (time of map)".
	@echo " POYNTINGFLUX=1.0e6, - set the Poynting flux, default is in the PARAM.in file."
	@echo " MAP=filename        - set the input map if desired. Default is 'NoMap'."
	@echo " REALIZATIONS=1,2    - list the realizations. Default is '1,2,3,4,5,6,7,8,9,10,11,12'"
	@echo " JOBNAME=amap        - set the job name. Default is 'amap' with "
	@echo "                       realization appensed, e.g. 'amap01'"
	@echo "                       Some systems limit the length of job name to 6 letters"
94
	@echo ""
Zhenguang Huang's avatar
Zhenguang Huang committed
95
	@echo "Notes:"
96
	@echo "User can set either TIME or MAP, or BOTH. And the following will occur:"
97
98
99
	@echo " 1. Both TIME and MAP are provided: use map and set start time to TIME. "
	@echo " 2. Only TIME is provided: download ADAPT map based on TIME and set start time to TIME."
	@echo " 3. Only MAP is provided: use MAP and set the start time based on map time."
100
	@echo ""
101

Zhenguang Huang's avatar
Zhenguang Huang committed
102
######################################################################################
103

104
105
adapt_run_w_compile:
	@echo "Submitting AWSoM runs with a ADAPT map with re-compiling the code."
106
	make compile
107
	make rundir_local
108
	make run
109
110
	@echo "Finished submitting AWSoM runs with a ADAPT map."

111
112
adapt_run:
	@echo "Submitting AWSoM runs with a ADAPT map without re-compiling the code."
113
	make rundir_local
114
115
116
	make run
	@echo "Finished submitting AWSoM runs with a ADAPT map."

117
118
119
120
121
122
123
124
install:
	-@(cp ${DIR}/util/DATAREAD/srcMagnetogram/remap_magnetogram.py ${MYDIR}/Scripts/;	\
	if([ -L ${MYDIR}/Scripts/pyfits ]); then					\
		rm -f ${MYDIR}/Scripts/pyfits; 						\
	fi;										\
	ln -s ${DIR}/share/Python/pyfits ${MYDIR}/Scripts/pyfits; 			\
	)

125
compile:
126
	-@(make install;								\
Lulu Zhao's avatar
Lulu Zhao committed
127
	if [[ "${MODEL}" == "$(filter ${MODEL},AWSoM AWSoM2T AWSoMR AWSoMR_SCIHOHSP)" ]]; then	\
128
		cd ${DIR}; 								\
129
130
131
132
		if [[ "${DOINSTALL}" == "T" ]]; then					\
			rm -f ${DIR}/bin/*.exe;						\
			./Config.pl -uninstall; 					\
			./Config.pl -install; 						\
zghuang's avatar
zghuang committed
133
			./Config.pl -v=Empty,SC/BATSRUS,IH/BATSRUS; 			\
134
		fi;									\
135
136
137
		if [[ "${MODEL}" == "AWSoM" ]]; then 					\
			./Config.pl -o=SC:u=Awsom,e=AwsomAnisoPi,nG=3,g=6,8,8; 		\
			./Config.pl -o=IH:u=Awsom,e=AwsomAnisoPi,nG=3,g=8,8,8; 		\
zghuang's avatar
zghuang committed
138
139
		fi;									\
		if [[ "${MODEL}" == "$(filter ${MODEL},AWSoM2T AWSoMR)" ]]; then	\
Lulu Zhao's avatar
Lulu Zhao committed
140
			./Config.pl -o=SC:u=Awsom,e=Awsom,nG=3,g=6,8,8;                 \
zghuang's avatar
zghuang committed
141
			./Config.pl -o=IH:u=Awsom,e=Awsom,nG=3,g=8,8,8;                 \
142
		fi; 									\
Lulu Zhao's avatar
Lulu Zhao committed
143
		if [[ "${MODEL}" == "AWSoMR_SCIHOHSP" ]]; then 				\
zghuang's avatar
zghuang committed
144
145
146
147
148
149
			./Config.pl -v=OH/BATSRUS,SP/MFLAMPA;				\
			./Config.pl -o=SC:u=Awsom,e=AwsomChGL,nG=2,g=6,8,8;             \
			./Config.pl -o=IH:u=Awsom,e=AwsomChGL,nG=2,g=8,8,8;             \
			./Config.pl -o=OH:u=Awsom,e=AwsomChGL,nG=2,g=4,4,4;             \
			./Config.pl -o=SP:g=20000;                                      \
		fi;									\
150
		make -j SWMF PIDL; 							\
151
		cp ${DIR}/bin/SWMF.exe ${DIR}/bin/${MODEL}.exe;				\
152
153
		cd ${DIR}/util/DATAREAD/srcMagnetogram; 				\
		make HARMONICS FDIPS; 							\
154
	else										\
Zhenguang Huang's avatar
Zhenguang Huang committed
155
		echo "MODEL = ${MODEL}";						\
Lulu Zhao's avatar
Lulu Zhao committed
156
		echo "ERROR: MODEL must be either AWSoM, AWSoM2T, AWSoMR or AWSoMR_SCIHOHSP.";	\
157
	fi;										\
158
159
	)

160
161
162
163
164
165
166
167
backup_run:
	-@if([ -d ${MYDIR}/${SIMDIR}/run01 ]); then					\
		rm -rf ${MYDIR}/${SIMDIR}/run_backup;					\
		mkdir -p ${MYDIR}/${SIMDIR}/run_backup;                   		\
		mv ${MYDIR}/${SIMDIR}/run[01]* ${MYDIR}/${SIMDIR}/run_backup/;          \
	fi

copy_param:
Lulu Zhao's avatar
Lulu Zhao committed
168
	-@(if [[ "${MODEL}" == "$(filter ${MODEL},AWSoM AWSoM2T AWSoMR AWSoMR_SCIHOHSP)" ]]; then	\
169
170
		if [[ "${MODEL}" == "AWSoMR" ]]; then					\
			cp Param/PARAM.in.awsomr PARAM.in; 				\
zghuang's avatar
zghuang committed
171
172
		fi; 									\
		if [[ "${MODEL}" == "$(filter ${MODEL},AWSoM AWSoM2T)" ]]; then		\
173
174
			cp Param/PARAM.in.awsom PARAM.in;				\
		fi;									\
Lulu Zhao's avatar
Lulu Zhao committed
175
		if [[ "${MODEL}" == "AWSoMR_SCIHOHSP" ]]; then				\
Lulu Zhao's avatar
Lulu Zhao committed
176
			cp Param/PARAM.in.awsomr.SCIHOHSP PARAM.in;				\
zghuang's avatar
zghuang committed
177
		fi;									\
178
179
180
		if [[ "${PARAM}" != "Default" ]]; then					\
			cp Param/${PARAM} PARAM.in;					\
		fi;									\
181
182
		cp Param/HARMONICS.in Param/FDIPS.in .; 				\
	else										\
Zhenguang Huang's avatar
Zhenguang Huang committed
183
		echo "MODEL = ${MODEL}";						\
Lulu Zhao's avatar
Lulu Zhao committed
184
		echo "ERROR: MODEL must be either AWSoM, AWSoM2T, AWSoMR or AWSoMR_SCIHOHSP.";	\
185
	fi;										\
186
187
188
189
190
191
192
193
194
195
	)

clean_rundir_tmp:
	-@(cd ${MYDIR};				\
	rm -f PARAM.in HARMONICS.in FDIPS.in;	\
	rm -f map_*.out; 			\
	)

rundir_realizations:
	-@for iRealization in ${REALIZATIONLIST}; do									\
196
197
		cd ${DIR}; 												\
		make rundir MACHINE=${MACHINE} RUNDIR=${MYDIR}/${SIMDIR}/run$${iRealization}; 				\
198
199
		rm ${MYDIR}/${SIMDIR}/run$${iRealization}/SWMF.exe;			 				\
		ln -s ${DIR}/bin/${MODEL}.exe ${MYDIR}/${SIMDIR}/run$${iRealization}/SWMF.exe;			 	\
200
201
202
203
204
		cp ${MYDIR}/PARAM.in     ${MYDIR}/${SIMDIR}/run$${iRealization}; 					\
		cp ${MYDIR}/HARMONICS.in ${MYDIR}/${SIMDIR}/run$${iRealization}/SC/; 					\
		cp ${MYDIR}/FDIPS.in     ${MYDIR}/${SIMDIR}/run$${iRealization}/SC/; 					\
		cp ${MYDIR}/JobScripts/job.${PFSS}.${MACHINE} ${MYDIR}/${SIMDIR}/run$${iRealization}/job.long;		\
		mv ${MYDIR}/map_$${iRealization}.out ${MYDIR}/${SIMDIR}/run$${iRealization}/SC/;  			\
205
206
207
208
209
210
211
212
		cd ${MYDIR}/${SIMDIR}/run$${iRealization}/SC/; 								\
		if [[ "${PFSS}"  == "HARMONICS" ]]; then								\
			perl -i -p -e "s/map_1/map_$${iRealization}/g" HARMONICS.in;					\
			./HARMONICS.exe; 										\
		fi; 													\
		if [[ "${PFSS}"  == "FDIPS" ]]; then									\
			perl -i -p -e "s/map_1/map_$${iRealization}/g" FDIPS.in;					\
		fi; 													\
213
214
	done

215
rundir_local:
216
217
218
	@echo "Creating rundirs"
	make backup_run
	make copy_param
Zhenguang Huang's avatar
Zhenguang Huang committed
219
	${MYDIR}/Scripts/change_awsom_param.py --map ${MAP} -t ${TIME} -B0 ${PFSS} -p ${POYNTINGFLUX}
220
221
	make rundir_realizations
	make clean_rundir_tmp
222

223
run:
224
	@echo "Submitting jobs"
225
226
	-@for iRealization in ${REALIZATIONLIST}; do              	        		\
		cd ${MYDIR}/${SIMDIR}/run$${iRealization}; 					\
227
		if [[ "${MACHINE}" == "frontera" ]];						\
228
			then perl -i -p -e "s/amap01/${JOBNAME}$${iRealization}/g" job.long;  	\
229
			sbatch job.long;							\
230
		fi;										\
231
		if [[ "${MACHINE}" == "pfe" ]];                         			\
232
			then ./qsub.pfe.pbspl.pl job.long ${JOBNAME}$${iRealization};      		\
233
		fi; 										\
234
235
	done

Zhenguang Huang's avatar
Zhenguang Huang committed
236
237
#########################################################################################

Zhenguang Huang's avatar
Zhenguang Huang committed
238
check_postproc:
239
240
241
	@if([ ! -d ${MYDIR}/Results/${RESDIR} ]); then                   			\
		rm -f error_postproc.log; 							\
		echo "Post processing simulation results to Results/${RESDIR}";			\
242
		mkdir -p ${MYDIR}/Results/${RESDIR}; 						\
243
		cp ${MYDIR}/${SIMDIR}/key_params.txt ${MYDIR}/Results/${RESDIR}/; 		\
244
		for RunDir in ${FullRunDirList};  do                              		\
245
246
247
			echo "processing rundir = $${RunDir}";					\
			cd $${RunDir};                                    			\
			if([ -f SWMF.SUCCESS ]); then                              		\
Zhenguang Huang's avatar
Zhenguang Huang committed
248
				mkdir -p ${FullResDir}/$${RunDir: -6:5};                      	\
249
				if([ ! -d RESULTS ]); then ./PostProc.pl -l=IH RESULTS; fi;   	\
250
251
				cp SC/map_*out ${FullResDir}/$${RunDir: -6:5}/;			\
				mv RESULTS/* ${FullResDir}/$${RunDir: -6:5}/;			\
252
				if [[ -f SC/fdips_bxyz.out ]]; then          			\
253
					mv SC/fdips_bxyz.out SC/FDIPS.in 			\
Zhenguang Huang's avatar
Zhenguang Huang committed
254
						${FullResDir}/$${RunDir: -6:5}/; 		\
255
256
				fi;								\
				if [[ -f SC/harmonics_adapt.dat ]]; then			\
257
					mv SC/harmonics_adapt.dat SC/HARMONICS.in harmonics_bxyz.out 	\
Zhenguang Huang's avatar
Zhenguang Huang committed
258
						${FullResDir}/$${RunDir: -6:5}/ ;		\
259
260
261
262
263
264
265
				fi;								\
			else									\
				echo "$${RunDir} crashed" >> ${MYDIR}/error_postproc.log;	\
			fi; 									\
		done;										\
	else                                                            			\
		echo "${RESDIR} already exists; skip post processing.";				\
Zhenguang Huang's avatar
Zhenguang Huang committed
266
267
	fi

Zhenguang Huang's avatar
Zhenguang Huang committed
268
269
270
#########################################################################################

check_compare:
271
272
273
274
275
	make check_compare_insitu
	make check_compare_remote

check_compare_insitu:
	-@(cd ${IDLDIR}; 									\
Zhenguang Huang's avatar
Zhenguang Huang committed
276
	for iRunDir in ${FullResRunDirList};  do 						\
Lulu Zhao's avatar
Lulu Zhao committed
277
		csh compare_insitu.sh ${DIR} $${iRunDir}/IH $${iRunDir} ${MODEL} 		\
278
			${MYDIR}/Results/obsdata ${EVTIME} ${DTEV};				\
279
	done)
Lulu Zhao's avatar
Lulu Zhao committed
280

281
check_compare_insitu_py:
Lulu Zhao's avatar
update    
Lulu Zhao committed
282
283
284
	-@(cd ${PYDIR};										\
	for iRunDir in ${FullResRunDirList};  do						\
		python compare_insitu.py ${DIR} $${iRunDir}/IH $${iRunDir} ${MODEL}		\
Lulu Zhao's avatar
Lulu Zhao committed
285
					 ${MYDIR}/Results/obsdata > $${iRunDir}/log_insitu;				\
286
287
	done)

288
289
290

check_compare_remote:
	-@(cd ${IDLDIR}; 									\
Zhenguang Huang's avatar
Zhenguang Huang committed
291
292
	for iRunDir in ${FullResRunDirList};  do 						\
		csh compare_remote.sh ${DIR} $${iRunDir}/SC $${iRunDir} ${MYDIR}/Results/obsdata; \
293
	done)
Zhenguang Huang's avatar
Zhenguang Huang committed
294

Lulu Zhao's avatar
Lulu Zhao committed
295
296
297
298
299
300
301
check_animate_xyz:
	-@(								\
	for iRunDir in ${FullResRunDirList};  do			\
		cd $${iRunDir};						\
		csh ${IDLDIR}/animate_xyz.sh;				\
	done)

Zhenguang Huang's avatar
Zhenguang Huang committed
302
303
304
305
306
307
308
309
clean_plot:
	@for RunDir in ${FullResRunDirList};  do 	\
		echo "cleaning $${RunDir}";		\
		cd $${RunDir}; 				\
		rm -f *eps; 				\
		rm -f log_insitu log_remote; 		\
	done

Zhenguang Huang's avatar
Zhenguang Huang committed
310
311
#########################################################################################

Zhenguang Huang's avatar
Zhenguang Huang committed
312
313
314
315
316
317
check_postproc_all:
	@for iSimDir in ${SimDirList}; do					\
		make check_postproc RESDIR=$${iSimDir} SIMDIR=$${iSimDir};	\
	done

check_compare_insitu_all:
318
319
320
	-@(cd ${IDLDIR}; 								\
	for iResDir in ${FullResDirList}; do						\
		csh compare_insitu.sh ${DIR} $${iResDir} $${iResDir} ${MODEL} 		\
321
			${MYDIR}/Results/obsdata ${EVTIME} ${DTEV}; 			\
322
	done)
Zhenguang Huang's avatar
Zhenguang Huang committed
323
324
325
326
327
328

check_compare_remote_all:
	@for iResDir in ${ResDirList};  do 				\
		make check_compare_remote RESDIR=$${iResDir};		\
	done

Lulu Zhao's avatar
Lulu Zhao committed
329
330
331
332
333
check_animate_xyz_all:
	@for iResDir in ${ResDirList};  do				\
		make check_animate_xyz RESDIR=$${iResDir};		\
	done

Zhenguang Huang's avatar
Zhenguang Huang committed
334
clean_plot_all:
335
	@(for iResDir in ${ResDirList};  do 				\
Zhenguang Huang's avatar
Zhenguang Huang committed
336
		make clean_plot RESDIR=$${iResDir};			\
337
338
339
340
341
342
343
	done;								\
	for iResDir in ${FullResDirList}; do				\
		echo "cleaning $${iResDir}"; 				\
		cd $${iResDir}; 					\
		rm -f *eps log_insitu log_remote; 			\
	done)

Zhenguang Huang's avatar
Zhenguang Huang committed
344
345

#########################################################################################