Revision 55

submit/namd/sub_namd_gpu_bash (revision 55)
1
#!/bin/bash
2
#$ -S /bin/bash
3
#$ -N  mutm_ap-oxog
4
#$ -q  E5-2670gpuK20deb128
5
#$ -l  h=!c82gpgpu33
6
#$ -pe mpi_debian 16
7
#$ -cwd
8
#$ -V
9

  
10
# Loading modules to set up environment
11
export MODULEPATH=/home/tjiang/modules/lmod/:${MODULEPATH}
12
module use ${MODULEPATH}
13
module load NAMD/multicore+CUDA/2.12
14

  
15
# Setting up home and scratch directories
16
HOMEDIR=$SGE_O_WORKDIR
17
cd $HOMEDIR
18
SCRATCHDIR=/scratch/$USER/$JOB_ID
19
/bin/mkdir -p $SCRATCHDIR
20
HOSTFILE=$TMPDIR/machines
21
cp $HOSTFILE .
22
sed -i -e 's/^/host\ /' machines
23

  
24
# Please provide the following files for namd
25
CONFIGFILE=run2.namd
26
TOPFILE=DDB2_CT_hmr.prmtop
27
PDBFILE=DDB2_CT_run1_0.coor
28
FIXFILE=DDB2_CT_new.fix
29
COLVARSFILE=dihedral_run1.in
30
LOGFILE=run2.out
31
#CONFIGFILE=run1.namd
32
#TOPFILE=ap-oxog3go8.prmtop
33
#PDBFILE=mutm_ap-oxog.coor
34
#COLVARSFILE=dihedral_run1.in
35
#LOGFILE=run1.out
36

  
37
# Copying files to scratchdir
38
rsync -c $CONFIGFILE $TOPFILE $PDBFILE $FIXFILE $COLVARSFILE machines runscript $SCRATCHDIR
39

  
40
# The executables for mpirun and namd2
41
MPIRUN=mpirun
42
COMMAND=`which namd2`
43

  
44
# Go to scratch directory and run calculation there
45
cd $SCRATCHDIR
46
echo 'group main ++shell ssh' > nodelist 
47
cat machines>>nodelist
48

  
49
#/applis/PSMN/generic/NAMD/2.12/NAMD_2.12_Linux-x86_64-ibverbs-smp-CUDA/charmrun ++p $NSLOTS ++ppn 8 ++nodelist nodelist ++remote-shell ssh /applis/PSMN/generic/NAMD/2.12/NAMD_2.12_Linux-x86_64-ibverbs-smp-CUDA/namd2 +setcpuaffinity +idlepoll $CONFIGFILE> ${LOGFILE}_2nodes
50
#/applis/PSMN/generic/NAMD/2.12/NAMD_2.12_Linux-x86_64-ibverbs-smp-CUDA/charmrun +p${NSLOTS} ++ppn 8 ++nodelist nodelist ++remote-shell ssh /applis/PSMN/generic/NAMD/2.12/NAMD_2.12_Linux-x86_64-ibverbs-smp-CUDA/namd2 +setcpuaffinity +idlepoll $CONFIGFILE> ${LOGFILE}_2nodes
51
#charmrun +p$NSLOTS  $MPIRUN  $COMMAND $CONFIGFILE> $LOGFILE
52
#namd2 +p 16  $CONFIGFILE> $LOGFILE
53
namd2 +idlepoll +p $NSLOTS +devices 0,1 $CONFIGFILE> ${LOGFILE}_${NSLOTS}cpu_2gpu_node31
54

  
55
# Copy back data
56
rsync -c --exclude '$CONFIGFILE $TOPFILE $PDBFILE $COLVARSFILE' * $HOMEDIR
57
cd $HOMEDIR
58

  
59
# Cleaning up scratch directory and hostfile
60
rm -fr $SCRATCHDIR
submit/crystal/sub_properties17.sh (revision 55)
1
#!/bin/bash
2
#$ -S /bin/bash
3
#$ -N  LTS6-eff-mass
4
#$ -q  monointeldeb48,monointeldeb128,r820deb768
5
#$ -V
6
#$ -cwd
7

  
8
module use /home/tjiang/modules/lmod
9
module load crystal/17
10

  
11
export HOMEDIR=$SGE_O_WORKDIR
12
cd $HOMEDIR
13
export CRY17_INP=$HOMEDIR
14
export CRY17_SCRDIR=/scratch/$USER/$JOB_ID
15
/bin/mkdir -p $CRY17_SCRDIR
16
export EXEC=$CRYSTALROOT/utils/runprop17
17

  
18
# Running crystal properties
19
# You should provide the name of .d3 and .f9 files, excluding the extension
20
# i.e., if there exist tmp1.d3 and tmp2.f9, then the command for running
21
# crystal properties should look like
22
# $EXEC tmp1 tmp2
23
$EXEC LaTaS6 LaTaS6
24

  
25
# Cleaning up scratch directory
26
rm -fr $CRY17_SCRDIR
submit/crystal/sub_crystal17.sh (revision 55)
1
#!/bin/bash
2
#$ -S /bin/bash
3
#$ -N sp-LaTaS6-HSE06
4
#$ -q M6*,E5-2667*,E5-2670*
5
#$ -pe mpi16_debian 16
6
#$ -V
7
#$ -cwd
8

  
9
module use /home/tjiang/modules/lmod
10
module load crystal/17
11

  
12
export HOMEDIR=$SGE_O_WORKDIR
13
cd $HOMEDIR
14

  
15
INPUT=LaTaS6
16
export CRY17_SCRDIR=/scratch/$USER/$JOB_ID
17
/bin/mkdir -p $CRY17_SCRDIR
18
export CRY17P_MACH=$HOMEDIR
19

  
20
export MPIRUN=mpirun
21
export HOSTFILE=$TMPDIR/machines
22
#export hosts=$HOMEDIR/hosts_`basename $INPUT .d12`
23
cp $HOSTFILE $HOMEDIR/machines.LINUX
24
cp $HOSTFILE $HOMEDIR/nodes.par
25
#Too lazy to do a test, but the file needs to be refreshed!
26
#rm -f $hosts
27
#/bin/cat $TMPDIR/machines > $hosts
28
export OMP_NUM_THREADS=1
29
export MKL_NUM_THREADS=1
30
$CRYSTALROOT/utils/runmpi17 $NSLOTS $INPUT
31
rm -fr $CRY17_SCRDIR
32
#rm $HOMEDIR/hosts_`basename $INPUT .d12`
submit/dalton/sub_dalton_bash (revision 55)
1
#!/bin/bash
2
#$ -S /bin/bash
3
#$ -N test
4
#$ -q E5-2670deb64A,E5-2670deb64B,E5-2670deb64C
5
#$ -pe mpi16_debian 16
6
#$ -V
7
#$ -cwd
8

  
9
## Environement (SGE, VASP, MPI)
10
source /usr/share/modules/init/bash
11
module use /home/tjiang/modules/lmod
12
module load dalton/2016-ompi
13
#module load dalton/2016-ompi-with_yttrium
14
#module load dalton/2016-omp
15

  
16
DALTON_TMPDIR=/scratch/$USER/$JOB_ID/
17
dalton -t $DALTON_TMPDIR -noappend -omp 16 -mb 16000 dalinp.dal moinp.mol
submit/amber/sub_amber17.sh (revision 55)
1
#!/bin/bash
2
#$ -S /bin/bash
3
#$ -N amber17
4
#$ -cwd
5
#$ -V
6
#$ -o npt4.out
7
#$ -j y
8
##=====================
9
##Choose between cpu queues and gpu queues, default gpu queues
10
##cpu queues
11
##$ -q M6*,E5-2667*,E5-2697*,E5-2670deb*
12
##$ -pe mpi16_debian 16
13
##gpu queues
14
#$ -q E5-2670gpu*
15
#$ -pe mpi_debian 1
16
##=====================
17

  
18
# PSMN: $Id$
19
#
20

  
21
export MODULEPATH=/home/tjiang/modules/lmod/:${MODULEPATH}
22
module use ${MODULEPATH}
23

  
24
#module file for gpu version of amber
25
module load amber/17_gcc5.4.0_cuda8.0ga2_openmpi1.6.4
26

  
27
#module file for cpu only version of amber
28
#module load amber/17_intel15.0.2_openmpi1.6.4
29

  
30
ambermod=$(basename $AMBERHOME)
31
echo $ambermod
32
if [ $ambermod = "gcc5.4.0_cuda8.0ga2_openmpi1.6.4" ]; then
33
    amberexe="pmemd.cuda"
34
else
35
    amberexe="pmemd"
36
fi
37
echo $amberexe
38

  
39
ulimit -v unlimited
40
ulimit -s unlimited
41

  
42
######################"
43
HOMEDIR=$SGE_O_WORKDIR
44
HOSTFILE=$TMPDIR/machines
45

  
46
cd $SGE_O_WORKDIR
47

  
48
echo NSLOTS=$NSLOTS
49

  
50
SCRATCHDIR=/scratch/$USER/$JOB_ID/
51
echo $SCRATCHDIR
52
/bin/mkdir -p $SCRATCHDIR
53
###################
54
Mac=`hostname | awk '{print substr($1,1,2)}' `
55

  
56
NbMac=`wc $PE_HOSTFILE | awk '{print $1}' `
57
rm ./machines
58
for i in `seq 1 $NbMac`
59
    do
60
        Mac=` head -n $i $PE_HOSTFILE | tail -1 | awk '{print $1}' `
61
        Nb=` head -n $i $PE_HOSTFILE | tail -1 | awk '{print $2}' `
62
        for j in `seq 1 $Nb`
63
             do
64
                 echo $Mac >> ./machines
65
                  done
66
                  done
67

  
68
                  cp $PE_HOSTFILE ./PE_HOSTFILE
69

  
70
                  cd $SCRATCHDIR
71

  
72
                  echo "start at" 
73
                  date
74

  
75
                  echo -n "Starting Script at: "
76
                  date
77
                  $amberexe -O -i $HOMEDIR/min.in \
78
                                -o $HOMEDIR/min.out \
79
                                -p $HOMEDIR/dna_TT.prmtop \
80
                                -c $HOMEDIR/dna3_TT.inpcrd \
81
                                -r $HOMEDIR/dna_TT_min.rst \
82
                            	-x $HOMEDIR/dna_TT_min.mdcrd 
83

  
84
                  echo "done at" 
85
                  date
86
                  cp -rf * $HOMEDIR
87

  
88

  
submit/gaussian/sub_gaussian09.sh (revision 55)
1
#!/bin/bash
2
#$ -S /bin/bash
3
#$ -N test
4
#$ -o $JOB_NAME.batch-log
5
#$ -j y
6
#$ -cwd
7
#$ -V
8
#$ -q E5*
9
##$ -pe gaussian8 8
10
#$ -pe openmp8 8
11

  
12
# Version: $Id$
13

  
14
# trace script (debug purpose, not mandatory)
15
set -x
16
# set globals
17
job=structure11
18
ExtIn=com
19
ExtOut=log
20

  
21
# Environment setting for gaussian 
22
g09root=/home/tjiang/softs/gaussian/g09d01
23
source $g09root/g09/bsd/g09.profile
24
export Gaussian=$g09root/g09/g09
25

  
26
export RSHCOMMAND=ssh
27
export P4_RSHCOMMAND=ssh
28

  
29
#export LD_LIBRARY_PATH=/softs/acml/acml2.7.0/pathscale64/lib:/softs/fftw/2.1.5/ps-ib/lib:$LD_LIBRARY_PATH:/usr/lib
30

  
31
cd ${SGE_O_WORKDIR}
32

  
33
echo "########## debut env##################"
34
env
35
echo "########## fin env ##################"
36

  
37
echo "########## Hostname ##################"
38
hostname
39

  
40
HOMEDIR=${SGE_O_WORKDIR}
41
cd ${HOMEDIR}
42

  
43
# Replace 'home' by 'scratch', from SGE_O_WORKDIR, store to SCRATCHDIR
44
SCRATCHDIR=/scratch/$USER/$JOB_ID/
45
# check if scratch exist, create or complain
46
if [[ -d "/scratch" ]]
47
then
48
    mkdir -p ${SCRATCHDIR}
49
else
50
    echo "/scratch not found, cannot create ${SCRATCHDIR}"
51
fi
52

  
53
# check if scratchdir is OK
54
echo "ok" >  ${SCRATCHDIR}/toto
55
Ok=$(cat  ${SCRATCHDIR}/toto)
56
if [[ "${Ok}" != "ok" ]]
57
then
58
    # if scratch is unavailable, fallback to home
59
    CalcDir="${SGE_O_WORKDIR}/scratch"_${job}
60
    mkdir -p ${CalcDir}
61
else
62
    CalcDir=${SCRATCHDIR}
63
fi
64
mkdir -p ${CalcDir}
65

  
66
# Gaussian specific ScratchDir
67
#export GAUSS_SCRDIR=${CalcDir}
68

  
69
Machine=$(hostname)
70

  
71
# check if there is a chk or chk.gz
72
# TODO/FIXME there's a problem here, watching the trace: if no chk, it create a .chk.chk
73
NChk=` grep -i "chk" ${job}.${ExtIn} | head -1 | sed 's/=/ /g' | awk '{print $2}'`
74
if [ "$NChk" != "" ]
75
then
76
NChk=` basename $NChk .chk`.chk
77
fi
78
if [[ -s ${HOMEDIR}/${NChk} ]]
79
then
80
    cp ${HOMEDIR}/${NChk} ${CalcDir}/${NChk}
81
fi
82
if [[ -s ${HOMEDIR}/${NChk}.gz ]]
83
then
84
    cp ${HOMEDIR}/${NChk}.gz ${CalcDir}/${NChk}.gz
85
    gunzip ${CalcDir}/${NChk}.gz
86
fi
87

  
88
cp ${HOMEDIR}/${job}.${ExtIn} ${CalcDir}/
89

  
90
# prepare postprocess script
91
# TODO/FIXME is this still usefull ?
92
# - compress .chk
93
# - copy chk from $SCRATCHDIR to $HOMEDIR
94
# - cleanup $SCRATCHDIR
95
# add exec mode on script
96
#echo "ssh ${SGE_O_LOGNAME}@${Machine} gzip ${CalcDir}/*.chk " > ${HOMEDIR}/${job}_${Machine}_CLEAN.${ExtOut}
97
#echo "scp ${SGE_O_LOGNAME}@${Machine}:${CalcDir}/* ." > ${HOMEDIR}/${job}_${Machine}_CLEAN.${ExtOut}
98
#echo "ssh ${SGE_O_LOGNAME}@${Machine} rm -fr ${CalcDir}" >> ${HOMEDIR}/${job}_${Machine}_CLEAN.${ExtOut}
99
#echo "rm ${job}_${Machine}_CLEAN.${ExtOut}" >> ${HOMEDIR}/${job}_${Machine}_CLEAN.${ExtOut}
100
#chmod u+x ${HOMEDIR}/${job}_${Machine}_CLEAN.${ExtOut}
101

  
102
# logrotate old logs
103
if [[ -s ${HOMEDIR}/${job}.${ExtOut} ]]
104
then
105
    Ext=1
106
    while [[ -s ${HOMEDIR}/${job}.${ExtOut}_${Ext} ]]
107
    do
108
        let Ext=Ext+1
109
    done
110
    mv ${HOMEDIR}/${job}.${ExtOut} ${HOMEDIR}/${job}.${ExtOut}_${Ext}
111
fi
112

  
113
cd ${CalcDir}
114
echo ${CalcDir}
115
ls -al
116

  
117
echo "time ${Gaussian} < ${job}.${ExtIn} > ${HOMEDIR}/${job}.${ExtOut}"
118

  
119
#/usr/bin/time ${Gaussian} < ${job}.${ExtIn} > ${HOMEDIR}/${job}.${ExtOut}
120
${Gaussian} < ${job}.${ExtIn} > ${HOMEDIR}/${job}.${ExtOut}
121

  
122
cp * ${HOMEDIR}/
123

  
124
if [[ -s ${NChk} ]]
125
then
126
    gzip -9 ${NChk}
127
    cp ${NChk}.gz ${HOMEDIR}/
128
fi
129

  
130
# final cleanup (commented during debug)
131
#rm -r ${GAUSS_SCRDIR}
132
#rm -r ${CalcDir}
submit/gaussian/sub_gaussian16.sh (revision 55)
1
#!/bin/bash
2
#$ -S /bin/bash
3
#$ -N test
4
#$ -o $JOB_NAME.batch-log
5
#$ -j y
6
#$ -cwd
7
#$ -V
8
#$ -q E5*
9
##$ -pe gaussian8 8
10
#$ -pe openmp8 8
11

  
12
# Version: $Id$
13

  
14
# trace script (debug purpose, not mandatory)
15
set -x
16
# set globals
17
job=structure11
18
ExtIn=com
19
ExtOut=log
20

  
21
# Environment setting for gaussian 
22
module use /home/tjiang/modules/lmod
23
module load gaussian/g16-avx
24
#module load gaussian/g16-avx2
25
source $g16root/g16/bsd/g16.profile
26
export Gaussian=$g16root/g16/g16
27
#export LD_LIBRARY_PATH=/softs/acml/acml2.7.0/pathscale64/lib:/softs/fftw/2.1.5/ps-ib/lib:$LD_LIBRARY_PATH:/usr/lib
28

  
29
cd ${SGE_O_WORKDIR}
30

  
31
echo "########## debut env##################"
32
env
33
echo "########## fin env ##################"
34

  
35
echo "########## Hostname ##################"
36
hostname
37

  
38
HOMEDIR=${SGE_O_WORKDIR}
39
cd ${HOMEDIR}
40

  
41
# Replace 'home' by 'scratch', from SGE_O_WORKDIR, store to SCRATCHDIR
42
SCRATCHDIR=/scratch/$USER/$JOB_ID/
43
# check if scratch exist, create or complain
44
if [[ -d "/scratch" ]]
45
then
46
    mkdir -p ${SCRATCHDIR}
47
else
48
    echo "/scratch not found, cannot create ${SCRATCHDIR}"
49
fi
50

  
51
# check if scratchdir is OK
52
echo "ok" >  ${SCRATCHDIR}/toto
53
Ok=$(cat  ${SCRATCHDIR}/toto)
54
if [[ "${Ok}" != "ok" ]]
55
then
56
    # if scratch is unavailable, fallback to home
57
    CalcDir="${SGE_O_WORKDIR}/scratch"_${job}
58
    mkdir -p ${CalcDir}
59
else
60
    CalcDir=${SCRATCHDIR}
61
fi
62
#mkdir -p ${CalcDir}
63

  
64
# Gaussian specific ScratchDir
65
export GAUSS_SCRDIR=${CalcDir}
66

  
67
Machine=$(hostname)
68

  
69
# check if there is a chk or chk.gz
70
# TODO/FIXME there's a problem here, watching the trace: if no chk, it create a .chk.chk
71
NChk=` grep -i "chk" ${job}.${ExtIn} | head -1 | sed 's/=/ /g' | awk '{print $2}'`
72
if [ "$NChk" != "" ]
73
then
74
NChk=` basename $NChk .chk`.chk
75
fi
76
if [[ -s ${HOMEDIR}/${NChk} ]]
77
then
78
    cp ${HOMEDIR}/${NChk} ${CalcDir}/${NChk}
79
fi
80
if [[ -s ${HOMEDIR}/${NChk}.gz ]]
81
then
82
    cp ${HOMEDIR}/${NChk}.gz ${CalcDir}/${NChk}.gz
83
    gunzip ${CalcDir}/${NChk}.gz
84
fi
85

  
86
cp ${HOMEDIR}/${job}.${ExtIn} ${CalcDir}/
87

  
88
# prepare postprocess script
89
# TODO/FIXME is this still usefull ?
90
# - compress .chk
91
# - copy chk from $SCRATCHDIR to $HOMEDIR
92
# - cleanup $SCRATCHDIR
93
# add exec mode on script
94
#echo "ssh ${SGE_O_LOGNAME}@${Machine} gzip ${CalcDir}/*.chk " > ${HOMEDIR}/${job}_${Machine}_CLEAN.${ExtOut}
95
#echo "scp ${SGE_O_LOGNAME}@${Machine}:${CalcDir}/* ." > ${HOMEDIR}/${job}_${Machine}_CLEAN.${ExtOut}
96
#echo "ssh ${SGE_O_LOGNAME}@${Machine} rm -fr ${CalcDir}" >> ${HOMEDIR}/${job}_${Machine}_CLEAN.${ExtOut}
97
#echo "rm ${job}_${Machine}_CLEAN.${ExtOut}" >> ${HOMEDIR}/${job}_${Machine}_CLEAN.${ExtOut}
98
#chmod u+x ${HOMEDIR}/${job}_${Machine}_CLEAN.${ExtOut}
99

  
100
# logrotate old logs
101
if [[ -s ${HOMEDIR}/${job}.${ExtOut} ]]
102
then
103
    Ext=1
104
    while [[ -s ${HOMEDIR}/${job}.${ExtOut}_${Ext} ]]
105
    do
106
        let Ext=Ext+1
107
    done
108
    mv ${HOMEDIR}/${job}.${ExtOut} ${HOMEDIR}/${job}.${ExtOut}_${Ext}
109
fi
110

  
111
cd ${CalcDir}
112
echo ${CalcDir}
113
ls -al
114

  
115
echo "time ${Gaussian} < ${job}.${ExtIn} > ${HOMEDIR}/${job}.${ExtOut}"
116

  
117
#/usr/bin/time ${Gaussian} < ${job}.${ExtIn} > ${HOMEDIR}/${job}.${ExtOut}
118
${Gaussian} < ${job}.${ExtIn} > ${HOMEDIR}/${job}.${ExtOut}
119

  
120
cp * ${HOMEDIR}/
121

  
122
if [[ -s ${NChk} ]]
123
then
124
    gzip -9 ${NChk}
125
    cp ${NChk}.gz ${HOMEDIR}/
126
fi
127

  
128
# final cleanup (commented during debug)
129
#rm -r ${GAUSS_SCRDIR}
130
#rm -r ${CalcDir}

Also available in: Unified diff