ProdAgent < Main

Tags: , view all tags
---++ !ProdAgent Installation
To install it:
<pre>
mkdir prodagent
export APT_VER=0.5.15lorg3.2-cmp; export VO_CMS_SW_DIR=/home/mdias/prodagent/
export SCRAM_ARCH=slc4_ia32_gcc345
export LANG "C"
wget -O $VO_CMS_SW_DIR/bootstrap.sh http://cmsrep.cern.ch/cmssw/cms/bootstrap.sh
sh -x $VO_CMS_SW_DIR/bootstrap.sh setup -repository comp -path $VO_CMS_SW_DIR -arch $SCRAM_ARCH >& $VO_CMS_SW_DIR/bootstrap_$SCRAM_ARCH.log
 sh -x $VO_CMS_SW_DIR/bootstrap.sh setup -repository comp -path $VO_CMS_SW_DIR -arch $SCRAM_ARCH >&
$VO_CMS_SW_DIR/bootstrap_$SCRAM_ARCH.log
source $VO_CMS_SW_DIR/$SCRAM_ARCH/external/apt/$APT_VER/etc/profile.d/init.sh
 apt-get update; apt-get install cms+prodagent+PRODAGENT_0_12_9-cmp
</pre>
Export necessary variables
<pre>
 cd /home/mdias/prodagent/slc4_ia32_gcc345/cms/prodagent/PRODAGENT_0_12_9-cmp/    
export PYTHONPATH=$PYTHONPATH:`pwd`/lib
 . /OSG/setup.sh 
 export PYTHONPATH=$PYTHONPATH:`pwd`/lib
 source /home/mdias/prodagent/slc4_ia32_gcc345/cms/prodagent/PRODAGENT_0_12_9-cmp/etc/profile.d/dependencies-setup.sh 
export PRODAGENT_WORKDIR=/home/mdias/work
PRODAGENT_CONFIG=$PRODAGENT_WORKDIR/ProdAgentConfig.xml
</pre>
Start to configure it
<pre>
 voms-proxy-init -hours 200 -voms cms:/cms/uscms
 prodAgent-new-config
 prodAgent-edit-config --component=ProdAgent --parameter=ProdAgentWorkDir --value=$PRODAGENT_WORKDIR
 prodAgent-edit-config --component=ProdAgent --parameter=ProdAgentName --value=T2_BR_SPRACE
 prodAgent-edit-config --component=JobCreator --parameter=CreatorName --value=OSGCreator
 prodAgent-edit-config --component=JobSubmitter --parameter=SubmitterName --value=OSGSubmitter
</pre>
To use the official production:
<pre>
 prodAgent-edit-config --component=LocalDBS --parameter=DBSURL --value=https://cmsdbsprod.cern.ch:8443/cms_dbs_prod_local_0X_writer/servlet/DBSServlet
 prodAgent-edit-config --component=LocalDBS --parameter=ReadDBSURL --value=http://cmsdbsprod.cern.ch/cms_dbs_prod_local_0X/servlet/DBSServlet
prodAgent-edit-config --component=DBSInterface --parameter=CloseBlockFiles --value=100
prodAgent-edit-config --component=MergeSensor --parameter=MinMergeFileSize --value=1000000000
prodAgent-edit-config --component=MergeSensor --parameter=MaxMergeFileSize --value=3000000000
prodAgent-edit-config --component=PhEDExConfig--parameter=PhEDExDropBox --value=/tmp
prodAgent-edit-config --component=PhEDExConfig--parameter=DBPARAM --value=/home/mdias/work/DBParam:Debug/SPRACE
</pre>
where the last file, =/home/mdias/work/DBParam=, has
<pre>
Section            Debug/SPRACE
Interface          Oracle
Database           cms_transfermgmt_sc
AuthDBUsername     cms_transfermgmt_sc_writer
AuthDBPassword     uNobtr3Ude9d
AuthRole           phedex_sprace_debug
AuthRolePassword   Ch452dfarMw
ConnectionLife     86400
LogConnection      on
LogSQL             off
</pre>
Continuing
<pre>
prodAgent-edit-config --component=DBSInterface  --parameter=skipPhEDExInjection --value="False"
prodAgent-edit-config --component=JobCleanup --parameter=SuccessArchive --value=/home/mdias/work/success
prodAgent-edit-config --component=JobCleanup --parameter=FailureArchive --value=/home/mdias/work/fail
prodAgent-edit-config --component=ProdMon --parameter=Team --value=T2_BR_SPRACE
prodAgent-edit-config --component=ProdMon --parameter=exportEnabled --value=True 
</pre>
Edit =$PRODAGENT_WORKDIR/SubmitterPluginConfig.xml= to point to condor
%CODE{"html"}% 
 <Parameter Name="SPRACE" Value="osg-ce.sprace.org.br/jobmanager-condor"/>
               <ConfigBlock Name="OSG">
                        <Parameter Name="GlobusScheduler" Value="condor"/>
                </ConfigBlock>
%ENDCODE%

Edit =$PRODAGENT_WORKDIR/CreatorPluginConfig.xml= to  configure the stage out
%CODE{"html"}% 
              <ConfigBlock Name="StageOut">
                        <Parameter Name="SetupCommand" Value=". $OSG_APP/cmssoft/cms/setup/bashrc"/>
                        <Parameter Name="TargetPathName" Value="/SRM?Manager=v2/pnfs/sprace.org.br/data/"/>
                        <Parameter Name="TargetHostName" Value="osg-se.sprace.org.br:8443"/>
                        <Parameter Name="TransportMethod" Value="srm"/>
                </ConfigBlock>
%ENDCODE%
and software setup
%CODE{"html"}% 
             <ConfigBlock Name="SoftwareSetup">
                        <Parameter Name="ScramCommand" Value="scramv1"/>
                        <Parameter Name="SetupCommand" Value=". $OSG_APP/cmssoft/cms/setup/bashrc"/>
                        <Parameter Name="ScramArch" Value="slc4_ia32_gcc345"/>
                </ConfigBlock>
%ENDCODE%
Start to configure the mysql database
<pre>
prodAgent-edit-config --component=ProdAgentDB --parameter=user --value=root
prodAgent-edit-config --component=ProdAgentDB --parameter=passwd --value=YourProdAgentPassHere
prodAgent-edit-config --component=ProdAgentDB --parameter=socketFileLocation --value=$PRODAGENT_WORKDIR/mysqldata/mysql.sock
prodAgent-edit-config --component=ProdAgentDB --parameter=dbName --value=ProdAgentDB
</pre>
Doing the database initialization
<pre>
mkdir $PRODAGENT_WORKDIR/mysqldata
mysql_install_db --datadir=$PRODAGENT_WORKDIR/mysqldata
mysqld_safe --datadir=$PRODAGENT_WORKDIR/mysqldata --socket=$PRODAGENT_WORKDIR/mysqldata/mysql.sock --skip-networking   --log-error=$PRODAGENT_WORKDIR/mysqldata/error.log --pid-file=$PRODAGENT_WORKDIR/mysqldata/mysqld.pid &
mysqladmin -u root password 'XXXXXXX' --socket=$PRODAGENT_WORKDIR/mysqldata/mysql.sock
</pre>
Install your databases
<pre>
prodAgent-install-db
prodAgent-install-bosslite-db
</pre>
---++ Starting !ProdAgent after install

To start it you can use the following script
%CODE{"sh"}%
#!/bin/bash
cd /home/mdias/prodagent/slc4_ia32_gcc345/cms/prodagent/PRODAGENT_0_12_9-cmp
export PRODAGENT_ROOT=`pwd`
export PATH=$PATH:`pwd`/bin
export PYTHONPATH=$PYTHONPATH:`pwd`/lib
export PRODAGENT_WORKDIR=/home/mdias/work
export PRODAGENT_CONFIG=$PRODAGENT_WORKDIR/ProdAgentConfig.xml
export MYTESTAREA=/home/mdias/prodagent
source /home/mdias/prodagent/slc4_ia32_gcc345/cms/prodagent/PRODAGENT_0_12_9-cmp/etc/profile.d/dependencies-setup.sh
/home/mdias/prodagent/slc4_ia32_gcc345/external/mysql/5.0.45-cmp3/bin/mysqld_safe\
--datadir=$PRODAGENT_WORKDIR/mysqldata --socket=$PRODAGENT_WORKDIR/mysqldata/mysql.sock --skip-networking \
-log-error=$PRODAGENT_WORKDIR/mysqldata/error.log --pid-file=$PRODAGENT_WORKDIR/mysqldata/mysqld.pid &
prodAgentd --start
%ENDCODE%
You can use also
<pre>
prodAgentd --status
prodAgentd --shutdown
</pre>
---++ !ProdAgent Mechanism
The =WorkflowInjector= component injects the workflow into the local PA DB, jobs are created, are injected into the =JobQueue=, get released by the the =ResourceMonitor=, are submitted by =JobSubmitter=, their Grid scheduling status is tracked by the =JobTracking= and when they are done the =GetOutput= component retrieves the output sandbox. Unmerged data is tracked by the =MergeAccountant= component and merge jobs are triggered by the =MergeSensor=. The =DBSInterface= component registers produced files/blocks/datasets in the local DBS, migrates closed merged blocks into global DBS and injects them into the PhEDEx database. The job archives containing the job logs of successful jobs end up in =$PRODAGENT_WORKDIR/SuccessArchive=. Jobs that failed are automatically resubmitted up to a configured maximum number of retries. After that they end up in =$PRODAGENT_WORKDIR/FailureArchive=


The basic monitoring is [[http://access.sprace.org.br:888][here ]].

---++ !AlpGen Production using !ProdAgent

!AlpGen uses a python script, !cmsGen to interface with !AlpGen inside CMSSW, following these steps:
   * 1rst step: get information from the config file
   * 2nd step: copy alpgen grid2 file
   * 3rd step: split the cfg file into two files
   * 4th step: make weighted events
   * 5th step: make unweighted events
   * 6th step: make links and providing some information 

First, set up your cms environment:
%CODE{"sh"}%
TOP=$PWD
export SCRAM_ARCH=slc4_ia32_gcc345
export BUILD_ARCH=slc4_ia32_gcc345
export VDT_LOCATION=/OSG
source $VDT_LOCATION/setup.sh
#Check for valid voms proxy, must be valid for at least 24hrs
voms-proxy-info -exists -valid 24:00 
if [ $? == 1 ] ; then
    voms-proxy-init -voms cms:/cms/uscms
fi
export X509_USER_PROXY=$(voms-proxy-info -path)
# define versions
project=CMSSW
version=2_1_12
# set up a CMSSW area
cd $TOP
export SW_DIR=/home/OSG_app/app/cmssoft/cms
source $SW_DIR/cmsset_default.sh ${project}_${version}
scramv1 project $project ${project}_${version}
cd ${project}_${version}/src
%ENDCODE%
Now we have to install the !AlpGen :
<pre>
cmsenv
export CVSROOT=:ext:mafd@cmscvs.cern.ch:/cvs_server/repositories/CMSSW
export CVS_RSH=/usr/bin/ssh
addpkg GeneratorInterface/AlpgenInterface V00-03-00-01
addpkg FWCore/Sources V00-15-01-01
cvs co -r 1.1.2.3 GeneratorInterface/AlpgenInterface/src/AlpgenExtractor.cc
scram b
</pre>
Create a configuration file , =w0j_0ptw100_alpgen.cfg=, with
   * First line: alpgen (*always*)
   * Second line: name of the executable
   * Third line: name of the grid2 file (*without ".grid2*)
   * All the rest: standard parameters, it's highly recommended not to include the seeds to avoid confusion (but it doesn't matter) 
 About =grid2 files=, they MUST be in the CMSSW release. They are in this area: =$HOME/CMSSW_2_1_12/src/GeneratorInterface/AlpgenInterface/data/=
<pre>
alpgen
wjet_0ptw100gen
w0j_0ptw100
njets 0
ebeam 7000
ih2 1
ickkw 1
iqopt 1
ptjmin 20
etajmax 5
drjmin 0.7
iwdecmode 4
</pre>
now you can run this
%CODE{"sh"}%
export PYTHONPATH=$PYTHONPATH:$HOME/prodagent/slc4_ia32_gcc345/cms/prodcommon/PRODCOMMON_0_12_9-cmp/lib/
mkdir $HOME/cmsRun1 $HOME/cmsGen1
python $HOME/prodagent/slc4_ia32_gcc345/cms/prodcommon/PRODCOMMON_0_12_9-cmp/lib/ProdCommon/CmsGen/cmsGen.py --generator=alpgen --number-of-events=1000 --seeds=10  --cfg=$HOME/w0j_0ptw100_alpgen.cfg
%ENDCODE%

---++ Create and Inject a workflow

Follow the steps bellow to create and inject into our farm:
<pre>
[mdias@access ~]$ cmsDriver.py Configuration/GenProduction/python/PYTHIA6_EWK_Zmumu_10TeV_cff.py -s GEN:ProductionFilterSequence,SIM,DIGI,L1,DIGI2RAW,HLT --eventcontent RAWSIM --datatier GEN-SIM-RAW --conditions FrontierConditions_GlobalTag,IDEAL_V11::All
[mdias@access ~]$ python /home/mdias/prodagent/slc4_ia32_gcc345/cms/prodagent/PRODAGENT_0_12_9-cmp/util/createProductionWorkflow.py  --py-cfg=PYTHIA6_EWK_Zmumu_10TeV_cff_py_GEN_SIM_DIGI_L1_DIGI2RAW_HLT_IDEAL.py --version=CMSSW_2_1_12 --channel=Zmumu
[mdias@access src]$ python $PRODAGENT_ROOT/util/publish.py WorkflowInjector:SetPlugin RequestFeeder
[mdias@access ~]$ python2.4 /home/mdias/prodagent/slc4_ia32_gcc345/cms/prodagent/PRODAGENT_0_12_9-cmp/util/publish.py RequestInjector:SetWorkflow /home/mdias/CMSSW_2_1_12/src/Test-Zmumu-mdias-1238180632-Workflow.xml 
[mdias@access ~]$ python2.4 /home/mdias/prodagent/slc4_ia32_gcc345/cms/prodagent/PRODAGENT_0_12_9-cmp/util/publish.py NewDataset /home/mdias/CMSSW_2_1_12/src/Test-Zmumu-mdias-1238180632-Workflow.xml
[mdias@access ~]$ python2.4 /home/mdias/prodagent/slc4_ia32_gcc345/cms/prodagent/PRODAGENT_0_12_9-cmp/util/publish.py RequestInjector:SetSitePref T2_BR_SPRACE
[mdias@access ~]$ python2.4 /home/mdias/prodagent/slc4_ia32_gcc345/cms/prodagent/PRODAGENT_0_12_9-cmp/util/publish.py RequestInjector:ResourcesAvailable
</pre>
Topic revision: r6 - 2009-04-01 - MarcoAndreFerreiraDias
antalya escort bursa escort eskisehir escort istanbul escort izmir escort