diff --git a/.github/workflows/bank-compress-workflow.yml b/.github/workflows/bank-compress-workflow.yml index d5cda78242e..d97b22ed798 100644 --- a/.github/workflows/bank-compress-workflow.yml +++ b/.github/workflows/bank-compress-workflow.yml @@ -48,7 +48,6 @@ jobs: cp examples/tmpltbank/compress_workflow/*.ini ./ bash -e examples/tmpltbank/compress_workflow/gen.sh cd output - bash -e ../examples/search/submit.sh python ../examples/search/check_job.py find submitdir/work/ -type f -name '*.tar.gz' -delete - name: store log files diff --git a/.github/workflows/inference-workflow.yml b/.github/workflows/inference-workflow.yml index fb6b6c329f9..0db7289855f 100644 --- a/.github/workflows/inference-workflow.yml +++ b/.github/workflows/inference-workflow.yml @@ -42,7 +42,6 @@ jobs: bash -e examples/workflow/inference/small_test/gen.sh condor_status cd gw_output - bash -e ../examples/search/submit.sh ./status python ../examples/search/check_job.py find submitdir/work/ -type f -name '*.tar.gz' -delete diff --git a/.github/workflows/search-workflow.yml b/.github/workflows/search-workflow.yml index cbe07bafe96..f3771636c28 100644 --- a/.github/workflows/search-workflow.yml +++ b/.github/workflows/search-workflow.yml @@ -52,7 +52,6 @@ jobs: bash -e examples/search/gen.sh cp *.gwf output/ cd output - bash -e ../examples/search/submit.sh python ../examples/search/check_job.py find submitdir/work/ -type f -name '*.tar.gz' -delete - name: store log files diff --git a/bin/bank/pycbc_geom_aligned_bank b/bin/bank/pycbc_geom_aligned_bank index 988f66e5c50..85580142fe3 100644 --- a/bin/bank/pycbc_geom_aligned_bank +++ b/bin/bank/pycbc_geom_aligned_bank @@ -611,5 +611,4 @@ workflow += bankverify_node workflow.save() logging.info("GENERATION SUCCESSFUL.") -logging.info("Submit the resulting dax file to generate your template bank.") -logging.info("See pycbc_submit_dax --help for instructions.") +logging.info("Submit the resulting workflow to generate your template bank.") diff --git a/bin/pycbc_submit_dax b/bin/pycbc_submit_dax deleted file mode 100644 index 5b5f3c230bf..00000000000 --- a/bin/pycbc_submit_dax +++ /dev/null @@ -1,251 +0,0 @@ -#!/bin/bash -# Redirect stdout ( > ) into a named pipe ( >() ) running "tee" -set -e - -mkdir -p workflow/planning -COMMAND_LINE="${0} ${@}" -echo "${COMMAND_LINE}" > workflow/planning/pycbc-submit-dax-command.txt -echo "Executing on `hostname` in directory `pwd` at `date`" > workflow/planning/pycbc-submit-dax-log.txt - -exec > >(tee -a workflow/pycbc-submit-dax-log.txt) - -# Without this, only stdout would be captured - i.e. your -# log file would not contain any error messages. -# SEE answer by Adam Spiers, which keeps STDERR a seperate stream - -# I did not want to steal from him by simply adding his answer to mine. -exec 2>&1 - -LOCAL_PEGASUS_DIR="" -PEGASUS_PROPERTIES="" -NO_QUERY_DB=0 -SUBMIT_DAX="--submit" -HTML_ENTITIES="{\"\'\": ''', '(': '(', ')': ')', '+': '+', '\"': '"'}" - -# These will be changed by the bundle builder -DATA_INLINE=False - -function expand_pegasus_files() { -( -base64 -d < extra-properties.conf -rm -f _reuse.cache -touch _reuse.cache -rm -f *-extra-site-properties.xml - -GETOPT_CMD=`getopt -o p:P:K:Q:n:l:G:h --long pegasus-properties:,append-pegasus-property:,no-create-proxy,no-query-db,no-submit,local-dir:,no-grid,help -n 'pycbc_submit_dax' -- "$@"` -eval set -- "$GETOPT_CMD" - -while true ; do - case "$1" in - -p|--pegasus-properties) - case "$2" in - "") shift 2 ;; - *) PEGASUS_PROPERTIES=$2 ; shift 2 ;; - esac ;; - -P|--append-pegasus-property) - case "$2" in - "") shift 2 ;; - *) echo $2 >> extra-properties.conf ; shift 2 ;; - esac ;; - -Q|--no-query-db) NO_QUERY_DB=1 ; shift ;; - -n|--no-submit) SUBMIT_DAX="" ; shift ;; - -l|--local-dir) - case "$2" in - "") shift 2 ;; - *) LOCAL_PEGASUS_DIR=$2 ; shift 2 ;; - esac ;; - -h|--help) - echo "usage: pycbc_submit_dax [-h] [optional arguments]" - echo - echo "optional arguments:" - echo " -h, --help show this help message and exit" - echo " -p, --pegasus-properties FILE use the specified file as" - echo " the pegasus properties file" - echo " -P, --append-pegasus-property STRING add the extra property" - echo " specified by the argument" - echo " -n, --no-submit Plan the DAX but do not submit it" - echo " -l, --local-dir Directory to put condor files under" - echo " -Q, --no-query-db Don't query the pegasus DB." - echo - echo "If the environment variable TMPDIR is set then this is prepended to the " - echo "path to the temporary workflow execute directory passed to pegasus-plan." - echo "If the --local-dir option is not given." - echo - echo "If the environment variable PEGASUS_FILE_DIRECTORY is set then the" - echo "script will look there for configuration, " - echo "otherwise the script will look for this directory by querying the" - echo "pycbc.workflow module." - echo - exit 0 ;; - --) shift ; break ;; - *) echo "Internal error!" ; exit 1 ;; - esac -done - -#Make a directory for the submit files -SUBMIT_DIR=`mktemp --tmpdir=${LOCAL_PEGASUS_DIR} -d pycbc-tmp.XXXXXXXXXX` - -#Make sure the directory is world readable -chmod 755 $SUBMIT_DIR - -# find the site-local template directory -if [ $DATA_INLINE == "True" ] ; then - expand_pegasus_files - PEGASUS_FILE_DIRECTORY=${PWD}/pegasus_files -elif [ -z $PEGASUS_FILE_DIRECTORY ] ; then - PEGASUS_FILE_DIRECTORY=`python -c 'from pycbc.workflow import PEGASUS_FILE_DIRECTORY;print(PEGASUS_FILE_DIRECTORY)'` -fi - -# Plan the workflow -echo "Generating concrete workflow" - -# cache the pegasus config -if [ -z ${PEGASUS_PROPERTIES} ] ; then - cp $PEGASUS_FILE_DIRECTORY/pegasus-properties.conf ./pegasus-properties.conf -else - cp ${PEGASUS_PROPERTIES} ./pegasus-properties.conf -fi - -echo >> pegasus-properties.conf -cat extra-properties.conf >> pegasus-properties.conf - - -# Ian's bash-fu is not good enough to wrap this in the nice error handling that -# it deserves! -STORED_PLANNER_ARGS=`cat additional_planner_args.dat` - -pegasus-plan --conf ./pegasus-properties.conf --dir $SUBMIT_DIR $SUBMIT_DAX ${STORED_PLANNER_ARGS} - -echo - -rm -f submitdir -ln -sf $SUBMIT_DIR submitdir - -echo "pegasus-status --long $SUBMIT_DIR/work \$@" > status -chmod 755 status - -echo "pegasus-analyzer -r -v $SUBMIT_DIR/work \$@" > debug -chmod 755 debug - -echo "pegasus-remove $SUBMIT_DIR/work \$@" > stop -chmod 755 stop - -echo "pegasus-run $SUBMIT_DIR/work \$@" > start - -chmod 755 start - -if [ -z ${SUBMIT_DAX} ] ; then - echo - echo "WARNING: DAX planned but not submitted. No dashboard entry has been created and" - echo " the workflow section of results page will not show a dashboard URL." - echo " You must run this script without the --no-submit option if this is a" - echo " production run." - echo - exit 0 -fi - -if [ ! -e ${HOME}/.pegasus/workflow.db ] ; then - echo "WARNING: Could not find Pegasus dashboard database in ${HOME}/.pegasus" - echo " Workflow has been submitted but the results page will not contain" - echo " a link to the dashboard page. If this is a production workflow," - echo " please remove the workflow, check for the origin of this error," - echo " and re-submit the workflow by re-running this script." - echo - exit 1 -fi - -WORKFLOW_ID_STRING="" -WORKFLOW_DB_CMD="sqlite3 -csv ${HOME}/.pegasus/workflow.db \"select submit_hostname,wf_id,wf_uuid from master_workflow where submit_dir = '${SUBMIT_DIR}/work';\"" -DB_TRY=0 - -# force a condor reschedule to get the workflow running -set +e -condor_reschedule &> /dev/null -set -e - -if [ $NO_QUERY_DB == 0 ]; then - - /bin/echo "Querying Pegasus database for workflow stored in ${SUBMIT_DIR}/work" - /bin/echo -n "This may take up to 120 seconds. Please wait..." - rm -f pegasus_db.log - touch pegasus_db.log - # querying the database sometimes fails, so allow retries - set +e - until [ $DB_TRY -ge 15 ] - do - /bin/echo -n "." - WORKFLOW_ID_STRING=`eval $WORKFLOW_DB_CMD 2>> pegasus_db.log` - if [ $? -eq 0 ] && [ ! -z $WORKFLOW_ID_STRING ] ; then - /bin/echo " Done." - DB_QUERY_SUCCESS=0 - break - else - DB_QUERY_SUCCESS=1 - fi - DB_TRY=$(( $DB_TRY + 1 )) - for s in `seq ${DB_TRY}` - do - /bin/echo -n "." - sleep 1 - done - done - set -e - - if [ ${DB_QUERY_SUCCESS} -eq 1 ] ; then - echo; echo - /bin/echo "Query failed: ${WORKFLOW_DB_CMD}" - cat pegasus_db.log - else - rm -f pegasus_db.log - fi - - if [ -z $WORKFLOW_ID_STRING ] ; then - echo "WARNING: Could not find the workflow in the Pegasus dashboard database." - echo " Workflow has been submitted but the results page will not contain" - echo " a link to the dashboard page. If this is a production workflow," - echo " please remove the workflow, check for the origin of this error," - echo " and re-submit the workflow by re-running this script." - echo - exit 1 - fi - -fi - -WORKFLOW_ID_ARRAY=(${WORKFLOW_ID_STRING//,/ }) -DASHBOARD_URL="https://${WORKFLOW_ID_ARRAY[0]}/pegasus/u/${USER}/r/${WORKFLOW_ID_ARRAY[1]}/w?wf_uuid=${WORKFLOW_ID_ARRAY[2]}" - -echo ${DASHBOARD_URL} > workflow/pegasus-dashboard-url.txt - -shopt -s nullglob -DASHBOARD_GLOB=(results/*_workflow/*DASHBOARD*.html) -DASHBOARD_PATH=${DASHBOARD_GLOB[0]} -if [[ -e ${DASHBOARD_PATH} ]] ; then - DASHBOARD_COPY=workflow/`basename ${DASHBOARD_PATH}`.orig - if [ -e ${DASHBOARD_COPY} ] ; then - cp ${DASHBOARD_COPY} ${DASHBOARD_PATH} - else - cp ${DASHBOARD_PATH} ${DASHBOARD_COPY} - fi - - COMMAND_LINE=$(python -c "import sys; from xml.sax.saxutils import (escape, unescape); print(escape(unescape(sys.stdin.read()), entities=${HTML_ENTITIES}))" <<< ${SITE_PROFILE}) - sed -i.bak -e "s+PYCBC_SUBMIT_DAX_ARGV+${COMMAND_LINE}+g" ${DASHBOARD_PATH} - sed -i.bak -e "s+PEGASUS_DASHBOARD_URL+${DASHBOARD_URL}+g" ${DASHBOARD_PATH} -fi -shopt -u nullglob - -echo "Workflow submission completed successfully." -echo -echo "The Pegasus dashboard URL for this workflow is:" -echo " ${DASHBOARD_URL}" -echo -echo "Note that it make take a while for the dashboard entry to appear while the workflow" -echo "is parsed by the dashboard. The delay can be on the order of one hour for very large" -echo "workflows." -echo - -exit 0 diff --git a/docs/tmpltbank.rst b/docs/tmpltbank.rst index 09ee2264070..f9e0c2070db 100644 --- a/docs/tmpltbank.rst +++ b/docs/tmpltbank.rst @@ -224,19 +224,21 @@ where ``accounting.tag`` should be replaced with one of the valid accounting tags. On non-LDG clusters, ``--supplement-config-file supplement.ini`` can be omitted if there are no extra options to give. -After running ``pycbc_geom_aligned_bank``, submit the workflow with +You can either submit the ``pycbc_geom_aligned_bank`` workflow immediately, +using .. code-block:: bash - pycbc_submit_dax - -Run + pycbc_geom_aligned_bank --submit-now + + +or submit the workflow after generation with the .. code-block:: bash - pycbc_submit_dax --help + ./start -for more details on options for submitting an abstract workflow to condor. +executable supplied in the output folder -------------------------- Command line options diff --git a/docs/upload_to_gracedb.rst b/docs/upload_to_gracedb.rst index 23f44972add..683b53202d8 100644 --- a/docs/upload_to_gracedb.rst +++ b/docs/upload_to_gracedb.rst @@ -25,7 +25,7 @@ You also need to be authorized on the gracedb end to do the upload. The initial Finding input files ============================ -To run this code requires some input files from your run. These can be found as follows where RUN_DIR is the place where you ran pycbc_submit_dax:: +To run this code requires some input files from your run. These can be found as follows where RUN_DIR is your run directory:: TRIG_FILE=${RUN_DIR}/results/7._result/H1L1-PAGE_FOREGROUND_XMLLOUDEST-1130754617-1019335.xml diff --git a/docs/workflow/pycbc_make_inference_inj_workflow.rst b/docs/workflow/pycbc_make_inference_inj_workflow.rst index 1d33a989914..9847f089897 100644 --- a/docs/workflow/pycbc_make_inference_inj_workflow.rst +++ b/docs/workflow/pycbc_make_inference_inj_workflow.rst @@ -123,21 +123,19 @@ to run the workflow. Plan and execute the workflow ----------------------------- -Change directory into the ``${WORKFLOW_NAME}-output`` directory:: - - cd ${WORKFLOW_NAME}-output +You can either submit the workflow immediately using the ``--submit-now`` option +or generate it, and then use the ``./start`` executable generated in the output directory. If you are on the ATLAS cluster (at AEI Hannover) or on an LDG cluster, you need to define an accounting group tag (talk to your cluster admins if you do -not know what this is). Once you know what accounting-group tag to use, plan -and submit the workflow with:: - - # submit workflow - pycbc_submit_dax --dax ${WORKFLOW_NAME}.dax \ - --no-grid \ - --no-create-proxy \ - --enable-shared-filesystem \ - --accounting-group ${ACCOUNTING_GROUP} +not know what this is). Once you know what accounting-group tag to use, add it +to your config files. ``request_disk`` and ``request_memory`` may also be required. + +.. code-block:: + + [pegasus_profile] + condor|accounting_group = accounting.tag + condor|request_disk = 1024 Here, ``${ACCOUNTING_GROUP}`` is the appropriate tag for your workflow. diff --git a/docs/workflow/pycbc_make_inference_workflow.rst b/docs/workflow/pycbc_make_inference_workflow.rst index ddd179b2379..1c501553c6c 100644 --- a/docs/workflow/pycbc_make_inference_workflow.rst +++ b/docs/workflow/pycbc_make_inference_workflow.rst @@ -178,6 +178,27 @@ same directory, you can generate the workflow by running the following script: :download:`Download <../../examples/workflow/inference/gw150914_gw170814-emcee_pt/create_workflow.sh>` +If you are on the ATLAS cluster (at AEI Hannover) or on an LDG cluster, you +need to define an accounting group tag (talk to your cluster admins if you do +not know what this is). Once you know what accounting-group tag to use, add it +to your config files. ``request_disk`` and ``request_memory`` may also be required. + +.. code-block:: + + [pegasus_profile] + condor|accounting_group = accounting.tag + condor|request_disk = 1024 + +Here, ``${ACCOUNTING_GROUP}`` is the appropriate tag for your workflow. + +Once it is running, you can monitor the status of the workflow by running +``./status`` from within the ``${WORKFLOW_NAME}-output`` directory. If your +workflow fails for any reason, you can see what caused the failure by running +``./debug``. If you need to stop the workflow at any point, run ``./stop``. +To resume a workflow, run ``./start``. If the ``pycbc_inference`` jobs were +still running, and they had checkpointed, they will resume from their last +checkpoint upon restart. + Note that you need to set the ``HTML_DIR`` before running. This tells the workflow where to save the results page when done. You can also change ``WORKFLOW_NAME`` if you like. @@ -187,39 +208,18 @@ This sets the seed that is passed to ``pycbc_inference`` (you set it here because it will be incremented for every ``pycbc_inference`` job that will be run in the workflow). -After the workflow has finished it will have created a directory named +After the workflow generator has finished it will have created a directory named ``${WORKFLOW_NAME}-output``. This contains the ``dax`` and all necessary files -to run the workflow. +to monitor the workflow. ------------------------------ -Plan and execute the workflow ------------------------------ +-------------------- +Monitor the workflow +-------------------- -Change directory into the ``${WORKFLOW_NAME}-output`` directory:: +Change directory into the ``${WORKFLOW_NAME}-output`` directory and run the ``status`` helper script:: cd ${WORKFLOW_NAME}-output - -If you are on the ATLAS cluster (at AEI Hannover) or on an LDG cluster, you -need to define an accounting group tag (talk to your cluster admins if you do -not know what this is). Once you know what accounting-group tag to use, plan -and submit the workflow with:: - - # submit workflow - pycbc_submit_dax --dax ${WORKFLOW_NAME}.dax \ - --no-grid \ - --no-create-proxy \ - --enable-shared-filesystem \ - --accounting-group ${ACCOUNTING_GROUP} - -Here, ``${ACCOUNTING_GROUP}`` is the appropriate tag for your workflow. - -Once it is running, you can monitor the status of the workflow by running -``./status`` from within the ``${WORKFLOW_NAME}-output`` directory. If your -workflow fails for any reason, you can see what caused the failure by running -``./debug``. If you need to stop the workflow at any point, run ``./stop``. -To resume a workflow, run ``./start``. If the ``pycbc_inference`` jobs were -still running, and they had checkpointed, they will resume from their last -checkpoint upon restart. + ./status ------------ Results page @@ -274,7 +274,7 @@ Note that we could have run both the ``emcee_pt`` analysis, above, and the need to remove any diagnostic plots that are unique to each sampler. Once you have downloaded the necessary files, create the workflow and launch -it using the same ``create_workflow.sh`` script and ``pycbc_submit_dax`` +it using the same ``create_workflow.sh`` script commands as above, making sure to change the ``WORKFLOW_NAME`` and ``SEED``. This will produce a results page that looks like the example diff --git a/docs/workflow/pycbc_make_offline_search_workflow.rst b/docs/workflow/pycbc_make_offline_search_workflow.rst index f963815b664..d8c3baf82b7 100644 --- a/docs/workflow/pycbc_make_offline_search_workflow.rst +++ b/docs/workflow/pycbc_make_offline_search_workflow.rst @@ -489,30 +489,27 @@ set the output web page location. Planning and Submitting the Workflow ==================================== -Pegasus is used to plan and submit the workflow. To involve Pegasus to plan a -PyCBC workflow, you use the command ``pycbc_submit_dax`` which takes the -command line arguments +Pegasus is used to plan and submit the workflow. To involve Pegasus to submit a +PyCBC workflow, you can use the argument ``--submit-now``. The workflow is planned +by default, and is submitted later using the +.. code-block:: -.. command-output:: pycbc_submit_dax --help + ./start -Note that you are running on a resource that mandates accounting, then you +executable which is made in the ``--output-dir`` directory you defined earlier. + +Note that if you are running on a resource that mandates accounting, then you will also need to add a valid tag with the ``--accounting-tag`` command line argument. Please see `the LDG accounting page `_. to -determine the correct tags. These can be applied by adding the following line -to your submit invocation. - -For example, to plan and submit the workflow in the example above, change to the directory that you specified with the ``--output`` -command line option to ``pycbc_make_offline_search_workflow`` and plan and submit -the workflow:: +determine the correct tags. Once you know what accounting-group tag to use, add it +to your config files. ``request_disk`` and ``request_memory`` may also be required. - cd output - pycbc_submit_dax --accounting-group ligo.dev.o1.cbc.explore.test --dax s6d_chunk3.dax - -.. note:: +.. code-block:: - The above example uses the accounting tag ``ligo.dev.o1.cbc.explore.test`` - which should not be used in practice. + [pegasus_profile] + condor|accounting_group = accounting.tag + condor|request_disk = 1024 You can monitor the status of the workflow with Pegasus Dashboard, or the other Pegasus tools described below. @@ -627,20 +624,19 @@ Setting up a workflow for data reuse ------------------------------------ The first step is to generate a new workflow that performs the analysis that -you would like to do. This workflow should be generated in a new directory so that it does not overwrite data from your previous workflows. -Data reuse happens at the ``pycbc_submit_dax`` step, so -first run ``pycbc_make_offline_search_workflow`` to build a new workflow, -following the instructions in the section :ref:`coincworkflowgenerate` of this -page. - -**Stop** before you plan and submit the workflow with ``pycbc_submit_dax``. -You will pass an additional file to ``pycbc_submit_dax`` using the +you would like to do. This workflow should be generated in a new directory so +that it does not overwrite data from your previous workflows. + +**Stop** before you generate the workflow. +You will pass an additional file to the generator using the ``--cache-file`` option with a list of files that Pegasus can re-use from a previous run. The Pegasus Workflow Planner will reduce the workflow using this cache file. Reduction works by deleting jobs from the workflow whose output files have been found in some location in this cache file. -The key to data reuse is building the cache file passed to ``pycbc_submit_dax``. This file maps a file created in the workflow to a URL and a site where that URL can be found. The syntax of the cache file is plain ASCII with each line in the file giving the location of a file in the format:: +The key to data reuse is building the cache file. This file maps a file created +in the workflow to a URL and a site where that URL can be found. The syntax of the cache +file is plain ASCII with each line in the file giving the location of a file in the format:: LOGICAL_FILE_NAME PHYSICAL_FILE_URL pool="SITE" @@ -657,7 +653,7 @@ the most useful. Pegasus will take care of adding transfer jobs for The string ``SITE`` is a hint that tells Pegasus on which site the ``PHYSICAL_FILE_URL`` can be found. The ``SITE`` string should be one of the -names used by ``pycbc_submit_dax`` to identify the cluster where jobs are run. +names used to identify the cluster where jobs are run. In practice there are only two execution sites used by PyCBC workflows: 1. ``local`` which is the regular Condor pool on the local cluster where the workflow is being run from. This is typically used when re-using data that exists on the filesystem of the local cluster. @@ -694,17 +690,24 @@ file, but for the two inspiral files it contains the subdirectory that the workflow uses to organize the files by GPS time. In the case of this file Pegasus will delete from the workflow the jobs that create the files ``H1-VETOTIME_CAT3-1169107218-1066800.xml``, ``L1-VETOTIME_CAT3-1169107218-1066800.xml``, ``116912/H1-INSPIRAL_FULL_DATA_JOB0-1169120586-1662.hdf``, and ``116912/H1-INSPIRAL_FULL_DATA_JOB1-1169120586-1662.hdf`` when it plans the workflow. Insted, the data will be re-used from the URLs specified in the cache. Since ``site="local"`` for these files, Pegasus expects that the files all exist on the host where the workflow is run from. Once a cache file has been constructed, to enable data re-use, you follow the -standard instructions for planning and submitting the workflow in the section +standard instructions for generating the workflow in the section :ref:`coincworkflowplan`, but add the ``--cache-file`` argument that points to the cache file that you have created. For example:: - pycbc_submit_dax --cache-file /path/to/prior_data.map --accounting-group ligo.dev.o1.cbc.explore.test --dax s6d_chunk3.dax + pycbc_make_offline_search_workflow --workflow-name s6d_chunk3 --output-dir output \ + --config-files ... \ + --config-overrides "results_page:output-path:${HOME}/public_html/s6/s6d-big-dog-weeks" \ + --cache-file /path/to/prior_data.map will use the URLs from the file ``/path/to/prior_data.map`` to implement data re-use and subsequent workflow reduction. If more than once cache file is -provided, pass the paths as a comma separated list to ``pycbc_submit_dax``:: +provided, pass the paths as a comma separated list:: + - pycbc_submit_dax --cache-file /path/to/prior_data.map,/path/to/other.map --accounting-group ligo.dev.o1.cbc.explore.test --dax s6d_chunk3.dax + pycbc_make_offline_search_workflow --workflow-name s6d_chunk3 --output-dir output \ + --config-files ... \ + --config-overrides "results_page:output-path:${HOME}/public_html/s6/s6d-big-dog-weeks" \ + --cache-file /path/to/prior_data.map,/path/to/other.map Which file URLs should be included in the reuse cache? There is no single correct way of deciding this, as it depends on exactly what you are trying to do. The sections @@ -775,7 +778,7 @@ will pull out all cache file lines for the outputs of ``pycbc_inspiral`` files a sed 's+/home/dbrown+gsiftp://sugwg-condor.phy.syr.edu/home/dbrown+g' inspiral_files.map.tmp > inspiral_files.map rm inspiral_files.map.tmp -6. Finally, copy the file ``inspiral_files.map`` to your new workflow directory and then run ``pycbc_submit_dax`` as usual, giving the path to ``inspiral_files.map`` as the ``--cache-file`` argument. +6. Finally, copy the file ``inspiral_files.map`` to your new workflow directory and then run ``pycbc_make_offline_search_workflow`` as usual, giving the path to ``inspiral_files.map`` as the ``--cache-file`` argument. --------------------------------------------------- Re-running a workflow using a new veto definer file @@ -822,7 +825,7 @@ This can be acomplished with the following command:: If category 1 vetoes have changed, you must also remove files matching ``PSD``, ``OPTIMAL``, and ``MERGE`` to remove the PSD estimation jobs, the jobs that compute the optimal SNR of injections, and the merged single-detector inspiral trigger files which may also change if the category 1 vetoes change. -6. Copy the file ``reuse_cache.map`` to your new workflow directory and then run ``pycbc_submit_dax`` as usual, giving the path to ``reuse_cache.map`` as the ``--cache-file`` argument. +6. Copy the file ``reuse_cache.map`` to your new workflow directory and then run ``pycbc_make_offline_search_workflow`` as usual, giving the path to ``reuse_cache.map`` as the ``--cache-file`` argument. ---------------------------- Re-running a failed workflow @@ -852,7 +855,7 @@ Once in the ``main_ID0000001`` directory, run the command:: changing ``/path/to`` to a location where you want to save the cache. -Now you can than use the ``partial_workflow.map`` cache file as the ``--cache-file`` argument to ``pycbc_submit_dax``. +Now you can than use the ``partial_workflow.map`` cache file as the ``--cache-file`` argument to ``pycbc_make_offline_search_workflow``. ----------------------------------------------- Using partial products from a previous workflow @@ -973,76 +976,4 @@ follwing lines to your ``executables.ini`` file:: -------------------- Running the workflow -------------------- - -Once you have planned the workflow as above, you must also modify the submission of the -workflow if it is to run successfully on the OSG. Add the following additional -arguments to ``pycbc_submit_dax``:: - - --no-create-proxy \ - --execution-sites osg \ - --append-pegasus-property 'pegasus.transfer.bypass.input.staging=true' \ - --local-staging-server gsiftp://`hostname -f` \ - --remote-staging-server gsiftp://`hostname -f` \ - -``hostname -f`` will give the correct value if there is a gsiftp server running on the -submit machine. If not, change this as needed. The ``remote-staging-server`` is the -intermediary computer than can pass files between the submitting computer and the computers -doing the work. ``hostname -f`` returns the full name of the computer. This full name has -to be one that is accessible to both the submit machine and the workers. The ``--no-create-proxy`` -may be omitted if you have LIGO.org credentials and will be retrieving data from authenticated -locations in CVMFS. - -You will also need to specify where the code should get the data needed to generate reduced -order model waveforms. To do this add the following additional arguments to ``pycbc_submit_dax``:: - - --append-site-profile 'local:env|LAL_DATA_PATH:/cvmfs/software.igwn.org/pycbc/lalsuite-extra/current/share/lalsimulation' \ - --append-site-profile 'osg:env|LAL_DATA_PATH:/cvmfs/software.igwn.org/pycbc/lalsuite-extra/current/share/lalsimulation' \ - -Here, ``current`` is a symbolic link to the latest version of the data and can be replaced with a -specific release (e.g. ``e02dab8c``) if required. - -It is also through arguments to ``pycbc_submit_dax`` that the workflow is made aware of which -Singularity image to use when running ``pycbc_inspiral``. This is done by including the following -argument to ``pycbc_submit_dax``:: - - --append-site-profile "osg:condor|+SingularityImage:\"/cvmfs/singularity.opensciencegrid.org/pycbc/pycbc-el8:latest\"" \ - -The precise line above will cause ``pycbc_inspiral`` to run using the code in the latest version of PyCBC -as found on the ``master`` branch. You may well prefer a specific version (for example, for a production -run) and each release will also have a corresponding Singularity image published to CVMFS. For example, -to use the ``1.14.3`` release of PyCBC, use instead the line:: - - --append-site-profile "osg:condor|+SingularityImage:\"/cvmfs/singularity.opensciencegrid.org/pycbc/pycbc-el8:v1.14.3\"" \ - -You may also direct the workflow to use a Singularity image of your own, if that has been published to CVMFS. - -When running on the OSG under Singularity, by default much of the environment of the host node where the -job runs will be inherited inside the container. In many cases this is desired, as some of the file-transfer -tools that Pegasus requires can come from that environment. In other cases, however, that environment may -interfere with what is in the container, and from release 1.14.3 onwards, the container itself includes -any necessary file transfer tools. If you want to be sure that it is the tools installed inside the container -that are used, then you must direct the workflow to have a clean environment inside the container, with nothing -in it that you have not specified using lines of the form ``--append-site-profile 'osg:env|VARNAME:VALUE`` -(there will also be present in the environment a few other variables that are needed for proper running of -Pegasus and HTCondor). To specify that you need your OSG jobs to run in a clean environment, also include -the following lines when invoking ``pycbc_submit_dax``:: - - --append-site-profile "osg:condor|+InitializeModulesEnv:False" \ - --append-site-profile "osg:condor|+SingularityCleanEnv:True" \ - --append-site-profile "osg:condor|getenv:False" \ - -In particular, it is recommended that LVK users run with the lines above. - -So far, we have described the arguments that will allow ``pycbc_inspiral`` to run on any OSG machine to which -you have access. If, in addition, you would like to run on an XSEDE resource on which you have an allocation, -then add the argument:: - - --append-site-profile 'osg:condor|+DESIRED_XSEDE_Sites:"Comet"' \ - -where you replace the desired site (in this example the Comet cluster) with wherever you have an allocation. If -you want to run **only** on that XSEDE cluster, then also add:: - - --append-site-profile 'osg:condor|+DESIRED_SITES:"Comet"' \ - -Shared file systems cannot be used with the OSG, so make sure that the ``--enable-shared-filesystem`` argument is -not provided to ``pycbc_submit_dax`` when running on the OSG. +FIXME: Add instructions here \ No newline at end of file diff --git a/docs/workflow/pycbc_make_psd_estimation_workflow.rst b/docs/workflow/pycbc_make_psd_estimation_workflow.rst index 8ee38904935..49e8021d3f9 100644 --- a/docs/workflow/pycbc_make_psd_estimation_workflow.rst +++ b/docs/workflow/pycbc_make_psd_estimation_workflow.rst @@ -149,6 +149,17 @@ The sections below control plotting jobs.:: analysis-title="PSD Estimation" analysis-subtitle="..." +If you are on the ATLAS cluster (at AEI Hannover) or on an LDG cluster, you +need to define an accounting group tag (talk to your cluster admins if you do +not know what this is). Once you know what accounting-group tag to use, add it +to your config files. ``request_disk`` and ``request_memory`` may also be required. + +.. code-block:: + + [pegasus_profile] + condor|accounting_group = accounting.tag + condor|request_disk = 1024 + =================================== Generating and running the workflow =================================== @@ -161,18 +172,19 @@ following way: pycbc_make_psd_estimation_workflow \ --workflow-name RUN_NAME \ --output-dir /path/to/run/directory \ - --config-files /path/to/ini/file + --config-files /path/to/ini/file \ + --submit-now ``RUN_NAME`` should be replaced with a meaningful descriptive name for the workflow and ``/path/to/run/directory`` should point to the directory where the -run is supposed to take place. Once the workflow is generated, move to +run is supposed to take place. + +If you don't want the workflow to start immediately, omit the ``--submit-now`` option. +Then after the workflow is generated, move to ``/path/to/run/directory`` and start the workflow with :: - - pycbc_submit_dax \ - --dax RUN_NAME.dax \ - --accounting-group ACCOUNTING_TAG + ./start where again ``RUN_NAME`` and ``ACCOUNTING_TAG`` should be given meaningful values. When the workflow completes, the average PSDs should be available in diff --git a/docs/workflow/pygrb.rst b/docs/workflow/pygrb.rst index ec7565fe6b5..cdbcfe672be 100644 --- a/docs/workflow/pygrb.rst +++ b/docs/workflow/pygrb.rst @@ -169,6 +169,21 @@ for the ``trig_combiner`` executable jobs, we may do this via:: This can be generalised to any executable or tagged jobs. +--------------- +Accounting Tags +--------------- + +If you are on the ATLAS cluster (at AEI Hannover) or on an LDG cluster, you +need to define an accounting group tag (talk to your cluster admins if you do +not know what this is). Once you know what accounting-group tag to use, add it +to your config files. ``request_disk`` and ``request_memory`` may also be required. + +.. code-block:: + + [pegasus_profile] + condor|accounting_group = accounting.tag + condor|request_disk = 1024 + .. _howtorunpygrb: ========== @@ -300,24 +315,15 @@ create the workflow from within the run directory using:: workflow:end-time:$(( GRB_TIME + 4096 )) \ workflow:html-dir:${HTML_DIR} -.. _pygrbplan: +You can submit to the job pool immediately by adding the option +.. code-block:: -==================================== -Planning and Submitting the Workflow -==================================== + --submit-now -Change directory into the directory where the dax was generated:: - - cd GRB${GRB_NAME} - -From the directory where the dax was created, run the submission script:: - - pycbc_submit_dax --dax pygrb_offline.dax --accounting-group - -.. note:: - - If running on the ARCCA cluster, please provide a suitable directory via - the option --local-dir, ie. /var/tmp/${USER} +or after generating, change to the working directory and run +.. code-block:: + + ./start ------------------------------------------------------------------------------------------------------------------------------------------- Monitor and Debug the Workflow (`Detailed Pegasus Documentation `_) @@ -376,14 +382,6 @@ One of the features of Pegasus is to reuse the data products of prior runs. This can be used to expand an analysis or recover a run with mistaken settings without duplicating work. ------------------------------------------ -Generate the full workflow you want to do ------------------------------------------ - -First generate the full workflow for the run you would like to do as normal, -following the instructions of this page from :ref:`howtorunpygrb`, but stop -before planning the workflow in :ref:`pygrbplan`. - ----------------------------------------------------- Select the files you want to reuse from the prior run ----------------------------------------------------- @@ -408,14 +406,6 @@ prior_data.map.:: You can include files in the prior data listing that wouldn't be generated anyway by your new run. These are simply ignored. ---------------------------- -Plan the workflow ---------------------------- - -From the directory where the dax was created, run the planning script:: - - pycbc_submit_dax --dax pygrb.dax --accounting-group --cache-file /path/to/prior_data.map - -Follow the remaining :ref:`pygrbplan` instructions to submit your reduced -workflow. +Add prior_data.map to your workflow generation, by adding the option +``--cache-file prior_data.map`` to ``pycbc_make_offline_grb_workflow`` diff --git a/examples/faith/pegasus_workflow_create.sh b/examples/faith/pegasus_workflow_create.sh index 4f2c5f1b2b7..72a3e28e740 100644 --- a/examples/faith/pegasus_workflow_create.sh +++ b/examples/faith/pegasus_workflow_create.sh @@ -3,4 +3,5 @@ WORKFLOW_NAME=example_faithsim_workflow pycbc_make_faithsim_workflow \ --workflow-name ${WORKFLOW_NAME} \ --config-files pegasus_workflow_conf.ini \ - --output-dir output + --output-dir output \ + --submit-now diff --git a/examples/faith/pegasus_workflow_submit.sh b/examples/faith/pegasus_workflow_submit.sh deleted file mode 100644 index 1496fc8a6b9..00000000000 --- a/examples/faith/pegasus_workflow_submit.sh +++ /dev/null @@ -1,3 +0,0 @@ -cd output - -pycbc_submit_dax diff --git a/examples/search/gen.sh b/examples/search/gen.sh index 1c94e68e21d..2ddfd34567d 100644 --- a/examples/search/gen.sh +++ b/examples/search/gen.sh @@ -5,4 +5,5 @@ pycbc_make_offline_search_workflow \ --workflow-name gw \ --output-dir output \ --config-files analysis.ini plotting.ini executables.ini injections_minimal.ini \ ---config-overrides results_page:output-path:$(pwd)/html +--config-overrides results_page:output-path:$(pwd)/html \ +--submit-now diff --git a/examples/search/master.sh b/examples/search/master.sh index b39534d7f49..498cffefa3c 100644 --- a/examples/search/master.sh +++ b/examples/search/master.sh @@ -8,5 +8,4 @@ bash -e gen.sh cp *.gwf output cd output -bash -e ../submit.sh python ../check_job.py diff --git a/examples/search/submit.sh b/examples/search/submit.sh deleted file mode 100644 index 0bd1f677bb6..00000000000 --- a/examples/search/submit.sh +++ /dev/null @@ -1,3 +0,0 @@ -pycbc_submit_dax \ ---local-dir ./ \ ---no-query-db \ diff --git a/examples/tmpltbank/compress_workflow/gen.sh b/examples/tmpltbank/compress_workflow/gen.sh index 087bff760d6..c8cfa2c933f 100644 --- a/examples/tmpltbank/compress_workflow/gen.sh +++ b/examples/tmpltbank/compress_workflow/gen.sh @@ -8,4 +8,5 @@ pycbc_make_bank_compression_workflow \ compress.ini \ executables.ini \ --config-overrides \ - results_page:output-path:$(pwd)/html + results_page:output-path:$(pwd)/html \ +--submit-now diff --git a/examples/workflow/inference/gw150914_gw170814-dynesty/create_workflow.sh b/examples/workflow/inference/gw150914_gw170814-dynesty/create_workflow.sh index d6a3353f559..64fd938c498 100755 --- a/examples/workflow/inference/gw150914_gw170814-dynesty/create_workflow.sh +++ b/examples/workflow/inference/gw150914_gw170814-dynesty/create_workflow.sh @@ -13,4 +13,5 @@ pycbc_make_inference_workflow \ --seed ${SEED} \ --config-files workflow_config.ini events.ini \ --workflow-name ${WORKFLOW_NAME} \ - --config-overrides results_page:output-path:${HTML_DIR}/${WORKFLOW_NAME} + --config-overrides results_page:output-path:${HTML_DIR}/${WORKFLOW_NAME} \ + --submit-now diff --git a/examples/workflow/inference/gw150914_gw170814-emcee_pt/create_workflow.sh b/examples/workflow/inference/gw150914_gw170814-emcee_pt/create_workflow.sh index 3bbf79974bc..9a39aa90c93 100755 --- a/examples/workflow/inference/gw150914_gw170814-emcee_pt/create_workflow.sh +++ b/examples/workflow/inference/gw150914_gw170814-emcee_pt/create_workflow.sh @@ -13,4 +13,5 @@ pycbc_make_inference_workflow \ --seed ${SEED} \ --config-files workflow_config.ini events.ini \ --workflow-name ${WORKFLOW_NAME} \ - --config-overrides results_page:output-path:${HTML_DIR}/${WORKFLOW_NAME} + --config-overrides results_page:output-path:${HTML_DIR}/${WORKFLOW_NAME} \ + --submit-now diff --git a/examples/workflow/inference/small_test/gen.sh b/examples/workflow/inference/small_test/gen.sh index 3d2ee3f92ee..e03117f01a7 100644 --- a/examples/workflow/inference/small_test/gen.sh +++ b/examples/workflow/inference/small_test/gen.sh @@ -8,7 +8,8 @@ pycbc_make_inference_workflow \ --workflow-name gw \ --config-overrides \ inference:config-overrides:data:"frame-files:'H1:${H1} L1:${L1} V1:${V1}'" \ - results_page:output-path:${PWD}/html + results_page:output-path:${PWD}/html \ + --submit-now # The above adds an option to the workflow.ini to set a config override for the inference jobs # the inference job then sets where it gets the frame files to our chosen location diff --git a/pycbc/workflow/core.py b/pycbc/workflow/core.py index b8351c48b25..7304bafd9ec 100644 --- a/pycbc/workflow/core.py +++ b/pycbc/workflow/core.py @@ -863,9 +863,9 @@ def save(self, filename=None, output_map_path=None, root=True): self.cp.write(fp) # save the sites file - #FIXME change to check also for submit_now if we drop pycbc_submit_dax - # this would prevent sub-workflows from making extra unused sites.yml - if not self.in_workflow: + # checking for submit_now prevents sub-workflows from making + # extra unused sites.yml + if not self.in_workflow and self.args.submit_now: catalog_path = os.path.join(self.out_dir, 'sites.yml') make_catalog(self.cp, self.out_dir).write(catalog_path) @@ -873,7 +873,6 @@ def save(self, filename=None, output_map_path=None, root=True): super(Workflow, self).save(filename=filename, output_map_path=output_map_path, submit_now=self.args.submit_now, - plan_now=self.args.plan_now, root=root) def save_config(self, fname, output_dir, cp=None): @@ -2332,11 +2331,6 @@ def add_workflow_settings_cli(parser, include_subdax_opts=False): wfgrp.add_argument("--cache-file", default=None, help="Path to input file containing list of files to " "be reused (the 'input_map' file)") - wfgrp.add_argument("--plan-now", default=False, action='store_true', - help="If given, workflow will immediately be planned " - "on completion of workflow generation but not " - "submitted to the condor pool. A start script " - "will be created to submit to condor.") wfgrp.add_argument("--submit-now", default=False, action='store_true', help="If given, workflow will immediately be submitted " "on completion of workflow generation") diff --git a/pycbc/workflow/pegasus_workflow.py b/pycbc/workflow/pegasus_workflow.py index 50f9ae6b2b8..08fb544f381 100644 --- a/pycbc/workflow/pegasus_workflow.py +++ b/pycbc/workflow/pegasus_workflow.py @@ -647,12 +647,11 @@ def save(self, filename=None, submit_now=False, plan_now=False, with open('additional_planner_args.dat', 'w') as f: stage_site_str = self.staging_site_str exec_sites = self.exec_sites_str - # For now we don't include --config as this can be added to - # in submit_dax. We should add an option to add additional + # We should add an option to add additional # pegasus properties (through the config files?) here. - #prop_file = os.path.join(PEGASUS_FILE_DIRECTORY, - # 'pegasus-properties.conf') - #f.write('--conf {} '.format(prop_file)) + prop_file = os.path.join(PEGASUS_FILE_DIRECTORY, + 'pegasus-properties.conf') + f.write('--conf {} '.format(prop_file)) if self.cache_file is not None: f.write('--cache {} '.format(self.cache_file)) @@ -672,7 +671,7 @@ def plan_and_submit(self, submit_now=True): """ Plan and submit the workflow now. """ # New functionality, this might still need some work. Here's things - # that this might want to do, that submit_dax does: + # that this might want to do, that submit_dax did: # * Checks proxy (ignore this, user should already have this done) # * Pulls properties file in (DONE) # * Send necessary options to the planner (DONE) diff --git a/tools/pycbc_test_suite.sh b/tools/pycbc_test_suite.sh index 4b42fa62df6..98b91ab458a 100755 --- a/tools/pycbc_test_suite.sh +++ b/tools/pycbc_test_suite.sh @@ -49,7 +49,7 @@ if [ "$PYCBC_TEST_TYPE" = "help" ] || [ -z ${PYCBC_TEST_TYPE+x} ]; then echo -e ">> [`date`] running $prog --help" $prog --help &> $LOG_FILE test_result - if [[ `echo $prog | egrep '(pycbc_copy_output_map|pycbc_submit_dax|pycbc_stageout_failed_workflow)'` ]] ; then + if [[ `echo $prog | egrep '(pycbc_copy_output_map|pycbc_stageout_failed_workflow)'` ]] ; then continue fi echo -e ">> [`date`] running $prog --version" diff --git a/tools/static/bundle_pycbc_submit_dax.sh b/tools/static/bundle_pycbc_submit_dax.sh deleted file mode 100755 index 0467079b325..00000000000 --- a/tools/static/bundle_pycbc_submit_dax.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -pushd ../../pycbc/workflow - -PEGASUS_FILE_DATA=`tar -zc pegasus_files | base64 | xargs echo | sed 's+ +ZZZZ+g'` - -popd - -mkdir -p dist - -sed -e 's+DATA_INLINE=False+DATA_INLINE=True+' \ - -e "s^PEGASUS_FILE_DATA^${PEGASUS_FILE_DATA}^" \ - -e "s+ZZZZ+\n+g" ../../bin/pycbc_submit_dax > dist/pycbc_submit_dax - - diff --git a/tools/test_coinc_search_workflow.sh b/tools/test_coinc_search_workflow.sh index 1f1784853ed..908ca9b4986 100755 --- a/tools/test_coinc_search_workflow.sh +++ b/tools/test_coinc_search_workflow.sh @@ -57,13 +57,8 @@ do pegasus-dax-validator $workflow done -echo -e "\\n>> [`date`] Planning test workflow" -pycbc_submit_dax \ - --force-no-accounting-group \ - --dax ${WORKFLOW_NAME}.dax \ - --no-create-proxy \ - --no-submit \ - --no-grid +echo -e "\\n>> [`date`] Submitting test workflow" +bash start popd popd