Merge pull request #673 from NCAR/WRF4_updates

Update WRF-DART interface to support WRF4

Author: hkershaw-brown

CHANGELOG.rst

@@ -22,6 +22,11 @@ individual files.
 
 The changes are now listed with the most recent at the top.
 
+**May 16 2024 :: WRF v4. Tag v11.5.0**
+
+- WRF-DART and WRF-DART Tutorial updated to WRFv4. Note, not backwards compatible with WRFv3.9. 
+- local particle filter default value for pf_enkf_hybrid=.false. *contributed by Jon Poterjoy*
+
 **April 23 2024 :: Bug-fix: WRF hybrid vertical coordinate. Tag v11.4.1**
 
 - DART now detects whether WRF is using Hybrid Vertical Coordinate (HVC) introduced in WRFv3.9 or terrain following (TF) system.

assimilation_code/modules/assimilation/assim_tools_mod.pf.f90

@@ -165,7 +165,7 @@ module assim_tools_mod
 real(r8) :: pf_alpha                        = 0.30_r8
 integer  :: pf_kddm                         = 0
 logical  :: sampling_weighted_prior         = .true.
-logical  :: pf_enkf_hybrid                  = .true.
+logical  :: pf_enkf_hybrid                  = .false.
 real(r8) :: min_residual                    = 0.5_r8
 integer  :: pf_maxiter                      = 3
 real(r8) :: pf_kf_rtps_coeff                = 0.0_r8

conf.py

@@ -21,7 +21,7 @@
 author = 'Data Assimilation Research Section'
 
 # The full version, including alpha/beta/rc tags
-release = '11.4.1'
+release = '11.5.0'
 root_doc = 'index'
 
 # -- General configuration ---------------------------------------------------

guide/matlab-observation-space.rst

@@ -2,6 +2,8 @@
 MATLAB observation space diagnostics
 ####################################
 
+.. _configMatlab:
+
 Configuring MATLAB
 ==================
 

models/wrf/model_mod.f90

@@ -697,7 +697,7 @@ subroutine static_init_model()
    wrf%dom(id)%type_u      = get_type_ind_from_type_string(id,'U')
    wrf%dom(id)%type_v      = get_type_ind_from_type_string(id,'V')
    wrf%dom(id)%type_w      = get_type_ind_from_type_string(id,'W')
-   wrf%dom(id)%type_t      = get_type_ind_from_type_string(id,'T')
+   wrf%dom(id)%type_t      = get_type_ind_from_type_string(id,'THM')
    wrf%dom(id)%type_gz     = get_type_ind_from_type_string(id,'PH')
    wrf%dom(id)%type_qv     = get_type_ind_from_type_string(id,'QVAPOR')
    wrf%dom(id)%type_qr     = get_type_ind_from_type_string(id,'QRAIN')
@@ -7465,7 +7465,7 @@ subroutine fill_default_state_table(default_table)
                           'UPDATE                    ', &
                           '999                       '  /)
 row = row+1
-default_table(:,row) = (/ 'T                         ', &
+default_table(:,row) = (/ 'THM                       ', &
                           'QTY_POTENTIAL_TEMPERATURE ', &
                           'TYPE_T                    ', &
                           'UPDATE                    ', &

models/wrf/readme.rst

@@ -9,6 +9,10 @@ DART interface module for the Weather Research and Forecasting
 `(WRF) <https://www.mmm.ucar.edu/weather-research-and-forecasting-model>`__
 model. This page documents the details of the
 module compiled into DART that interfaces with the WRF data in the state vector.
+**The WRF-DART interface is compatible with WRF versions 4 and later, and is 
+no longer backwards compatible with WRFv3.9 and earlier.**  
+For more information on the interface changes required between 
+different WRF versions see the WRF tutorial link in the next section.
 
 WRF+DART Tutorial
 -----------------
@@ -362,4 +366,4 @@ Files
 References
 ----------
 
-http://www2.mmm.ucar.edu/wrf/users/docs/user_guide_V3/contents.html
+https://www2.mmm.ucar.edu/wrf/users/docs/user_guide_v4/contents.html

models/wrf/shell_scripts/add_bank_perts.ncl

@@ -43,16 +43,19 @@ begin
  asciiwrite("mem"+MEM_NUM+"_pert_bank_num",ens_mem_num)
  print ("bank member number "+ens_mem_num)
 
+
+;For WRFv4 or later prognostic temp variable is THM
  pert_fields = (/"U", "V", "T", "QVAPOR","MU"/)
+ wrf_fields  = (/"U", "V", "THM", "QVAPOR","MU"/)
  pert_scale  = (/scale_U,scale_V,scale_T,scale_Q,scale_M/)
  nperts = dimsizes(pert_fields)
  pert_in = addfile(pert_bank_path+"/"+pert_bank_file,"r")
  wrf_in  = addfile(wrf_file,"w")
  do n=0,nperts-1
-  temp_w = wrf_in->$pert_fields(n)$ 
+  temp_w = wrf_in->$wrf_fields(n)$
   temp_p = pert_in->$pert_fields(n)$
   temp_c = temp_w+(temp_p * pert_scale(n))
-  wrf_in->$pert_fields(n)$ = temp_c
+  wrf_in->$wrf_fields(n)$ = temp_c
   delete(temp_w)
   delete(temp_p)
   delete(temp_c)

models/wrf/shell_scripts/driver.csh

@@ -473,6 +473,11 @@ while ( 1 == 1 )
 
             else if ( $SUPER_PLATFORM == 'derecho' ) then
 
+               # Prevent double submission for member 1 only               
+               if ( $n == 1) then
+               sleep 5
+               endif
+
                if ( `qstat -wa | grep assim_advance_${n} | wc -l` == 0 ) then
 
                   echo "assim_advance_${n} is missing from the queue"
@@ -515,12 +520,12 @@ while ( 1 == 1 )
                else if ( $SUPER_PLATFORM == 'derecho' ) then
 
                   qsub assim_advance_mem${n}.csh
-
+                  sleep 5
                endif
       	       break
 
             endif
-            sleep 10    # this might need to be longer, though I moved the done flag lower in the
+            sleep 15    # this might need to be longer, though I moved the done flag lower in the
                         # advance_model.csh to hopefully avoid the file moves below failing
 
          end

models/wrf/shell_scripts/param.csh

@@ -34,7 +34,7 @@ set POST_STAGE_DIR   = ${BASE_DIR}/post
 set OBS_DIAG_DIR     = ${BASE_DIR}/obs_diag
 set PERTS_DIR        = ${BASE_DIR}/perts
 
-#  Directories that can be used by many things
+#  Assign path to DART, WRF, WPS and WRFDA build
 set SHELL_SCRIPTS_DIR = ${BASE_DIR}/scripts
 set DART_DIR          = /glade/work/USER/DART                     # set this appropriately #%%%#
 set WRF_DM_SRC_DIR    = /glade/work/USER/WRFV3                    # set this appropriately #%%%#
@@ -47,14 +47,14 @@ set GRIB_DATA_DIR     = ${ICBC_DIR}/grib_data                     # set this app
 set GRIB_SRC          = 'GFS'                                     # set this appropriately #%%%#
 
 # list of variables for extraction and cycling
-set extract_vars_a   = ( U V PH T MU QVAPOR QCLOUD QRAIN QICE QSNOW QGRAUP QNICE QNRAIN \
+set extract_vars_a   = ( U V PH THM MU QVAPOR QCLOUD QRAIN QICE QSNOW QGRAUP QNICE QNRAIN \
                          U10 V10 T2 Q2 PSFC TSLB SMOIS TSK RAINC RAINNC GRAUPELNC )
-set extract_vars_b   = ( U V W PH T MU QVAPOR QCLOUD QRAIN QICE QSNOW QGRAUP QNICE QNRAIN \
+set extract_vars_b   = ( U V W PH THM MU QVAPOR QCLOUD QRAIN QICE QSNOW QGRAUP QNICE QNRAIN \
                          U10 V10 T2 Q2 PSFC TSLB SMOIS TSK RAINC RAINNC GRAUPELNC \
                          REFL_10CM VT_DBZ_WT )
-set cycle_vars_a     =   ( U V PH T MU QVAPOR QCLOUD QRAIN QICE QSNOW QGRAUP QNICE QNRAIN \
+set cycle_vars_a     =   ( U V PH THM MU QVAPOR QCLOUD QRAIN QICE QSNOW QGRAUP QNICE QNRAIN \
                            U10 V10 T2 Q2 PSFC TSLB SMOIS TSK )
-set increment_vars_a = ( U V PH T MU QVAPOR QCLOUD QRAIN QICE QSNOW QGRAUP QNICE QNRAIN U10 V10 T2 Q2 PSFC )
+set increment_vars_a = ( U V PH THM MU QVAPOR QCLOUD QRAIN QICE QSNOW QGRAUP QNICE QNRAIN U10 V10 T2 Q2 PSFC )
 
 #  Diagnostic parameters
 set OBS_VERIF_DAYS      = 7

models/wrf/tutorial/README.rst

@@ -12,13 +12,33 @@ and only covers the  WRF-specific aspects of coupling with DART.
 It is not wise to try to run WRF-DART if you have no experience with
 either WRF or DART.
 
-This tutorial was designed to be compatible with WRF V3.9.1 and was
-tested with DART V11.0.2. Other releases of WRF may or may not be
-backwards or forwards compatible with this tutorial.
+.. Important ::
+
+  This tutorial was designed to be compatible with WRF Version 4 and was
+  tested with WRFv4.5.2. This tutorial should not be used with DART
+  versions 11.4.0 and earlier because those older versions do not account
+  for different coordinate systems including the sigma hybrid coordinates as 
+  described in `DART Issue #650 <https://github.com/NCAR/DART/pull/650>`__.
+  Furthermore, older versions do not account for the prognostic temperature variable
+  switch from ``T`` (perturbation potential temperature) to ``THM``, (either perturbation
+  potential temperature or perturbation moist potential temperature) as described in
+  `DART issue #661 <https://github.com/NCAR/DART/issues/661>`__. The current implementation
+  of the code sets ``T=THM`` because within &dynamics section of ``namelist.input``
+  ``use_theta_m=0``.
+
+  Earlier version of WRF (v3.9) may run without errors with more recent versions of
+  DART (later than 11.4.0), but the assimilation performance will be deprecated.  
+  If you need to run with earlier versions of WRF, please review the changes required
+  to switch from WRFv4 to WRFv3 as documented within 
+  `DART issue #661 <https://github.com/NCAR/DART/issues/661>`__,
+  or contact the DART team.  Earlier WRF versions also require different settings
+  within the WRF ``namelist.input`` file to promote vertical stability for the tutorial 
+  example. These settings are also described in DART Issue #661.
+
 Prior to running this tutorial, we urge the users to familarize themselves with the
 `WRF system <https://www2.mmm.ucar.edu/wrf/users/model_overview.html>`__
-(WRF_ARW, WPS and WRFDA), and to read through the `WRFv3.9  User's Guide
-<https://www2.mmm.ucar.edu/wrf/users/docs/user_guide_V3/user_guide_V3.9/contents.html>`__
+(WRF_ARW, WPS and WRFDA), and to read through the `WRFv4.5  User's Guide
+<https://www2.mmm.ucar.edu/wrf/users/docs/user_guide_v4/contents.html>`__
 and the `WRF model tutorials <https://www2.mmm.ucar.edu/wrf/users/tutorial/tutorial.html>`__
 
 The DART team is not responsible for and does not maintain the WRF code. For WRF related issues check out the
@@ -233,13 +253,16 @@ bash  ``export BASE_DIR=<path_to_your_working_directory>``
    ::
 
        cd $BASE_DIR
-       wget http://www.image.ucar.edu/wrfdart/tutorial/wrf_dart_tutorial_23May2018_v3.tar.gz
-       tar -xzvf wrf_dart_tutorial_23May2018_v3.tar.gz
+       wget http://www.image.ucar.edu/wrfdart/tutorial/wrf_dart_tutorial_29Apr2024.tar.gz
+       tar -xzvf wrf_dart_tutorial_29Apr2024.tar.gz
 
    After untarring the file you should see the following directories:
    *icbc, output, perts,* and *template.* The directory names (case
    sensitive) are important, as the scripts rely on these local paths
-   and file names.
+   and file names. Please note that the perturbation, surface and initial 
+   condition files were derived from an earlier version (pre-4.0) of WRF/WPS/WRFDA
+   but still maintains compatibility with the (post-4.0, post-11.4.0) 
+   WRF-DART versions recommended to run this WRF assimilation example.  
 
 2. You will need template WRF namelists from the
    ``$DART_DIR/models/wrf/tutorial/template`` directory:
@@ -923,10 +946,10 @@ between the background (prior) and the analysis (posterior) after running
 
 
 The ``analysis_increment.nc`` file includes the following atmospheric variables: 
-``MU, PH, PSFC, QRAIN, QCLOUD, QGRAUP, QICE, QNICE, QSNOW, QVAPOR, T`` and ``T2``.
-The example figure below shows the increments for temperature (T) only. You can 
-use **ncview** to advance through all 11 atmospheric pressure levels. You should
-see spatial patterns that look something like the meteorology of the day.
+``MU, PH, PSFC, QRAIN, QCLOUD, QGRAUP, QICE, QNICE, QSNOW, QVAPOR, THM`` and ``T2``.
+The example figure below shows the increments for THM (perturbation potential temperature)
+only. You can use **ncview** to advance through all 11 atmospheric pressure levels. 
+You should see spatial patterns that look something like the meteorology of the day.
 
 +--------------------------+--------------------------------+
 | |ncview1|                | |ncview2|                      |
@@ -1297,9 +1320,9 @@ period of the assimilation.
    calendar       = 'Gregorian',
    first_bin_start =  1601, 1, 1, 0, 0, 0,
    first_bin_end   =  2999, 1, 1, 0, 0, 0,
-   last_bin_end   =  2999, 1, 1, 0, 0, 0,
-   bin_interval_days    = 0,
-   bin_interval_seconds = 21600,
+   last_bin_end   =   2999, 1, 1, 0, 0, 0,
+   bin_interval_days    = 1000000,
+   bin_interval_seconds = 0,
    max_num_bins         = 1000,
    print_table          = .true
    /
@@ -1380,10 +1403,8 @@ More Resources
 --------------
 
 -  `Check or Submit DART Issues <https://github.com/NCAR/DART/issues>`__
--  `DAReS website <ttp://dart.ucar.edu>`__
--  `Preparing
-   MATLAB <https://dart.ucar.edu/pages/Getting_Started.html#matlab>`__
-   to use with DART.
+-  `DAReS website <http://dart.ucar.edu>`__
+-  :ref:`Preparing MATLAB<configMatlab>` to use with DART.
 -  `WRF model users page <http://www.mmm.ucar.edu/wrf/users>`__
 
 .. |ncview1| image:: ../../../guide/images/WRF_tutorial_ncview1.png

models/wrf/tutorial/template/input.nml.template

@@ -19,8 +19,8 @@
    first_obs_seconds        = -1,
    last_obs_days            = -1,
    last_obs_seconds         = -1,
-   num_output_state_members = 0,
-   num_output_obs_members   = 3,
+   num_output_state_members = 50,
+   num_output_obs_members   = 50,
    output_interval          = 1,
    num_groups               = 1,
    output_forward_op_errors = .false.,
@@ -117,7 +117,7 @@
    wrf_state_variables     = 'U','QTY_U_WIND_COMPONENT','TYPE_U','UPDATE','999',
                              'V','QTY_V_WIND_COMPONENT','TYPE_V','UPDATE','999',
                              'W','QTY_VERTICAL_VELOCITY','TYPE_W','UPDATE','999',
-                             'T','QTY_POTENTIAL_TEMPERATURE','TYPE_T','UPDATE','999',
+                             'THM','QTY_POTENTIAL_TEMPERATURE','TYPE_T','UPDATE','999',
                              'PH','QTY_GEOPOTENTIAL_HEIGHT','TYPE_GZ','UPDATE','999',
                              'MU','QTY_PRESSURE','TYPE_MU','UPDATE','999',
                              'QVAPOR','QTY_VAPOR_MIXING_RATIO','TYPE_QV','UPDATE','999',

models/wrf/tutorial/template/namelist.input.meso

@@ -109,6 +109,8 @@
  diff_6th_opt                        = 2, 2, 
  diff_6th_factor                     = 0.25, 0.12,
  epssm                               = 0.1
+ use_theta_m                         = 0
+ zadvect_implicit                    = 1
  /
 
  &bdy_control

models/wrf/work/input.nml

@@ -48,8 +48,8 @@
    first_obs_seconds        = -1,
    last_obs_days            = -1,
    last_obs_seconds         = -1,
-   num_output_state_members = 0,
-   num_output_obs_members   = 32,
+   num_output_state_members = 3,
+   num_output_obs_members   = 3,
    output_interval          = 1,
    num_groups               = 1,
    distributed_state        = .true.
@@ -205,7 +205,7 @@
                          'V',     'QTY_V_WIND_COMPONENT',     'TYPE_V',    'UPDATE','999',
                          'W',     'QTY_VERTICAL_VELOCITY',    'TYPE_W',    'UPDATE','999',
                          'PH',    'QTY_GEOPOTENTIAL_HEIGHT',  'TYPE_GZ',   'UPDATE','999',
-                         'T',     'QTY_POTENTIAL_TEMPERATURE','TYPE_T',    'UPDATE','999',
+                         'THM',     'QTY_POTENTIAL_TEMPERATURE','TYPE_T',    'UPDATE','999',
                          'MU',    'QTY_PRESSURE',             'TYPE_MU',   'UPDATE','999',
                          'QVAPOR','QTY_VAPOR_MIXING_RATIO',   'TYPE_QV',   'UPDATE','999',
                          'PSFC',  'QTY_SURFACE_PRESSURE',     'TYPE_PSFC', 'UPDATE','999',