A lot of changes to build and packaging scripts. Fixes #210 (finalizes).

Author: myurkin

devtools/README.md

@@ -0,0 +1,14 @@
+Tools used by ADDA developers for building, testing, and packaging of ADDA:
+* `build` - script to build ADDA with various flags and optionally copy the obtained executables
+* `build_degug` - compiles ADDA (in debug mode) with various compilation flags
+* `build_misc` - builds misc tools and optionally copies (installs) the obtained executables
+* `commit_docs` - script to commit `docs/` and `src/const.h` for release
+* `release_sequence.txt` - sequence of tasks to be made during release
+* `svg_images.txt` - typical workflow (guidelines) for producing `.svg` images
+* `test_new` - simple wrapper to build and test ADDA
+* `versions.txt` - note on files that need to be checked prior to release (e.g. contain version number)
+* `win_all.sh` - script to perform all Windows-specific tasks during release
+* `win_commit.sh` - script to run a sample simulation and commit `sample/` and  `win64/`
+* `zip_packages` - script to export (archive) source and binary packages for a given version
+
+

devtools/build

@@ -1,7 +1,9 @@
 #!/bin/bash
+# Builds several versions of ADDA and optionally copies (installs) the obtained executables
 # First optional parameter is compilation modes: seq, mpi, ocl, spa, any combination of them (separated by space in
 # quotes, implies cross modes), or all (default). 'spa' is sparse and requires combination with something.
 # Second optional parameter is destination path to copy executables (otherwise, they are left in compilation folders)
+# Compilation is performed silently with (-s flag)
 #
 # Copyright (C) ADDA contributors
 # GNU General Public License version 3
@@ -11,7 +13,7 @@ MODES=${1:-all}
 DEST=""
 if [ -n "$2" ]; then
   if [ -d "$2" ]; then
-    DEST=`cd "$2"; pwd`
+    DEST="`cd "$2"; pwd`"
   else
     echo "ERROR: could not find the directory '$2'" >&2
     exit 1
@@ -43,7 +45,7 @@ function check_mode {
 }
 function compile_copy {
   # $1 is mode, $2 - name of executable, $3 - extra options
-  if ! make $1 $3 $XFL; then
+  if ! make -s $1 $3 $XFL; then
     echo "ERROR during compiling ADDA with options '$1 $3 $XFL'" >&2
     exit 1
   fi

devtools/build_debug

@@ -0,0 +1,38 @@
+#!/bin/bash
+# Compiles ADDA (in debug mode) with various compilation flags - used to find compilation warnings
+# Compilation is performed silently with (-s flag)
+#
+# All warnings and errors go to stderr, so one may look mostly at this stream to catch failures 
+# (i.e. redirect it to file), however it is commonly colorized by make, so should be OK as is
+#
+# Copyright (C) ADDA contributors
+# GNU General Public License version 3
+
+function compile_debug {
+  # $1 is mode, $2 - extra options
+  if ! make -s $1 OPTIONS="DEBUG $2"; then
+    echo "ERROR during compiling ADDA with options '$1 OPTIONS=\"DEBUG $2\"'" >&2
+    exit 1
+  fi
+}
+
+# main code; we do not do any special cleaning, since believe that make tracks all dependences
+cd ./../src
+compile_debug all ""
+compile_debug all "DEBUGFULL"
+compile_debug all "FFT_TEMPERTON"
+compile_debug all "NO_FORTRAN"
+compile_debug all "PRECISE_TIMING"
+compile_debug all "USE_SSE3"
+
+compile_debug seq "NOT_USE_LOCK"
+compile_debug seq "ONLY_LOCKFILE"
+compile_debug seq "OVERRIDE_STDC_TEST"
+compile_debug seq "NO_GITHASH"
+
+compile_debug ocl "OCL_READ_SOURCE_RUNTIME"
+compile_debug ocl "OCL_BLAS"
+compile_debug ocl "CLFFT_APPLE"
+
+compile_debug "seq mpi" "SPARSE"
+compile_debug "seq mpi" "SPARSE USE_SSE3"

devtools/build_misc

@@ -0,0 +1,41 @@
+#!/bin/bash
+# Builds misc tools and optionally copies (installs) the obtained executables
+# The only optional parameter is destination path to copy executables (otherwise, they are left in compilation folders)
+# Executables of each misc package are copied into a separate directory in this path
+#
+# Copyright (C) ADDA contributors
+# GNU General Public License version 3
+
+# process input variables
+MISCDIR="`pwd`/../misc"
+DEST="$MISCDIR"
+if [ -n "$1" ]; then
+  if [ -d "$1" ]; then
+    DEST="`cd "$1"; pwd`"
+  else
+    echo "ERROR: could not find the directory '$1'" >&2
+    exit 1
+  fi
+fi
+
+MISCDIR="`pwd`/../misc"
+
+for dir in `ls "$MISCDIR"`; do
+  MAKEFILE="$MISCDIR/$dir/Makefile"
+  if [ -f "$MAKEFILE" ]; then
+    echo "Processing misc/$dir"
+    # The following is a bit complicated. Ideally should be replaced by something like
+    # make install
+    dest="$DEST/$dir"
+    if [ ! -d "$dest" ]; then
+        mkdir "$dest"
+    fi
+    cd "$dest"
+    make -f "$MAKEFILE" srcdir="$MISCDIR/$dir"
+    if [ $? -ne 0 ]; then
+      echo "ERROR: compilation in 'misc/$dir' failed"
+      exit 1
+    fi
+    rm -f *.o
+  fi
+done

devtools/commit_docs

@@ -0,0 +1,23 @@
+#!/bin/bash
+# Commits docs and const.h for release
+# Accepts exactly one argument - version number (like 1.4.0)
+
+# Copyright (C) ADDA contributors
+# GNU General Public License version 3
+
+# Test arguments
+if [ $# -ne 1 ]; then
+  echo "ERROR: requires 1 argument"
+  exit 1
+fi
+
+git pull
+if [ $? -ne 0 ]; then
+  echo "ERROR: error during git pull"
+  exit 1
+fi
+git commit -m "Preparing file for release $1: docs and const.h" ../doc/manual.docx ../doc/manual.pdf ../doc/history ../src/const.h 
+if [ $? -ne 0 ]; then
+  echo "ERROR: error during commiting docs"
+  exit 1
+fi

devtools/make_branch

@@ -1,41 +1,51 @@
-1) test the current source (standard tests)
-> cd ..../adda/src
-> make -s
-> cd ../tests/2exec
-edit comp2exec to compare against previous version
-> sh comp2exec seq, mpi, ocl + mpi_seq, ocl_seq
-
-# extra surface tests (optional, causes a number of false positives)
-edit comp2exec to enable SURF_EXT
-> sh comp2exec seq, mpi, ocl
-edit comp2exec to enable SURF_STANDARD (instead of SURF_EXT) and to compare against adda, etc. in the test directory (.)
-> cp ../../src/seq/adda .
-> cp ../../src/mpi/adda_mpi .
-> cp ../../src/ocl/adda_ocl .
-> sh comp2exec seq, mpi, ocl
-
-# sparse tests (first two lines are not required if done above)
-#> cp ../../src/seq/adda .
-#> cp ../../src/mpi/adda_mpi .
-> cd ../../src
-> make seq OPTIONS=SPARSE -s
-> cd ../tests/2exec
-edit comp2exec to enable SPARSE
-> sh comp2exec seq, mpi
-edit comp2exec to compare against adda and adda_mpi + enable SPARSE_STANDARD
-> sh comp2exec seq, mpi
-
-2) make sure that manual.doc/pdf, history, and const.h are up to date. In particular, version number in const.h should
-not contain "a" or "b" (if not explicitly making beta release).
-
-3) Update CodeDesign and corresponding schemes in doc/gv/
-
-(on 64-bit Windows)
-4) sh win_all.sh #.#
-
-(on Linux)
-5) ./zip_packages #.#
-
-6) Upload packages to GoogleCode
-7) Update Wiki pages: PackageDescription, ReleaseNotes, Features
-8) Send announcement to users.
+Use some normal shell, under Windows - this can be MSYS. Then 'sh' can be omitted in the following commands.
+
+1) Test for compiler warnings (about 10 min)
+> sh build_debug 2> log_debug
+
+2) Build and test the current source (a few hours)
+> sh test_new 2> log_test
+Before running the tests, make sure that ../tests/2exec/comp2exec script is properly set up. This includes
+the REFPATH (when comparing against previous version) and GUIDIFF. If the latter is used, you generally do
+not need to redirect stdout.
+It is also possible to separately run build and ../tests/2exec/test_all scripts (see help inside them)
+
+3) If any changes are made to the code or tests, commit them now. 
+   It is also a good idea to repeat some of the above on different systems.
+
+4) make sure that manual.doc/pdf, history, and const.h are up to date. See also versions.txt
+- version number in const.h should not contain "alpha" or "beta" (if not explicitly making beta release).
+- history should contain version date of the in-progress release.
+- manual should have correct version (including links on the first page). Look through the whole manual
+  (for figure placement, etc.). Update sample outputs in appendices, if needed. 
+- use doiLink macro in Word and produce pdf (better with Actobat plugin - leads to twice smaller size). 
+  Look through the pdf once more.
+
+5) Update CodeDesign and corresponding schemes in doc/gv/
+
+6) Make sure that the DLLs in win64/ correspond to the latest executables (to be copied there). They can be 
+   staged for commit (including deleted older ones) - then they will be committed together with executables.  
+
+7) Run the following aggregate script (on Windows), inserting appropriate version number:
+> sh win_all.sh #.#.#
+You can also go through it line by line.
+
+8) It is a good idea to test that the resulting executables can be run with given DLLs. For example you can run
+'set PATH=""' or 'export PATH=""' in a terminal before running the executables (to make other DLLs on your system
+unavailable).
+
+9) Now you are ready to go live (check your commits):
+> git push --tags
+
+10) Prepare binary and source packages
+> zip_packages #.#.#
+
+11) Create release at GitHub using this tag (Release notes is a shortened version of history). Attach the above 
+packages to it.
+
+12) Update wiki page Features
+
+13) Send announcement to users.
+
+14) With the next commit update ignore patterns in tests/2exec/comp2exec and, possibly, testing suites to compare 
+against the latest release

devtools/make_tag

@@ -26,7 +26,7 @@ function test_error {
   fi
 }
 
-# uncomment any of the following for quicker tests
+# comment out any of the following for quicker tests
 test_error prev
 test_error cross
 test_error extra

devtools/release_sequence.txt

@@ -1,7 +1,10 @@
 #!/bin/bash
-# Master script to builds and commit Windows executables, all functionality is inside child scripts
-# Accepts exactly one argument - version number (like 1.0)
-# Should be executed on Windows 64-bit
+# Master script to builds and commit Windows executables, most functionality is inside child scripts
+# Accepts exactly one argument - version number (like 1.4.0)
+# Should be executed on Windows
+#
+# Copyright (C) ADDA contributors
+# GNU General Public License version 3
 
 # Test arguments
 if [ $# -ne 1 ]; then
@@ -11,33 +14,27 @@ fi
 
 SHELL=/bin/bash
 
-$SHELL win_build_adda.sh 64
+$SHELL commit_docs $1
 if [ $? -ne 0 ]; then
-  echo "ERROR: error during building 64-bit ADDA"
+  echo "ERROR: error during commiting docs"
   exit 1
 fi
-$SHELL win_build_adda.sh 32
+$SHELL build all ../win64
 if [ $? -ne 0 ]; then
-  echo "ERROR: error during building 32-bit ADDA"
+  echo "ERROR: error during building ADDA"
   exit 1
 fi
-$SHELL win_build_misc.sh 64
+$SHELL build_misc ../win64/misc
 if [ $? -ne 0 ]; then
-  echo "ERROR: error during building 64-bit misc tools"
+  echo "ERROR: error during building misc tools"
   exit 1
 fi
-$SHELL win_build_misc.sh 32
-if [ $? -ne 0 ]; then
-  echo "ERROR: error during building 32-bit misc tools"
-  exit 1
-fi
-
 $SHELL win_commit.sh $1
 if [ $? -ne 0 ]; then
-  echo "ERROR: error during commit"
+  echo "ERROR: error during commit of binaries"
   exit 1
 fi
-$SHELL make_tag $1
+git tag -a v$1 -m "New release $1"
 if [ $? -ne 0 ]; then
   echo "ERROR: creating tag failed"
   exit 1