Generate html manuals with OUTGOING, copy html manuals for 3.21

BBS-make-OUTGOING.py

@@ -11,6 +11,7 @@
 import os
 import time
 import shutil
+import subprocess
 
 import bbs.fileutils
 import bbs.parse
@@ -119,6 +120,7 @@ def copy_outgoing_pkgs(products_in_subdir, source_node):
         if BBSvars.buildtype in ['workflows', 'books', 'bioc-mac-arm64']:
             pass
         elif source_node:
+            # pdf manuals
             pdf_file = os.path.join(BBSvars.products_in_rdir.path,
                                     BBSutils.getSourceNode(),
                                     'checksrc',
@@ -131,6 +133,20 @@ def copy_outgoing_pkgs(products_in_subdir, source_node):
                 os.link(pdf_file, dst) # create hard link to avoid making a copy
             else:
                 print("BBS> [stage6b]     SKIPPED (file %s doesn't exist)" % pdf_file)
+            # html manuals
+            rel_link = "../../%s/man/%s.html"
+            hooks = f"hooks <- list(pkg_href = function(pkg) sprintf('{rel_link}', pkg, pkg));"
+            html_file = os.path.join(BBSvars.Central_rdir.path,
+                                     "OUTGOING/manuals",
+                                    '%s.html' % pkg)
+            Rexpr = hooks + f"tools::pkg2HTML('{pkg_path}',out='{html_file}',hooks=hooks)"
+            cmd = BBSbase.Rexpr2syscmd(Rexpr)
+            try:
+                print(Rexpr)
+                subprocess.run(cmd, stdout=None, stderr=subprocess.STDOUT,
+                              shell=True, check=True)
+            except subprocess.CalledProcessError as e:
+                print("BBS> [stage6b]     SKIPPED (could not generate %s)" % html_file)
     print('BBS> [stage6b] END copying outgoing packages from %s.' % srcdir)
     return
 

propagation/3.20/updateReposPkgs-bioc.sh

@@ -17,6 +17,9 @@ WIN_CONTRIB="$REPOS_ROOT/bin/windows/contrib/$R_VERSION"
 MAC_BIG_SUR_x86_64_CONTRIB="$REPOS_ROOT/bin/macosx/big-sur-x86_64/contrib/$R_VERSION"
 MAC_BIG_SUR_arm64_CONTRIB="$REPOS_ROOT/bin/macosx/big-sur-arm64/contrib/$R_VERSION"
 
+META_SRC="$SRC_CONTRIB/Meta"
+META_R_EXPR="source('/home/biocbuild/BBS/utils/makeMetaDbs.R')"
+
 if [ ! -f "$PROPAGATION_DB_FILE" ]; then
         echo "ERROR: $PROPAGATION_DB_FILE not found. Did postrun.sh run?"
         exit 1
@@ -58,6 +61,17 @@ update_repo "$MAC_BIG_SUR_arm64_CONTRIB" "mac.binary.big-sur-arm64" "tgz"
 
 echo ""
 
+echo ""
+echo "========================================================================"
+/bin/date
+echo "------------------------------------------------------------------------"
+
+echo "Updating $META_SRC with aliases and cross refences dbs..."
+$Rscript -e "$META_R_EXPR; try(makeMetaDbs('$PROPAGATION_DB_FILE', '$REPOS_ROOT', '$META_SRC'))"
+
+echo ""
+
+
 ## FIXME: Why aren't manuals propagated based on the same criteria as source
 ## packages? Looks like the former are propagated based on their timestamps
 ## only (see below) while for source packages we use the more refined
@@ -68,7 +82,7 @@ MANUALS_DEST="$REPOS_ROOT/manuals"
 MANUALS_SRC="$BBS_OUTGOING_DIR/manuals"
 echo "Updating $BIOC_VERSION/bioc repo with reference manuals..."
 for i in `ls $MANUALS_SRC`; do
-	pkg=`echo $i| awk '{split($0,a,".pdf"); print(a[1])}'`
+	pkg=`echo $i| awk '{split($0,a,".(html|pdf)"); print(a[1])}'`
 	mkdir -p $MANUALS_DEST/$pkg/man
 	cp --update --verbose $MANUALS_SRC/$i $MANUALS_DEST/$pkg/man
 done

propagation/3.20/updateReposPkgs-data-annotation.sh

@@ -54,7 +54,7 @@ MANUALS_DEST="$REPOS_ROOT/manuals"
 MANUALS_SRC="$BBS_OUTGOING_DIR/manuals"
 echo "Updating $BIOC_VERSION/data/annotation repo with reference manuals..."
 for i in `ls $MANUALS_SRC`; do
-	pkg=`echo $i| awk '{split($0,a,".pdf"); print(a[1])}'`
+	pkg=`echo $i| awk '{split($0,a,".(html|pdf)"); print(a[1])}'`
 	mkdir -p $MANUALS_DEST/$pkg/man
 	cp --update --verbose $MANUALS_SRC/$i $MANUALS_DEST/$pkg/man
 done

propagation/3.20/updateReposPkgs-data-experiment.sh

@@ -54,7 +54,7 @@ MANUALS_DEST="$REPOS_ROOT/manuals"
 MANUALS_SRC="$BBS_OUTGOING_DIR/manuals"
 echo "Updating $BIOC_VERSION/data/experiment repo with reference manuals..."
 for i in `ls $MANUALS_SRC`; do
-	pkg=`echo $i| awk '{split($0,a,".pdf"); print(a[1])}'`
+	pkg=`echo $i| awk '{split($0,a,".(html|pdf)"); print(a[1])}'`
 	mkdir -p $MANUALS_DEST/$pkg/man
 	cp --update --verbose $MANUALS_SRC/$i $MANUALS_DEST/$pkg/man
 done

propagation/3.21/updateReposPkgs-bioc.sh

@@ -1,5 +1,4 @@
 #!/bin/bash
-
 set -e  # exit immediately if a simple command returns a non-zero status
 
 cd "$HOME/propagation/3.21"
@@ -17,6 +16,10 @@ WIN_CONTRIB="$REPOS_ROOT/bin/windows/contrib/$R_VERSION"
 MAC_BIG_SUR_x86_64_CONTRIB="$REPOS_ROOT/bin/macosx/big-sur-x86_64/contrib/$R_VERSION"
 MAC_BIG_SUR_arm64_CONTRIB="$REPOS_ROOT/bin/macosx/big-sur-arm64/contrib/$R_VERSION"
 
+META_SRC="$SRC_CONTRIB/Meta"
+META_R_EXPR="source('/home/biocbuild/BBS/utils/makeMetaDbs.R')"
+MEAT_PATH="/home/biocbuild/bbs-$BIOC_VERSION-bioc/meat"
+
 if [ ! -f "$PROPAGATION_DB_FILE" ]; then
         echo "ERROR: $PROPAGATION_DB_FILE not found. Did postrun.sh run?"
         exit 1
@@ -58,6 +61,17 @@ update_repo "$MAC_BIG_SUR_arm64_CONTRIB" "mac.binary.big-sur-arm64" "tgz"
 
 echo ""
 
+echo ""
+echo "========================================================================"
+/bin/date
+echo "------------------------------------------------------------------------"
+
+echo "Updating $META_SRC with aliases and cross refences dbs..."
+$Rscript -e "$META_R_EXPR; try(makeMetaDbs('$PROPAGATION_DB_FILE', '$MEAT_PATH', '$REPOS_ROOT', '$META_SRC'))"
+
+echo ""
+
+
 ## FIXME: Why aren't manuals propagated based on the same criteria as source
 ## packages? Looks like the former are propagated based on their timestamps
 ## only (see below) while for source packages we use the more refined
@@ -68,7 +82,7 @@ MANUALS_DEST="$REPOS_ROOT/manuals"
 MANUALS_SRC="$BBS_OUTGOING_DIR/manuals"
 echo "Updating $BIOC_VERSION/bioc repo with reference manuals..."
 for i in `ls $MANUALS_SRC`; do
-	pkg=`echo $i| awk '{split($0,a,".pdf"); print(a[1])}'`
+	pkg=`echo $i| awk '{split($0,a,".(html|pdf)"); print(a[1])}'`
 	mkdir -p $MANUALS_DEST/$pkg/man
 	cp --update --verbose $MANUALS_SRC/$i $MANUALS_DEST/$pkg/man
 done

propagation/3.21/updateReposPkgs-data-annotation.sh

@@ -54,7 +54,7 @@ MANUALS_DEST="$REPOS_ROOT/manuals"
 MANUALS_SRC="$BBS_OUTGOING_DIR/manuals"
 echo "Updating $BIOC_VERSION/data/annotation repo with reference manuals..."
 for i in `ls $MANUALS_SRC`; do
-	pkg=`echo $i| awk '{split($0,a,".pdf"); print(a[1])}'`
+	pkg=`echo $i| awk '{split($0,a,".(html|pdf)"); print(a[1])}'`
 	mkdir -p $MANUALS_DEST/$pkg/man
 	cp --update --verbose $MANUALS_SRC/$i $MANUALS_DEST/$pkg/man
 done

propagation/3.21/updateReposPkgs-data-experiment.sh

@@ -54,7 +54,7 @@ MANUALS_DEST="$REPOS_ROOT/manuals"
 MANUALS_SRC="$BBS_OUTGOING_DIR/manuals"
 echo "Updating $BIOC_VERSION/data/experiment repo with reference manuals..."
 for i in `ls $MANUALS_SRC`; do
-	pkg=`echo $i| awk '{split($0,a,".pdf"); print(a[1])}'`
+	pkg=`echo $i| awk '{split($0,a,".(html|pdf)"); print(a[1])}'`
 	mkdir -p $MANUALS_DEST/$pkg/man
 	cp --update --verbose $MANUALS_SRC/$i $MANUALS_DEST/$pkg/man
 done

utils/makeMetaDbs.R

@@ -0,0 +1,33 @@
+makeMetaDbs <- function(db_filepath, repos_root, meta_path) {
+
+  meat_dir <- Sys.getenv("BBS_MEAT_PATH")
+  prop_status <- read.dcf(db_filepath)
+
+  pkgs <- c()
+  for (i in 1:dim(prop_status)[2]) {
+    pkg_type_stage <- strsplit(colnames(prop_status)[i], "#")[[1]]
+    if ("source" %in% pkg_type_stage &&
+        strsplit(prop_status[i], ",")[[1]][1] %in% c("YES", "UNNEEDED")) {
+        pkgs <- c(pkgs, pkg_type_stage[1])
+    }
+  }
+
+  web_dir <- file.path(repos_root, "web", "packages")
+  meta_dir <- file.path(meta_path)
+  if (!dir.exists(meta_dir)) {
+    dir.create(meta_dir, recursive = TRUE)
+  }
+
+  pkg_paths <- file.path(meat_dir, pkgs)
+  for (pkg_path in pkg_paths) {
+    biocViews::build_db_from_source(pkg_path, repos_root)
+  }
+
+  aliases_db_file <- file.path(meta_dir, "aliases.rds")
+  meta_aliases_db <- biocViews::build_meta_aliases_db(web_dir, aliases_db_file)
+  saveRDS(meta_aliases_db, aliases_db_file, version = 2)
+
+  rdxrefs_db_file <- file.path(meta_dir, "rdxrefs.rds")
+  meta_rdxrefs_db <- biocViews::build_meta_rdxrefs_db(web_dir, rdxrefs_db_file)
+  saveRDS(meta_rdxrefs_db, rdxrefs_db_file, version = 2)
+}