WIP: Packaging v3

.gitlab/ci/test.gitlab-ci.yml

@@ -71,10 +71,15 @@ test-questions:
   script:
     - python3 -m pytest src/tests/test_questions.py
 
-test-app-config:
+test-app-configpanel:
   extends: .test-stage
   script:
-    - python3 -m pytest src/tests/test_app_config.py
+    - python3 -m pytest src/tests/test_app_configpanel.py
+
+test-app-regenconf:
+  extends: .test-stage
+  script:
+    - python3 -m pytest src/tests/test_app_regenconf.py
 
 test-app-resources:
   extends: .test-stage
@@ -147,7 +152,9 @@ coverage:
       artifacts: true
     - job: test-questions
       artifacts: true
-    - job: test-app-config
+    - job: test-app-configpanel
+      artifacts: true
+    - job: test-app-regenconf
       artifacts: true
     - job: test-app-resources
       artifacts: true

helpers/helpers.v1.d/sources

@@ -18,6 +18,8 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
 
+readonly YNH_SOURCES_CACHE_DIR="/var/tmp/yunohost/download"
+
 # Download, check integrity, uncompress and patch the source from app.src
 #
 # usage: ynh_setup_source --dest_dir=dest_dir [--source_id=source_id] [--keep="file1 file2"] [--full_replace]
@@ -160,8 +162,8 @@ ynh_setup_source() {
     local local_src="/opt/yunohost-apps-src/${YNH_APP_ID}/${source_id}"
 
     # Gotta use this trick with 'dirname' because source_id may contain slashes x_x
-    mkdir -p $(dirname /var/cache/yunohost/download/${YNH_APP_ID}/${source_id})
-    src_filename="/var/cache/yunohost/download/${YNH_APP_ID}/${source_id}"
+    mkdir -p "$(dirname "$YNH_SOURCES_CACHE_DIR/${YNH_APP_ID}/${source_id}")"
+    src_filename="$YNH_SOURCES_CACHE_DIR/${YNH_APP_ID}/${source_id}"
 
     if [ "$src_format" = "docker" ]; then
         src_platform="${src_platform:-"linux/$YNH_ARCH"}"
@@ -199,10 +201,9 @@ ynh_setup_source() {
 
     # Keep files to be backup/restored at the end of the helper
     # Assuming $dest_dir already exists
-    rm -rf /var/cache/yunohost/files_to_keep_during_setup_source/
+    local keep_dir="$(mktemp --directory --tmpdir="/var/tmp/" ynh_files_to_keep_during_setup_source.$YNH_APP_ID.XXXXX)"
     if [ -n "$keep" ] && [ -e "$dest_dir" ]; then
-        local keep_dir=/var/cache/yunohost/files_to_keep_during_setup_source/${YNH_APP_ID}
-        mkdir -p $keep_dir
+        mkdir -p "$keep_dir"
         local stuff_to_keep
         for stuff_to_keep in $keep; do
             if [ -e "$dest_dir/$stuff_to_keep" ]; then
@@ -279,13 +280,12 @@ ynh_setup_source() {
 
     # Add supplementary files
     if test -e "$YNH_APP_BASEDIR/sources/extra_files/${source_id}"; then
-        cp --archive $YNH_APP_BASEDIR/sources/extra_files/$source_id/. "$dest_dir"
+        cp --archive "$YNH_APP_BASEDIR/sources/extra_files/$source_id/." "$dest_dir"
     fi
 
     # Keep files to be backup/restored at the end of the helper
     # Assuming $dest_dir already exists
     if [ -n "$keep" ]; then
-        local keep_dir=/var/cache/yunohost/files_to_keep_during_setup_source/${YNH_APP_ID}
         local stuff_to_keep
         for stuff_to_keep in $keep; do
             if [ -e "$keep_dir/$stuff_to_keep" ]; then
@@ -300,5 +300,5 @@ ynh_setup_source() {
             fi
         done
     fi
-    rm -rf /var/cache/yunohost/files_to_keep_during_setup_source/
-}
+    rm -rf "$keep_dir"
+}

helpers/helpers.v1.d/utils

@@ -41,7 +41,7 @@ ynh_exit_properly() {
     local exit_code=$?
 
     if [[ "${YNH_APP_ACTION:-}" =~ ^install$|^upgrade$|^restore$ ]]; then
-        rm -rf "/var/cache/yunohost/download/"
+        rm -rf "${YNH_SOURCES_CACHE_DIR}/${YNH_APP_ID}"
     fi
 
     if [ "$exit_code" -eq 0 ]; then

helpers/helpers.v2.1.d/0-utils

@@ -27,8 +27,8 @@ YNH_APP_BASEDIR=${YNH_APP_BASEDIR:-$(realpath ..)}
 ynh_exit_properly() {
     local exit_code=$?
 
-    if [[ "${YNH_APP_ACTION:-}" =~ ^install$|^upgrade$|^restore$ ]]; then
-        rm -rf "/var/cache/yunohost/download/"
+    if [[ "${YNH_APP_ACTION:-}" =~ ^install$|^upgrade$|^restore$|^build$|^init$|^migrate$ ]]; then
+        rm -rf "${YNH_SOURCES_CACHE_DIR}/${YNH_APP_ID}"
     fi
 
     if [ "$exit_code" -eq 0 ]; then
@@ -270,13 +270,21 @@ _ynh_apply_default_permissions() {
             # Files inside should be owned by $app with rw-r----- (+x for folders or files that already have +x)
             # The group needs read/dirtraversal (in particular if it's www-data)
             chmod -R u=rwX,g=rX,o=--- "$target"
-            chown -R "$app:$group" "$target"
+            # FIXME : packaging v3 ... can't change group of a file to a group the user ain't part of (in particular, www-data)
+            if [[ "$group" == "www-data" ]] && [[ "$(whoami)" == "root" ]]
+            then
+                chown -R "$app:$group" "$target"
+            fi
             return
         elif [ "$target" == "${data_dir:-}" ]; then
             # Read the group from the data manifest resource
             local group="$(ynh_read_manifest 'resources.data_dir.group' | sed 's/null//g' | sed "s/__APP__/$app/g" | cut -f1 -d:)"
             chmod 750 "$target"
-            chown -R "$app:${group:-$app}" "$target"
+            # FIXME : packaging v3 ... can't change group of a file to a group the user ain't part of (in particular, www-data)
+            if [[ "$group" == "www-data" ]] && [[ "$(whoami)" == "root" ]]
+            then
+                chown -R "$app:${group:-$app}" "$target"
+            fi
             return
         fi
     fi

helpers/helpers.v2.1.d/composer

@@ -55,9 +55,18 @@ ynh_composer_install() {
 ynh_composer_exec() {
     local workdir="${composer_workdir:-$install_dir}"
 
-    COMPOSER_HOME="$workdir/.composer" \
+    if [[ "$(whoami)" == "root" ]]
+    then
+        COMPOSER_HOME="$workdir/.composer" \
         COMPOSER_MEMORY_LIMIT=-1 \
         sudo -E -u "${composer_user:-$app}" \
         "php$php_version" "$workdir/composer.phar" "$@" \
         -d "$workdir" --no-interaction --no-ansi 2>&1
+    else
+        COMPOSER_HOME="$workdir/.composer" \
+        COMPOSER_MEMORY_LIMIT=-1 \
+        "php$php_version" "$workdir/composer.phar" "$@" \
+        -d "$workdir" --no-interaction --no-ansi 2>&1
+    fi
+
 }

helpers/helpers.v2.1.d/sources

@@ -18,6 +18,8 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
 
+readonly YNH_SOURCES_CACHE_DIR="/var/tmp/yunohost/download"
+
 # Download, check integrity, uncompress and patch upstream sources
 #
 # usage: ynh_setup_source --dest_dir=dest_dir [--source_id=source_id] [--keep="file1 file2"] [--full_replace]
@@ -92,6 +94,11 @@ ynh_setup_source() {
     full_replace="${full_replace:-0}"
     source_id="${source_id:-main}"
     # ===========================================
+    # Make sure to keep composer.phar that may have been provisioned prior to this, otherwise it'll get deleted right after when using --full-replace?
+    if [ -e "$dest_dir/composer.phar" ]
+    then
+        keep+=" composer.phar"
+    fi
 
     # Make sure to keep composer.phar that may have been provisioned prior to this,
     # otherwise it'll get deleted right after when using --full-replace?
@@ -143,8 +150,8 @@ ynh_setup_source() {
     fi
 
     # Gotta use this trick with 'dirname' because source_id may contain slashes x_x
-    mkdir -p "$(dirname "/var/cache/yunohost/download/$YNH_APP_ID/$source_id")"
-    src_filename="/var/cache/yunohost/download/$YNH_APP_ID/$source_id"
+    mkdir -p "$(dirname "$YNH_SOURCES_CACHE_DIR/$YNH_APP_ID/$source_id")"
+    src_filename="$YNH_SOURCES_CACHE_DIR/$YNH_APP_ID/$source_id"
 
     if [ "$src_format" = "docker" ]; then
         src_platform="${src_platform:-"linux/$YNH_ARCH"}"
@@ -178,9 +185,8 @@ ynh_setup_source() {
 
     # Keep files to be backup/restored at the end of the helper
     # Assuming $dest_dir already exists
-    rm -rf /var/cache/yunohost/files_to_keep_during_setup_source/
+    local keep_dir="$(mktemp --directory --tmpdir="/var/tmp/" ynh_files_to_keep_during_setup_source.$YNH_APP_ID.XXXXX)"
     if [ -n "$keep" ] && [ -e "$dest_dir" ]; then
-        local keep_dir=/var/cache/yunohost/files_to_keep_during_setup_source/${YNH_APP_ID}
         mkdir -p "$keep_dir"
         local stuff_to_keep
         for stuff_to_keep in $keep; do
@@ -192,7 +198,14 @@ ynh_setup_source() {
     fi
 
     if [ "$full_replace" -eq 1 ]; then
-        ynh_safe_rm "$dest_dir"
+        if [[ "$dest_dir" == "$install_dir" ]]
+        then
+            # Delete all the content except the directory itself ... because we may run as $app (not root)
+            # and $app can't delete its own install dir
+            find "$dest_dir" -mindepth 1 -delete
+        else
+            ynh_safe_rm "$dest_dir"
+        fi
     fi
 
     # Extract source into the app dir
@@ -256,7 +269,6 @@ ynh_setup_source() {
     # Keep files to be backup/restored at the end of the helper
     # Assuming $dest_dir already exists
     if [ -n "$keep" ]; then
-        local keep_dir=/var/cache/yunohost/files_to_keep_during_setup_source/${YNH_APP_ID}
         local stuff_to_keep
         for stuff_to_keep in $keep; do
             if [ -e "$keep_dir/$stuff_to_keep" ]; then
@@ -271,7 +283,8 @@ ynh_setup_source() {
             fi
         done
     fi
-    rm -rf /var/cache/yunohost/files_to_keep_during_setup_source/
+
+    rm -rf "$keep_dir"
 
     if [ -n "${install_dir:-}" ] && [ "$dest_dir" == "$install_dir" ]; then
         _ynh_apply_default_permissions "$dest_dir"

hooks/conf_regen/01-yunohost

@@ -153,6 +153,14 @@ base_folder_and_perm_init() {
     chown root:root /var/cache/yunohost
     chmod 700 /var/cache/yunohost
 
+    mkdir -p /var/tmp/yunohost
+    mkdir /var/tmp/yunohost/download
+    chown root:root /var/tmp/yunohost
+    chown root:root /var/tmp/yunohost/download
+    # 711 because we want $app-s to be able to access their assets folder (as non-root) during scripts
+    chmod 711 /var/tmp/yunohost
+    chmod 711 /var/tmp/yunohost/download
+
     [ ! -e /var/www/.well-known/ynh-diagnosis/ ] || chmod 775 /var/www/.well-known/ynh-diagnosis/
 
     if test -e /etc/yunohost/installed; then

locales/en.json

@@ -77,6 +77,7 @@
     "app_not_upgraded_broken_system": "The app '{failed_app}' failed to upgrade and put the system in a broken state, and as a consequence the following apps' upgrades have been cancelled: {apps}",
     "app_not_upgraded_broken_system_continue": "The app '{failed_app}' failed to upgrade and put the system in a broken state (so --continue-on-failure is ignored), and as a consequence the following apps' upgrades have been cancelled: {apps}",
     "app_packaging_format_not_supported": "This app cannot be installed because its packaging format is not supported by your YunoHost version. You should probably consider upgrading your system.",
+    "app_regenconf_failed": "(Re)Generating configurations for {app} failed: {error}",
     "app_remove_after_failed_install": "Removing the app after installation failure…",
     "app_removed": "{app} uninstalled",
     "app_requirements_checking": "Checking requirements for {app}…",
@@ -89,6 +90,7 @@
     "app_start_remove": "Removing {app}…",
     "app_start_restore": "Restoring {app}…",
     "app_unknown": "Unknown app",
+    "app_uninitialized_variables": "Some variables are not initialized: {vars}",
     "app_unsupported_remote_type": "Unsupported remote type used for the app",
     "app_upgrade_app_name": "Now upgrading {app}…",
     "app_upgrade_failed": "Could not upgrade {app}: {error}",

share/actionsmap.yml

@@ -894,6 +894,22 @@ app:
                     help: Also remove all application data
                     action: store_true
 
+        ### app_regenconf()
+        regen-conf:
+            action_help: Regen configurations related to an app
+            api: PUT /apps/<app>/regenconf
+            arguments:
+                app:
+                    help: App to regen configurations for
+                -d:
+                    full: --with-diff
+                    help: Show differences in case of configuration changes
+                    action: store_true
+                -n:
+                    full: --dry-run
+                    help: Show what would have been regenerated
+                    action: store_true
+
         ### app_upgrade()
         upgrade:
             action_help: Upgrade app