Put Python resources into a bundle inside a framework (#139)

* Put python resources into bundle

* echo "$SERIOUS_PYTHON_SITE_PACKAGES"

* Fix setting readme

* iOS packaging improved with bundles

* macOS to store site-packages in python.bundle, not app archive

* Improved structure and reliability

* Fix macOS test

* Fix .origin path

* site-packages-hash arg

* Windows: include site-packages to an app directory, not app.zip

* Fix Windows test

* Copy site-packages to app for Linux

* Fix Linux tests

* Install packages to site-packages for all platforms

* --skip-site-packages option

* Rename --cleanup-packages-files -> --cleanup-package-files

* --arch option to allow a list

* Version bumped to 0.9.0, changelog updated

* Try to fix Android test

Author: FeodorFitsner

.appveyor.yml

@@ -1,7 +1,7 @@
 skip_branch_with_pr: true
 
 environment:
-  FLUTTER_VERSION: 3.24.5
+  FLUTTER_VERSION: 3.27.4
   GITHUB_TOKEN:
     secure: 9SKIwc3VSfYJ5IChvNR74mEv2nb0ZFftUzn3sGRdXipXEfKSxY50DoodChHvlqZduQNhjg0oyLWAAa3n+iwWvVM2yI7Cgb14lFNClijz/kHI/PibnjDMNvLKaAygcfAc
 
@@ -59,6 +59,7 @@ for:
     build: off
 
     test_script:
+      - export SERIOUS_PYTHON_SITE_PACKAGES=$APPVEYOR_BUILD_FOLDER/site-packages
       - cd src/serious_python/example/flet_example
       - dart run serious_python:main package app/src -p Darwin -r flet,--pre
       - flutter test integration_test -d macos
@@ -149,6 +150,7 @@ for:
     build: off
 
     test_script:
+      - set SERIOUS_PYTHON_SITE_PACKAGES=%APPVEYOR_BUILD_FOLDER%\site-packages
       - cd src/serious_python/example/flet_example
       - dart run serious_python:main package app/src -p Windows -r flet,--pre
       - flutter test integration_test -d windows
@@ -169,6 +171,7 @@ for:
     build: off
 
     test_script:
+      - export SERIOUS_PYTHON_SITE_PACKAGES=$APPVEYOR_BUILD_FOLDER/site-packages
       - cd src/serious_python/example/flet_example
       - dart run serious_python:main package app/src -p Linux -r flet,--pre
       - xvfb-run flutter test integration_test -d linux
@@ -193,6 +196,7 @@ for:
     build: off
 
     test_script:
+      - export SERIOUS_PYTHON_SITE_PACKAGES=$APPVEYOR_BUILD_FOLDER/site-packages
       - cd src/serious_python/example/flet_example
       - dart run serious_python:main package app/src -p Linux -r flet,--pre
       - xvfb-run flutter test integration_test -d linux

src/serious_python/CHANGELOG.md

@@ -1,8 +1,14 @@
-## 0.8.8
+## 0.9.0
 
 * Set `MinimumOSVersion` to `13.0` for generated Python frameworks.
+* iOS and macOS packaging: Python system libraries are put into `python.bundle` to pass App Store verification.
+* On macOS, Windows, and Linux, application site packages are copied in an unpacked state to the application bundle to speed up the first launch.
 * Pyodide 0.27.2
 * Python 3.12.9
+* Packaging `--cleanup` option replaced with two separate `--cleanup-app` and `--cleanup-packages` options.
+* New packaging options `--cleanup-app-files` and `--cleanup-package-files` to specify a list of globs to exclude files and directories from app and site packages.
+* New packaging `--skip-site-packages` option to skip site packages installation for faster re-builds.
+* Packaging `--arch` option accepts a list now.
 
 ## 0.8.7
 

src/serious_python/bin/package_command.dart

@@ -20,7 +20,6 @@ const pyodideLockFile = "pyodide-lock.json";
 
 const buildPythonVersion = "3.12.9";
 const buildPythonReleaseDate = "20250205";
-const defaultSitePackagesDir = "__pypackages__";
 const sitePackagesEnvironmentVariable = "SERIOUS_PYTHON_SITE_PACKAGES";
 const flutterPackagesFlutterEnvironmentVariable =
     "SERIOUS_PYTHON_FLUTTER_PACKAGES";
@@ -100,9 +99,9 @@ class PackageCommand extends Command {
         allowed: ["iOS", "Android", "Pyodide", "Windows", "Linux", "Darwin"],
         mandatory: true,
         help: "Install dependencies for specific platform, e.g. 'Android'.");
-    argParser.addOption('arch',
+    argParser.addMultiOption('arch',
         help:
-            "Install dependencies for specific architecture only. Leave empty to install all supported architectures.");
+            "Install dependencies for specific architectures only. Leave empty to install all supported architectures.");
     argParser.addMultiOption('requirements',
         abbr: "r",
         help:
@@ -114,6 +113,8 @@ class PackageCommand extends Command {
     argParser.addMultiOption('exclude',
         help:
             "List of relative paths to exclude from app package, e.g. \"assets,build\".");
+    argParser.addFlag("skip-site-packages",
+        help: "Skip installation of site packages.", negatable: false);
     argParser.addFlag("compile-app",
         help: "Compile Python application before packaging.", negatable: false);
     argParser.addFlag("compile-packages",
@@ -131,7 +132,7 @@ class PackageCommand extends Command {
     argParser.addFlag("cleanup-packages",
         help: "Cleanup packages from unneccessary files and directories.",
         negatable: false);
-    argParser.addMultiOption('cleanup-packages-files',
+    argParser.addMultiOption('cleanup-package-files',
         help: "List of globs to delete extra packages files and directories.");
     argParser.addFlag("verbose", help: "Verbose output.", negatable: false);
   }
@@ -157,17 +158,18 @@ class PackageCommand extends Command {
       // args
       String? sourceDirPath = argResults!.rest.first;
       String platform = argResults?['platform'];
-      String? archArg = argResults?['arch'];
+      List<String> archArg = argResults?['arch'];
       List<String> requirements = argResults?['requirements'];
       String? assetPath = argResults?['asset'];
       List<String> exclude = argResults?['exclude'];
+      bool skipSitePackages = argResults?["skip-site-packages"];
       bool compileApp = argResults?["compile-app"];
       bool compilePackages = argResults?["compile-packages"];
       bool cleanup = argResults?["cleanup"];
       bool cleanupApp = argResults?["cleanup-app"];
       List<String> cleanupAppFiles = argResults?['cleanup-app-files'];
       bool cleanupPackages = argResults?["cleanup-packages"];
-      List<String> cleanupPackagesFiles = argResults?['cleanup-packages-files'];
+      List<String> cleanupPackageFiles = argResults?['cleanup-package-files'];
       _verbose = argResults?["verbose"];
 
       if (path.isRelative(sourceDirPath)) {
@@ -259,13 +261,29 @@ class PackageCommand extends Command {
       }
 
       // install requirements
-      if (requirements.isNotEmpty) {
+      if (requirements.isNotEmpty && !skipSitePackages) {
         String? sitePackagesRoot;
-        bool sitePackagesRootDeleted = false;
+
+        if (Platform.environment.containsKey(sitePackagesEnvironmentVariable)) {
+          sitePackagesRoot =
+              Platform.environment[sitePackagesEnvironmentVariable];
+        }
+        if (sitePackagesRoot == null || sitePackagesRoot.isEmpty) {
+          sitePackagesRoot = path.join(currentPath, "build", "site-packages");
+        }
+
+        if (await Directory(sitePackagesRoot).exists()) {
+          await for (var f in Directory(sitePackagesRoot)
+              .list()
+              .where((f) => !path.basename(f.path).startsWith("."))) {
+            await f.delete(recursive: true);
+          }
+        }
+
         bool flutterPackagesCopied = false;
         // invoke pip for every platform arch
         for (var arch in platforms[platform]!.entries) {
-          if (archArg != null && arch.key != archArg) {
+          if (archArg.isNotEmpty && !archArg.contains(arch.key)) {
             continue;
           }
           String? sitePackagesDir;
@@ -300,28 +318,6 @@ class PackageCommand extends Command {
                   [sitecustomizeDir.path].join(Platform.isWindows ? ";" : ":"),
             };
 
-            if (isMobile) {
-              if (!Platform.environment
-                  .containsKey(sitePackagesEnvironmentVariable)) {
-                throw "Environment variable is not set: $sitePackagesEnvironmentVariable";
-              }
-              sitePackagesRoot =
-                  Platform.environment[sitePackagesEnvironmentVariable];
-              if (sitePackagesRoot!.isEmpty) {
-                throw "Environment variable cannot be empty: $sitePackagesEnvironmentVariable";
-              }
-            } else {
-              sitePackagesRoot =
-                  path.join(tempDir.path, defaultSitePackagesDir);
-            }
-
-            if (!sitePackagesRootDeleted) {
-              if (await Directory(sitePackagesRoot).exists()) {
-                await Directory(sitePackagesRoot).delete(recursive: true);
-              }
-              sitePackagesRootDeleted = true;
-            }
-
             sitePackagesDir = arch.key.isNotEmpty
                 ? path.join(sitePackagesRoot, arch.key)
                 : sitePackagesRoot;
@@ -394,7 +390,7 @@ class PackageCommand extends Command {
 
             // cleanup packages
             if (cleanupPackages || cleanup) {
-              var allJunkFiles = [...junkFiles, ...cleanupPackagesFiles];
+              var allJunkFiles = [...junkFiles, ...cleanupPackageFiles];
               if (_verbose) {
                 verbose(
                     "Delete unnecessary package files and directories: $allJunkFiles");
@@ -414,11 +410,18 @@ class PackageCommand extends Command {
 
         if (platform == "Darwin") {
           await macos_utils.mergeMacOsSitePackages(
-              path.join(sitePackagesRoot!, "arm64"),
+              path.join(sitePackagesRoot, "arm64"),
               path.join(sitePackagesRoot, "x86_64"),
               path.join(sitePackagesRoot),
               _verbose);
         }
+
+        // synchronize pod
+        var syncSh =
+            File(path.join(sitePackagesRoot, ".pod", "sync_site_packages.sh"));
+        if (await syncSh.exists()) {
+          await runExec("/bin/sh", [syncSh.path]);
+        }
       }
 
       // create archive