From 120940a8efba6045ce39a0939c4c049209d239d6 Mon Sep 17 00:00:00 2001 From: Hari Rana Date: Tue, 31 Dec 2024 13:18:03 -0500 Subject: [PATCH 1/8] flatpak: Improve updating process This adds req2flatpak as a submodule for easy access to generating PyPi dependencies. --- .gitmodules | 3 + build-aux/com.usebottles.bottles.Devel.json | 2 +- build-aux/flatpak-pip-generator.py | 523 ------------------ ....bottles.pypi-deps.yaml => pypi-deps.yaml} | 2 +- build-aux/req2flatpak | 1 + 5 files changed, 6 insertions(+), 525 deletions(-) create mode 100644 .gitmodules delete mode 100644 build-aux/flatpak-pip-generator.py rename build-aux/{com.usebottles.bottles.pypi-deps.yaml => pypi-deps.yaml} (98%) create mode 160000 build-aux/req2flatpak diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000000..8dd3cb3803 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "build-aux/req2flatpak"] + path = build-aux/req2flatpak + url = https://github.com/johannesjh/req2flatpak.git diff --git a/build-aux/com.usebottles.bottles.Devel.json b/build-aux/com.usebottles.bottles.Devel.json index 326c23d365..8fcf441952 100644 --- a/build-aux/com.usebottles.bottles.Devel.json +++ b/build-aux/com.usebottles.bottles.Devel.json @@ -80,7 +80,7 @@ "mkdir -p /app/share/vulkan/implicit_layer.d/" ], "modules": [ - "com.usebottles.bottles.pypi-deps.yaml", + "pypi-deps.yaml", { "name": "vmtouch", "buildsystem": "simple", diff --git a/build-aux/flatpak-pip-generator.py b/build-aux/flatpak-pip-generator.py deleted file mode 100644 index e3ff138572..0000000000 --- a/build-aux/flatpak-pip-generator.py +++ /dev/null @@ -1,523 +0,0 @@ -#!/usr/bin/env python3 -# original: -# https://github.com/flatpak/flatpak-builder-tools/blob/67455d214028f1562edf4fa4bcef8ba9d2617d23/pip/flatpak-pip-generator -# but modified with: -# - will use arch-dependent whl directly -# - will still install packages if they already exist in org.freedesktop.Sdk - -__license__ = "MIT" - -import argparse -import hashlib -import json -import os -import shutil -import subprocess -import sys -import tempfile -import urllib.request -from collections import OrderedDict - -try: - import requirements -except ImportError: - exit('Requirements modules is not installed. Run "pip install requirements-parser"') - -parser = argparse.ArgumentParser() -parser.add_argument("packages", nargs="*") -parser.add_argument( - "--python2", action="store_true", help="Look for a Python 2 package" -) -parser.add_argument( - "--cleanup", choices=["scripts", "all"], help="Select what to clean up after build" -) -parser.add_argument("--requirements-file", "-r", help="Specify requirements.txt file") -parser.add_argument( - "--build-only", - action="store_const", - dest="cleanup", - const="all", - help="Clean up all files after build", -) -parser.add_argument( - "--build-isolation", - action="store_true", - default=False, - help=( - "Do not disable build isolation. " - "Mostly useful on pip that does't " - "support the feature." - ), -) -parser.add_argument( - "--ignore-installed", - type=lambda s: s.split(","), - default="", - help="Comma-separated list of package names for which pip " - "should ignore already installed packages. Useful when " - "the package is installed in the SDK but not in the " - "runtime.", -) -parser.add_argument( - "--checker-data", - action="store_true", - help='Include x-checker-data in output for the "Flatpak External Data Checker"', -) -parser.add_argument("--output", "-o", help="Specify output file name") -parser.add_argument( - "--runtime", - help="Specify a flatpak to run pip inside of a sandbox, ensures python version compatibility", -) -parser.add_argument( - "--yaml", action="store_true", help="Use YAML as output format instead of JSON" -) -opts = parser.parse_args() - -if opts.yaml: - try: - import yaml - except ImportError: - exit('PyYAML modules is not installed. Run "pip install PyYAML"') - - -def get_pypi_url(name: str, filename: str) -> str: - url = "https://pypi.org/pypi/{}/json".format(name) - print("Extracting download url for", name) - with urllib.request.urlopen(url) as response: - body = json.loads(response.read().decode("utf-8")) - for release in body["releases"].values(): - for source in release: - if source["filename"] == filename: - return source["url"] - raise Exception("Failed to extract url from {}".format(url)) - - -def get_tar_package_url_pypi(name: str, version: str) -> str: - url = "https://pypi.org/pypi/{}/{}/json".format(name, version) - with urllib.request.urlopen(url) as response: - body = json.loads(response.read().decode("utf-8")) - for ext in ["bz2", "gz", "xz", "zip"]: - for source in body["urls"]: - if source["url"].endswith(ext): - return source["url"] - err = "Failed to get {}-{} source from {}".format(name, version, url) - raise Exception(err) - - -def get_package_name(filename: str) -> str: - if filename.endswith(("bz2", "gz", "xz", "zip")): - segments = filename.split("-") - if len(segments) == 2: - return segments[0] - return "-".join(segments[: len(segments) - 1]) - elif filename.endswith("whl"): - segments = filename.split("-") - if len(segments) == 5: - return segments[0] - candidate = segments[: len(segments) - 4] - # Some packages list the version number twice - # e.g. PyQt5-5.15.0-5.15.0-cp35.cp36.cp37.cp38-abi3-manylinux2014_x86_64.whl - if candidate[-1] == segments[len(segments) - 4]: - return "-".join(candidate[:-1]) - return "-".join(candidate) - else: - raise Exception( - "Downloaded filename: {} does not end with bz2, gz, xz, zip, or whl".format( - filename - ) - ) - - -def get_file_version(filename: str) -> str: - name = get_package_name(filename) - segments = filename.split(name + "-") - version = segments[1].split("-")[0] - for ext in ["tar.gz", "whl", "tar.xz", "tar.gz", "tar.bz2", "zip"]: - version = version.replace("." + ext, "") - return version - - -def get_file_hash(filename: str) -> str: - sha = hashlib.sha256() - print("Generating hash for", filename.split("/")[-1]) - with open(filename, "rb") as f: - while True: - data = f.read(1024 * 1024 * 32) - if not data: - break - sha.update(data) - return sha.hexdigest() - - -def download_tar_pypi(url: str, tempdir: str) -> None: - with urllib.request.urlopen(url) as response: - file_path = os.path.join(tempdir, url.split("/")[-1]) - with open(file_path, "x+b") as tar_file: - shutil.copyfileobj(response, tar_file) - - -def parse_continuation_lines(fin): - for line in fin: - line = line.rstrip("\n") - while line.endswith("\\"): - try: - line = line[:-1] + next(fin).rstrip("\n") - except StopIteration: - exit( - 'Requirements have a wrong number of line continuation characters "\\"' - ) - yield line - - -def fprint(string: str) -> None: - separator = "=" * 72 # Same as `flatpak-builder` - print(separator) - print(string) - print(separator) - - -packages = [] -if opts.requirements_file: - requirements_file = os.path.expanduser(opts.requirements_file) - try: - with open(requirements_file, "r") as req_file: - reqs = parse_continuation_lines(req_file) - reqs_as_str = "\n".join([r.split("--hash")[0] for r in reqs]) - packages = list(requirements.parse(reqs_as_str)) - except FileNotFoundError: - pass - -elif opts.packages: - packages = list(requirements.parse("\n".join(opts.packages))) - with tempfile.NamedTemporaryFile( - "w", delete=False, prefix="requirements." - ) as req_file: - req_file.write("\n".join(opts.packages)) - requirements_file = req_file.name -else: - exit("Please specifiy either packages or requirements file argument") - -for i in packages: - if i["name"].lower().startswith("pyqt"): - print("PyQt packages are not supported by flapak-pip-generator") - print("However, there is a BaseApp for PyQt available, that you should use") - print( - "Visit https://github.com/flathub/com.riverbankcomputing.PyQt.BaseApp for more information" - ) - sys.exit(0) - -with open(requirements_file, "r") as req_file: - use_hash = "--hash=" in req_file.read() - -python_version = "2" if opts.python2 else "3" -if opts.python2: - pip_executable = "pip2" -else: - pip_executable = "pip3" - -if opts.runtime: - flatpak_cmd = [ - "flatpak", - "--devel", - "--share=network", - "--filesystem=/tmp", - "--command={}".format(pip_executable), - "run", - opts.runtime, - ] - if opts.requirements_file: - requirements_file = os.path.expanduser(opts.requirements_file) - if os.path.exists(requirements_file): - prefix = os.path.realpath(requirements_file) - flag = "--filesystem={}".format(prefix) - flatpak_cmd.insert(1, flag) -else: - flatpak_cmd = [pip_executable] - -if opts.output: - output_package = opts.output -elif opts.requirements_file: - output_package = "python{}-{}".format( - python_version, - os.path.basename(opts.requirements_file).replace(".txt", ""), - ) -elif len(packages) == 1: - output_package = "python{}-{}".format( - python_version, - packages[0].name, - ) -else: - output_package = "python{}-modules".format(python_version) -if opts.yaml: - output_filename = output_package + ".yaml" -else: - output_filename = output_package + ".json" - -modules = [] -vcs_modules = [] -sources = {} - -tempdir_prefix = "pip-generator-{}".format(os.path.basename(output_package)) -with tempfile.TemporaryDirectory(prefix=tempdir_prefix) as tempdir: - pip_download = flatpak_cmd + [ - "download", - "--exists-action=i", - "--dest", - tempdir, - "-r", - requirements_file, - ] - if use_hash: - pip_download.append("--require-hashes") - - fprint("Downloading sources") - cmd = " ".join(pip_download) - print('Running: "{}"'.format(cmd)) - try: - subprocess.run(pip_download, check=True) - except subprocess.CalledProcessError: - print("Failed to download") - print("Please fix the module manually in the generated file") - - if not opts.requirements_file: - try: - os.remove(requirements_file) - except FileNotFoundError: - pass - - fprint("Downloading arch independent packages") - for filename in os.listdir(tempdir): - if not filename.endswith( - ("bz2", "whl", "gz", "xz", "zip") - ): # modified 'any.whl' to 'whl' - version = get_file_version(filename) - name = get_package_name(filename) - url = get_tar_package_url_pypi(name, version) - print("Deleting", filename) - try: - os.remove(os.path.join(tempdir, filename)) - except FileNotFoundError: - pass - print("Downloading {}".format(url)) - download_tar_pypi(url, tempdir) - - files = {get_package_name(f): [] for f in os.listdir(tempdir)} - - for filename in os.listdir(tempdir): - name = get_package_name(filename) - files[name].append(filename) - - # Delete redundant sources, for vcs sources - for name in files: - if len(files[name]) > 1: - zip_source = False - for f in files[name]: - if f.endswith(".zip"): - zip_source = True - if zip_source: - for f in files[name]: - if not f.endswith(".zip"): - try: - os.remove(os.path.join(tempdir, f)) - except FileNotFoundError: - pass - - vcs_packages = { - x.name: {"vcs": x.vcs, "revision": x.revision, "uri": x.uri} - for x in packages - if x.vcs - } - - fprint("Obtaining hashes and urls") - for filename in os.listdir(tempdir): - name = get_package_name(filename) - sha256 = get_file_hash(os.path.join(tempdir, filename)) - - if name in vcs_packages: - uri = vcs_packages[name]["uri"] - revision = vcs_packages[name]["revision"] - vcs = vcs_packages[name]["vcs"] - url = "https://" + uri.split("://", 1)[1] - s = "commit" - if vcs == "svn": - s = "revision" - source = OrderedDict( - [ - ("type", vcs), - ("url", url), - (s, revision), - ] - ) - is_vcs = True - else: - url = get_pypi_url(name, filename) - source = OrderedDict([("type", "file"), ("url", url), ("sha256", sha256)]) - if opts.checker_data: - source["x-checker-data"] = {"type": "pypi", "name": name} - if url.endswith(".whl"): - source["x-checker-data"]["packagetype"] = "bdist_wheel" - is_vcs = False - sources[name] = {"source": source, "vcs": is_vcs} - -# Python3 packages that come as part of org.freedesktop.Sdk. -system_packages = [ - "cython", - "easy_install", - "mako", - "markdown", - "meson", - "pip", - "pygments", - "setuptools", - "six", - "wheel", -] - -fprint("Generating dependencies") -for package in packages: - if package.name is None: - print( - "Warning: skipping invalid requirement specification {} because it is missing a name".format( - package.line - ), - file=sys.stderr, - ) - print( - "Append #egg= to the end of the requirement line to fix", - file=sys.stderr, - ) - continue - elif package.name.casefold() in system_packages: - # modified - print(f"{package.name} is in system_packages. Proceed anyway.") - - if len(package.extras) > 0: - extras = "[" + ",".join(extra for extra in package.extras) + "]" - else: - extras = "" - - version_list = [x[0] + x[1] for x in package.specs] - version = ",".join(version_list) - - if package.vcs: - revision = "" - if package.revision: - revision = "@" + package.revision - pkg = package.uri + revision + "#egg=" + package.name - else: - pkg = package.name + extras + version - - dependencies = [] - # Downloads the package again to list dependencies - - tempdir_prefix = "pip-generator-{}".format(package.name) - with tempfile.TemporaryDirectory( - prefix="{}-{}".format(tempdir_prefix, package.name) - ) as tempdir: - pip_download = flatpak_cmd + [ - "download", - "--exists-action=i", - "--dest", - tempdir, - ] - try: - print("Generating dependencies for {}".format(package.name)) - subprocess.run(pip_download + [pkg], check=True, stdout=subprocess.DEVNULL) - for filename in sorted(os.listdir(tempdir)): - dep_name = get_package_name(filename) - if dep_name.casefold() in system_packages: - pass # modified - dependencies.append(dep_name) - - except subprocess.CalledProcessError: - print("Failed to download {}".format(package.name)) - - is_vcs = True if package.vcs else False - package_sources = [] - for dependency in dependencies: - if dependency in sources: - source = sources[dependency] - elif dependency.replace("_", "-") in sources: - source = sources[dependency.replace("_", "-")] - else: - continue - - if not (not source["vcs"] or is_vcs): - continue - - package_sources.append(source["source"]) - - if package.vcs: - name_for_pip = "." - else: - name_for_pip = pkg - - module_name = "python{}-{}".format(python_version, package.name) - - pip_command = [ - pip_executable, - "install", - "--verbose", - "--exists-action=i", - "--no-index", - '--find-links="file://${PWD}"', - "--prefix=${FLATPAK_DEST}", - '"{}"'.format(name_for_pip), - ] - if package.name in opts.ignore_installed: - pip_command.append("--ignore-installed") - if package.name.casefold() in system_packages: # modified - print(f"{package.name} is in system_packages, adding --ignore-installed") - pip_command.append("--ignore-installed") - if not opts.build_isolation: - pip_command.append("--no-build-isolation") - - module = OrderedDict( - [ - ("name", module_name), - ("buildsystem", "simple"), - ("build-commands", [" ".join(pip_command)]), - ("sources", package_sources), - ] - ) - if opts.cleanup == "all": - module["cleanup"] = ["*"] - elif opts.cleanup == "scripts": - module["cleanup"] = ["/bin", "/share/man/man1"] - - if package.vcs: - vcs_modules.append(module) - else: - modules.append(module) - -modules = vcs_modules + modules -if len(modules) == 1: - pypi_module = modules[0] -else: - pypi_module = { - "name": output_package, - "buildsystem": "simple", - "build-commands": [], - "modules": modules, - } - -print() -with open(output_filename, "w") as output: - if opts.yaml: - - class OrderedDumper(yaml.Dumper): - def increase_indent(self, flow=False, indentless=False): - return super(OrderedDumper, self).increase_indent(flow, False) - - def dict_representer(dumper, data): - return dumper.represent_dict(data.items()) - - OrderedDumper.add_representer(OrderedDict, dict_representer) - - output.write( - "# Generated with flatpak-pip-generator " + " ".join(sys.argv[1:]) + "\n" - ) - yaml.dump(pypi_module, output, Dumper=OrderedDumper) - else: - output.write(json.dumps(pypi_module, indent=4)) - print("Output saved to {}".format(output_filename)) diff --git a/build-aux/com.usebottles.bottles.pypi-deps.yaml b/build-aux/pypi-deps.yaml similarity index 98% rename from build-aux/com.usebottles.bottles.pypi-deps.yaml rename to build-aux/pypi-deps.yaml index 442eeae28d..851de3245c 100644 --- a/build-aux/com.usebottles.bottles.pypi-deps.yaml +++ b/build-aux/pypi-deps.yaml @@ -1,4 +1,4 @@ -# Generated by req2flatpak.py --requirements-file requirements.txt --yaml --target-platforms 312-x86_64 -o com.usebottles.bottles.pypi-deps.yaml +# Generated by req2flatpak.py --requirements-file requirements.txt --yaml --target-platforms 312-x86_64 -o build-aux/pypi-deps.yaml name: python3-package-installation buildsystem: simple build-commands: diff --git a/build-aux/req2flatpak b/build-aux/req2flatpak new file mode 160000 index 0000000000..2c715eb5cb --- /dev/null +++ b/build-aux/req2flatpak @@ -0,0 +1 @@ +Subproject commit 2c715eb5cb493f606500f2b9c17c0c3675a36454 From 2ebb31e3a4d371e1a76f8df6bc6ac26ca519ad4e Mon Sep 17 00:00:00 2001 From: EmoonX Date: Tue, 31 Dec 2024 18:03:39 -0300 Subject: [PATCH 2/8] midi: register instruments in increments of 16 libfluidsynth creates 16 banks by default for each soundfont (all repeated). So 0~15 are all copies of the 1st soundfont, 16~31 copies of the 2nd, and so on --- bottles/backend/utils/midi.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/bottles/backend/utils/midi.py b/bottles/backend/utils/midi.py index 86b9c5a269..71fef38478 100644 --- a/bottles/backend/utils/midi.py +++ b/bottles/backend/utils/midi.py @@ -36,9 +36,11 @@ def __init__(self, soundfont_path: str): @classmethod def __get_vacant_id(cls) -> int: - """Get smallest 0-indexed ID currently not being used by a SoundFont.""" - n = len(cls.__active_instances) - return next(i for i in range(n + 1) if i not in cls.__active_instances) + """Get smallest 16-step ID (0, 16, 32...) not currently in use by a SoundFont.""" + i = 0 + while i in cls.__active_instances: + i += 16 + return i def __run_server(self): """Create Synth object and start server with loaded SoundFont.""" From b89291083e1dd0b1b9dedf068d78780cb3c82d14 Mon Sep 17 00:00:00 2001 From: EmoonX Date: Tue, 31 Dec 2024 18:07:33 -0300 Subject: [PATCH 3/8] chore: update `pypi-deps.yaml` --- build-aux/pypi-deps.yaml | 35 ++++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/build-aux/pypi-deps.yaml b/build-aux/pypi-deps.yaml index 851de3245c..055b61a6f2 100644 --- a/build-aux/pypi-deps.yaml +++ b/build-aux/pypi-deps.yaml @@ -5,7 +5,7 @@ build-commands: - pip3 install --verbose --exists-action=i --no-index --find-links="file://${PWD}" --prefix=${FLATPAK_DEST} --no-build-isolation wheel PyYAML pycurl chardet requests Markdown icoextract patool pathvalidate FVS orjson pycairo PyGObject charset-normalizer - pyfluidsynth idna urllib3 certifi pefile + numpy pyfluidsynth idna urllib3 certifi pefile sources: - type: file url: https://files.pythonhosted.org/packages/f7/2f/cc09899755f94b36e7f570b9f9ca19a5fdff536e2614fd3ac1c28bb777f6/FVS-0.3.4.tar.gz @@ -22,14 +22,14 @@ sources: only-arches: - x86_64 - type: file - url: https://files.pythonhosted.org/packages/12/90/3c9ff0512038035f59d279fddeb79f5f1eccd8859f06d6163c58798b9487/certifi-2024.8.30-py3-none-any.whl - sha256: 922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8 + url: https://files.pythonhosted.org/packages/a5/32/8f6669fc4798494966bf446c8c4a162e0b5d893dff088afddf76414f70e1/certifi-2024.12.14-py3-none-any.whl + sha256: 1275f7a45be9464efc1173084eaa30f866fe2e47d389406136d332ed4967ec56 - type: file url: https://files.pythonhosted.org/packages/38/6f/f5fbc992a329ee4e0f288c1fe0e2ad9485ed064cac731ed2fe47dcc38cbf/chardet-5.2.0-py3-none-any.whl sha256: e1cf59446890a00105fe7b7912492ea04b6e6f06d4b742b2c788469e34c82970 - type: file - url: https://files.pythonhosted.org/packages/ee/fb/14d30eb4956408ee3ae09ad34299131fb383c47df355ddb428a7331cfa1e/charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - sha256: 90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b + url: https://files.pythonhosted.org/packages/3e/a2/513f6cbe752421f16d969e32f3583762bfd583848b763913ddab8d9bfd4f/charset_normalizer-3.4.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + sha256: bc2722592d8998c870fa4e290c2eec2c1569b87fe58618e67d38b4665dfa680d only-arches: - x86_64 - type: file @@ -39,16 +39,21 @@ sources: url: https://files.pythonhosted.org/packages/76/c6/c88e154df9c4e1a2a66ccf0005a88dfb2650c1dffb6f5ce603dfbd452ce3/idna-3.10-py3-none-any.whl sha256: 946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3 - type: file - url: https://files.pythonhosted.org/packages/a0/6b/34e6904ac99df811a06e42d8461d47b6e0c9b86e2fe7ee84934df6e35f0d/orjson-3.10.7-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - sha256: a0c6a008e91d10a2564edbb6ee5069a9e66df3fbe11c9a005cb411f441fd2c09 + url: https://files.pythonhosted.org/packages/7f/a7/c1f1d978166eb6b98ad009503e4d93a8c1962d0eb14a885c352ee0276a54/numpy-2.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + sha256: 5062dc1a4e32a10dc2b8b13cedd58988261416e811c1dc4dbdea4f57eea61b0d + only-arches: + - x86_64 +- type: file + url: https://files.pythonhosted.org/packages/48/90/e583d6e29937ec30a164f1d86a0439c1a2477b5aae9f55d94b37a4f5b5f0/orjson-3.10.13-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + sha256: 064b9dbb0217fd64a8d016a8929f2fae6f3312d55ab3036b00b1d17399ab2f3e only-arches: - x86_64 - type: file url: https://files.pythonhosted.org/packages/d3/5e/76a9d08b4b4e4583f269cb9f64de267f9aeae0dacef23307f53a14211716/pathvalidate-3.2.1-py3-none-any.whl sha256: 9a6255eb8f63c9e2135b9be97a5ce08f10230128c4ae7b3e935378b82b22c4c9 - type: file - url: https://files.pythonhosted.org/packages/0e/44/192ede8c7f935643e4c8a56545fcac6ae1b8c50a77f54b2b1c4ab9fcae49/patool-3.0.0-py2.py3-none-any.whl - sha256: 928070d5f82a776534a290a52f4758e2c0dd9cd5a633e3f63f7270c8982833b8 + url: https://files.pythonhosted.org/packages/a3/68/c1a6597c901b1f750d2fcf562181c18c0d6c284908e4df57f1029d8b8887/patool-3.1.0-py2.py3-none-any.whl + sha256: 401a918bdbf65434fd59c038bdb2c15ff7185675aedddb4494330c3e8e4fe80d - type: file url: https://files.pythonhosted.org/packages/54/16/12b82f791c7f50ddec566873d5bdd245baa1491bac11d15ffb98aecc8f8b/pefile-2024.8.26-py3-none-any.whl sha256: 76f8b485dcd3b1bb8166f1128d395fa3d87af26360c2358fb75b80019b957c6f @@ -56,8 +61,8 @@ sources: url: https://files.pythonhosted.org/packages/07/4a/42b26390181a7517718600fa7d98b951da20be982a50cd4afb3d46c2e603/pycairo-1.27.0.tar.gz sha256: 5cb21e7a00a2afcafea7f14390235be33497a2cce53a98a19389492a60628430 - type: file - url: https://files.pythonhosted.org/packages/65/80/8791945007e2295806bfd0e982e00fee023517b17d5b2d845ca64c81878c/pycurl-7.45.3-cp312-cp312-manylinux_2_28_x86_64.whl - sha256: 3d07c5daef2d0d85949e32ec254ee44232bb57febb0634194379dd14d1ff4f87 + url: https://files.pythonhosted.org/packages/2c/4c/07e7192f0d7fc549dab2784c6448ffa98412acb942a365adce4e14d1a143/pycurl-7.45.4-cp312-cp312-manylinux_2_28_x86_64.whl + sha256: 688d09ba2c6a0d4a749d192c43422839d73c40c85143c50cc65c944258fe0ba8 only-arches: - x86_64 - type: file @@ -67,8 +72,8 @@ sources: url: https://files.pythonhosted.org/packages/f9/9b/335f9764261e915ed497fcdeb11df5dfd6f7bf257d4a6a2a686d80da4d54/requests-2.32.3-py3-none-any.whl sha256: 70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6 - type: file - url: https://files.pythonhosted.org/packages/ce/d9/5f4c13cecde62396b0d3fe530a50ccea91e7dfc1ccf0e09c228841bb5ba8/urllib3-2.2.3-py3-none-any.whl - sha256: ca899ca043dcb1bafa3e262d73aa25c465bfb49e0bd9dd5d59f1d0acba2f8fac + url: https://files.pythonhosted.org/packages/c8/19/4ec628951a74043532ca2cf5d97b7b14863931476d117c471e8e2b1eb39f/urllib3-2.3.0-py3-none-any.whl + sha256: 1cee9ad369867bfdbbb48b7dd50374c0967a0bb7710050facf0dd6911440e3df - type: file - url: https://files.pythonhosted.org/packages/1b/d1/9babe2ccaecff775992753d8686970b1e2755d21c8a63be73aba7a4e7d77/wheel-0.44.0-py3-none-any.whl - sha256: 2376a90c98cc337d18623527a97c31797bd02bad0033d41547043a1cbfbe448f + url: https://files.pythonhosted.org/packages/0b/2c/87f3254fd8ffd29e4c02732eee68a83a1d3c346ae39bc6822dcbcb697f2b/wheel-0.45.1-py3-none-any.whl + sha256: 708e7481cc80179af0e556bbf0cc00b8444c7321e2700b8d8580231d13017248 From d4421518abe34d198ea64539e55af0c026d79e9c Mon Sep 17 00:00:00 2001 From: EmoonX Date: Tue, 31 Dec 2024 14:08:56 -0300 Subject: [PATCH 4/8] chore: add `ignore` pragma fallback for `import fluidsynth` --- bottles/backend/utils/midi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bottles/backend/utils/midi.py b/bottles/backend/utils/midi.py index 71fef38478..42de3952b4 100644 --- a/bottles/backend/utils/midi.py +++ b/bottles/backend/utils/midi.py @@ -1,4 +1,4 @@ -import fluidsynth +import fluidsynth # type: ignore[import-not-found] from bottles.backend.logger import Logger from bottles.backend.models.config import BottleConfig From ac69e91c752a4692ce4afcb64085e597f5b85d25 Mon Sep 17 00:00:00 2001 From: EmoonX Date: Tue, 31 Dec 2024 18:07:33 -0300 Subject: [PATCH 5/8] chore: update `pypi-deps.yaml` --- build-aux/pypi-deps.yaml | 35 ++++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/build-aux/pypi-deps.yaml b/build-aux/pypi-deps.yaml index 851de3245c..055b61a6f2 100644 --- a/build-aux/pypi-deps.yaml +++ b/build-aux/pypi-deps.yaml @@ -5,7 +5,7 @@ build-commands: - pip3 install --verbose --exists-action=i --no-index --find-links="file://${PWD}" --prefix=${FLATPAK_DEST} --no-build-isolation wheel PyYAML pycurl chardet requests Markdown icoextract patool pathvalidate FVS orjson pycairo PyGObject charset-normalizer - pyfluidsynth idna urllib3 certifi pefile + numpy pyfluidsynth idna urllib3 certifi pefile sources: - type: file url: https://files.pythonhosted.org/packages/f7/2f/cc09899755f94b36e7f570b9f9ca19a5fdff536e2614fd3ac1c28bb777f6/FVS-0.3.4.tar.gz @@ -22,14 +22,14 @@ sources: only-arches: - x86_64 - type: file - url: https://files.pythonhosted.org/packages/12/90/3c9ff0512038035f59d279fddeb79f5f1eccd8859f06d6163c58798b9487/certifi-2024.8.30-py3-none-any.whl - sha256: 922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8 + url: https://files.pythonhosted.org/packages/a5/32/8f6669fc4798494966bf446c8c4a162e0b5d893dff088afddf76414f70e1/certifi-2024.12.14-py3-none-any.whl + sha256: 1275f7a45be9464efc1173084eaa30f866fe2e47d389406136d332ed4967ec56 - type: file url: https://files.pythonhosted.org/packages/38/6f/f5fbc992a329ee4e0f288c1fe0e2ad9485ed064cac731ed2fe47dcc38cbf/chardet-5.2.0-py3-none-any.whl sha256: e1cf59446890a00105fe7b7912492ea04b6e6f06d4b742b2c788469e34c82970 - type: file - url: https://files.pythonhosted.org/packages/ee/fb/14d30eb4956408ee3ae09ad34299131fb383c47df355ddb428a7331cfa1e/charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - sha256: 90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b + url: https://files.pythonhosted.org/packages/3e/a2/513f6cbe752421f16d969e32f3583762bfd583848b763913ddab8d9bfd4f/charset_normalizer-3.4.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + sha256: bc2722592d8998c870fa4e290c2eec2c1569b87fe58618e67d38b4665dfa680d only-arches: - x86_64 - type: file @@ -39,16 +39,21 @@ sources: url: https://files.pythonhosted.org/packages/76/c6/c88e154df9c4e1a2a66ccf0005a88dfb2650c1dffb6f5ce603dfbd452ce3/idna-3.10-py3-none-any.whl sha256: 946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3 - type: file - url: https://files.pythonhosted.org/packages/a0/6b/34e6904ac99df811a06e42d8461d47b6e0c9b86e2fe7ee84934df6e35f0d/orjson-3.10.7-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - sha256: a0c6a008e91d10a2564edbb6ee5069a9e66df3fbe11c9a005cb411f441fd2c09 + url: https://files.pythonhosted.org/packages/7f/a7/c1f1d978166eb6b98ad009503e4d93a8c1962d0eb14a885c352ee0276a54/numpy-2.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + sha256: 5062dc1a4e32a10dc2b8b13cedd58988261416e811c1dc4dbdea4f57eea61b0d + only-arches: + - x86_64 +- type: file + url: https://files.pythonhosted.org/packages/48/90/e583d6e29937ec30a164f1d86a0439c1a2477b5aae9f55d94b37a4f5b5f0/orjson-3.10.13-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + sha256: 064b9dbb0217fd64a8d016a8929f2fae6f3312d55ab3036b00b1d17399ab2f3e only-arches: - x86_64 - type: file url: https://files.pythonhosted.org/packages/d3/5e/76a9d08b4b4e4583f269cb9f64de267f9aeae0dacef23307f53a14211716/pathvalidate-3.2.1-py3-none-any.whl sha256: 9a6255eb8f63c9e2135b9be97a5ce08f10230128c4ae7b3e935378b82b22c4c9 - type: file - url: https://files.pythonhosted.org/packages/0e/44/192ede8c7f935643e4c8a56545fcac6ae1b8c50a77f54b2b1c4ab9fcae49/patool-3.0.0-py2.py3-none-any.whl - sha256: 928070d5f82a776534a290a52f4758e2c0dd9cd5a633e3f63f7270c8982833b8 + url: https://files.pythonhosted.org/packages/a3/68/c1a6597c901b1f750d2fcf562181c18c0d6c284908e4df57f1029d8b8887/patool-3.1.0-py2.py3-none-any.whl + sha256: 401a918bdbf65434fd59c038bdb2c15ff7185675aedddb4494330c3e8e4fe80d - type: file url: https://files.pythonhosted.org/packages/54/16/12b82f791c7f50ddec566873d5bdd245baa1491bac11d15ffb98aecc8f8b/pefile-2024.8.26-py3-none-any.whl sha256: 76f8b485dcd3b1bb8166f1128d395fa3d87af26360c2358fb75b80019b957c6f @@ -56,8 +61,8 @@ sources: url: https://files.pythonhosted.org/packages/07/4a/42b26390181a7517718600fa7d98b951da20be982a50cd4afb3d46c2e603/pycairo-1.27.0.tar.gz sha256: 5cb21e7a00a2afcafea7f14390235be33497a2cce53a98a19389492a60628430 - type: file - url: https://files.pythonhosted.org/packages/65/80/8791945007e2295806bfd0e982e00fee023517b17d5b2d845ca64c81878c/pycurl-7.45.3-cp312-cp312-manylinux_2_28_x86_64.whl - sha256: 3d07c5daef2d0d85949e32ec254ee44232bb57febb0634194379dd14d1ff4f87 + url: https://files.pythonhosted.org/packages/2c/4c/07e7192f0d7fc549dab2784c6448ffa98412acb942a365adce4e14d1a143/pycurl-7.45.4-cp312-cp312-manylinux_2_28_x86_64.whl + sha256: 688d09ba2c6a0d4a749d192c43422839d73c40c85143c50cc65c944258fe0ba8 only-arches: - x86_64 - type: file @@ -67,8 +72,8 @@ sources: url: https://files.pythonhosted.org/packages/f9/9b/335f9764261e915ed497fcdeb11df5dfd6f7bf257d4a6a2a686d80da4d54/requests-2.32.3-py3-none-any.whl sha256: 70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6 - type: file - url: https://files.pythonhosted.org/packages/ce/d9/5f4c13cecde62396b0d3fe530a50ccea91e7dfc1ccf0e09c228841bb5ba8/urllib3-2.2.3-py3-none-any.whl - sha256: ca899ca043dcb1bafa3e262d73aa25c465bfb49e0bd9dd5d59f1d0acba2f8fac + url: https://files.pythonhosted.org/packages/c8/19/4ec628951a74043532ca2cf5d97b7b14863931476d117c471e8e2b1eb39f/urllib3-2.3.0-py3-none-any.whl + sha256: 1cee9ad369867bfdbbb48b7dd50374c0967a0bb7710050facf0dd6911440e3df - type: file - url: https://files.pythonhosted.org/packages/1b/d1/9babe2ccaecff775992753d8686970b1e2755d21c8a63be73aba7a4e7d77/wheel-0.44.0-py3-none-any.whl - sha256: 2376a90c98cc337d18623527a97c31797bd02bad0033d41547043a1cbfbe448f + url: https://files.pythonhosted.org/packages/0b/2c/87f3254fd8ffd29e4c02732eee68a83a1d3c346ae39bc6822dcbcb697f2b/wheel-0.45.1-py3-none-any.whl + sha256: 708e7481cc80179af0e556bbf0cc00b8444c7321e2700b8d8580231d13017248 From 4dfe566e187b6934879e2cc36c11f160c858335d Mon Sep 17 00:00:00 2001 From: EmoonX Date: Tue, 31 Dec 2024 14:08:56 -0300 Subject: [PATCH 6/8] chore: add `ignore` pragma fallback for `import fluidsynth` --- bottles/backend/utils/midi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bottles/backend/utils/midi.py b/bottles/backend/utils/midi.py index 86b9c5a269..62c7cab7f8 100644 --- a/bottles/backend/utils/midi.py +++ b/bottles/backend/utils/midi.py @@ -1,4 +1,4 @@ -import fluidsynth +import fluidsynth # type: ignore[import-not-found] from bottles.backend.logger import Logger from bottles.backend.models.config import BottleConfig From dbee2e5c75ffe4fc6885d5e201040f265f64a48a Mon Sep 17 00:00:00 2001 From: EmoonX Date: Tue, 31 Dec 2024 21:55:59 -0300 Subject: [PATCH 7/8] midi: reduce channels from 256 to 16 (and thus banks from 16 to 1) --- bottles/backend/utils/midi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bottles/backend/utils/midi.py b/bottles/backend/utils/midi.py index 62c7cab7f8..1f45a6bbe0 100644 --- a/bottles/backend/utils/midi.py +++ b/bottles/backend/utils/midi.py @@ -46,7 +46,7 @@ def __run_server(self): "Starting new FluidSynth server with SoundFont" f" #{self.instrument_set_id} ('{self.soundfont_path}')…" ) - synth = fluidsynth.Synth() + synth = fluidsynth.Synth(channels=16) synth.start() sfid = synth.sfload(self.soundfont_path) synth.program_select(0, sfid, 0, 0) From 89520d238c0bca311b3f1e9fd52e1acdf7f074de Mon Sep 17 00:00:00 2001 From: EmoonX Date: Tue, 31 Dec 2024 23:17:46 -0300 Subject: [PATCH 8/8] midi: FluidSynth instance correctly terminated on executor's exit Done through garbage collector's reference counting. In case of multiple programs running with the same soundfont at once, instance is only terminated after ALL of them exit. --- bottles/backend/utils/midi.py | 51 +++++++++++++++++++------------- bottles/backend/wine/executor.py | 8 +++++ 2 files changed, 39 insertions(+), 20 deletions(-) diff --git a/bottles/backend/utils/midi.py b/bottles/backend/utils/midi.py index 1f45a6bbe0..6b608e6de7 100644 --- a/bottles/backend/utils/midi.py +++ b/bottles/backend/utils/midi.py @@ -1,4 +1,5 @@ -import fluidsynth # type: ignore[import-not-found] +from ctypes import c_void_p +from fluidsynth import cfunc, Synth from bottles.backend.logger import Logger from bottles.backend.models.config import BottleConfig @@ -8,7 +9,7 @@ class FluidSynth: - """Manages a FluidSynth server bounded to a SoundFont (.sf2, .sf3) file.""" + """Manages a FluidSynth instance bounded to an unique SoundFont (.sf2, .sf3) file.""" __active_instances: dict[int, "FluidSynth"] = {} """Active FluidSynth instances (i.e currently in use by one or more programs).""" @@ -17,7 +18,7 @@ class FluidSynth: def find_or_create(cls, soundfont_path: str) -> "FluidSynth": """ Search for running FluidSynth instance and return it. - If nonexistent, create and add SoundFont to active ones' dict beforehand. + If nonexistent, create and add it to active ones beforehand. """ for fs in cls.__active_instances.values(): @@ -25,54 +26,64 @@ def find_or_create(cls, soundfont_path: str) -> "FluidSynth": return fs fs = cls(soundfont_path) - cls.__active_instances[fs.instrument_set_id] = fs + cls.__active_instances[fs.id] = fs return fs def __init__(self, soundfont_path: str): """Build a new FluidSynth object from SoundFont file path.""" self.soundfont_path = soundfont_path - self.instrument_set_id = self.__get_vacant_id() - self.__run_server() + self.id = self.__get_vacant_id() + self.__start() @classmethod def __get_vacant_id(cls) -> int: - """Get smallest 0-indexed ID currently not being used by a SoundFont.""" + """Get smallest 0-indexed ID currently not in use by a SoundFont.""" n = len(cls.__active_instances) return next(i for i in range(n + 1) if i not in cls.__active_instances) - def __run_server(self): - """Create Synth object and start server with loaded SoundFont.""" + def __start(self): + """Start FluidSynth synthetizer with loaded SoundFont.""" logging.info( "Starting new FluidSynth server with SoundFont" - f" #{self.instrument_set_id} ('{self.soundfont_path}')…" + f" #{self.id} ('{self.soundfont_path}')…" ) - synth = fluidsynth.Synth(channels=16) + synth = Synth(channels=16) synth.start() sfid = synth.sfload(self.soundfont_path) synth.program_select(0, sfid, 0, 0) - self.server = synth + self.synth = synth def register_as_current(self, config: BottleConfig): """ - Update Wine registry with SoundFont's ID, instructing + Update Wine registry with this instance's ID, instructing MIDI mapping to load the correct instrument set on program startup. """ reg = Reg(config) reg.add( key="HKEY_CURRENT_USER\\Software\\Microsoft\\Windows\\CurrentVersion\\Multimedia\\MIDIMap", value="CurrentInstrument", - data=f"#{self.instrument_set_id}", + data=f"#{self.id}", value_type="REG_SZ", ) - def __del__(self): + def delete(self): """ - Kill underlying server and remove SoundFont from dict - when object instance is deallocated (i.e no programs using it anymore). + Kill underlying synthetizer and remove FluidSynth instance from dict. + Should be called only when no more programs are using it. """ + + def __delete_synth(synth: Synth): + """Bind missing function and run deletion routines.""" + delete_fluid_midi_driver = cfunc( + 'delete_fluid_midi_driver', c_void_p, ('driver', c_void_p, 1) + ) + delete_fluid_midi_driver(synth.midi_driver) + synth.delete() + logging.info( "Killing FluidSynth server with SoundFont" - f" #{self.instrument_set_id} ('{self.soundfont_path}')…" + f" #{self.id} ('{self.soundfont_path}')…" ) - self.server.delete() - self.__active_soundfonts.pop(self.soundfont_path) + __delete_synth(self.synth) + self.__active_instances.pop(self.id) + diff --git a/bottles/backend/wine/executor.py b/bottles/backend/wine/executor.py index f6a242532a..dc8daeb3f4 100644 --- a/bottles/backend/wine/executor.py +++ b/bottles/backend/wine/executor.py @@ -1,3 +1,4 @@ +import gc import os import shlex import uuid @@ -363,3 +364,10 @@ def __set_monitors(self): winedbg = WineDbg(self.config, silent=True) for m in self.monitoring: winedbg.wait_for_process(name=m) + + def __del__(self): + """On exit, kill FluidSynth instance if this was the last executor using it.""" + if hasattr(self, "fluidsynth") and self.fluidsynth is not None: + ref_count = len(gc.get_referrers(self.fluidsynth)) + if ref_count == 2: + self.fluidsynth.delete()