+ Open in your Web browser +
{}
'.format(self.description)) - - def __ge__(self, other): - return not self.__lt__(self, other) - - def __lt__(self, other): - if self.parent.sortType == "alphabetical": - return self.name.lower() > other.name.lower() - elif self.parent.sortType == "rating": - if self.rating == other.rating: - if self.reviews == other.reviews: - return self.name.lower() > other.name.lower() - return self.reviews < other.reviews - return self.rating < other.rating - elif self.parent.sortType == "size": - if self.height * self.width == other.height * other.width: - return self.name.lower() > other.name.lower() - return self.height * self.width < other.height * other.width - elif self.parent.sortType == "date": - if self.date is None: - return other.date is not None - if self.date == other.date: - return self.name.lower() > other.name.lower() - return self.date < other.date diff --git a/src/vaults/vaultitem.py b/src/vaults/vaultitem.py new file mode 100644 index 000000000..85d023803 --- /dev/null +++ b/src/vaults/vaultitem.py @@ -0,0 +1,166 @@ +from PyQt5 import QtCore, QtGui, QtWidgets + +import util +from downloadManager import DownloadRequest + + +class VaultItem(QtWidgets.QListWidgetItem): + TEXTWIDTH = 230 + ICONSIZE = 100 + PADDING = 10 + + def __init__(self, parent, *args, **kwargs): + QtWidgets.QListWidgetItem.__init__(self, *args, **kwargs) + self.parent = parent + + self.name = "" + self.description = "" + self.trimmedDescription = "" + self.version = 0 + self.rating = 0 + self.reviews = 0 + self.date = None + + self.itemType_ = "" + self.color = "white" + + self.link = "" + self.setHidden(True) + + self._item_dl_request = DownloadRequest() + self._item_dl_request.done.connect(self._on_item_downloaded) + + def update(self): + self.ensureIcon() + self.updateVisibility() + + def setItemIcon(self, filename, themed=True): + icon = util.THEME.icon(filename) + if not themed: + pixmap = QtGui.QPixmap(filename) + if not pixmap.isNull(): + icon.addPixmap( + pixmap.scaled( + QtCore.QSize(self.ICONSIZE, self.ICONSIZE), + ), + ) + self.setIcon(icon) + + def ensureIcon(self): + if self.icon() is None or self.icon().isNull(): + self.setItemIcon("games/unknown_map.png") + + def _on_item_downloaded(self, mapname, result): + filename, themed = result + self.setItemIcon(filename, themed) + self.ensureIcon() + + def updateVisibility(self): + self.setHidden(not self.shouldBeVisible()) + if len(self.description) < 200: + self.trimmedDescription = self.description + else: + self.trimmedDescription = self.description[:197] + "..." + + self.setToolTip('{}
'.format(self.description)) + + def __ge__(self, other): + return not self.__lt__(self, other) + + def __lt__(self, other): + if self.parent.sortType == "alphabetical": + return self.name.lower() > other.name.lower() + elif self.parent.sortType == "rating": + if self.rating == other.rating: + if self.reviews == other.reviews: + return self.name.lower() > other.name.lower() + return self.reviews < other.reviews + return self.rating < other.rating + elif self.parent.sortType == "size": + if self.height * self.width == other.height * other.width: + return self.name.lower() > other.name.lower() + return self.height * self.width < other.height * other.width + elif self.parent.sortType == "date": + if self.date is None: + return other.date is not None + if self.date == other.date: + return self.name.lower() > other.name.lower() + return self.date < other.date + + +class VaultItemDelegate(QtWidgets.QStyledItemDelegate): + + def __init__(self, *args, **kwargs): + QtWidgets.QStyledItemDelegate.__init__(self, *args, **kwargs) + + def paint(self, painter, option, index, *args, **kwargs): + self.initStyleOption(option, index) + + painter.save() + + html = QtGui.QTextDocument() + html.setHtml(option.text) + + icon = QtGui.QIcon(option.icon) + iconsize = QtCore.QSize(VaultItem.ICONSIZE, VaultItem.ICONSIZE) + + # clear icon and text before letting the control draw itself because + # we're rendering these parts ourselves + option.icon = QtGui.QIcon() + option.text = "" + option.widget.style().drawControl( + QtWidgets.QStyle.CE_ItemViewItem, option, painter, option.widget, + ) + + # Shadow + painter.fillRect( + option.rect.left() + 7, + option.rect.top() + 7, + iconsize.width(), + iconsize.height(), + QtGui.QColor("#202020"), + ) + + iconrect = QtCore.QRect(option.rect.adjusted(3, 3, 0, 0)) + iconrect.setSize(iconsize) + # Icon + icon.paint( + painter, iconrect, QtCore.Qt.AlignLeft | QtCore.Qt.AlignVCenter, + ) + + # Frame around the icon + pen = QtGui.QPen() + pen.setWidth(1) + # FIXME: This needs to come from theme. + pen.setBrush(QtGui.QColor("#303030")) + + pen.setCapStyle(QtCore.Qt.RoundCap) + painter.setPen(pen) + painter.drawRect(iconrect) + + # Description + painter.translate( + option.rect.left() + iconsize.width() + 10, option.rect.top() + 4, + ) + clip = QtCore.QRectF( + 0, 0, option.rect.width() - iconsize.width() - 15, + option.rect.height(), + ) + html.drawContents(painter, clip) + + painter.restore() + + def sizeHint(self, option, index, *args, **kwargs): + self.initStyleOption(option, index) + + html = QtGui.QTextDocument() + html.setHtml(option.text) + html.setTextWidth(VaultItem.TEXTWIDTH) + return QtCore.QSize( + ( + VaultItem.ICONSIZE + + VaultItem.TEXTWIDTH + + VaultItem.PADDING + ), + VaultItem.ICONSIZE + VaultItem.PADDING, + ) From af582e24f628335c13bd24f3f6d9cad0c40352ea Mon Sep 17 00:00:00 2001 From: gatsik <74517072+Gatsik@users.noreply.github.com> Date: Sat, 4 Mar 2023 23:13:54 +0200 Subject: [PATCH 006/123] Force single line for imports --- setup.cfg | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index 218b455de..e1d35b6d8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,3 +1,2 @@ [isort] -multi_line_output=3 -include_trailing_comma=True +force_single_line=True From 262dffb0c56dc8f43828efebdd12f6bb2a020055 Mon Sep 17 00:00:00 2001 From: gatsik <74517072+Gatsik@users.noreply.github.com> Date: Thu, 14 Mar 2024 19:59:53 +0200 Subject: [PATCH 007/123] Make .ui files compatible with PyQt6 --- res/client/client.ui | 3 --- res/client/oauth.ui | 3 --- 2 files changed, 6 deletions(-) diff --git a/res/client/client.ui b/res/client/client.ui index b4f2a915f..477e67e8b 100644 --- a/res/client/client.ui +++ b/res/client/client.ui @@ -31,9 +31,6 @@ '
- .format(url=_file['url'])
+ .format(url=_file['cacheableUrl'])
),
category='Update',
silent=False,
+ request_params={_file["hmacParameter"]: _file["hmacToken"]},
)
if not downloaded:
diff --git a/src/vaults/dialogs.py b/src/vaults/dialogs.py
index 142a7afad..2dbce1fe7 100644
--- a/src/vaults/dialogs.py
+++ b/src/vaults/dialogs.py
@@ -204,7 +204,14 @@ def downloadVaultAsset(url, target_dir, exist_handler, name, category, silent):
return ret
-def downloadFile(url, target_dir, name, category, silent):
+def downloadFile(
+ url: str,
+ target_dir: str,
+ name: str,
+ category: str,
+ silent: bool,
+ request_params: dict | None = None,
+) -> None:
"""
Basically a copy of downloadVaultAssetNoMsg without zip
"""
@@ -214,7 +221,7 @@ def downloadFile(url, target_dir, name, category, silent):
output = io.BytesIO()
capitCat = category[0].upper() + category[1:]
- dler = FileDownload(_global_nam, url, output)
+ dler = FileDownload(_global_nam, url, output, request_params=request_params)
ddialog = VaultDownloadDialog(
dler, "Downloading {}".format(category), name, silent,
)
From 2aada998315ea8bc7d29370c41fb4dcab81758bd Mon Sep 17 00:00:00 2001
From: gatsik <74517072+Gatsik@users.noreply.github.com>
Date: Thu, 11 Apr 2024 21:39:02 +0300
Subject: [PATCH 021/123] Add more download progress info
downloaded size and total size
---
src/vaults/dialogs.py | 29 ++++++++++++++++-------------
1 file changed, 16 insertions(+), 13 deletions(-)
diff --git a/src/vaults/dialogs.py b/src/vaults/dialogs.py
index 2dbce1fe7..a75ef6f66 100644
--- a/src/vaults/dialogs.py
+++ b/src/vaults/dialogs.py
@@ -48,29 +48,32 @@ def __init__(self, dler, title, label, silent=False):
progressBar.setAlignment(QtCore.Qt.AlignmentFlag.AlignCenter)
self._progress.setBar(progressBar)
+ self.progress_measure_interaval = 250
self.timer = QtCore.QTimer()
- self.timer.setInterval(500)
+ self.timer.setInterval(self.progress_measure_interaval)
self.timer.timeout.connect(self.updateLabel)
self.bytes_prev = 0
- def updateLabel(self):
- self._progress.setLabelText(
- '{label}\n\n{downloaded} MiB ({speed} MiB/s)'
- .format(
- label=self.label,
- downloaded=self.getDownloadProgressMiB(),
- speed=self.getDownloadSpeed(),
- ),
- )
+ def updateLabel(self) -> None:
+ label_text = f"{self.label}\n\n{self.get_download_progress_mb()}"
+ speed_text = f"({self.get_download_speed()} MB/s)"
+ if self._dler.bytes_total > 0:
+ label_text = f"{label_text}/{self.get_download_size_mb()} MB\n\n{speed_text}"
+ else:
+ label_text = f"{label_text} MB {speed_text}"
+ self._progress.setLabelText(label_text)
- def getDownloadSpeed(self):
+ def get_download_speed(self) -> float:
bytes_diff = self._dler.bytes_progress - self.bytes_prev
self.bytes_prev = self._dler.bytes_progress
- return round(bytes_diff * 2 / 1024 / 1024, 2)
+ return round(bytes_diff * (1000 / self.progress_measure_interaval) / 1024 / 1024, 2)
- def getDownloadProgressMiB(self):
+ def get_download_progress_mb(self) -> float:
return round(self._dler.bytes_progress / 1024 / 1024, 2)
+ def get_download_size_mb(self) -> float:
+ return round(self._dler.bytes_total / 1024 / 1024, 2)
+
def run(self):
self.updateLabel()
self.timer.start()
From bb82911573d128e3147536171d0e5a161d592fdc Mon Sep 17 00:00:00 2001
From: gatsik <74517072+Gatsik@users.noreply.github.com>
Date: Thu, 11 Apr 2024 22:38:29 +0300
Subject: [PATCH 022/123] Remove deprecated replay filter parameters
and use approximation to filter replays by rating
---
src/replays/_replayswidget.py | 33 +++++----------------------------
1 file changed, 5 insertions(+), 28 deletions(-)
diff --git a/src/replays/_replayswidget.py b/src/replays/_replayswidget.py
index 2d2be09ae..377a2d4be 100644
--- a/src/replays/_replayswidget.py
+++ b/src/replays/_replayswidget.py
@@ -853,34 +853,11 @@ def prepareFilters(
.format(leaderboardId),
)
- if minRating and minRating > 0:
- if leaderboardId == 1:
- filters.append(
- 'playerStats.player.globalRating.rating=ge="{}"'
- .format(minRating),
- )
- elif leaderboardId == 2:
- filters.append(
- 'playerStats.player.ladder1v1Rating.rating=ge="{}"'
- .format(minRating),
- )
- else:
- filters.append(
- 'playerStats.ratingChanges.meanBefore=ge="{}"'
- .format(minRating + 300),
- )
- else:
- if minRating and minRating > 0:
- if modListIndex == "ladder1v1":
- filters.append(
- 'playerStats.player.ladder1v1Rating.rating=ge="{}"'
- .format(minRating),
- )
- else:
- filters.append(
- 'playerStats.player.globalRating.rating=ge="{}"'
- .format(minRating),
- )
+ if minRating and minRating > 0:
+ filters.append(
+ 'playerStats.ratingChanges.meanBefore=ge="{}"'
+ .format(minRating + 300),
+ )
if mapName:
filters.append(
From c479a2d95748b89736aa249fb326f8b783240b30 Mon Sep 17 00:00:00 2001
From: gatsik <74517072+Gatsik@users.noreply.github.com>
Date: Fri, 12 Apr 2024 20:26:55 +0300
Subject: [PATCH 023/123] Remove QtWebEngine usage from NewsWidget
---
res/news/news.ui | 21 +--
res/news/news_page.html | 14 ++
res/news/{news_webview.css => news_style.css} | 4 +-
res/news/news_webview_frame.html | 10 --
src/news/_newswidget.py | 151 ++++++++----------
5 files changed, 89 insertions(+), 111 deletions(-)
create mode 100644 res/news/news_page.html
rename res/news/{news_webview.css => news_style.css} (89%)
delete mode 100644 res/news/news_webview_frame.html
diff --git a/res/news/news.ui b/res/news/news.ui
index b3021b139..05c95e5fe 100644
--- a/res/news/news.ui
+++ b/res/news/news.ui
@@ -217,32 +217,13 @@
").removesuffix("
") + html = self.HTML.format( + style=self.CSS, + title=current.newsPost['title'], + content=content, + img_source=current.newsPost["img_url"], + external_link=external_link, + ) + self.newsTextBrowser.setHtml(html) def showAll(self): for item in self.newsItems: item.setHidden(False) self.updateLabel(0) - def showEditToolTip(self): + def showEditToolTip(self) -> None: """ Default tooltips are too slow and disappear when user starts typing """ widget = self.hideNewsEdit position = widget.mapToGlobal( - QtCore.QPoint(0 + widget.width(), 0 - widget.height() / 2), + QPoint(0 + widget.width(), 0 - widget.height() / 2), ) QtWidgets.QToolTip.showText( position, From 3f8c5dd6fcb1445491d2c879af6c066be2536796 Mon Sep 17 00:00:00 2001 From: gatsik <74517072+Gatsik@users.noreply.github.com> Date: Sat, 13 Apr 2024 15:50:15 +0300 Subject: [PATCH 024/123] Remove QtWebEngine from UnitDB tab just open those urls in Web browser --- res/unitdb/unitdb.ui | 42 ++++++------ src/client/_clientwindow.py | 4 +- src/config/__init__.py | 1 - src/unitdb/unitdbtab.py | 133 ++++-------------------------------- 4 files changed, 33 insertions(+), 147 deletions(-) diff --git a/res/unitdb/unitdb.ui b/res/unitdb/unitdb.ui index 2c8fd5118..03c2f5156 100644 --- a/res/unitdb/unitdb.ui +++ b/res/unitdb/unitdb.ui @@ -25,6 +25,12 @@' - .format(url=_file['cacheableUrl']) - ), - category='Update', + downloaded = download_file( + url=url, + target_dir=target_dir, + name=name, + category="Update", silent=False, - request_params={_file["hmacParameter"]: _file["hmacToken"]}, + request_params={file_info["hmacParameter"]: file_info["hmacToken"]}, + label=f"Downloading FA file : {url}
",
)
if not downloaded:
@@ -294,7 +293,7 @@ def updateFiles(self, filegroup, files):
if self.keep_cache or self.in_session_cache:
files_to_check.append(_file)
else:
- self.fetchFile(_file, filegroup)
+ self.fetch_file(_file, filegroup)
self.filesToUpdate.remove(_file)
self.updatedFiles.append(_file['name'])
@@ -305,7 +304,7 @@ def updateFiles(self, filegroup, files):
self.replaceFromCache(replaceable_files, filegroup)
for _file in need_to_download:
self.moveToCache([_file], filegroup)
- self.fetchFile(_file, filegroup)
+ self.fetch_file(_file, filegroup)
self.filesToUpdate.remove(_file)
self.updatedFiles.append(_file['name'])
@@ -375,13 +374,12 @@ def prepareBinFAF(self):
# need to patch them
os.chmod(dst_file, st.st_mode | stat.S_IWRITE)
- def doUpdate(self):
+ def doUpdate(self) -> None:
""" The core function that does most of the actual update work."""
try:
- if self.sim:
- if utils.downloadMod(
- self.requestSimUrlByUid(self.featured_mod),
- ):
+ if self.sim_mod:
+ uid, name = self.sim_mod
+ if utils.downloadMod(self.request_sim_url_by_uid(uid), name):
self.result = self.RESULT_SUCCESS
else:
self.result = self.RESULT_FAILURE
diff --git a/src/mapGenerator/mapgenManager.py b/src/mapGenerator/mapgenManager.py
index 1fd325521..6bca9efb6 100644
--- a/src/mapGenerator/mapgenManager.py
+++ b/src/mapGenerator/mapgenManager.py
@@ -13,7 +13,7 @@
from fa.maps import getUserMapsFolder
from mapGenerator.mapgenProcess import MapGeneratorProcess
from mapGenerator.mapgenUtils import generatedMapPattern
-from vaults.dialogs import downloadFile
+from vaults.dialogs import download_file
logger = logging.getLogger(__name__)
@@ -132,21 +132,21 @@ def generateRandomMap(self):
return self.generateMap(mapName)
- def versionController(self, version):
+ def versionController(self, version: str) -> str:
name = GENERATOR_JAR_NAME.format(version)
- filePath = os.path.join(util.MAPGEN_DIR, name)
+ file_path = os.path.join(util.MAPGEN_DIR, name)
# Check if required version is already in folder
if os.path.isdir(util.MAPGEN_DIR):
for infile in os.listdir(util.MAPGEN_DIR):
if infile.lower() == name.lower():
- return filePath
+ return file_path
# Download from github if not
url = RELEASE_URL + RELEASE_VERSION_PATH.format(version=version)
- return downloadFile(
- url, filePath, name, "map generator", silent=False,
- )
+ if download_file(url, util.MAPGEN_DIR, name, "map generator", silent=False):
+ return file_path
+ return ""
def checkUpdates(self):
'''
diff --git a/src/model/game.py b/src/model/game.py
index 98c560a2e..0a575b869 100644
--- a/src/model/game.py
+++ b/src/model/game.py
@@ -167,9 +167,7 @@ def url(self, player_id):
else:
gtype = GameUrlType.LIVE_REPLAY
- return GameUrl(
- gtype, self.mapname, self.featured_mod, self.uid, player_id,
- )
+ return GameUrl(gtype, self.mapname, self.featured_mod, self.uid, player_id, self.sim_mods)
# Utility functions start here.
diff --git a/src/util/gameurl.py b/src/util/gameurl.py
index a7ed29cd5..0e653ae1a 100644
--- a/src/util/gameurl.py
+++ b/src/util/gameurl.py
@@ -28,8 +28,8 @@ def to_url(self):
query = QUrlQuery()
query.addQueryItem("map", self.map)
query.addQueryItem("mod", self.mod)
- if self.mods is not None:
- query.addQueryItem("mods", self.mods)
+ if self.mods:
+ query.addQueryItem("mods", ";".join(self.mods))
if self.game_type == GameUrlType.OPEN_GAME:
url.setPath("/{}".format(self.player))
diff --git a/src/vaults/dialogs.py b/src/vaults/dialogs.py
index a75ef6f66..a73d49932 100644
--- a/src/vaults/dialogs.py
+++ b/src/vaults/dialogs.py
@@ -1,13 +1,13 @@
-import io
import logging
import os
-import zipfile
+from typing import Callable
from PyQt6 import QtCore
from PyQt6 import QtNetwork
from PyQt6 import QtWidgets
from downloadManager import FileDownload
+from downloadManager import ZipDownloadExtract
logger = logging.getLogger(__name__)
@@ -19,7 +19,13 @@ class VaultDownloadDialog(object):
DL_ERROR = 2
UNKNOWN_ERROR = 3
- def __init__(self, dler, title, label, silent=False):
+ def __init__(
+ self,
+ dler: FileDownload | ZipDownloadExtract,
+ title: str,
+ label: str,
+ silent: bool = False,
+ ) -> None:
self._silent = silent
self._result = None
@@ -122,8 +128,15 @@ def _finished(self, dler):
def downloadVaultAssetNoMsg(
- url, target_dir, exist_handler, name, category, silent,
-):
+ url: str,
+ target_dir: str,
+ exist_handler: Callable[[str, str], bool],
+ name: str,
+ category: str,
+ silent: bool,
+ request_params: dict | None = None,
+ label: str = "",
+) -> tuple[bool, Callable[[], None] | None]:
"""
Download and unpack a zip from the vault, interacting with the user and
logging things.
@@ -132,69 +145,39 @@ def downloadVaultAssetNoMsg(
msg = None
msg_title = ""
msg_text = ""
- output = io.BytesIO()
- capitCat = category[0].upper() + category[1:]
+ capit_cat = f"{category[0].upper()}{category[1:]}"
- dler = FileDownload(_global_nam, url, output)
- ddialog = VaultDownloadDialog(
- dler, "Downloading {}".format(category), name, silent,
- )
+ if os.path.exists(os.path.join(target_dir, name)):
+ proceed = exist_handler(target_dir, name)
+ if not proceed:
+ return False, msg
+
+ dler = ZipDownloadExtract(target_dir, _global_nam, url, request_params)
+ ddialog = VaultDownloadDialog(dler, f"Downloading {category}", label or name, silent)
result = ddialog.run()
if result == VaultDownloadDialog.CANCELED:
- logger.warning("{} Download canceled for: {}".format(capitCat, url))
+ logger.warning(f"{capit_cat} Download canceled for: {url}")
if result in [
VaultDownloadDialog.DL_ERROR,
VaultDownloadDialog.UNKNOWN_ERROR,
]:
- logger.warning(
- "Vault download failed, {} probably not in vault "
- "(or broken).".format(category),
- )
- msg_title = "{} not downloadable".format(capitCat)
+ logger.warning(f"Vault download failed, {category} probably not in vault (or broken).")
+ msg_title = "{} not downloadable".format(capit_cat)
msg_text = (
- "This {} was not found in the vault (or is broken)."
- "
You need to get it from somewhere else in order to "
- "use it.".format(category)
+ f"This {category} was not found in the vault (or is broken)."
+ f"
You need to get it from somewhere else in order to "
+ f"use it."
)
- if msg_title and msg_text:
def msg():
QtWidgets.QMessageBox.information(None, msg_title, msg_text)
if result != VaultDownloadDialog.SUCCESS:
return False, msg
- try:
- zfile = zipfile.ZipFile(output)
- # FIXME - nothing in python 2.7 that can do that
- dirname = zfile.namelist()[0].split(os.path.sep, 1)[0]
-
- if os.path.exists(os.path.join(target_dir, dirname)):
- proceed = exist_handler(target_dir, dirname)
- if not proceed:
- return False
- zfile.extractall(target_dir)
- logger.debug(
- "Successfully downloaded and extracted {} from: {}"
- .format(category, url),
- )
- return True, msg
-
- except BaseException:
- logger.error("Extract error")
-
- def msg():
- QtWidgets.QMessageBox.information(
- None,
- "{} installation failed".format(capitCat),
- (
- "This {} could not be installed (please report this {} "
- "or bug).".format(category, category)
- ),
- )
- return False, msg
+ return True, msg
def downloadVaultAsset(url, target_dir, exist_handler, name, category, silent):
@@ -207,56 +190,43 @@ def downloadVaultAsset(url, target_dir, exist_handler, name, category, silent):
return ret
-def downloadFile(
+def download_file(
url: str,
target_dir: str,
name: str,
category: str,
silent: bool,
request_params: dict | None = None,
-) -> None:
+ label: str = "",
+) -> bool:
"""
Basically a copy of downloadVaultAssetNoMsg without zip
"""
global _global_nam
- msg = None
- output = io.BytesIO()
- capitCat = category[0].upper() + category[1:]
+ capit_cat = f"{category[0].upper()}{category[1:]}"
- dler = FileDownload(_global_nam, url, output, request_params=request_params)
- ddialog = VaultDownloadDialog(
- dler, "Downloading {}".format(category), name, silent,
- )
+ os.makedirs(target_dir, exist_ok=True)
+
+ target_path = os.path.join(target_dir, name)
+ dler = FileDownload(target_path, _global_nam, url, request_params)
+ ddialog = VaultDownloadDialog(dler, f"Downloading {category}", label or name, silent)
result = ddialog.run()
if result == VaultDownloadDialog.CANCELED:
- logger.warning("{} Download canceled for: {}".format(capitCat, url))
+ logger.warning(f"{capit_cat} Download canceled for: {url}")
if result in [
VaultDownloadDialog.DL_ERROR,
VaultDownloadDialog.UNKNOWN_ERROR,
]:
- logger.warning("Download failed. {}".format(url))
-
- def msg():
- QtWidgets.QMessageBox.information(
- None,
- "{} not downloadable".format(capitCat),
- (
- "Failed to download {} from
"
- "{}".format(category, url)
- ),
- )
+ logger.warning(f"Download failed. {url}")
+ QtWidgets.QMessageBox.information(
+ None,
+ f"{capit_cat} not downloadable",
+ f"Failed to download {category} from
{url}",
+ )
if result != VaultDownloadDialog.SUCCESS:
- if msg:
- msg()
return False
- if not os.path.exists(os.path.dirname(target_dir)):
- os.makedirs(os.path.dirname(target_dir))
-
- with open(target_dir, "w+b") as f:
- f.write(output.getvalue())
-
- return target_dir
+ return True
diff --git a/src/vaults/modvault/modvault.py b/src/vaults/modvault/modvault.py
index 7cc1f6ebe..3ffc0c738 100644
--- a/src/vaults/modvault/modvault.py
+++ b/src/vaults/modvault/modvault.py
@@ -172,8 +172,8 @@ def openUploadForm(self):
"This folder doesn't contain a mod_info.lua file",
)
- def downloadMod(self, mod):
- if utils.downloadMod(mod):
+ def downloadMod(self, link: str, name: str) -> bool:
+ if utils.downloadMod(link, name):
self.uids = [mod.uid for mod in utils.getInstalledMods()]
self.updateVisibilities()
return True
diff --git a/src/vaults/modvault/modwidget.py b/src/vaults/modvault/modwidget.py
index 796dd600e..74b97b3a6 100644
--- a/src/vaults/modvault/modwidget.py
+++ b/src/vaults/modvault/modwidget.py
@@ -81,9 +81,9 @@ def load_stylesheet(self):
self.setStyleSheet(util.THEME.readstylesheet("client/client.css"))
@QtCore.pyqtSlot()
- def download(self):
+ def download(self) -> None:
if self.mod.uid not in self.parent.uids:
- self.parent.downloadMod(self.mod)
+ self.parent.downloadMod(self.mod.link, self.mod.name)
self.done(1)
else:
show = QtWidgets.QMessageBox.question(
diff --git a/src/vaults/modvault/utils.py b/src/vaults/modvault/utils.py
index 886335817..294741c91 100644
--- a/src/vaults/modvault/utils.py
+++ b/src/vaults/modvault/utils.py
@@ -420,14 +420,10 @@ def generateThumbnail(sourcename, destname):
return False
-def downloadMod(item):
- if isinstance(item, str):
- link = item
- else:
- link = item.link
- logger.debug("Getting mod from: {}".format(link))
+def downloadMod(link: str, name: str) -> bool:
+ logger.debug(f"Getting mod from: {link}")
- def handle_exist(path, modname):
+ def handle_exist(path: str, modname: str) -> bool:
modpath = os.path.join(path, modname)
oldmod = getModInfoFromFolder(modpath)
result = QtWidgets.QMessageBox.question(
@@ -446,9 +442,7 @@ def handle_exist(path, modname):
removeMod(oldmod)
return True
- return downloadVaultAsset(
- link, MODFOLDER, handle_exist, link, "mod", False,
- )
+ return downloadVaultAsset(link, MODFOLDER, handle_exist, name, "mod", silent=False)
def removeMod(mod):
From dbad8493fec69ea6e4635eb741f7cb94eef4b0f8 Mon Sep 17 00:00:00 2001
From: gatsik <74517072+Gatsik@users.noreply.github.com>
Date: Wed, 17 Apr 2024 19:50:53 +0300
Subject: [PATCH 036/123] Use FileDownload instead of BaseDownload in
DownloadWrapper
so that wrapper doesn't need to create and cleanup files
---
src/downloadManager/__init__.py | 46 ++++++++++++---------------------
src/vaults/vaultitem.py | 4 +--
2 files changed, 19 insertions(+), 31 deletions(-)
diff --git a/src/downloadManager/__init__.py b/src/downloadManager/__init__.py
index afc006392..d5b0ad049 100644
--- a/src/downloadManager/__init__.py
+++ b/src/downloadManager/__init__.py
@@ -160,8 +160,10 @@ def __init__(
addr: str,
request_params: dict | None = None,
) -> None:
- self._target_path = f"{target_path}.part"
- self._output = QFile(target_path)
+ self._target_path = target_path
+ self._cache_path = f"{target_path}.part"
+
+ self._output = QFile(self._cache_path)
self._output.open(QIODevice.OpenModeFlag.WriteOnly)
super().__init__(nam, addr, self._output, request_params=request_params)
@@ -172,11 +174,11 @@ def _about_to_finish(self) -> None:
def cleanup(self) -> None:
self._output.close()
if self.failed():
- logger.debug(f"Download failed for: {self.name}")
- os.unlink(self._target_path)
+ logger.debug(f"Download failed for: {self._target_path}")
+ os.unlink(self._cache_path)
else:
logger.debug(f"Finished download from {self.addr}")
- self._output.rename(self._target_path.removesuffix(".part"))
+ self._output.rename(self._target_path)
class ZipDownloadExtract(BaseDownload):
@@ -225,7 +227,7 @@ def cleanup(self) -> None:
self._output.close()
-class GeneralDownload(QObject):
+class DownloadWrapper(QObject):
done = pyqtSignal(object, object)
def __init__(
@@ -255,34 +257,20 @@ def _start_download(self) -> None:
self._dl = self._prepare_dl()
self._dl.run()
- def _prepare_dl(self) -> BaseDownload:
- file_obj = self._get_cachefile(self.name + ".part")
- dl = BaseDownload(self._nam, self._url, file_obj)
+ def _prepare_dl(self) -> FileDownload:
+ filepath = os.path.join(self._target_dir, self.name)
+ dl = FileDownload(filepath, self._nam, self._url)
dl.finished.connect(self._finished)
dl.blocksize = None
return dl
- def _get_cachefile(self, name: str) -> QFile:
- filepath = os.path.join(self._target_dir, name)
- file_obj = QFile(filepath)
- file_obj.open(QIODevice.OpenModeFlag.WriteOnly)
- return file_obj
-
def remove_request(self, req: DownloadRequest) -> None:
self.requests.remove(req)
def add_request(self, req: DownloadRequest) -> None:
self.requests.add(req)
- def _finished(self, dl: BaseDownload) -> None:
- dl.dest.close()
- destpath = dl.dest.fileName()
- if self.failed():
- logger.debug(f"Download failed for: {self.name}")
- os.unlink(destpath)
- else:
- logger.debug("Finished download from " + dl.addr)
- dl.dest.rename(destpath.removesuffix(".part"))
+ def _finished(self, dl: FileDownload) -> None:
self.done.emit(self, dl.dest.fileName())
def failed(self):
@@ -312,7 +300,7 @@ def finished(self, name, result):
self.done.emit(name, result)
-class GeneralDownloader(QObject):
+class Downloader(QObject):
"""
Class for downloading. Clients ask to download by giving download
requests, which are stored by name. After download is complete, all
@@ -328,7 +316,7 @@ def __init__(self, target_dir: str) -> None:
super().__init__()
self._nam = QNetworkAccessManager(self)
self._target_dir = target_dir
- self._downloads: dict[str, GeneralDownload] = {}
+ self._downloads: dict[str, DownloadWrapper] = {}
self._timeouts = DownloadTimeouts(self.REDOWNLOAD_TIMEOUT, self.DOWNLOAD_FAILS_TO_TIMEOUT)
def set_target_dir(self, target_dir: str) -> None:
@@ -348,11 +336,11 @@ def _add_download(self, name: str, url: str) -> None:
delay = self._timeouts.timer
else:
delay = None
- dl = GeneralDownload(self._nam, name, url, self._target_dir, delay)
+ dl = DownloadWrapper(self._nam, name, url, self._target_dir, delay)
dl.done.connect(self._finished_download)
self._downloads[name] = dl
- def _finished_download(self, download: GeneralDownload, download_path: str) -> None:
+ def _finished_download(self, download: DownloadWrapper, download_path: str) -> None:
self._timeouts.update_fail_count(download.name, download.failed())
requests = set(download.requests) # Don't change it during iteration
for req in requests:
@@ -362,7 +350,7 @@ def _finished_download(self, download: GeneralDownload, download_path: str) -> N
req.finished(download.name, (download_path, download.failed()))
-class MapPreviewDownloader(GeneralDownloader):
+class MapPreviewDownloader(Downloader):
def __init__(self, target_dir: str, size: str) -> None:
super().__init__(target_dir)
self.size = size
diff --git a/src/vaults/vaultitem.py b/src/vaults/vaultitem.py
index 11c321491..db04d38d5 100644
--- a/src/vaults/vaultitem.py
+++ b/src/vaults/vaultitem.py
@@ -3,8 +3,8 @@
from PyQt6 import QtWidgets
import util
+from downloadManager import Downloader
from downloadManager import DownloadRequest
-from downloadManager import GeneralDownloader
class VaultItem(QtWidgets.QListWidgetItem):
@@ -30,7 +30,7 @@ def __init__(self, parent, *args, **kwargs):
self.link = ""
self.setHidden(True)
- self._preview_dler = GeneralDownloader(util.CACHE_DIR)
+ self._preview_dler = Downloader(util.CACHE_DIR)
self._item_dl_request = DownloadRequest()
self._item_dl_request.done.connect(self._on_item_downloaded)
From 2ec6f51683b8722328e2d0469d39728340ddc963 Mon Sep 17 00:00:00 2001
From: gatsik <74517072+Gatsik@users.noreply.github.com>
Date: Wed, 17 Apr 2024 19:59:43 +0300
Subject: [PATCH 037/123] NewsWidget: Download news images into cache folder
---
src/news/_newswidget.py | 55 +++++++++++++++++++++--------------------
src/util/__init__.py | 5 +++-
2 files changed, 32 insertions(+), 28 deletions(-)
diff --git a/src/news/_newswidget.py b/src/news/_newswidget.py
index 39604c2f5..e34836ee3 100644
--- a/src/news/_newswidget.py
+++ b/src/news/_newswidget.py
@@ -1,18 +1,18 @@
import logging
+import os.path
from PyQt6 import QtWidgets
-from PyQt6.QtCore import QByteArray
from PyQt6.QtCore import QPoint
from PyQt6.QtCore import QSize
from PyQt6.QtCore import QUrl
from PyQt6.QtGui import QImage
from PyQt6.QtGui import QTextDocument
from PyQt6.QtNetwork import QNetworkAccessManager
-from PyQt6.QtNetwork import QNetworkReply
-from PyQt6.QtNetwork import QNetworkRequest
import util
from config import Settings
+from downloadManager import Downloader
+from downloadManager import DownloadRequest
from .newsitem import NewsItem
from .newsitem import NewsItemDelegate
@@ -35,11 +35,12 @@ def __init__(self, *args, **kwargs) -> None:
self.setupUi(self)
self.nam = QNetworkAccessManager()
- self.reply: QNetworkReply | None = None
+ self._downloader = Downloader(util.NEWS_CACHE_DIR)
+ self._images_dl_request = DownloadRequest()
+ self._images_dl_request.done.connect(self.item_image_downloaded)
self.newsManager = NewsManager(self)
self.newsItems = []
- self.images = {}
# open all links in external browser
self.newsTextBrowser.setOpenExternalLinks(True)
@@ -65,37 +66,36 @@ def updateNews(self) -> None:
self.newsList.clear()
self.newsManager.WpApi.download()
- def download_image(self, img_url: QUrl) -> None:
- request = QNetworkRequest(img_url)
- self.reply = self.nam.get(request)
- self.reply.finished.connect(self.item_image_downloaded)
+ def download_image(self, img_url: str) -> None:
+ name = os.path.basename(img_url)
+ self._downloader.download(name, self._images_dl_request, img_url)
- def add_image_resource(self, img_url: QUrl, image_data: QByteArray) -> None:
- img = QImage()
- img.loadFromData(image_data)
+ def add_image_resource(self, image_name: str, image_path: str) -> None:
+ doc = self.newsTextBrowser.document()
+ if doc.resource(QTextDocument.ResourceType.ImageResource, QUrl(image_name)):
+ return
+ img = QImage(image_path)
scaled = img.scaled(QSize(900, 500))
+ doc.addResource(QTextDocument.ResourceType.ImageResource, QUrl(image_name), scaled)
- self.images[img_url] = scaled
- self.newsTextBrowser.document().addResource(
- QTextDocument.ResourceType.ImageResource,
- img_url,
- scaled,
- )
-
- def item_image_downloaded(self) -> None:
- if self.reply.error() is not self.reply.NetworkError.NoError:
- return
- self.add_image_resource(self.reply.request().url(), self.reply.readAll())
+ def item_image_downloaded(self, image_name: str, result: tuple[str, bool]) -> None:
+ image_path, download_failed = result
+ if not download_failed:
+ self.add_image_resource(image_name, image_path)
self.show_newspage()
def itemChanged(self, current: NewsItem | None, previous: NewsItem | None) -> None:
if current is None:
return
- url = QUrl(current.newsPost["img_url"])
- if url in self.images:
+
+ url = current.newsPost["img_url"]
+ image_name = os.path.basename(url)
+ image_path = os.path.join(util.NEWS_CACHE_DIR, image_name)
+ if os.path.isfile(image_path):
+ self.add_image_resource(image_name, image_path)
self.show_newspage()
else:
- self.download_image(url)
+ self._downloader.download(image_name, self._images_dl_request, url)
def show_newspage(self) -> None:
current = self.newsList.currentItem()
@@ -105,12 +105,13 @@ def show_newspage(self) -> None:
else:
external_link = current.newsPost['external_link']
+ image_name = os.path.basename(current.newsPost["img_url"])
content = current.newsPost["excerpt"].strip().removeprefix("
").removesuffix("
") html = self.HTML.format( style=self.CSS, title=current.newsPost['title'], content=content, - img_source=current.newsPost["img_url"], + img_source=image_name, external_link=external_link, ) self.newsTextBrowser.setHtml(html) diff --git a/src/util/__init__.py b/src/util/__init__.py index 91b3be673..6586f2e04 100644 --- a/src/util/__init__.py +++ b/src/util/__init__.py @@ -55,6 +55,9 @@ MOD_PREVIEW_DIR = os.path.join(CACHE_DIR, "mod_previews") +# Cache for news images +NEWS_CACHE_DIR = os.path.join(CACHE_DIR, "news") + # This contains cached data downloaded for FA extras EXTRA_DIR = os.path.join(APPDATA_DIR, "extra") @@ -159,7 +162,7 @@ def setPersonalDir(): for data_dir in [ APPDATA_DIR, PERSONAL_DIR, LUA_DIR, CACHE_DIR, MAP_PREVIEW_SMALL_DIR, MAP_PREVIEW_LARGE_DIR, MOD_PREVIEW_DIR, - THEME_DIR, REPLAY_DIR, LOG_DIR, EXTRA_DIR, + THEME_DIR, REPLAY_DIR, LOG_DIR, EXTRA_DIR, NEWS_CACHE_DIR, ]: if not os.path.isdir(data_dir): os.makedirs(data_dir) From 587bff73228b1e8b831f6d3c0522d33f585ea457 Mon Sep 17 00:00:00 2001 From: gatsik <74517072+Gatsik@users.noreply.github.com> Date: Thu, 18 Apr 2024 18:38:19 +0300 Subject: [PATCH 038/123] vaults/dialogs: Extract some repeatable code into a function --- src/util/__init__.py | 7 +++ src/vaults/dialogs.py | 108 +++++++++++++++++++++++------------------- 2 files changed, 67 insertions(+), 48 deletions(-) diff --git a/src/util/__init__.py b/src/util/__init__.py index 6586f2e04..6d758ce27 100644 --- a/src/util/__init__.py +++ b/src/util/__init__.py @@ -501,3 +501,10 @@ def strtodate(s): def datetostr(d): return d.strftime("%Y-%m-%d %H:%M:%S") + + +def capitalize(string: str) -> str: + """ + Capitalize the first letter only, leave the rest as it is + """ + return f"{string[0].upper()}{string[1:]}" diff --git a/src/vaults/dialogs.py b/src/vaults/dialogs.py index a73d49932..b892759d6 100644 --- a/src/vaults/dialogs.py +++ b/src/vaults/dialogs.py @@ -1,5 +1,7 @@ import logging import os +from enum import Enum +from enum import auto from typing import Callable from PyQt6 import QtCore @@ -8,16 +10,19 @@ from downloadManager import FileDownload from downloadManager import ZipDownloadExtract +from util import capitalize logger = logging.getLogger(__name__) +class VaultDownloadResult(Enum): + SUCCESS = auto() + CANCELED = auto() + DL_ERROR = auto() + UNKNOWN_ERROR = auto() + + class VaultDownloadDialog(object): - # Result codes - SUCCESS = 0 - CANCELED = 1 - DL_ERROR = 2 - UNKNOWN_ERROR = 3 def __init__( self, @@ -102,24 +107,25 @@ def _cont(self, dler): QtWidgets.QApplication.processEvents() - def _finished(self, dler): + def _finished(self, dler: FileDownload | ZipDownloadExtract) -> None: self.timer.stop() self._progress.reset() + self._set_result(dler) - if not dler.succeeded(): + def _set_result(self, dler: FileDownload | ZipDownloadExtract) -> None: + if dler.failed(): if dler.canceled: - self._result = self.CANCELED + self._result = VaultDownloadResult.CANCELED return - elif dler.error: - self._result = self.DL_ERROR + self._result = VaultDownloadResult.DL_ERROR return else: logger.error('Unknown download error') - self._result = self.UNKNOWN_ERROR + self._result = VaultDownloadResult.UNKNOWN_ERROR return - self._result = self.SUCCESS + self._result = VaultDownloadResult.SUCCESS return @@ -127,6 +133,25 @@ def _finished(self, dler): _global_nam = QtNetwork.QNetworkAccessManager() +def _download_asset( + dler: FileDownload | ZipDownloadExtract, + category: str, + silent: bool, + label: str = "", +) -> VaultDownloadResult: + ddialog = VaultDownloadDialog(dler, f"Downloading {category}", label, silent) + result = ddialog.run() + + if result == VaultDownloadResult.CANCELED: + logger.warning(f"{category} Download canceled for: {dler.addr}") + if result in [ + VaultDownloadResult.DL_ERROR, + VaultDownloadResult.UNKNOWN_ERROR, + ]: + logger.warning(f"Download failed. {dler.addr}") + return result + + def downloadVaultAssetNoMsg( url: str, target_dir: str, @@ -142,28 +167,24 @@ def downloadVaultAssetNoMsg( logging things. """ global _global_nam + msg = None - msg_title = "" - msg_text = "" - capit_cat = f"{category[0].upper()}{category[1:]}" + capit_cat = capitalize(category) if os.path.exists(os.path.join(target_dir, name)): proceed = exist_handler(target_dir, name) if not proceed: return False, msg + os.makedirs(target_dir, exist_ok=True) dler = ZipDownloadExtract(target_dir, _global_nam, url, request_params) - ddialog = VaultDownloadDialog(dler, f"Downloading {category}", label or name, silent) - result = ddialog.run() - - if result == VaultDownloadDialog.CANCELED: - logger.warning(f"{capit_cat} Download canceled for: {url}") + result = _download_asset(dler, capit_cat, silent, label or name) if result in [ - VaultDownloadDialog.DL_ERROR, - VaultDownloadDialog.UNKNOWN_ERROR, + VaultDownloadResult.DL_ERROR, + VaultDownloadResult.UNKNOWN_ERROR, ]: - logger.warning(f"Vault download failed, {category} probably not in vault (or broken).") + logger.warning(f"Vault download failed, {category} is probably not in vault (or broken).") msg_title = "{} not downloadable".format(capit_cat) msg_text = ( f"This {category} was not found in the vault (or is broken)." @@ -174,19 +195,20 @@ def downloadVaultAssetNoMsg( def msg(): QtWidgets.QMessageBox.information(None, msg_title, msg_text) - if result != VaultDownloadDialog.SUCCESS: - return False, msg + return result == VaultDownloadResult.SUCCESS, msg - return True, msg - -def downloadVaultAsset(url, target_dir, exist_handler, name, category, silent): - ret, dialog = downloadVaultAssetNoMsg( - url, target_dir, exist_handler, name, category, silent, - ) +def downloadVaultAsset( + url: str, + target_dir: str, + exist_handler: Callable[[str, str], bool], + name: str, + category: str, + silent: bool, +) -> bool: + ret, dialog = downloadVaultAssetNoMsg(url, target_dir, exist_handler, name, category, silent) if dialog is not None: dialog() - return ret @@ -202,31 +224,21 @@ def download_file( """ Basically a copy of downloadVaultAssetNoMsg without zip """ - - global _global_nam - capit_cat = f"{category[0].upper()}{category[1:]}" + capit_cat = capitalize(category) os.makedirs(target_dir, exist_ok=True) - target_path = os.path.join(target_dir, name) + dler = FileDownload(target_path, _global_nam, url, request_params) - ddialog = VaultDownloadDialog(dler, f"Downloading {category}", label or name, silent) - result = ddialog.run() + result = _download_asset(dler, capit_cat, silent, label or name) - if result == VaultDownloadDialog.CANCELED: - logger.warning(f"{capit_cat} Download canceled for: {url}") if result in [ - VaultDownloadDialog.DL_ERROR, - VaultDownloadDialog.UNKNOWN_ERROR, + VaultDownloadResult.DL_ERROR, + VaultDownloadResult.UNKNOWN_ERROR, ]: - logger.warning(f"Download failed. {url}") QtWidgets.QMessageBox.information( None, f"{capit_cat} not downloadable", f"Failed to download {category} from{}
'.format(self.description)) + def should_be_visible(self) -> bool: + return True - def __ge__(self, other): + def update_visibility(self) -> None: + self.setHidden(self.should_be_hidden()) + if len(self.item_version.description) < 200: + trimmed_description = self.item_version.description + else: + trimmed_description = f"{self.item_version.description[:197]}..." + self.setToolTip('{}
'.format(trimmed_description)) + + def __ge__(self, other: VaultListItem) -> bool: return not self.__lt__(self, other) - def __lt__(self, other): + def __lt__(self, other: VaultListItem) -> bool: if self.parent.sortType == "alphabetical": - return self.name.lower() > other.name.lower() + return self._lt_alphabetical(other) elif self.parent.sortType == "rating": - if self.rating == other.rating: - if self.reviews == other.reviews: - return self.name.lower() > other.name.lower() - return self.reviews < other.reviews - return self.rating < other.rating - elif self.parent.sortType == "size": - if self.height * self.width == other.height * other.width: - return self.name.lower() > other.name.lower() - return self.height * self.width < other.height * other.width + return self._lt_rating(other) elif self.parent.sortType == "date": - if self.date is None: - return other.date is not None - if self.date == other.date: - return self.name.lower() > other.name.lower() - return self.date < other.date + return self._lt_date(other) + return True + + def _lt_date(self, other: VaultListItem) -> bool: + if self.item_version.create_time == other.item_version.create_time: + if self.item_version.update_time == other.item_version.update_time: + return self._lt_alphabetical(other) + return self.item_version.update_time < other.item_version.update_time + return self.item_version.create_time < other.item_version.create_time + + def _lt_alphabetical(self, other: VaultListItem) -> bool: + return self.item_info.display_name.lower() > other.item_info.display_name.lower() + def _lt_rating(self, other: VaultListItem) -> bool: + review = self.item_info.reviews_summary + other_review = other.item_info.reviews_summary -class VaultItemDelegate(QtWidgets.QStyledItemDelegate): + if review is None: + return other_review is not None + if other_review is None: + return review is None - def __init__(self, *args, **kwargs): - QtWidgets.QStyledItemDelegate.__init__(self, *args, **kwargs) + if review.average_score == other_review.average_score: + if review.num_reviews == other_review.num_reviews: + return self._lt_alphabetical(other) + return review.num_reviews < other_review.num_reviews + return review.average_score < other_review.average_score + + +class VaultItemDelegate(QStyledItemDelegate): def paint(self, painter, option, index, *args, **kwargs): self.initStyleOption(option, index) @@ -106,15 +120,14 @@ def paint(self, painter, option, index, *args, **kwargs): html.setHtml(option.text) icon = QtGui.QIcon(option.icon) - iconsize = QtCore.QSize(VaultItem.ICONSIZE, VaultItem.ICONSIZE) + iconsize = QtCore.QSize(VaultListItem.ICONSIZE, VaultListItem.ICONSIZE) # clear icon and text before letting the control draw itself because # we're rendering these parts ourselves option.icon = QtGui.QIcon() option.text = "" - option.widget.style().drawControl( - QtWidgets.QStyle.ControlElement.CE_ItemViewItem, option, painter, option.widget, - ) + control_element = QStyle.ControlElement.CE_ItemViewItem + option.widget.style().drawControl(control_element, option, painter, option.widget) # Shadow painter.fillRect( @@ -158,12 +171,12 @@ def sizeHint(self, option, index, *args, **kwargs): html = QtGui.QTextDocument() html.setHtml(option.text) - html.setTextWidth(VaultItem.TEXTWIDTH) + html.setTextWidth(VaultListItem.TEXTWIDTH) return QtCore.QSize( ( - VaultItem.ICONSIZE - + VaultItem.TEXTWIDTH - + VaultItem.PADDING + VaultListItem.ICONSIZE + + VaultListItem.TEXTWIDTH + + VaultListItem.PADDING ), - VaultItem.ICONSIZE + VaultItem.PADDING, + VaultListItem.ICONSIZE + VaultListItem.PADDING, ) From 0e297cf321afca573f9367f4d8bd620ed99ea68f Mon Sep 17 00:00:00 2001 From: gatsik <74517072+Gatsik@users.noreply.github.com> Date: Mon, 22 Apr 2024 21:00:06 +0300 Subject: [PATCH 043/123] FileDownload: Except possible permission error when deleting cache file --- src/downloadManager/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/downloadManager/__init__.py b/src/downloadManager/__init__.py index 9c67ffd0a..2f475f7c7 100644 --- a/src/downloadManager/__init__.py +++ b/src/downloadManager/__init__.py @@ -174,7 +174,10 @@ def _about_to_finish(self) -> None: def cleanup(self) -> None: self._output.close() if self.failed(): - os.unlink(self._cache_path) + try: + os.unlink(self._cache_path) + except OSError as e: + logger.warning(f"Couldn't remove {self._cache_path}: {e}") else: logger.debug(f"Finished download from {self.addr}") self._output.rename(self._target_path) From e148c51d9bb2e7e3bb7d69d0c7335c6aa424df86 Mon Sep 17 00:00:00 2001 From: gatsik <74517072+Gatsik@users.noreply.github.com> Date: Tue, 23 Apr 2024 22:12:18 +0300 Subject: [PATCH 044/123] Factor out a superclass for vault api connectors --- src/api/vaults_api.py | 105 +++++++++++++++++++++++------------------- src/vaults/vault.py | 2 +- 2 files changed, 58 insertions(+), 49 deletions(-) diff --git a/src/api/vaults_api.py b/src/api/vaults_api.py index ba69edc93..2fff9d01a 100644 --- a/src/api/vaults_api.py +++ b/src/api/vaults_api.py @@ -1,4 +1,5 @@ import logging +from collections.abc import Sequence from api.ApiAccessors import DataApiAccessor from api.parsers.MapParser import MapParser @@ -9,30 +10,55 @@ logger = logging.getLogger(__name__) -class ModApiConnector(DataApiAccessor): - def __init__(self, dispatch: Dispatcher) -> None: - super().__init__('/data/mod') - self.dispatch = dispatch +class VaultsApiConnector(DataApiAccessor): + def __init__(self, route: str) -> None: + super().__init__(route) + self._includes = ("latestVersion", "reviewsSummary") + + def _extend_query_options(self, query_options: dict) -> dict: + self._add_default_includes(query_options) + self._apply_default_filters(query_options) + return query_options + + def _copy_query_options(self, query_options: dict | None) -> dict: + query_options = query_options or {} + return query_options.copy() + + def request_data(self, query_options: dict | None = None) -> None: + query = self._copy_query_options(query_options) + self._extend_query_options(query) + self.get_by_query(query, self.parse_data) - def requestData(self, params: dict | None = None) -> None: - params = params or {} - self._add_default_include(params) - self._extend_filters(params) - self.get_by_query(params, self.handle_data) + def _add_default_includes(self, query_options: dict) -> dict: + return self._extend_includes(query_options, self._includes) - def _add_default_include(self, params: dict) -> dict: - params["include"] = ",".join(("latestVersion", "reviewsSummary", "uploader")) - return params + def _extend_includes(self, query_options: dict, to_include: Sequence[str]) -> dict: + cur_includes = query_options.get("include", "") + to_include_str = ",".join((cur_includes, *to_include)).removeprefix(",") + query_options["include"] = to_include_str + return query_options - def _extend_filters(self, params: dict) -> dict: + def _apply_default_filters(self, query_options: dict) -> dict: + cur_filters = query_options.get("filter", "") additional_filter = "latestVersion.hidden=='false'" - if cur_filters := params.get("filter", ""): - params["filter"] = f"{cur_filters};{additional_filter}" - else: - params["filter"] = additional_filter - return params + query_options["filter"] = ";".join((cur_filters, additional_filter)).removeprefix(";") + return query_options - def handle_data(self, message: dict) -> None: + def parse_data(self, message: dict) -> None: + raise NotImplementedError + + +class ModApiConnector(VaultsApiConnector): + def __init__(self, dispatch: Dispatcher) -> None: + super().__init__("/data/mod") + self.dispatch = dispatch + + def _extend_query_options(self, query_options: dict) -> dict: + super()._extend_query_options(query_options) + self._extend_includes(query_options, ["uploader"]) + return query_options + + def parse_data(self, message: dict) -> None: parsed_data = { "command": "modvault_info", "values": ModParser.parse_many(message["data"]), @@ -41,28 +67,14 @@ def handle_data(self, message: dict) -> None: self.dispatch.dispatch(parsed_data) -class MapApiConnector(DataApiAccessor): +class MapApiConnector(VaultsApiConnector): def __init__(self, dispatch: Dispatcher) -> None: super().__init__("/data/map") self.dispatch = dispatch - def requestData(self, params: dict | None = None) -> None: - params = params or {} - self._add_default_include(params) - self._extend_filters(params) - self.get_by_query(params, self.parse_data) - - def _extend_filters(self, params: dict) -> dict: - additional_filter = "latestVersion.hidden=='false'" - if cur_filters := params.get("filter", ""): - params["filter"] = f"{cur_filters};{additional_filter}" - else: - params["filter"] = additional_filter - return params - - def _add_default_include(self, params: dict) -> dict: - params["include"] = ",".join(("latestVersion", "reviewsSummary", "author")) - return params + def _extend_query_options(self, query_options: dict) -> dict: + super()._extend_query_options(query_options) + self._extend_includes(query_options, ["author"]) def parse_data(self, message: dict) -> None: prepared_data = { @@ -73,23 +85,20 @@ def parse_data(self, message: dict) -> None: self.dispatch.dispatch(prepared_data) -class MapPoolApiConnector(DataApiAccessor): +class MapPoolApiConnector(VaultsApiConnector): def __init__(self, dispatch: Dispatcher) -> None: - super().__init__('/data/mapPoolAssignment') + super().__init__("/data/mapPoolAssignment") self.dispatch = dispatch - - def requestData(self, params: dict | None) -> None: - params = params or {} - self.get_by_query(self._add_default_include(params), self.parse_data) - - def _add_default_include(self, params: dict) -> dict: - params["include"] = ",".join(( + self._includes = ( "mapVersion", "mapVersion.map", "mapVersion.map.author", "mapVersion.map.reviewsSummary", - )) - return params + ) + + def _extend_query_options(self, query_options: dict) -> dict: + self._add_default_includes(query_options) + return query_options def parse_data(self, message: dict) -> None: prepared_data = { diff --git a/src/vaults/vault.py b/src/vaults/vault.py index 9b621f571..55792e318 100644 --- a/src/vaults/vault.py +++ b/src/vaults/vault.py @@ -92,7 +92,7 @@ def goToPage(self, page: int) -> None: self.pageBox.setValue(page) self.pageNumber = self.pageBox.value() self.updateQuery(self.pageNumber) - self.apiConnector.requestData(self.searchQuery) + self.apiConnector.request_data(self.searchQuery) self.update_visibilities() def create_item(self, item_key: str) -> VaultListItem: From ad040107c289bda7daf10a4a1072446bec2a079b Mon Sep 17 00:00:00 2001 From: gatsik <74517072+Gatsik@users.noreply.github.com> Date: Wed, 24 Apr 2024 18:46:23 +0300 Subject: [PATCH 045/123] Do not use Dispatcher to transmit vaults' signals use bare pyqtSignal for that --- src/api/vaults_api.py | 37 +++++++++++++++------------------ src/client/connection.py | 24 ++++----------------- src/vaults/mapvault/mapvault.py | 10 ++++----- src/vaults/modvault/modvault.py | 4 ++-- 4 files changed, 28 insertions(+), 47 deletions(-) diff --git a/src/api/vaults_api.py b/src/api/vaults_api.py index 2fff9d01a..5ff34089b 100644 --- a/src/api/vaults_api.py +++ b/src/api/vaults_api.py @@ -1,16 +1,19 @@ import logging from collections.abc import Sequence +from PyQt6.QtCore import pyqtSignal + from api.ApiAccessors import DataApiAccessor from api.parsers.MapParser import MapParser from api.parsers.MapPoolAssignmentParser import MapPoolAssignmentParser from api.parsers.ModParser import ModParser -from client.connection import Dispatcher logger = logging.getLogger(__name__) class VaultsApiConnector(DataApiAccessor): + data_ready = pyqtSignal(dict) + def __init__(self, route: str) -> None: super().__init__(route) self._includes = ("latestVersion", "reviewsSummary") @@ -27,7 +30,7 @@ def _copy_query_options(self, query_options: dict | None) -> dict: def request_data(self, query_options: dict | None = None) -> None: query = self._copy_query_options(query_options) self._extend_query_options(query) - self.get_by_query(query, self.parse_data) + self.get_by_query(query, self.handle_response) def _add_default_includes(self, query_options: dict) -> dict: return self._extend_includes(query_options, self._includes) @@ -44,51 +47,47 @@ def _apply_default_filters(self, query_options: dict) -> dict: query_options["filter"] = ";".join((cur_filters, additional_filter)).removeprefix(";") return query_options - def parse_data(self, message: dict) -> None: - raise NotImplementedError + def parse_data(self, message: dict) -> dict: + return message + + def handle_response(self, message: dict) -> None: + self.data_ready.emit(self.parse_data(message)) class ModApiConnector(VaultsApiConnector): - def __init__(self, dispatch: Dispatcher) -> None: + def __init__(self) -> None: super().__init__("/data/mod") - self.dispatch = dispatch def _extend_query_options(self, query_options: dict) -> dict: super()._extend_query_options(query_options) self._extend_includes(query_options, ["uploader"]) return query_options - def parse_data(self, message: dict) -> None: - parsed_data = { - "command": "modvault_info", + def parse_data(self, message: dict) -> dict: + return { "values": ModParser.parse_many(message["data"]), "meta": message["meta"], } - self.dispatch.dispatch(parsed_data) class MapApiConnector(VaultsApiConnector): - def __init__(self, dispatch: Dispatcher) -> None: + def __init__(self) -> None: super().__init__("/data/map") - self.dispatch = dispatch def _extend_query_options(self, query_options: dict) -> dict: super()._extend_query_options(query_options) self._extend_includes(query_options, ["author"]) def parse_data(self, message: dict) -> None: - prepared_data = { - "command": "mapvault_info", + return { "values": MapParser.parse_many(message["data"]), "meta": message["meta"], } - self.dispatch.dispatch(prepared_data) class MapPoolApiConnector(VaultsApiConnector): - def __init__(self, dispatch: Dispatcher) -> None: + def __init__(self) -> None: super().__init__("/data/mapPoolAssignment") - self.dispatch = dispatch self._includes = ( "mapVersion", "mapVersion.map", @@ -101,9 +100,7 @@ def _extend_query_options(self, query_options: dict) -> dict: return query_options def parse_data(self, message: dict) -> None: - prepared_data = { - "command": "mapvault_info", + return { "values": MapPoolAssignmentParser.parse_many_to_maps(message["data"]), "meta": message["meta"], } - self.dispatch.dispatch(prepared_data) diff --git a/src/client/connection.py b/src/client/connection.py index f8928169b..b4b3b63f1 100644 --- a/src/client/connection.py +++ b/src/client/connection.py @@ -392,13 +392,11 @@ class LobbyInfo(QtCore.QObject): coopInfo = QtCore.pyqtSignal(dict) tutorialsInfo = QtCore.pyqtSignal(dict) modInfo = QtCore.pyqtSignal(dict) - modVaultInfo = QtCore.pyqtSignal(dict) replayVault = QtCore.pyqtSignal(dict) coopLeaderBoard = QtCore.pyqtSignal(dict) avatarList = QtCore.pyqtSignal(list) social = QtCore.pyqtSignal(dict) serverSession = QtCore.pyqtSignal(dict) - mapVaultInfo = QtCore.pyqtSignal(dict) aliasInfo = QtCore.pyqtSignal(dict) matchmakerQueueInfo = QtCore.pyqtSignal(dict) @@ -418,24 +416,10 @@ def __init__(self, dispatcher, gameset, playerset): self._dispatcher["session"] = self._simple_emit(self.serverSession) self._dispatcher["alias_info"] = self._simple_emit(self.aliasInfo) self._dispatcher["modvault_list_info"] = self.handle_modvault_list_info - self._dispatcher["modvault_info"] = self._simple_emit( - self.modVaultInfo, - ) - self._dispatcher["mapvault_info"] = self._simple_emit( - self.mapVaultInfo, - ) - self._dispatcher["updated_achievements"] = ( - self.handle_updated_achievements - ) - self._dispatcher["coop_leaderboard"] = self._simple_emit( - self.coopLeaderBoard, - ) - self._dispatcher["tutorials_info"] = self._simple_emit( - self.tutorialsInfo, - ) - self._dispatcher["matchmaker_queue_info"] = self._simple_emit( - self.matchmakerQueueInfo, - ) + self._dispatcher["updated_achievements"] = (self.handle_updated_achievements) + self._dispatcher["coop_leaderboard"] = self._simple_emit(self.coopLeaderBoard) + self._dispatcher["tutorials_info"] = self._simple_emit(self.tutorialsInfo) + self._dispatcher["matchmaker_queue_info"] = self._simple_emit(self.matchmakerQueueInfo) self._gameset = gameset self._playerset = playerset diff --git a/src/vaults/mapvault/mapvault.py b/src/vaults/mapvault/mapvault.py index ba56e90da..8ae503926 100644 --- a/src/vaults/mapvault/mapvault.py +++ b/src/vaults/mapvault/mapvault.py @@ -37,7 +37,6 @@ def __init__(self, client: ClientWindow, *args, **kwargs) -> None: logger.debug("Map Vault tab instantiating") self.itemList.itemDoubleClicked.connect(self.itemClicked) - self.client.lobby_info.mapVaultInfo.connect(self.mapInfo) self.client.authorized.connect(self.busy_entered) self.installed_maps = maps.getUserMaps() @@ -47,10 +46,11 @@ def __init__(self, client: ClientWindow, *args, **kwargs) -> None: for type_ in ["Unranked Only", "Ranked Only", "Installed"]: self.ShowTypeList.addItem(type_) - self.mapApiConnector = MapApiConnector(self.client.lobby_dispatch) - self.mapPoolApiConnector = MapPoolApiConnector( - self.client.lobby_dispatch, - ) + self.mapApiConnector = MapApiConnector() + self.mapPoolApiConnector = MapPoolApiConnector() + self.mapApiConnector.data_ready.connect(self.mapInfo) + self.mapPoolApiConnector.data_ready.connect(self.mapInfo) + self.apiConnector = self.mapApiConnector self.items_uid = "folderName" diff --git a/src/vaults/modvault/modvault.py b/src/vaults/modvault/modvault.py index 6a435da9d..1d3e8c0bb 100644 --- a/src/vaults/modvault/modvault.py +++ b/src/vaults/modvault/modvault.py @@ -66,14 +66,14 @@ def __init__(self, client, *args, **kwargs): self.itemList.itemDoubleClicked.connect(self.modClicked) self.UIButton.clicked.connect(self.openUIModForm) - self.client.lobby_info.modVaultInfo.connect(self.modInfo) self.uids = [mod.uid for mod in utils.getInstalledMods()] for type_ in ["UI Only", "Sim Only", "Uploaded by You", "Installed"]: self.ShowTypeList.addItem(type_) - self.apiConnector = ModApiConnector(self.client.lobby_dispatch) + self.apiConnector = ModApiConnector() + self.apiConnector.data_ready.connect(self.modInfo) self.items_uid = "uid" From 1db3a0232d15848f32838098e4656bfd591d4437 Mon Sep 17 00:00:00 2001 From: gatsik <74517072+Gatsik@users.noreply.github.com> Date: Thu, 25 Apr 2024 22:20:35 +0300 Subject: [PATCH 046/123] Do not use Dispatcher to dispatch data from API i think it was originally supposed to dispatch server messages also, move some common parts of ApiAcessors into their base class --- src/api/ApiAccessors.py | 14 ++++ src/api/featured_mod_api.py | 21 ++---- src/api/featured_mod_updater.py | 9 +-- src/api/matchmaker_queue_api.py | 18 ++--- src/api/player_api.py | 34 ++------- src/api/replaysapi.py | 22 +----- src/api/sim_mod_updater.py | 3 - src/api/stats_api.py | 38 ++-------- src/api/vaults_api.py | 18 +---- src/client/aliasviewer.py | 6 +- src/client/connection.py | 13 ---- src/games/_gameswidget.py | 5 +- src/games/automatchframe.py | 12 +-- src/replays/_replayswidget.py | 40 +++++----- src/stats/_statswidget.py | 90 ++++------------------- src/stats/leaderboard_widget.py | 39 ++++------ src/stats/models/leaderboardtablemodel.py | 6 +- 17 files changed, 103 insertions(+), 285 deletions(-) diff --git a/src/api/ApiAccessors.py b/src/api/ApiAccessors.py index 874a02cb4..bc17ed6c5 100644 --- a/src/api/ApiAccessors.py +++ b/src/api/ApiAccessors.py @@ -1,5 +1,7 @@ import logging +from PyQt6.QtCore import pyqtSignal + from api.ApiBase import ApiBase logger = logging.getLogger(__name__) @@ -18,6 +20,8 @@ def __init__(self, route: str = "") -> None: class DataApiAccessor(ApiAccessor): + data_ready = pyqtSignal(dict) + def parse_message(self, message: dict) -> dict: included = self.parseIncluded(message) result = {} @@ -87,3 +91,13 @@ def parseMeta(self, message: dict) -> dict: if "meta" in message: return message["meta"] return {} + + def requestData(self, query_dict: dict | None = None) -> None: + query_dict = query_dict or {} + self.get_by_query(query_dict, self.handle_response) + + def prepare_data(self, message: dict) -> dict: + return message + + def handle_response(self, message: dict) -> None: + self.data_ready.emit(self.prepare_data(message)) diff --git a/src/api/featured_mod_api.py b/src/api/featured_mod_api.py index 203aad324..1577fc4cb 100644 --- a/src/api/featured_mod_api.py +++ b/src/api/featured_mod_api.py @@ -1,27 +1,18 @@ import logging from api.ApiAccessors import DataApiAccessor -from client.connection import Dispatcher logger = logging.getLogger(__name__) class FeaturedModApiConnector(DataApiAccessor): - def __init__(self, dispatch: Dispatcher) -> None: + def __init__(self) -> None: super().__init__('/data/featuredMod') - self.dispatch = dispatch - def requestData(self) -> None: - self.get_by_query({}, self.handleData) - - def handleData(self, message: dict) -> None: - preparedData = { - "command": "mod_info_api", - "values": [], - } + def prepare_data(self, message: dict) -> None: + values = [] for mod in message["data"]: - preparedMod = { - "command": "mod_info_api", + prepared_mod = { "name": mod["technicalName"], "fullname": mod["displayName"], "publish": mod.get("visible", False), @@ -31,5 +22,5 @@ def handleData(self, message: dict) -> None: "No description provided", ), } - preparedData["values"].append(preparedMod) - self.dispatch.dispatch(preparedData) + values.append(prepared_mod) + return {"values": values} diff --git a/src/api/featured_mod_updater.py b/src/api/featured_mod_updater.py index e1d2cbd6e..3a9b96163 100644 --- a/src/api/featured_mod_updater.py +++ b/src/api/featured_mod_updater.py @@ -10,10 +10,7 @@ def __init__(self, mod_id: int, version: str) -> None: super().__init__('/featuredMods/{}/files/{}'.format(mod_id, version)) self.featuredModFiles = [] - def requestData(self) -> None: - self.get_by_query({}, self.handleData) - - def handleData(self, message): + def handle_response(self, message): self.featuredModFiles = message["data"] def getFiles(self): @@ -27,10 +24,6 @@ def __init__(self) -> None: super().__init__('/data/featuredMod') self.featuredModId = 0 - def requestData(self, queryDict: dict | None = None) -> None: - queryDict = queryDict or {} - self.get_by_query(queryDict, self.handleData) - def handleFeaturedModId(self, message): self.featuredModId = message['data'][0]['id'] diff --git a/src/api/matchmaker_queue_api.py b/src/api/matchmaker_queue_api.py index e4ca62d43..576a85a31 100644 --- a/src/api/matchmaker_queue_api.py +++ b/src/api/matchmaker_queue_api.py @@ -1,22 +1,16 @@ import logging from api.ApiAccessors import DataApiAccessor -from client.connection import Dispatcher logger = logging.getLogger(__name__) -class matchmakerQueueApiConnector(DataApiAccessor): - def __init__(self, dispatch: Dispatcher) -> None: +class MatchmakerQueueApiConnector(DataApiAccessor): + def __init__(self) -> None: super().__init__('/data/matchmakerQueue') - self.dispatch = dispatch - def requestData(self, queryDict: dict | None = None) -> None: - queryDict = queryDict or {} - self.get_by_query(queryDict, self.handleData) - - def handleData(self, message: dict) -> None: - preparedData = { + def prepare_data(self, message: dict) -> None: + prepared_data = { "command": "matchmaker_queue_info", "values": [], "meta": message["meta"], @@ -28,5 +22,5 @@ def handleData(self, message: dict) -> None: "id": queue["id"], "leaderboardId": queue["leaderboard"]["id"], } - preparedData["values"].append(preparedQueue) - self.dispatch.dispatch(preparedData) + prepared_data["values"].append(preparedQueue) + return prepared_data diff --git a/src/api/player_api.py b/src/api/player_api.py index 235a22638..5253c8854 100644 --- a/src/api/player_api.py +++ b/src/api/player_api.py @@ -1,34 +1,17 @@ import logging +from PyQt6.QtCore import pyqtSignal + from api.ApiAccessors import DataApiAccessor -from client.connection import Dispatcher logger = logging.getLogger(__name__) class PlayerApiConnector(DataApiAccessor): - def __init__(self, dispatch: Dispatcher) -> None: + alias_info = pyqtSignal(dict) + + def __init__(self) -> None: super().__init__('/data/player') - self.dispatch = dispatch - - def requestDataForLeaderboard( - self, - leaderboardName: str, - queryDict: dict | None = None, - ) -> None: - queryDict = queryDict or {} - self.leaderboardName = leaderboardName - self.get_by_query(queryDict, self.handleDataForLeaderboard) - - def handleDataForLeaderboard(self, message: dict) -> None: - preparedData = dict( - command='stats', - type='player', - leaderboardName=self.leaderboardName, - values=message['data'], - meta=message['meta'], - ) - self.dispatch.dispatch(preparedData) def requestDataForAliasViewer(self, nameToFind: str) -> None: queryDict = { @@ -42,9 +25,4 @@ def requestDataForAliasViewer(self, nameToFind: str) -> None: self.get_by_query(queryDict, self.handleDataForAliasViewer) def handleDataForAliasViewer(self, message: dict) -> None: - preparedData = dict( - command='alias_info', - values=message['data'], - meta=message['meta'], - ) - self.dispatch.dispatch(preparedData) + self.alias_info.emit(message) diff --git a/src/api/replaysapi.py b/src/api/replaysapi.py index f76ebd17e..a9a8e99b7 100644 --- a/src/api/replaysapi.py +++ b/src/api/replaysapi.py @@ -1,30 +1,10 @@ import logging from api.ApiAccessors import DataApiAccessor -from client.connection import Dispatcher logger = logging.getLogger(__name__) class ReplaysApiConnector(DataApiAccessor): - def __init__(self, dispatch: Dispatcher) -> None: + def __init__(self) -> None: super().__init__('/data/game') - self.dispatch = dispatch - - def requestData(self, params: dict) -> None: - self.get_by_query(params, self.handleData) - - def handleData(self, message): - preparedData = dict( - command="replay_vault", - action="search_result", - replays={}, - featuredMods={}, - maps={}, - players={}, - playerStats={}, - ) - - preparedData["replays"] = message["data"] - - self.dispatch.dispatch(preparedData) diff --git a/src/api/sim_mod_updater.py b/src/api/sim_mod_updater.py index a38b44456..9b5bacbd2 100644 --- a/src/api/sim_mod_updater.py +++ b/src/api/sim_mod_updater.py @@ -10,9 +10,6 @@ def __init__(self) -> None: super().__init__('/data/modVersion') self.mod_url = "" - def requestData(self, queryDict: dict) -> None: - self.get_by_query(queryDict, self.handleData) - def get_url_from_message(self, message: dict) -> str: self.mod_url = message["data"][0]["downloadUrl"] diff --git a/src/api/stats_api.py b/src/api/stats_api.py index ad8f1fc0c..af4301ab1 100644 --- a/src/api/stats_api.py +++ b/src/api/stats_api.py @@ -1,46 +1,20 @@ import logging from api.ApiAccessors import DataApiAccessor -from client.connection import Dispatcher logger = logging.getLogger(__name__) class LeaderboardRatingApiConnector(DataApiAccessor): - def __init__(self, dispatch: Dispatcher, leaderboardName: str) -> None: + def __init__(self, leaderboard_name: str) -> None: super().__init__('/data/leaderboardRating') - self.dispatch = dispatch - self.leadeboardName = leaderboardName + self.leaderboard_name = leaderboard_name - def requestData(self, queryDict: dict | None = None) -> None: - queryDict = queryDict or {} - self.get_by_query(queryDict, self.handleData) - - def handleData(self, message: dict) -> None: - preparedData = dict( - command='stats', - type='leaderboardRating', - leaderboardName=self.leadeboardName, - values=message["data"], - meta=message["meta"], - ) - self.dispatch.dispatch(preparedData) + def prepare_data(self, message: dict) -> None: + message["leaderboard"] = self.leaderboard_name + return message class LeaderboardApiConnector(DataApiAccessor): - def __init__(self, dispatch: Dispatcher | None = None) -> None: + def __init__(self) -> None: super().__init__('/data/leaderboard') - self.dispatch = dispatch - - def requestData(self, queryDict: dict | None = None) -> None: - queryDict = queryDict or {} - self.get_by_query(queryDict, self.handleData) - - def handleData(self, message: dict) -> None: - preparedData = dict( - command='stats', - type='leaderboard', - values=message["data"], - meta=message["meta"], - ) - self.dispatch.dispatch(preparedData) diff --git a/src/api/vaults_api.py b/src/api/vaults_api.py index 5ff34089b..c1cf6504a 100644 --- a/src/api/vaults_api.py +++ b/src/api/vaults_api.py @@ -1,8 +1,6 @@ import logging from collections.abc import Sequence -from PyQt6.QtCore import pyqtSignal - from api.ApiAccessors import DataApiAccessor from api.parsers.MapParser import MapParser from api.parsers.MapPoolAssignmentParser import MapPoolAssignmentParser @@ -12,8 +10,6 @@ class VaultsApiConnector(DataApiAccessor): - data_ready = pyqtSignal(dict) - def __init__(self, route: str) -> None: super().__init__(route) self._includes = ("latestVersion", "reviewsSummary") @@ -47,12 +43,6 @@ def _apply_default_filters(self, query_options: dict) -> dict: query_options["filter"] = ";".join((cur_filters, additional_filter)).removeprefix(";") return query_options - def parse_data(self, message: dict) -> dict: - return message - - def handle_response(self, message: dict) -> None: - self.data_ready.emit(self.parse_data(message)) - class ModApiConnector(VaultsApiConnector): def __init__(self) -> None: @@ -63,9 +53,9 @@ def _extend_query_options(self, query_options: dict) -> dict: self._extend_includes(query_options, ["uploader"]) return query_options - def parse_data(self, message: dict) -> dict: + def prepare_data(self, message: dict) -> dict: return { - "values": ModParser.parse_many(message["data"]), + "values": ModParser.parse_many(message["data"]), "meta": message["meta"], } @@ -78,7 +68,7 @@ def _extend_query_options(self, query_options: dict) -> dict: super()._extend_query_options(query_options) self._extend_includes(query_options, ["author"]) - def parse_data(self, message: dict) -> None: + def prepare_data(self, message: dict) -> None: return { "values": MapParser.parse_many(message["data"]), "meta": message["meta"], @@ -99,7 +89,7 @@ def _extend_query_options(self, query_options: dict) -> dict: self._add_default_includes(query_options) return query_options - def parse_data(self, message: dict) -> None: + def prepare_data(self, message: dict) -> None: return { "values": MapPoolAssignmentParser.parse_many_to_maps(message["data"]), "meta": message["meta"], diff --git a/src/client/aliasviewer.py b/src/client/aliasviewer.py index 58a2d1355..6996511ce 100644 --- a/src/client/aliasviewer.py +++ b/src/client/aliasviewer.py @@ -14,8 +14,8 @@ class AliasViewer: def __init__(self, client, alias_formatter): self.client = client self.formatter = alias_formatter - self.api_connector = PlayerApiConnector(self.client.lobby_dispatch) - self.client.lobby_info.aliasInfo.connect(self.process_alias_info) + self.api_connector = PlayerApiConnector() + self.api_connector.alias_info.connect(self.process_alias_info) self.name_to_find = "" self.searching = False self.timer = QTimer() @@ -37,7 +37,7 @@ def process_alias_info(self, message): self.stop_alias_search() player_aliases, other_users = [], [] - for player in message["values"]: + for player in message["data"]: if player["login"].lower() == self.name_to_find.lower(): player_aliases.append({ "name": player["login"], diff --git a/src/client/connection.py b/src/client/connection.py index b4b3b63f1..e1257b76e 100644 --- a/src/client/connection.py +++ b/src/client/connection.py @@ -391,14 +391,11 @@ class LobbyInfo(QtCore.QObject): statsInfo = QtCore.pyqtSignal(dict) coopInfo = QtCore.pyqtSignal(dict) tutorialsInfo = QtCore.pyqtSignal(dict) - modInfo = QtCore.pyqtSignal(dict) replayVault = QtCore.pyqtSignal(dict) coopLeaderBoard = QtCore.pyqtSignal(dict) avatarList = QtCore.pyqtSignal(list) social = QtCore.pyqtSignal(dict) serverSession = QtCore.pyqtSignal(dict) - aliasInfo = QtCore.pyqtSignal(dict) - matchmakerQueueInfo = QtCore.pyqtSignal(dict) def __init__(self, dispatcher, gameset, playerset): QtCore.QObject.__init__(self) @@ -406,20 +403,15 @@ def __init__(self, dispatcher, gameset, playerset): self._dispatcher = dispatcher self._dispatcher["stats"] = self._simple_emit(self.statsInfo) self._dispatcher["coop_info"] = self._simple_emit(self.coopInfo) - self._dispatcher["mod_info_api"] = self._simple_emit(self.modInfo) - self._dispatcher["mod_info"] = lambda _: None self._dispatcher["game_info"] = self.handle_game_info self._dispatcher["replay_vault"] = self._simple_emit(self.replayVault) self._dispatcher["avatar"] = self.handle_avatar self._dispatcher["admin"] = self.handle_admin self._dispatcher["social"] = self._simple_emit(self.social) self._dispatcher["session"] = self._simple_emit(self.serverSession) - self._dispatcher["alias_info"] = self._simple_emit(self.aliasInfo) - self._dispatcher["modvault_list_info"] = self.handle_modvault_list_info self._dispatcher["updated_achievements"] = (self.handle_updated_achievements) self._dispatcher["coop_leaderboard"] = self._simple_emit(self.coopLeaderBoard) self._dispatcher["tutorials_info"] = self._simple_emit(self.tutorialsInfo) - self._dispatcher["matchmaker_queue_info"] = self._simple_emit(self.matchmakerQueueInfo) self._gameset = gameset self._playerset = playerset @@ -454,11 +446,6 @@ def _update_game(self, m): else: self._gameset[uid].update(**m) - def handle_modvault_list_info(self, message): - modList = message["modList"] - for mod in modList: - self.modVaultInfo.emit(mod) - def handle_avatar(self, message): if "avatarlist" in message: self.avatarList.emit(message["avatarlist"]) diff --git a/src/games/_gameswidget.py b/src/games/_gameswidget.py index 72283fe59..650778b8c 100644 --- a/src/games/_gameswidget.py +++ b/src/games/_gameswidget.py @@ -97,7 +97,8 @@ def __init__( self._game_model = CustomGameFilterModel(self._me, game_model) self._game_launcher = game_launcher - self.apiConnector = FeaturedModApiConnector(self.client.lobby_dispatch) + self.apiConnector = FeaturedModApiConnector() + self.apiConnector.data_ready.connect(self.processModInfo) self.gameview = gameview_builder(self._game_model, self.gameList) self.gameview.game_double_clicked.connect(self.gameDoubleClicked) @@ -106,8 +107,6 @@ def __init__( self.ispassworded = False self.party = None - self.client.lobby_info.modInfo.connect(self.processModInfo) - self.client.matchmaker_info.connect(self.handleMatchmakerInfo) self.client.game_enter.connect(self.stopSearch) self.client.viewing_replay.connect(self.stopSearch) diff --git a/src/games/automatchframe.py b/src/games/automatchframe.py index 2b8f6cec5..2a2aa75ea 100644 --- a/src/games/automatchframe.py +++ b/src/games/automatchframe.py @@ -10,7 +10,7 @@ import fa import util -from api.matchmaker_queue_api import matchmakerQueueApiConnector +from api.matchmaker_queue_api import MatchmakerQueueApiConnector from config import Settings from fa.factions import Factions @@ -89,13 +89,9 @@ def __init__( self.secondsToAutomatch = 0 self.ratingType = "" - self.client.lobby_info.matchmakerQueueInfo.connect( - self.handleApiQueueInfo, - ) - self.apiConnector = matchmakerQueueApiConnector( - self.client.lobby_dispatch, - ) - self.apiConnector.requestData(queryDict=dict(include="leaderboard")) + self.apiConnector = MatchmakerQueueApiConnector() + self.apiConnector.data_ready.connect(self.handleApiQueueInfo) + self.apiConnector.requestData({"include": "leaderboard"}) title = self.queueName.replace("_", " ").capitalize() self.automatchTitle.setText(title) diff --git a/src/replays/_replayswidget.py b/src/replays/_replayswidget.py index 8eb0288ae..d7ad42e63 100644 --- a/src/replays/_replayswidget.py +++ b/src/replays/_replayswidget.py @@ -683,8 +683,8 @@ def __init__(self, widget, dispatcher, client, gameset, playerset): self.onlineReplays = {} self.selectedReplay = None - self.apiConnector = ReplaysApiConnector(self._dispatcher) - self.client.lobby_info.replayVault.connect(self.replayVault) + self.apiConnector = ReplaysApiConnector() + self.apiConnector.data_ready.connect(self.process_replays_data) self.replayDownload = QNetworkAccessManager() self.replayDownload.finished.connect(self.onDownloadFinished) self.toolboxHandler = ReplayToolboxHandler( @@ -1038,27 +1038,25 @@ def onDownloadFinished(self, reply): faf_replay.close() replay(os.path.join(util.CACHE_DIR, "temp.fafreplay")) - def replayVault(self, message): - action = message["action"] + def process_replays_data(self, message: dict) -> None: self.stopSearchVault() self._w.replayInfos.clear() - if action == "search_result": - self.onlineReplays = {} - replays = message["replays"] - for replay_item in replays: - uid = int(replay_item["id"]) - if uid not in self.onlineReplays: - self.onlineReplays[uid] = ReplayItem(uid, self._w) - self.onlineReplays[uid].update(replay_item, self.client) - self.updateOnlineTree() - - if len(message["replays"]) == 0: - self._w.searchInfoLabel.setText( - "No replays found", - ) - self._w.advSearchInfoLabel.setText( - "No replays found", - ) + self.onlineReplays = {} + replays = message["data"] + for replay_item in replays: + uid = int(replay_item["id"]) + if uid not in self.onlineReplays: + self.onlineReplays[uid] = ReplayItem(uid, self._w) + self.onlineReplays[uid].update(replay_item, self.client) + self.updateOnlineTree() + + if len(message["data"]) == 0: + self._w.searchInfoLabel.setText( + "No replays found", + ) + self._w.advSearchInfoLabel.setText( + "No replays found", + ) def updateOnlineTree(self): self.selectedReplay = None # clear, it won't be part of the new tree diff --git a/src/stats/_statswidget.py b/src/stats/_statswidget.py index b8f7f494e..a13bef632 100644 --- a/src/stats/_statswidget.py +++ b/src/stats/_statswidget.py @@ -29,8 +29,6 @@ def __init__(self, client): self.client = client - self.client.lobby_info.statsInfo.connect(self.processStatsInfos) - self.selected_player = None self.selected_player_loaded = False self.leagues.currentChanged.connect(self.leagueUpdate) @@ -56,8 +54,9 @@ def __init__(self, client): # setup other tabs - self.apiConnector = LeaderboardApiConnector(self.client.lobby_dispatch) - self.apiConnector.requestData(dict(sort="id")) + self.apiConnector = LeaderboardApiConnector() + self.apiConnector.data_ready.connect(self.process_leaderboards_info) + self.apiConnector.requestData({"sort": "id"}) # hiding some non-functional tabs self.removeTab(self.indexOf(self.ladderTab)) @@ -242,78 +241,19 @@ def leaderboardsTabChanged(self, curr): if self.leaderboards.widget(curr) is not None: self.leaderboards.widget(curr).entered() - @QtCore.pyqtSlot(dict) - def processStatsInfos(self, message): - - typeStat = message["type"] - if typeStat == "divisions": - self.currentLeague = message["league"] - tab = self.currentLeague - 1 - - if tab not in self.pagesDivisions: - self.pagesDivisions[tab] = self.createDivisionsTabs( - message["values"], - ) - leagueTab = self.leagues.widget(tab).findChild( - QtWidgets.QTabWidget, "league" + str(tab), - ) - leagueTab.widget(1).layout().addWidget( - self.pagesDivisions[tab], - ) - - elif typeStat == "division_table": - self.currentLeague = message["league"] - self.currentDivision = message["division"] - - if self.currentLeague in self.pagesDivisionsResults: - if ( - self.currentDivision - in self.pagesDivisionsResults[self.currentLeague] - ): - self.createResults( - message["values"], - ( - self.pagesDivisionsResults[self.currentLeague] - [self.currentDivision] - ), - ) - - elif typeStat == "league_table": - self.currentLeague = message["league"] - tab = self.currentLeague - 1 - if tab not in self.pagesAllLeagues: - table = QtWidgets.QTextBrowser() - self.pagesAllLeagues[tab] = self.createResults( - message["values"], table, - ) - leagueTab = self.leagues.widget(tab).findChild( - QtWidgets.QTabWidget, "league" + str(tab), - ) - leagueTab.currentChanged.connect(self.divisionsUpdate) - leagueTab.widget(0).layout().addWidget( - self.pagesAllLeagues[tab], - ) - - elif typeStat == "leaderboard": - self.leaderboardNames.clear() - for value in message["values"]: - self.leaderboardNames.append(value["technicalName"]) - for i in range(len(self.leaderboardNames)): - self.leaderboards.insertTab( - i, - LeaderboardWidget( - self.client, self, self.leaderboardNames[i], - ), - self.leaderboardNames[i].capitalize().replace("_", " "), - ) - self.client.replays.leaderboardList.addItem( - self.leaderboardNames[i], - ) - - self.leaderboards.setCurrentIndex(1) - self.leaderboards.currentChanged.connect( - self.leaderboardsTabChanged, + def process_leaderboards_info(self, message: dict) -> None: + self.leaderboardNames.clear() + for value in message["data"]: + self.leaderboardNames.append(value["technicalName"]) + for index, name in enumerate(self.leaderboardNames): + self.leaderboards.insertTab( + index, + LeaderboardWidget(self.client, self, name), + name.capitalize().replace("_", " "), ) + self.client.replays.leaderboardList.addItem(name) + self.leaderboards.setCurrentIndex(1) + self.leaderboards.currentChanged.connect(self.leaderboardsTabChanged) @QtCore.pyqtSlot() def busy_entered(self): diff --git a/src/stats/leaderboard_widget.py b/src/stats/leaderboard_widget.py index a8a8d1636..a10dc476d 100644 --- a/src/stats/leaderboard_widget.py +++ b/src/stats/leaderboard_widget.py @@ -41,12 +41,9 @@ def __init__( self.client = client self.parent = parent self.leaderboardName = leaderboardName - self.apiConnector = LeaderboardRatingApiConnector( - self.client.lobby_dispatch, self.leaderboardName, - ) - self.playerApiConnector = PlayerApiConnector( - self.client.lobby_dispatch, - ) + self.apiConnector = LeaderboardRatingApiConnector(self.leaderboardName) + self.apiConnector.data_ready.connect(self.process_rating_info) + self.playerApiConnector = PlayerApiConnector() self.onlyActive = True self.pageNumber = 1 self.totalPages = 1 @@ -77,7 +74,6 @@ def __init__( self.pageBox.valueChanged.connect(self.checkTotalPages) self.refreshButton.clicked.connect(self.refreshLeaderboard) - self.client.lobby_info.statsInfo.connect(self.processStatsInfos) self.findInPageLine.textChanged.connect(self.findEntry) self.findInPageLine.returnPressed.connect( lambda: self.findEntry(self.findInPageLine.text()), @@ -169,16 +165,12 @@ def showColumns(self): self.showColumnCheckBoxes[index].blockSignals(False) - def processStatsInfos(self, message): - if message["type"] == "leaderboardRating": - if message["leaderboardName"] == self.leaderboardName: - self.createLeaderboard(message) - self.processMeta(message["meta"]) - self.resetLoading() - self.timer.stop() - elif message["type"] == "player": - if message["leaderboardName"] == self.leaderboardName: - self.createPlayerCompleter(message) + def process_rating_info(self, message: dict) -> None: + if message["leaderboard"] == self.leaderboardName: + self.createLeaderboard(message) + self.processMeta(message["meta"]) + self.resetLoading() + self.timer.stop() def createLeaderboard(self, data): self.model = LeaderboardTableModel(data) @@ -220,20 +212,15 @@ def findEntry(self, text): self.tableView.selectRow(self.model.logins.index(row)) break - def searchPlayer(self): + def searchPlayer(self) -> None: query = { "filter": 'login=="{}*"'.format(self.searchPlayerLine.text()), "page[size]": 10, } - self.playerApiConnector.requestDataForLeaderboard( - self.leaderboardName, query, - ) - - def createPlayerCompleter(self, message): - logins = [] - for value in message["values"]: - logins.append(value["login"]) + self.playerApiConnector.get_by_query(query, self.createPlayerCompleter) + def createPlayerCompleter(self, message: dict) -> None: + logins = [player["login"] for player in message["data"]] self.searchPlayerLine.set_completion_list(logins) completer = QtWidgets.QCompleter( sorted(logins, key=lambda login: login.lower()), diff --git a/src/stats/models/leaderboardtablemodel.py b/src/stats/models/leaderboardtablemodel.py index 18b04dd69..c01c95d06 100644 --- a/src/stats/models/leaderboardtablemodel.py +++ b/src/stats/models/leaderboardtablemodel.py @@ -9,14 +9,14 @@ def __init__(self, data=None): QAbstractTableModel.__init__(self) self.load_data(data) - def load_data(self, data): - self.values = data["values"] + def load_data(self, data: dict) -> None: + self.values = data["data"] self.meta = data["meta"] self.logins = [] for value in self.values: self.logins.append(value["player"]["login"]) self.column_count = 9 - self.row_count = len(data["values"]) + self.row_count = len(data["data"]) def rowCount(self, parent=QModelIndex()): return self.row_count From 53bdebcd36d6aeeca7ab265809af6def82e50ade Mon Sep 17 00:00:00 2001 From: gatsik <74517072+Gatsik@users.noreply.github.com> Date: Thu, 25 Apr 2024 22:28:49 +0300 Subject: [PATCH 047/123] pre-commit autoupdate --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e0eb7f6ad..2a40de6c3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 + rev: v4.6.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer From 4426540b88de2559b9e585bcaa21fdc99f843efe Mon Sep 17 00:00:00 2001 From: gatsik <74517072+Gatsik@users.noreply.github.com> Date: Fri, 26 Apr 2024 23:05:31 +0300 Subject: [PATCH 048/123] Create api model for FeaturedMod --- src/api/featured_mod_api.py | 16 ++-------------- src/api/models/FeaturedMod.py | 10 ++++++++++ src/api/parsers/FeaturedModParser.py | 21 +++++++++++++++++++++ src/games/_gameswidget.py | 14 +++++++------- src/games/moditem.py | 18 +++++++++--------- 5 files changed, 49 insertions(+), 30 deletions(-) create mode 100644 src/api/models/FeaturedMod.py create mode 100644 src/api/parsers/FeaturedModParser.py diff --git a/src/api/featured_mod_api.py b/src/api/featured_mod_api.py index 1577fc4cb..164e9332d 100644 --- a/src/api/featured_mod_api.py +++ b/src/api/featured_mod_api.py @@ -1,6 +1,7 @@ import logging from api.ApiAccessors import DataApiAccessor +from api.parsers.FeaturedModParser import FeaturedModParser logger = logging.getLogger(__name__) @@ -10,17 +11,4 @@ def __init__(self) -> None: super().__init__('/data/featuredMod') def prepare_data(self, message: dict) -> None: - values = [] - for mod in message["data"]: - prepared_mod = { - "name": mod["technicalName"], - "fullname": mod["displayName"], - "publish": mod.get("visible", False), - "order": mod.get("order", 0), - "desc": mod.get( - "description", - "No description provided", - ), - } - values.append(prepared_mod) - return {"values": values} + return {"values": FeaturedModParser.parse_many(message["data"])} diff --git a/src/api/models/FeaturedMod.py b/src/api/models/FeaturedMod.py new file mode 100644 index 000000000..abb8f9030 --- /dev/null +++ b/src/api/models/FeaturedMod.py @@ -0,0 +1,10 @@ +from dataclasses import dataclass + + +@dataclass +class FeaturedMod: + name: str + fullname: str + visible: bool + order: int + description: str diff --git a/src/api/parsers/FeaturedModParser.py b/src/api/parsers/FeaturedModParser.py new file mode 100644 index 000000000..2ebfcdd9a --- /dev/null +++ b/src/api/parsers/FeaturedModParser.py @@ -0,0 +1,21 @@ +from api.models.FeaturedMod import FeaturedMod + + +class FeaturedModParser: + + @staticmethod + def parse(data: dict) -> FeaturedMod: + return FeaturedMod( + name=data["technicalName"], + fullname=data["displayName"], + visible=data.get("visible", False), + order=data.get("order", 0), + description=data.get( + "description", + "No description provided", + ), + ) + + @staticmethod + def parse_many(data: list[dict]) -> list[FeaturedMod]: + return [FeaturedModParser.parse(info) for info in data] diff --git a/src/games/_gameswidget.py b/src/games/_gameswidget.py index 650778b8c..9d1a2b057 100644 --- a/src/games/_gameswidget.py +++ b/src/games/_gameswidget.py @@ -98,7 +98,7 @@ def __init__( self._game_launcher = game_launcher self.apiConnector = FeaturedModApiConnector() - self.apiConnector.data_ready.connect(self.processModInfo) + self.apiConnector.data_ready.connect(self.process_mod_info) self.gameview = gameview_builder(self._game_model, self.gameList) self.gameview.game_double_clicked.connect(self.gameDoubleClicked) @@ -172,14 +172,14 @@ def onLogOut(self): self.matchmakerFramesInitialized = False @pyqtSlot(dict) - def processModInfo(self, message): + def process_mod_info(self, message: dict) -> None: """ Slot that interprets and propagates mod_info messages into the mod list """ - for value in message["values"]: - mod = value['name'] + for featured_mod in message["values"]: + mod = featured_mod.name old_mod = self.mods.get(mod, None) - self.mods[mod] = ModItem(value) + self.mods[mod] = ModItem(featured_mod) if old_mod: if mod in mod_invisible: @@ -191,12 +191,12 @@ def processModInfo(self, message): if self.client.replays.modList.itemText(i) == old_mod.mod: self.client.replays.modList.removeItem(i) - if value["publish"]: + if featured_mod.visible: self.modList.addItem(self.mods[mod]) else: mod_invisible[mod] = self.mods[mod] - self.client.replays.modList.addItem(value["name"]) + self.client.replays.modList.addItem(mod) @pyqtSlot(int) def togglePrivateGames(self, state): diff --git a/src/games/moditem.py b/src/games/moditem.py index 93fd64680..7142083f1 100644 --- a/src/games/moditem.py +++ b/src/games/moditem.py @@ -5,6 +5,7 @@ import client import util +from api.models.FeaturedMod import FeaturedMod # Maps names of featured mods to ModItem objects. mods = {} @@ -18,18 +19,17 @@ class ModItem(QtWidgets.QListWidgetItem): - def __init__(self, message, *args, **kwargs): - QtWidgets.QListWidgetItem.__init__(self, *args, **kwargs) + def __init__(self, mod_info: FeaturedMod, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) - self.mod = message["name"] - self.order = message.get("order", 0) - self.name = message["fullname"] + self.mod = mod_info.name + self.order = mod_info.order + self.name = mod_info.fullname # Load Icon and Tooltip - tip = message["desc"] - self.setToolTip(tip) + self.setToolTip(mod_info.description) - icon = util.THEME.icon(os.path.join("games/mods/", self.mod + ".png")) + icon = util.THEME.icon(os.path.join("games/mods/", f"{self.mod}.png")) if icon.isNull(): icon = util.THEME.icon("games/mods/default.png") self.setIcon(icon) @@ -56,6 +56,6 @@ def __lt__(self, other): if self.order == other.order: # Default: Alphabetical - return self.name.lower() < other.mod.lower() + return self.name.lower() < other.name.lower() return self.order < other.order From ab20f214128d482dfb9faef6ced40b567d2f4ba4 Mon Sep 17 00:00:00 2001 From: gatsik <74517072+Gatsik@users.noreply.github.com> Date: Sat, 27 Apr 2024 21:47:24 +0300 Subject: [PATCH 049/123] Clean up updater a little * remove unused code from Updater class * fix duplications in updating logic * do not redownload ForgedAlliance.exe file -- patch it if needed So we take files of specific version from API and update all except ForgedAlliance.exe -- versions for this file are broken (at least for fafbeta and fafdevelop featured mods) -- we patch it ourselves: patching is just writing version to special addresses --- src/config/production.py | 2 + src/fa/check.py | 14 +- src/fa/mods.py | 2 +- src/fa/replayparser.py | 2 +- src/fa/updater.py | 444 +++++++++++++++------------------------ 5 files changed, 176 insertions(+), 288 deletions(-) diff --git a/src/config/production.py b/src/config/production.py index dc872ba18..e0f805a5a 100644 --- a/src/config/production.py +++ b/src/config/production.py @@ -28,6 +28,8 @@ 'game/logs/path': join(APPDATA_DIR, 'logs'), 'game/mods/path': join(join(APPDATA_DIR, 'repo'), 'mods'), 'game/maps/path': join(join(APPDATA_DIR, 'repo'), 'maps'), + 'game/exe-url': 'https://content.{host}/faf/updaterNew/updates_faf_files/ForgedAlliance.exe', + 'game/exe-name': "ForgedAlliance.exe", 'host': 'faforever.com', 'proxy/host': 'proxy.{host}', 'proxy/port': 9124, diff --git a/src/fa/check.py b/src/fa/check.py index 166f5eec5..6e9275d8d 100644 --- a/src/fa/check.py +++ b/src/fa/check.py @@ -150,12 +150,12 @@ def checkMovies(files): def check( - featured_mod, - mapname=None, - version=None, - modVersions=None, - sim_mods=None, - silent=False, + featured_mod: str, + mapname: str | None = None, + version: int | None = None, + modVersions: dict | None = None, + sim_mods: dict[str, str] | None = None, + silent: bool = False, ): """ This checks whether the mods are properly updated and player has the @@ -182,7 +182,7 @@ def check( ) result = game_updater.run() - if result != fa.updater.Updater.RESULT_SUCCESS: + if result != fa.updater.UpdaterResult.SUCCESS: return False try: diff --git a/src/fa/mods.py b/src/fa/mods.py index ff367ea60..a05b78afd 100644 --- a/src/fa/mods.py +++ b/src/fa/mods.py @@ -48,7 +48,7 @@ def checkMods(mods: dict[str, str]) -> bool: # mods is a dictionary of uid-name # Spawn an update for the required mod updater = fa.updater.Updater("sim", sim_mod=item) result = updater.run() - if result != fa.updater.Updater.RESULT_SUCCESS: + if result != fa.updater.UpdaterResult.SUCCESS: logger.warning(f"Failure getting {item}") return False diff --git a/src/fa/replayparser.py b/src/fa/replayparser.py index a6d647da8..6bc708e1b 100644 --- a/src/fa/replayparser.py +++ b/src/fa/replayparser.py @@ -35,7 +35,7 @@ def getVersion(self): if not supcomVersion.startswith("Supreme Commander v1"): return None else: - return supcomVersion.split(".")[-1] + return int(supcomVersion.split(".")[-1]) def getMapName(self): with open(self.file, 'rb') as f: diff --git a/src/fa/updater.py b/src/fa/updater.py index 77f2d7d9b..950388034 100644 --- a/src/fa/updater.py +++ b/src/fa/updater.py @@ -13,11 +13,11 @@ import shutil import stat import time +from enum import Enum from PyQt6 import QtCore from PyQt6 import QtWidgets -import config import util from api.featured_mod_updater import FeaturedModFiles from api.featured_mod_updater import FeaturedModId @@ -93,21 +93,20 @@ class UpdaterTimeout(Exception): pass +class UpdaterResult(Enum): + SUCCESS = 0 # Update successful + NONE = -1 # Update operation is still ongoing + FAILURE = 1 # An error occured during updating + CANCEL = 2 # User cancelled the download process + ILLEGAL = 3 # User has the wrong version of FA + BUSY = 4 # Server is currently busy + PASS = 5 # User refuses to update by canceling the wizard + + class Updater(QtCore.QObject): """ This is the class that does the actual installation work. """ - # Network configuration - TIMEOUT = 20 # seconds - - # Return codes to expect from run() - RESULT_SUCCESS = 0 # Update successful - RESULT_NONE = -1 # Update operation is still ongoing - RESULT_FAILURE = 1 # An error occured during updating - RESULT_CANCEL = 2 # User cancelled the download process - RESULT_ILLEGAL = 3 # User has the wrong version of FA - RESULT_BUSY = 4 # Server is currently busy - RESULT_PASS = 5 # User refuses to update by canceling the wizard def __init__( self, @@ -124,19 +123,13 @@ def __init__( """ QtCore.QObject.__init__(self, *args, **kwargs) - self.filesToUpdate = [] - self.updatedFiles = [] - - self.lastData = time.time() - self.featured_mod = featured_mod self.version = version self.modversions = modversions - self.sim_mod = sim_mod - self.modpath = None + self.silent = silent - self.result = self.RESULT_NONE + self.result = UpdaterResult.NONE self.keep_cache = not Settings.get( 'cache/do_not_keep', type=bool, default=True, @@ -144,12 +137,7 @@ def __init__( self.in_session_cache = Settings.get( 'cache/in_session', type=bool, default=False, ) - self.fmod_cache_dir = os.path.join(util.CACHE_DIR, 'featured_mod') - if self.keep_cache or self.in_session_cache: - if not os.path.exists(self.fmod_cache_dir): - os.mkdir(self.fmod_cache_dir) - self.silent = silent self.progress = QtWidgets.QProgressDialog() if self.silent: self.progress.setCancelButton(None) @@ -161,11 +149,7 @@ def __init__( self.progress.setAutoClose(False) self.progress.setAutoReset(False) self.progress.setModal(1) - self.progress.setWindowTitle( - "Updating {}".format(self.featured_mod.upper()), - ) - - self.bytesToSend = 0 + self.progress.setWindowTitle(f"Updating {self.featured_mod.upper()}") def run(self, *args, **kwargs): clearLog() @@ -181,29 +165,41 @@ def run(self, *args, **kwargs): if not self.progress.wasCanceled(): log("Connected to update server at {}".format(timestamp())) - self.doUpdate() + self.do_update() self.progress.setLabelText("Cleaning up.") self.progress.close() else: log("Cancelled connecting to server.") - self.result = self.RESULT_CANCEL + self.result = UpdaterResult.CANCEL log("Update finished at {}".format(timestamp())) return self.result - def getFilesToUpdate(self, id_, version): - return FeaturedModFiles(id_, version).getFiles() + def get_files_to_update(self, mod_id: int, version: str) -> list[dict]: + return FeaturedModFiles(mod_id, version).getFiles() - def getFeaturedModIdByName(self, technicalName): - return FeaturedModId().requestAndGetFeaturedModIdByName(technicalName) + def get_featured_mod_id_by_name(self, technical_name: str) -> int: + return FeaturedModId().requestAndGetFeaturedModIdByName(technical_name) def request_sim_url_by_uid(self, uid: str) -> str: return SimModFiles().request_and_get_sim_mod_url_by_id(uid) - def fetch_file(self, file_info: dict, filegroup: str) -> None: + @staticmethod + def _file_needs_update(file_info: dict) -> bool: + filegroup = file_info["group"] + filename = file_info["name"] + filepath = os.path.join(util.APPDATA_DIR, filegroup, filename) + return filename != Settings.get("game/exe-name") and util.md5(filepath) != file_info["md5"] + + def fetch_files(self, files: list[dict]) -> None: + for file in files: + self.fetch_single_file(file) + + def fetch_single_file(self, file_info: dict) -> None: name = file_info["name"] + filegroup = file_info["group"] url = file_info["cacheableUrl"] target_dir = os.path.join(util.APPDATA_DIR, filegroup) @@ -226,122 +222,84 @@ def fetch_file(self, file_info: dict, filegroup: str) -> None: "Operation aborted while waiting for data.", ) - def moveFromCache(self, files, filegroup): - src_dir = os.path.join(util.APPDATA_DIR, filegroup) - cache_dir = os.path.join(self.fmod_cache_dir, filegroup) - for _file in files: - if os.path.exists(os.path.join(cache_dir, _file['md5'])): - shutil.move( - os.path.join(cache_dir, _file['md5']), - os.path.join(src_dir, _file['name']), - ) - - def moveToCache(self, files, filegroup): - src_dir = os.path.join(util.APPDATA_DIR, filegroup) - cache_dir = os.path.join(self.fmod_cache_dir, filegroup) - for _file in files: - if os.path.exists(os.path.join(src_dir, _file['name'])): - md5 = util.md5(os.path.join(src_dir, _file['name'])) - shutil.move( - os.path.join(src_dir, _file['name']), - os.path.join(cache_dir, md5), - ) - util.setAccessTime(os.path.join(cache_dir, md5)) - - def replaceFromCache(self, files, filegroup): - self.moveToCache(files, filegroup) - self.moveFromCache(files, filegroup) - - def checkCache(self, filegroup, files_to_check): - cache_dir = os.path.join(self.fmod_cache_dir, filegroup) - if not os.path.exists(cache_dir): - os.mkdir(cache_dir) - for src_dir, _, _files in os.walk(cache_dir): - files_in_cache = _files - replaceable_files, need_to_download = [], [] - for _file in files_to_check: - if _file['md5'] in files_in_cache: - replaceable_files.append(_file) - self.filesToUpdate.remove(_file) - else: - need_to_download.append(_file) - return replaceable_files, need_to_download - - def updateFiles(self, filegroup, files): - """ - Updates the files in a given file group, in the destination - subdirectory of the Forged Alliance path. - """ - QtWidgets.QApplication.processEvents() + def move_many_from_cache(self, files: list[dict]) -> None: + for file in files: + self.move_from_cache(file) + + def move_from_cache(self, file_info: dict) -> None: + src_dir = os.path.join(util.APPDATA_DIR, file_info["group"]) + cache_dir = os.path.join(util.GAME_CACHE_DIR, file_info["group"]) + if os.path.exists(os.path.join(cache_dir, file_info["md5"])): + shutil.move( + os.path.join(cache_dir, file_info["md5"]), + os.path.join(src_dir, file_info["name"]), + ) - self.progress.setLabelText("Updating files: " + filegroup) + def move_many_to_cache(self, files: list[dict]) -> None: + for file in files: + self.move_to_cache(file) + + def move_to_cache(self, file_info: dict) -> None: + src_dir = os.path.join(util.APPDATA_DIR, file_info["group"]) + cache_dir = os.path.join(util.GAME_CACHE_DIR, file_info["group"]) + if os.path.exists(os.path.join(src_dir, file_info["name"])): + md5 = util.md5(os.path.join(src_dir, file_info["name"])) + shutil.move( + os.path.join(src_dir, file_info["name"]), + os.path.join(cache_dir, md5), + ) + util.setAccessTime(os.path.join(cache_dir, md5)) - targetdir = os.path.join(util.APPDATA_DIR, filegroup) - if not os.path.exists(targetdir): - os.makedirs(targetdir) + def replace_from_cache(self, file: dict) -> None: + self.move_to_cache(file) + self.move_from_cache(file) - files_to_check = [] + def replace_many_from_cache(self, files: list[dict]) -> None: + for file in files: + self.replace_from_cache(file) - for _file in files: - md5File = util.md5( - os.path.join(util.APPDATA_DIR, filegroup, _file['name']), - ) - md5NewFile = _file['md5'] - if md5File == md5NewFile: - self.filesToUpdate.remove(_file) + def check_cache(self, files_to_check: list[dict]) -> None: + replaceable_files, need_to_download = [], [] + for file in files_to_check: + cache_dir = os.path.join(util.GAME_CACHE_DIR, file["group"]) + os.makedirs(cache_dir, exist_ok=True) + if self._is_cached(file): + replaceable_files.append(file) else: - if self.keep_cache or self.in_session_cache: - files_to_check.append(_file) - else: - self.fetch_file(_file, filegroup) - self.filesToUpdate.remove(_file) - self.updatedFiles.append(_file['name']) + need_to_download.append(file) + return replaceable_files, need_to_download - if len(files_to_check) > 0: - replaceable_files, need_to_download = ( - self.checkCache(filegroup, files_to_check) - ) - self.replaceFromCache(replaceable_files, filegroup) - for _file in need_to_download: - self.moveToCache([_file], filegroup) - self.fetch_file(_file, filegroup) - self.filesToUpdate.remove(_file) - self.updatedFiles.append(_file['name']) + @staticmethod + def _is_cached(file_info: dict) -> bool: + cached_file = os.path.join(util.GAME_CACHE_DIR, file_info["group"], file_info["name"]) + return os.path.isfile(cached_file) - self.waitUntilFilesAreUpdated() + def create_cache_subdirs(self, files: list[dict]) -> None: + for file in files: + target = os.path.join(util.GAME_CACHE_DIR, file["group"]) + os.makedirs(target, exist_ok=True) - def waitUntilFilesAreUpdated(self): + def update_files(self, files: list[dict]) -> None: """ - A simple loop that updates the progress bar while the server sends - actual file data + Updates the files in the destination + subdirectory of the Forged Alliance path. """ - self.lastData = time.time() - - self.progress.setValue(0) - self.progress.setMinimum(0) - self.progress.setMaximum(0) - - while len(self.filesToUpdate) > 0: - if self.progress.wasCanceled(): - raise UpdaterCancellation( - "Operation aborted while waiting for data.", - ) + self.create_cache_subdirs(files) + self.patch_fa_exe_if_needed(files) - if self.result != self.RESULT_NONE: - raise UpdaterFailure( - "Operation failed while waiting for data.", - ) + to_update = list(filter(self._file_needs_update, files)) + replacable_files, need_to_download = self.check_cache(to_update) - if time.time() - self.lastData > self.TIMEOUT: - raise UpdaterTimeout( - "Connection timed out while waiting for data.", - ) - - QtWidgets.QApplication.processEvents() + if self.keep_cache or self.in_session_cache: + self.replace_many_from_cache(replacable_files) + self.move_many_to_cache(need_to_download) + else: + self.move_many_from_cache(replacable_files) + self.fetch_files(need_to_download) log("Updates applied successfully.") - def prepareBinFAF(self): + def prepare_bin_FAF(self) -> None: """ Creates all necessary files in the binFAF folder, which contains a modified copy of all that is in the standard bin folder of @@ -350,9 +308,7 @@ def prepareBinFAF(self): self.progress.setLabelText("Preparing binFAF...") # now we check if we've got a binFAF folder - FABindir = os.path.join( - config.Settings.get("ForgedAlliance/app/path"), 'bin', - ) + FABindir = os.path.join(Settings.get("ForgedAlliance/app/path"), "bin") FAFdir = util.BIN_DIR # Try to copy without overwriting, but fill in any missing files, @@ -374,170 +330,100 @@ def prepareBinFAF(self): # need to patch them os.chmod(dst_file, st.st_mode | stat.S_IWRITE) - def doUpdate(self) -> None: + self.download_fa_executable() + + def download_fa_executable(self) -> bool: + fa_exe_name = Settings.get("game/exe-name") + fa_exe = os.path.join(util.BIN_DIR, fa_exe_name) + + if os.path.isfile(fa_exe): + return True + + url = Settings.get("game/exe-url") + return download_file( + url=url, + target_dir=util.BIN_DIR, + name=fa_exe_name, + category="Update", + silent=False, + label=f"Downloading FA file : {url}",
+ )
+
+ def patch_fa_executable(self, version: int) -> None:
+ exe_path = os.path.join(util.BIN_DIR, "ForgedAlliance.exe")
+ version_addresses = (0xd3d40, 0x47612d, 0x476666)
+ with open(exe_path, "rb+") as file:
+ for address in version_addresses:
+ file.seek(address)
+ file.write(version.to_bytes(4, "little"))
+
+ def patch_fa_exe_if_needed(self, files: list[dict]) -> None:
+ for file in files:
+ if file["name"] == Settings.get("game/exe-name"):
+ version = int(self._resolve_base_version(file))
+ self.patch_fa_executable(version)
+ return
+
+ def update_featured_mod(self, modname: str, modversion: str) -> list[dict]:
+ fmod_id = self.get_featured_mod_id_by_name(modname)
+ files = self.get_files_to_update(fmod_id, modversion)
+ self.update_files(files)
+ return files
+
+ def _resolve_modversion(self) -> str:
+ if self.modversions:
+ return str(max(self.modversions.values()))
+ return "latest"
+
+ def _resolve_base_version(self, base_info: dict | None) -> str:
+ if self.version:
+ return str(self.version)
+ if base_info:
+ return str(base_info["version"])
+ return "latest"
+
+ def do_update(self) -> None:
""" The core function that does most of the actual update work."""
try:
if self.sim_mod:
uid, name = self.sim_mod
if utils.downloadMod(self.request_sim_url_by_uid(uid), name):
- self.result = self.RESULT_SUCCESS
+ self.result = UpdaterResult.SUCCESS
else:
- self.result = self.RESULT_FAILURE
+ self.result = UpdaterResult.FAILURE
else:
# Prepare FAF directory & all necessary files
- self.prepareBinFAF()
-
- toUpdate = []
- filesToUpdateInBin = []
- filesToUpdateInGamedata = []
-
+ self.prepare_bin_FAF()
# Update the mod if it's requested
- if (
- self.featured_mod == "faf"
- # HACK - ladder1v1 "is" FAF. :-)
- or self.featured_mod == "ladder1v1"
- ):
- if self.version:
- # id for faf (or ladder1v1) is 0
- toUpdate = self.getFilesToUpdate('0', self.version)
- else:
- toUpdate = self.getFilesToUpdate('0', 'latest')
-
- for _file in toUpdate:
- if _file['group'] == 'bin':
- filesToUpdateInBin.append(_file)
- else:
- filesToUpdateInGamedata.append(_file)
-
- self.filesToUpdate = filesToUpdateInBin.copy()
- self.updateFiles("bin", filesToUpdateInBin)
- self.filesToUpdate = filesToUpdateInGamedata.copy()
- self.updateFiles("gamedata", filesToUpdateInGamedata)
-
- elif (
- self.featured_mod == 'fafbeta'
- or self.featured_mod == 'fafdevelop'
- ):
- # no need to update faf first for these mods
- id_ = self.getFeaturedModIdByName(self.featured_mod)
- if self.modversions:
- modversion = sorted(
- self.modversions.items(),
- key=lambda item: item[1],
- reverse=True,
- )[0][1]
- else:
- modversion = 'latest'
-
- toUpdate = self.getFilesToUpdate(id_, modversion)
- toUpdate.sort(
- key=lambda item: item['version'],
- reverse=True,
- )
-
- # file lists for fafbeta and fafdevelop contain wrong
- # version of ForgedAlliance.exe
- if self.version:
- faf_version = self.version
- else:
- faf_version = toUpdate[0]['version']
-
- for _file in toUpdate:
- if _file['group'] == 'bin':
- if _file['name'] != 'ForgedAlliance.exe':
- filesToUpdateInBin.append(_file)
- else:
- filesToUpdateInGamedata.append(_file)
-
- self.filesToUpdate = filesToUpdateInBin.copy()
- self.updateFiles('bin', filesToUpdateInBin)
- self.filesToUpdate = filesToUpdateInGamedata.copy()
- self.updateFiles('gamedata', filesToUpdateInGamedata)
-
- filesToUpdateInBin.clear()
- filesToUpdateInGamedata.clear()
-
- # update proper version of bin
- toUpdate = self.getFilesToUpdate('0', faf_version)
-
- for _file in toUpdate:
- if _file['group'] == 'bin':
- filesToUpdateInBin.append(_file)
-
- self.filesToUpdate = filesToUpdateInBin.copy()
- self.updateFiles('bin', filesToUpdateInBin)
-
+ if self.featured_mod in ("faf", "fafbeta", "fafdevelop", "ladder1v1"):
+ self.update_featured_mod(self.featured_mod, self._resolve_base_version())
else:
# update faf first
- # id for faf (or ladder1v1) is 0
- if self.version:
- toUpdate = self.getFilesToUpdate('0', self.version)
- else:
- toUpdate = self.getFilesToUpdate('0', 'latest')
-
- for _file in toUpdate:
- if _file['group'] == 'bin':
- filesToUpdateInBin.append(_file)
- else:
- filesToUpdateInGamedata.append(_file)
-
- self.filesToUpdate = filesToUpdateInBin.copy()
- self.updateFiles("bin", filesToUpdateInBin)
- self.filesToUpdate = filesToUpdateInGamedata.copy()
- self.updateFiles("gamedata", filesToUpdateInGamedata)
-
- filesToUpdateInBin.clear()
- filesToUpdateInGamedata.clear()
-
- # update featuredMod then
- id_ = self.getFeaturedModIdByName(self.featured_mod)
- if self.modversions:
- modversion = sorted(
- self.modversions.items(),
- key=lambda item: item[1],
- reverse=True,
- )[0][1]
- else:
- modversion = 'latest'
-
- toUpdate = self.getFilesToUpdate(id_, modversion)
-
- for _file in toUpdate:
- if _file['group'] == 'bin':
- filesToUpdateInBin.append(_file)
- else:
- filesToUpdateInGamedata.append(_file)
-
- self.filesToUpdate = filesToUpdateInBin.copy()
- self.updateFiles("bin", filesToUpdateInBin)
- self.filesToUpdate = filesToUpdateInGamedata.copy()
- self.updateFiles("gamedata", filesToUpdateInGamedata)
-
- except UpdaterTimeout as e:
- log("TIMEOUT: {}".format(e))
- self.result = self.RESULT_FAILURE
+ self.update_featured_mod("faf", self._resolve_base_version())
+ # update featured mod then
+ self.update_featured_mod(self.featured_mod, self._resolve_modversion())
except UpdaterCancellation as e:
log("CANCELLED: {}".format(e))
- self.result = self.RESULT_CANCEL
+ self.result = UpdaterResult.CANCEL
except BaseException as e:
log("EXCEPTION: {}".format(e))
- self.result = self.RESULT_FAILURE
+ self.result = UpdaterResult.FAILURE
else:
- self.result = self.RESULT_SUCCESS
+ self.result = UpdaterResult.SUCCESS
# Hide progress dialog if it's still showing.
self.progress.close()
# Integrated handlers for the various things that could go wrong
- if self.result == self.RESULT_CANCEL:
+ if self.result == UpdaterResult.CANCEL:
pass # The user knows damn well what happened here.
- elif self.result == self.RESULT_PASS:
+ elif self.result == UpdaterResult.PASS:
QtWidgets.QMessageBox.information(
QtWidgets.QApplication.activeWindow(),
"Installation Required",
"You can't play without a legal version of Forged Alliance.",
)
- elif self.result == self.RESULT_BUSY:
+ elif self.result == UpdaterResult.BUSY:
QtWidgets.QMessageBox.information(
QtWidgets.QApplication.activeWindow(),
"Server Busy",
@@ -546,7 +432,7 @@ def doUpdate(self) -> None:
"again later."
),
)
- elif self.result == self.RESULT_FAILURE:
+ elif self.result == UpdaterResult.FAILURE:
failureDialog()
# If nothing terribly bad happened until now,
From e0f4d78c68794ffb9f0b8e7964e56b99ca17daa2 Mon Sep 17 00:00:00 2001
From: gatsik <74517072+Gatsik@users.noreply.github.com>
Date: Sun, 28 Apr 2024 10:11:00 +0300
Subject: [PATCH 050/123] Updater: Be more specific with Qt imports
---
src/fa/updater.py | 35 ++++++++++++++++++++---------------
1 file changed, 20 insertions(+), 15 deletions(-)
diff --git a/src/fa/updater.py b/src/fa/updater.py
index 950388034..78d716c40 100644
--- a/src/fa/updater.py
+++ b/src/fa/updater.py
@@ -15,8 +15,13 @@
import time
from enum import Enum
-from PyQt6 import QtCore
-from PyQt6 import QtWidgets
+from PyQt6.QtCore import QObject
+from PyQt6.QtCore import Qt
+from PyQt6.QtCore import pyqtSlot
+from PyQt6.QtWidgets import QApplication
+from PyQt6.QtWidgets import QDialog
+from PyQt6.QtWidgets import QMessageBox
+from PyQt6.QtWidgets import QProgressDialog
import util
from api.featured_mod_updater import FeaturedModFiles
@@ -43,22 +48,22 @@ def __init__(self, parent):
self.adjustSize()
self.watches = []
- @QtCore.pyqtSlot(str)
+ @pyqtSlot(str)
def appendLog(self, text):
self.logPlainTextEdit.appendPlainText(text)
- @QtCore.pyqtSlot(QtCore.QObject)
+ @pyqtSlot(QObject)
def addWatch(self, watch):
self.watches.append(watch)
watch.finished.connect(self.watchFinished)
- @QtCore.pyqtSlot()
+ @pyqtSlot()
def watchFinished(self) -> None:
for watch in self.watches:
if not watch.isFinished():
return
# equivalent to self.accept(), but clearer
- self.done(QtWidgets.QDialog.DialogCode.Accepted)
+ self.done(QDialog.DialogCode.Accepted)
def clearLog():
@@ -103,7 +108,7 @@ class UpdaterResult(Enum):
PASS = 5 # User refuses to update by canceling the wizard
-class Updater(QtCore.QObject):
+class Updater(QObject):
"""
This is the class that does the actual installation work.
"""
@@ -121,7 +126,7 @@ def __init__(
"""
Constructor
"""
- QtCore.QObject.__init__(self, *args, **kwargs)
+ super().__init__(*args, **kwargs)
self.featured_mod = featured_mod
self.version = version
@@ -138,13 +143,13 @@ def __init__(
'cache/in_session', type=bool, default=False,
)
- self.progress = QtWidgets.QProgressDialog()
+ self.progress = QProgressDialog()
if self.silent:
self.progress.setCancelButton(None)
else:
self.progress.setCancelButtonText("Cancel")
self.progress.setWindowFlags(
- QtCore.Qt.WindowType.CustomizeWindowHint | QtCore.Qt.WindowType.WindowTitleHint,
+ Qt.WindowType.CustomizeWindowHint | Qt.WindowType.WindowTitleHint,
)
self.progress.setAutoClose(False)
self.progress.setAutoReset(False)
@@ -157,7 +162,7 @@ def run(self, *args, **kwargs):
log("Using appdata: " + util.APPDATA_DIR)
self.progress.show()
- QtWidgets.QApplication.processEvents()
+ QApplication.processEvents()
# Actual network code adapted from previous version
self.progress.setLabelText("Connecting to update server...")
@@ -418,14 +423,14 @@ def do_update(self) -> None:
if self.result == UpdaterResult.CANCEL:
pass # The user knows damn well what happened here.
elif self.result == UpdaterResult.PASS:
- QtWidgets.QMessageBox.information(
- QtWidgets.QApplication.activeWindow(),
+ QMessageBox.information(
+ QApplication.activeWindow(),
"Installation Required",
"You can't play without a legal version of Forged Alliance.",
)
elif self.result == UpdaterResult.BUSY:
- QtWidgets.QMessageBox.information(
- QtWidgets.QApplication.activeWindow(),
+ QMessageBox.information(
+ QApplication.activeWindow(),
"Server Busy",
(
"The Server is busy preparing new patch files.
Try "
From 638ab1b135cb08ed4be02541652b5ee9a7e6855d Mon Sep 17 00:00:00 2001
From: gatsik <74517072+Gatsik@users.noreply.github.com>
Date: Mon, 29 Apr 2024 19:57:41 +0300
Subject: [PATCH 051/123] Do not require optional argument
---
src/fa/updater.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/fa/updater.py b/src/fa/updater.py
index 78d716c40..e58d10802 100644
--- a/src/fa/updater.py
+++ b/src/fa/updater.py
@@ -380,7 +380,7 @@ def _resolve_modversion(self) -> str:
return str(max(self.modversions.values()))
return "latest"
- def _resolve_base_version(self, base_info: dict | None) -> str:
+ def _resolve_base_version(self, base_info: dict | None = None) -> str:
if self.version:
return str(self.version)
if base_info:
From 2c36817ecc0af2ade61220d94c07504383c42c9f Mon Sep 17 00:00:00 2001
From: gatsik <74517072+Gatsik@users.noreply.github.com>
Date: Tue, 30 Apr 2024 19:44:27 +0300
Subject: [PATCH 052/123] Unpack movies inside Updater
because it has the information about files being updated
---
src/fa/check.py | 57 -----------------------------------------------
src/fa/updater.py | 3 +++
src/fa/utils.py | 54 ++++++++++++++++++++++++++++++++++++++++++++
3 files changed, 57 insertions(+), 57 deletions(-)
create mode 100644 src/fa/utils.py
diff --git a/src/fa/check.py b/src/fa/check.py
index 6e9275d8d..7bf6465cc 100644
--- a/src/fa/check.py
+++ b/src/fa/check.py
@@ -1,9 +1,6 @@
from __future__ import annotations
-import binascii
import logging
-import os
-import zipfile
from typing import TYPE_CHECKING
from PyQt6 import QtWidgets
@@ -102,53 +99,6 @@ def game(parent):
return True
-def crc32(fname):
- try:
- with open(fname) as stream:
- return binascii.crc32(stream.read())
- except BaseException:
- logger.exception('CRC check fail!')
- return None
-
-
-def checkMovies(files):
- """
- Unpacks movies (based on path in zipfile) to the movies folder.
- Movies must be unpacked for FA to be able to play them.
- This is a hack needed because the game updater can only handle bin and
- gamedata.
- """
-
- logger.info('checking updated files: {}'.format(files))
-
- # construct dirs
- gd = os.path.join(util.APPDATA_DIR, 'gamedata')
-
- for fname in files:
- origpath = os.path.join(gd, fname)
-
- if os.path.exists(origpath) and zipfile.is_zipfile(origpath):
- try:
- zf = zipfile.ZipFile(origpath)
- except BaseException:
- logger.exception(
- 'Failed to open Game File {}'.format(origpath),
- )
- continue
-
- for zi in zf.infolist():
- if zi.filename.startswith('movies'):
- tgtpath = os.path.join(util.APPDATA_DIR, zi.filename)
- # copy only if file is different - check first if file
- # exists, then if size is changed, then crc
- if (
- not os.path.exists(tgtpath)
- or os.stat(tgtpath).st_size != zi.file_size
- or crc32(tgtpath) != zi.CRC
- ):
- zf.extract(zi, util.APPDATA_DIR)
-
-
def check(
featured_mod: str,
mapname: str | None = None,
@@ -185,13 +135,6 @@ def check(
if result != fa.updater.UpdaterResult.SUCCESS:
return False
- try:
- if len(game_updater.updatedFiles) > 0:
- checkMovies(game_updater.updatedFiles)
- except BaseException:
- logger.exception('Error checking game files for movies')
- return False
-
# Now it's down to having the right map
if mapname:
if not map_(mapname, silent=silent):
diff --git a/src/fa/updater.py b/src/fa/updater.py
index e58d10802..755759f83 100644
--- a/src/fa/updater.py
+++ b/src/fa/updater.py
@@ -28,6 +28,7 @@
from api.featured_mod_updater import FeaturedModId
from api.sim_mod_updater import SimModFiles
from config import Settings
+from fa.utils import unpack_movies
from vaults.dialogs import download_file
from vaults.modvault import utils
@@ -302,6 +303,8 @@ def update_files(self, files: list[dict]) -> None:
self.move_many_from_cache(replacable_files)
self.fetch_files(need_to_download)
+
+ unpack_movies(to_update)
log("Updates applied successfully.")
def prepare_bin_FAF(self) -> None:
diff --git a/src/fa/utils.py b/src/fa/utils.py
new file mode 100644
index 000000000..9cd0a59e7
--- /dev/null
+++ b/src/fa/utils.py
@@ -0,0 +1,54 @@
+import binascii
+import logging
+import os
+import zipfile
+
+from util import APPDATA_DIR
+
+logger = logging.getLogger(__name__)
+
+
+def crc32(filepath: str) -> int | None:
+ try:
+ with open(filepath) as stream:
+ return binascii.crc32(stream.read())
+ except Exception as e:
+ logger.exception(f"CRC check fail! Details: {e}")
+ return None
+
+
+def unpack_movies(files: list[dict]) -> None:
+ """
+ Unpacks movies (based on path in zipfile) to the movies folder.
+ Movies must be unpacked for FA to be able to play them.
+ This is a hack needed because the game updater can only handle bin and
+ gamedata.
+ """
+
+ logger.info(f"checking updated files: {files}")
+
+ # construct dirs
+ gd = os.path.join(APPDATA_DIR, "gamedata")
+
+ for file in files:
+ fname = file["name"]
+ origpath = os.path.join(gd, fname)
+
+ if os.path.exists(origpath) and zipfile.is_zipfile(origpath):
+ try:
+ zf = zipfile.ZipFile(origpath)
+ except Exception as e:
+ logger.exception(f"Failed to open Game File {origpath!r}: {e}")
+ continue
+
+ for zi in zf.infolist():
+ if zi.filename.startswith("movies"):
+ tgtpath = os.path.join(APPDATA_DIR, zi.filename)
+ # copy only if file is different - check first if file
+ # exists, then if size is changed, then crc
+ if (
+ not os.path.exists(tgtpath)
+ or os.stat(tgtpath).st_size != zi.file_size
+ or crc32(tgtpath) != zi.CRC
+ ):
+ zf.extract(zi, APPDATA_DIR)
From 8628d66e9a5e84a305325d0d1a8033e27f549a7b Mon Sep 17 00:00:00 2001
From: gatsik <74517072+Gatsik@users.noreply.github.com>
Date: Wed, 1 May 2024 03:50:48 +0300
Subject: [PATCH 053/123] Use correct PyQt namespace in SecondaryServer
this server doesn't work anyway and was deprecated ages ago
and shoudl be pruned together with unused widgets
---
src/secondaryServer/secondaryserver.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/src/secondaryServer/secondaryserver.py b/src/secondaryServer/secondaryserver.py
index d51557a2f..49da8bd26 100644
--- a/src/secondaryServer/secondaryserver.py
+++ b/src/secondaryServer/secondaryserver.py
@@ -164,19 +164,19 @@ def handleServerError(self, socketError):
Simple error handler that flags the whole operation as failed, not
very graceful but what can you do...
"""
- if socketError == QtNetwork.QAbstractSocket.RemoteHostClosedError:
+ if socketError == QtNetwork.QAbstractSocket.SocketError.RemoteHostClosedError:
log(
"FA Server down: The server is down for maintenance, please "
"try later.",
)
- elif socketError == QtNetwork.QAbstractSocket.HostNotFoundError:
+ elif socketError == QtNetwork.QAbstractSocket.SocketError.HostNotFoundError:
log(
"Connection to Host lost. Please check the host name and port "
"settings.",
)
- elif socketError == QtNetwork.QAbstractSocket.ConnectionRefusedError:
+ elif socketError == QtNetwork.QAbstractSocket.SocketError.ConnectionRefusedError:
log("The connection was refused by the peer.")
else:
log(
From 67f237fe560db48fe7e79526538d9b5cdf2a8cd7 Mon Sep 17 00:00:00 2001
From: gatsik <74517072+Gatsik@users.noreply.github.com>
Date: Wed, 1 May 2024 20:03:53 +0300
Subject: [PATCH 054/123] Prune featured_mod_updater
remove FeaturedModId class -- use existing FeaturedModApiConnector
---
src/api/featured_mod_api.py | 31 +++++++++++++++++++++--
src/api/featured_mod_updater.py | 37 ----------------------------
src/api/models/FeaturedMod.py | 1 +
src/api/parsers/FeaturedModParser.py | 1 +
src/fa/updater.py | 17 +++++++------
5 files changed, 40 insertions(+), 47 deletions(-)
delete mode 100644 src/api/featured_mod_updater.py
diff --git a/src/api/featured_mod_api.py b/src/api/featured_mod_api.py
index 164e9332d..c779c9c2a 100644
--- a/src/api/featured_mod_api.py
+++ b/src/api/featured_mod_api.py
@@ -1,6 +1,7 @@
import logging
from api.ApiAccessors import DataApiAccessor
+from api.models.FeaturedMod import FeaturedMod
from api.parsers.FeaturedModParser import FeaturedModParser
logger = logging.getLogger(__name__)
@@ -8,7 +9,33 @@
class FeaturedModApiConnector(DataApiAccessor):
def __init__(self) -> None:
- super().__init__('/data/featuredMod')
+ super().__init__("/data/featuredMod")
- def prepare_data(self, message: dict) -> None:
+ def prepare_data(self, message: dict) -> dict[str, list[FeaturedMod]]:
return {"values": FeaturedModParser.parse_many(message["data"])}
+
+ def handle_featured_mod(self, message: dict) -> None:
+ self.featured_mod = FeaturedModParser.parse(message["data"][0])
+
+ def request_fmod_by_name(self, technical_name: str) -> None:
+ queryDict = {"filter": f"technicalName=={technical_name}"}
+ self.get_by_query(queryDict, self.handle_featured_mod)
+
+ def request_and_get_fmod_by_name(self, technicalName) -> FeaturedMod:
+ self.request_fmod_by_name(technicalName)
+ self.waitForCompletion()
+ return self.featured_mod
+
+
+class FeaturedModFiles(DataApiAccessor):
+ def __init__(self, mod_id: str, version: str) -> None:
+ super().__init__(f"/featuredMods/{mod_id}/files/{version}")
+ self.featuredModFiles = []
+
+ def handle_response(self, message):
+ self.featuredModFiles = message["data"]
+
+ def get_files(self):
+ self.requestData()
+ self.waitForCompletion()
+ return self.featuredModFiles
diff --git a/src/api/featured_mod_updater.py b/src/api/featured_mod_updater.py
deleted file mode 100644
index 3a9b96163..000000000
--- a/src/api/featured_mod_updater.py
+++ /dev/null
@@ -1,37 +0,0 @@
-import logging
-
-from api.ApiAccessors import DataApiAccessor
-
-logger = logging.getLogger(__name__)
-
-
-class FeaturedModFiles(DataApiAccessor):
- def __init__(self, mod_id: int, version: str) -> None:
- super().__init__('/featuredMods/{}/files/{}'.format(mod_id, version))
- self.featuredModFiles = []
-
- def handle_response(self, message):
- self.featuredModFiles = message["data"]
-
- def getFiles(self):
- self.requestData()
- self.waitForCompletion()
- return self.featuredModFiles
-
-
-class FeaturedModId(DataApiAccessor):
- def __init__(self) -> None:
- super().__init__('/data/featuredMod')
- self.featuredModId = 0
-
- def handleFeaturedModId(self, message):
- self.featuredModId = message['data'][0]['id']
-
- def requestFeaturedModIdByName(self, technicalName: str) -> None:
- queryDict = dict(filter='technicalName=={}'.format(technicalName))
- self.get_by_query(queryDict, self.handleFeaturedModId)
-
- def requestAndGetFeaturedModIdByName(self, technicalName):
- self.requestFeaturedModIdByName(technicalName)
- self.waitForCompletion()
- return self.featuredModId
diff --git a/src/api/models/FeaturedMod.py b/src/api/models/FeaturedMod.py
index abb8f9030..0d08d37b8 100644
--- a/src/api/models/FeaturedMod.py
+++ b/src/api/models/FeaturedMod.py
@@ -3,6 +3,7 @@
@dataclass
class FeaturedMod:
+ uid: str
name: str
fullname: str
visible: bool
diff --git a/src/api/parsers/FeaturedModParser.py b/src/api/parsers/FeaturedModParser.py
index 2ebfcdd9a..11fb83f09 100644
--- a/src/api/parsers/FeaturedModParser.py
+++ b/src/api/parsers/FeaturedModParser.py
@@ -6,6 +6,7 @@ class FeaturedModParser:
@staticmethod
def parse(data: dict) -> FeaturedMod:
return FeaturedMod(
+ uid=data["id"],
name=data["technicalName"],
fullname=data["displayName"],
visible=data.get("visible", False),
diff --git a/src/fa/updater.py b/src/fa/updater.py
index 755759f83..f864e284a 100644
--- a/src/fa/updater.py
+++ b/src/fa/updater.py
@@ -24,8 +24,9 @@
from PyQt6.QtWidgets import QProgressDialog
import util
-from api.featured_mod_updater import FeaturedModFiles
-from api.featured_mod_updater import FeaturedModId
+from api.featured_mod_api import FeaturedModApiConnector
+from api.featured_mod_api import FeaturedModFiles
+from api.models.FeaturedMod import FeaturedMod
from api.sim_mod_updater import SimModFiles
from config import Settings
from fa.utils import unpack_movies
@@ -183,11 +184,11 @@ def run(self, *args, **kwargs):
log("Update finished at {}".format(timestamp()))
return self.result
- def get_files_to_update(self, mod_id: int, version: str) -> list[dict]:
- return FeaturedModFiles(mod_id, version).getFiles()
+ def get_files_to_update(self, mod_id: str, version: str) -> list[dict]:
+ return FeaturedModFiles(mod_id, version).get_files()
- def get_featured_mod_id_by_name(self, technical_name: str) -> int:
- return FeaturedModId().requestAndGetFeaturedModIdByName(technical_name)
+ def get_featured_mod_by_name(self, technical_name: str) -> FeaturedMod:
+ return FeaturedModApiConnector().request_and_get_fmod_by_name(technical_name)
def request_sim_url_by_uid(self, uid: str) -> str:
return SimModFiles().request_and_get_sim_mod_url_by_id(uid)
@@ -373,8 +374,8 @@ def patch_fa_exe_if_needed(self, files: list[dict]) -> None:
return
def update_featured_mod(self, modname: str, modversion: str) -> list[dict]:
- fmod_id = self.get_featured_mod_id_by_name(modname)
- files = self.get_files_to_update(fmod_id, modversion)
+ fmod = self.get_featured_mod_by_name(modname)
+ files = self.get_files_to_update(fmod.uid, modversion)
self.update_files(files)
return files
From 7df55f17c2440c3cb5b033f7c3077102eb4dfa01 Mon Sep 17 00:00:00 2001
From: gatsik <74517072+Gatsik@users.noreply.github.com>
Date: Thu, 2 May 2024 22:17:17 +0300
Subject: [PATCH 055/123] CoopWidget: Fix sim_mods being incorrectly checked on
game join
---
src/coop/_coopwidget.py | 9 ++++-----
1 file changed, 4 insertions(+), 5 deletions(-)
diff --git a/src/coop/_coopwidget.py b/src/coop/_coopwidget.py
index 035cc8620..de70bd265 100644
--- a/src/coop/_coopwidget.py
+++ b/src/coop/_coopwidget.py
@@ -13,6 +13,7 @@
from coop.coopmapitem import CoopMapItemDelegate
from coop.coopmodel import CoopGameFilterModel
from fa.replay import replay
+from model.game import Game
from ui.busy_widget import BusyWidget
logger = logging.getLogger(__name__)
@@ -52,7 +53,7 @@ def __init__(
self.coopList.setItemDelegate(CoopMapItemDelegate(self))
self.gameview = self._gameview_builder(self._game_model, self.gameList)
- self.gameview.game_double_clicked.connect(self.gameDoubleClicked)
+ self.gameview.game_double_clicked.connect(self.game_double_clicked)
self.coopList.itemDoubleClicked.connect(self.coopListDoubleClicked)
self.coopList.itemClicked.connect(self.coopListClicked)
@@ -278,16 +279,14 @@ def processCoopInfo(self, message):
self.coop[uid] = itemCoop
- def gameDoubleClicked(self, game):
+ def game_double_clicked(self, game: Game) -> None:
"""
Slot that attempts to join a game.
"""
if not fa.instance.available():
return
- if not fa.check.check(
- game.featured_mod, game.mapname, None, game.sim_mods,
- ):
+ if not fa.check.check(game.featured_mod, game.mapname, sim_mods=game.sim_mods):
return
if game.password_protected:
From 62177ca65d75e64bf7f1a656857465c5e5ebcdc4 Mon Sep 17 00:00:00 2001
From: gatsik <74517072+Gatsik@users.noreply.github.com>
Date: Fri, 3 May 2024 17:06:37 +0300
Subject: [PATCH 056/123] Make checking md5 and unpacking movies less
UI-blocking
spawn/use QProgressDialog to block other windows from receving inputs
and call setValue() on progress done (which calls processEvents interally)
---
src/fa/updater.py | 32 +++++++++++++++++----
src/fa/utils.py | 73 +++++++++++++++++++++++++++++------------------
2 files changed, 72 insertions(+), 33 deletions(-)
diff --git a/src/fa/updater.py b/src/fa/updater.py
index f864e284a..27a7392d1 100644
--- a/src/fa/updater.py
+++ b/src/fa/updater.py
@@ -195,10 +195,29 @@ def request_sim_url_by_uid(self, uid: str) -> str:
@staticmethod
def _file_needs_update(file_info: dict) -> bool:
- filegroup = file_info["group"]
- filename = file_info["name"]
- filepath = os.path.join(util.APPDATA_DIR, filegroup, filename)
- return filename != Settings.get("game/exe-name") and util.md5(filepath) != file_info["md5"]
+ return (
+ file_info["name"] != Settings.get("game/exe-name")
+ and file_info["old_md5"] != file_info["md5"]
+ )
+
+ def _calc_md5s(self, files: list[dict]) -> None:
+ self.progress.setMaximum(len(files))
+
+ for index, file_info in enumerate(files, start=1):
+
+ if self.progress.wasCanceled():
+ raise UpdaterCancellation()
+
+ filegroup = file_info["group"]
+ filename = file_info["name"]
+ filepath = os.path.join(util.APPDATA_DIR, filegroup, filename)
+
+ self.progress.setLabelText(f"Calculating md5 for {filename}...")
+
+ file_info["old_md5"] = util.md5(filepath)
+
+ self.progress.setValue(index)
+ self.progress.setMaximum(0)
def fetch_files(self, files: list[dict]) -> None:
for file in files:
@@ -293,6 +312,9 @@ def update_files(self, files: list[dict]) -> None:
"""
self.create_cache_subdirs(files)
self.patch_fa_exe_if_needed(files)
+ self._calc_md5s(files)
+
+ self.progress.setLabelText("Updating files...")
to_update = list(filter(self._file_needs_update, files))
replacable_files, need_to_download = self.check_cache(to_update)
@@ -305,7 +327,7 @@ def update_files(self, files: list[dict]) -> None:
self.fetch_files(need_to_download)
- unpack_movies(to_update)
+ unpack_movies(files)
log("Updates applied successfully.")
def prepare_bin_FAF(self) -> None:
diff --git a/src/fa/utils.py b/src/fa/utils.py
index 9cd0a59e7..5167601c0 100644
--- a/src/fa/utils.py
+++ b/src/fa/utils.py
@@ -3,6 +3,9 @@
import os
import zipfile
+from PyQt6.QtCore import Qt
+from PyQt6.QtWidgets import QProgressDialog
+
from util import APPDATA_DIR
logger = logging.getLogger(__name__)
@@ -10,45 +13,59 @@
def crc32(filepath: str) -> int | None:
try:
- with open(filepath) as stream:
+ with open(filepath, "rb") as stream:
return binascii.crc32(stream.read())
except Exception as e:
- logger.exception(f"CRC check fail! Details: {e}")
+ logger.exception(f"CRC check for {filepath!r} fail! Details: {e}")
return None
-def unpack_movies(files: list[dict]) -> None:
+def unpack_movies_from_file(file_info: dict) -> None:
"""
Unpacks movies (based on path in zipfile) to the movies folder.
Movies must be unpacked for FA to be able to play them.
This is a hack needed because the game updater can only handle bin and
gamedata.
"""
-
- logger.info(f"checking updated files: {files}")
-
# construct dirs
gd = os.path.join(APPDATA_DIR, "gamedata")
- for file in files:
- fname = file["name"]
- origpath = os.path.join(gd, fname)
-
- if os.path.exists(origpath) and zipfile.is_zipfile(origpath):
- try:
- zf = zipfile.ZipFile(origpath)
- except Exception as e:
- logger.exception(f"Failed to open Game File {origpath!r}: {e}")
- continue
-
- for zi in zf.infolist():
- if zi.filename.startswith("movies"):
- tgtpath = os.path.join(APPDATA_DIR, zi.filename)
- # copy only if file is different - check first if file
- # exists, then if size is changed, then crc
- if (
- not os.path.exists(tgtpath)
- or os.stat(tgtpath).st_size != zi.file_size
- or crc32(tgtpath) != zi.CRC
- ):
- zf.extract(zi, APPDATA_DIR)
+ fname = file_info["name"]
+ origpath = os.path.join(gd, fname)
+
+ if os.path.exists(origpath) and zipfile.is_zipfile(origpath):
+ try:
+ zf = zipfile.ZipFile(origpath)
+ except Exception as e:
+ logger.exception(f"Failed to open Game File {origpath!r}: {e}")
+ return
+
+ for zi in zf.infolist():
+ if zi.filename.startswith("movies") and not zi.is_dir():
+ tgtpath = os.path.join(APPDATA_DIR, zi.filename)
+ # copy only if file is different - check first if file
+ # exists, then if size is changed, then crc
+ if (
+ not os.path.exists(tgtpath)
+ or os.stat(tgtpath).st_size != zi.file_size
+ or crc32(tgtpath) != zi.CRC
+ ):
+ zf.extract(zi, APPDATA_DIR)
+
+
+def unpack_movies(files: list[dict]) -> None:
+ logger.info("Checking files for movies")
+
+ progress = QProgressDialog()
+ progress.setWindowTitle("Updating Movies")
+ progress.setWindowFlags(Qt.WindowType.CustomizeWindowHint | Qt.WindowType.WindowTitleHint)
+ progress.setModal(True)
+ progress.setCancelButton(None)
+ progress.setMaximum(len(files))
+ progress.setValue(0)
+
+ for index, file in enumerate(files, start=1):
+ filename = file["name"]
+ progress.setLabelText(f"Checking for movies in {filename}...")
+ unpack_movies_from_file(file)
+ progress.setValue(index)
From c25637d1ab2ce72f0147b3cb4b0ea0d123df0489 Mon Sep 17 00:00:00 2001
From: gatsik <74517072+Gatsik@users.noreply.github.com>
Date: Fri, 3 May 2024 18:03:33 +0300
Subject: [PATCH 057/123] Unpack sounds along with movies from gamefiles
---
src/fa/updater.py | 4 ++--
src/fa/utils.py | 22 +++++++++++++---------
2 files changed, 15 insertions(+), 11 deletions(-)
diff --git a/src/fa/updater.py b/src/fa/updater.py
index 27a7392d1..e3d012c73 100644
--- a/src/fa/updater.py
+++ b/src/fa/updater.py
@@ -29,7 +29,7 @@
from api.models.FeaturedMod import FeaturedMod
from api.sim_mod_updater import SimModFiles
from config import Settings
-from fa.utils import unpack_movies
+from fa.utils import unpack_movies_and_sounds
from vaults.dialogs import download_file
from vaults.modvault import utils
@@ -327,7 +327,7 @@ def update_files(self, files: list[dict]) -> None:
self.fetch_files(need_to_download)
- unpack_movies(files)
+ unpack_movies_and_sounds(files)
log("Updates applied successfully.")
def prepare_bin_FAF(self) -> None:
diff --git a/src/fa/utils.py b/src/fa/utils.py
index 5167601c0..33500a777 100644
--- a/src/fa/utils.py
+++ b/src/fa/utils.py
@@ -20,10 +20,10 @@ def crc32(filepath: str) -> int | None:
return None
-def unpack_movies_from_file(file_info: dict) -> None:
+def unpack_movies_and_sounds_from_file(file_info: dict) -> None:
"""
- Unpacks movies (based on path in zipfile) to the movies folder.
- Movies must be unpacked for FA to be able to play them.
+ Unpacks movies and sounds (based on path in zipfile) to the corresponding
+ folder. Movies must be unpacked for FA to be able to play them.
This is a hack needed because the game updater can only handle bin and
gamedata.
"""
@@ -41,7 +41,11 @@ def unpack_movies_from_file(file_info: dict) -> None:
return
for zi in zf.infolist():
- if zi.filename.startswith("movies") and not zi.is_dir():
+ movie_or_sound = (
+ zi.filename.startswith("movies")
+ or zi.filename.startswith("sounds")
+ )
+ if movie_or_sound and not zi.is_dir():
tgtpath = os.path.join(APPDATA_DIR, zi.filename)
# copy only if file is different - check first if file
# exists, then if size is changed, then crc
@@ -53,11 +57,11 @@ def unpack_movies_from_file(file_info: dict) -> None:
zf.extract(zi, APPDATA_DIR)
-def unpack_movies(files: list[dict]) -> None:
- logger.info("Checking files for movies")
+def unpack_movies_and_sounds(files: list[dict]) -> None:
+ logger.info("Checking files for movies and sounds")
progress = QProgressDialog()
- progress.setWindowTitle("Updating Movies")
+ progress.setWindowTitle("Updating Movies and Sounds")
progress.setWindowFlags(Qt.WindowType.CustomizeWindowHint | Qt.WindowType.WindowTitleHint)
progress.setModal(True)
progress.setCancelButton(None)
@@ -66,6 +70,6 @@ def unpack_movies(files: list[dict]) -> None:
for index, file in enumerate(files, start=1):
filename = file["name"]
- progress.setLabelText(f"Checking for movies in {filename}...")
- unpack_movies_from_file(file)
+ progress.setLabelText(f"Checking for movies and sounds in {filename}...")
+ unpack_movies_and_sounds_from_file(file)
progress.setValue(index)
From 4c81a0d42e4ad246a4e17df3929d3964634dd16a Mon Sep 17 00:00:00 2001
From: gatsik <74517072+Gatsik@users.noreply.github.com>
Date: Fri, 3 May 2024 18:16:13 +0300
Subject: [PATCH 058/123] Updater: Do not close progress prematurely
---
src/fa/updater.py | 3 ---
1 file changed, 3 deletions(-)
diff --git a/src/fa/updater.py b/src/fa/updater.py
index e3d012c73..2e2758a5f 100644
--- a/src/fa/updater.py
+++ b/src/fa/updater.py
@@ -442,9 +442,6 @@ def do_update(self) -> None:
else:
self.result = UpdaterResult.SUCCESS
- # Hide progress dialog if it's still showing.
- self.progress.close()
-
# Integrated handlers for the various things that could go wrong
if self.result == UpdaterResult.CANCEL:
pass # The user knows damn well what happened here.
From d1e795f076bd213340d3f79e50274fc5eb3b99ee Mon Sep 17 00:00:00 2001
From: gatsik <74517072+Gatsik@users.noreply.github.com>
Date: Sat, 4 May 2024 09:41:36 +0300
Subject: [PATCH 059/123] Create api model for FeaturedModFile
---
src/api/featured_mod_api.py | 14 ++--
src/api/models/FeaturedModFile.py | 14 ++++
src/api/parsers/FeaturedModFileParser.py | 21 ++++++
src/fa/updater.py | 93 ++++++++++++------------
src/fa/utils.py | 11 ++-
5 files changed, 93 insertions(+), 60 deletions(-)
create mode 100644 src/api/models/FeaturedModFile.py
create mode 100644 src/api/parsers/FeaturedModFileParser.py
diff --git a/src/api/featured_mod_api.py b/src/api/featured_mod_api.py
index c779c9c2a..9d1f67b7c 100644
--- a/src/api/featured_mod_api.py
+++ b/src/api/featured_mod_api.py
@@ -2,6 +2,8 @@
from api.ApiAccessors import DataApiAccessor
from api.models.FeaturedMod import FeaturedMod
+from api.models.FeaturedModFile import FeaturedModFile
+from api.parsers.FeaturedModFileParser import FeaturedModFileParser
from api.parsers.FeaturedModParser import FeaturedModParser
logger = logging.getLogger(__name__)
@@ -27,15 +29,15 @@ def request_and_get_fmod_by_name(self, technicalName) -> FeaturedMod:
return self.featured_mod
-class FeaturedModFiles(DataApiAccessor):
+class FeaturedModFilesApiConnector(DataApiAccessor):
def __init__(self, mod_id: str, version: str) -> None:
super().__init__(f"/featuredMods/{mod_id}/files/{version}")
- self.featuredModFiles = []
+ self.featured_mod_files = []
- def handle_response(self, message):
- self.featuredModFiles = message["data"]
+ def handle_response(self, message: dict) -> None:
+ self.featured_mod_files = FeaturedModFileParser.parse_many(message["data"])
- def get_files(self):
+ def get_files(self) -> list[FeaturedModFile]:
self.requestData()
self.waitForCompletion()
- return self.featuredModFiles
+ return self.featured_mod_files
diff --git a/src/api/models/FeaturedModFile.py b/src/api/models/FeaturedModFile.py
new file mode 100644
index 000000000..8162c8540
--- /dev/null
+++ b/src/api/models/FeaturedModFile.py
@@ -0,0 +1,14 @@
+from dataclasses import dataclass
+
+
+@dataclass
+class FeaturedModFile:
+ uid: str
+ version: int
+ group: str
+ name: str
+ md5: str
+ url: str
+ cacheable_url: str
+ hmac_token: str
+ hmac_parameter: str
diff --git a/src/api/parsers/FeaturedModFileParser.py b/src/api/parsers/FeaturedModFileParser.py
new file mode 100644
index 000000000..ee2acecc7
--- /dev/null
+++ b/src/api/parsers/FeaturedModFileParser.py
@@ -0,0 +1,21 @@
+from api.models.FeaturedModFile import FeaturedModFile
+
+
+class FeaturedModFileParser:
+ @staticmethod
+ def parse(api_result: dict) -> FeaturedModFile:
+ return FeaturedModFile(
+ uid=api_result["id"],
+ version=api_result["version"],
+ group=api_result["group"],
+ name=api_result["name"],
+ md5=api_result["md5"],
+ url=api_result["url"],
+ cacheable_url=api_result["cacheableUrl"],
+ hmac_token=api_result["hmacToken"],
+ hmac_parameter=api_result["hmacParameter"],
+ )
+
+ @staticmethod
+ def parse_many(api_result: list[dict]) -> list[FeaturedModFile]:
+ return [FeaturedModFileParser.parse(file_info) for file_info in api_result]
diff --git a/src/fa/updater.py b/src/fa/updater.py
index 2e2758a5f..2b9293a3c 100644
--- a/src/fa/updater.py
+++ b/src/fa/updater.py
@@ -25,8 +25,9 @@
import util
from api.featured_mod_api import FeaturedModApiConnector
-from api.featured_mod_api import FeaturedModFiles
+from api.featured_mod_api import FeaturedModFilesApiConnector
from api.models.FeaturedMod import FeaturedMod
+from api.models.FeaturedModFile import FeaturedModFile
from api.sim_mod_updater import SimModFiles
from config import Settings
from fa.utils import unpack_movies_and_sounds
@@ -185,7 +186,7 @@ def run(self, *args, **kwargs):
return self.result
def get_files_to_update(self, mod_id: str, version: str) -> list[dict]:
- return FeaturedModFiles(mod_id, version).get_files()
+ return FeaturedModFilesApiConnector(mod_id, version).get_files()
def get_featured_mod_by_name(self, technical_name: str) -> FeaturedMod:
return FeaturedModApiConnector().request_and_get_fmod_by_name(technical_name)
@@ -194,50 +195,46 @@ def request_sim_url_by_uid(self, uid: str) -> str:
return SimModFiles().request_and_get_sim_mod_url_by_id(uid)
@staticmethod
- def _file_needs_update(file_info: dict) -> bool:
+ def _file_needs_update(file: FeaturedModFile) -> bool:
return (
- file_info["name"] != Settings.get("game/exe-name")
- and file_info["old_md5"] != file_info["md5"]
+ file.name != Settings.get("game/exe-name")
+ and file.old_md5 != file.md5
)
- def _calc_md5s(self, files: list[dict]) -> None:
+ def _calc_md5s(self, files: list[FeaturedModFile]) -> None:
self.progress.setMaximum(len(files))
- for index, file_info in enumerate(files, start=1):
+ for index, file in enumerate(files, start=1):
if self.progress.wasCanceled():
raise UpdaterCancellation()
- filegroup = file_info["group"]
- filename = file_info["name"]
- filepath = os.path.join(util.APPDATA_DIR, filegroup, filename)
+ filepath = os.path.join(util.APPDATA_DIR, file.group, file.name)
- self.progress.setLabelText(f"Calculating md5 for {filename}...")
+ self.progress.setLabelText(f"Calculating md5 for {file.name}...")
- file_info["old_md5"] = util.md5(filepath)
+ file.old_md5 = util.md5(filepath)
self.progress.setValue(index)
self.progress.setMaximum(0)
- def fetch_files(self, files: list[dict]) -> None:
+ def fetch_files(self, files: list[FeaturedModFile]) -> None:
for file in files:
self.fetch_single_file(file)
- def fetch_single_file(self, file_info: dict) -> None:
- name = file_info["name"]
- filegroup = file_info["group"]
- url = file_info["cacheableUrl"]
- target_dir = os.path.join(util.APPDATA_DIR, filegroup)
+ def fetch_single_file(self, file: FeaturedModFile) -> None:
+ target_dir = os.path.join(util.APPDATA_DIR, file.group)
+ url = file.cacheable_url
logger.info(f"Updater: Downloading {url}")
downloaded = download_file(
url=url,
target_dir=target_dir,
- name=name,
+ name=file.name,
category="Update",
silent=False,
- request_params={file_info["hmacParameter"]: file_info["hmacToken"]},
+ request_params={file.hmac_parameter: file.hmac_token},
label=f"Downloading FA file : {url}
",
)
@@ -248,46 +245,46 @@ def fetch_single_file(self, file_info: dict) -> None:
"Operation aborted while waiting for data.",
)
- def move_many_from_cache(self, files: list[dict]) -> None:
+ def move_many_from_cache(self, files: list[FeaturedModFile]) -> None:
for file in files:
self.move_from_cache(file)
- def move_from_cache(self, file_info: dict) -> None:
- src_dir = os.path.join(util.APPDATA_DIR, file_info["group"])
- cache_dir = os.path.join(util.GAME_CACHE_DIR, file_info["group"])
- if os.path.exists(os.path.join(cache_dir, file_info["md5"])):
+ def move_from_cache(self, file: FeaturedModFile) -> None:
+ src_dir = os.path.join(util.APPDATA_DIR, file.group)
+ cache_dir = os.path.join(util.GAME_CACHE_DIR, file.group)
+ if os.path.exists(os.path.join(cache_dir, file.md5)):
shutil.move(
- os.path.join(cache_dir, file_info["md5"]),
- os.path.join(src_dir, file_info["name"]),
+ os.path.join(cache_dir, file.md5),
+ os.path.join(src_dir, file.name),
)
def move_many_to_cache(self, files: list[dict]) -> None:
for file in files:
self.move_to_cache(file)
- def move_to_cache(self, file_info: dict) -> None:
- src_dir = os.path.join(util.APPDATA_DIR, file_info["group"])
- cache_dir = os.path.join(util.GAME_CACHE_DIR, file_info["group"])
- if os.path.exists(os.path.join(src_dir, file_info["name"])):
- md5 = util.md5(os.path.join(src_dir, file_info["name"]))
+ def move_to_cache(self, file: FeaturedModFile) -> None:
+ src_dir = os.path.join(util.APPDATA_DIR, file.group)
+ cache_dir = os.path.join(util.GAME_CACHE_DIR, file.group)
+ if os.path.exists(os.path.join(src_dir, file.name)):
+ md5 = util.md5(os.path.join(src_dir, file.name))
shutil.move(
- os.path.join(src_dir, file_info["name"]),
+ os.path.join(src_dir, file.name),
os.path.join(cache_dir, md5),
)
util.setAccessTime(os.path.join(cache_dir, md5))
- def replace_from_cache(self, file: dict) -> None:
+ def replace_from_cache(self, file: FeaturedModFile) -> None:
self.move_to_cache(file)
self.move_from_cache(file)
- def replace_many_from_cache(self, files: list[dict]) -> None:
+ def replace_many_from_cache(self, files: list[FeaturedModFile]) -> None:
for file in files:
self.replace_from_cache(file)
- def check_cache(self, files_to_check: list[dict]) -> None:
+ def check_cache(self, files_to_check: list[FeaturedModFile]) -> None:
replaceable_files, need_to_download = [], []
for file in files_to_check:
- cache_dir = os.path.join(util.GAME_CACHE_DIR, file["group"])
+ cache_dir = os.path.join(util.GAME_CACHE_DIR, file.group)
os.makedirs(cache_dir, exist_ok=True)
if self._is_cached(file):
replaceable_files.append(file)
@@ -296,16 +293,16 @@ def check_cache(self, files_to_check: list[dict]) -> None:
return replaceable_files, need_to_download
@staticmethod
- def _is_cached(file_info: dict) -> bool:
- cached_file = os.path.join(util.GAME_CACHE_DIR, file_info["group"], file_info["name"])
+ def _is_cached(file: FeaturedModFile) -> bool:
+ cached_file = os.path.join(util.GAME_CACHE_DIR, file.group, file.name)
return os.path.isfile(cached_file)
- def create_cache_subdirs(self, files: list[dict]) -> None:
+ def create_cache_subdirs(self, files: list[FeaturedModFile]) -> None:
for file in files:
- target = os.path.join(util.GAME_CACHE_DIR, file["group"])
+ target = os.path.join(util.GAME_CACHE_DIR, file.group)
os.makedirs(target, exist_ok=True)
- def update_files(self, files: list[dict]) -> None:
+ def update_files(self, files: list[FeaturedModFile]) -> None:
"""
Updates the files in the destination
subdirectory of the Forged Alliance path.
@@ -388,14 +385,14 @@ def patch_fa_executable(self, version: int) -> None:
file.seek(address)
file.write(version.to_bytes(4, "little"))
- def patch_fa_exe_if_needed(self, files: list[dict]) -> None:
+ def patch_fa_exe_if_needed(self, files: list[FeaturedModFile]) -> None:
for file in files:
- if file["name"] == Settings.get("game/exe-name"):
+ if file.name == Settings.get("game/exe-name"):
version = int(self._resolve_base_version(file))
self.patch_fa_executable(version)
return
- def update_featured_mod(self, modname: str, modversion: str) -> list[dict]:
+ def update_featured_mod(self, modname: str, modversion: str) -> list[FeaturedModFile]:
fmod = self.get_featured_mod_by_name(modname)
files = self.get_files_to_update(fmod.uid, modversion)
self.update_files(files)
@@ -406,11 +403,11 @@ def _resolve_modversion(self) -> str:
return str(max(self.modversions.values()))
return "latest"
- def _resolve_base_version(self, base_info: dict | None = None) -> str:
+ def _resolve_base_version(self, exe_info: FeaturedModFile | None = None) -> str:
if self.version:
return str(self.version)
- if base_info:
- return str(base_info["version"])
+ if exe_info:
+ return str(exe_info.version)
return "latest"
def do_update(self) -> None:
diff --git a/src/fa/utils.py b/src/fa/utils.py
index 33500a777..78355342a 100644
--- a/src/fa/utils.py
+++ b/src/fa/utils.py
@@ -6,6 +6,7 @@
from PyQt6.QtCore import Qt
from PyQt6.QtWidgets import QProgressDialog
+from api.models.FeaturedModFile import FeaturedModFile
from util import APPDATA_DIR
logger = logging.getLogger(__name__)
@@ -20,7 +21,7 @@ def crc32(filepath: str) -> int | None:
return None
-def unpack_movies_and_sounds_from_file(file_info: dict) -> None:
+def unpack_movies_and_sounds_from_file(file: FeaturedModFile) -> None:
"""
Unpacks movies and sounds (based on path in zipfile) to the corresponding
folder. Movies must be unpacked for FA to be able to play them.
@@ -30,8 +31,7 @@ def unpack_movies_and_sounds_from_file(file_info: dict) -> None:
# construct dirs
gd = os.path.join(APPDATA_DIR, "gamedata")
- fname = file_info["name"]
- origpath = os.path.join(gd, fname)
+ origpath = os.path.join(gd, file.name)
if os.path.exists(origpath) and zipfile.is_zipfile(origpath):
try:
@@ -57,7 +57,7 @@ def unpack_movies_and_sounds_from_file(file_info: dict) -> None:
zf.extract(zi, APPDATA_DIR)
-def unpack_movies_and_sounds(files: list[dict]) -> None:
+def unpack_movies_and_sounds(files: list[FeaturedModFile]) -> None:
logger.info("Checking files for movies and sounds")
progress = QProgressDialog()
@@ -69,7 +69,6 @@ def unpack_movies_and_sounds(files: list[dict]) -> None:
progress.setValue(0)
for index, file in enumerate(files, start=1):
- filename = file["name"]
- progress.setLabelText(f"Checking for movies and sounds in {filename}...")
+ progress.setLabelText(f"Checking for movies and sounds in {file.name}...")
unpack_movies_and_sounds_from_file(file)
progress.setValue(index)
From 6773cc56fbda12234ecf9ea6a78aeb84119bda52 Mon Sep 17 00:00:00 2001
From: gatsik <74517072+Gatsik@users.noreply.github.com>
Date: Sat, 4 May 2024 09:44:29 +0300
Subject: [PATCH 060/123] Updater: Get fa exe name from settings everywhere
---
src/fa/updater.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/fa/updater.py b/src/fa/updater.py
index 2b9293a3c..81cd8eb06 100644
--- a/src/fa/updater.py
+++ b/src/fa/updater.py
@@ -378,7 +378,7 @@ def download_fa_executable(self) -> bool:
)
def patch_fa_executable(self, version: int) -> None:
- exe_path = os.path.join(util.BIN_DIR, "ForgedAlliance.exe")
+ exe_path = os.path.join(util.BIN_DIR, Settings.get("game/exe-name"))
version_addresses = (0xd3d40, 0x47612d, 0x476666)
with open(exe_path, "rb+") as file:
for address in version_addresses:
From d30ba146f39ca9c84fe3041af95314846e0fbe6a Mon Sep 17 00:00:00 2001
From: gatsik <74517072+Gatsik@users.noreply.github.com>
Date: Sat, 4 May 2024 11:32:27 +0300
Subject: [PATCH 061/123] Do not add attributes to FeaturedModFile after it is
initialized
and keep the ability of it (and other api models)
to become immutable (frozen) in the future
---
src/fa/updater.py | 20 +++++++++++---------
1 file changed, 11 insertions(+), 9 deletions(-)
diff --git a/src/fa/updater.py b/src/fa/updater.py
index 81cd8eb06..2c8b94378 100644
--- a/src/fa/updater.py
+++ b/src/fa/updater.py
@@ -14,6 +14,7 @@
import stat
import time
from enum import Enum
+from functools import partial
from PyQt6.QtCore import QObject
from PyQt6.QtCore import Qt
@@ -195,15 +196,15 @@ def request_sim_url_by_uid(self, uid: str) -> str:
return SimModFiles().request_and_get_sim_mod_url_by_id(uid)
@staticmethod
- def _file_needs_update(file: FeaturedModFile) -> bool:
- return (
- file.name != Settings.get("game/exe-name")
- and file.old_md5 != file.md5
- )
+ def _file_needs_update(file: FeaturedModFile, md5s: dict[str, str]) -> bool:
+ incoming_md5 = file.md5
+ current_md5 = md5s[file.md5]
+ return file.name != Settings.get("game/exe-name") and current_md5 != incoming_md5
- def _calc_md5s(self, files: list[FeaturedModFile]) -> None:
+ def _calc_md5s(self, files: list[FeaturedModFile]) -> dict[str, str]:
self.progress.setMaximum(len(files))
+ result = {}
for index, file in enumerate(files, start=1):
if self.progress.wasCanceled():
@@ -213,10 +214,11 @@ def _calc_md5s(self, files: list[FeaturedModFile]) -> None:
self.progress.setLabelText(f"Calculating md5 for {file.name}...")
- file.old_md5 = util.md5(filepath)
+ result[file.md5] = util.md5(filepath)
self.progress.setValue(index)
self.progress.setMaximum(0)
+ return result
def fetch_files(self, files: list[FeaturedModFile]) -> None:
for file in files:
@@ -309,11 +311,11 @@ def update_files(self, files: list[FeaturedModFile]) -> None:
"""
self.create_cache_subdirs(files)
self.patch_fa_exe_if_needed(files)
- self._calc_md5s(files)
+ md5s = self._calc_md5s(files)
self.progress.setLabelText("Updating files...")
- to_update = list(filter(self._file_needs_update, files))
+ to_update = list(filter(partial(self._file_needs_update, md5s=md5s), files))
replacable_files, need_to_download = self.check_cache(to_update)
if self.keep_cache or self.in_session_cache:
From 2764bf7c65bee214f5477bcd6100789bdc0feef0 Mon Sep 17 00:00:00 2001
From: gatsik <74517072+Gatsik@users.noreply.github.com>
Date: Sun, 5 May 2024 13:04:14 +0300
Subject: [PATCH 062/123] Fail update if sim mod wasn't downloaded
download function doesn't (normally) raise exceptions and
'result' attribute gets assigned to UpdaterResult.SUCCESS
thanks to try..else block
---
src/fa/updater.py | 6 ++----
1 file changed, 2 insertions(+), 4 deletions(-)
diff --git a/src/fa/updater.py b/src/fa/updater.py
index 2c8b94378..dbbb2e53c 100644
--- a/src/fa/updater.py
+++ b/src/fa/updater.py
@@ -417,10 +417,8 @@ def do_update(self) -> None:
try:
if self.sim_mod:
uid, name = self.sim_mod
- if utils.downloadMod(self.request_sim_url_by_uid(uid), name):
- self.result = UpdaterResult.SUCCESS
- else:
- self.result = UpdaterResult.FAILURE
+ if not utils.downloadMod(self.request_sim_url_by_uid(uid), name):
+ raise UpdaterFailure("Sim mod wasn't downloaded")
else:
# Prepare FAF directory & all necessary files
self.prepare_bin_FAF()
From 817425a01fb61a06849e8a2fe3f9d51ac571f5b1 Mon Sep 17 00:00:00 2001
From: gatsik <74517072+Gatsik@users.noreply.github.com>
Date: Mon, 6 May 2024 22:12:25 +0300
Subject: [PATCH 063/123] Check possible map preview files more thoroughly
coop missions preview files do not contain version (e.g. '.v0039')
and also can be in different case
Windows doesn't care about cases, but Linux does
although client most likely doesn't work on Linux at the moment
we don't want to break Linux compatibility even further
---
src/fa/maps.py | 29 ++++++++++++++++++++++-------
1 file changed, 22 insertions(+), 7 deletions(-)
diff --git a/src/fa/maps.py b/src/fa/maps.py
index fc49c8d4f..c973e9b87 100644
--- a/src/fa/maps.py
+++ b/src/fa/maps.py
@@ -218,7 +218,10 @@ def genPrevFromDDS(sourcename: str, destname: str, small: bool = False) -> None:
raise
-def exportPreviewFromMap(mapname, positions=None):
+def export_preview_from_map(
+ mapname: str | None,
+ positions: dict | None = None,
+) -> None | dict[str, None | str | list[str]]:
"""
This method auto-upgrades the maps to have small and large preview images
"""
@@ -240,10 +243,11 @@ def exportPreviewFromMap(mapname, positions=None):
return previews
mapname = os.path.basename(mapdir).lower()
+ mapname_no_version, *_ = mapname.partition(".")
if isGeneratedMap(mapname):
mapfilename = os.path.join(mapdir, mapname + ".scmap")
else:
- mapfilename = os.path.join(mapdir, mapname.split(".")[0] + ".scmap")
+ mapfilename = os.path.join(mapdir, f"{mapname_no_version}.scmap")
mode = os.stat(mapdir)[0]
if not (mode and stat.S_IWRITE):
@@ -253,9 +257,20 @@ def exportPreviewFromMap(mapname, positions=None):
if not os.path.isdir(mapdir):
os.mkdir(mapdir)
- previewsmallname = os.path.join(mapdir, mapname + ".small.png")
- previewlargename = os.path.join(mapdir, mapname + ".large.png")
- previewddsname = os.path.join(mapdir, mapname + ".dds")
+ def plausible_mapname_preview_name(suffix: str) -> str:
+ casefold_names = (
+ f"{mapname}{suffix}".casefold(),
+ f"{mapname_no_version}{suffix}".casefold(),
+ )
+ for entry in os.listdir(mapdir):
+ plausible_preview = os.path.join(mapdir, entry)
+ if os.path.isfile(plausible_preview) and entry.casefold() in casefold_names:
+ return plausible_preview
+ return suffix
+
+ previewsmallname = plausible_mapname_preview_name(".small.png")
+ previewlargename = plausible_mapname_preview_name(".large.png")
+ previewddsname = plausible_mapname_preview_name(".dds")
cachepngname = os.path.join(util.MAP_PREVIEW_SMALL_DIR, mapname + ".png")
logger.debug("Generating preview from user maps for: " + mapname)
@@ -419,7 +434,7 @@ def preview(mapname, pixmap=False):
return util.THEME.icon(img, False, pixmap)
# Try to find in local map folder
- img = exportPreviewFromMap(mapname)
+ img = export_preview_from_map(mapname)
if (
img
@@ -471,7 +486,7 @@ def processMapFolderForUpload(mapDir, positions):
Zipping the file and creating thumbnails
"""
# creating thumbnail
- files = exportPreviewFromMap(mapDir, positions)["tozip"]
+ files = export_preview_from_map(mapDir, positions)["tozip"]
# abort zipping if there is insufficient previews
if len(files) != 3:
logger.debug("Insufficient previews for making an archive.")
From a50f36b4c53cc8cd165997156d7e3a2a3fc2ef4c Mon Sep 17 00:00:00 2001
From: gatsik <74517072+Gatsik@users.noreply.github.com>
Date: Tue, 7 May 2024 13:17:26 +0300
Subject: [PATCH 064/123] Do not ping server constantly while in game
ping server only if there was no message from it within a minute
currently 'keepalive' functionality is only enabled during game
and it was constantly pinging server every 3 seconds
which sometimes caused infinite reconnect loop
when client sends 'ping', doesn't get 'pong' within those 3 secs
and forces disonnect to start over again
the main version of why this happens is:
1. There is some connection error for whatever reason
2. Client tries to reconnect
- Reconnecting means exchanging 'startup' messages
between client and server, which includes some long messages
(e.g. player_info, which contains a list of all players
and is so long that actually requires several messages
to be fully sent) or messages that could be delayed
(e.g. ask_session)
- Keepalive timer continues to ping server every 3 seconds
as it stops only when game closes
3. The above connection/authentication process takes more
than 3 seconds, or catches bad timing
4. Client forces disconnect
and the loop repeats
probably this current keepalive logic should be reviewed
and checked, because there are 2 timers: _keepalive_timer
and _reconnect_timer, and _keepalive_timer doesn't
stop on disconnect (maybe it should)
anyway, looks like increasing 'ping' interval and pinging
only when there is no apparent conversation between client
and server is the right thing to do
---
src/client/connection.py | 34 +++++++++++++++++++---------------
1 file changed, 19 insertions(+), 15 deletions(-)
diff --git a/src/client/connection.py b/src/client/connection.py
index e1257b76e..620ec307d 100644
--- a/src/client/connection.py
+++ b/src/client/connection.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
import json
import logging
import sys
@@ -27,11 +29,11 @@ class ConnectionState(IntEnum):
class ServerReconnecter(QtCore.QObject):
- def __init__(self, connection):
+ def __init__(self, connection: ServerConnection) -> None:
QtCore.QObject.__init__(self)
self._connection = connection
connection.state_changed.connect(self.on_state_changed)
- connection.received_pong.connect(self._receive_pong)
+ connection.message_received.connect(self._receive_message)
self._connection_attempts = 0
self._reconnect_timer = QtCore.QTimer(self)
@@ -44,8 +46,8 @@ def __init__(self, connection):
self._keepalive = False
self._keepalive_timer = QtCore.QTimer(self)
self._keepalive_timer.timeout.connect(self._ping_connection)
- self.keepalive_interval = 3 * 1000
- self._waiting_for_pong = False
+ self.keepalive_interval = 60 * 1000
+ self._waiting_for_message = False
@property
def enabled(self):
@@ -71,7 +73,7 @@ def keepalive(self, value):
def _disable_keepalive(self):
self._keepalive_timer.stop()
- self._waiting_for_pong = False
+ self._waiting_for_message = False
def _enable_keepalive(self):
if not self._keepalive_timer.isActive():
@@ -126,25 +128,27 @@ def _ping_connection(self):
not self._enabled
or self._connection.state != ConnectionState.CONNECTED
):
- self._waiting_for_pong = False
+ self._waiting_for_message = False
return
# Prepare to reconnect immediately
self._connection_attempts = 0
- if self._waiting_for_pong:
- self._waiting_for_pong = False
+ if self._waiting_for_message:
+ self._waiting_for_message = False
# Force disconnect
# Note that it will force disconnect and reconnect if we
# reconnected on our own since last ping!
self._connection.disconnect_()
else:
- self._waiting_for_pong = True
- self._connection.send(dict(command="ping"))
+ self._waiting_for_message = True
+ self._connection.send({"command": "ping"})
- def _receive_pong(self):
- self._waiting_for_pong = False
+ def _receive_message(self):
+ self._waiting_for_message = False
+ if self.keepalive:
+ self._keepalive_timer.start() # restart
class ServerConnection(QtCore.QObject):
@@ -154,13 +158,14 @@ class ServerConnection(QtCore.QObject):
state_changed = QtCore.pyqtSignal(object)
connected = QtCore.pyqtSignal()
disconnected = QtCore.pyqtSignal()
- received_pong = QtCore.pyqtSignal()
+ message_received = QtCore.pyqtSignal()
access_url_ready = QtCore.pyqtSignal(QtCore.QUrl)
def __init__(self, host, port, dispatch):
QtCore.QObject.__init__(self)
self.socket = QWebSocket()
self.socket.binaryMessageReceived.connect(self.on_binary_message_received)
+ self.socket.binaryMessageReceived.connect(lambda: self.message_received.emit())
self.socket.errorOccurred.connect(self.socketError)
self.socket.stateChanged.connect(self.on_socket_state_change)
@@ -267,7 +272,7 @@ def set_upnp(self, port):
self.socket.localAddress().toString(), port, "UDP",
)
- def processDataFromServer(self, data):
+ def processDataFromServer(self, data: str) -> None:
self._data = ""
for line in data.splitlines():
action = json.loads(line)
@@ -277,7 +282,6 @@ def processDataFromServer(self, data):
self.send(dict(command="pong"))
elif command == "pong":
logger.debug("Server: PONG")
- self.received_pong.emit()
else:
try:
self._dispatch(action)
From aae49fa564df0f04fffbb3b12b0f60577179ece3 Mon Sep 17 00:00:00 2001
From: gatsik <74517072+Gatsik@users.noreply.github.com>
Date: Tue, 7 May 2024 14:18:15 +0300
Subject: [PATCH 065/123] Stop reconnect_timer when connection is established
so it doesn't continue to call connection.do_connect()
after successful reconnect
---
src/client/connection.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/client/connection.py b/src/client/connection.py
index 620ec307d..697581390 100644
--- a/src/client/connection.py
+++ b/src/client/connection.py
@@ -100,6 +100,7 @@ def on_state_changed(self, state):
def handle_connected(self):
self._connection_attempts = 0
+ self._reconnect_timer.stop()
def handle_reconnecting(self):
self._connection_attempts += 1
@@ -110,7 +111,6 @@ def handle_disconnected(self):
if self._connection_attempts < 3:
logger.info("Reconnecting immediately")
- self._reconnect_timer.stop()
self._connection.do_connect()
elif self._reconnect_timer.isActive():
return
From 4773a966bb8e600b9103a21bcfaca25a83197955 Mon Sep 17 00:00:00 2001
From: gatsik <74517072+Gatsik@users.noreply.github.com>
Date: Tue, 7 May 2024 20:36:56 +0300
Subject: [PATCH 066/123] util: Add GAME_CACHE_DIR constant
which is needed since ab20f214128d482dfb9faef6ced40b567d2f4ba4
ridiculous!
---
src/util/__init__.py | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/src/util/__init__.py b/src/util/__init__.py
index 6d758ce27..5ee297d77 100644
--- a/src/util/__init__.py
+++ b/src/util/__init__.py
@@ -58,6 +58,9 @@
# Cache for news images
NEWS_CACHE_DIR = os.path.join(CACHE_DIR, "news")
+# This contains cached game files
+GAME_CACHE_DIR = os.path.join(CACHE_DIR, "featured_mod")
+
# This contains cached data downloaded for FA extras
EXTRA_DIR = os.path.join(APPDATA_DIR, "extra")
@@ -163,6 +166,7 @@ def setPersonalDir():
APPDATA_DIR, PERSONAL_DIR, LUA_DIR, CACHE_DIR,
MAP_PREVIEW_SMALL_DIR, MAP_PREVIEW_LARGE_DIR, MOD_PREVIEW_DIR,
THEME_DIR, REPLAY_DIR, LOG_DIR, EXTRA_DIR, NEWS_CACHE_DIR,
+ GAME_CACHE_DIR,
]:
if not os.path.isdir(data_dir):
os.makedirs(data_dir)
From 35e5f96f6b046f37821060b0c6bc27fe1a83239a Mon Sep 17 00:00:00 2001
From: gatsik <74517072+Gatsik@users.noreply.github.com>
Date: Wed, 8 May 2024 15:08:38 +0300
Subject: [PATCH 067/123] replayserver: Fix PyQt's namespaces
---
src/fa/replayserver.py | 15 ++++++++++-----
1 file changed, 10 insertions(+), 5 deletions(-)
diff --git a/src/fa/replayserver.py b/src/fa/replayserver.py
index d315e9cb5..02eceea7c 100644
--- a/src/fa/replayserver.py
+++ b/src/fa/replayserver.py
@@ -1,3 +1,4 @@
+from __future__ import annotations
import json
import logging
@@ -26,13 +27,17 @@ class ReplayRecorder(QtCore.QObject):
"""
__logger = logging.getLogger(__name__)
- def __init__(self, parent, local_socket, *args, **kwargs):
+ def __init__(
+ self,
+ parent: ReplayServer,
+ local_socket: QtNetwork.QTcpSocket,
+ *args,
+ **kwargs,
+ ) -> None:
QtCore.QObject.__init__(self, *args, **kwargs)
self.parent = parent
self.inputSocket = local_socket
- self.inputSocket.setSocketOption(
- QtNetwork.QTcpSocket.KeepAliveOption, 1,
- )
+ self.inputSocket.setSocketOption(QtNetwork.QTcpSocket.SocketOption.KeepAliveOption, 1)
self.inputSocket.readyRead.connect(self.readDatas)
self.inputSocket.disconnected.connect(self.inputDisconnected)
self.__logger.info("FA connected locally.")
@@ -168,7 +173,7 @@ def writeReplayFile(self):
)
replay = QtCore.QFile(filename)
- replay.open(QtCore.QIODevice.OpenModeFlag.WriteOnly | QtCore.QIODevice.Text)
+ replay.open(QtCore.QIODevice.OpenModeFlag.WriteOnly | QtCore.QIODevice.OpenModeFlag.Text)
replay.write(json.dumps(self.replayInfo).encode('utf-8'))
replay.write(b'\n')
replay.write(QtCore.qCompress(self.replayData).toBase64())
From 4e96ec3713caa51dfcf28fd70abe4f26a24acae8 Mon Sep 17 00:00:00 2001
From: gatsik <74517072+Gatsik@users.noreply.github.com>
Date: Thu, 9 May 2024 20:30:38 +0300
Subject: [PATCH 068/123] Remove Qt ANGLE workaround
as it is not supported and was removed in Qt6
---
src/__main__.py | 16 ----------------
1 file changed, 16 deletions(-)
diff --git a/src/__main__.py b/src/__main__.py
index 8e3d3d06a..df9860c0a 100644
--- a/src/__main__.py
+++ b/src/__main__.py
@@ -36,24 +36,8 @@
cmd_parser = argparse.ArgumentParser(
description='FAF client commandline arguments.',
)
-cmd_parser.add_argument(
- '--qt-angle-workaround',
- action='store_true',
- help=(
- 'Use Qt5 ANGLE backend. Enable if some client '
- 'tabs appear frozen. On by default.'
- ),
-)
-cmd_parser.add_argument(
- '--no-qt-angle-workaround',
- action='store_true',
- help='Do not use Qt5 ANGLE backend.',
-)
args, trailing_args = cmd_parser.parse_known_args()
-if sys.platform == 'win32' and not args.no_qt_angle_workaround:
- os.environ.setdefault('QT_OPENGL', 'angle')
- os.environ.setdefault('QT_ANGLE_PLATFORM', 'd3d9')
path = os.path.join(os.path.dirname(sys.argv[0]), "PyQt6.uic.widget-plugins")
From 453fe7042e1cfd67fb06b9167328709b6ddaa5ba Mon Sep 17 00:00:00 2001
From: gatsik <74517072+Gatsik@users.noreply.github.com>
Date: Fri, 10 May 2024 15:13:06 +0300
Subject: [PATCH 069/123] Refactor updater
* run Updater's tasks in a separate thread
to not block the GUI thread
* make use of UpdaterProgressDialog which was in the code
since like project creation, but was never used
* use QEventLoop to processEvents in Updater (it is still undesirable
behaviour, but preferred to using QApplication.processEvents())
* factor out downloading sim mods from updater -- it doesn't fit
into updater, and also spawns GUI windows (and Updater runs in non-GUI thread)
now updater doesn't block GUI at all!
---
res/fa/updater/updater.ui | 397 +++++++++++++++------------
src/fa/mods.py | 14 +-
src/fa/update_processor.py | 285 ++++++++++++++++++++
src/fa/updater.py | 532 ++++++++++---------------------------
src/fa/updater_misc.py | 75 ++++++
src/fa/utils.py | 22 +-
6 files changed, 732 insertions(+), 593 deletions(-)
create mode 100644 src/fa/update_processor.py
create mode 100644 src/fa/updater_misc.py
diff --git a/res/fa/updater/updater.ui b/res/fa/updater/updater.ui
index ea13a733a..7c7ade422 100644
--- a/res/fa/updater/updater.ui
+++ b/res/fa/updater/updater.ui
@@ -6,159 +6,273 @@
",
+ )
+
+ def patch_fa_executable(self, version: int) -> None:
+ exe_path = os.path.join(util.BIN_DIR, Settings.get("game/exe-name"))
+ version_addresses = (0xd3d40, 0x47612d, 0x476666)
+ with open(exe_path, "rb+") as file:
+ for address in version_addresses:
+ file.seek(address)
+ file.write(version.to_bytes(4, "little"))
+
+ def patch_fa_exe_if_needed(self, files: list[FeaturedModFile]) -> None:
+ for file in files:
+ if file.name == Settings.get("game/exe-name"):
+ version = int(self._resolve_base_version(file))
+ self.patch_fa_executable(version)
+ return
+
+ def update_featured_mod(self, modname: str, modversion: str) -> list[FeaturedModFile]:
+ fmod = self.get_featured_mod_by_name(modname)
+ files = self.get_files_to_update(fmod.uid, modversion)
+ self.update_files(files)
+ return files
+
+ def _resolve_modversion(self) -> str:
+ if self.modversions:
+ return str(max(self.modversions.values()))
+ return "latest"
+
+ def _resolve_base_version(self, exe_info: FeaturedModFile | None = None) -> str:
+ if self.version:
+ return str(self.version)
+ if exe_info:
+ return str(exe_info.version)
+ return "latest"
+
+ def do_update(self) -> None:
+ """ The core function that does most of the actual update work."""
+ try:
+ # Prepare FAF directory & all necessary files
+ self.prepare_bin_FAF()
+ # Update the mod if it's requested
+ if self.featured_mod in ("faf", "fafbeta", "fafdevelop", "ladder1v1"):
+ self.update_featured_mod(self.featured_mod, self._resolve_base_version())
+ else:
+ # update faf first
+ self.update_featured_mod("faf", self._resolve_base_version())
+ # update featured mod then
+ self.update_featured_mod(self.featured_mod, self._resolve_modversion())
+ except UpdaterCancellation as e:
+ log("CANCELLED: {}".format(e), logger)
+ self.result = UpdaterResult.CANCEL
+ except Exception as e:
+ log("EXCEPTION: {}".format(e), logger)
+ logger.exception(f"EXCEPTION: {e}")
+ self.result = UpdaterResult.FAILURE
+ else:
+ self.result = UpdaterResult.SUCCESS
+ self.done.emit(self.result)
diff --git a/src/fa/updater.py b/src/fa/updater.py
index dbbb2e53c..402e64c2c 100644
--- a/src/fa/updater.py
+++ b/src/fa/updater.py
@@ -8,471 +8,205 @@
@author thygrrr
"""
+from __future__ import annotations
+
import logging
-import os
-import shutil
-import stat
-import time
-from enum import Enum
-from functools import partial
+from PyQt6.QtCore import QEventLoop
from PyQt6.QtCore import QObject
-from PyQt6.QtCore import Qt
+from PyQt6.QtCore import QThread
+from PyQt6.QtCore import pyqtSignal
from PyQt6.QtCore import pyqtSlot
-from PyQt6.QtWidgets import QApplication
+from PyQt6.QtGui import QTextCursor
from PyQt6.QtWidgets import QDialog
-from PyQt6.QtWidgets import QMessageBox
-from PyQt6.QtWidgets import QProgressDialog
import util
-from api.featured_mod_api import FeaturedModApiConnector
-from api.featured_mod_api import FeaturedModFilesApiConnector
-from api.models.FeaturedMod import FeaturedMod
-from api.models.FeaturedModFile import FeaturedModFile
-from api.sim_mod_updater import SimModFiles
-from config import Settings
-from fa.utils import unpack_movies_and_sounds
-from vaults.dialogs import download_file
-from vaults.modvault import utils
+from downloadManager import FileDownload
+from fa.updater_misc import ProgressInfo
+from fa.updater_misc import UpdaterResult
+from fa.updater_misc import clear_log
+from fa.updater_misc import failure_dialog
+from fa.updater_misc import log
+from fa.updater_misc import timestamp
+from src.fa.update_processor import UpdateProcessor
logger = logging.getLogger(__name__)
-# This contains a complete dump of everything that was supplied to logOutput
-debugLog = []
-
FormClass, BaseClass = util.THEME.loadUiType("fa/updater/updater.ui")
class UpdaterProgressDialog(FormClass, BaseClass):
- def __init__(self, parent):
+ aborted = pyqtSignal()
+
+ def __init__(self, parent, abortable: bool) -> None:
BaseClass.__init__(self, parent)
self.setupUi(self)
- self.logPlainTextEdit.setVisible(False)
+ self.setModal(True)
+ self.logPlainTextEdit.setLineWrapMode(self.logPlainTextEdit.LineWrapMode.NoWrap)
+ self.logFrame.setVisible(False)
self.adjustSize()
self.watches = []
+ if not abortable:
+ self.abortButton.hide()
+
+ self.rejected.connect(self.abort)
+ self.abortButton.clicked.connect(self.reject)
+ self.detailsButton.clicked.connect(self.change_details_visibility)
+
+ def change_details_visibility(self) -> None:
+ visible = self.logFrame.isVisible()
+ self.logFrame.setVisible(not visible)
+ self.adjustSize()
+
+ def abort(self) -> None:
+ self.aborted.emit()
+
@pyqtSlot(str)
- def appendLog(self, text):
+ def append_log(self, text: str) -> None:
self.logPlainTextEdit.appendPlainText(text)
+ def replace_last_log_line(self, text: str) -> None:
+ self.logPlainTextEdit.moveCursor(
+ QTextCursor.MoveOperation.StartOfLine,
+ QTextCursor.MoveMode.KeepAnchor,
+ )
+ self.logPlainTextEdit.textCursor().removeSelectedText()
+ self.logPlainTextEdit.insertPlainText(text)
+
@pyqtSlot(QObject)
- def addWatch(self, watch):
+ def add_watch(self, watch: QObject) -> None:
self.watches.append(watch)
- watch.finished.connect(self.watchFinished)
+ watch.finished.connect(self.watch_finished)
@pyqtSlot()
- def watchFinished(self) -> None:
+ def watch_finished(self) -> None:
for watch in self.watches:
- if not watch.isFinished():
+ if not watch.is_finished():
return
# equivalent to self.accept(), but clearer
self.done(QDialog.DialogCode.Accepted)
-def clearLog():
- global debugLog
- debugLog = []
-
-
-def log(string):
- logger.debug(string)
- debugLog.append(str(string))
-
-
-def dumpPlainText():
- return "\n".join(debugLog)
-
-
-def dumpHTML():
- return "
".join(debugLog)
-
-
-# A set of exceptions we use to see what goes wrong during asynchronous data
-# transfer waits
-class UpdaterCancellation(Exception):
- pass
-
-
-class UpdaterFailure(Exception):
- pass
-
-
-class UpdaterTimeout(Exception):
- pass
-
-
-class UpdaterResult(Enum):
- SUCCESS = 0 # Update successful
- NONE = -1 # Update operation is still ongoing
- FAILURE = 1 # An error occured during updating
- CANCEL = 2 # User cancelled the download process
- ILLEGAL = 3 # User has the wrong version of FA
- BUSY = 4 # Server is currently busy
- PASS = 5 # User refuses to update by canceling the wizard
-
-
class Updater(QObject):
"""
This is the class that does the actual installation work.
"""
+ finished = pyqtSignal()
+
def __init__(
- self,
- featured_mod: str,
- version: int | None = None,
- modversions: dict | None = None,
- sim_mod: tuple[str, str] | None = None,
- silent: bool = False,
- *args,
- **kwargs,
+ self,
+ featured_mod: str,
+ version: int | None = None,
+ modversions: dict | None = None,
+ silent: bool = False,
+ *args,
+ **kwargs,
):
"""
Constructor
"""
super().__init__(*args, **kwargs)
- self.featured_mod = featured_mod
- self.version = version
- self.modversions = modversions
- self.sim_mod = sim_mod
- self.silent = silent
-
- self.result = UpdaterResult.NONE
-
- self.keep_cache = not Settings.get(
- 'cache/do_not_keep', type=bool, default=True,
- )
- self.in_session_cache = Settings.get(
- 'cache/in_session', type=bool, default=False,
- )
-
- self.progress = QProgressDialog()
- if self.silent:
- self.progress.setCancelButton(None)
- else:
- self.progress.setCancelButtonText("Cancel")
- self.progress.setWindowFlags(
- Qt.WindowType.CustomizeWindowHint | Qt.WindowType.WindowTitleHint,
- )
- self.progress.setAutoClose(False)
- self.progress.setAutoReset(False)
- self.progress.setModal(1)
- self.progress.setWindowTitle(f"Updating {self.featured_mod.upper()}")
-
- def run(self, *args, **kwargs):
- clearLog()
- log("Update started at " + timestamp())
- log("Using appdata: " + util.APPDATA_DIR)
-
- self.progress.show()
- QApplication.processEvents()
-
- # Actual network code adapted from previous version
- self.progress.setLabelText("Connecting to update server...")
-
- if not self.progress.wasCanceled():
- log("Connected to update server at {}".format(timestamp()))
-
- self.do_update()
-
- self.progress.setLabelText("Cleaning up.")
-
- self.progress.close()
- else:
- log("Cancelled connecting to server.")
- self.result = UpdaterResult.CANCEL
-
- log("Update finished at {}".format(timestamp()))
- return self.result
-
- def get_files_to_update(self, mod_id: str, version: str) -> list[dict]:
- return FeaturedModFilesApiConnector(mod_id, version).get_files()
-
- def get_featured_mod_by_name(self, technical_name: str) -> FeaturedMod:
- return FeaturedModApiConnector().request_and_get_fmod_by_name(technical_name)
-
- def request_sim_url_by_uid(self, uid: str) -> str:
- return SimModFiles().request_and_get_sim_mod_url_by_id(uid)
-
- @staticmethod
- def _file_needs_update(file: FeaturedModFile, md5s: dict[str, str]) -> bool:
- incoming_md5 = file.md5
- current_md5 = md5s[file.md5]
- return file.name != Settings.get("game/exe-name") and current_md5 != incoming_md5
+ self.worker_thread = QThread()
+ self.update_processor = UpdateProcessor(featured_mod, version, modversions, silent)
+ self.update_processor.moveToThread(self.worker_thread)
- def _calc_md5s(self, files: list[FeaturedModFile]) -> dict[str, str]:
- self.progress.setMaximum(len(files))
+ self.update_processor.done.connect(self.on_update_done)
+ self.update_processor.md5_progress.connect(self.on_md5_progress)
+ self.update_processor.movies_progress.connect(self.on_movies_progress)
+ self.update_processor.game_progress.connect(self.on_game_progress)
+ self.update_processor.featured_mod_progress.connect(self.on_feat_mod_progress)
+ self.update_processor.download_progress.connect(self.on_download_progress)
+ self.update_processor.download_finished.connect(self.on_download_finished)
+ self.update_processor.download_started.connect(self.on_download_started)
+ self.worker_thread.started.connect(self.update_processor.do_update)
- result = {}
- for index, file in enumerate(files, start=1):
+ self.progress = UpdaterProgressDialog(None, silent)
+ self.progress.aborted.connect(self.abort)
- if self.progress.wasCanceled():
- raise UpdaterCancellation()
-
- filepath = os.path.join(util.APPDATA_DIR, file.group, file.name)
+ self.result = UpdaterResult.NONE
- self.progress.setLabelText(f"Calculating md5 for {file.name}...")
+ def on_movies_progress(self, info: ProgressInfo) -> None:
+ self.progress.moviesProgress.setMaximum(info.total)
+ self.progress.moviesProgress.setValue(info.progress)
+ self.progress.append_log(f"Checking for movies and sounds: {info.description}")
- result[file.md5] = util.md5(filepath)
+ def on_md5_progress(self, info: ProgressInfo) -> None:
+ self.progress.md5Progress.setMaximum(info.total)
+ self.progress.md5Progress.setValue(info.progress)
+ self.progress.append_log(f"Calculating md5: {info.description}")
- self.progress.setValue(index)
- self.progress.setMaximum(0)
- return result
+ def on_game_progress(self, info: ProgressInfo) -> None:
+ self.progress.gameProgress.setMaximum(info.total)
+ self.progress.gameProgress.setValue(info.progress)
+ self.progress.append_log(f"Copying game files: {info.description}")
- def fetch_files(self, files: list[FeaturedModFile]) -> None:
- for file in files:
- self.fetch_single_file(file)
+ def on_feat_mod_progress(self, info: ProgressInfo) -> None:
+ self.progress.modProgress.setMaximum(info.total)
+ self.progress.modProgress.setValue(info.progress)
+ self.progress.append_log(f"Processing something: {info.description}")
- def fetch_single_file(self, file: FeaturedModFile) -> None:
- target_dir = os.path.join(util.APPDATA_DIR, file.group)
+ def on_download_progress(self, dler: FileDownload) -> None:
+ if dler.bytes_total == 0:
+ return
- url = file.cacheable_url
- logger.info(f"Updater: Downloading {url}")
+ total = dler.bytes_total
+ ready = dler.bytes_progress
- downloaded = download_file(
- url=url,
- target_dir=target_dir,
- name=file.name,
- category="Update",
- silent=False,
- request_params={file.hmac_parameter: file.hmac_token},
- label=f"Downloading FA file : {url}
", - ) + total_mb = round(total / (1024 ** 2), 2) + ready_mb = round(ready / (1024 ** 2), 2) - if not downloaded: - # FIXME: the information about the reason is already given in the - # dowloadFile function, need to come up with better way probably - raise UpdaterCancellation( - "Operation aborted while waiting for data.", - ) - - def move_many_from_cache(self, files: list[FeaturedModFile]) -> None: - for file in files: - self.move_from_cache(file) - - def move_from_cache(self, file: FeaturedModFile) -> None: - src_dir = os.path.join(util.APPDATA_DIR, file.group) - cache_dir = os.path.join(util.GAME_CACHE_DIR, file.group) - if os.path.exists(os.path.join(cache_dir, file.md5)): - shutil.move( - os.path.join(cache_dir, file.md5), - os.path.join(src_dir, file.name), - ) - - def move_many_to_cache(self, files: list[dict]) -> None: - for file in files: - self.move_to_cache(file) - - def move_to_cache(self, file: FeaturedModFile) -> None: - src_dir = os.path.join(util.APPDATA_DIR, file.group) - cache_dir = os.path.join(util.GAME_CACHE_DIR, file.group) - if os.path.exists(os.path.join(src_dir, file.name)): - md5 = util.md5(os.path.join(src_dir, file.name)) - shutil.move( - os.path.join(src_dir, file.name), - os.path.join(cache_dir, md5), - ) - util.setAccessTime(os.path.join(cache_dir, md5)) - - def replace_from_cache(self, file: FeaturedModFile) -> None: - self.move_to_cache(file) - self.move_from_cache(file) - - def replace_many_from_cache(self, files: list[FeaturedModFile]) -> None: - for file in files: - self.replace_from_cache(file) - - def check_cache(self, files_to_check: list[FeaturedModFile]) -> None: - replaceable_files, need_to_download = [], [] - for file in files_to_check: - cache_dir = os.path.join(util.GAME_CACHE_DIR, file.group) - os.makedirs(cache_dir, exist_ok=True) - if self._is_cached(file): - replaceable_files.append(file) - else: - need_to_download.append(file) - return replaceable_files, need_to_download - - @staticmethod - def _is_cached(file: FeaturedModFile) -> bool: - cached_file = os.path.join(util.GAME_CACHE_DIR, file.group, file.name) - return os.path.isfile(cached_file) - - def create_cache_subdirs(self, files: list[FeaturedModFile]) -> None: - for file in files: - target = os.path.join(util.GAME_CACHE_DIR, file.group) - os.makedirs(target, exist_ok=True) - - def update_files(self, files: list[FeaturedModFile]) -> None: - """ - Updates the files in the destination - subdirectory of the Forged Alliance path. - """ - self.create_cache_subdirs(files) - self.patch_fa_exe_if_needed(files) - md5s = self._calc_md5s(files) + def construct_bar(blockchar: str = "=", fillchar: str = " ") -> str: + num_blocks = round(20 * ready / total) + empty_blocks = 20 - num_blocks + return f"[{blockchar * num_blocks}{fillchar * empty_blocks}]" - self.progress.setLabelText("Updating files...") + bar = construct_bar() + percent_text = f"{100 * ready / total:.1f}%" + text = f"{bar} {percent_text} ({ready_mb} MB / {total_mb} MB)" + self.progress.replace_last_log_line(text) - to_update = list(filter(partial(self._file_needs_update, md5s=md5s), files)) - replacable_files, need_to_download = self.check_cache(to_update) + def on_download_finished(self, dler: FileDownload) -> None: + self.progress.append_log("Finished downloading.") - if self.keep_cache or self.in_session_cache: - self.replace_many_from_cache(replacable_files) - self.move_many_to_cache(need_to_download) - else: - self.move_many_from_cache(replacable_files) + def on_download_started(self, dler: FileDownload) -> None: + self.progress.append_log(f"Downloading file from {dler.addr}\n") - self.fetch_files(need_to_download) + def run(self) -> UpdaterResult: + clear_log() + log(f"Update started at {timestamp()}", logger) + log(f"Using appdata: {util.APPDATA_DIR}", logger) - unpack_movies_and_sounds(files) - log("Updates applied successfully.") + self.progress.show() + self.worker_thread.start() - def prepare_bin_FAF(self) -> None: - """ - Creates all necessary files in the binFAF folder, which contains - a modified copy of all that is in the standard bin folder of - Forged Alliance - """ - self.progress.setLabelText("Preparing binFAF...") - - # now we check if we've got a binFAF folder - FABindir = os.path.join(Settings.get("ForgedAlliance/app/path"), "bin") - FAFdir = util.BIN_DIR - - # Try to copy without overwriting, but fill in any missing files, - # otherwise it might miss some files to update - root_src_dir = FABindir - root_dst_dir = FAFdir - - for src_dir, _, files in os.walk(root_src_dir): - dst_dir = src_dir.replace(root_src_dir, root_dst_dir) - if not os.path.exists(dst_dir): - os.mkdir(dst_dir) - for file_ in files: - src_file = os.path.join(src_dir, file_) - dst_file = os.path.join(dst_dir, file_) - if not os.path.exists(dst_file): - shutil.copy(src_file, dst_dir) - st = os.stat(dst_file) - # make all files we were considering writable, because we may - # need to patch them - os.chmod(dst_file, st.st_mode | stat.S_IWRITE) - - self.download_fa_executable() - - def download_fa_executable(self) -> bool: - fa_exe_name = Settings.get("game/exe-name") - fa_exe = os.path.join(util.BIN_DIR, fa_exe_name) - - if os.path.isfile(fa_exe): - return True - - url = Settings.get("game/exe-url") - return download_file( - url=url, - target_dir=util.BIN_DIR, - name=fa_exe_name, - category="Update", - silent=False, - label=f"Downloading FA file : {url}
",
- )
+ loop = QEventLoop()
+ self.worker_thread.finished.connect(loop.quit)
+ loop.exec()
- def patch_fa_executable(self, version: int) -> None:
- exe_path = os.path.join(util.BIN_DIR, Settings.get("game/exe-name"))
- version_addresses = (0xd3d40, 0x47612d, 0x476666)
- with open(exe_path, "rb+") as file:
- for address in version_addresses:
- file.seek(address)
- file.write(version.to_bytes(4, "little"))
-
- def patch_fa_exe_if_needed(self, files: list[FeaturedModFile]) -> None:
- for file in files:
- if file.name == Settings.get("game/exe-name"):
- version = int(self._resolve_base_version(file))
- self.patch_fa_executable(version)
- return
+ self.progress.accept()
+ log(f"Update finished at {timestamp()}", logger)
+ return self.result
- def update_featured_mod(self, modname: str, modversion: str) -> list[FeaturedModFile]:
- fmod = self.get_featured_mod_by_name(modname)
- files = self.get_files_to_update(fmod.uid, modversion)
- self.update_files(files)
- return files
-
- def _resolve_modversion(self) -> str:
- if self.modversions:
- return str(max(self.modversions.values()))
- return "latest"
-
- def _resolve_base_version(self, exe_info: FeaturedModFile | None = None) -> str:
- if self.version:
- return str(self.version)
- if exe_info:
- return str(exe_info.version)
- return "latest"
-
- def do_update(self) -> None:
- """ The core function that does most of the actual update work."""
- try:
- if self.sim_mod:
- uid, name = self.sim_mod
- if not utils.downloadMod(self.request_sim_url_by_uid(uid), name):
- raise UpdaterFailure("Sim mod wasn't downloaded")
- else:
- # Prepare FAF directory & all necessary files
- self.prepare_bin_FAF()
- # Update the mod if it's requested
- if self.featured_mod in ("faf", "fafbeta", "fafdevelop", "ladder1v1"):
- self.update_featured_mod(self.featured_mod, self._resolve_base_version())
- else:
- # update faf first
- self.update_featured_mod("faf", self._resolve_base_version())
- # update featured mod then
- self.update_featured_mod(self.featured_mod, self._resolve_modversion())
- except UpdaterCancellation as e:
- log("CANCELLED: {}".format(e))
- self.result = UpdaterResult.CANCEL
- except BaseException as e:
- log("EXCEPTION: {}".format(e))
- self.result = UpdaterResult.FAILURE
- else:
- self.result = UpdaterResult.SUCCESS
+ def on_update_done(self, result: UpdaterResult) -> None:
+ self.result = result
+ self.handle_result_if_needed(result)
+ self.stop_thread()
+ def handle_result_if_needed(self, result: UpdaterResult) -> None:
# Integrated handlers for the various things that could go wrong
- if self.result == UpdaterResult.CANCEL:
+ if result == UpdaterResult.CANCEL:
pass # The user knows damn well what happened here.
- elif self.result == UpdaterResult.PASS:
- QMessageBox.information(
- QApplication.activeWindow(),
- "Installation Required",
- "You can't play without a legal version of Forged Alliance.",
- )
- elif self.result == UpdaterResult.BUSY:
- QMessageBox.information(
- QApplication.activeWindow(),
- "Server Busy",
- (
- "The Server is busy preparing new patch files.
Try "
- "again later."
- ),
- )
- elif self.result == UpdaterResult.FAILURE:
- failureDialog()
-
- # If nothing terribly bad happened until now,
- # the operation is a success and/or the client can display what's up.
- return self.result
+ elif result == UpdaterResult.FAILURE:
+ failure_dialog()
+ def abort(self) -> None:
+ self.result = UpdaterResult.CANCEL
+ self.stop_thread()
-def timestamp():
- return time.strftime("%Y-%m-%d %H:%M:%S")
-
-
-# This is a pretty rough port of the old installer wizard. It works, but will
-# need some work later
-def failureDialog():
- """
- The dialog that shows the user the log if something went wrong.
- """
- raise Exception(dumpPlainText())
+ def stop_thread(self) -> None:
+ self.worker_thread.quit()
+ self.worker_thread.wait(1000)
diff --git a/src/fa/updater_misc.py b/src/fa/updater_misc.py
new file mode 100644
index 000000000..09774e5a9
--- /dev/null
+++ b/src/fa/updater_misc.py
@@ -0,0 +1,75 @@
+import logging
+import time
+from enum import Enum
+from typing import NamedTuple
+
+from PyQt6.QtCore import Qt
+from PyQt6.QtWidgets import QApplication
+from PyQt6.QtWidgets import QMessageBox
+
+
+# A set of exceptions we use to see what goes wrong during asynchronous data
+# transfer waits
+class UpdaterCancellation(Exception):
+ pass
+
+
+class UpdaterFailure(Exception):
+ pass
+
+
+class UpdaterTimeout(Exception):
+ pass
+
+
+class UpdaterResult(Enum):
+ SUCCESS = 0 # Update successful
+ NONE = -1 # Update operation is still ongoing
+ FAILURE = 1 # An error occured during updating
+ CANCEL = 2 # User cancelled the download process
+
+
+class ProgressInfo(NamedTuple):
+ progress: int
+ total: int
+ description: str = ""
+
+
+# This contains a complete dump of everything that was supplied to logOutput
+debug_log = []
+
+
+def clear_log() -> None:
+ global debug_log
+ debug_log = []
+
+
+def log(string: str, loger: logging.Logger) -> None:
+ loger.debug(string)
+ debug_log.append(str(string))
+
+
+def dump_plain_text() -> str:
+ return "\n".join(debug_log)
+
+
+def dump_HTML() -> str:
+ return "
".join(debug_log)
+
+
+def timestamp() -> str:
+ return time.strftime("%Y-%m-%d %H:%M:%S")
+
+
+# It works, but will need some work later
+def failure_dialog() -> None:
+ """
+ The dialog that shows the user the log if something went wrong.
+ """
+ mbox = QMessageBox()
+ mbox.setParent(QApplication.activeWindow())
+ mbox.setWindowFlags(Qt.WindowType.Dialog)
+ mbox.setWindowTitle("Update Failed")
+ mbox.setText("An error occurred during downloading/copying/moving files")
+ mbox.setDetailedText(dump_plain_text())
+ mbox.exec()
diff --git a/src/fa/utils.py b/src/fa/utils.py
index 78355342a..f45e1ef52 100644
--- a/src/fa/utils.py
+++ b/src/fa/utils.py
@@ -3,9 +3,6 @@
import os
import zipfile
-from PyQt6.QtCore import Qt
-from PyQt6.QtWidgets import QProgressDialog
-
from api.models.FeaturedModFile import FeaturedModFile
from util import APPDATA_DIR
@@ -21,7 +18,7 @@ def crc32(filepath: str) -> int | None:
return None
-def unpack_movies_and_sounds_from_file(file: FeaturedModFile) -> None:
+def unpack_movies_and_sounds(file: FeaturedModFile) -> None:
"""
Unpacks movies and sounds (based on path in zipfile) to the corresponding
folder. Movies must be unpacked for FA to be able to play them.
@@ -55,20 +52,3 @@ def unpack_movies_and_sounds_from_file(file: FeaturedModFile) -> None:
or crc32(tgtpath) != zi.CRC
):
zf.extract(zi, APPDATA_DIR)
-
-
-def unpack_movies_and_sounds(files: list[FeaturedModFile]) -> None:
- logger.info("Checking files for movies and sounds")
-
- progress = QProgressDialog()
- progress.setWindowTitle("Updating Movies and Sounds")
- progress.setWindowFlags(Qt.WindowType.CustomizeWindowHint | Qt.WindowType.WindowTitleHint)
- progress.setModal(True)
- progress.setCancelButton(None)
- progress.setMaximum(len(files))
- progress.setValue(0)
-
- for index, file in enumerate(files, start=1):
- progress.setLabelText(f"Checking for movies and sounds in {file.name}...")
- unpack_movies_and_sounds_from_file(file)
- progress.setValue(index)
From 20af1d525ded73c9b167e7771d84212e720b72ae Mon Sep 17 00:00:00 2001
From: gatsik <74517072+Gatsik@users.noreply.github.com>
Date: Fri, 10 May 2024 15:47:49 +0300
Subject: [PATCH 070/123] Fix up BaseDownload
* use QEventLoop to wait for finish (looks like it works
from within non-GUI thread, while QApplication.processEvents()
does not)
* emit progress signal on QNetworkReply's progress,
not on every 8 KB of data written to disk, which caused an
enormous signal spam
---
src/downloadManager/__init__.py | 17 ++++++++++-------
1 file changed, 10 insertions(+), 7 deletions(-)
diff --git a/src/downloadManager/__init__.py b/src/downloadManager/__init__.py
index 2f475f7c7..ba4be5a27 100644
--- a/src/downloadManager/__init__.py
+++ b/src/downloadManager/__init__.py
@@ -6,7 +6,6 @@
from io import BytesIO
from PyQt6 import QtGui
-from PyQt6 import QtWidgets
from PyQt6.QtCore import QEventLoop
from PyQt6.QtCore import QFile
from PyQt6.QtCore import QIODevice
@@ -113,9 +112,10 @@ def _atFinished(self):
self._sock_finished = True
self._kick_read()
- def _atProgress(self, recv, total):
+ def _atProgress(self, recv: int, total: int) -> None:
self.bytes_progress = recv
self.bytes_total = total
+ self.progress.emit(self)
def _kick_read(self): # Don't run the read loop more than once at a time
if self._reading:
@@ -138,7 +138,6 @@ def _readloop(self):
else:
bs = self.blocksize
self.dest.write(self._dfile.read(bs))
- self.progress.emit(self)
def succeeded(self):
return not self.error and not self.canceled
@@ -146,10 +145,14 @@ def succeeded(self):
def failed(self) -> bool:
return not self.succeeded()
- def waitForCompletion(self):
- waitFlag = QEventLoop.ProcessEventsFlag.WaitForMoreEvents
- while self._running:
- QtWidgets.QApplication.processEvents(waitFlag)
+ def waitForCompletion(self) -> None:
+ if not self._running:
+ return
+
+ wait_flag = QEventLoop.ProcessEventsFlag.WaitForMoreEvents
+ loop = QEventLoop()
+ self.finished.connect(loop.quit)
+ loop.exec(wait_flag)
class FileDownload(BaseDownload):
From b0889877bed244b01764189abfa2de83d1aa8d92 Mon Sep 17 00:00:00 2001
From: gatsik <74517072+Gatsik@users.noreply.github.com>
Date: Sat, 11 May 2024 16:56:11 +0300
Subject: [PATCH 071/123] Provide clearer info about update process
* add label, which tells about current and total featured mods to be updated
---
res/fa/updater/updater.ui | 27 ++++++++++-------
src/fa/update_processor.py | 32 ++++++++++++--------
src/fa/updater.py | 61 +++++++++++++++++++++++---------------
3 files changed, 73 insertions(+), 47 deletions(-)
diff --git a/res/fa/updater/updater.ui b/res/fa/updater/updater.ui
index 7c7ade422..415cd055f 100644
--- a/res/fa/updater/updater.ui
+++ b/res/fa/updater/updater.ui
@@ -200,7 +200,7 @@
| Size: {width} x {height} km |
| Rating: {rating} |
| {reviews} Rated this |
| Uploaded {date} |
| Size: {width} x {height} km |
| Rating: {rating} ({reviews}) |
| Uploaded: {date} |
| Author: {author} |
| Rating: {rating} |
| {reviews} Rated this |
| Uploaded {date} |
| Author: {author} |
| Rating: {rating} ({reviews}) |
| Uploaded {date} |
uc&N&2;2>4(HR@tg($USV%#3EOGFCnDAJklUdM;2QZU*>0~^g9pK>nI(gG7
z=*qN?#{mkHnAfHm0vA1!S?^W74zd+STORK?4W5SMt}JXbXd-9nWuT2t8Xf^Rt6w5zi_tr