diff --git a/bioconda_utils/_version.py b/bioconda_utils/_version.py index cd999ec6eb..8bccee040a 100644 --- a/bioconda_utils/_version.py +++ b/bioconda_utils/_version.py @@ -1,4 +1,3 @@ - # This file helps to compute a version number in source trees obtained from # git-archive tarball (such as those provided by githubs download-from-tag # feature). Distribution tarballs (built by setup.py sdist) and build @@ -58,17 +57,18 @@ class NotThisMethod(Exception): def register_vcs_handler(vcs, method): # decorator """Decorator to mark a method as the handler for a particular VCS.""" + def decorate(f): """Store f in HANDLERS[vcs][method].""" if vcs not in HANDLERS: HANDLERS[vcs] = {} HANDLERS[vcs][method] = f return f + return decorate -def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, - env=None): +def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, env=None): """Call the given command(s).""" assert isinstance(commands, list) p = None @@ -76,10 +76,13 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, try: dispcmd = str([c] + args) # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen([c] + args, cwd=cwd, env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr - else None)) + p = subprocess.Popen( + [c] + args, + cwd=cwd, + env=env, + stdout=subprocess.PIPE, + stderr=(subprocess.PIPE if hide_stderr else None), + ) break except EnvironmentError: e = sys.exc_info()[1] @@ -116,16 +119,22 @@ def versions_from_parentdir(parentdir_prefix, root, verbose): for i in range(3): dirname = os.path.basename(root) if dirname.startswith(parentdir_prefix): - return {"version": dirname[len(parentdir_prefix):], - "full-revisionid": None, - "dirty": False, "error": None, "date": None} + return { + "version": dirname[len(parentdir_prefix) :], + "full-revisionid": None, + "dirty": False, + "error": None, + "date": None, + } else: rootdirs.append(root) root = os.path.dirname(root) # up a level if verbose: - print("Tried directories %s but none started with prefix %s" % - (str(rootdirs), parentdir_prefix)) + print( + "Tried directories %s but none started with prefix %s" + % (str(rootdirs), parentdir_prefix) + ) raise NotThisMethod("rootdir doesn't start with parentdir_prefix") @@ -181,7 +190,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of # just "foo-1.0". If we see a "tag: " prefix, prefer those. TAG = "tag: " - tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)]) + tags = set([r[len(TAG) :] for r in refs if r.startswith(TAG)]) if not tags: # Either we're using git < 1.8.3, or there really are no tags. We use # a heuristic: assume all version tags have a digit. The old git %d @@ -190,7 +199,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): # between branches and tags. By ignoring refnames without digits, we # filter out many common branch names like "release" and # "stabilization", as well as "HEAD" and "master". - tags = set([r for r in refs if re.search(r'\d', r)]) + tags = set([r for r in refs if re.search(r"\d", r)]) if verbose: print("discarding '%s', no digits" % ",".join(refs - tags)) if verbose: @@ -198,19 +207,26 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): for ref in sorted(tags): # sorting will prefer e.g. "2.0" over "2.0rc1" if ref.startswith(tag_prefix): - r = ref[len(tag_prefix):] + r = ref[len(tag_prefix) :] if verbose: print("picking %s" % r) - return {"version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": None, - "date": date} + return { + "version": r, + "full-revisionid": keywords["full"].strip(), + "dirty": False, + "error": None, + "date": date, + } # no suitable tags, so version is "0+unknown", but full hex is still there if verbose: print("no suitable tags, using unknown + full revision id") - return {"version": "0+unknown", - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": "no suitable tags", "date": None} + return { + "version": "0+unknown", + "full-revisionid": keywords["full"].strip(), + "dirty": False, + "error": "no suitable tags", + "date": None, + } @register_vcs_handler("git", "pieces_from_vcs") @@ -225,8 +241,7 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): if sys.platform == "win32": GITS = ["git.cmd", "git.exe"] - out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, - hide_stderr=True) + out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, hide_stderr=True) if rc != 0: if verbose: print("Directory %s not under git control" % root) @@ -234,10 +249,19 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = run_command(GITS, ["describe", "--tags", "--dirty", - "--always", "--long", - "--match", "%s*" % tag_prefix], - cwd=root) + describe_out, rc = run_command( + GITS, + [ + "describe", + "--tags", + "--dirty", + "--always", + "--long", + "--match", + "%s*" % tag_prefix, + ], + cwd=root, + ) # --long was added in git-1.5.5 if describe_out is None: raise NotThisMethod("'git describe' failed") @@ -260,17 +284,16 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): dirty = git_describe.endswith("-dirty") pieces["dirty"] = dirty if dirty: - git_describe = git_describe[:git_describe.rindex("-dirty")] + git_describe = git_describe[: git_describe.rindex("-dirty")] # now we have TAG-NUM-gHEX or HEX if "-" in git_describe: # TAG-NUM-gHEX - mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) + mo = re.search(r"^(.+)-(\d+)-g([0-9a-f]+)$", git_describe) if not mo: # unparseable. Maybe git-describe is misbehaving? - pieces["error"] = ("unable to parse git-describe output: '%s'" - % describe_out) + pieces["error"] = "unable to parse git-describe output: '%s'" % describe_out return pieces # tag @@ -279,10 +302,12 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): if verbose: fmt = "tag '%s' doesn't start with prefix '%s'" print(fmt % (full_tag, tag_prefix)) - pieces["error"] = ("tag '%s' doesn't start with prefix '%s'" - % (full_tag, tag_prefix)) + pieces["error"] = "tag '%s' doesn't start with prefix '%s'" % ( + full_tag, + tag_prefix, + ) return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix):] + pieces["closest-tag"] = full_tag[len(tag_prefix) :] # distance: number of commits since tag pieces["distance"] = int(mo.group(2)) @@ -293,13 +318,13 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): else: # HEX: no tags pieces["closest-tag"] = None - count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], - cwd=root) + count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], cwd=root) pieces["distance"] = int(count_out) # total number of commits # commit date: see ISO-8601 comment in git_versions_from_keywords() - date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], - cwd=root)[0].strip() + date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[ + 0 + ].strip() pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) return pieces @@ -330,8 +355,7 @@ def render_pep440(pieces): rendered += ".dirty" else: # exception #1 - rendered = "0+untagged.%d.g%s" % (pieces["distance"], - pieces["short"]) + rendered = "0+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) if pieces["dirty"]: rendered += ".dirty" return rendered @@ -445,11 +469,13 @@ def render_git_describe_long(pieces): def render(pieces, style): """Render the given version pieces into the requested style.""" if pieces["error"]: - return {"version": "unknown", - "full-revisionid": pieces.get("long"), - "dirty": None, - "error": pieces["error"], - "date": None} + return { + "version": "unknown", + "full-revisionid": pieces.get("long"), + "dirty": None, + "error": pieces["error"], + "date": None, + } if not style or style == "default": style = "pep440" # the default @@ -469,9 +495,13 @@ def render(pieces, style): else: raise ValueError("unknown style '%s'" % style) - return {"version": rendered, "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], "error": None, - "date": pieces.get("date")} + return { + "version": rendered, + "full-revisionid": pieces["long"], + "dirty": pieces["dirty"], + "error": None, + "date": pieces.get("date"), + } def get_versions(): @@ -485,8 +515,7 @@ def get_versions(): verbose = cfg.verbose try: - return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, - verbose) + return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, verbose) except NotThisMethod: pass @@ -495,13 +524,16 @@ def get_versions(): # versionfile_source is the relative path from the top of the source # tree (where the .git directory might live) to this file. Invert # this to find the root from __file__. - for i in cfg.versionfile_source.split('/'): + for i in cfg.versionfile_source.split("/"): root = os.path.dirname(root) except NameError: - return {"version": "0+unknown", "full-revisionid": None, - "dirty": None, - "error": "unable to find root of source tree", - "date": None} + return { + "version": "0+unknown", + "full-revisionid": None, + "dirty": None, + "error": "unable to find root of source tree", + "date": None, + } try: pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) @@ -515,6 +547,10 @@ def get_versions(): except NotThisMethod: pass - return {"version": "0+unknown", "full-revisionid": None, - "dirty": None, - "error": "unable to compute version", "date": None} + return { + "version": "0+unknown", + "full-revisionid": None, + "dirty": None, + "error": "unable to compute version", + "date": None, + } diff --git a/bioconda_utils/aiopipe.py b/bioconda_utils/aiopipe.py index 75eb64c81e..662fec93c9 100644 --- a/bioconda_utils/aiopipe.py +++ b/bioconda_utils/aiopipe.py @@ -8,6 +8,7 @@ import signal from concurrent.futures import ProcessPoolExecutor + try: from concurrent.futures import BrokenExecutor except ImportError: @@ -29,14 +30,17 @@ logger = logging.getLogger(__name__) # pylint: disable=invalid-name -ITEM = TypeVar('ITEM') +ITEM = TypeVar("ITEM") + class EndProcessing(BaseException): """Raised by `AsyncFilter` to tell `AsyncPipeline` to stop processing""" + class EndProcessingItem(Exception): """Raised to indicate that an item should not be processed further""" - __slots__ = ['item', 'args'] + + __slots__ = ["item", "args"] template = "broken: %s" level = logging.INFO @@ -62,6 +66,7 @@ def name(self): class AsyncFilter(abc.ABC, Generic[ITEM]): """Function object type called by Scanner""" + def __init__(self, pipeline: "AsyncPipeline", *_args, **_kwargs) -> None: self.pipeline = pipeline @@ -109,7 +114,9 @@ async def shutdown(self, sig=None) -> None: if sig == signal.SIGINT: logger.error("Ctrl-C pressed - aborting...") self.proc_pool_executor.shutdown() - tasks = [t for t in asyncio.Task.all_tasks() if t != asyncio.Task.current_task()] + tasks = [ + t for t in asyncio.Task.all_tasks() if t != asyncio.Task.current_task() + ] for t in tasks: t.cancel() res_and_excs = await asyncio.gather(*tasks, return_exceptions=True) @@ -119,8 +126,9 @@ def run(self) -> bool: """Enters the asyncio loop and manages shutdown.""" # We need to handle KeyboardInterrupt "manually" to get clean shutdown # for the ProcessPoolExecutor - self.loop.add_signal_handler(signal.SIGINT, - lambda: asyncio.ensure_future(self.shutdown(signal.SIGINT))) + self.loop.add_signal_handler( + signal.SIGINT, lambda: asyncio.ensure_future(self.shutdown(signal.SIGINT)) + ) try: task = asyncio.ensure_future(self._async_run()) self.loop.run_until_complete(task) @@ -156,8 +164,10 @@ async def _async_run(self) -> bool: tasks.append(asyncio.ensure_future(self.show_progress(progress_q, return_q))) # setup workers - tasks.extend(asyncio.ensure_future(self.worker(source_q, progress_q)) - for n in range(self.threads)) + tasks.extend( + asyncio.ensure_future(self.worker(source_q, progress_q)) + for n in range(self.threads) + ) # send items await self.queue_items(source_q, return_q) @@ -222,9 +232,8 @@ async def run_sp(self, func, *args): return await self.loop.run_in_executor(self.proc_pool_executor, func, *args) -class AsyncRequests(): - """Provides helpers for async access to URLs - """ +class AsyncRequests: + """Provides helpers for async access to URLs""" #: Used as user agent in http requests and as requester in github API requests USER_AGENT = "bioconda/bioconda-utils" @@ -236,8 +245,8 @@ def __init__(self, cache_fn: str = None) -> None: #: cache self.cache: Optional[Dict[str, Dict[str, str]]] = None - async def __aenter__(self) -> 'AsyncRequests': - session = aiohttp.ClientSession(headers={'User-Agent': self.USER_AGENT}) + async def __aenter__(self) -> "AsyncRequests": + session = aiohttp.ClientSession(headers={"User-Agent": self.USER_AGENT}) await session.__aenter__() self.session = session if self.cache_fn: @@ -261,8 +270,12 @@ async def __aexit__(self, ext_type, exc, trace): with open(self.cache_fn, "wb") as stream: pickle.dump(self.cache, stream) - @backoff.on_exception(backoff.fibo, aiohttp.ClientResponseError, max_tries=20, - giveup=lambda ex: ex.code not in [429, 502, 503, 504]) + @backoff.on_exception( + backoff.fibo, + aiohttp.ClientResponseError, + max_tries=20, + giveup=lambda ex: ex.code not in [429, 502, 503, 504], + ) async def get_text_from_url(self, url: str) -> str: """Fetch content at **url** and return as text @@ -302,8 +315,12 @@ async def get_checksum_from_url(self, url: str, desc: str) -> str: return res - @backoff.on_exception(backoff.fibo, aiohttp.ClientResponseError, max_tries=20, - giveup=lambda ex: ex.code not in [429, 502, 503, 504]) + @backoff.on_exception( + backoff.fibo, + aiohttp.ClientResponseError, + max_tries=20, + giveup=lambda ex: ex.code not in [429, 502, 503, 504], + ) async def get_checksum_from_http(self, url: str, desc: str) -> str: """Compute sha256 checksum of content at http **url** @@ -313,18 +330,30 @@ async def get_checksum_from_http(self, url: str, desc: str) -> str: async with self.session.get(url) as resp: resp.raise_for_status() size = int(resp.headers.get("Content-Length", 0)) - with tqdm(total=size, unit='B', unit_scale=True, unit_divisor=1024, - desc=desc, miniters=1, leave=False, disable=None) as progress: + with tqdm( + total=size, + unit="B", + unit_scale=True, + unit_divisor=1024, + desc=desc, + miniters=1, + leave=False, + disable=None, + ) as progress: while True: - block = await resp.content.read(1024*1024) + block = await resp.content.read(1024 * 1024) if not block: break progress.update(len(block)) checksum.update(block) return checksum.hexdigest() - @backoff.on_exception(backoff.fibo, aiohttp.ClientResponseError, max_tries=20, - giveup=lambda ex: ex.code not in [429, 502, 503, 504]) + @backoff.on_exception( + backoff.fibo, + aiohttp.ClientResponseError, + max_tries=20, + giveup=lambda ex: ex.code not in [429, 502, 503, 504], + ) async def get_file_from_url(self, fname: str, url: str, desc: str) -> None: """Fetch file at **url** into **fname** @@ -333,11 +362,19 @@ async def get_file_from_url(self, fname: str, url: str, desc: str) -> None: async with self.session.get(url) as resp: resp.raise_for_status() size = int(resp.headers.get("Content-Length", 0)) - with tqdm(total=size, unit='B', unit_scale=True, unit_divisor=1024, - desc=desc, miniters=1, leave=False, disable=None) as progress: + with tqdm( + total=size, + unit="B", + unit_scale=True, + unit_divisor=1024, + desc=desc, + miniters=1, + leave=False, + disable=None, + ) as progress: with open(fname, "wb") as out: while True: - block = await resp.content.read(1024*1024) + block = await resp.content.read(1024 * 1024) if not block: break out.write(block) @@ -350,8 +387,9 @@ async def get_ftp_listing(self, url): return self.cache["ftp_list"][url] parsed = urlparse(url) - async with aioftp.ClientSession(parsed.netloc, - password=self.USER_AGENT+"@") as client: + async with aioftp.ClientSession( + parsed.netloc, password=self.USER_AGENT + "@" + ) as client: res = [str(path) for path, _info in await client.list(parsed.path)] if self.cache: self.cache["ftp_list"][url] = res @@ -365,8 +403,9 @@ async def get_checksum_from_ftp(self, url, _desc=None): """ parsed = urlparse(url) checksum = sha256() - async with aioftp.ClientSession(parsed.netloc, - password=self.USER_AGENT+"@") as client: + async with aioftp.ClientSession( + parsed.netloc, password=self.USER_AGENT + "@" + ) as client: async with client.download_stream(parsed.path) as stream: async for block in stream.iter_by_block(): checksum.update(block) diff --git a/bioconda_utils/autobump.py b/bioconda_utils/autobump.py index b999468f98..b1312c0def 100644 --- a/bioconda_utils/autobump.py +++ b/bioconda_utils/autobump.py @@ -50,8 +50,18 @@ from collections import defaultdict, Counter from functools import partial from urllib.parse import urlparse -from typing import (Any, Dict, Iterable, List, Mapping, Optional, Sequence, - Set, Tuple, TYPE_CHECKING) +from typing import ( + Any, + Dict, + Iterable, + List, + Mapping, + Optional, + Sequence, + Set, + Tuple, + TYPE_CHECKING, +) import aiofiles from aiohttp import ClientResponseError @@ -72,7 +82,13 @@ from .utils import ensure_list, RepoData from .recipe import Recipe from .recipe import load_parallel_iter as recipes_load_parallel_iter -from .aiopipe import AsyncFilter, AsyncPipeline, AsyncRequests, EndProcessingItem, EndProcessing +from .aiopipe import ( + AsyncFilter, + AsyncPipeline, + AsyncRequests, + EndProcessingItem, + EndProcessing, +) from .githandler import GitHandler from .githubhandler import AiohttpGitHubHandler, GitHubHandler @@ -94,8 +110,14 @@ class RecipeSource: packages: list of packages, may contain globs shuffle: If true, package order will be randomized. """ - def __init__(self, recipe_base: str, packages: List[str], exclude: List[str], - shuffle: bool=True) -> None: + + def __init__( + self, + recipe_base: str, + packages: List[str], + exclude: List[str], + shuffle: bool = True, + ) -> None: self.recipe_base = recipe_base self.recipe_dirs = list(utils.get_recipes(recipe_base, packages, exclude)) if shuffle: @@ -123,8 +145,15 @@ def get_item_count(self): class RecipeGraphSource(RecipeSource): - def __init__(self, recipe_base: str, packages: List[str], exclude: List[str], - shuffle: bool, config: Dict[str, str], cache_fn: str = None) -> None: + def __init__( + self, + recipe_base: str, + packages: List[str], + exclude: List[str], + shuffle: bool, + config: Dict[str, str], + cache_fn: str = None, + ) -> None: super().__init__(recipe_base, packages, exclude, shuffle) self.config = config self.cache_fn = cache_fn @@ -163,7 +192,8 @@ def load_graph(self): else: blacklist = utils.get_blacklist(self.config, self.recipe_base) dag = graph.build_from_recipes( - recipe for recipe in recipes_load_parallel_iter(self.recipe_base, "*") + recipe + for recipe in recipes_load_parallel_iter(self.recipe_base, "*") if recipe.reldir not in blacklist ) if self.cache_fn: @@ -180,9 +210,13 @@ class Scanner(AsyncPipeline[Recipe]): cache_fn: Filename prefix for caching status_fn: Filename for status output """ - def __init__(self, recipe_source: Iterable[Recipe], - cache_fn: str = None, - status_fn: str = None) -> None: + + def __init__( + self, + recipe_source: Iterable[Recipe], + cache_fn: str = None, + status_fn: str = None, + ) -> None: super().__init__() #: recipe source self.recipe_source = recipe_source @@ -199,7 +233,7 @@ def run(self) -> bool: """Runs scanner""" logger.info("Running pipeline with these steps:") for n, filt in enumerate(self.filters): - logger.info(" %i. %s", n+1, filt.get_info()) + logger.info(" %i. %s", n + 1, filt.get_info()) res = super().run() logger.info("") logger.info("Recipe status statistics:") @@ -209,7 +243,7 @@ def run(self) -> bool: if self.status_fn: with open(self.status_fn, "w") as out: for rname, result in self.status: - out.write(f'{rname}\t{result.name}\n') + out.write(f"{rname}\t{result.name}\n") return res async def queue_items(self, send_q, return_q): @@ -240,6 +274,7 @@ async def process(self, recipe: Recipe) -> bool: class Filter(AsyncFilter[Recipe]): """Filter for Scanner - class exists primarily to silence mypy""" + pipeline: Scanner def get_info(self) -> str: @@ -258,25 +293,24 @@ class ExcludeOtherChannel(Filter): class OtherChannel(EndProcessingItem): """This recipe builds one or more packages that are also present in at least one other channel""" + template = "builds package found in other channel(s)" level = logging.DEBUG - def __init__(self, scanner: Scanner, channels: Sequence[str], - cache: str) -> None: + def __init__(self, scanner: Scanner, channels: Sequence[str], cache: str) -> None: super().__init__(scanner) self.channels = channels logger.info("Loading package lists for %s", channels) repo = utils.RepoData() if cache: repo.set_cache(cache) - self.other = set(repo.get_package_data('name', channels=channels)) + self.other = set(repo.get_package_data("name", channels=channels)) def get_info(self) -> str: - return super().get_info().replace('**channels**', ', '.join(self.channels)) + return super().get_info().replace("**channels**", ", ".join(self.channels)) async def apply(self, recipe: Recipe) -> None: - if any(package in self.other - for package in recipe.package_names): + if any(package in self.other for package in recipe.package_names): raise self.OtherChannel(recipe) @@ -313,6 +347,7 @@ class ExcludeSubrecipe(Filter, AutoBumpConfigMixin): class IsSubRecipe(EndProcessingItem): """This recipe is a sub-recipe (i.e. blast/2.4.0).""" + template = "is a subrecipe" level = logging.DEBUG @@ -332,6 +367,7 @@ class ExcludeDisabled(Filter, AutoBumpConfigMixin): class IsDisabled(EndProcessingItem): """This recipe was explicitly disabled""" + template = "is disabled for autobump" async def apply(self, recipe: Recipe) -> None: @@ -345,18 +381,21 @@ class ExcludeBlacklisted(Filter): class Blacklisted(EndProcessingItem): """This recipe has been blacklisted (fails to build, see blacklist file)""" + template = "is blacklisted" level = logging.DEBUG def __init__(self, scanner: Scanner, recipe_base: str, config: Dict) -> None: super().__init__(scanner) - self.blacklists = config.get('blacklists') + self.blacklists = config.get("blacklists") self.blacklisted = utils.get_blacklist(config, recipe_base) logger.warning("Excluding %i blacklisted recipes", len(self.blacklisted)) def get_info(self) -> str: - return (super().get_info() + - f": {', '.join(self.blacklists)} / {len(self.blacklisted)} recipes") + return ( + super().get_info() + + f": {', '.join(self.blacklists)} / {len(self.blacklisted)} recipes" + ) async def apply(self, recipe: Recipe) -> None: if recipe.reldir in self.blacklisted: @@ -368,6 +407,7 @@ class ExcludeDependencyPending(Filter): class DependencyPending(EndProcessingItem): """A dependency of this recipe is pending rebuild""" + template = "deferred pending rebuild of dependencies: %s" def __init__(self, scanner: Scanner, dag: nx.DiGraph) -> None: @@ -376,11 +416,10 @@ def __init__(self, scanner: Scanner, dag: nx.DiGraph) -> None: async def apply(self, recipe: Recipe) -> None: pending_deps = [ - dep for dep in nx.ancestors(self.dag, recipe) - if dep.is_modified() + dep for dep in nx.ancestors(self.dag, recipe) if dep.is_modified() ] if pending_deps: - msg = ", ".join(str(x) for x in pending_deps) + msg = ", ".join(str(x) for x in pending_deps) raise self.DependencyPending(recipe, msg) @@ -405,25 +444,27 @@ def match_version(spec, version): True if the spec is satisfied by the version. """ mspec = MatchSpec(version=spec.replace(" ", "")) - return mspec.match({'name': '', 'build': '', 'build_number': 0, - 'version': version}) + return mspec.match( + {"name": "", "build": "", "build_number": 0, "version": version} + ) async def apply(self, recipe: Recipe) -> None: - reason = await self.scanner.run_sp( - self._sp_apply, - (self.build_config, recipe) - ) + reason = await self.scanner.run_sp(self._sp_apply, (self.build_config, recipe)) if reason: - recipe.data['pinning'] = reason + recipe.data["pinning"] = reason new_buildno = recipe.build_number + 1 - logger.info("%s needs rebuild. Bumping buildnumber to %i", recipe, new_buildno) + logger.info( + "%s needs rebuild. Bumping buildnumber to %i", recipe, new_buildno + ) recipe.reset_buildnumber(new_buildno) recipe.render() @classmethod def _sp_apply(cls, data) -> None: config, recipe = data - status, recipe = update_pinnings.check(recipe, build_config=config, keep_metas=True) + status, recipe = update_pinnings.check( + recipe, build_config=config, keep_metas=True + ) if status.needs_bump(): metas = recipe.conda_render(config=config) reason = cls.find_reason(recipe, metas) @@ -449,12 +490,16 @@ def find_reason(cls, recipe, metas): # doesn't exist and needs to be extended to 1.2.11. resolved_vers = {} for var, vers in pinnings.items(): - avail_vers = RepoData().get_package_data('version', name=var) + avail_vers = RepoData().get_package_data("version", name=var) if avail_vers: - resolved_vers[var] = list(set( - avs for avs in list(set(avail_vers)) - for pvs in vers - if avs.startswith(pvs))) + resolved_vers[var] = list( + set( + avs + for avs in list(set(avail_vers)) + for pvs in vers + if avs.startswith(pvs) + ) + ) else: resolved_vers[var] = vers @@ -462,26 +507,34 @@ def find_reason(cls, recipe, metas): # Iterate over extant builds for this recipe at this version and build number # It's one for noarch, two if osx and linux are built, more if we have # variant pins such as for python. - package_data = RepoData().get_package_data(['build', 'depends', 'platform'], - name=recipe.name, - version=recipe.version, - build_number=recipe.build_number) + package_data = RepoData().get_package_data( + ["build", "depends", "platform"], + name=recipe.name, + version=recipe.version, + build_number=recipe.build_number, + ) for _build_id, package_deps, _platform in package_data: for item in package_deps: package, _, constraint = item.partition(" ") if not constraint or package not in pinnings: continue - if any(cls.match_version(constraint, version) - for version in resolved_vers[package]): + if any( + cls.match_version(constraint, version) + for version in resolved_vers[package] + ): continue causes.setdefault(package, set()).add( f"Pin `{package} {','.join(pinnings[package])}` not within `{constraint}`" ) if not causes: - compiler_pins = [i for k, v in pinnings.items() for i in v if 'compiler' in k] + compiler_pins = [ + i for k, v in pinnings.items() for i in v if "compiler" in k + ] if compiler_pins: - return {'compiler': set([f"Recompiling with {' / '.join(compiler_pins)}"])} + return { + "compiler": set([f"Recompiling with {' / '.join(compiler_pins)}"]) + } return causes # must return not None! @@ -506,10 +559,12 @@ class UpdateVersion(Filter, AutoBumpConfigMixin): class Metapackage(EndProcessingItem): """This recipe only builds a meta package - it has no upstream sources.""" + template = "builds a meta package (recipe has no sources)" class UpToDate(EndProcessingItem): """This recipe appears to be up to date!""" + template = "is up to date" level = logging.DEBUG @@ -519,6 +574,7 @@ class UpdateVersionFailure(EndProcessingItem): Something probably went wrong trying to replace the version number in the meta.yaml. """ + template = "could not be updated from %s to %s" class NoUrlInSource(EndProcessingItem): @@ -526,16 +582,19 @@ class NoUrlInSource(EndProcessingItem): Most likely, this is due to a git-url. """ + template = "has no URL in source %i" class NoRecognizedSourceUrl(EndProcessingItem): """None of the source URLs in this recipe were recognized.""" + template = "has no URL in source %i recognized by any Hoster class" level = logging.DEBUG class UrlNotVersioned(EndProcessingItem): """This recipe has an URL unaffected by the version change. the recipe""" + template = "has URL not modified by version change" class NoReleases(EndProcessingItem): @@ -544,10 +603,12 @@ class NoReleases(EndProcessingItem): This is unusual - something probably went wrong finding releases for the source urls in this recipe. """ + template = "has no releases?!" - def __init__(self, scanner: Scanner, hoster_factory, - unparsed_file: Optional[str] = None) -> None: + def __init__( + self, scanner: Scanner, hoster_factory, unparsed_file: Optional[str] = None + ) -> None: super().__init__(scanner) #: output file name for unparsed urls self.unparsed_urls: List[str] = [] @@ -556,8 +617,7 @@ def __init__(self, scanner: Scanner, hoster_factory, #: function selecting hoster self.hoster_factory = hoster_factory #: conda build config - self.build_config: conda_build.config.Config = \ - utils.load_conda_build_config() + self.build_config: conda_build.config.Config = utils.load_conda_build_config() def finalize(self) -> None: """Save unparsed urls to file and print stats to log""" @@ -597,13 +657,15 @@ async def apply(self, recipe: Recipe) -> None: # Update `url:`s without Jinja expressions (plain text) for fname in versions[latest]: - recipe.replace(fname, versions[latest][fname]['link'], within=["source"]) + recipe.replace(fname, versions[latest][fname]["link"], within=["source"]) # Update the version number itself. This will also usually update # `url:`s expressed with `{{version}}` tags. if not recipe.replace(recipe.version, latest, within=["package"]): # allow changes between dash/dot/underscore - if recipe.replace(recipe.version, latest, within=["package"], with_fuzz=True): + if recipe.replace( + recipe.version, latest, within=["package"], with_fuzz=True + ): logger.warning("Recipe %s: replaced version with fuzz", recipe) recipe.reset_buildnumber() @@ -614,10 +676,10 @@ async def apply(self, recipe: Recipe) -> None: raise self.UpdateVersionFailure(recipe, recipe.orig.version, latest) # Verify that every url was modified - for src, osrc in zip(ensure_list(recipe.meta['source']), - ensure_list(recipe.orig.meta['source'])): - for url, ourl in zip(ensure_list(src['url']), - ensure_list(osrc['url'])): + for src, osrc in zip( + ensure_list(recipe.meta["source"]), ensure_list(recipe.orig.meta["source"]) + ): + for url, ourl in zip(ensure_list(src["url"]), ensure_list(osrc["url"])): if url == ourl: raise self.UrlNotVersioned(recipe) @@ -632,7 +694,7 @@ async def get_version_map(self, recipe: Recipe): source_iter = iter(sources) versions = await self.get_versions(recipe, next(source_iter), 0) for num, source in enumerate(source_iter): - add_versions = await self.get_versions(recipe, source, num+1) + add_versions = await self.get_versions(recipe, source, num + 1) for vers, files in add_versions.items(): for fname, data in files.items(): versions[vers][fname] = data @@ -643,12 +705,13 @@ async def get_version_map(self, recipe: Recipe): raise self.NoReleases(recipe) return versions - async def get_versions(self, recipe: Recipe, source: Mapping[Any, Any], - source_idx: int): + async def get_versions( + self, recipe: Recipe, source: Mapping[Any, Any], source_idx: int + ): """Select hosters and retrieve versions for this source""" urls = source.get("url") if not urls: - raise self.NoUrlInSource(recipe, source_idx+1) + raise self.NoUrlInSource(recipe, source_idx + 1) if isinstance(urls, str): urls = [urls] @@ -661,15 +724,17 @@ async def get_versions(self, recipe: Recipe, source: Mapping[Any, Any], continue logger.debug("Scanning with %s", hoster.__class__.__name__) try: - versions = await hoster.get_versions(self.pipeline.req, recipe.orig.version) + versions = await hoster.get_versions( + self.pipeline.req, recipe.orig.version + ) for match in versions: - match['hoster'] = hoster + match["hoster"] = hoster version_map[match["version"]][url] = match except ClientResponseError as exc: logger.debug("HTTP %s when getting %s", exc, url) if not version_map: - raise self.NoRecognizedSourceUrl(recipe, source_idx+1) + raise self.NoRecognizedSourceUrl(recipe, source_idx + 1) return version_map @@ -709,8 +774,9 @@ def select_version(current: str, versions: Sequence[str]) -> str: return latest - def check_version_pin_conflict(self, recipe: Recipe, - versions: Dict[str, Any]) -> None: + def check_version_pin_conflict( + self, recipe: Recipe, versions: Dict[str, Any] + ) -> None: """Find items in **versions** conflicting with pins Example: @@ -719,7 +785,8 @@ def check_version_pin_conflict(self, recipe: Recipe, TODO: currently, this only logs an error """ variants = conda_build.variants.get_package_variants( - recipe.path, self.build_config) + recipe.path, self.build_config + ) def check_pins(depends): for pkg, spec in depends: @@ -727,19 +794,30 @@ def check_pins(depends): try: mspec = MatchSpec(version=spec.replace(" ", "")) except conda.exceptions.InvalidVersionSpecError: - logger.error("Recipe %s: invalid upstream spec %s %s", - recipe, pkg, repr(spec)) + logger.error( + "Recipe %s: invalid upstream spec %s %s", + recipe, + pkg, + repr(spec), + ) continue if norm_pkg in variants[0]: for variant in variants: - if not mspec.match({'name': '', 'build': '', 'build_number': '0', - 'version': variant[norm_pkg]}): - logger.error("Recipe %s: %s %s conflicts pins", - recipe, pkg, spec) + if not mspec.match( + { + "name": "", + "build": "", + "build_number": "0", + "version": variant[norm_pkg], + } + ): + logger.error( + "Recipe %s: %s %s conflicts pins", recipe, pkg, spec + ) for files in versions.values(): for data in files.values(): - depends = data.get('depends') + depends = data.get("depends") if depends: check_pins(depends.items()) @@ -751,22 +829,25 @@ class FetchUpstreamDependencies(Filter): determined by ``setup.py`` at installation time and need not be static in many cases (there is work to change this going on). """ + def __init__(self, scanner: Scanner) -> None: super().__init__(scanner) #: conda build config - self.build_config: conda_build.config.Config = \ - utils.load_conda_build_config() + self.build_config: conda_build.config.Config = utils.load_conda_build_config() async def apply(self, recipe: Recipe) -> None: - await asyncio.gather(*[ - data["hoster"].get_deps(self.pipeline, self.build_config, - recipe.name, data) - for r in (recipe, recipe.orig) - for fn, data in r.version_data.items() - if "depends" not in data - and "hoster" in data - and hasattr(data["hoster"], "get_deps") - ]) + await asyncio.gather( + *[ + data["hoster"].get_deps( + self.pipeline, self.build_config, recipe.name, data + ) + for r in (recipe, recipe.orig) + for fn, data in r.version_data.items() + if "depends" not in data + and "hoster" in data + and hasattr(data["hoster"], "get_deps") + ] + ) class UpdateChecksums(Filter): @@ -791,22 +872,25 @@ class UpdateChecksums(Filter): class NoValidUrls(EndProcessingItem): """Failed to download any file for a source to generate new checksum""" + template = "has no valid urls in source %i" class SourceUrlMismatch(EndProcessingItem): """The URLs in a source point to different files after update""" + template = "has mismatching cheksums for alternate URLs in source %i" class ChecksumReplaceFailed(EndProcessingItem): """The checksum could not be updated after version bump""" + template = "- failed to replace checksum" class ChecksumUnchanged(EndProcessingItem): """The checksum did not change after version bump""" + template = "had no change to checksum after update?!" - def __init__(self, scanner: Scanner, - failed_file: Optional[str] = None) -> None: + def __init__(self, scanner: Scanner, failed_file: Optional[str] = None) -> None: super().__init__(scanner) #: failed urls - for later inspection self.failed_urls: List[str] = [] @@ -816,8 +900,10 @@ def __init__(self, scanner: Scanner, def finalize(self): """Store list of URLs that could not be downloaded""" if self.failed_urls: - logger.warning("Encountered %i download failures while computing checksums", - len(self.failed_urls)) + logger.warning( + "Encountered %i download failures while computing checksums", + len(self.failed_urls), + ) if self.failed_urls and self.failed_file: with open(self.failed_file, "w") as out: out.write("\n".join(self.failed_urls)) @@ -834,7 +920,9 @@ async def apply(self, recipe: Recipe) -> None: await self.update_source(recipe, source, source_idx) recipe.render() - async def update_source(self, recipe: Recipe, source: Mapping, source_idx: int) -> None: + async def update_source( + self, recipe: Recipe, source: Mapping, source_idx: int + ) -> None: """Updates one source Each source has its own checksum, but all urls in one source must have @@ -862,18 +950,27 @@ async def update_source(self, recipe: Recipe, source: Mapping, source_idx: int) try: new_checksums.add( await self.pipeline.req.get_checksum_from_url( - url, f"{recipe} [{source_idx}.{url_idx}]") + url, f"{recipe} [{source_idx}.{url_idx}]" + ) ) except ClientResponseError as exc: - logger.info("Recipe %s: HTTP %s while downloading url %i", - recipe, exc.code, url_idx) + logger.info( + "Recipe %s: HTTP %s while downloading url %i", + recipe, + exc.code, + url_idx, + ) self.failed_urls += ["\t".join((str(exc.code), url))] if not new_checksums: raise self.NoValidUrls(recipe, source_idx) if len(new_checksums) > 1: - logger.error("Recipe %s source %i: checksum mismatch between alternate urls - %s", - recipe, source_idx, new_checksums) + logger.error( + "Recipe %s source %i: checksum mismatch between alternate urls - %s", + recipe, + source_idx, + new_checksums, + ) raise self.SourceUrlMismatch(recipe, source_idx) new_checksum = new_checksums.pop() @@ -908,7 +1005,9 @@ def branch_name(cls, recipe: Recipe) -> str: ``bump/toolx/1.2.x``. Note: this will break if we have ``recipe/x`` and ``recipe/x.d`` in the repo. """ - return f"{cls.branch_prefix}{recipe.reldir.replace('-', '_').replace('/', '.d/')}" + return ( + f"{cls.branch_prefix}{recipe.reldir.replace('-', '_').replace('/', '.d/')}" + ) # placate pylint by reiterating abstract method @abc.abstractmethod @@ -923,8 +1022,10 @@ class ExcludeNoActiveUpdate(GitFilter): updates are those for which a branch with commits newer than master exists (which is checked by GitLoadRecipe). """ + class NoActiveUpdate(EndProcessingItem): """Recipe is not currently being updated""" + template = "is not currently being updated" level = logging.DEBUG @@ -936,13 +1037,13 @@ async def apply(self, recipe: Recipe) -> None: class LoadRecipe(Filter): """Load the recipe from the filesystem""" + def __init__(self, scanner: Scanner) -> None: super().__init__(scanner) self.sem = asyncio.Semaphore(64) async def apply(self, recipe: Recipe) -> None: - async with self.sem, \ - aiofiles.open(recipe.path, encoding="utf-8") as fdes: + async with self.sem, aiofiles.open(recipe.path, encoding="utf-8") as fdes: recipe_text = await fdes.read() recipe.load_from_string(recipe_text) recipe.set_original() @@ -969,7 +1070,7 @@ async def apply(self, recipe: Recipe) -> None: branch_name = self.branch_name(recipe) remote_branch = self.git.get_remote_branch(branch_name) local_branch = self.git.get_local_branch(branch_name) - master_branch = self.git.get_local_branch('master') + master_branch = self.git.get_local_branch("master") if local_branch: logger.debug("Recipe %s: removing local branch %s", recipe, branch_name) @@ -977,32 +1078,39 @@ async def apply(self, recipe: Recipe) -> None: logger.debug("Recipe %s: loading from master", recipe) recipe_text = await self.pipeline.run_io( - self.git.read_from_branch, master_branch, recipe.path) + self.git.read_from_branch, master_branch, recipe.path + ) recipe.load_from_string(recipe_text) recipe.set_original() if remote_branch: if await self.git.branch_is_current(remote_branch, recipe.dir): logger.info("Recipe %s: updating from remote %s", recipe, branch_name) - recipe_text = await self.pipeline.run_io(self.git.read_from_branch, - remote_branch, recipe.path) + recipe_text = await self.pipeline.run_io( + self.git.read_from_branch, remote_branch, recipe.path + ) recipe.load_from_string(recipe_text) await self.pipeline.run_io(self.git.create_local_branch, branch_name) recipe.on_branch = True else: # Note: If a PR still exists for this, it is silently closed by deleting # the branch. - logger.info("Recipe %s: deleting outdated remote %s", recipe, branch_name) + logger.info( + "Recipe %s: deleting outdated remote %s", recipe, branch_name + ) await self.pipeline.run_io(self.git.delete_remote_branch, branch_name) + class WritingFilter: class NoChanges(EndProcessingItem): """Recipe had no changes after applying updates""" + template = "had no changes" class WriteRecipe(Filter, WritingFilter): """Write recipe to filesystem""" + def __init__(self, scanner: Scanner) -> None: super().__init__(scanner) self.sem = asyncio.Semaphore(64) @@ -1010,8 +1118,7 @@ def __init__(self, scanner: Scanner) -> None: async def apply(self, recipe: Recipe) -> None: if not recipe.is_modified(): raise self.NoChanges(recipe) - async with self.sem, \ - aiofiles.open(recipe.path, "w", encoding="utf-8") as fdes: + async with self.sem, aiofiles.open(recipe.path, "w", encoding="utf-8") as fdes: await fdes.write(recipe.dump()) @@ -1026,8 +1133,7 @@ async def apply(self, recipe: Recipe) -> None: changed = False async with self.git.lock_working_dir: self.git.prepare_branch(branch_name) - async with aiofiles.open(recipe.path, "w", - encoding="utf-8") as fdes: + async with aiofiles.open(recipe.path, "w", encoding="utf-8") as fdes: await fdes.write(recipe.dump()) if recipe.version != recipe.orig.version: msg = f"Update {recipe} to {recipe.version}" @@ -1049,6 +1155,7 @@ class CreatePullRequest(GitFilter): class UpdateInProgress(EndProcessingItem): """The recipe has an active update PR""" + template = "has update in progress" class UpdateRejected(EndProcessingItem): @@ -1059,14 +1166,20 @@ class UpdateRejected(EndProcessingItem): indication that the update should not be repeated """ + template = "had latest updated rejected manually" class FailedToCreatePR(EndProcessingItem): """Something went wrong trying to create a new PR on Github.""" + template = "had failure in PR create" - def __init__(self, scanner: Scanner, git_handler: "GitHandler", - github_handler: "GitHubHandler") -> None: + def __init__( + self, + scanner: Scanner, + git_handler: "GitHandler", + github_handler: "GitHubHandler", + ) -> None: super().__init__(scanner, git_handler) self.ghub = github_handler @@ -1087,21 +1200,27 @@ def render_deps_diff(recipe): or filled in later by the `FetchUpstreamDependencies` filter (currently for PyPi). """ - diffset: Dict[str, Set[str]] = {'host': set(), 'run': set()} + diffset: Dict[str, Set[str]] = {"host": set(), "run": set()} if not recipe.version_data: - logger.debug("Recipe %s: dependency diff not rendered (no version_data)", recipe) + logger.debug( + "Recipe %s: dependency diff not rendered (no version_data)", recipe + ) for fname in recipe.version_data: if fname not in recipe.orig.version_data: - logger.debug("Recipe %s: dependency diff not rendered (no orig.version_data)", - recipe) + logger.debug( + "Recipe %s: dependency diff not rendered (no orig.version_data)", + recipe, + ) continue - new = recipe.version_data[fname].get('depends') - orig = recipe.orig.version_data[fname].get('depends') + new = recipe.version_data[fname].get("depends") + orig = recipe.orig.version_data[fname].get("depends") if not new or not orig: - logger.debug("Recipe %s: dependency diff not rendered (no depends in version_data)", - recipe) + logger.debug( + "Recipe %s: dependency diff not rendered (no depends in version_data)", + recipe, + ) continue - for kind in ('host', 'run'): + for kind in ("host", "run"): deps: Set[str] = set() deps.update(new[kind].keys()) deps.update(orig[kind].keys()) @@ -1111,9 +1230,12 @@ def render_deps_diff(recipe): elif dep not in orig[kind]: diffset[kind].add("+ - {} {}".format(dep, new[kind][dep])) elif orig[kind][dep] != new[kind][dep]: - diffset[kind].add("- - {dep} {}\n" - "+ - {dep} {}".format(orig[kind][dep], new[kind][dep], - dep=dep)) + diffset[kind].add( + "- - {dep} {}\n" + "+ - {dep} {}".format( + orig[kind][dep], new[kind][dep], dep=dep + ) + ) text = "" for kind, lines in diffset.items(): if lines: @@ -1134,8 +1256,10 @@ def get_github_author(recipe) -> Optional[str]: if not recipe.version_data: return None for ver in recipe.version_data.values(): - if 'hoster' in ver and ver['hoster'].__class__.__name__.startswith('Github'): - return ver['vals']['account'] + if "hoster" in ver and ver["hoster"].__class__.__name__.startswith( + "Github" + ): + return ver["vals"]["account"] return None async def apply(self, recipe: Recipe) -> None: @@ -1150,29 +1274,36 @@ async def apply(self, recipe: Recipe) -> None: template_name = "autobump_update_pinning_pr.md" template = utils.jinja.get_template(template_name) - context = template.new_context({ - 'r': recipe, - 'recipe_relurl': self.ghub.get_file_relurl(recipe.dir, branch_name), - 'author': author, - 'author_is_member': await self.ghub.is_member(author), - 'dependency_diff': self.render_deps_diff(recipe), - 'version': __version__ - }) - - labels = (''.join(template.blocks['labels'](context))).splitlines() - title = ''.join(template.blocks['title'](context)).replace('\n',' ') + context = template.new_context( + { + "r": recipe, + "recipe_relurl": self.ghub.get_file_relurl(recipe.dir, branch_name), + "author": author, + "author_is_member": await self.ghub.is_member(author), + "dependency_diff": self.render_deps_diff(recipe), + "version": __version__, + } + ) + + labels = ("".join(template.blocks["labels"](context))).splitlines() + title = "".join(template.blocks["title"](context)).replace("\n", " ") body = template.render(context) # check if we already have an open PR (=> update in progress) - pullreqs = await self.ghub.get_prs(from_branch=branch_name, from_user="bioconda") + pullreqs = await self.ghub.get_prs( + from_branch=branch_name, from_user="bioconda" + ) if pullreqs: if len(pullreqs) > 1: - logger.error("Multiple PRs updating %s: %s", - recipe, - ", ".join(str(pull['number']) for pull in pullreqs)) + logger.error( + "Multiple PRs updating %s: %s", + recipe, + ", ".join(str(pull["number"]) for pull in pullreqs), + ) for pull in pullreqs: - logger.debug("Found PR %i updating %s: %s", - pull["number"], recipe, pull["title"]) + logger.debug( + "Found PR %i updating %s: %s", pull["number"], recipe, pull["title"] + ) # update the PR if title or body changed pull = pullreqs[0] if body == pull["body"]: @@ -1180,38 +1311,54 @@ async def apply(self, recipe: Recipe) -> None: if title == pull["title"]: title = None if not (body is None and title is None): - if await self.ghub.modify_issue(number=pull['number'], body=body, title=title): - logger.info("Updated PR %i updating %s to %s", - pull['number'], recipe, recipe.version) + if await self.ghub.modify_issue( + number=pull["number"], body=body, title=title + ): + logger.info( + "Updated PR %i updating %s to %s", + pull["number"], + recipe, + recipe.version, + ) else: - logger.error("Failed to update PR %i with title=%s and body=%s", - pull['number'], title, body) + logger.error( + "Failed to update PR %i with title=%s and body=%s", + pull["number"], + title, + body, + ) else: - logger.debug("Not updating PR %i updating %s - no changes", - pull['number'], recipe) + logger.debug( + "Not updating PR %i updating %s - no changes", + pull["number"], + recipe, + ) raise self.UpdateInProgress(recipe) # check for matching closed PR (=> update rejected) - pullreqs = await self.ghub.get_prs(from_branch=branch_name, state=self.ghub.STATE.closed) - if any(recipe.version in pull['title'] for pull in pullreqs): + pullreqs = await self.ghub.get_prs( + from_branch=branch_name, state=self.ghub.STATE.closed + ) + if any(recipe.version in pull["title"] for pull in pullreqs): raise self.UpdateRejected(recipe) # finally, create new PR - pull = await self.ghub.create_pr(title=title, - from_branch=branch_name, - body=body) + pull = await self.ghub.create_pr( + title=title, from_branch=branch_name, body=body + ) if not pull: raise self.FailedToCreatePR(recipe) # set labels (can't do that in create_pr as they are part of issue API) - await self.ghub.modify_issue(number=pull['number'], labels=labels) + await self.ghub.modify_issue(number=pull["number"], labels=labels) - logger.info("Created PR %i: %s", pull['number'], title) + logger.info("Created PR %i: %s", pull["number"], title) class MaxUpdates(Filter): """Terminate pipeline after **max_updates** recipes have been updated.""" + def __init__(self, scanner: Scanner, max_updates: int = 0) -> None: super().__init__(scanner) self.max_updates = max_updates @@ -1219,7 +1366,7 @@ def __init__(self, scanner: Scanner, max_updates: int = 0) -> None: logger.warning("Will exit after %s updated recipes", max_updates) def get_info(self) -> str: - return super().get_info().replace('**max_updates**', str(self.max_updates)) + return super().get_info().replace("**max_updates**", str(self.max_updates)) async def apply(self, recipe: Recipe) -> None: if not recipe.is_modified(): diff --git a/bioconda_utils/bioconductor_skeleton.py b/bioconda_utils/bioconductor_skeleton.py index b7a8fa4031..676c558baf 100755 --- a/bioconda_utils/bioconductor_skeleton.py +++ b/bioconda_utils/bioconductor_skeleton.py @@ -26,7 +26,7 @@ logger = logging.getLogger(__name__) -base_url = 'https://bioconductor.org/packages/' +base_url = "https://bioconductor.org/packages/" # Packages that might be specified in the DESCRIPTION of a package as # dependencies, but since they're built-in we don't need to specify them in @@ -36,88 +36,142 @@ # # conda create -n rtest -c r r # R -e "rownames(installed.packages())" -BASE_R_PACKAGES = ["base", "compiler", "datasets", "graphics", "grDevices", - "grid", "methods", "parallel", "splines", "stats", "stats4", - "tcltk", "tools", "utils"] +BASE_R_PACKAGES = [ + "base", + "compiler", + "datasets", + "graphics", + "grDevices", + "grid", + "methods", + "parallel", + "splines", + "stats", + "stats4", + "tcltk", + "tools", + "utils", +] # These CRAN packages directly or indirectly depend on X, requiring specialized # build and test-time requirements as well as the extended container. # These can be found here: https://github.com/search?p=2&q=cdt%28%27mesa-libgl-devel%27%29+user%3Aconda-forge+r-base&type=Code # and here: https://github.com/search?l=YAML&q=r-rgl+user%3Aconda-forge&type=Code -CRAN_X_PACKAGES = set(["rgl", "tsdist", "tsclust", "plot3drgl", "pals", "longitudinaldata", - "bpca", "bcrocsurface", "feature", "pca3d", "forestfloor", "oceanview", - "clustersim", "hiver", "lidr", "mixomics", "snpls", "matlib", "qpcr"]) +CRAN_X_PACKAGES = set( + [ + "rgl", + "tsdist", + "tsclust", + "plot3drgl", + "pals", + "longitudinaldata", + "bpca", + "bcrocsurface", + "feature", + "pca3d", + "forestfloor", + "oceanview", + "clustersim", + "hiver", + "lidr", + "mixomics", + "snpls", + "matlib", + "qpcr", + ] +) # This maps items in a package's SystemRequirement to conda packages # There can be multiple resulting packages, all of which should then # be included in recipes -SysReqs = {'and egrep are required for some functionalities': ['grep'], - 'bowtie': ['bowtie2'], - 'bowtie and samtools are required for some functionalities': ['bowtie2', 'samtools'], - 'clustalo': ['clustalo'], - 'cwltool (>= 1.0.2018)': ['cwltool >=1.0.2018'], - 'Cytoscape (>= 3.3.0)': ['cytoscape >=3.3.0'], - 'Cytoscape (>= 3.6.1) (if used for visualization of results': ['cytoscape >=3.6.1'], - 'Cytoscape (>= 3.7.1)': ['cytoscape >=3.7.1'], - 'Ensembl VEP (API version 96) and the Perl modules DBI and DBD::mysql must be installed. See the package README and Ensembl installation instructions: http://www.ensembl.org/info/docs/tools/vep/script/vep_download.html#installer': ['ensembl-vep', 'perl-dbd-mysql', 'perl-dbi'], - 'GEOS (>= 3.2.0);for building from source: GEOS from http://trac.osgeo.org/geos/; GEOS OSX frameworks built by William Kyngesburye at http://www.kyngchaos.com/ may be used for source installs on OSX.': ['geos >=3.2.0'], - 'GLPK (>= 4.42)': ['glpk >=4.42'], - 'GNU Scientific Library >= 1.6 (http://www.gnu.org/software/gsl/)': ['gsl >=4.42'], - 'graphviz': ['graphviz'], - 'Graphviz version >= 2.2': ['graphviz >=2.2'], - 'gs': ['ghostscript'], - 'gsl': ['gsl'], - 'GSL and OpenMP': ['gsl'], - 'GSL (GNU Scientific Library)': ['gsl'], - 'gsl. Note: users should have GSL installed. Windows users: \'consult the README file available in the inst directory of the source distribution for necessary configuration instructions\'.': ['gsl'], - 'h5py': ['h5py'], - 'HMMER3': ['hmmer >=3'], - 'ImageMagick': ['imagemagick'], - 'JAGS 4.x.y': ['jags 4.*.*'], - 'Java': ['openjdk'], - 'Java (>= 1.5)': ['openjdk'], - 'Java (>= 1.6)': ['openjdk'], - 'Java (>= 1.7)': ['openjdk'], - 'Java (>= 1.8)': ['openjdk'], - 'Java (>= 8)': ['openjdk'], - 'Java Runtime Environment (>= 6)': ['openjdk'], - 'Java version >= 1.6': ['openjdk'], - 'Java version >= 1.7': ['openjdk'], - 'jQuery': ['jquery'], - 'jQueryUI': ['jquery-ui'], - 'libsbml (==5.10.2)': ['libsbml 5.10.2'], - 'libSBML (>= 5.5)': ['libsbml >=5.5'], - 'libxml2': ['libxml2'], - 'MAFFT (>= 7.305)': ['mafft >=7.305'], - 'mofapy': ['mofapy'], - 'Netpbm': ['netpbm'], - 'nodejs': ['nodejs'], - 'numpy': ['numpy'], - 'OpenBabel': ['openbabel'], - 'OpenBabel (>= 2.3.1) with headers (http://openbabel.org).': ['openbabel >=2.3.1'], - 'pandas': ['pandas'], - 'Pandoc': ['pandoc'], - 'pandoc (>= 1.12.3)': ['pandoc >=1.12.3'], - 'Pandoc (>= 1.12.3)': ['pandoc >=1.12.3'], - 'pandoc (>= 1.19.2.1)': ['pandoc >=1.19.2.1'], - 'pandoc (http://pandoc.org/installing.html) for generating reports from markdown files.': ['pandoc'], - 'perl': ['perl >=5.6.0'], - 'Perl': ['perl >=5.6.0'], - 'Perl (>= 5.6.0)': ['perl >=5.6.0'], - 'python (>= 2.7)': ['python >=2.7'], - 'Python (>=2.7.0)': ['python >=2.7'], - 'python (< 3.7)': ['python <3.7'], - 'root_v5.34.36 - See README file for installation instructions.': ['root5 5.34.36'], - 'rTANDEM uses expat and pthread libraries. See the README file for details.': ['expat'], - 'samtools': ['samtools'], - 'scipy': ['scipiy'], - 'sklearn': ['scikit-learn'], - 'STAR': ['star'], - 'tensorflow': ['tensorflow'], - 'To generate html reports pandoc (http://pandoc.org/installing.html) is required.': ['pandoc'], - 'TopHat': ['tophat2'], - 'ViennaRNA (>= 2.4.1)': ['viennarna >=2.4.1'], - 'xml2': ['libxml2']} +SysReqs = { + "and egrep are required for some functionalities": ["grep"], + "bowtie": ["bowtie2"], + "bowtie and samtools are required for some functionalities": [ + "bowtie2", + "samtools", + ], + "clustalo": ["clustalo"], + "cwltool (>= 1.0.2018)": ["cwltool >=1.0.2018"], + "Cytoscape (>= 3.3.0)": ["cytoscape >=3.3.0"], + "Cytoscape (>= 3.6.1) (if used for visualization of results": ["cytoscape >=3.6.1"], + "Cytoscape (>= 3.7.1)": ["cytoscape >=3.7.1"], + "Ensembl VEP (API version 96) and the Perl modules DBI and DBD::mysql must be installed. See the package README and Ensembl installation instructions: http://www.ensembl.org/info/docs/tools/vep/script/vep_download.html#installer": [ + "ensembl-vep", + "perl-dbd-mysql", + "perl-dbi", + ], + "GEOS (>= 3.2.0);for building from source: GEOS from http://trac.osgeo.org/geos/; GEOS OSX frameworks built by William Kyngesburye at http://www.kyngchaos.com/ may be used for source installs on OSX.": [ + "geos >=3.2.0" + ], + "GLPK (>= 4.42)": ["glpk >=4.42"], + "GNU Scientific Library >= 1.6 (http://www.gnu.org/software/gsl/)": ["gsl >=4.42"], + "graphviz": ["graphviz"], + "Graphviz version >= 2.2": ["graphviz >=2.2"], + "gs": ["ghostscript"], + "gsl": ["gsl"], + "GSL and OpenMP": ["gsl"], + "GSL (GNU Scientific Library)": ["gsl"], + "gsl. Note: users should have GSL installed. Windows users: 'consult the README file available in the inst directory of the source distribution for necessary configuration instructions'.": [ + "gsl" + ], + "h5py": ["h5py"], + "HMMER3": ["hmmer >=3"], + "ImageMagick": ["imagemagick"], + "JAGS 4.x.y": ["jags 4.*.*"], + "Java": ["openjdk"], + "Java (>= 1.5)": ["openjdk"], + "Java (>= 1.6)": ["openjdk"], + "Java (>= 1.7)": ["openjdk"], + "Java (>= 1.8)": ["openjdk"], + "Java (>= 8)": ["openjdk"], + "Java Runtime Environment (>= 6)": ["openjdk"], + "Java version >= 1.6": ["openjdk"], + "Java version >= 1.7": ["openjdk"], + "jQuery": ["jquery"], + "jQueryUI": ["jquery-ui"], + "libsbml (==5.10.2)": ["libsbml 5.10.2"], + "libSBML (>= 5.5)": ["libsbml >=5.5"], + "libxml2": ["libxml2"], + "MAFFT (>= 7.305)": ["mafft >=7.305"], + "mofapy": ["mofapy"], + "Netpbm": ["netpbm"], + "nodejs": ["nodejs"], + "numpy": ["numpy"], + "OpenBabel": ["openbabel"], + "OpenBabel (>= 2.3.1) with headers (http://openbabel.org).": ["openbabel >=2.3.1"], + "pandas": ["pandas"], + "Pandoc": ["pandoc"], + "pandoc (>= 1.12.3)": ["pandoc >=1.12.3"], + "Pandoc (>= 1.12.3)": ["pandoc >=1.12.3"], + "pandoc (>= 1.19.2.1)": ["pandoc >=1.19.2.1"], + "pandoc (http://pandoc.org/installing.html) for generating reports from markdown files.": [ + "pandoc" + ], + "perl": ["perl >=5.6.0"], + "Perl": ["perl >=5.6.0"], + "Perl (>= 5.6.0)": ["perl >=5.6.0"], + "python (>= 2.7)": ["python >=2.7"], + "Python (>=2.7.0)": ["python >=2.7"], + "python (< 3.7)": ["python <3.7"], + "root_v5.34.36 - See README file for installation instructions.": [ + "root5 5.34.36" + ], + "rTANDEM uses expat and pthread libraries. See the README file for details.": [ + "expat" + ], + "samtools": ["samtools"], + "scipy": ["scipiy"], + "sklearn": ["scikit-learn"], + "STAR": ["star"], + "tensorflow": ["tensorflow"], + "To generate html reports pandoc (http://pandoc.org/installing.html) is required.": [ + "pandoc" + ], + "TopHat": ["tophat2"], + "ViennaRNA (>= 2.4.1)": ["viennarna >=2.4.1"], + "xml2": ["libxml2"], +} HERE = os.path.abspath(os.path.dirname(__file__)) @@ -141,7 +195,9 @@ def bioconductor_versions(): bioc_config = yaml.safe_load(response.text) versions = list(bioc_config["r_ver_for_bioc_ver"].keys()) # Handle semantic version sorting like 3.10 and 3.9 - versions = sorted(versions, key=lambda v: list(map(int, v.split('.'))), reverse=True) + versions = sorted( + versions, key=lambda v: list(map(int, v.split("."))), reverse=True + ) return versions @@ -175,8 +231,8 @@ def bioconductor_tarball_url(package, pkg_version, bioc_version): Bioconductor release version """ return ( - 'https://bioconductor.org/packages/{bioc_version}' - '/bioc/src/contrib/{package}_{pkg_version}.tar.gz'.format(**locals()) + "https://bioconductor.org/packages/{bioc_version}" + "/bioc/src/contrib/{package}_{pkg_version}.tar.gz".format(**locals()) ) @@ -196,8 +252,8 @@ def bioconductor_annotation_data_url(package, pkg_version, bioc_version): Bioconductor release version """ return ( - 'https://bioconductor.org/packages/{bioc_version}' - '/data/annotation/src/contrib/{package}_{pkg_version}.tar.gz'.format(**locals()) + "https://bioconductor.org/packages/{bioc_version}" + "/data/annotation/src/contrib/{package}_{pkg_version}.tar.gz".format(**locals()) ) @@ -217,8 +273,8 @@ def bioconductor_experiment_data_url(package, pkg_version, bioc_version): Bioconductor release version """ return ( - 'https://bioconductor.org/packages/{bioc_version}' - '/data/experiment/src/contrib/{package}_{pkg_version}.tar.gz'.format(**locals()) + "https://bioconductor.org/packages/{bioc_version}" + "/data/experiment/src/contrib/{package}_{pkg_version}.tar.gz".format(**locals()) ) @@ -238,9 +294,8 @@ def bioarchive_url(package, pkg_version, bioc_version=None): Bioconductor release version. Not used;, only included for API compatibility with other url funcs """ - return ( - 'https://bioarchive.galaxyproject.org/{0}_{1}.tar.gz' - .format(package, pkg_version) + return "https://bioarchive.galaxyproject.org/{0}_{1}.tar.gz".format( + package, pkg_version ) @@ -262,8 +317,8 @@ def cargoport_url(package, pkg_version, bioc_version=None): """ package = package.lower() return ( - 'https://depot.galaxyproject.org/software/bioconductor-{0}/bioconductor-{0}_' - '{1}_src_all.tar.gz'.format(package, pkg_version) + "https://depot.galaxyproject.org/software/bioconductor-{0}/bioconductor-{0}_" + "{1}_src_all.tar.gz".format(package, pkg_version) ) @@ -286,22 +341,27 @@ def find_best_bioc_version(package, version): """ for bioc_version in bioconductor_versions(): for kind, func in zip( - ('package', 'data'), + ("package", "data"), ( - bioconductor_tarball_url, bioconductor_annotation_data_url, + bioconductor_tarball_url, + bioconductor_annotation_data_url, bioconductor_experiment_data_url, ), ): url = func(package, version, bioc_version) if requests.head(url).status_code == 200: - logger.debug('success: %s', url) + logger.debug("success: %s", url) logger.info( - 'A working URL for %s==%s was identified for Bioconductor version %s: %s', - package, version, bioc_version, url) + "A working URL for %s==%s was identified for Bioconductor version %s: %s", + package, + version, + bioc_version, + url, + ) found_version = bioc_version return found_version else: - logger.debug('missing: %s', url) + logger.debug("missing: %s", url) raise PackageNotFoundError( "Cannot find any Bioconductor versions for {0}=={1}".format(package, version) ) @@ -322,9 +382,17 @@ def fetchPackages(bioc_version): } """ d = dict() - packages_urls = [(os.path.join(base_url, bioc_version, 'bioc', 'VIEWS'), 'bioc'), - (os.path.join(base_url, bioc_version, 'data', 'annotation', 'VIEWS'), 'data/annotation'), - (os.path.join(base_url, bioc_version, 'data', 'experiment', 'VIEWS'), 'data/experiment')] + packages_urls = [ + (os.path.join(base_url, bioc_version, "bioc", "VIEWS"), "bioc"), + ( + os.path.join(base_url, bioc_version, "data", "annotation", "VIEWS"), + "data/annotation", + ), + ( + os.path.join(base_url, bioc_version, "data", "experiment", "VIEWS"), + "data/experiment", + ), + ] for url, prefix in packages_urls: req = requests.get(url) if not req.ok: @@ -335,13 +403,15 @@ def fetchPackages(bioc_version): for line in pkg.split("\n"): if line.startswith(" "): pkgDict[lastKey] += " {}".format(line.strip()) - pkgDict[lastKey] = pkgDict[lastKey].strip() # Prevent prepending a space when content begins on the next line + pkgDict[lastKey] = pkgDict[ + lastKey + ].strip() # Prevent prepending a space when content begins on the next line elif ":" in line: idx = line.index(":") lastKey = line[:idx] - pkgDict[lastKey] = line[idx + 2:].strip() - pkgDict['URLprefix'] = prefix.strip() - d[pkgDict['Package']] = pkgDict + pkgDict[lastKey] = line[idx + 2 :].strip() + pkgDict["URLprefix"] = prefix.strip() + d[pkgDict["Package"]] = pkgDict return d @@ -403,7 +473,9 @@ def __init__(self, package, bioc_version=None, pkg_version=None, packages=None): if not self._pkg_version: self.bioc_version = latest_bioconductor_release_version() else: - self.bioc_version = find_best_bioc_version(self.package, self._pkg_version) + self.bioc_version = find_best_bioc_version( + self.package, self._pkg_version + ) # Fetch a list of all packages, so dependency versions can be calculated if not packages: @@ -412,27 +484,36 @@ def __init__(self, package, bioc_version=None, pkg_version=None, packages=None): self.packages = packages if package not in self.packages: - raise PackageNotFoundError('{} does not exist in this bioconductor release!'.format(package)) + raise PackageNotFoundError( + "{} does not exist in this bioconductor release!".format(package) + ) if not pkg_version: - self.version = self.packages[package]['Version'] + self.version = self.packages[package]["Version"] self.depends_on_gcc = False # Determine the URL - url = os.path.join(base_url, self.bioc_version, self.packages[package]['URLprefix'], 'html', package + '.html') + url = os.path.join( + base_url, + self.bioc_version, + self.packages[package]["URLprefix"], + "html", + package + ".html", + ) request = requests.get(url) if not request: raise PageNotFoundError( - 'Could not find HTML page for {0.package}. Tried: ' - '{1}'.format(self, url)) + "Could not find HTML page for {0.package}. Tried: " + "{1}".format(self, url) + ) # Since we provide the "short link" we will get redirected. Using # requests allows us to keep track of the final destination URL, # which we need for reconstructing the tarball URL. self.url = request.url - if self.packages[package]['URLprefix'] != 'bioc': + if self.packages[package]["URLprefix"] != "bioc": self.is_data_package = True @property @@ -462,14 +543,20 @@ def cargoport_url(self): return url else: raise PageNotFoundError( - "Unexpected error: {0.status_code} ({0.reason})".format(response)) + "Unexpected error: {0.status_code} ({0.reason})".format(response) + ) @property def bioconductor_tarball_url(self): """ Return the url to the tarball from the bioconductor site. """ - url = os.path.join(base_url, self.bioc_version, self.packages[self.package]['URLprefix'], self.packages[self.package]['source.ver']) + url = os.path.join( + base_url, + self.bioc_version, + self.packages[self.package]["URLprefix"], + self.packages[self.package]["source.ver"], + ) response = requests.head(url) if response.status_code == 200: return url @@ -477,9 +564,11 @@ def bioconductor_tarball_url(self): @property def tarball_url(self): if not self._tarball_url: - urls = [self.bioconductor_tarball_url, - self.bioarchive_url, - self.cargoport_url] + urls = [ + self.bioconductor_tarball_url, + self.bioarchive_url, + self.cargoport_url, + ] for url in urls: if url is not None: response = requests.head(url) @@ -488,9 +577,11 @@ def tarball_url(self): return url logger.error( - 'No working URL for %s==%s in Bioconductor %s. ' - 'Tried the following:\n\t' + '\n\t'.join(urls), - self.package, self.version, self.bioc_version + "No working URL for %s==%s in Bioconductor %s. " + "Tried the following:\n\t" + "\n\t".join(urls), + self.package, + self.version, + self.bioc_version, ) if self._auto: @@ -498,7 +589,8 @@ def tarball_url(self): if self._tarball_url is None: raise ValueError( - "No working URLs found for this version in any bioconductor version") + "No working URLs found for this version in any bioconductor version" + ) return self._tarball_url @property @@ -516,7 +608,7 @@ def cached_tarball(self): """ if self._cached_tarball: return self._cached_tarball - cache_dir = os.path.join(tempfile.gettempdir(), 'cached_bioconductor_tarballs') + cache_dir = os.path.join(tempfile.gettempdir(), "cached_bioconductor_tarballs") if not os.path.exists(cache_dir): os.makedirs(cache_dir) fn = os.path.join(cache_dir, self.tarball_basename) @@ -524,14 +616,15 @@ def cached_tarball(self): self._cached_tarball = fn return fn tmp = tempfile.NamedTemporaryFile(delete=False).name - with open(tmp, 'wb') as fout: - logger.info('Downloading {0} to {1}'.format(self.tarball_url, fn)) + with open(tmp, "wb") as fout: + logger.info("Downloading {0} to {1}".format(self.tarball_url, fn)) response = requests.get(self.tarball_url) if response.status_code == 200: fout.write(response.content) else: raise PageNotFoundError( - 'Unexpected error {0.status_code} ({0.reason})'.format(response)) + "Unexpected error {0.status_code} ({0.reason})".format(response) + ) shutil.move(tmp, fn) self._cached_tarball = fn return fn @@ -541,18 +634,18 @@ def title(self): """ The Title section fromt he VIEW file """ - return self.packages[self.package]['Title'] + return self.packages[self.package]["Title"] @property def description(self): """ The "Description" from the VIEW file """ - return self.packages[self.package]['Description'] + return self.packages[self.package]["Description"] @property def license(self): - return self.packages[self.package]['License'] + return self.packages[self.package]["License"] def license_file_location(self): """ @@ -569,14 +662,16 @@ def license_file_location(self): if "LICENSE" in license: return "LICENSE" - licenses = {'GPL-2': '{{ environ["PREFIX"] }}/lib/R/share/licenses/GPL-2', - 'GPL (== 2)': '{{ environ["PREFIX"] }}/lib/R/share/licenses/GPL-2', - 'GPL (==2)': '{{ environ["PREFIX"] }}/lib/R/share/licenses/GPL-2', - 'GPL version 2': '{{ environ["PREFIX"] }}/lib/R/share/licenses/GPL-2', - 'AGPL-3': '{{ environ["PREFIX"] }}/lib/R/share/licenses/AGPL-3', - 'Artisitic-2.0': '{{ environ["PREFIX"] }}/lib/R/share/licenses/Artistic-2.0', - 'LGPL-2': '{{ environ["PREFIX"] }}/lib/R/share/licenses/LGPL-2', - 'LGPL-2.1': '{{ environ["PREFIX"] }}/lib/R/share/licenses/LGPL-2.1'} + licenses = { + "GPL-2": '{{ environ["PREFIX"] }}/lib/R/share/licenses/GPL-2', + "GPL (== 2)": '{{ environ["PREFIX"] }}/lib/R/share/licenses/GPL-2', + "GPL (==2)": '{{ environ["PREFIX"] }}/lib/R/share/licenses/GPL-2', + "GPL version 2": '{{ environ["PREFIX"] }}/lib/R/share/licenses/GPL-2', + "AGPL-3": '{{ environ["PREFIX"] }}/lib/R/share/licenses/AGPL-3', + "Artisitic-2.0": '{{ environ["PREFIX"] }}/lib/R/share/licenses/Artistic-2.0', + "LGPL-2": '{{ environ["PREFIX"] }}/lib/R/share/licenses/LGPL-2', + "LGPL-2.1": '{{ environ["PREFIX"] }}/lib/R/share/licenses/LGPL-2.1', + } if license in licenses: return licenses[license] @@ -588,14 +683,18 @@ def license_file_location(self): return '{{ environ["PREFIX"] }}/lib/R/share/licenses/GPL-3' return None - @property def imports(self): """ List of "imports" from the VIEW file """ try: - return [i.strip() for i in self.packages[self.package]['Imports'].replace(' ', '').split(',')] + return [ + i.strip() + for i in self.packages[self.package]["Imports"] + .replace(" ", "") + .split(",") + ] except KeyError: return [] @@ -605,7 +704,7 @@ def systemrequirements(self): List of "SystemRequirements" from the VIEW file """ try: - return self.packages[self.package]['SystemRequirements'] + return self.packages[self.package]["SystemRequirements"] except KeyError: return [] @@ -615,7 +714,12 @@ def depends(self): List of "depends" from the VIEW file """ try: - return [i.strip() for i in self.packages[self.package]['Depends'].replace(' ', '').split(',')] + return [ + i.strip() + for i in self.packages[self.package]["Depends"] + .replace(" ", "") + .split(",") + ] except KeyError: return [] @@ -625,7 +729,12 @@ def linkingto(self): List of "linkingto" from the VIEW file """ try: - return [i.strip() for i in self.packages[self.package]['LinkingTo'].replace(' ', '').split(',')] + return [ + i.strip() + for i in self.packages[self.package]["LinkingTo"] + .replace(" ", "") + .split(",") + ] except KeyError: return [] @@ -647,12 +756,12 @@ def _parse_dependencies(self, items): """ results = [] for item in items: - toks = [i.strip() for i in item.split('(')] + toks = [i.strip() for i in item.split("(")] if len(toks) == 1: results.append((toks[0], "")) elif len(toks) == 2: - assert ')' in toks[1] - toks[1] = toks[1].replace(')', '').replace(' ', '') + assert ")" in toks[1] + toks[1] = toks[1].replace(")", "").replace(" ", "") results.append(tuple(toks)) else: raise ValueError("Found {0} toks: {1}".format(len(toks), toks)) @@ -671,7 +780,7 @@ def pin_version(self, name): - BSgenome.Vvinifera.URGI.IGGP12Xv0 - BSgenome.Vvinifera.URGI.IGGP12Xv2 """ - v = str(self.packages[name]['Version']) + v = str(self.packages[name]["Version"]) # There are a few packages with only major.minor versions! s = v.split(".") if len(s) == 3: @@ -681,9 +790,7 @@ def pin_version(self, name): @property def dependencies(self): - """ - - """ + """ """ if self._dependencies: return self._dependencies @@ -693,9 +800,9 @@ def dependencies(self): # `imports`. We keep the most specific version of each. version_specs = list( set( - self._parse_dependencies(self.imports) + - self._parse_dependencies(self.depends) + - self._parse_dependencies(self.linkingto) + self._parse_dependencies(self.imports) + + self._parse_dependencies(self.depends) + + self._parse_dependencies(self.linkingto) ) ) specific_r_version = False @@ -716,14 +823,16 @@ def dependencies(self): # Try finding the dependency on the bioconductor site; if it can't # be found then we assume it's in CRAN. if name in self.packages: - prefix = 'bioconductor-' + prefix = "bioconductor-" version = self.pin_version(name) else: - prefix = 'r-' + prefix = "r-" - logger.info('{0:>12} dependency: name="{1}" version="{2}"'.format( - {'r-': 'R', 'bioconductor-': 'BioConductor'}[prefix], - name, version)) + logger.info( + '{0:>12} dependency: name="{1}" version="{2}"'.format( + {"r-": "R", "bioconductor-": "BioConductor"}[prefix], name, version + ) + ) # add padding to version string if version: @@ -733,8 +842,7 @@ def dependencies(self): # the dependency `r-base` for that version. Otherwise, R is # implied, and we make sure that r-base is added as a dependency at # the end. - if name.lower() == 'r': - + if name.lower() == "r": # Had some issues with CONDA_R finding the right version if "r" # had version restrictions. Since we're generally building # up-to-date packages, we can just use "r". @@ -742,45 +850,49 @@ def dependencies(self): # # "r >=2.5" rather than "r-r >=2.5" specific_r_version = True - dependency_mapping[name.lower() + '-base'] = 'r-base' + dependency_mapping[name.lower() + "-base"] = "r-base" else: dependency_mapping[prefix + name.lower() + version] = name # Check SystemRequirements in the DESCRIPTION file to make sure # packages with such reqquirements are provided correct recipes. - if (self.packages[self.package].get('SystemRequirements') is not None): + if self.packages[self.package].get("SystemRequirements") is not None: logger.warning( "The 'SystemRequirements' {} are needed".format( - self.packages[self.package].get('SystemRequirements')) + - " by the recipe for the package to work as intended." + self.packages[self.package].get("SystemRequirements") + ) + + " by the recipe for the package to work as intended." ) - if ( - (self.packages[self.package].get('NeedsCompilation', 'no') == 'yes') or - (self.packages[self.package].get('LinkingTo', None) is not None) + if (self.packages[self.package].get("NeedsCompilation", "no") == "yes") or ( + self.packages[self.package].get("LinkingTo", None) is not None ): # Modified from conda_build.skeletons.cran # with tarfile.open(self.cached_tarball) as tf: - need_f = any(f.name.lower().endswith(('.f', '.f90', '.f77')) for f in tf) + need_f = any( + f.name.lower().endswith((".f", ".f90", ".f77")) for f in tf + ) if need_f: need_c = True else: - need_c = any(f.name.lower().endswith('.c') for f in tf) + need_c = any(f.name.lower().endswith(".c") for f in tf) need_cxx = any( - f.name.lower().endswith(('.cxx', '.cpp', '.cc', '.c++')) for f in tf) - need_autotools = any(f.name.lower().endswith('/configure') for f in tf) + f.name.lower().endswith((".cxx", ".cpp", ".cc", ".c++")) for f in tf + ) + need_autotools = any(f.name.lower().endswith("/configure") for f in tf) if any((need_autotools, need_f, need_cxx, need_c)): need_make = True else: need_make = any( - f.name.lower().endswith(('/makefile', '/makevars')) for f in tf) + f.name.lower().endswith(("/makefile", "/makevars")) for f in tf + ) else: need_c = need_cxx = need_f = need_autotools = need_make = False for name, version in sorted(versions.items()): - if name in ['Rcpp', 'RcppArmadillo']: + if name in ["Rcpp", "RcppArmadillo"]: need_cxx = True if need_cxx: @@ -788,24 +900,24 @@ def dependencies(self): self._cb3_build_reqs = {} if need_c: - self._cb3_build_reqs['c'] = "{{ compiler('c') }}" + self._cb3_build_reqs["c"] = "{{ compiler('c') }}" if need_cxx: - self._cb3_build_reqs['cxx'] = "{{ compiler('cxx') }}" + self._cb3_build_reqs["cxx"] = "{{ compiler('cxx') }}" if need_f: - self._cb3_build_reqs['fortran'] = "{{ compiler('fortran') }}" + self._cb3_build_reqs["fortran"] = "{{ compiler('fortran') }}" if need_autotools: - self._cb3_build_reqs['automake'] = 'automake' + self._cb3_build_reqs["automake"] = "automake" if need_make: - self._cb3_build_reqs['make'] = 'make' + self._cb3_build_reqs["make"] = "make" # Add R itself if not specific_r_version: - dependency_mapping['r-base'] = 'r-base' + dependency_mapping["r-base"] = "r-base" # Sometimes empty dependencies make it into the list from a trailing # comma in DESCRIPTION; remove them here. for k in list(dependency_mapping.keys()): - if k == 'r-': + if k == "r-": dependency_mapping.pop(k) self._dependencies = dependency_mapping @@ -817,7 +929,7 @@ def md5(self): Calculate the md5 hash of the tarball so it can be filled into the meta.yaml. """ - return self.packages[self.package]['MD5sum'] + return self.packages[self.package]["MD5sum"] def pacified_text(self, section="Description"): """ @@ -829,15 +941,15 @@ def pacified_text(self, section="Description"): By default, this will pacify the Description section """ description = self.packages[self.package][section] - for vcs in ['HG', 'SVN', 'GIT']: - description = description.replace('{}_'.format(vcs), '{} '.format(vcs)) + for vcs in ["HG", "SVN", "GIT"]: + description = description.replace("{}_".format(vcs), "{} ".format(vcs)) return description def parseSystemRequirements(self, reqs): """ Parse the text version of system requirements and return a list of conda packages """ - reqs = reqs.split(',') + reqs = reqs.split(",") packages = [SysReqs[r.strip()] for r in reqs if r.strip() in SysReqs] packages = [] for req in reqs: @@ -869,23 +981,22 @@ def meta_yaml(self): the yaml is written. """ - version_placeholder = '{{ version }}' - package_placeholder = '{{ name }}' - package_lower_placeholder = '{{ name|lower }}' - bioc_placeholder = '{{ bioc }}' + version_placeholder = "{{ version }}" + package_placeholder = "{{ name }}" + package_lower_placeholder = "{{ name|lower }}" + bioc_placeholder = "{{ bioc }}" def sub_placeholders(x): return ( - x - .replace(self.version, version_placeholder) + x.replace(self.version, version_placeholder) .replace(self.package, package_placeholder) .replace(self.package_lower, package_lower_placeholder) .replace(self.bioc_version, bioc_placeholder) ) url = [ - sub_placeholders(u) for u in - [ + sub_placeholders(u) + for u in [ # keep the one that was found self.bioconductor_tarball_url, # use the built URL, regardless of whether it was found or not. @@ -902,104 +1013,144 @@ def sub_placeholders(x): # Handle libblas and liblapack, which all compiled packages # are assumed to need additional_host_deps = [] - if self.linkingto != [] or len(set(['c', 'cxx', 'fortran']).intersection(self._cb3_build_reqs.keys())) > 0: - additional_host_deps.append('libblas') - additional_host_deps.append('liblapack') + if ( + self.linkingto != [] + or len( + set(["c", "cxx", "fortran"]).intersection(self._cb3_build_reqs.keys()) + ) + > 0 + ): + additional_host_deps.append("libblas") + additional_host_deps.append("liblapack") additional_run_deps = [] if self.is_data_package: - additional_run_deps.append('curl') - additional_run_deps.append('bioconductor-data-packages>={}'.format(date.today().strftime('%Y%m%d'))) + additional_run_deps.append("curl") + additional_run_deps.append( + "bioconductor-data-packages>={}".format(date.today().strftime("%Y%m%d")) + ) - d = OrderedDict(( - ( - 'package', OrderedDict(( - ('name', 'bioconductor-{{ name|lower }}'), - ('version', '{{ version }}'), - )), - ), - ( - 'source', OrderedDict(( - ('url', url), - ('md5', self.md5), - )), - ), + d = OrderedDict( ( - 'build', OrderedDict(( - ('number', self.build_number), - ('rpaths', ['lib/R/lib/', 'lib/']), - )), - ), - ( - 'requirements', OrderedDict(( - # If you don't make copies, pyaml sees these as the same - # object and tries to make a shortcut, causing an error in - # decoding unicode. Possible pyaml bug? Anyway, this fixes - # it. - ('host', DEPENDENCIES[:] + additional_host_deps), - ('run', DEPENDENCIES[:] + additional_run_deps), - )), - ), - ( - 'test', OrderedDict(( - ( - 'commands', ['''$R -e "library('{{ name }}')"'''] + ( + "package", + OrderedDict( + ( + ("name", "bioconductor-{{ name|lower }}"), + ("version", "{{ version }}"), + ) ), - )), - ), - ( - 'about', OrderedDict(( - ('home', sub_placeholders(self.url)), - ('license', self.license), - ('summary', self.pacified_text(section="Title")), - ('description', self.pacified_text(section="Description")), - )), - ), - )) + ), + ( + "source", + OrderedDict( + ( + ("url", url), + ("md5", self.md5), + ) + ), + ), + ( + "build", + OrderedDict( + ( + ("number", self.build_number), + ("rpaths", ["lib/R/lib/", "lib/"]), + ) + ), + ), + ( + "requirements", + OrderedDict( + ( + # If you don't make copies, pyaml sees these as the same + # object and tries to make a shortcut, causing an error in + # decoding unicode. Possible pyaml bug? Anyway, this fixes + # it. + ("host", DEPENDENCIES[:] + additional_host_deps), + ("run", DEPENDENCIES[:] + additional_run_deps), + ) + ), + ), + ( + "test", + OrderedDict((("commands", ['''$R -e "library('{{ name }}')"''']),)), + ), + ( + "about", + OrderedDict( + ( + ("home", sub_placeholders(self.url)), + ("license", self.license), + ("summary", self.pacified_text(section="Title")), + ("description", self.pacified_text(section="Description")), + ) + ), + ), + ) + ) if self.license_file_location(): - d['about']['license_file'] = self.license_file_location() - - if self.packages[self.package].get('SystemRequirements', None): - if self.parseSystemRequirements(self.packages[self.package]['SystemRequirements']): - d['requirements']['host'].extend(self.parseSystemRequirements(self.packages[self.package]['SystemRequirements'])) - d['requirements']['run'].extend(self.parseSystemRequirements(self.packages[self.package]['SystemRequirements'])) + d["about"]["license_file"] = self.license_file_location() + + if self.packages[self.package].get("SystemRequirements", None): + if self.parseSystemRequirements( + self.packages[self.package]["SystemRequirements"] + ): + d["requirements"]["host"].extend( + self.parseSystemRequirements( + self.packages[self.package]["SystemRequirements"] + ) + ) + d["requirements"]["run"].extend( + self.parseSystemRequirements( + self.packages[self.package]["SystemRequirements"] + ) + ) if self.needsX: # Anything that causes rgl to get imported needs X around if not self.extra: self.extra = OrderedDict() - self.extra['container'] = OrderedDict([('extended-base', True)]) + self.extra["container"] = OrderedDict([("extended-base", True)]) - if 'build' not in d['requirements']: + if "build" not in d["requirements"]: # This is filled in manually later since pyaml.dumps will mess of the formatting otherwise - d['requirements']['build'] = ["PLACEHOLDER"] - - d['test']['commands'] = ['''LD_LIBRARY_PATH="${BUILD_PREFIX}/x86_64-conda-linux-gnu/sysroot/usr/lib64:${BUILD_PREFIX}/lib" $R -e "library('{{ name }}')"'''] + d["requirements"]["build"] = ["PLACEHOLDER"] + d["test"]["commands"] = [ + '''LD_LIBRARY_PATH="${BUILD_PREFIX}/x86_64-conda-linux-gnu/sysroot/usr/lib64:${BUILD_PREFIX}/lib" $R -e "library('{{ name }}')"''' + ] if self.extra: - d['extra'] = self.extra + d["extra"] = self.extra if self._cb3_build_reqs: - d['requirements']['build'] = [] + d["requirements"]["build"] = [] else: - d['build']['noarch'] = 'generic' + d["build"]["noarch"] = "generic" for k, v in self._cb3_build_reqs.items(): - d['requirements']['build'].append(k + '_' + "PLACEHOLDER") + d["requirements"]["build"].append(k + "_" + "PLACEHOLDER") - rendered = pyaml.dumps(d, width=1e6).decode('utf-8') + rendered = pyaml.dumps(d, width=1e6).decode("utf-8") # Add Suggests: and SystemRequirements: - renderedsplit = rendered.split('\n') - idx = renderedsplit.index('requirements:') - if self.packages[self.package].get('SystemRequirements', None): - renderedsplit.insert(idx, '# SystemRequirements: {}'.format(self.packages[self.package]['SystemRequirements'])) - if self.packages[self.package].get('Suggests', None): - renderedsplit.insert(idx, '# Suggests: {}'.format(self.packages[self.package]['Suggests'])) + renderedsplit = rendered.split("\n") + idx = renderedsplit.index("requirements:") + if self.packages[self.package].get("SystemRequirements", None): + renderedsplit.insert( + idx, + "# SystemRequirements: {}".format( + self.packages[self.package]["SystemRequirements"] + ), + ) + if self.packages[self.package].get("Suggests", None): + renderedsplit.insert( + idx, "# Suggests: {}".format(self.packages[self.package]["Suggests"]) + ) # Fix the core dependencies if this needsX if self.needsX: - idx = renderedsplit.index(' build:') + 1 + idx = renderedsplit.index(" build:") + 1 renderedsplit.insert(idx, " - xorg-libxfixes # [linux]") renderedsplit.insert(idx, " - {{ cdt('libxxf86vm') }} # [linux]") renderedsplit.insert(idx, " - {{ cdt('libxdamage') }} # [linux]") @@ -1008,26 +1159,42 @@ def sub_placeholders(x): renderedsplit.insert(idx, " - {{ cdt('mesa-libgl-devel') }} # [linux]") if " - PLACEHOLDER" in renderedsplit: del renderedsplit[renderedsplit.index(" - PLACEHOLDER")] - rendered = '\n'.join(renderedsplit) + '\n' + rendered = "\n".join(renderedsplit) + "\n" rendered = ( - '{% set version = "' + self.version + '" %}\n' + - '{% set name = "' + self.package + '" %}\n' + - '{% set bioc = "' + self.bioc_version + '" %}\n\n' + - rendered + '{% set version = "' + + self.version + + '" %}\n' + + '{% set name = "' + + self.package + + '" %}\n' + + '{% set bioc = "' + + self.bioc_version + + '" %}\n\n' + + rendered ) for k, v in self._cb3_build_reqs.items(): - rendered = rendered.replace(k + '_' + "PLACEHOLDER", v) + rendered = rendered.replace(k + "_" + "PLACEHOLDER", v) tmpdir = tempfile.mkdtemp() - with open(os.path.join(tmpdir, 'meta.yaml'), 'w') as fout: + with open(os.path.join(tmpdir, "meta.yaml"), "w") as fout: fout.write(rendered) return fout.name -def write_recipe_recursive(proj, seen_dependencies, recipe_dir, config, bioc_data_packages, force, - bioc_version, pkg_version, versioned, recursive): +def write_recipe_recursive( + proj, + seen_dependencies, + recipe_dir, + config, + bioc_data_packages, + force, + bioc_version, + pkg_version, + versioned, + recursive, +): """ Parameters ---------- @@ -1050,25 +1217,26 @@ def write_recipe_recursive(proj, seen_dependencies, recipe_dir, config, bioc_dat force : bool If True, any recipes that already exist will be overwritten. """ - logger.debug('%s has dependencies: %s', proj.package, proj.dependencies) + logger.debug("%s has dependencies: %s", proj.package, proj.dependencies) for conda_name, cran_or_bioc_name in proj.dependencies.items(): - # For now, this function is version-agnostic. so we strip out any # version info when checking if we've seen this dep before. - conda_name_without_version = re.sub(r' >=.*$', '', conda_name) - if conda_name.startswith('r-base'): + conda_name_without_version = re.sub(r" >=.*$", "", conda_name) + if conda_name.startswith("r-base"): continue if conda_name_without_version in seen_dependencies: logger.debug( - "{} already created or in existing channels, skipping" - .format(conda_name_without_version)) + "{} already created or in existing channels, skipping".format( + conda_name_without_version + ) + ) continue seen_dependencies.update([conda_name_without_version]) - if conda_name_without_version.startswith('r-'): - #writer = cran_skeleton.write_recipe # We should only create bioconductor dependencies, otherwise we duplicate what's on conda-forge! + if conda_name_without_version.startswith("r-"): + # writer = cran_skeleton.write_recipe # We should only create bioconductor dependencies, otherwise we duplicate what's on conda-forge! continue else: writer = write_recipe @@ -1099,13 +1267,25 @@ def updateDataPackages(bioc_data_packages, pkg, urls, md5, tarball): jsContent = dict() if os.path.exists(jsPath): jsContent = json.load(open(jsPath)) - jsContent[pkg] = {'urls': urls, 'md5': md5, 'fn': tarball} + jsContent[pkg] = {"urls": urls, "md5": md5, "fn": tarball} json.dump(jsContent, open(jsPath, "w")) -def write_recipe(package, recipe_dir, config, bioc_data_packages=None, force=False, bioc_version=None, - pkg_version=None, versioned=False, recursive=False, seen_dependencies=None, - packages=None, skip_if_in_channels=None, needs_x=None): +def write_recipe( + package, + recipe_dir, + config, + bioc_data_packages=None, + force=False, + bioc_version=None, + pkg_version=None, + versioned=False, + recursive=False, + seen_dependencies=None, + packages=None, + skip_if_in_channels=None, + needs_x=None, +): """ Write the meta.yaml and build.sh files. If the package is detected to be a data package (bsed on the detected URL from Bioconductor), then also @@ -1161,7 +1341,7 @@ def write_recipe(package, recipe_dir, config, bioc_data_packages=None, force=Fal """ config = utils.load_config(config) proj = BioCProjectPage(package, bioc_version, pkg_version, packages=packages) - logger.info('Making recipe for: {}'.format(package)) + logger.info("Making recipe for: {}".format(package)) if bioc_data_packages is None: bioc_data_packages = os.path.join(recipe_dir, "bioconductor-data-packages") @@ -1177,58 +1357,71 @@ def write_recipe(package, recipe_dir, config, bioc_data_packages=None, force=Fal if recursive: # get a list of existing packages in channels if skip_if_in_channels is not None: - for name in set(utils.RepoData().get_package_data("name", channels=skip_if_in_channels)): - if name.startswith(('r-', 'bioconductor-')): + for name in set( + utils.RepoData().get_package_data("name", channels=skip_if_in_channels) + ): + if name.startswith(("r-", "bioconductor-")): seen_dependencies.add(name) - write_recipe_recursive(proj, seen_dependencies, recipe_dir, config, - bioc_data_packages, force, bioc_version, pkg_version, versioned, - recursive) + write_recipe_recursive( + proj, + seen_dependencies, + recipe_dir, + config, + bioc_data_packages, + force, + bioc_version, + pkg_version, + versioned, + recursive, + ) - logger.debug('%s==%s, BioC==%s', proj.package, proj.version, proj.bioc_version) - logger.info('Using tarball from %s', proj.tarball_url) + logger.debug("%s==%s, BioC==%s", proj.package, proj.version, proj.bioc_version) + logger.info("Using tarball from %s", proj.tarball_url) if versioned: - recipe_dir = os.path.join(recipe_dir, 'bioconductor-' + proj.package.lower(), proj.version) + recipe_dir = os.path.join( + recipe_dir, "bioconductor-" + proj.package.lower(), proj.version + ) else: - recipe_dir = os.path.join(recipe_dir, 'bioconductor-' + proj.package.lower()) + recipe_dir = os.path.join(recipe_dir, "bioconductor-" + proj.package.lower()) if os.path.exists(recipe_dir) and not force: raise ValueError("{0} already exists, aborting".format(recipe_dir)) else: if not os.path.exists(recipe_dir): - logger.info('creating %s' % recipe_dir) + logger.info("creating %s" % recipe_dir) os.makedirs(recipe_dir) # If the version number has not changed but something else in the recipe # *has* changed, then bump the version number. - meta_file = os.path.join(recipe_dir, 'meta.yaml') + meta_file = os.path.join(recipe_dir, "meta.yaml") if os.path.exists(meta_file): updated_meta = utils.load_first_metadata(proj.meta_yaml, finalize=False).meta current_meta = utils.load_first_metadata(meta_file, finalize=False).meta # pop off the version and build numbers so we can compare the rest of # the dicts - updated_version = updated_meta['package']['version'] - current_version = current_meta['package']['version'] + updated_version = updated_meta["package"]["version"] + current_version = current_meta["package"]["version"] # updated_build_number = updated_meta['build'].pop('number') - current_build_number = current_meta['build'].pop('number', 0) + current_build_number = current_meta["build"].pop("number", 0) - if ( - (updated_version == current_version) and - (updated_meta != current_meta) - ): + if (updated_version == current_version) and (updated_meta != current_meta): proj.build_number = int(current_build_number) + 1 - if 'extra' in current_meta: - exclude = set(['final', 'copy_test_source_files']) - proj.extra = {x: y for x, y in current_meta['extra'].items() if x not in exclude} + if "extra" in current_meta: + exclude = set(["final", "copy_test_source_files"]) + proj.extra = { + x: y for x, y in current_meta["extra"].items() if x not in exclude + } - with open(os.path.join(recipe_dir, 'meta.yaml'), 'w') as fout: + with open(os.path.join(recipe_dir, "meta.yaml"), "w") as fout: fout.write(open(proj.meta_yaml).read()) if not proj.is_data_package: - with open(os.path.join(recipe_dir, 'build.sh'), 'w') as fout: - fout.write(dedent( - '''\ + with open(os.path.join(recipe_dir, "build.sh"), "w") as fout: + fout.write( + dedent( + """\ #!/bin/bash mv DESCRIPTION DESCRIPTION.old grep -v '^Priority: ' DESCRIPTION.old > DESCRIPTION @@ -1239,35 +1432,48 @@ def write_recipe(package, recipe_dir, config, bioc_data_packages=None, force=Fal CXX98=$CXX CXX11=$CXX CXX14=$CXX" > ~/.R/Makevars - ''')) + """ + ) + ) if needs_x: - fout.write(dedent( - '''export LD_LIBRARY_PATH=${BUILD_PREFIX}/x86_64-conda-linux-gnu/sysroot/usr/lib64:${BUILD_PREFIX}/lib - ''')) - fout.write(dedent('''$R CMD INSTALL --build .''')) + fout.write( + dedent( + """export LD_LIBRARY_PATH=${BUILD_PREFIX}/x86_64-conda-linux-gnu/sysroot/usr/lib64:${BUILD_PREFIX}/lib + """ + ) + ) + fout.write(dedent("""$R CMD INSTALL --build .""")) else: urls = [ - '{0}'.format(u) for u in [ + "{0}".format(u) + for u in [ proj.bioconductor_tarball_url, bioarchive_url(proj.package, proj.version, proj.bioc_version), cargoport_url(proj.package, proj.version, proj.bioc_version), - proj.cargoport_url + proj.cargoport_url, ] if u is not None ] - recipeName = '{}-{}'.format(proj.package.lower(), proj.version) + recipeName = "{}-{}".format(proj.package.lower(), proj.version) post_link_template = dedent( - '''\ + """\ #!/bin/bash installBiocDataPackage.sh "{recipeName}" - '''.format(recipeName=recipeName)) + """.format( + recipeName=recipeName + ) + ) - with open(os.path.join(recipe_dir, 'post-link.sh'), 'w') as fout: + with open(os.path.join(recipe_dir, "post-link.sh"), "w") as fout: fout.write(dedent(post_link_template)) - pre_unlink_template = "R CMD REMOVE --library=$PREFIX/lib/R/library/ {0}\n".format(package) - with open(os.path.join(recipe_dir, 'pre-unlink.sh'), 'w') as fout: + pre_unlink_template = ( + "R CMD REMOVE --library=$PREFIX/lib/R/library/ {0}\n".format(package) + ) + with open(os.path.join(recipe_dir, "pre-unlink.sh"), "w") as fout: fout.write(pre_unlink_template) - updateDataPackages(bioc_data_packages, recipeName, urls, proj.md5, proj.tarball_basename) + updateDataPackages( + bioc_data_packages, recipeName, urls, proj.md5, proj.tarball_basename + ) - logger.info('Wrote recipe in %s', recipe_dir) + logger.info("Wrote recipe in %s", recipe_dir) diff --git a/bioconda_utils/bot/chat.py b/bioconda_utils/bot/chat.py index e3f60bd5d4..ee7522062a 100644 --- a/bioconda_utils/bot/chat.py +++ b/bioconda_utils/bot/chat.py @@ -30,11 +30,18 @@ class GitterListener: api: Gitter API object rooms: Map containing rooms and their respective github user/repo """ - def __init__(self, app: aiohttp.web.Application, token: str, rooms: Dict[str, str], - session: aiohttp.ClientSession, ghappapi) -> None: + + def __init__( + self, + app: aiohttp.web.Application, + token: str, + rooms: Dict[str, str], + session: aiohttp.ClientSession, + ghappapi, + ) -> None: self.rooms = rooms self._ghappapi = ghappapi - self._api = AioGitterAPI(app['client_session'], token) + self._api = AioGitterAPI(app["client_session"], token) self._user: gitter.User = None self._tasks: List[Any] = [] self._session = session @@ -53,8 +60,7 @@ async def start(self, app: aiohttp.web.Application) -> None: logger.debug("%s: Room Info: %s", self, room) logger.debug("%s: Groups Info: %s", self, await self._api.list_groups()) - self._tasks = [app.loop.create_task(self.listen(room)) - for room in self.rooms] + self._tasks = [app.loop.create_task(self.listen(room)) for room in self.rooms] async def shutdown(self, _app: aiohttp.web.Application) -> None: """Send cancel signal to listener""" @@ -68,7 +74,7 @@ async def shutdown(self, _app: aiohttp.web.Application) -> None: async def listen(self, room_name: str) -> None: """Main run loop""" try: - user, repo = self.rooms[room_name].split('/') + user, repo = self.rooms[room_name].split("/") logger.error("Listening in %s for repo %s/%s", room_name, user, repo) message = None while True: @@ -89,16 +95,21 @@ async def listen(self, room_name: str) -> None: # http errors just get logged except aiohttp.ClientResponseError as exc: - logger.exception("HTTP Error Code %s while listening to room %s", - exc.code, room_name) + logger.exception( + "HTTP Error Code %s while listening to room %s", + exc.code, + room_name, + ) # asyncio cancellation needs to be passed up - except asyncio.CancelledError: # pylint: disable=try-except-raise + except asyncio.CancelledError: # pylint: disable=try-except-raise raise # the rest, we just log so that we remain online after an error except Exception: # pylint: disable=broad-except - logger.exception("Unexpected exception caught. Last message: '%s'", message) + logger.exception( + "Unexpected exception caught. Last message: '%s'", message + ) await asyncio.sleep(1) except asyncio.CancelledError: @@ -111,31 +122,42 @@ async def listen(self, room_name: str) -> None: await self._api.leave_room(self._user, room) logger.error("%s: left room %s", self, room_name) - async def handle_msg(self, room: gitter.Room, message: gitter.Message, ghapi) -> None: + async def handle_msg( + self, room: gitter.Room, message: gitter.Message, ghapi + ) -> None: """Parse Gitter message and dispatch via command_routes""" await self._api.mark_as_read(self._user, room, [message.id]) if self._user.id not in (m.userId for m in message.mentions): - if self._user.username.lower() in (m.screenName.lower() for m in message.mentions): - await self._api.send_message(room, "@%s - are you talking to me?", - message.fromUser.username) + if self._user.username.lower() in ( + m.screenName.lower() for m in message.mentions + ): + await self._api.send_message( + room, "@%s - are you talking to me?", message.fromUser.username + ) return - command = message.text.strip().lstrip('@'+self._user.username).strip() + command = message.text.strip().lstrip("@" + self._user.username).strip() if command == message.text.strip(): - await self._api.send_message(room, "Hmm? Someone talking about me?", - message.fromUser.username) + await self._api.send_message( + room, "Hmm? Someone talking about me?", message.fromUser.username + ) return cmd, *args = command.split() issue_number = None try: - if args[-1][0] == '#': + if args[-1][0] == "#": issue_number = int(args[-1][1:]) args.pop() except (ValueError, IndexError): pass - response = await command_routes.dispatch(cmd.lower(), ghapi, issue_number, - message.fromUser.username, *args) + response = await command_routes.dispatch( + cmd.lower(), ghapi, issue_number, message.fromUser.username, *args + ) if response: - await self._api.send_message(room, "@%s: %s", message.fromUser.username, response) + await self._api.send_message( + room, "@%s: %s", message.fromUser.username, response + ) else: - await self._api.send_message(room, "@%s: command failed", message.fromUser.username) + await self._api.send_message( + room, "@%s: command failed", message.fromUser.username + ) diff --git a/bioconda_utils/bot/commands.py b/bioconda_utils/bot/commands.py index e6ac38b5e2..4062a15d96 100644 --- a/bioconda_utils/bot/commands.py +++ b/bioconda_utils/bot/commands.py @@ -20,15 +20,20 @@ class CommandDispatch: >>> def command_hello(event, ghapi, *args): >>> logger.info("Got command 'hello %s'", " ".join(args)) """ + def __init__(self): self.mapping: Dict[str, Callable] = {} - def register(self, cmd: str, description: str = None) -> Callable[[Callable], Callable]: + def register( + self, cmd: str, description: str = None + ) -> Callable[[Callable], Callable]: """Decorator adding decorated function to dispatcher""" + def decorator(func): desc = description or getdoc(func) self.mapping[cmd] = (func, desc) return func + return decorator async def dispatch(self, cmd, *args, **kwargs): @@ -46,9 +51,11 @@ async def dispatch(self, cmd, *args, **kwargs): command_routes = CommandDispatch() # pylint: disable=invalid-name -def permissions(member: Optional[bool] = None, - author: Optional[bool] = None, - team: Optional[str] = None): +def permissions( + member: Optional[bool] = None, + author: Optional[bool] = None, + team: Optional[str] = None, +): """Decorator for commands checking for permissions Permissions are OR combined. Repeat the decorator to get AND. @@ -57,6 +64,7 @@ def permissions(member: Optional[bool] = None, member: if true, user must be organization member author: if true, user must be author, if false user must not be author """ + def decorator(func): @functools.wraps(func) async def wrapper(ghapi, issue_number, user, *args): @@ -71,7 +79,7 @@ async def wrapper(ghapi, issue_number, user, *args): err += [f"a member of {ghapi.user.capitalize()}"] if not okmsg and author is not None: prq = await ghapi.get_prs(number=issue_number) - pr_author = prq['user']['login'] + pr_author = prq["user"]["login"] negate = "" if author else "not " if (pr_author == user) == author: okmsg = f"User is {negate}PR author" @@ -85,13 +93,21 @@ async def wrapper(ghapi, issue_number, user, *args): msg = okmsg or f"Permission denied. You need to be {' or '.join(err)}." - logger.warning("Access %s: %s wants to run %s on %s#%s ('%s')", - "GRANTED" if okmsg else "DENIED", - user, func.__name__, ghapi, issue_number, msg) + logger.warning( + "Access %s: %s wants to run %s on %s#%s ('%s')", + "GRANTED" if okmsg else "DENIED", + user, + func.__name__, + ghapi, + issue_number, + msg, + ) if okmsg: return await func(ghapi, issue_number, user, *args) return msg + return wrapper + return decorator @@ -111,7 +127,7 @@ async def command_hello(ghapi, issue_number, user, *_args): @command_routes.register("bump") @permissions(member=True, author=True) async def command_bump(ghapi, issue_number, _user, *_args): - """Bump the build number of a recipe """ + """Bump the build number of a recipe""" tasks.bump.apply_async((issue_number, ghapi)) return f"Scheduled bump of build number for #{issue_number}" @@ -120,8 +136,8 @@ async def command_bump(ghapi, issue_number, _user, *_args): async def command_lint(ghapi, issue_number, _user, *_args): """Lint the current recipe""" ( - tasks.get_latest_pr_commit.s(issue_number, ghapi) | - tasks.create_check_run.s(ghapi) + tasks.get_latest_pr_commit.s(issue_number, ghapi) + | tasks.create_check_run.s(ghapi) ).apply_async() return f"Scheduled creation of new check run for latest commit in #{issue_number}" @@ -133,8 +149,8 @@ async def command_recheck(ghapi, issue_number, _user, *_args): tasks.check_circle_artifacts.s(issue_number, ghapi).apply_async() # queue lint check ( - tasks.get_latest_pr_commit.s(issue_number, ghapi) | - tasks.create_check_run.s(ghapi) + tasks.get_latest_pr_commit.s(issue_number, ghapi) + | tasks.create_check_run.s(ghapi) ).apply_async() return f"Scheduled check run and circle artificats verification on #{issue_number}" @@ -153,8 +169,8 @@ async def command_merge(ghapi, issue_number, user, *_args): """Merge PR""" comment_id = await ghapi.create_comment(issue_number, "Scheduled Upload & Merge") ( - tasks.merge_pr.si(issue_number, comment_id, ghapi) | - tasks.post_result.s(issue_number, comment_id, "merge", user, ghapi) + tasks.merge_pr.si(issue_number, comment_id, ghapi) + | tasks.post_result.s(issue_number, comment_id, "merge", user, ghapi) ).apply_async() return f"Scheduled merge of #{issue_number}" @@ -163,7 +179,7 @@ async def command_merge(ghapi, issue_number, user, *_args): @permissions(member=True) async def command_autobump(ghapi, _issue_number, _user, *args): """Run immediate Autobump on recipes""" - if any('*' in arg for arg in args): + if any("*" in arg for arg in args): return f"Wildcards in autobump not allowed" if len(args) > 5: return f"Please don't schedule more than 5 updates at once" @@ -190,27 +206,29 @@ async def command_schedule(ghapi, issue_number, user, *args): max_updates = int(args[1]) else: max_updates = 5 - params = {'AUTOBUMP_OPTS': f'--max-updates {max_updates}'} + params = {"AUTOBUMP_OPTS": f"--max-updates {max_updates}"} from .config import CIRCLE_TOKEN from ..circleci import AsyncCircleAPI + capi = AsyncCircleAPI(ghapi.session, token=CIRCLE_TOKEN) - res = await capi.trigger_job(project='bioconda-utils', job='autobump', - params=params) + res = await capi.trigger_job( + project="bioconda-utils", job="autobump", params=params + ) return "Scheduled Autobump: " + res else: err = "Unknown command" - return err + (" Options are:\n" - " - `autobump [n=5]`: schedule *n* updates of autobump") + return err + ( + " Options are:\n" " - `autobump [n=5]`: schedule *n* updates of autobump" + ) + @command_routes.register("update") @permissions(member=True, author=True) async def command_update(ghapi, issue_number, user, *args): - """Update PR branch with it's target branch ("base") - """ + """Update PR branch with it's target branch ("base")""" if await ghapi.pr_update_branch(issue_number): msg = "Ok. Branch update triggered." else: msg = "Sorry, I was unable to trigger branch update." await ghapi.create_comment(issue_number, msg) - diff --git a/bioconda_utils/bot/config.py b/bioconda_utils/bot/config.py index 1fb4e8cf42..0040922797 100644 --- a/bioconda_utils/bot/config.py +++ b/bioconda_utils/bot/config.py @@ -6,6 +6,7 @@ import re import sys + def get_secret(name): """Load a secret from file or env @@ -19,13 +20,14 @@ def get_secret(name): try: return os.environ[name] except KeyError: - if os.path.basename(sys.argv[0]) == 'sphinx-build': + if os.path.basename(sys.argv[0]) == "sphinx-build": # We won't have nor need secrets when building docs return None raise ValueError( f"Missing secrets: configure {name} or {name}_FILE to contain or point at secret" ) from None + #: PEM signing key for APP requests APP_KEY = get_secret("APP_KEY") @@ -58,18 +60,18 @@ def get_secret(name): #: Gitter Channels GITTER_CHANNELS = { - 'bioconda/Lobby': 'bioconda/bioconda-recipes', - 'bioconda/bot': 'bioconda/bioconda-recipes' + "bioconda/Lobby": "bioconda/bioconda-recipes", + "bioconda/bot": "bioconda/bioconda-recipes", } -#GITTER_CHANNELS = { +# GITTER_CHANNELS = { # 'bioconda/bot_test': 'epruesse/bioconda-recipes' -#} +# } #: Name of bot BOT_NAME = "BiocondaBot" #: Bot alias regex - this is what it'll react to in comments -BOT_ALIAS_RE = re.compile(r'@bioconda[- ]?bot', re.IGNORECASE) +BOT_ALIAS_RE = re.compile(r"@bioconda[- ]?bot", re.IGNORECASE) #: Email address used in commits. Needs to match the account under #: which the CODE_SIGNING_KEY was registered. @@ -83,5 +85,5 @@ def get_secret(name): #: Assign PRs to project columns by label PROJECT_COLUMN_LABEL_MAP = { - 5706816: set(('please review & merge',)), + 5706816: set(("please review & merge",)), } diff --git a/bioconda_utils/bot/events.py b/bioconda_utils/bot/events.py index 799d846058..97b9f14b5f 100644 --- a/bioconda_utils/bot/events.py +++ b/bioconda_utils/bot/events.py @@ -20,7 +20,7 @@ logger = logging.getLogger(__name__) # pylint: disable=invalid-name event_routes = gidgethub.routing.Router() # pylint: disable=invalid-name -BOT_ALIAS_RE = re.compile(r'@bioconda[- ]?bot', re.IGNORECASE) +BOT_ALIAS_RE = re.compile(r"@bioconda[- ]?bot", re.IGNORECASE) @event_routes.register("issue_comment", action="created") @@ -30,13 +30,13 @@ async def handle_comment_created(event, ghapi, *args, **_kwargs): - dispatches @biocondabot commands - re-iterates commenets from non-members attempting to @mention @bioconda/xxx """ - issue_number = event.get('issue/number', "NA") + issue_number = event.get("issue/number", "NA") comment_author = event.get("comment/user/login", "") comment_body = event.get("comment/body", "") # Ignore self mentions. This is important not only to avoid loops, # but critical because we are repeating what non-members say below. - if BOT_ALIAS_RE.match('@'+comment_author): + if BOT_ALIAS_RE.match("@" + comment_author): return commands = [ @@ -48,23 +48,25 @@ async def handle_comment_created(event, ghapi, *args, **_kwargs): logger.info("Dispatching command from #%s: '%s' %s", issue_number, cmd, args) await command_routes.dispatch(cmd, ghapi, issue_number, comment_author, *args) - if '@bioconda/' in comment_body.lower() and not await ghapi.is_member(comment_author): - if '@bioconda/all' in comment_body.lower(): + if "@bioconda/" in comment_body.lower() and not await ghapi.is_member( + comment_author + ): + if "@bioconda/all" in comment_body.lower(): return # not pinging everyone logger.info("Repeating comment from %s on #%s to allow ping") - quoted_msg = '\n'.join('> ' + line for line in comment_body.splitlines()) + quoted_msg = "\n".join("> " + line for line in comment_body.splitlines()) await ghapi.create_comment( issue_number, f"Repeating comment from @{comment_author} to enable @mention:\n" - + quoted_msg) + + quoted_msg, + ) @event_routes.register("check_suite") async def handle_check_suite(event, ghapi): - """Handle check suite event - """ - action = event.get('action') - if action not in ['requested', 'rerequested']: + """Handle check suite event""" + action = event.get("action") + if action not in ["requested", "rerequested"]: return head_sha = event.get("check_suite/head_sha") tasks.create_check_run.apply_async((head_sha, ghapi)) @@ -74,14 +76,16 @@ async def handle_check_suite(event, ghapi): @event_routes.register("check_run") async def handle_check_run(event, ghapi): """Handle check run event""" - action = event.get('action') + action = event.get("action") app_owner = event.get("check_run/check_suite/app/owner/login", None) head_sha = event.get("check_run/head_sha") event_repo = event.get("repository/id") if action == "completed" and app_owner == "circleci": - pr_numbers = [int(pr["number"]) - for pr in event.get("check_run/check_suite/pull_requests", []) - if pr["base"]["repo"]["id"] == event_repo] + pr_numbers = [ + int(pr["number"]) + for pr in event.get("check_run/check_suite/pull_requests", []) + if pr["base"]["repo"]["id"] == event_repo + ] if not pr_numbers: pr_numbers = await ghapi.get_prs_from_sha(head_sha, only_open=True) for pr_number in pr_numbers: @@ -97,12 +101,12 @@ async def handle_check_run(event, ghapi): tasks.create_check_run.apply_async((head_sha, ghapi)) logger.info("Scheduled create_check_run for %s", head_sha) elif action == "created": - check_run_number = event.get('check_run/id') + check_run_number = event.get("check_run/id") tasks.lint_check.apply_async((check_run_number, head_sha, ghapi)) logger.info("Scheduled lint_check for %s %s", check_run_number, head_sha) elif action == "requested_action": - head_branch = event.get('check_run/check_suite/head_branch') - requested_action = event.get('requested_action/identifier') + head_branch = event.get("check_run/check_suite/head_branch") + requested_action = event.get("requested_action/identifier") if requested_action == "lint_fix": tasks.lint_fix.s(head_branch, head_sha, ghapi).apply_async() logger.info("Scheduled lint_fix for %s %s", head_branch, head_sha) @@ -111,6 +115,7 @@ async def handle_check_run(event, ghapi): else: logger.error("Unknown action %s", action) + @event_routes.register("pull_request") async def handle_pull_request(event, ghapi): """ @@ -124,16 +129,18 @@ async def handle_pull_request(event, ghapi): - ready_for_review - synchronize """ - action = event.get('action') - pr_number = int(event.get('number')) - head_sha = event.get('pull_request/head/sha') + action = event.get("action") + pr_number = int(event.get("number")) + head_sha = event.get("pull_request/head/sha") logger.info("Handling pull_request/%s #%s (%s)", action, pr_number, head_sha) if action == "opened": tasks.create_welcome_post.s(pr_number, ghapi).apply_async() - if action in ('opened', 'reopened', 'synchronize'): - tasks.create_check_run.s(head_sha, ghapi, recreate=False).apply_async(countdown=30) + if action in ("opened", "reopened", "synchronize"): + tasks.create_check_run.s(head_sha, ghapi, recreate=False).apply_async( + countdown=30 + ) logger.info("Scheduled create_check_run(recreate=False) for %s", head_sha) - if action in ('labeled', 'unlabeled', 'closed'): + if action in ("labeled", "unlabeled", "closed"): tasks.update_pr_project_columns.s(pr_number, ghapi).apply_async() diff --git a/bioconda_utils/bot/tasks.py b/bioconda_utils/bot/tasks.py index 0b69d9571a..1c0305eb4a 100644 --- a/bioconda_utils/bot/tasks.py +++ b/bioconda_utils/bot/tasks.py @@ -28,8 +28,12 @@ from .worker import capp from .config import ( - BOT_NAME, BOT_EMAIL, CIRCLE_TOKEN, QUAY_LOGIN, ANACONDA_TOKEN, - PROJECT_COLUMN_LABEL_MAP + BOT_NAME, + BOT_EMAIL, + CIRCLE_TOKEN, + QUAY_LOGIN, + ANACONDA_TOKEN, + PROJECT_COLUMN_LABEL_MAP, ) from .. import utils from .. import autobump @@ -46,8 +50,8 @@ logger = get_task_logger(__name__) # pylint: disable=invalid-name -Image = namedtuple('Image', "url name tag") -Package = namedtuple('Package', "arch fname url repodata_md") +Image = namedtuple("Image", "url name tag") +Package = namedtuple("Package", "arch fname url repodata_md") PACKAGE_RE = re.compile(r"(.*packages)/(osx-64|linux-64|noarch)/(.+\.tar\.bz2)$") IMAGE_RE = re.compile(r".*images/(.+)(?::|%3A)(.+)\.tar\.gz$") @@ -70,6 +74,7 @@ class Checkout: >>> else: >>> for filename in git.list_changed_files(): """ + def __init__(self, ghapi, ref=None, issue_number=None, branch_name="master"): self.ghapi = ghapi self.orig_cwd = None @@ -79,15 +84,20 @@ def __init__(self, ghapi, ref=None, issue_number=None, branch_name="master"): self.issue_number = issue_number async def __aenter__(self): - logger.info("Preparing checkout: pr=%s ref=%s branch=%s repo=%s", - self.issue_number, self.ref, self.branch_name, self.ghapi) + logger.info( + "Preparing checkout: pr=%s ref=%s branch=%s repo=%s", + self.issue_number, + self.ref, + self.branch_name, + self.ghapi, + ) try: if self.issue_number: prs = await self.ghapi.get_prs(number=self.issue_number) - fork_user = prs['head']['user']['login'] - fork_repo = prs['head']['repo']['name'] - branch_name = prs['head']['ref'] + fork_user = prs["head"]["user"]["login"] + fork_repo = prs["head"]["repo"]["name"] + branch_name = prs["head"]["ref"] ref = None elif self.ref: fork_user = None @@ -108,7 +118,7 @@ async def __aenter__(self): home_user=self.ghapi.user, home_repo=self.ghapi.repo, fork_user=fork_user, - fork_repo=fork_repo + fork_repo=fork_repo, ) self.git.set_user(BOT_NAME, BOT_EMAIL) @@ -124,15 +134,15 @@ async def __aenter__(self): else: branch = self.git.create_local_branch(branch_name, ref) if not branch: - logger.error("-- Failed to checkout - no branch? (git=%s)", - self.git) + logger.error("-- Failed to checkout - no branch? (git=%s)", self.git) return None branch.checkout() return self.git except Exception: - logger.exception("-- Failed to checkout - caught exception: (git=%s)", - self.git) + logger.exception( + "-- Failed to checkout - caught exception: (git=%s)", self.git + ) return None async def __aexit__(self, _exc_type, _exc, _tb): @@ -145,11 +155,11 @@ async def __aexit__(self, _exc_type, _exc, _tb): @capp.task(acks_late=True, ignore_result=False) async def get_latest_pr_commit(issue_number: int, ghapi): """Returns last commit""" - commit = {'sha': None} + commit = {"sha": None} async for commit in ghapi.iter_pr_commits(issue_number): pass - logger.info("Latest SHA on #%s is %s", issue_number, commit['sha']) - return commit['sha'] + logger.info("Latest SHA on #%s is %s", issue_number, commit["sha"]) + return commit["sha"] @capp.task(acks_late=True) @@ -167,9 +177,8 @@ async def create_check_run(head_sha: str, ghapi, recreate=True): LINT_CHECK_NAME = "Linting Recipe(s)" if not recreate: for check_run in await ghapi.get_check_runs(head_sha): - if check_run.get('name') == LINT_CHECK_NAME: - logger.warning("Check run for %s exists - not recreating", - head_sha) + if check_run.get("name") == LINT_CHECK_NAME: + logger.warning("Check run for %s exists - not recreating", head_sha) return check_run_number = await ghapi.create_check_run(LINT_CHECK_NAME, head_sha) logger.warning("Created check run %s for %s", check_run_number, head_sha) @@ -189,7 +198,7 @@ async def bump(issue_number: int, ghapi): return recipes = git.get_changed_recipes() for meta_fn in recipes: - recipe = Recipe.from_file('recipes', meta_fn) + recipe = Recipe.from_file("recipes", meta_fn) recipe.reset_buildnumber(recipe.build_number + 1) recipe.save() msg = f"Bump {recipe} buildno to {recipe.build_number}" @@ -215,9 +224,8 @@ async def lint_check(check_run_number: int, ref: str, ghapi): check_run_number, status=CheckRunStatus.completed, conclusion=CheckRunConclusion.neutral, - output_title= - f"Failed to check out " - f"{ghapi.user}/{ghapi.repo}:{ref_label}" + output_title=f"Failed to check out " + f"{ghapi.user}/{ghapi.repo}:{ref_label}", ) return @@ -228,16 +236,15 @@ async def lint_check(check_run_number: int, ref: str, ghapi): status=CheckRunStatus.completed, conclusion=CheckRunConclusion.neutral, output_title="No recipes modified", - output_summary= - "This branch does not modify any recipes! " + output_summary="This branch does not modify any recipes! " "Please make sure this is what you intend. Upon merge, " - "no packages would be built." + "no packages would be built.", ) return # Here we call the actual linter code - config = utils.load_config('config.yml') - linter = lint.Linter(config, 'recipes') # fixme, should be configurable + config = utils.load_config("config.yml") + linter = lint.Linter(config, "recipes") # fixme, should be configurable res = linter.lint(recipes) messages = linter.get_messages() @@ -247,8 +254,10 @@ async def lint_check(check_run_number: int, ref: str, ghapi): summary += " - `{}`\n".format(recipe) summary += "\n" - details = ['Severity | Location | Check (links to docs) | Info ', - '---------|----------|-----------------------|------'] + details = [ + "Severity | Location | Check (links to docs) | Info ", + "---------|----------|-----------------------|------", + ] annotations = [] if not messages: # no errors, success @@ -264,31 +273,35 @@ async def lint_check(check_run_number: int, ref: str, ghapi): title = "Some recipes had problems" summary += "Please fix the issues listed below." - url = 'https://bioconda.github.io/contributor/linting.html' + url = "https://bioconda.github.io/contributor/linting.html" for msg in messages: - annotations.append({ - 'path': msg.fname, - 'start_line': msg.start_line, - 'end_line': msg.end_line, - 'annotation_level': msg.get_level(), - 'title': msg.title, - 'message': msg.body, - }) + annotations.append( + { + "path": msg.fname, + "start_line": msg.start_line, + "end_line": msg.end_line, + "annotation_level": msg.get_level(), + "title": msg.title, + "message": msg.body, + } + ) # 'raw_details' can also be sent as annotation, contents are hidden # and unfold if 'raw details' blue button clicked. details.append( - f'{msg.get_level()}|{msg.fname}:{msg.start_line}|' + f"{msg.get_level()}|{msg.fname}:{msg.start_line}|" f'[{msg.check}]({url}#{str(msg.check).replace("_","-")})|{msg.title}' ) actions = [] if any(msg.canfix for msg in messages): - actions.append({ - 'label': "Fix Issues", - 'description': "Some issues can be fixed automatically", - 'identifier': 'lint_fix' - }) + actions.append( + { + "label": "Fix Issues", + "description": "Some issues can be fixed automatically", + "identifier": "lint_fix", + } + ) await ghapi.modify_check_run( check_run_number, @@ -296,9 +309,10 @@ async def lint_check(check_run_number: int, ref: str, ghapi): conclusion=conclusion, output_title=title, output_summary=summary, - output_text='\n'.join(details) if messages else None, + output_text="\n".join(details) if messages else None, output_annotations=annotations, - actions=actions) + actions=actions, + ) @capp.task(acks_late=True) @@ -318,8 +332,8 @@ async def lint_fix(head_branch: str, head_sha: str, ghapi): if not recipes: logger.error("No recipes? Internal error") return - config = utils.load_config('config.yml') - linter = lint.Linter(config, 'recipes') # fixme, should be configurable + config = utils.load_config("config.yml") + linter = lint.Linter(config, "recipes") # fixme, should be configurable linter.lint(recipes, fix=True) msg = "Fixed Lint Checks" @@ -342,9 +356,9 @@ async def check_circle_artifacts(pr_number: int, ghapi): """ logger.info("Starting check for artifacts on #%s as of %s", pr_number, ghapi) pr = await ghapi.get_prs(number=pr_number) - head_ref = pr['head']['ref'] - head_sha = pr['head']['sha'] - head_user = pr['head']['repo']['owner']['login'] + head_ref = pr["head"]["ref"] + head_sha = pr["head"]["sha"] + head_user = pr["head"]["repo"]["owner"]["login"] # get path for Circle if head_user == ghapi.user: branch = head_ref @@ -365,7 +379,7 @@ async def check_circle_artifacts(pr_number: int, ghapi): # base /fname # repo/arch/fname repo_url, arch, fname = match.groups() - repodata_url = '/'.join((repo_url, arch, 'repodata.json')) + repodata_url = "/".join((repo_url, arch, "repodata.json")) repodata_md = "" if repodata_url in artifact_urls: repos.setdefault(repo_url, set()).add(arch) @@ -382,7 +396,7 @@ async def check_circle_artifacts(pr_number: int, ghapi): msg_head, _, _ = msg.partition("\n") async for comment in await ghapi.iter_comments(pr_number): - if comment['body'].startswith(msg_head): + if comment["body"].startswith(msg_head): await ghapi.update_comment(comment["id"], msg) break else: @@ -398,9 +412,9 @@ async def trigger_circle_rebuild(pr_number: int, ghapi): """ logger.info("Triggering rebuild of #%s", pr_number) pr = await ghapi.get_prs(number=pr_number) - head_ref = pr['head']['ref'] - head_sha = pr['head']['sha'] - head_user = pr['head']['repo']['owner']['login'] + head_ref = pr["head"]["ref"] + head_sha = pr["head"]["sha"] + head_user = pr["head"]["repo"]["owner"]["login"] capi = AsyncCircleAPI(ghapi.session, token=CIRCLE_TOKEN) if head_user == ghapi.user: @@ -427,7 +441,7 @@ async def merge_pr(self, pr_number: int, comment_id: int, ghapi) -> Tuple[bool, comment_id: ID of comment in PR to use for posting progress """ pr = await ghapi.get_prs(number=pr_number) - state, message = await ghapi.check_protections(pr_number, pr['head']['sha']) + state, message = await ghapi.check_protections(pr_number, pr["head"]["sha"]) if state is None: try: raise self.retry(countdown=20, max_retries=15) @@ -435,13 +449,14 @@ async def merge_pr(self, pr_number: int, comment_id: int, ghapi) -> Tuple[bool, return False, "PR cannot be merged at this time. Please try again later" if not state: return state, message - comment = ("Upload & Merge started. Reload page to view progress.\n" - "- [x] Checks OK\n") + comment = ( + "Upload & Merge started. Reload page to view progress.\n" "- [x] Checks OK\n" + ) await ghapi.update_comment(comment_id, comment) - head_ref = pr['head']['ref'] - head_sha = pr['head']['sha'] - head_user = pr['head']['repo']['owner']['login'] + head_ref = pr["head"]["ref"] + head_sha = pr["head"]["sha"] + head_user = pr["head"]["repo"]["owner"]["login"] # get path for Circle if head_user == ghapi.user: branch = head_ref @@ -473,17 +488,19 @@ async def merge_pr(self, pr_number: int, comment_id: int, ghapi) -> Tuple[bool, if not files: return False, "PR did not build any packages." - comment += "- [x] Fetching {} packages and {} images\n".format(len(packages), len(images)) + comment += "- [x] Fetching {} packages and {} images\n".format( + len(packages), len(images) + ) await ghapi.update_comment(comment_id, comment) - logger.info("Downloading %s", ', '.join(f for _, f in files)) + logger.info("Downloading %s", ", ".join(f for _, f in files)) done = False with tempfile.TemporaryDirectory() as tmpdir: # Download files try: fds = [] urls = [] - for url,path in files: + for url, path in files: fpath = os.path.join(tmpdir, path) fdir = os.path.dirname(fpath) if not os.path.exists(fdir): @@ -503,7 +520,7 @@ async def merge_pr(self, pr_number: int, comment_id: int, ghapi) -> Tuple[bool, uploaded = [] for fname, dref in images: fpath = os.path.join(tmpdir, fname) - ndref = "biocontainers/"+dref + ndref = "biocontainers/" + dref for _ in range(5): logger.info("Uploading: %s", ndref) if skopeo_upload(fpath, ndref, creds=QUAY_LOGIN): @@ -540,19 +557,19 @@ async def merge_pr(self, pr_number: int, comment_id: int, ghapi) -> Tuple[bool, lines.append("") # collect authors - pr_author = pr['user']['login'] + pr_author = pr["user"]["login"] coauthors: Set[str] = set() coauthor_logins: Set[str] = set() last_sha: str = None async for commit in ghapi.iter_pr_commits(pr_number): - last_sha = commit['sha'] - author_login = (commit['author'] or {}).get('login') + last_sha = commit["sha"] + author_login = (commit["author"] or {}).get("login") if author_login != pr_author: - name = commit['commit']['author']['name'] - email = commit['commit']['author']['email'] + name = commit["commit"]["author"]["name"] + email = commit["commit"]["author"]["email"] coauthors.add(f"Co-authored-by: {name} <{email}>") if author_login: - coauthor_logins.add("@"+author_login) + coauthor_logins.add("@" + author_login) else: coauthor_logins.add(name) lines.extend(list(coauthors)) @@ -564,20 +581,26 @@ async def merge_pr(self, pr_number: int, comment_id: int, ghapi) -> Tuple[bool, comment += "\n" await ghapi.update_comment(comment_id, comment) - res, msg = await ghapi.merge_pr(pr_number, sha=last_sha, - message="\n".join(lines) if lines else None) + res, msg = await ghapi.merge_pr( + pr_number, sha=last_sha, message="\n".join(lines) if lines else None + ) if not res: return res, msg - if not branch.startswith('pull/'): + if not branch.startswith("pull/"): await ghapi.delete_branch(branch) return res, msg - @capp.task(acks_late=True, ignore_result=True) -async def post_result(result: Tuple[bool, str], pr_number: int, _comment_id: int, - prefix: str, user: str, ghapi) -> None: +async def post_result( + result: Tuple[bool, str], + pr_number: int, + _comment_id: int, + prefix: str, + user: str, + ghapi, +) -> None: logger.error("post result: result=%s, issue=%s", result, pr_number) status = "succeeded" if result[0] else "failed" message = f"@{user}, your request to {prefix} {status}: {result[1]}" @@ -589,7 +612,7 @@ async def post_result(result: Tuple[bool, str], pr_number: int, _comment_id: int async def create_welcome_post(pr_number: int, ghapi): """Post welcome message for first timers""" prq = await ghapi.get_prs(number=pr_number) - pr_author = prq['user']['login'] + pr_author = prq["user"]["login"] if await ghapi.get_pr_count(pr_author) > 1: logger.error("PR %#s is not %s's first", pr_number, pr_author) return @@ -611,7 +634,7 @@ async def run_autobump(package_names, ghapi, *_args): if not git: logger.error("failed to checkout master") return - recipe_source = autobump.RecipeSource('recipes', package_names, []) + recipe_source = autobump.RecipeSource("recipes", package_names, []) scanner = autobump.Scanner(recipe_source) scanner.add(autobump.ExcludeSubrecipe) scanner.add(autobump.GitLoadRecipe, git) @@ -627,17 +650,19 @@ async def update_pr_project_columns(issue_number, ghapi, *_args): """Updates project columns for PR according to labels""" pr = await ghapi.get_prs(number=issue_number) if not pr: - logger.error("Failed to update projects from labels: #%s is not a PR?", - issue_number) + logger.error( + "Failed to update projects from labels: #%s is not a PR?", issue_number + ) return - logger.info("Updating projects from labels for #%s '%s'", - issue_number, pr['title']) - pr_labels = set(label['name'] for label in pr['labels']) - pr_closed = pr['state'] == 'closed' + logger.info("Updating projects from labels for #%s '%s'", issue_number, pr["title"]) + pr_labels = set(label["name"] for label in pr["labels"]) + pr_closed = pr["state"] == "closed" for column_id, col_labels in PROJECT_COLUMN_LABEL_MAP.items(): - have_card = any(card.get('issue_number') == issue_number - for card in await ghapi.list_project_cards(column_id)) + have_card = any( + card.get("issue_number") == issue_number + for card in await ghapi.list_project_cards(column_id) + ) if not pr_closed and pr_labels.intersection(col_labels): if not have_card: await ghapi.create_project_card(column_id, number=issue_number) diff --git a/bioconda_utils/bot/views.py b/bioconda_utils/bot/views.py index cef35e75a0..d1eec722e8 100644 --- a/bioconda_utils/bot/views.py +++ b/bioconda_utils/bot/views.py @@ -6,7 +6,13 @@ from aiohttp import web from aiohttp_session import get_session -from aiohttp_security import check_authorized, forget, permits, remember, authorized_userid +from aiohttp_security import ( + check_authorized, + forget, + permits, + remember, + authorized_userid, +) from aiohttp_jinja2 import template, render_template from .events import event_routes @@ -35,10 +41,12 @@ def add_to_navbar(title): and the name in the navbar. """ + def wrapper(func): route = web_routes[-1] - navigation_bar.append((route.path, route.kwargs['name'], title)) + navigation_bar.append((route.path, route.kwargs["name"], title)) return func + return wrapper @@ -56,11 +64,11 @@ async def check_permission(request, permission, context=None): await check_authorized(request) allowed = await permits(request, permission, context) if not allowed: - request['permission_required'] = permission + request["permission_required"] = permission raise web.HTTPForbidden() -@web_routes.post('/_gh') +@web_routes.post("/_gh") async def github_webhook_dispatch(request): """View for incoming webhooks from Github @@ -86,31 +94,38 @@ async def github_webhook_dispatch(request): return web.Response(status=200) # Log Event - installation = event.get('installation/id') - to_user = event.get('repository/owner/login', None) - to_repo = event.get('repository/name', None) - action = event.get('action', None) - action_msg = '/' + action if action else '' - logger.info("Received GH Event '%s%s' (%s) for %s (%s/%s)", - event.event, action_msg, - event.delivery_id, - installation, to_user, to_repo) + installation = event.get("installation/id") + to_user = event.get("repository/owner/login", None) + to_repo = event.get("repository/name", None) + action = event.get("action", None) + action_msg = "/" + action if action else "" + logger.info( + "Received GH Event '%s%s' (%s) for %s (%s/%s)", + event.event, + action_msg, + event.delivery_id, + installation, + to_user, + to_repo, + ) # Get GithubAPI object for this installation - ghapi = await request.app['ghappapi'].get_github_api( + ghapi = await request.app["ghappapi"].get_github_api( dry_run=False, installation=installation, to_user=to_user, to_repo=to_repo ) # Dispatch the Event try: await event_routes.dispatch(event, ghapi) - logger.info("Event '%s%s' (%s) done", event.event, action_msg, event.delivery_id) + logger.info( + "Event '%s%s' (%s) done", event.event, action_msg, event.delivery_id + ) except Exception: # pylint: disable=broad-except logger.exception("Failed to dispatch %s", event.delivery_id) # Remember the rate limit # FIXME: remove this, we have many tokens in many places, this no longer works sensibly. - request.app['gh_rate_limit'] = ghapi.rate_limit + request.app["gh_rate_limit"] = ghapi.rate_limit return web.Response(status=200) except Exception: # pylint: disable=broad-except @@ -118,7 +133,7 @@ async def github_webhook_dispatch(request): return web.Response(status=500) -@web_routes.post('/hooks/circleci') +@web_routes.post("/hooks/circleci") async def generic_circleci_dispatch(request): """View for incoming webhooks from CircleCI @@ -136,7 +151,7 @@ async def generic_circleci_dispatch(request): return web.Response(status=500) -@web_routes.post('/hooks/{source}') +@web_routes.post("/hooks/{source}") async def generic_webhook_dispatch(request): """View for all other incoming webhooks @@ -145,19 +160,19 @@ async def generic_webhook_dispatch(request): """ try: - source = request.match_info['source'] + source = request.match_info["source"] body = await request.read() logger.error("Got generic webhook for %s", source) logger.error(" Data: %s", body) return web.Response(status=200) - except Exception: # pylint: disable=broad-except + except Exception: # pylint: disable=broad-except logger.exception("Failure in generic webhook dispatch") return web.Response(status=500) @add_to_navbar(title="Home") @web_routes.get("/", name="home") -@template('bot_index.html') +@template("bot_index.html") async def show_index(_request): """View for the Bot's home page. @@ -178,30 +193,26 @@ async def show_status(request): within that time. """ - await check_permission(request, 'bioconda') + await check_permission(request, "bioconda") worker_status = capp.control.inspect(timeout=0.1) if not worker_status: - return { - 'error': 'Could not get worker status' - } + return {"error": "Could not get worker status"} alive = worker_status.ping() if not alive: - return { - 'error': 'No workers found' - } + return {"error": "No workers found"} return { - 'workers': { + "workers": { worker: { - 'active': worker_status.active(worker), - 'reserved': worker_status.reserved(worker), + "active": worker_status.active(worker), + "reserved": worker_status.reserved(worker), } for worker in sorted(alive.keys()) } } -@web_routes.get('/logout', name="logout") +@web_routes.get("/logout", name="logout") async def logout(request): """View for logging out user @@ -210,13 +221,13 @@ async def logout(request): """ await check_authorized(request) - nexturl = request.query.get('next', '/') + nexturl = request.query.get("next", "/") response = web.HTTPFound(nexturl) await forget(request, response) return response -@web_routes.get('/login') +@web_routes.get("/login") async def login(request): """View for login page @@ -224,10 +235,10 @@ async def login(request): methods supported. """ - return web.HTTPFound('/auth/github') + return web.HTTPFound("/auth/github") -@web_routes.get('/auth/github', name="login") +@web_routes.get("/auth/github", name="login") async def auth_github(request): """View for signing in with Github @@ -237,15 +248,15 @@ async def auth_github(request): necessary. """ - if 'error' in request.query: + if "error" in request.query: logger.error(request.query) web.HTTPUnauthorized(body="Encountered an error. ") session = await get_session(request) - nexturl = request.query.get('next') or '/' + nexturl = request.query.get("next") or "/" baseurl = BOT_BASEURL + "/auth/github?next=" + nexturl try: - ghappapi = request.app['ghappapi'] + ghappapi = request.app["ghappapi"] ghapi = await ghappapi.oauth_github_user(baseurl, session, request.query) if ghapi.username: await remember(request, web.HTTPFound(nexturl), ghapi.token) @@ -258,13 +269,13 @@ async def auth_github(request): @add_to_navbar(title="Commands") -@web_routes.get('/commands', name="commands") -@template('bot_commands.html') +@web_routes.get("/commands", name="commands") +@template("bot_commands.html") async def list_commands(request): """Self documents available commands""" return { - 'commands': [ - {'name': name, 'description': desc} + "commands": [ + {"name": name, "description": desc} for name, (func, desc) in command_routes.mapping.items() ] } diff --git a/bioconda_utils/bot/web.py b/bioconda_utils/bot/web.py index 878328fd03..e9bf434ac1 100644 --- a/bioconda_utils/bot/web.py +++ b/bioconda_utils/bot/web.py @@ -14,11 +14,21 @@ import markupsafe from aiohttp_session import setup as setup_session from aiohttp_session.cookie_storage import EncryptedCookieStorage -from aiohttp_security import (authorized_userid, - setup as setup_security, - SessionIdentityPolicy, AbstractAuthorizationPolicy) -from .config import (BOT_NAME, APP_KEY, APP_ID, GITTER_TOKEN, GITTER_CHANNELS, - APP_CLIENT_ID, APP_CLIENT_SECRET) +from aiohttp_security import ( + authorized_userid, + setup as setup_security, + SessionIdentityPolicy, + AbstractAuthorizationPolicy, +) +from .config import ( + BOT_NAME, + APP_KEY, + APP_ID, + GITTER_TOKEN, + GITTER_CHANNELS, + APP_CLIENT_ID, + APP_CLIENT_SECRET, +) from .views import web_routes, navigation_bar from .chat import GitterListener from .. import utils @@ -30,11 +40,12 @@ #: Override this to get more verbose logging of web app (and, if launched #: with web frontend, the worker). -LOGLEVEL = 'INFO' +LOGLEVEL = "INFO" class AuthorizationPolicy(AbstractAuthorizationPolicy): """Authorization policy for web interface""" + def __init__(self, app): self.app = app @@ -46,7 +57,7 @@ async def authorized_userid(self, identity: str) -> AiohttpGitHubHandler: Returns: Logged in Github API client. """ - return await self.app['ghappapi'].get_github_user_api(identity) + return await self.app["ghappapi"].get_github_user_api(identity) async def permits(self, identity: str, permission: str, context=None) -> bool: """Check user permissions. @@ -59,7 +70,7 @@ async def permits(self, identity: str, permission: str, context=None) -> bool: if identity is None: return False - org, _, team = permission.partition('/') + org, _, team = permission.partition("/") # Fail if no permissions requested if not org: @@ -67,7 +78,7 @@ async def permits(self, identity: str, permission: str, context=None) -> bool: return False # Fail if not logged in - userapi = await self.app['ghappapi'].get_github_user_api(identity) + userapi = await self.app["ghappapi"].get_github_user_api(identity) if not userapi: return False @@ -100,25 +111,28 @@ async def jinja_defaults(request): try: title = next(item for item in navigation_bar if item[1] == active_page)[2] except StopIteration: - title = 'Unknown' + title = "Unknown" ghapi = await authorized_userid(request) return { - 'user': ghapi, - 'version': VERSION, - 'navigation_bar': navigation_bar, - 'active_page': active_page, - 'title': title, - 'request': request, + "user": ghapi, + "version": VERSION, + "navigation_bar": navigation_bar, + "active_page": active_page, + "title": title, + "request": request, } -md2html = markdown.Markdown(extensions=[ - 'markdown.extensions.fenced_code', - 'markdown.extensions.tables', - 'markdown.extensions.admonition', - 'markdown.extensions.codehilite', - 'markdown.extensions.sane_lists', -]) +md2html = markdown.Markdown( + extensions=[ + "markdown.extensions.fenced_code", + "markdown.extensions.tables", + "markdown.extensions.admonition", + "markdown.extensions.codehilite", + "markdown.extensions.sane_lists", + ] +) + def jinja2_filter_markdown(text): return markupsafe.Markup(md2html.reset().convert(text)) @@ -132,10 +146,11 @@ async def handle_errors(request, handler): if exc.status in (302,): raise try: - return aiohttp_jinja2.render_template('bot_40x.html', request, {'exc':exc}) + return aiohttp_jinja2.render_template("bot_40x.html", request, {"exc": exc}) except KeyError as XYZ: raise exc + async def start(): """Initialize App @@ -146,11 +161,11 @@ async def start(): --worker-class aiohttp.worker.GunicornWebWorker \ --reload """ - utils.setup_logger('bioconda_utils', LOGLEVEL, prefix="") + utils.setup_logger("bioconda_utils", LOGLEVEL, prefix="") logger.info("Starting bot (version=%s)", VERSION) app = aiohttp.web.Application() - app['name'] = BOT_NAME + app["name"] = BOT_NAME # Set up session storage fernet_key = fernet.Fernet.generate_key() @@ -162,39 +177,48 @@ async def start(): setup_security(app, SessionIdentityPolicy(), AuthorizationPolicy(app)) # Set up jinja2 rendering - loader = jinja2.PackageLoader('bioconda_utils', 'templates') - aiohttp_jinja2.setup(app, loader=loader, - context_processors=[jinja_defaults], - filters={'markdown': jinja2_filter_markdown}) + loader = jinja2.PackageLoader("bioconda_utils", "templates") + aiohttp_jinja2.setup( + app, + loader=loader, + context_processors=[jinja_defaults], + filters={"markdown": jinja2_filter_markdown}, + ) # Set up error handlers app.middlewares.append(handle_errors) # Prepare persistent client session - app['client_session'] = aiohttp.ClientSession() + app["client_session"] = aiohttp.ClientSession() # Create Github client - app['ghappapi'] = GitHubAppHandler(app['client_session'], - BOT_NAME, APP_KEY, APP_ID, - APP_CLIENT_ID, APP_CLIENT_SECRET) + app["ghappapi"] = GitHubAppHandler( + app["client_session"], + BOT_NAME, + APP_KEY, + APP_ID, + APP_CLIENT_ID, + APP_CLIENT_SECRET, + ) # Create Gitter Client (background process) - app['gitter_listener'] = GitterListener( - app, GITTER_TOKEN, GITTER_CHANNELS, app['client_session'], - app['ghappapi']) + app["gitter_listener"] = GitterListener( + app, GITTER_TOKEN, GITTER_CHANNELS, app["client_session"], app["ghappapi"] + ) # Add routes collected above app.add_routes(web_routes) # Set up static files utils_path = os.path.dirname(os.path.dirname(__file__)) - app.router.add_static("/css", os.path.join(utils_path, 'templates/css')) + app.router.add_static("/css", os.path.join(utils_path, "templates/css")) # Close session - this needs to be at the end of the # on shutdown pieces so the client session remains available # until everything is done. async def close_session(app): - await app['client_session'].close() + await app["client_session"].close() + app.on_shutdown.append(close_session) return app @@ -208,19 +232,25 @@ async def start_with_celery(): """ app = await start() - proc = subprocess.Popen([ - 'celery', - '-A', 'bioconda_utils.bot.worker', - 'worker', - '-l', LOGLEVEL, - '--without-heartbeat', - '-c', '1', - ]) - app['celery_worker'] = proc + proc = subprocess.Popen( + [ + "celery", + "-A", + "bioconda_utils.bot.worker", + "worker", + "-l", + LOGLEVEL, + "--without-heartbeat", + "-c", + "1", + ] + ) + app["celery_worker"] = proc + async def collect_worker(app): # We don't use celery.broadcast('shutdown') as that seems to trigger # an immediate reload. Instead, just send a sigterm. - proc = app['celery_worker'] + proc = app["celery_worker"] logger.info("Terminating celery worker: sending sigterm") proc.terminate() wait = 10 @@ -234,7 +264,7 @@ async def collect_worker(app): logger.info("Terminating celery worker: failed. Sending sigkill") proc.kill() logger.info("Terminating celery worker: collecting process") - app['celery_worker'].wait() + app["celery_worker"].wait() logger.info("Terminating celery worker: done") app.on_shutdown.append(collect_worker) diff --git a/bioconda_utils/bot/worker.py b/bioconda_utils/bot/worker.py index 76ca0b7145..5f3133f1b2 100644 --- a/bioconda_utils/bot/worker.py +++ b/bioconda_utils/bot/worker.py @@ -23,8 +23,13 @@ from ..githandler import install_gpg_key from ..utils import RepoData, setup_logger from .config import ( - APP_ID, APP_KEY, CODE_SIGNING_KEY, BOT_NAME, REPODATA_TIMEOUT, - APP_CLIENT_ID, APP_CLIENT_SECRET + APP_ID, + APP_KEY, + CODE_SIGNING_KEY, + BOT_NAME, + REPODATA_TIMEOUT, + APP_CLIENT_ID, + APP_CLIENT_SECRET, ) logger = logging.getLogger(__name__) # pylint: disable=invalid-name @@ -59,6 +64,7 @@ class detects if the method provided is a coroutine and runs it is so that spawned tasks can survive a shutdown of the app. """ + #: Our tasks should be re-run if they don't finish acks_late = True @@ -79,6 +85,7 @@ def bind(self, app=None): executes the original method inside the asyncio loop. """ if asyncio.iscoroutinefunction(self.run): # only for async funcs + @wraps(self.run) def sync_run(*args, **kwargs): largs = list(args) # need list so that pre-run can modify @@ -98,10 +105,14 @@ async def async_init(self): This happens during binding -> on load. """ if not self.ghappapi: - self.ghappapi = GitHubAppHandler(aiohttp.ClientSession(), BOT_NAME, - APP_KEY, APP_ID, - APP_CLIENT_ID, APP_CLIENT_SECRET) - + self.ghappapi = GitHubAppHandler( + aiohttp.ClientSession(), + BOT_NAME, + APP_KEY, + APP_ID, + APP_CLIENT_ID, + APP_CLIENT_SECRET, + ) async def async_pre_run(self, args, _kwargs): """Per-call async initialization @@ -113,8 +124,8 @@ async def async_pre_run(self, args, _kwargs): for num, arg in enumerate(args): if isinstance(arg, GitHubHandler): args[num] = await self.ghappapi.get_github_api( - False, arg.user, arg.repo, - arg.installation) + False, arg.user, arg.repo, arg.installation + ) @abc.abstractmethod def run(self, *_args, **_kwargs): @@ -144,31 +155,36 @@ def custom_loads(string): the type, passing the result dict from obj.for_json() to __init__(). """ + def decode(obj): if isinstance(obj, dict): try: - typ = obj.pop('__type__') - mod = import_module(obj.pop('__module__')) + typ = obj.pop("__type__") + mod = import_module(obj.pop("__module__")) klass = getattr(mod, typ) return klass(**obj) except KeyError: pass return obj + return simplejson.loads(string, object_hook=decode) + # Register a custom serializer. We do this so we can conveniently # transfer objects without resorting to pickling. -serialization.register('custom_json', - custom_dumps, custom_loads, - content_type='application/x-bioconda-json', - content_encoding='utf8') +serialization.register( + "custom_json", + custom_dumps, + custom_loads, + content_type="application/x-bioconda-json", + content_encoding="utf8", +) # Instantiate Celery app, setting our AsyncTask as default # task class and loading the tasks from tasks.py capp = Celery( # pylint: disable=invalid-name - task_cls=AsyncTask, - include=['bioconda_utils.bot.tasks'] + task_cls=AsyncTask, include=["bioconda_utils.bot.tasks"] ) @@ -176,25 +192,22 @@ def decode(obj): # Settings are suggestions from CloudAMPQ capp.conf.update( # Set the URL to the AMQP broker using environment variable - broker_url=os.environ.get('CLOUDAMQP_URL'), - + broker_url=os.environ.get("CLOUDAMQP_URL"), # Limit the number of connections to the pool. This should # be 2 when running on Heroku to avoid running out of free # connections on CloudAMPQ. # # broker_pool_limit=2, # need two so we can inspect - broker_heartbeat=None, broker_connection_timeout=30, - # We don't feed back our tasks results - result_backend='rpc://', + result_backend="rpc://", event_queue_expires=60, worker_prefetch_multiplier=1, worker_concurrency=1, - task_serializer='custom_json', - accept_content=['custom_json', 'json'] - #task_acks_late=true + task_serializer="custom_json", + accept_content=["custom_json", "json"] + # task_acks_late=true ) diff --git a/bioconda_utils/build.py b/bioconda_utils/build.py index e09cf60db8..177fd6e0e4 100644 --- a/bioconda_utils/build.py +++ b/bioconda_utils/build.py @@ -41,16 +41,21 @@ def conda_build_purge() -> None: if free_mb < 300: logger.info("CLEANING UP PACKAGE CACHE (free space: %iMB).", free_mb) utils.run(["conda", "clean", "--all"], mask=False) - logger.info("CLEANED UP PACKAGE CACHE (free space: %iMB).", - utils.get_free_space()) - - -def build(recipe: str, pkg_paths: List[str] = None, - testonly: bool = False, mulled_test: bool = True, - channels: List[str] = None, - docker_builder: docker_utils.RecipeBuilder = None, - raise_error: bool = False, - linter=None) -> BuildResult: + logger.info( + "CLEANED UP PACKAGE CACHE (free space: %iMB).", utils.get_free_space() + ) + + +def build( + recipe: str, + pkg_paths: List[str] = None, + testonly: bool = False, + mulled_test: bool = True, + channels: List[str] = None, + docker_builder: docker_utils.RecipeBuilder = None, + raise_error: bool = False, + linter=None, +) -> BuildResult: """ Build a single recipe for a single env @@ -69,12 +74,15 @@ def build(recipe: str, pkg_paths: List[str] = None, linter: Linter to use for checking recipes """ if linter: - logger.info('Linting recipe %s', recipe) + logger.info("Linting recipe %s", recipe) linter.clear_messages() if linter.lint([recipe]): - logger.error('\n\nThe recipe %s failed linting. See ' - 'https://bioconda.github.io/contributor/linting.html for details:\n\n%s\n', - recipe, linter.get_report()) + logger.error( + "\n\nThe recipe %s failed linting. See " + "https://bioconda.github.io/contributor/linting.html for details:\n\n%s\n", + recipe, + linter.get_report(), + ) return BuildResult(False, None) logger.info("Lint checks passed") @@ -87,43 +95,46 @@ def build(recipe: str, pkg_paths: List[str] = None, logger.info("BUILD START %s", recipe) - args = ['--override-channels'] + args = ["--override-channels"] if testonly: args += ["--test"] else: args += ["--no-anaconda-upload"] - for channel in channels or ['local']: - args += ['-c', channel] + for channel in channels or ["local"]: + args += ["-c", channel] - logger.debug('Build and Channel Args: %s', args) + logger.debug("Build and Channel Args: %s", args) # Even though there may be variants of the recipe that will be built, we # will only be checking attributes that are independent of variants (pkg # name, version, noarch, whether or not an extended container was used) meta = utils.load_first_metadata(recipe, finalize=False) - is_noarch = bool(meta.get_value('build/noarch', default=False)) - use_base_image = meta.get_value('extra/container', {}).get('extended-base', False) + is_noarch = bool(meta.get_value("build/noarch", default=False)) + use_base_image = meta.get_value("extra/container", {}).get("extended-base", False) if use_base_image: - base_image = 'quay.io/bioconda/base-glibc-debian-bash:2.1.0' + base_image = "quay.io/bioconda/base-glibc-debian-bash:2.1.0" else: - base_image = 'quay.io/bioconda/base-glibc-busybox-bash:2.1.0' + base_image = "quay.io/bioconda/base-glibc-busybox-bash:2.1.0" try: if docker_builder is not None: - docker_builder.build_recipe(recipe_dir=os.path.abspath(recipe), - build_args=' '.join(args), - env=whitelisted_env, - noarch=is_noarch) + docker_builder.build_recipe( + recipe_dir=os.path.abspath(recipe), + build_args=" ".join(args), + env=whitelisted_env, + noarch=is_noarch, + ) # Use presence of expected packages to check for success for pkg_path in pkg_paths: if not os.path.exists(pkg_path): logger.error( - "BUILD FAILED: the built package %s " - "cannot be found", pkg_path) + "BUILD FAILED: the built package %s " "cannot be found", + pkg_path, + ) return BuildResult(False, None) else: - conda_build_cmd = [utils.bin_for('conda'), 'mambabuild'] + conda_build_cmd = [utils.bin_for("conda"), "mambabuild"] # - Temporarily reset os.environ to avoid leaking env vars # - Also pass filtered env to run() # - Point conda-build to meta.yaml, to avoid building subdirs @@ -131,27 +142,28 @@ def build(recipe: str, pkg_paths: List[str] = None, cmd = conda_build_cmd + args for config_file in utils.get_conda_build_config_files(): cmd += [config_file.arg, config_file.path] - cmd += [os.path.join(recipe, 'meta.yaml')] + cmd += [os.path.join(recipe, "meta.yaml")] with utils.Progress(): utils.run(cmd, env=os.environ, mask=False) - logger.info('BUILD SUCCESS %s', - ' '.join(os.path.basename(p) for p in pkg_paths)) + logger.info( + "BUILD SUCCESS %s", " ".join(os.path.basename(p) for p in pkg_paths) + ) except (docker_utils.DockerCalledProcessError, sp.CalledProcessError) as exc: - logger.error('BUILD FAILED %s', recipe) + logger.error("BUILD FAILED %s", recipe) if raise_error: raise exc return BuildResult(False, None) if mulled_test: - logger.info('TEST START via mulled-build %s', recipe) + logger.info("TEST START via mulled-build %s", recipe) mulled_images = [] for pkg_path in pkg_paths: try: pkg_test.test_package(pkg_path, base_image=base_image) except sp.CalledProcessError: - logger.error('TEST FAILED: %s', recipe) + logger.error("TEST FAILED: %s", recipe) return BuildResult(False, None) logger.info("TEST SUCCESS %s", recipe) mulled_images.append(pkg_test.get_image_name(pkg_path)) @@ -163,15 +175,18 @@ def build(recipe: str, pkg_paths: List[str] = None, def remove_cycles(dag, name2recipes, failed, skip_dependent): nodes_in_cycles = set() for cycle in list(nx.simple_cycles(dag)): - logger.error('BUILD ERROR: dependency cycle found: %s', cycle) + logger.error("BUILD ERROR: dependency cycle found: %s", cycle) nodes_in_cycles.update(cycle) for name in sorted(nodes_in_cycles): cycle_fail_recipes = sorted(name2recipes[name]) - logger.error('BUILD ERROR: cannot build recipes for %s since ' - 'it cyclically depends on other packages in the ' - 'current build job. Failed recipes: %s', - name, cycle_fail_recipes) + logger.error( + "BUILD ERROR: cannot build recipes for %s since " + "it cyclically depends on other packages in the " + "current build job. Failed recipes: %s", + name, + cycle_fail_recipes, + ) failed.extend(cycle_fail_recipes) for node in nx.algorithms.descendants(dag, name): if node not in nodes_in_cycles: @@ -183,7 +198,8 @@ def get_subdags(dag, n_workers, worker_offset): if n_workers > 1 and worker_offset >= n_workers: raise ValueError( "n-workers is less than the worker-offset given! " - "Either decrease --n-workers or decrease --worker-offset!") + "Either decrease --n-workers or decrease --worker-offset!" + ) # Get connected subdags and sort by nodes if n_workers > 1: @@ -206,26 +222,36 @@ def get_subdags(dag, n_workers, worker_offset): for child in children: found.add(child) subdags = dag.subgraph(list(nodes)) - logger.info("Building and testing sub-DAGs %i in each group of %i, which is %i packages", worker_offset, n_workers, len(subdags.nodes())) + logger.info( + "Building and testing sub-DAGs %i in each group of %i, which is %i packages", + worker_offset, + n_workers, + len(subdags.nodes()), + ) else: subdags = dag return subdags -def build_recipes(recipe_folder: str, config_path: str, recipes: List[str], - mulled_test: bool = True, testonly: bool = False, - force: bool = False, - docker_builder: docker_utils.RecipeBuilder = None, - label: str = None, - anaconda_upload: bool = False, - mulled_upload_target=None, - check_channels: List[str] = None, - do_lint: bool = None, - lint_exclude: List[str] = None, - n_workers: int = 1, - worker_offset: int = 0, - keep_old_work: bool = False): +def build_recipes( + recipe_folder: str, + config_path: str, + recipes: List[str], + mulled_test: bool = True, + testonly: bool = False, + force: bool = False, + docker_builder: docker_utils.RecipeBuilder = None, + label: str = None, + anaconda_upload: bool = False, + mulled_upload_target=None, + check_channels: List[str] = None, + do_lint: bool = None, + lint_exclude: List[str] = None, + n_workers: int = 1, + worker_offset: int = 0, + keep_old_work: bool = False, +): """ Build one or many bioconda packages. @@ -262,14 +288,14 @@ def build_recipes(recipe_folder: str, config_path: str, recipes: List[str], # get channels to check if check_channels is None: - if config['channels']: - check_channels = [c for c in config['channels'] if c != "defaults"] + if config["channels"]: + check_channels = [c for c in config["channels"] if c != "defaults"] else: check_channels = [] # setup linting if do_lint: - always_exclude = ('build_number_needs_bump',) + always_exclude = ("build_number_needs_bump",) if not lint_exclude: lint_exclude = always_exclude else: @@ -291,17 +317,20 @@ def build_recipes(recipe_folder: str, config_path: str, recipes: List[str], if not subdag: logger.info("Nothing to be done.") return True - logger.info("%i recipes to build and test: \n%s", len(subdag), "\n".join(subdag.nodes())) + logger.info( + "%i recipes to build and test: \n%s", len(subdag), "\n".join(subdag.nodes()) + ) recipe2name = {} for name, recipe_list in name2recipes.items(): for recipe in recipe_list: recipe2name[recipe] = name - recipes = [(recipe, recipe2name[recipe]) - for package in nx.topological_sort(subdag) - for recipe in name2recipes[package]] - + recipes = [ + (recipe, recipe2name[recipe]) + for package in nx.topological_sort(subdag) + for recipe in name2recipes[package] + ] built_recipes = [] skipped_recipes = [] @@ -309,26 +338,35 @@ def build_recipes(recipe_folder: str, config_path: str, recipes: List[str], for recipe, name in recipes: if name in skip_dependent: - logger.info('BUILD SKIP: skipping %s because it depends on %s ' - 'which had a failed build.', - recipe, skip_dependent[name]) + logger.info( + "BUILD SKIP: skipping %s because it depends on %s " + "which had a failed build.", + recipe, + skip_dependent[name], + ) skipped_recipes.append(recipe) continue - logger.info('Determining expected packages for %s', recipe) + logger.info("Determining expected packages for %s", recipe) try: pkg_paths = utils.get_package_paths(recipe, check_channels, force=force) except utils.DivergentBuildsError as exc: - logger.error('BUILD ERROR: packages with divergent build strings in repository ' - 'for recipe %s. A build number bump is likely needed: %s', - recipe, exc) + logger.error( + "BUILD ERROR: packages with divergent build strings in repository " + "for recipe %s. A build number bump is likely needed: %s", + recipe, + exc, + ) failed.append(recipe) for pkg in nx.algorithms.descendants(subdag, name): skip_dependent[pkg].append(recipe) continue except (UnsatisfiableError, DependencyNeedsBuildingError) as exc: - logger.error('BUILD ERROR: could not determine dependencies for recipe %s: %s', - recipe, exc) + logger.error( + "BUILD ERROR: could not determine dependencies for recipe %s: %s", + recipe, + exc, + ) failed.append(recipe) for pkg in nx.algorithms.descendants(subdag, name): skip_dependent[pkg].append(recipe) @@ -337,13 +375,15 @@ def build_recipes(recipe_folder: str, config_path: str, recipes: List[str], logger.info("Nothing to be done for recipe %s", recipe) continue - res = build(recipe=recipe, - pkg_paths=pkg_paths, - testonly=testonly, - mulled_test=mulled_test, - channels=config['channels'], - docker_builder=docker_builder, - linter=linter) + res = build( + recipe=recipe, + pkg_paths=pkg_paths, + testonly=testonly, + mulled_test=mulled_test, + channels=config["channels"], + docker_builder=docker_builder, + linter=linter, + ) if not res.success: failed.append(recipe) @@ -366,24 +406,38 @@ def build_recipes(recipe_folder: str, config_path: str, recipes: List[str], conda_build_purge() if failed or failed_uploads: - logger.error('BUILD SUMMARY: of %s recipes, ' - '%s failed and %s were skipped. ' - 'Details of recipes and environments follow.', - len(recipes), len(failed), len(skipped_recipes)) + logger.error( + "BUILD SUMMARY: of %s recipes, " + "%s failed and %s were skipped. " + "Details of recipes and environments follow.", + len(recipes), + len(failed), + len(skipped_recipes), + ) if built_recipes: - logger.error('BUILD SUMMARY: while the entire build failed, ' - 'the following recipes were built successfully:\n%s', - '\n'.join(built_recipes)) + logger.error( + "BUILD SUMMARY: while the entire build failed, " + "the following recipes were built successfully:\n%s", + "\n".join(built_recipes), + ) for recipe in failed: - logger.error('BUILD SUMMARY: FAILED recipe %s', recipe) + logger.error("BUILD SUMMARY: FAILED recipe %s", recipe) for name, dep in skip_dependent.items(): - logger.error('BUILD SUMMARY: SKIPPED recipe %s ' - 'due to failed dependencies %s', name, dep) + logger.error( + "BUILD SUMMARY: SKIPPED recipe %s " "due to failed dependencies %s", + name, + dep, + ) if failed_uploads: - logger.error('UPLOAD SUMMARY: the following packages failed to upload:\n%s', - '\n'.join(failed_uploads)) + logger.error( + "UPLOAD SUMMARY: the following packages failed to upload:\n%s", + "\n".join(failed_uploads), + ) return False - logger.info("BUILD SUMMARY: successfully built %s of %s recipes", - len(built_recipes), len(recipes)) + logger.info( + "BUILD SUMMARY: successfully built %s of %s recipes", + len(built_recipes), + len(recipes), + ) return True diff --git a/bioconda_utils/circleci.py b/bioconda_utils/circleci.py index 0a77ff3577..767bc494d8 100644 --- a/bioconda_utils/circleci.py +++ b/bioconda_utils/circleci.py @@ -15,20 +15,23 @@ logger = logging.getLogger(__name__) # pylint: disable=invalid-name + class CircleAPI(abc.ABC): - """CircleCI API - """ + """CircleCI API""" + CIRCLE_API = "https://circleci.com/api/v1.1" LIST_ARTIFACTS = "/project/{vcs_type}/{username}/{project}/{build_num}/artifacts" TRIGGER_REBUILD = "/project/{vcs_type}/{username}/{project}/build" BUILDS = "/project/{vcs_type}/{username}/{project}/tree/{path}" - def __init__(self, - token: Optional[str] = None, - vcs_type: str = 'github', - username: str = 'bioconda', - project: str = 'bioconda-recipes') -> None: + def __init__( + self, + token: Optional[str] = None, + vcs_type: str = "github", + username: str = "bioconda", + project: str = "bioconda-recipes", + ) -> None: self.token = token self.vcs_type = vcs_type self.username = username @@ -39,35 +42,40 @@ def __init__(self, def var_data(self): """Defaults for this API instance""" return { - 'vcs_type': self.vcs_type, - 'username': self.username, - 'project': self.project, - 'circle-token': self.token, + "vcs_type": self.vcs_type, + "username": self.username, + "project": self.project, + "circle-token": self.token, } @abc.abstractmethod - async def _request(self, method: str, url: str, - headers: Mapping[str, str], - body: bytes = b'') -> Tuple[int, Mapping[str, str], bytes]: + async def _request( + self, method: str, url: str, headers: Mapping[str, str], body: bytes = b"" + ) -> Tuple[int, Mapping[str, str], bytes]: """Execute HTTP request (overriden by IO providing subclass)""" - async def _make_request(self, method: str, url: str, var_dict: Mapping[str, str], - data: Any = None, - accept: str = "application/json") -> Any: + async def _make_request( + self, + method: str, + url: str, + var_dict: Mapping[str, str], + data: Any = None, + accept: str = "application/json", + ) -> Any: """Make HTTP request""" - url = ''.join((self.CIRCLE_API, url, "{?circle-token}")) + url = "".join((self.CIRCLE_API, url, "{?circle-token}")) url = uritemplate.expand(url, var_dict=var_dict) headers = {} - headers['accept'] = accept - charset = 'utf-8' + headers["accept"] = accept + charset = "utf-8" - body = b'' + body = b"" if isinstance(data, str): body = data.encode(charset) elif isinstance(data, Mapping): body = json.dumps(data).encode(charset) - headers['content-type'] = "application/json; charset=" + charset - headers['content-length'] = str(len(body)) + headers["content-type"] = "application/json; charset=" + charset + headers["content-length"] = str(len(body)) status, res_headers, response = await self._request(method, url, headers, body) if self.debug_once: @@ -86,9 +94,11 @@ async def _make_request(self, method: str, url: str, var_dict: Mapping[str, str] try: return json.loads(response_text) except json.decoder.JSONDecodeError as exc: - logger.error("Call to '%s' yielded text '%s' - not JSON", - url.replace(self.token, "******"), - response_text.replace(self.token, "******")) + logger.error( + "Call to '%s' yielded text '%s' - not JSON", + url.replace(self.token, "******"), + response_text.replace(self.token, "******"), + ) return response_text async def list_artifacts(self, build_number: int) -> List[Mapping[str, Any]]: @@ -98,11 +108,12 @@ async def list_artifacts(self, build_number: int) -> List[Mapping[str, Any]]: List of artifacts described by ``path`` and ``url`` """ var_data = self.var_data - var_data['build_num'] = build_number - return await self._make_request('GET', self.LIST_ARTIFACTS, var_data) + var_data["build_num"] = build_number + return await self._make_request("GET", self.LIST_ARTIFACTS, var_data) - async def list_recent_builds(self, path: str, sha: str = None, - skip_rebuilt: bool = True) -> List[Mapping[str, Any]]: + async def list_recent_builds( + self, path: str, sha: str = None, skip_rebuilt: bool = True + ) -> List[Mapping[str, Any]]: """List recent builds for **path** (branch or pr) Note: skip rebuild seems to only apply to jobs, not workflow reruns. @@ -117,21 +128,25 @@ async def list_recent_builds(self, path: str, sha: str = None, ``vc_revision`, ``build_time_millis`` """ var_data = self.var_data - var_data['path'] = path - res = await self._make_request('GET', self.BUILDS, var_data) + var_data["path"] = path + res = await self._make_request("GET", self.BUILDS, var_data) if sha is not None: res = [build for build in res if build["vcs_revision"] == sha] if skip_rebuilt: # try using 'retry_of` to remove builds - rebuilt = set(build['retry_of'] for build in res if 'retry_of' in build) + rebuilt = set(build["retry_of"] for build in res if "retry_of" in build) res = [build for build in res if build["build_num"] not in rebuilt] # now just pick the newest of each workflow_name/job_name new_res = [] job_types = set() - for build in sorted(res, key=lambda build: build['build_num'], reverse=True): - job_type = (build['workflows']['workflow_name'], - build['workflows']['job_name']) + for build in sorted( + res, key=lambda build: build["build_num"], reverse=True + ): + job_type = ( + build["workflows"]["workflow_name"], + build["workflows"]["job_name"], + ) if job_type in job_types: continue job_types.add(job_type) @@ -146,11 +161,10 @@ async def trigger_rebuild(self, branch: str, sha: str): branch: Must be ``pull/123`` if on fork, otherwise name of branch sha: The SHA to rebuild """ - data = { - 'revision': sha, - 'branch': branch - } - return await self._make_request('POST', self.TRIGGER_REBUILD, self.var_data, data=data) + data = {"revision": sha, "branch": branch} + return await self._make_request( + "POST", self.TRIGGER_REBUILD, self.var_data, data=data + ) async def trigger_job(self, branch="master", project=None, job=None, params=None): """Trigger specific job @@ -162,21 +176,21 @@ async def trigger_job(self, branch="master", project=None, job=None, params=None params: Optional dict of parameters (envvars) to override """ var_data = self.var_data - var_data['path'] = branch + var_data["path"] = branch if project: - var_data['project'] = project + var_data["project"] = project - data = { - 'build_parameters': params or {} - } + data = {"build_parameters": params or {}} if job: - data['build_parameters']['CIRCLE_JOB'] = job + data["build_parameters"]["CIRCLE_JOB"] = job - res = await self._make_request('POST', self.BUILDS, var_data, data=data) - return res.get('build_url') + res = await self._make_request("POST", self.BUILDS, var_data, data=data) + return res.get("build_url") - async def get_artifacts(self, path: str, head_sha: str) -> List[Tuple[str, str, int]]: + async def get_artifacts( + self, path: str, head_sha: str + ) -> List[Tuple[str, str, int]]: """Get artifacts for specific branch and head_sha For each artifact built for this sha, get the latest URL. Multiple builds @@ -188,12 +202,14 @@ async def get_artifacts(self, path: str, head_sha: str) -> List[Tuple[str, str, Returns: Mapping of relative path to full URL """ - build_numbers = [build["build_num"] - for build in await self.list_recent_builds(path, head_sha)] + build_numbers = [ + build["build_num"] + for build in await self.list_recent_builds(path, head_sha) + ] artifacts = [] for buildno in sorted(build_numbers): artifacts.extend( - (artifact['path'], artifact['url'], buildno) + (artifact["path"], artifact["url"], buildno) for artifact in await self.list_artifacts(buildno) ) return artifacts @@ -201,42 +217,53 @@ async def get_artifacts(self, path: str, head_sha: str) -> List[Tuple[str, str, class SlackMessage: """Parses a Slack message as sent by CircleCI""" + def __init__(self, _headers: Mapping[str, str], data: bytes): - response_text = data.decode('utf-8') + response_text = data.decode("utf-8") try: data = json.loads(response_text) except json.decoder.JSONDecodeError: raise RuntimeError("Unable to decore CircleCI Slack message") self.parsed = [] err = False - for attachment in data['attachments']: - text = attachment['text'] - if text.startswith('Success:'): + for attachment in data["attachments"]: + text = attachment["text"] + if text.startswith("Success:"): success = True - elif text.startswith('Failed:'): + elif text.startswith("Failed:"): success = False else: err = True continue - urls = {key: url for url, key in re.findall(r'<(http[^|>]+)\|([^>]+)>', text)} - self.parsed.append({ - 'urls': urls, - 'success': success, - }) + urls = { + key: url for url, key in re.findall(r"<(http[^|>]+)\|([^>]+)>", text) + } + self.parsed.append( + { + "urls": urls, + "success": success, + } + ) def __str__(self): - return '|'.join(f"success={x['success']} {':'.join(x['urls'].keys())}" - for x in self.parsed) + return "|".join( + f"success={x['success']} {':'.join(x['urls'].keys())}" for x in self.parsed + ) class AsyncCircleAPI(CircleAPI): """CircleCI API using `aiohttp`""" - def __init__(self, session: aiohttp.ClientSession, *args: Any, **kwargs: Any) -> None: + + def __init__( + self, session: aiohttp.ClientSession, *args: Any, **kwargs: Any + ) -> None: self._session = session super().__init__(*args, **kwargs) - async def _request(self, method: str, url: str, - headers: Mapping[str, str], - body: bytes = b'') -> Tuple[int, Mapping[str, str], bytes]: - async with self._session.request(method, url, headers=headers, data=body) as response: + async def _request( + self, method: str, url: str, headers: Mapping[str, str], body: bytes = b"" + ) -> Tuple[int, Mapping[str, str], bytes]: + async with self._session.request( + method, url, headers=headers, data=body + ) as response: return response.status, response.headers, await response.read() diff --git a/bioconda_utils/cli.py b/bioconda_utils/cli.py index c160abdc97..ea56df95aa 100644 --- a/bioconda_utils/cli.py +++ b/bioconda_utils/cli.py @@ -7,6 +7,7 @@ # ".../importlib/_bootstrap.py:219: RuntimeWarning: numpy.dtype size \ # changed, may indicate binary incompatibility. Expected 96, got 88" import warnings + warnings.filterwarnings("ignore", message="numpy.dtype size changed") import sys @@ -38,33 +39,48 @@ logger = logging.getLogger(__name__) -def enable_logging(default_loglevel='info', default_file_loglevel='debug'): +def enable_logging(default_loglevel="info", default_file_loglevel="debug"): """Adds the parameter ``--loglevel`` and sets up logging Args: default_loglevel: loglevel used when --loglevel is not passed """ + def decorator(func): - @arg('--loglevel', help="Set logging level (debug, info, warning, error, critical)") - @arg('--logfile', help="Write log to file") - @arg('--logfile-level', help="Log level for log file") - @arg('--log-command-max-lines', help="Limit lines emitted for commands executed") + @arg( + "--loglevel", + help="Set logging level (debug, info, warning, error, critical)", + ) + @arg("--logfile", help="Write log to file") + @arg("--logfile-level", help="Log level for log file") + @arg( + "--log-command-max-lines", help="Limit lines emitted for commands executed" + ) @utils.wraps(func) - def wrapper(*args, loglevel=default_loglevel, logfile=None, - logfile_level=default_file_loglevel, - log_command_max_lines=None, **kwargs): + def wrapper( + *args, + loglevel=default_loglevel, + logfile=None, + logfile_level=default_file_loglevel, + log_command_max_lines=None, + **kwargs, + ): max_lines = int(log_command_max_lines) if log_command_max_lines else None - utils.setup_logger('bioconda_utils', loglevel, logfile, logfile_level, - max_lines) + utils.setup_logger( + "bioconda_utils", loglevel, logfile, logfile_level, max_lines + ) func(*args, **kwargs) + return wrapper + return decorator def enable_debugging(): """Adds the paremeter ``--pdb`` (or ``-P``) to enable dropping into PDB""" + def decorator(func): - @arg('-P', '--pdb', help="Drop into debugger on exception") + @arg("-P", "--pdb", help="Drop into debugger on exception") @utils.wraps(func) def wrapper(*args, pdb=False, **kwargs): try: @@ -73,22 +89,28 @@ def wrapper(*args, pdb=False, **kwargs): logger.exception("Dropping into debugger") if pdb: import pdb + pdb.post_mortem() else: raise + return wrapper + return decorator def enable_threads(): """Adds the parameter ``--threads`` (or ``-t``) to limit parallelism""" + def decorator(func): - @arg('-t', '--threads', help="Limit maximum number of processes used.") + @arg("-t", "--threads", help="Limit maximum number of processes used.") @utils.wraps(func) def wrapper(*args, threads=16, **kwargs): utils.set_max_threads(threads) func(*args, **kwargs) + return wrapper + return decorator @@ -97,6 +119,7 @@ def recipe_folder_and_config(allow_missing_for=None): Requires that func has synopsis ``def x(recipe_folder, config,...)``. """ + def check_arg(args, idx, name, default, allow_missing): val = args[idx] if not val: @@ -112,23 +135,29 @@ def check_arg(args, idx, name, default, allow_missing): def decorator(func): args = inspect.getfullargspec(func).args try: - recipe_folder_idx = args.index('recipe_folder') - config_idx = args.index('config') - allow_missing_idx = [args.index(field) - for field in allow_missing_for or []] + recipe_folder_idx = args.index("recipe_folder") + config_idx = args.index("config") + allow_missing_idx = [args.index(field) for field in allow_missing_for or []] except ValueError: sys.exit(f"Function {func} must have 'recipe_folder' and 'config' args") - @arg('recipe_folder', nargs='?', - help='Path to folder containing recipes (default: recipes/)') - @arg('config', nargs='?', - help='Path to Bioconda config (default: config.yml)') + + @arg( + "recipe_folder", + nargs="?", + help="Path to folder containing recipes (default: recipes/)", + ) + @arg("config", nargs="?", help="Path to Bioconda config (default: config.yml)") @utils.wraps(func) def wrapper(*args, **kwargs): allow = any(args[idx] for idx in allow_missing_idx) - args = check_arg(args, recipe_folder_idx, 'recipe_folder', 'recipes/', allow) - args = check_arg(args, config_idx, 'config', 'config.yml', allow) + args = check_arg( + args, recipe_folder_idx, "recipe_folder", "recipes/", allow + ) + args = check_arg(args, config_idx, "config", "config.yml", allow) func(*args, **kwargs) + return wrapper + return decorator @@ -161,17 +190,26 @@ def get_recipes(config, recipe_folder, packages, git_range) -> List[str]: """ recipes = list(utils.get_recipes(recipe_folder, packages)) - logger.info("Considering total of %s recipes%s.", - len(recipes), utils.ellipsize_recipes(recipes, recipe_folder)) + logger.info( + "Considering total of %s recipes%s.", + len(recipes), + utils.ellipsize_recipes(recipes, recipe_folder), + ) if git_range: changed_recipes = get_recipes_to_build(git_range, recipe_folder) - logger.info("Constraining to %s git modified recipes%s.", len(changed_recipes), - utils.ellipsize_recipes(changed_recipes, recipe_folder)) + logger.info( + "Constraining to %s git modified recipes%s.", + len(changed_recipes), + utils.ellipsize_recipes(changed_recipes, recipe_folder), + ) recipes = [recipe for recipe in recipes if recipe in set(changed_recipes)] if len(recipes) != len(changed_recipes): - logger.info("Overlap was %s recipes%s.", len(recipes), - utils.ellipsize_recipes(recipes, recipe_folder)) + logger.info( + "Overlap was %s recipes%s.", + len(recipes), + utils.ellipsize_recipes(recipes, recipe_folder), + ) blacklist = utils.get_blacklist(config, recipe_folder) blacklisted = [] @@ -179,11 +217,17 @@ def get_recipes(config, recipe_folder, packages, git_range) -> List[str]: if os.path.relpath(recipe, recipe_folder) in blacklist: blacklisted.append(recipe) if blacklisted: - logger.info("Ignoring %s blacklisted recipes%s.", len(blacklisted), - utils.ellipsize_recipes(blacklisted, recipe_folder)) + logger.info( + "Ignoring %s blacklisted recipes%s.", + len(blacklisted), + utils.ellipsize_recipes(blacklisted, recipe_folder), + ) recipes = [recipe for recipe in recipes if recipe not in set(blacklisted)] - logger.info("Processing %s recipes%s.", len(recipes), - utils.ellipsize_recipes(recipes, recipe_folder)) + logger.info( + "Processing %s recipes%s.", + len(recipes), + utils.ellipsize_recipes(recipes, recipe_folder), + ) return recipes @@ -195,138 +239,199 @@ def get_recipes(config, recipe_folder, packages, git_range) -> List[str]: # `recipes/bowtie` or `recipes/bowtie/1.0.1`. -@arg('config', help='Path to yaml file specifying the configuration') -@arg('--strict-version', action='store_true', help='Require version to strictly match.') -@arg('--strict-build', action='store_true', help='Require version and build to strictly match.') -@arg('--remove', action='store_true', help='Remove packages from anaconda.') -@arg('--dryrun', '-n', action='store_true', help='Only print removal plan.') -@arg('--url', action='store_true', help='Print anaconda urls.') -@arg('--channel', help="Channel to check for duplicates") +@arg("config", help="Path to yaml file specifying the configuration") +@arg("--strict-version", action="store_true", help="Require version to strictly match.") +@arg( + "--strict-build", + action="store_true", + help="Require version and build to strictly match.", +) +@arg("--remove", action="store_true", help="Remove packages from anaconda.") +@arg("--dryrun", "-n", action="store_true", help="Only print removal plan.") +@arg("--url", action="store_true", help="Print anaconda urls.") +@arg("--channel", help="Channel to check for duplicates") @enable_logging() -def duplicates(config, - strict_version=False, - strict_build=False, - dryrun=False, - remove=False, - url=False, - channel='bioconda'): +def duplicates( + config, + strict_version=False, + strict_build=False, + dryrun=False, + remove=False, + url=False, + channel="bioconda", +): """ Detect packages in bioconda that have duplicates in the other defined channels. """ if remove and not strict_build: - raise ValueError('Removing packages is only supported in case of ' - '--strict-build.') + raise ValueError( + "Removing packages is only supported in case of " "--strict-build." + ) config = utils.load_config(config) - if channel not in config['channels']: + if channel not in config["channels"]: raise ValueError("Channel given with --channel must be in config channels") our_channel = channel - channels = [c for c in config['channels'] if c != our_channel] - logger.info("Checking for packages from %s also present in %s", - our_channel, channels) + channels = [c for c in config["channels"] if c != our_channel] + logger.info( + "Checking for packages from %s also present in %s", our_channel, channels + ) - check_fields = ['name'] + check_fields = ["name"] if strict_version or strict_build: - check_fields += ['version'] + check_fields += ["version"] if strict_build: - check_fields += ['build'] + check_fields += ["build"] def remove_package(spec): - fn = '{}-{}-{}.tar.bz2'.format(*spec) + fn = "{}-{}-{}.tar.bz2".format(*spec) name, version = spec[:2] subcmd = [ - 'remove', '-f', - '{channel}/{name}/{version}/{fn}'.format( + "remove", + "-f", + "{channel}/{name}/{version}/{fn}".format( name=name, version=version, fn=fn, channel=our_channel - ) + ), ] if dryrun: - logger.info(" ".join([utils.bin_for('anaconda')] + subcmd)) + logger.info(" ".join([utils.bin_for("anaconda")] + subcmd)) else: - token = os.environ.get('ANACONDA_TOKEN') + token = os.environ.get("ANACONDA_TOKEN") if token is None: token = [] else: - token = ['-t', token] - logger.info(utils.run([utils.bin_for('anaconda')] + token + subcmd, mask=[token]).stdout) + token = ["-t", token] + logger.info( + utils.run( + [utils.bin_for("anaconda")] + token + subcmd, mask=[token] + ).stdout + ) # packages in our channel repodata = utils.RepoData() our_package_specs = set(repodata.get_package_data(check_fields, our_channel)) - logger.info("%s unique packages specs to consider in %s", - len(our_package_specs), our_channel) + logger.info( + "%s unique packages specs to consider in %s", + len(our_package_specs), + our_channel, + ) # packages in channels we depend on duplicate = defaultdict(list) for channel in channels: package_specs = set(repodata.get_package_data(check_fields, channel)) - logger.info("%s unique packages specs to consider in %s", - len(package_specs), channel) + logger.info( + "%s unique packages specs to consider in %s", len(package_specs), channel + ) dups = our_package_specs & package_specs logger.info(" (of which %s are duplicate)", len(dups)) for spec in dups: duplicate[spec].append(channel) - print('\t'.join(check_fields + ['channels'])) + print("\t".join(check_fields + ["channels"])) for spec, dup_channels in sorted(duplicate.items()): if remove: remove_package(spec) else: if url: if not strict_version and not strict_build: - print('https://anaconda.org/{}/{}'.format( - our_channel, spec[0])) - print('https://anaconda.org/{}/{}/files?version={}'.format( - our_channel, *spec)) + print("https://anaconda.org/{}/{}".format(our_channel, spec[0])) + print( + "https://anaconda.org/{}/{}/files?version={}".format( + our_channel, *spec + ) + ) else: - print(*spec, ','.join(dup_channels), sep='\t') + print(*spec, ",".join(dup_channels), sep="\t") -@recipe_folder_and_config(allow_missing_for=['list_checks']) +@recipe_folder_and_config(allow_missing_for=["list_checks"]) @arg( - '--packages', + "--packages", nargs="+", - help='Glob for package[s] to build. Default is to build all packages. Can ' - 'be specified more than once') -@arg('--cache', help='''To speed up debugging, use repodata cached locally in + help="Glob for package[s] to build. Default is to build all packages. Can " + "be specified more than once", +) +@arg( + "--cache", + help="""To speed up debugging, use repodata cached locally in the provided filename. If the file does not exist, it will be created the - first time.''') -@arg('--list-checks', help='''List the linting functions to be used and then - exit''') -@arg('--exclude', nargs='+', help='''Exclude this linting function. Can be used - multiple times.''') -@arg('--push-status', action='store_true', help='''If set, the lint status will + first time.""", +) +@arg( + "--list-checks", + help="""List the linting functions to be used and then + exit""", +) +@arg( + "--exclude", + nargs="+", + help="""Exclude this linting function. Can be used + multiple times.""", +) +@arg( + "--push-status", + action="store_true", + help="""If set, the lint status will be sent to the current commit on github. Also needs --user and --repo to be set. Requires the env var GITHUB_TOKEN to be set. Note that pull requests from forks will not have access to encrypted variables on - ci, so this feature may be of limited use.''') -@arg('--commit', help='Commit on github on which to update status') -@arg('--push-comment', action='store_true', help='''If set, the lint status + ci, so this feature may be of limited use.""", +) +@arg("--commit", help="Commit on github on which to update status") +@arg( + "--push-comment", + action="store_true", + help="""If set, the lint status will be posted as a comment in the corresponding pull request (given by --pull-request). Also needs --user and --repo to be set. Requires the env - var GITHUB_TOKEN to be set.''') -@arg('--pull-request', type=int, help='''Pull request id on github on which to - post a comment.''') -@arg('--user', help='Github user') -@arg('--repo', help='Github repo') -@arg('--git-range', nargs='+', - help='''Git range (e.g. commits or something like + var GITHUB_TOKEN to be set.""", +) +@arg( + "--pull-request", + type=int, + help="""Pull request id on github on which to + post a comment.""", +) +@arg("--user", help="Github user") +@arg("--repo", help="Github repo") +@arg( + "--git-range", + nargs="+", + help="""Git range (e.g. commits or something like "master HEAD" to check commits in HEAD vs master, or just "HEAD" to include uncommitted changes). All recipes modified within this range will - be built if not present in the channel.''') -@arg('--full-report', action='store_true', help='''Default behavior is to + be built if not present in the channel.""", +) +@arg( + "--full-report", + action="store_true", + help="""Default behavior is to summarize the linting results; use this argument to get the full - results as a TSV printed to stdout.''') -@arg('--try-fix', help='''Attempt to fix problems where found''') + results as a TSV printed to stdout.""", +) +@arg("--try-fix", help="""Attempt to fix problems where found""") @enable_logging() @enable_debugging() -@named('lint') -def do_lint(recipe_folder, config, packages="*", cache=None, list_checks=False, - exclude=None, push_status=False, user='bioconda', - commit=None, push_comment=False, pull_request=None, - repo='bioconda-recipes', git_range=None, full_report=False, - try_fix=False): +@named("lint") +def do_lint( + recipe_folder, + config, + packages="*", + cache=None, + list_checks=False, + exclude=None, + push_status=False, + user="bioconda", + commit=None, + push_comment=False, + pull_request=None, + repo="bioconda-recipes", + git_range=None, + full_report=False, + try_fix=False, +): """ Lint recipes @@ -334,7 +439,7 @@ def do_lint(recipe_folder, config, packages="*", cache=None, list_checks=False, Otherwise pushes a commit status to the specified commit on github. """ if list_checks: - print('\n'.join(str(check) for check in lint.get_checks())) + print("\n".join(str(check) for check in lint.get_checks())) sys.exit(0) config = utils.load_config(config) @@ -358,76 +463,147 @@ def do_lint(recipe_folder, config, packages="*", cache=None, list_checks=False, @recipe_folder_and_config() -@arg('--packages', - nargs="+", - help='Glob for package[s] to build. Default is to build all packages. Can ' - 'be specified more than once') -@arg('--git-range', nargs='+', - help='''Git range (e.g. commits or something like +@arg( + "--packages", + nargs="+", + help="Glob for package[s] to build. Default is to build all packages. Can " + "be specified more than once", +) +@arg( + "--git-range", + nargs="+", + help="""Git range (e.g. commits or something like "master HEAD" to check commits in HEAD vs master, or just "HEAD" to include uncommitted changes). All recipes modified within this range will - be built if not present in the channel.''') -@arg('--testonly', help='Test packages instead of building') -@arg('--force', - help='''Force building the recipe even if it already exists in the + be built if not present in the channel.""", +) +@arg("--testonly", help="Test packages instead of building") +@arg( + "--force", + help="""Force building the recipe even if it already exists in the bioconda channel. If --force is specified, --git-range is ignored and only - those packages matching --packages globs will be built.''') -@arg('--docker', action='store_true', - help='Build packages in docker container.') -@arg('--mulled-test', action='store_true', help="Run a mulled-build test on the built package") -@arg('--mulled-upload-target', help="Provide a quay.io target to push mulled docker images to.") -@arg('--build_script_template', help='''Filename to optionally replace build + those packages matching --packages globs will be built.""", +) +@arg("--docker", action="store_true", help="Build packages in docker container.") +@arg( + "--mulled-test", + action="store_true", + help="Run a mulled-build test on the built package", +) +@arg( + "--mulled-upload-target", + help="Provide a quay.io target to push mulled docker images to.", +) +@arg( + "--build_script_template", + help="""Filename to optionally replace build script template used by the Docker container. By default use - docker_utils.BUILD_SCRIPT_TEMPLATE. Only used if --docker is True.''') -@arg('--pkg_dir', help='''Specifies the directory to which container-built + docker_utils.BUILD_SCRIPT_TEMPLATE. Only used if --docker is True.""", +) +@arg( + "--pkg_dir", + help="""Specifies the directory to which container-built packages should be stored on the host. Default is to use the host's conda-bld dir. If --docker is not specified, then this argument is - ignored.''') -@arg('--anaconda-upload', action='store_true', help='''After building recipes, upload - them to Anaconda. This requires $ANACONDA_TOKEN to be set.''') -@arg('--build-image', action='store_true', help='''Build temporary docker build - image with conda/conda-build version matching local versions''') -@arg('--keep-image', action='store_true', help='''After building recipes, the + ignored.""", +) +@arg( + "--anaconda-upload", + action="store_true", + help="""After building recipes, upload + them to Anaconda. This requires $ANACONDA_TOKEN to be set.""", +) +@arg( + "--build-image", + action="store_true", + help="""Build temporary docker build + image with conda/conda-build version matching local versions""", +) +@arg( + "--keep-image", + action="store_true", + help="""After building recipes, the created Docker image is removed by default to save disk space. Use this - argument to disable this behavior.''') -@arg('--lint', '--prelint', action='store_true', help='''Just before each recipe, apply + argument to disable this behavior.""", +) +@arg( + "--lint", + "--prelint", + action="store_true", + help="""Just before each recipe, apply the linting functions to it. This can be used as an alternative to linting all recipes before any building takes place with the `bioconda-utils lint` - command.''') -@arg('--lint-exclude', nargs='+', - help='''Exclude this linting function. Can be used multiple times.''') -@arg('--check-channels', nargs='+', - help='''Channels to check recipes against before building. Any recipe + command.""", +) +@arg( + "--lint-exclude", + nargs="+", + help="""Exclude this linting function. Can be used multiple times.""", +) +@arg( + "--check-channels", + nargs="+", + help="""Channels to check recipes against before building. Any recipe already present in one of these channels will be skipped. The default is the first two channels specified in the config file. Note that this is - ignored if you specify --git-range.''') -@arg('--n-workers', type=int, default=1, - help='''The number of parallel workers that are in use. This is intended + ignored if you specify --git-range.""", +) +@arg( + "--n-workers", + type=int, + default=1, + help="""The number of parallel workers that are in use. This is intended for use in cases such as the "bulk" branch, where there are multiple parallel workers building and uploading recipes. In essence, this causes bioconda-utils to process every Nth sub-DAG, where N is the value you give to this option. The default is 1, which is intended for cases where there are NOT parallel workers (i.e., the majority of cases). This should generally NOT be used in conjunctions with the --packages or --git-range - options!''') -@arg('--worker-offset', type=int, default=0, - help='''This is only used if --nWorkers is >1. In that case, then each + options!""", +) +@arg( + "--worker-offset", + type=int, + default=0, + help="""This is only used if --nWorkers is >1. In that case, then each instance of bioconda-utils will process every Nth sub-DAG. This option gives the 0-based offset for that. For example, if "--n-workers 5 --worker-offset 0" is used, then this instance of bioconda-utils will process the 1st, 6th, 11th, etc. sub-DAGs. Equivalently, using "--n-workers 5 --worker-offset 1" will result in sub-DAGs 2, 7, 12, etc. being processed. If you use more - than one worker, then make sure to give each a different offset!''') -@arg('--keep-old-work', action='store_true', help='''Do not remove anything -from environment, even after successful build and test.''') + than one worker, then make sure to give each a different offset!""", +) +@arg( + "--keep-old-work", + action="store_true", + help="""Do not remove anything +from environment, even after successful build and test.""", +) @enable_logging() -def build(recipe_folder, config, packages="*", git_range=None, testonly=False, - force=False, docker=None, mulled_test=False, build_script_template=None, - pkg_dir=None, anaconda_upload=False, mulled_upload_target=None, - build_image=False, keep_image=False, lint=False, lint_exclude=None, - check_channels=None, n_workers=1, worker_offset=0, keep_old_work=False): +def build( + recipe_folder, + config, + packages="*", + git_range=None, + testonly=False, + force=False, + docker=None, + mulled_test=False, + build_script_template=None, + pkg_dir=None, + anaconda_upload=False, + mulled_upload_target=None, + build_image=False, + keep_image=False, + lint=False, + lint_exclude=None, + check_channels=None, + n_workers=1, + worker_offset=0, + keep_old_work=False, +): cfg = utils.load_config(config) - setup = cfg.get('setup', None) + setup = cfg.get("setup", None) if setup: logger.debug("Running setup: %s", setup) for cmd in setup: @@ -456,43 +632,55 @@ def build(recipe_folder, config, packages="*", git_range=None, testonly=False, docker_builder = None if lint_exclude and not lint: - logger.warning('--lint-exclude has no effect unless --lint is specified.') - - label = os.getenv('BIOCONDA_LABEL', None) or None - - success = build_recipes(recipe_folder, config, recipes, - testonly=testonly, - force=force, - mulled_test=mulled_test, - docker_builder=docker_builder, - anaconda_upload=anaconda_upload, - mulled_upload_target=mulled_upload_target, - do_lint=lint, - lint_exclude=lint_exclude, - check_channels=check_channels, - label=label, - n_workers=n_workers, - worker_offset=worker_offset, - keep_old_work=keep_old_work) + logger.warning("--lint-exclude has no effect unless --lint is specified.") + + label = os.getenv("BIOCONDA_LABEL", None) or None + + success = build_recipes( + recipe_folder, + config, + recipes, + testonly=testonly, + force=force, + mulled_test=mulled_test, + docker_builder=docker_builder, + anaconda_upload=anaconda_upload, + mulled_upload_target=mulled_upload_target, + do_lint=lint, + lint_exclude=lint_exclude, + check_channels=check_channels, + label=label, + n_workers=n_workers, + worker_offset=worker_offset, + keep_old_work=keep_old_work, + ) exit(0 if success else 1) @recipe_folder_and_config() -@arg('--packages', - nargs="+", - help='Glob for package[s] to show in DAG. Default is to show all ' - 'packages. Can be specified more than once') -@arg('--format', choices=['gml', 'dot', 'txt'], help='''Set format to print +@arg( + "--packages", + nargs="+", + help="Glob for package[s] to show in DAG. Default is to show all " + "packages. Can be specified more than once", +) +@arg( + "--format", + choices=["gml", "dot", "txt"], + help="""Set format to print graph. "gml" and "dot" can be imported into graph visualization tools (graphviz, gephi, cytoscape). "txt" will print out recipes grouped by independent subdags, largest subdag first, each in topologically sorted order. Singleton subdags (if not hidden with --hide-singletons) are - reported as one large group at the end.''') -@arg('--hide-singletons', - action='store_true', - help='Hide singletons in the printed graph.') + reported as one large group at the end.""", +) +@arg( + "--hide-singletons", + action="store_true", + help="Hide singletons in the printed graph.", +) @enable_logging() -def dag(recipe_folder, config, packages="*", format='gml', hide_singletons=False): +def dag(recipe_folder, config, packages="*", format="gml", hide_singletons=False): """ Export the DAG of packages to a graph format file for visualization """ @@ -503,11 +691,11 @@ def dag(recipe_folder, config, packages="*", format='gml', hide_singletons=False for node in nx.nodes(dag): if dag.degree(node) == 0: dag.remove_node(node) - if format == 'gml': + if format == "gml": nx.write_gml(dag, sys.stdout.buffer) - elif format == 'dot': + elif format == "dot": write_dot(dag, sys.stdout) - elif format == 'txt': + elif format == "txt": subdags = sorted(map(sorted, nx.connected_components(dag.to_undirected()))) subdags = sorted(subdags, key=len, reverse=True) singletons = [] @@ -518,50 +706,64 @@ def dag(recipe_folder, config, packages="*", format='gml', hide_singletons=False print("# subdag {0}".format(i)) subdag = dag.subgraph(s) recipes = [ - recipe for package in nx.topological_sort(subdag) - for recipe in name2recipes[package]] - print('\n'.join(recipes) + '\n') + recipe + for package in nx.topological_sort(subdag) + for recipe in name2recipes[package] + ] + print("\n".join(recipes) + "\n") if not hide_singletons: - print('# singletons') - recipes = [recipe for package in singletons for recipe in - name2recipes[package]] - print('\n'.join(recipes) + '\n') + print("# singletons") + recipes = [ + recipe for package in singletons for recipe in name2recipes[package] + ] + print("\n".join(recipes) + "\n") @recipe_folder_and_config() -@arg('--packages', - nargs="+", - help='Glob for package[s] to update, as needed due to a change in pinnings') -@arg('--skip-additional-channels', - nargs='*', - help="""Skip updating/bumping packges that are already built with +@arg( + "--packages", + nargs="+", + help="Glob for package[s] to update, as needed due to a change in pinnings", +) +@arg( + "--skip-additional-channels", + nargs="*", + help="""Skip updating/bumping packges that are already built with compatible pinnings in one of the given channels in addition to those - listed in 'config'.""") -@arg('--skip-variants', - nargs='*', - help='Skip packages that use one of the given variant keys.') -@arg('--max-bumps', type=int, - help='Maximum number of recipes that will be updated.') -@arg('--no-leaves', - help='Only update recipes with dependent packages.') -@arg('--cache', help='''To speed up debugging, use repodata cached locally in + listed in 'config'.""", +) +@arg( + "--skip-variants", + nargs="*", + help="Skip packages that use one of the given variant keys.", +) +@arg("--max-bumps", type=int, help="Maximum number of recipes that will be updated.") +@arg("--no-leaves", help="Only update recipes with dependent packages.") +@arg( + "--cache", + help="""To speed up debugging, use repodata cached locally in the provided filename. If the file does not exist, it will be created the - first time.''') + first time.""", +) @enable_logging() @enable_threads() @enable_debugging() -def update_pinning(recipe_folder, config, packages="*", - skip_additional_channels=None, - skip_variants=None, - max_bumps=None, - no_leaves=False, - cache=None): +def update_pinning( + recipe_folder, + config, + packages="*", + skip_additional_channels=None, + skip_variants=None, + max_bumps=None, + no_leaves=False, + cache=None, +): """Bump a package build number and all dependencies as required due to a change in pinnings """ config = utils.load_config(config) if skip_additional_channels: - config['channels'] += skip_additional_channels + config["channels"] += skip_additional_channels skip_variants = frozenset(skip_variants or ()) if cache: @@ -572,9 +774,12 @@ def update_pinning(recipe_folder, config, packages="*", blacklist = utils.get_blacklist(config, recipe_folder) from . import recipe + dag = graph.build_from_recipes( - recip for recip in recipe.load_parallel_iter(recipe_folder, "*") - if recip.reldir not in blacklist) + recip + for recip in recipe.load_parallel_iter(recipe_folder, "*") + if recip.reldir not in blacklist + ) dag = graph.filter_recipe_dag(dag, packages, []) if no_leaves: @@ -592,7 +797,9 @@ def update_pinning(recipe_folder, config, packages="*", bumpErrors = set() needs_bump = partial( - update_pinnings.check, build_config=build_config, skip_variant_keys=skip_variants, + update_pinnings.check, + build_config=build_config, + skip_variant_keys=skip_variants, ) State = update_pinnings.State @@ -605,25 +812,27 @@ def update_pinning(recipe_folder, config, packages="*", num_recipes_needing_bump += 1 if num_recipes_needing_bump <= max_bumps: logger.info("Bumping %s", recip) - recip.reset_buildnumber(int(recip['build']['number'])+1) + recip.reset_buildnumber(int(recip["build"]["number"]) + 1) recip.save() else: logger.info( "Bumping %s -- theoretically (%d out of %d allowed bumps)", - recip, num_recipes_needing_bump, max_bumps, + recip, + num_recipes_needing_bump, + max_bumps, ) elif status.failed(): logger.info("Failed to inspect %s", recip) hadErrors.add(recip) else: - logger.info('OK: %s', recip) + logger.info("OK: %s", recip) # Print some information print("Packages requiring the following:") print(stats) - #print(" No build number change needed: {}".format(stats[STATE.ok])) - #print(" A rebuild for a new python version: {}".format(stats[STATE.bump_python])) - #print(" A build number increment: {}".format(stats[STATE.bump])) + # print(" No build number change needed: {}".format(stats[STATE.ok])) + # print(" A rebuild for a new python version: {}".format(stats[STATE.bump_python])) + # print(" A build number increment: {}".format(stats[STATE.bump])) if num_recipes_needing_bump > max_bumps: print( @@ -631,40 +840,57 @@ def update_pinning(recipe_folder, config, packages="*", " that needed a build number bump." ) if hadErrors: - print("{} packages produced an error " - "in conda-build: {}".format(len(hadErrors), list(hadErrors))) + print( + "{} packages produced an error " + "in conda-build: {}".format(len(hadErrors), list(hadErrors)) + ) if bumpErrors: - print("The build numbers in the following recipes " - "could not be incremented: {}".format(list(bumpErrors))) + print( + "The build numbers in the following recipes " + "could not be incremented: {}".format(list(bumpErrors)) + ) @recipe_folder_and_config() -@arg('--dependencies', nargs='+', - help='''Return recipes in `recipe_folder` in the dependency chain for the - packages listed here. Answers the question "what does PACKAGE need?"''') -@arg('--reverse-dependencies', nargs='+', - help='''Return recipes in `recipe_folder` in the reverse dependency chain +@arg( + "--dependencies", + nargs="+", + help='''Return recipes in `recipe_folder` in the dependency chain for the + packages listed here. Answers the question "what does PACKAGE need?"''', +) +@arg( + "--reverse-dependencies", + nargs="+", + help='''Return recipes in `recipe_folder` in the reverse dependency chain for packages listed here. Answers the question "what depends on - PACKAGE?"''') -@arg('--restrict', - help='''Restrict --dependencies to packages in `recipe_folder`. Has no + PACKAGE?"''', +) +@arg( + "--restrict", + help="""Restrict --dependencies to packages in `recipe_folder`. Has no effect if --reverse-dependencies, which always looks just in the recipe - dir.''') + dir.""", +) @enable_logging() -def dependent(recipe_folder, config, restrict=False, - dependencies=None, reverse_dependencies=None): +def dependent( + recipe_folder, config, restrict=False, dependencies=None, reverse_dependencies=None +): """ Print recipes dependent on a package """ if dependencies and reverse_dependencies: raise ValueError( - '`dependencies` and `reverse_dependencies` are mutually exclusive') + "`dependencies` and `reverse_dependencies` are mutually exclusive" + ) if not any([dependencies, reverse_dependencies]): raise ValueError( - 'One of `--dependencies` or `--reverse-dependencies` is required.') + "One of `--dependencies` or `--reverse-dependencies` is required." + ) - d, n2r = graph.build(utils.get_recipes(recipe_folder, "*"), config, restrict=restrict) + d, n2r = graph.build( + utils.get_recipes(recipe_folder, "*"), config, restrict=restrict + ) if reverse_dependencies is not None: func, packages = nx.algorithms.descendants, reverse_dependencies @@ -674,39 +900,75 @@ def dependent(recipe_folder, config, restrict=False, pkgs = [] for pkg in packages: pkgs.extend(list(func(d, pkg))) - print('\n'.join(sorted(list(set(pkgs))))) + print("\n".join(sorted(list(set(pkgs))))) -@arg('package', help='''Bioconductor package name. This is case-sensitive, and +@arg( + "package", + help="""Bioconductor package name. This is case-sensitive, and must match the package name on the Bioconductor site. If "update-all-packages" is specified, then all packages in a given bioconductor release will be - created/updated (--force is then implied).''') + created/updated (--force is then implied).""", +) @recipe_folder_and_config() -@arg('bioc_data_packages', nargs='?', - help='''Path to folder containing the recipe for the bioconductor-data-packages - (default: recipes/bioconductor-data-packages)''') -@arg('--versioned', action='store_true', help='''If specified, recipe will be - created in RECIPES//''') -@arg('--force', action='store_true', help='''Overwrite the contents of an +@arg( + "bioc_data_packages", + nargs="?", + help="""Path to folder containing the recipe for the bioconductor-data-packages + (default: recipes/bioconductor-data-packages)""", +) +@arg( + "--versioned", + action="store_true", + help="""If specified, recipe will be + created in RECIPES//""", +) +@arg( + "--force", + action="store_true", + help="""Overwrite the contents of an existing recipe. If --recursive is also used, then overwrite *all* recipes - created.''') -@arg('--pkg-version', help='''Package version to use instead of the current - one''') -@arg('--bioc-version', help="""Version of Bioconductor to target. If not + created.""", +) +@arg( + "--pkg-version", + help="""Package version to use instead of the current + one""", +) +@arg( + "--bioc-version", + help="""Version of Bioconductor to target. If not specified, then automatically finds the latest version of Bioconductor with the specified version in --pkg-version, or if --pkg-version not specified, then finds the the latest package version in the latest - Bioconductor version""") -@arg('--recursive', action='store_true', help="""Creates the recipes for all - Bioconductor and CRAN dependencies of the specified package.""") -@arg('--skip-if-in-channels', nargs='*', help="""When --recursive is used, it will build + Bioconductor version""", +) +@arg( + "--recursive", + action="store_true", + help="""Creates the recipes for all + Bioconductor and CRAN dependencies of the specified package.""", +) +@arg( + "--skip-if-in-channels", + nargs="*", + help="""When --recursive is used, it will build *all* recipes. Use this argument to skip recursive building for packages - that already exist in the packages listed here.""") -@enable_logging('debug') + that already exist in the packages listed here.""", +) +@enable_logging("debug") def bioconductor_skeleton( - recipe_folder, config, package, bioc_data_packages, versioned=False, force=False, - pkg_version=None, bioc_version=None, recursive=False, - skip_if_in_channels=['conda-forge', 'bioconda']): + recipe_folder, + config, + package, + bioc_data_packages, + versioned=False, + force=False, + pkg_version=None, + bioc_version=None, + recursive=False, + skip_if_in_channels=["conda-forge", "bioconda"], +): """ Build a Bioconductor recipe. The recipe will be created in the 'recipes' directory and will be prefixed by "bioconductor-". If --recursive is set, @@ -738,27 +1000,54 @@ def bioconductor_skeleton( for k, v in packages.items(): try: _bioconductor_skeleton.write_recipe( - k, recipe_folder, config, bioc_data_packages=bioc_data_packages, force=True, bioc_version=bioc_version, - pkg_version=v['Version'], versioned=versioned, packages=packages, - skip_if_in_channels=skip_if_in_channels, needs_x = k in needs_x) + k, + recipe_folder, + config, + bioc_data_packages=bioc_data_packages, + force=True, + bioc_version=bioc_version, + pkg_version=v["Version"], + versioned=versioned, + packages=packages, + skip_if_in_channels=skip_if_in_channels, + needs_x=k in needs_x, + ) except: problems.append(k) if len(problems): - sys.exit("The following recipes had problems and were not finished: {}".format(", ".join(problems))) + sys.exit( + "The following recipes had problems and were not finished: {}".format( + ", ".join(problems) + ) + ) else: _bioconductor_skeleton.write_recipe( - package, recipe_folder, config, bioc_data_packages, force=force, bioc_version=bioc_version, - pkg_version=pkg_version, versioned=versioned, recursive=recursive, + package, + recipe_folder, + config, + bioc_data_packages, + force=force, + bioc_version=bioc_version, + pkg_version=pkg_version, + versioned=versioned, + recursive=recursive, seen_dependencies=seen_dependencies, - skip_if_in_channels=skip_if_in_channels) - sys.stderr.write("Warning! Make sure to bump bioconductor-data-packages if needed!\n") + skip_if_in_channels=skip_if_in_channels, + ) + sys.stderr.write( + "Warning! Make sure to bump bioconductor-data-packages if needed!\n" + ) -@arg('recipe', help='''Path to recipe to be cleaned''') -@arg('--no-windows', action='store_true', help="""Use this when submitting an +@arg("recipe", help="""Path to recipe to be cleaned""") +@arg( + "--no-windows", + action="store_true", + help="""Use this when submitting an R package to Bioconda. After a CRAN skeleton is created, any Windows-related lines will be removed and the bld.bat file will be - removed.""") + removed.""", +) @enable_logging() def clean_cran_skeleton(recipe, no_windows=False): """ @@ -773,67 +1062,115 @@ def clean_cran_skeleton(recipe, no_windows=False): cran_skeleton.clean_skeleton_files(recipe, no_windows=no_windows) -@arg('recipe_folder', help='Path to recipes directory') -@arg('config', help='Path to yaml file specifying the configuration') +@arg("recipe_folder", help="Path to recipes directory") +@arg("config", help="Path to yaml file specifying the configuration") @recipe_folder_and_config() -@arg('--packages', nargs="+", - help='Glob(s) for package[s] to scan. Can be specified more than once') -@arg('--exclude', nargs="+", - help='Globs for package[s] to exclude from scan. Can be specified more than once') -@arg('--exclude-subrecipes', help='''By default, only subrecipes explicitly +@arg( + "--packages", + nargs="+", + help="Glob(s) for package[s] to scan. Can be specified more than once", +) +@arg( + "--exclude", + nargs="+", + help="Globs for package[s] to exclude from scan. Can be specified more than once", +) +@arg( + "--exclude-subrecipes", + help="""By default, only subrecipes explicitly enabled for watch in meta.yaml are considered. Set to 'always' to - exclude all subrecipes. Set to 'never' to include all subrecipes''') -@arg('--exclude-channels', nargs="+", help='''Exclude recipes + exclude all subrecipes. Set to 'never' to include all subrecipes""", +) +@arg( + "--exclude-channels", + nargs="+", + help="""Exclude recipes building packages present in other channels. Set to 'none' to disable - check.''') -@arg('--ignore-blacklists', help='''Do not exclude recipes from blacklist''') -@arg('--fetch-requirements', - help='''Try to fetch python requirements. Please note that this requires + check.""", +) +@arg("--ignore-blacklists", help="""Do not exclude recipes from blacklist""") +@arg( + "--fetch-requirements", + help="""Try to fetch python requirements. Please note that this requires downloading packages and executing setup.py, so presents a potential - security problem.''') -@arg('--cache', help='''To speed up debugging, use repodata cached locally in + security problem.""", +) +@arg( + "--cache", + help="""To speed up debugging, use repodata cached locally in the provided filename. If the file does not exist, it will be created the first time. Caution: The cache will not be updated if - exclude-channels is changed''') -@arg('--unparsed-urls', help='''Write unrecognized urls to this file''') -@arg('--failed-urls', help='''Write urls with permanent failure to this file''') -@arg('--recipe-status', help='''Write status for each recipe to this file''') -@arg('--check-branch', help='''Check if recipe has active branch''') + exclude-channels is changed""", +) +@arg("--unparsed-urls", help="""Write unrecognized urls to this file""") +@arg("--failed-urls", help="""Write urls with permanent failure to this file""") +@arg("--recipe-status", help="""Write status for each recipe to this file""") +@arg("--check-branch", help="""Check if recipe has active branch""") @arg("--only-active", action="store_true", help="Check only recipes with active update") -@arg("--create-branch", action="store_true", help='''Create branch for each - update''') -@arg("--create-pr", action="store_true", help='''Create PR for each update. - Implies create-branch.''') -@arg("--max-updates", help='''Exit after ARG updates''') -@arg("--no-shuffle", help='''Do not shuffle recipe order''') +@arg( + "--create-branch", + action="store_true", + help="""Create branch for each + update""", +) +@arg( + "--create-pr", + action="store_true", + help="""Create PR for each update. + Implies create-branch.""", +) +@arg("--max-updates", help="""Exit after ARG updates""") +@arg("--no-shuffle", help="""Do not shuffle recipe order""") @arg("--dry-run", help='''Don't update remote git or github"''') -@arg("--no-check-pinnings", help='''Don't check for pinning updates''') -@arg("--no-follow-graph", - help='''Don't process recipes in graph order or add dependent recipes - to checks. Implies --no-skip-pending-deps.''') -@arg("--no-check-pending-deps", - help='''Don't check for recipes having a dependency with a pending update. - Update all recipes, including those having deps in need or rebuild.''') -@arg("--no-check-version-update", - help='''Don't check for version updates to recipes''') -@arg('--sign', nargs="?", help='''Enable signing. Optionally takes keyid.''') -@arg('--commit-as', nargs=2, help='''Set user and email to use for committing. ''' - '''Takes exactly two arguments.''') +@arg("--no-check-pinnings", help="""Don't check for pinning updates""") +@arg( + "--no-follow-graph", + help="""Don't process recipes in graph order or add dependent recipes + to checks. Implies --no-skip-pending-deps.""", +) +@arg( + "--no-check-pending-deps", + help="""Don't check for recipes having a dependency with a pending update. + Update all recipes, including those having deps in need or rebuild.""", +) +@arg("--no-check-version-update", help="""Don't check for version updates to recipes""") +@arg("--sign", nargs="?", help="""Enable signing. Optionally takes keyid.""") +@arg( + "--commit-as", + nargs=2, + help="""Set user and email to use for committing. """ + """Takes exactly two arguments.""", +) @enable_logging() @enable_debugging() @enable_threads() -def autobump(recipe_folder, config, packages='*', exclude=None, cache=None, - failed_urls=None, unparsed_urls=None, recipe_status=None, - exclude_subrecipes=None, exclude_channels='conda-forge', - ignore_blacklists=False, - fetch_requirements=False, - check_branch=False, create_branch=False, create_pr=False, - only_active=False, no_shuffle=False, - max_updates=0, dry_run=False, - no_check_pinnings=False, no_follow_graph=False, - no_check_version_update=False, - no_check_pending_deps=False, - sign=0, commit_as=None): +def autobump( + recipe_folder, + config, + packages="*", + exclude=None, + cache=None, + failed_urls=None, + unparsed_urls=None, + recipe_status=None, + exclude_subrecipes=None, + exclude_channels="conda-forge", + ignore_blacklists=False, + fetch_requirements=False, + check_branch=False, + create_branch=False, + create_pr=False, + only_active=False, + no_shuffle=False, + max_updates=0, + dry_run=False, + no_check_pinnings=False, + no_follow_graph=False, + no_check_version_update=False, + no_check_pending_deps=False, + sign=0, + commit_as=None, +): """ Updates recipes in recipe_folder """ @@ -845,17 +1182,23 @@ def autobump(recipe_folder, config, packages='*', exclude=None, cache=None, if no_follow_graph: recipe_source = autobump.RecipeSource( - recipe_folder, packages, exclude or [], not no_shuffle) + recipe_folder, packages, exclude or [], not no_shuffle + ) no_skip_pending_deps = True else: recipe_source = autobump.RecipeGraphSource( - recipe_folder, packages, exclude or [], not no_shuffle, - config_dict, cache_fn=cache and cache + "_dag.pkl") + recipe_folder, + packages, + exclude or [], + not no_shuffle, + config_dict, + cache_fn=cache and cache + "_dag.pkl", + ) # Setup scanning pipeline - scanner = autobump.Scanner(recipe_source, - cache_fn=cache and cache + "_scan.pkl", - status_fn=recipe_status) + scanner = autobump.Scanner( + recipe_source, cache_fn=cache and cache + "_scan.pkl", status_fn=recipe_status + ) # Always exclude recipes that were explicitly disabled scanner.add(autobump.ExcludeDisabled) @@ -866,8 +1209,7 @@ def autobump(recipe_folder, config, packages='*', exclude=None, cache=None, # Exclude sub-recipes if exclude_subrecipes != "never": - scanner.add(autobump.ExcludeSubrecipe, - always=exclude_subrecipes == "always") + scanner.add(autobump.ExcludeSubrecipe, always=exclude_subrecipes == "always") # Exclude recipes with dependencies pending an update if not no_check_pending_deps and not no_follow_graph: @@ -893,8 +1235,7 @@ def autobump(recipe_folder, config, packages='*', exclude=None, cache=None, try: git_handler.enable_signing(install_gpg_key(env_key)) except ValueError as exc: - logger.error("Failed to use CODE_SIGNING_KEY from environment: %s", - exc) + logger.error("Failed to use CODE_SIGNING_KEY from environment: %s", exc) if commit_as: git_handler.set_user(*commit_as) else: @@ -907,8 +1248,11 @@ def autobump(recipe_folder, config, packages='*', exclude=None, cache=None, if exclude_channels != ["none"]: if not isinstance(exclude_channels, list): exclude_channels = [exclude_channels] - scanner.add(autobump.ExcludeOtherChannel, exclude_channels, - cache and cache + "_repodata.txt") + scanner.add( + autobump.ExcludeOtherChannel, + exclude_channels, + cache and cache + "_repodata.txt", + ) # Test if due to pinnings, the package hash would change and a rebuild # has become necessary. If so, bump the buildnumber. @@ -938,7 +1282,8 @@ def autobump(recipe_folder, config, packages='*', exclude=None, cache=None, logger.critical("GITHUB_TOKEN required to create PRs") exit(1) github_handler = githubhandler.AiohttpGitHubHandler( - token, dry_run, "bioconda", "bioconda-recipes") + token, dry_run, "bioconda", "bioconda-recipes" + ) scanner.add(autobump.CreatePullRequest, git_handler, github_handler) # Terminate the scanning pipeline after x recipes have reached this point. @@ -953,8 +1298,8 @@ def autobump(recipe_folder, config, packages='*', exclude=None, cache=None, git_handler.close() -@arg('--loglevel', default='info', help='Log level') -def bot(loglevel='info'): +@arg("--loglevel", default="info", help="Log level") +def bot(loglevel="info"): """Locally accedd bioconda-bot command API To run the bot locally, use: @@ -964,15 +1309,26 @@ def bot(loglevel='info'): You can append --reload to have gunicorn reload if any of the python files change. """ - utils.setup_logger('bioconda_utils', loglevel) + utils.setup_logger("bioconda_utils", loglevel) logger.error("Nothing here yet") + def main(): - if '--version' in sys.argv: + if "--version" in sys.argv: print("This is bioconda-utils version", VERSION) sys.exit(0) - argh.dispatch_commands([ - build, dag, dependent, do_lint, duplicates, update_pinning, - bioconductor_skeleton, clean_cran_skeleton, autobump, bot - ]) + argh.dispatch_commands( + [ + build, + dag, + dependent, + do_lint, + duplicates, + update_pinning, + bioconductor_skeleton, + clean_cran_skeleton, + autobump, + bot, + ] + ) diff --git a/bioconda_utils/cran_skeleton.py b/bioconda_utils/cran_skeleton.py index dae6523b70..b43c9fa3d5 100644 --- a/bioconda_utils/cran_skeleton.py +++ b/bioconda_utils/cran_skeleton.py @@ -18,7 +18,7 @@ # Some dependencies are listed in CRAN that are actually in Bioconductor. Use # this dict to map these to bioconductor package names. INVALID_NAME_MAP = { - 'r-edger': 'bioconductor-edger', + "r-edger": "bioconductor-edger", } # Raw strings needed to support the awkward backslashes needed when adding the @@ -28,75 +28,84 @@ license_family: GPL2 license_file: '{{ environ["PREFIX"] }}/lib/R/share/licenses/GPL-2' # [unix] license_file: '{{ environ["PREFIX"] }}\\R\\share\\licenses\\GPL-2' # [win] -""".strip('\n') +""".strip( + "\n" +) gpl3_short = r" license_family: GPL3" gpl3_long = r""" license_family: GPL3 license_file: '{{ environ["PREFIX"] }}/lib/R/share/licenses/GPL-3' # [unix] license_file: '{{ environ["PREFIX"] }}\\R\\share\\licenses\\GPL-3' # [win] -""".strip('\n') +""".strip( + "\n" +) -win32_string = 'number: 0\n skip: true # [win32]' +win32_string = "number: 0\n skip: true # [win32]" -def write_recipe(package, recipe_dir='.', recursive=False, force=False, - no_windows=False, **kwargs): - """ - Call out to to ``conda skeleton cran``. +def write_recipe( + package, recipe_dir=".", recursive=False, force=False, no_windows=False, **kwargs +): + """ + Call out to to ``conda skeleton cran``. - Kwargs are accepted for uniformity with - `bioconductor_skeleton.write_recipe`; the only one handled here is - ``recursive``. + Kwargs are accepted for uniformity with + `bioconductor_skeleton.write_recipe`; the only one handled here is + ``recursive``. - Parameters - ---------- + Parameters + ---------- - package : str - Package name. Can be case-sensitive CRAN name, or sanitized - "r-pkgname" conda package name. + package : str + Package name. Can be case-sensitive CRAN name, or sanitized + "r-pkgname" conda package name. - recipe_dir : str - Recipe will be created as a subdirectory in ``recipe_dir`` + recipe_dir : str + Recipe will be created as a subdirectory in ``recipe_dir`` - recursive : bool - Add the ``--recursive`` argument to ``conda skeleton cran`` to - recursively build CRAN recipes. + recursive : bool + Add the ``--recursive`` argument to ``conda skeleton cran`` to + recursively build CRAN recipes. - force : bool - If True, then remove the directory ``/``, where - ```` the sanitized conda version of the package name, - regardless of which format was provided as ``package``. + force : bool + If True, then remove the directory ``/``, where + ```` the sanitized conda version of the package name, + regardless of which format was provided as ``package``. - no_windows : bool - If True, then after creating the skeleton the files are then - cleaned of any Windows-related lines and the bld.bat file is - removed from the recipe. - """ - logger.debug('Building skeleton for %s', package) - conda_version = package.startswith('r-') - if not conda_version: - outdir = os.path.join( - recipe_dir, 'r-' + package.lower()) + no_windows : bool + If True, then after creating the skeleton the files are then + cleaned of any Windows-related lines and the bld.bat file is + removed from the recipe. + """ + logger.debug("Building skeleton for %s", package) + conda_version = package.startswith("r-") + if not conda_version: + outdir = os.path.join(recipe_dir, "r-" + package.lower()) + else: + outdir = os.path.join(recipe_dir, package) + if os.path.exists(outdir): + if force: + logger.warning("Removing %s", outdir) + run(["rm", "-r", outdir], mask=False) else: - outdir = os.path.join( - recipe_dir, package) - if os.path.exists(outdir): - if force: - logger.warning('Removing %s', outdir) - run(['rm', '-r', outdir], mask=False) - else: - logger.warning('%s exists, skipping', outdir) - return - - try: - skeletonize( - package, repo='cran', output_dir=recipe_dir, version=None, recursive=recursive) - clean_skeleton_files( - package=os.path.join(recipe_dir, 'r-' + package.lower()), - no_windows=no_windows) - except NotImplementedError as e: - logger.error('%s had dependencies that specified versions: skipping.', package) + logger.warning("%s exists, skipping", outdir) + return + + try: + skeletonize( + package, + repo="cran", + output_dir=recipe_dir, + version=None, + recursive=recursive, + ) + clean_skeleton_files( + package=os.path.join(recipe_dir, "r-" + package.lower()), + no_windows=no_windows, + ) + except NotImplementedError as e: + logger.error("%s had dependencies that specified versions: skipping.", package) def clean_skeleton_files(package, no_windows=True): @@ -133,19 +142,19 @@ def clean_yaml_file(package, no_windows): If True, then adds a "build: skip: True # [win32]" line to skip Windows builds. """ - path = os.path.join(package, 'meta.yaml') - with open(path, 'r') as yaml: + path = os.path.join(package, "meta.yaml") + with open(path, "r") as yaml: lines = list(yaml.readlines()) # Remove lines consisting only of comments - lines = filter_lines_regex(lines, r'^\s*#.*$', '') + lines = filter_lines_regex(lines, r"^\s*#.*$", "") lines = remove_empty_lines(lines) # Remove file license - lines = filter_lines_regex(lines, r' [+|] file LICEN[SC]E', '') + lines = filter_lines_regex(lines, r" [+|] file LICEN[SC]E", "") # Remove file name lines, which aren't needed as of conda-build3 - lines = filter_lines_regex(lines, r'^\s+fn:\s*.*$', '') + lines = filter_lines_regex(lines, r"^\s+fn:\s*.*$", "") # Replace GPL2 or GPL3 string created by conda skeleton cran with long # format @@ -155,14 +164,14 @@ def clean_yaml_file(package, no_windows): if no_windows: # Inserts `skip: true # [win32]` after `number: 0` to skip windows # builds - lines = filter_lines_regex(lines, r'number: 0', win32_string) + lines = filter_lines_regex(lines, r"number: 0", win32_string) # Add contents of maintainers file to the end of the recipe add_maintainers(lines) - with open(path, 'w') as yaml: + with open(path, "w") as yaml: out = "".join(lines) - out = out.replace('{indent}', '\n - ') + out = out.replace("{indent}", "\n - ") # Edit INVALID_NAME_MAP if additional fixes needed for wrong, correct in INVALID_NAME_MAP.items(): @@ -185,22 +194,22 @@ def clean_build_file(package, no_windows=False): any effect for this function. """ - path = os.path.join(package, 'build.sh') - with open(path, 'r') as build: + path = os.path.join(package, "build.sh") + with open(path, "r") as build: lines = list(build.readlines()) # Remove lines with mv commands - lines = filter_lines_regex(lines, r'^mv\s.*$', '') + lines = filter_lines_regex(lines, r"^mv\s.*$", "") # Remove lines with grep commands - lines = filter_lines_regex(lines, r'^grep\s.*$', '') + lines = filter_lines_regex(lines, r"^grep\s.*$", "") # Removes the lines consisting of only comments - lines = filter_lines_regex(lines, r'^\s*#.*$', '') + lines = filter_lines_regex(lines, r"^\s*#.*$", "") lines = remove_empty_lines(lines) - with open(path, 'w') as build: + with open(path, "w") as build: build.write("".join(lines)) @@ -217,20 +226,20 @@ def clean_bld_file(package, no_windows): no_windows : bool If True, then the bld.bat file will be removed. """ - path = os.path.join(package, 'bld.bat') + path = os.path.join(package, "bld.bat") if not os.path.exists(path): return if no_windows: os.unlink(path) return - with open(path, 'r') as bld: + with open(path, "r") as bld: lines = list(bld.readlines()) # Removes the lines that start with @ - lines = filter_lines_regex(lines, r'^@.*$', '') + lines = filter_lines_regex(lines, r"^@.*$", "") lines = remove_empty_lines(lines) - with open(path, 'w') as bld: + with open(path, "w") as bld: bld.write("".join(lines)) @@ -260,9 +269,8 @@ def remove_empty_lines(lines): """ cleaned_lines = [] for line, next_line in zip_longest(lines, lines[1:]): - if ( - (line.isspace() and next_line is None) or - (line.isspace() and next_line.isspace()) + if (line.isspace() and next_line is None) or ( + line.isspace() and next_line.isspace() ): pass else: @@ -278,27 +286,34 @@ def add_maintainers(lines): Append the contents of "maintainers.yaml" to the end of a YAML file. """ HERE = os.path.abspath(os.path.dirname(__file__)) - maintainers_yaml = os.path.join(HERE, 'maintainers.yaml') - with open(maintainers_yaml, 'r') as yaml: + maintainers_yaml = os.path.join(HERE, "maintainers.yaml") + with open(maintainers_yaml, "r") as yaml: extra_lines = list(yaml.readlines()) lines.extend(extra_lines) def main(): - """ Adding support for arguments here """ + """Adding support for arguments here""" setup_logger() parser = argparse.ArgumentParser() - parser.add_argument('package', help='name of the cran package') - parser.add_argument('output_dir', help='output directory for the recipe') - parser.add_argument('--no-win', action="store_true", - help='runs the skeleton and removes windows specific information') - parser.add_argument('--force', action='store_true', - help='If a directory exists for any recipe, overwrite it') + parser.add_argument("package", help="name of the cran package") + parser.add_argument("output_dir", help="output directory for the recipe") + parser.add_argument( + "--no-win", + action="store_true", + help="runs the skeleton and removes windows specific information", + ) + parser.add_argument( + "--force", + action="store_true", + help="If a directory exists for any recipe, overwrite it", + ) args = parser.parse_args() - write_recipe(args.package, args.output_dir, no_windows=args.no_win, - force=args.force) + write_recipe( + args.package, args.output_dir, no_windows=args.no_win, force=args.force + ) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/bioconda_utils/docker_utils.py b/bioconda_utils/docker_utils.py index a43d864646..020bff8b6c 100644 --- a/bioconda_utils/docker_utils.py +++ b/bioconda_utils/docker_utils.py @@ -64,6 +64,7 @@ from . import __version__ import logging + logger = logging.getLogger(__name__) @@ -80,8 +81,7 @@ # can add additional attributes to the RecipeBuilder instance and have them # filled in here. # -BUILD_SCRIPT_TEMPLATE = \ -""" +BUILD_SCRIPT_TEMPLATE = """ #!/bin/bash set -eo pipefail @@ -125,8 +125,7 @@ # The default image is created automatically for releases using the Dockerfile # in the bioconda-utils repo. -DOCKERFILE_TEMPLATE = \ -r""" +DOCKERFILE_TEMPLATE = r""" FROM {docker_base_image} {proxies} RUN \ @@ -148,7 +147,6 @@ class DockerBuildError(Exception): pass - def get_host_conda_bld(): """ Identifies the conda-bld directory on the host. @@ -164,8 +162,8 @@ def get_host_conda_bld(): class RecipeBuilder(object): def __init__( self, - tag='tmp-bioconda-builder', - container_recipe='/opt/recipe', + tag="tmp-bioconda-builder", + container_recipe="/opt/recipe", container_staging="/opt/host-conda-bld", requirements=None, build_script_template=BUILD_SCRIPT_TEMPLATE, @@ -175,7 +173,9 @@ def __init__( keep_image=False, build_image=False, image_build_dir=None, - docker_base_image='quay.io/bioconda/bioconda-utils-build-env-cos7:{}'.format(__version__.replace('+', '_')) + docker_base_image="quay.io/bioconda/bioconda-utils-build-env-cos7:{}".format( + __version__.replace("+", "_") + ), ): """ Class to handle building a custom docker container that can be used for @@ -276,7 +276,8 @@ def __init__( uid=uid, gid=usr.pw_gid, groupname=grp.getgrgid(usr.pw_gid).gr_name, - username=usr.pw_name) + username=usr.pw_name, + ) self.container_recipe = container_recipe self.container_staging = container_staging @@ -305,7 +306,9 @@ def __init__( def _get_config_path(self, staging_prefix, i, config_file): src_basename = os.path.basename(config_file.path) - dst_basename = 'conda_build_config_{}_{}_{}'.format(i, config_file.arg, src_basename) + dst_basename = "conda_build_config_{}_{}_{}".format( + i, config_file.arg, src_basename + ) return os.path.join(staging_prefix, dst_basename) def __del__(self): @@ -313,11 +316,10 @@ def __del__(self): def _find_proxy_settings(self): res = {} - for var in ('http_proxy', 'https_proxy'): - values = set([ - os.environ.get(var, None), - os.environ.get(var.upper(), None) - ]).difference([None]) + for var in ("http_proxy", "https_proxy"): + values = set( + [os.environ.get(var, None), os.environ.get(var.upper(), None)] + ).difference([None]) if len(values) == 1: res[var] = next(iter(values)) elif len(values) > 1: @@ -336,28 +338,36 @@ def _build_image(self): else: build_dir = self.image_build_dir - logger.info('DOCKER: Building image "%s" from %s', self.docker_temp_image, build_dir) - with open(os.path.join(build_dir, 'requirements.txt'), 'w') as fout: + logger.info( + 'DOCKER: Building image "%s" from %s', self.docker_temp_image, build_dir + ) + with open(os.path.join(build_dir, "requirements.txt"), "w") as fout: if self.requirements: fout.write(open(self.requirements).read()) else: - fout.write(open(pkg_resources.resource_filename( - 'bioconda_utils', - 'bioconda_utils-requirements.txt') - ).read()) - - proxies = "\n".join("ENV {} {}".format(k, v) - for k, v in self._find_proxy_settings()) - - with open(os.path.join(build_dir, "Dockerfile"), 'w') as fout: - fout.write(self.dockerfile_template.format( - docker_base_image=self.docker_base_image, - proxies=proxies, - conda_ver=conda.__version__, - conda_build_ver=conda_build.__version__) + fout.write( + open( + pkg_resources.resource_filename( + "bioconda_utils", "bioconda_utils-requirements.txt" + ) + ).read() + ) + + proxies = "\n".join( + "ENV {} {}".format(k, v) for k, v in self._find_proxy_settings() + ) + + with open(os.path.join(build_dir, "Dockerfile"), "w") as fout: + fout.write( + self.dockerfile_template.format( + docker_base_image=self.docker_base_image, + proxies=proxies, + conda_ver=conda.__version__, + conda_build_ver=conda_build.__version__, + ) ) - logger.debug('Dockerfile:\n' + open(fout.name).read()) + logger.debug("Dockerfile:\n" + open(fout.name).read()) # Check if the installed version of docker supports the --network flag # (requires version >= 1.13.0) @@ -368,39 +378,43 @@ def _build_image(self): try: s = sp.check_output(["docker", "--version"]).decode() except FileNotFoundError: - logger.error('DOCKER FAILED: Error checking docker version, is it installed?') + logger.error( + "DOCKER FAILED: Error checking docker version, is it installed?" + ) raise except sp.CalledProcessError: - logger.error('DOCKER FAILED: Error checking docker version.') + logger.error("DOCKER FAILED: Error checking docker version.") raise - p = re.compile(r"\d+\.\d+\.\d+") # three groups of at least on digit separated by dots + p = re.compile( + r"\d+\.\d+\.\d+" + ) # three groups of at least on digit separated by dots version_string = re.search(p, s).group(0) if LooseVersion(version_string) >= LooseVersion("1.13.0"): cmd = [ - 'docker', 'build', - # xref #5027 - '--network', 'host', - '-t', self.docker_temp_image, - build_dir + "docker", + "build", + # xref #5027 + "--network", + "host", + "-t", + self.docker_temp_image, + build_dir, ] else: # Network flag was added in 1.13.0, do not add it for lower versions. xref #5387 - cmd = [ - 'docker', 'build', - '-t', self.docker_temp_image, - build_dir - ] + cmd = ["docker", "build", "-t", self.docker_temp_image, build_dir] try: with utils.Progress(): p = utils.run(cmd, mask=False) except sp.CalledProcessError as e: logger.error( - 'DOCKER FAILED: Error building docker container %s. ', - self.docker_temp_image) + "DOCKER FAILED: Error building docker container %s. ", + self.docker_temp_image, + ) raise e - logger.info('DOCKER: Built docker image tag=%s', self.docker_temp_image) + logger.info("DOCKER: Built docker image tag=%s", self.docker_temp_image) if self.image_build_dir is None: shutil.rmtree(build_dir) return p @@ -433,61 +447,70 @@ def build_recipe(self, recipe_dir, build_args, env, noarch=False): # Attach the build args to self so that it can be filled in by the # template. if not isinstance(build_args, str): - raise ValueError('build_args must be str') + raise ValueError("build_args must be str") build_args_list = [build_args] for i, config_file in enumerate(utils.get_conda_build_config_files()): dst_file = self._get_config_path(self.container_staging, i, config_file) build_args_list.extend([config_file.arg, quote(dst_file)]) - self.conda_build_args = ' '.join(build_args_list) + self.conda_build_args = " ".join(build_args_list) # Write build script to tempfile build_dir = os.path.realpath(tempfile.mkdtemp()) script = self.build_script_template.format( - self=self, arch='noarch' if noarch else 'linux-64') - with open(os.path.join(build_dir, 'build_script.bash'), 'w') as fout: + self=self, arch="noarch" if noarch else "linux-64" + ) + with open(os.path.join(build_dir, "build_script.bash"), "w") as fout: fout.write(script) build_script = fout.name - logger.debug('DOCKER: Container build script: \n%s', open(fout.name).read()) + logger.debug("DOCKER: Container build script: \n%s", open(fout.name).read()) # Build the args for env vars. Note can also write these to tempfile # and use --env-file arg, but using -e seems clearer in debug output. env_list = [] for k, v in env.items(): - env_list.append('-e') - env_list.append('{0}={1}'.format(k, v)) + env_list.append("-e") + env_list.append("{0}={1}".format(k, v)) - env_list.append('-e') - env_list.append('{0}={1}'.format('HOST_USER_ID', self.user_info['uid'])) + env_list.append("-e") + env_list.append("{0}={1}".format("HOST_USER_ID", self.user_info["uid"])) cmd = [ - 'docker', 'run', '-t', - '--net', 'host', - '--rm', - '-v', '{0}:/opt/build_script.bash'.format(build_script), - '-v', '{0}:{1}'.format(self.pkg_dir, self.container_staging), - '-v', '{0}:{1}'.format(recipe_dir, self.container_recipe), + "docker", + "run", + "-t", + "--net", + "host", + "--rm", + "-v", + "{0}:/opt/build_script.bash".format(build_script), + "-v", + "{0}:{1}".format(self.pkg_dir, self.container_staging), + "-v", + "{0}:{1}".format(recipe_dir, self.container_recipe), ] cmd += env_list if self.build_image: cmd += [self.docker_temp_image] else: cmd += [self.docker_base_image] - cmd += ['/bin/bash', '/opt/build_script.bash'] + cmd += ["/bin/bash", "/opt/build_script.bash"] - logger.debug('DOCKER: cmd: %s', cmd) + logger.debug("DOCKER: cmd: %s", cmd) with utils.Progress(): p = utils.run(cmd, mask=False) return p def cleanup(self): if self.build_image and not self.keep_image: - cmd = ['docker', 'rmi', self.docker_temp_image] + cmd = ["docker", "rmi", self.docker_temp_image] utils.run(cmd, mask=False) def purgeImage(mulled_upload_target, img): pkg_name_and_version, pkg_build_string = img.rsplit("--", 1) pkg_name, pkg_version = pkg_name_and_version.rsplit("=", 1) - pkg_container_image = f"quay.io/{mulled_upload_target}/{pkg_name}:{pkg_version}--{pkg_build_string}" - cmd = ['docker', 'rmi', pkg_container_image] + pkg_container_image = ( + f"quay.io/{mulled_upload_target}/{pkg_name}:{pkg_version}--{pkg_build_string}" + ) + cmd = ["docker", "rmi", pkg_container_image] o = utils.run(cmd, mask=False) diff --git a/bioconda_utils/githandler.py b/bioconda_utils/githandler.py index b5137c89cc..f53ba9703d 100644 --- a/bioconda_utils/githandler.py +++ b/bioconda_utils/githandler.py @@ -30,21 +30,23 @@ def install_gpg_key(key) -> str: Raises: ValueError if importing the key failed """ - proc = subprocess.run(['gpg', '--import'], - input=key, stderr=subprocess.PIPE, - encoding='ascii') + proc = subprocess.run( + ["gpg", "--import"], input=key, stderr=subprocess.PIPE, encoding="ascii" + ) for line in proc.stderr.splitlines(): - match = re.match(r'gpg: key ([\dA-F]{8,16}): ' - r'(secret key imported|already in secret keyring)', - line) + match = re.match( + r"gpg: key ([\dA-F]{8,16}): " + r"(secret key imported|already in secret keyring)", + line, + ) if match: keyid = match.group(1) break else: # If the key has escaped newlines (\n literally), replace those # and try again - if r'\n' in key: - return install_gpg_key(key.replace(r'\n', '\n')) + if r"\n" in key: + return install_gpg_key(key.replace(r"\n", "\n")) raise ValueError(f"Unable to import GPG key: {proc.stderr}") return keyid @@ -53,7 +55,7 @@ class GitHandlerFailure(Exception): """Something went wrong interacting with git""" -class GitHandlerBase(): +class GitHandlerBase: """GitPython abstraction We have to work with three git repositories, the local checkout, @@ -68,11 +70,15 @@ class GitHandlerBase(): fork: string occurring in remote url marking forked repo allow_dirty: don't bail out if repo is dirty """ - def __init__(self, repo: git.Repo, - dry_run: bool, - home='bioconda/bioconda-recipes', - fork=None, - allow_dirty=False) -> None: + + def __init__( + self, + repo: git.Repo, + dry_run: bool, + home="bioconda/bioconda-recipes", + fork=None, + allow_dirty=False, + ) -> None: #: GitPython Repo object representing our repository self.repo: git.Repo = repo if not allow_dirty and self.repo.is_dirty(): @@ -103,7 +109,8 @@ def close(self): def __str__(self): def get_name(remote): url = next(remote.urls) - return url[url.rfind('/', 0, url.rfind('/'))+1:] + return url[url.rfind("/", 0, url.rfind("/")) + 1 :] + name = get_name(self.home_remote) if self.fork_remote != self.home_remote: name = f"{name} <- {get_name(self.fork_remote)}" @@ -131,13 +138,14 @@ def get_remote(self, desc: str): if section.startswith("url "): new = section.lstrip("url ").strip('"') try: - old = reader.get(section, 'insteadOf') + old = reader.get(section, "insteadOf") desc = desc.replace(old, new) except KeyError: pass # now try if any remote matches the url - remotes = [r for r in self.repo.remotes - if any(filter(lambda x: desc in x, r.urls))] + remotes = [ + r for r in self.repo.remotes if any(filter(lambda x: desc in x, r.urls)) + ] if not remotes: raise KeyError(f"No remote matching '{desc}' found") @@ -150,11 +158,18 @@ async def branch_is_current(self, branch, path: str, master="master") -> bool: """Checks if **branch** has the most recent commit modifying **path** as compared to **master**""" proc = await asyncio.create_subprocess_exec( - 'git', 'log', '-1', '--oneline', '--decorate', - f'{master}...{branch.name}', '--', path, - stdout=asyncio.subprocess.PIPE) + "git", + "log", + "-1", + "--oneline", + "--decorate", + f"{master}...{branch.name}", + "--", + path, + stdout=asyncio.subprocess.PIPE, + ) stdout, _ = await proc.communicate() - return branch.name in stdout.decode('ascii') + return branch.name in stdout.decode("ascii") def delete_local_branch(self, branch) -> None: """Deletes **branch** locally""" @@ -221,24 +236,23 @@ def get_remote_branch(self, branch_name: str, try_fetch=False): return remote_ref.ref def get_latest_master(self): - return self.home_remote.fetch('master')[0].commit + return self.home_remote.fetch("master")[0].commit def read_from_branch(self, branch, file_name: str) -> str: """Reads contents of file **file_name** from git branch **branch**""" abs_file_name = os.path.abspath(file_name) abs_repo_root = os.path.abspath(self.repo.working_dir) if not abs_file_name.startswith(abs_repo_root): - raise RuntimeError( - f"File {abs_file_name} not inside {abs_repo_root}" - ) - rel_file_name = abs_file_name[len(abs_repo_root):].lstrip("/") - commit = getattr(branch, 'commit', branch) + raise RuntimeError(f"File {abs_file_name} not inside {abs_repo_root}") + rel_file_name = abs_file_name[len(abs_repo_root) :].lstrip("/") + commit = getattr(branch, "commit", branch) blob = commit.tree / rel_file_name if blob: return blob.data_stream.read().decode("utf-8") - logger.error("File %s not found on branch %s commit %s", - rel_file_name, branch, commit) + logger.error( + "File %s not found on branch %s commit %s", rel_file_name, branch, commit + ) return None def create_local_branch(self, branch_name: str, remote_branch: str = None): @@ -281,17 +295,20 @@ def get_merge_base(self, ref=None, other=None, try_fetch=False): for depth in depths: if depth: self.fork_remote.fetch(ref, depth=depth) - self.home_remote.fetch('master', depth=depth) + self.home_remote.fetch("master", depth=depth) merge_bases = self.repo.merge_base(other, ref) if merge_bases: break - logger.debug("No merge base found for %s and master at depth %i", ref, depth) + logger.debug( + "No merge base found for %s and master at depth %i", ref, depth + ) else: logger.error("No merge base found for %s and master", ref) - return None # FIXME: This should raise + return None # FIXME: This should raise if len(merge_bases) > 1: - logger.error("Multiple merge bases found for %s and master: %s", - ref, merge_bases) + logger.error( + "Multiple merge bases found for %s and master: %s", ref, merge_bases + ) return merge_bases[0] def list_changed_files(self, ref=None, other=None): @@ -332,8 +349,9 @@ def prepare_branch(self, branch_name: str) -> None: branch = self.repo.heads[branch_name] branch.checkout() - def commit_and_push_changes(self, files: List[str], branch_name: str, - msg: str, sign=False) -> bool: + def commit_and_push_changes( + self, files: List[str], branch_name: str, msg: str, sign=False + ) -> bool: """Create recipe commit and pushes to upstream remote Returns: @@ -352,11 +370,12 @@ def commit_and_push_changes(self, files: List[str], branch_name: str, if sign: # Gitpyhon does not support signing, so we use the command line client here args = [ - '-S' + sign if isinstance(sign, str) else '-S', - '-m', msg, + "-S" + sign if isinstance(sign, str) else "-S", + "-m", + msg, ] if self.actor: - args += ['--author', f'{self.actor.name} <{self.actor.email}>'] + args += ["--author", f"{self.actor.name} <{self.actor.email}>"] self.repo.index.write() self.repo.git.commit(*args) else: @@ -369,7 +388,9 @@ def commit_and_push_changes(self, files: List[str], branch_name: str, logger.info("Pushing branch %s", branch_name) try: res = self.fork_remote.push(branch_name) - failed = res[0].flags & ~(git.PushInfo.FAST_FORWARD | git.PushInfo.NEW_HEAD) + failed = res[0].flags & ~( + git.PushInfo.FAST_FORWARD | git.PushInfo.NEW_HEAD + ) text = res[0].summary except git.GitCommandError as exc: failed = True @@ -409,7 +430,7 @@ def get_changed_recipes(self, ref=None, other=None, files=None): ``recipes_folder`` are ignored. """ if files is None: - files = ['meta.yaml', 'build.sh'] + files = ["meta.yaml", "build.sh"] changed = set() for path in self.list_changed_files(ref, other): if not path.startswith(self.recipes_folder): @@ -436,7 +457,7 @@ def get_blacklisted(self, ref=None): branch = ref config_data = self.read_from_branch(branch, self.config_file) config = yaml.safe_load(config_data) - blacklists = config['blacklists'] + blacklists = config["blacklists"] blacklisted = set() for blacklist in blacklists: blacklist_data = self.read_from_branch(branch, blacklist) @@ -472,10 +493,13 @@ def get_recipes_to_build(self, ref=None, other=None): `list` of recipes that should be built """ tobuild = set(self.get_changed_recipes(ref, other)) - tobuild.update([recipe - for recipe in self.get_unblacklisted(ref, other) - if recipe.startswith(self.recipes_folder) - and os.path.exists(recipe)]) + tobuild.update( + [ + recipe + for recipe in self.get_unblacklisted(ref, other) + if recipe.startswith(self.recipes_folder) and os.path.exists(recipe) + ] + ) return list(tobuild) @@ -484,12 +508,16 @@ class GitHandler(GitHandlerBase): Restores the branch active when created upon calling `close()`. """ - def __init__(self, folder: str=".", - dry_run=False, - home='bioconda/bioconda-recipes', - fork=None, - allow_dirty=True, - depth=1) -> None: + + def __init__( + self, + folder: str = ".", + dry_run=False, + home="bioconda/bioconda-recipes", + fork=None, + allow_dirty=True, + depth=1, + ) -> None: if os.path.exists(folder): repo = git.Repo(folder, search_parent_directories=True) else: @@ -507,7 +535,9 @@ def __init__(self, folder: str=".", self.prev_active_branch = self.repo.active_branch except: # This will fail on CI nodes from forks, but we don't need to switch back and forth between branches there - logger.warning("Couldn't get the active branch name, we must be on detached HEAD") + logger.warning( + "Couldn't get the active branch name, we must be on detached HEAD" + ) pass def checkout_master(self): @@ -574,8 +604,8 @@ def _get_local_mirror(cls, url: str) -> git.Repo: atexit.register(cls._local_mirror_tmpdir.cleanup) # Make location of repo in tmpdir from url - _, _, fname = url.rpartition('@') - tmpname = getattr(cls._local_mirror_tmpdir, 'name', cls._local_mirror_tmpdir) + _, _, fname = url.rpartition("@") + tmpname = getattr(cls._local_mirror_tmpdir, "name", cls._local_mirror_tmpdir) mirror_name = os.path.join(tmpname, fname) # Re-use or create mirror of remote repo @@ -588,35 +618,37 @@ def _get_local_mirror(cls, url: str) -> git.Repo: # Update the remote url, in case password changed logger.info("Updating Bare Mirror %s", fname) - m_origin = mirror.remote('origin') + m_origin = mirror.remote("origin") m_origin.set_url(url, next(m_origin.urls)) logger.info("Updating Bare Mirror %s -- DONE", fname) # Update the remote repo - mirror.remote('origin').update() + mirror.remote("origin").update() return mirror @classmethod def _clone_with_mirror(cls, home_url, todir): """Prepares a clone of **home_url** in **todir** using mirror cache""" repo = cls._get_local_mirror(home_url).clone(todir) - r_origin = repo.remote('origin') + r_origin = repo.remote("origin") r_origin.set_url(home_url, next(r_origin.urls)) - _, _, fname = home_url.rpartition('@') + _, _, fname = home_url.rpartition("@") logger.info("Fetching %s", fname) r_origin.fetch() logger.info("Fetching %s - DONE", fname) return repo - def __init__(self, - username: str = None, - password: str = None, - url_format="https://{userpass}github.com/{user}/{repo}.git", - home_user="bioconda", - home_repo="bioconda-recipes", - fork_user=None, - fork_repo=None, - dry_run=False) -> None: + def __init__( + self, + username: str = None, + password: str = None, + url_format="https://{userpass}github.com/{user}/{repo}.git", + home_user="bioconda", + home_repo="bioconda-recipes", + fork_user=None, + fork_repo=None, + dry_run=False, + ) -> None: userpass = "" if password is not None and username is None: username = "x-access-token" @@ -633,14 +665,15 @@ def censor(string): return string return string.replace(password, "******") - home_url = url_format.format(userpass=userpass, - user=home_user, repo=home_repo) + home_url = url_format.format(userpass=userpass, user=home_user, repo=home_repo) logger.info("Cloning %s to %s", censor(home_url), self.tempdir.name) repo = self._clone_with_mirror(home_url, self.tempdir.name) if fork_repo is not None: - fork_url = url_format.format(userpass=userpass, user=fork_user, repo=fork_repo) + fork_url = url_format.format( + userpass=userpass, user=fork_user, repo=fork_repo + ) if fork_url != home_url: logger.warning("Adding remote fork %s", censor(fork_url)) fork_remote = repo.create_remote("fork", fork_url) @@ -650,7 +683,6 @@ def censor(string): logger.info("Finished setting up repo in %s", self.tempdir) super().__init__(repo, dry_run, home_url, fork_url) - def close(self) -> None: """Remove temporary clone and cleanup resources""" super().close() @@ -661,5 +693,6 @@ def close(self) -> None: class BiocondaRepo(GitHandler, BiocondaRepoMixin): pass + class TempBiocondaRepo(TempGitHandler, BiocondaRepoMixin): pass diff --git a/bioconda_utils/githubhandler.py b/bioconda_utils/githubhandler.py index 278e476ed2..d20b6a9446 100644 --- a/bioconda_utils/githubhandler.py +++ b/bioconda_utils/githubhandler.py @@ -31,23 +31,27 @@ CheckRunStatus = Enum("CheckRunState", "queued in_progress completed") #: Conclusion of Github Check Run -CheckRunConclusion = Enum("CheckRunConclusion", - "success failure neutral cancelled timed_out action_required") +CheckRunConclusion = Enum( + "CheckRunConclusion", "success failure neutral cancelled timed_out action_required" +) #: Merge method MergeMethod = Enum("MergeMethod", "merge squash rebase") #: Pull request review state -ReviewState = Enum("ReviewState", "APPROVED CHANGES_REQUESTED COMMENTED DISMISSED PENDING") +ReviewState = Enum( + "ReviewState", "APPROVED CHANGES_REQUESTED COMMENTED DISMISSED PENDING" +) #: ContentType for Project Cards CardContentType = Enum("CardContentType", "Issue PullRequest") + def iso_now() -> str: """Creates ISO 8601 timestamp in format ``YYYY-MM-DDTHH:MM:SSZ`` as required by Github """ - return datetime.datetime.utcnow().replace(microsecond=0).isoformat()+'Z' + return datetime.datetime.utcnow().replace(microsecond=0).isoformat() + "Z" class GitHubHandler: @@ -59,45 +63,49 @@ class GitHubHandler: to_user: Target User/Org for PRs to_repo: Target repository within **to_user** """ - USER = "/user" - USER_APPS = "/user/installations" - USER_ORGS = "/user/orgs" - USER_TEAMS = "/user/teams" - - PULLS = "/repos/{user}/{repo}/pulls{/number}{?head,base,state}" - PULL_FILES = "/repos/{user}/{repo}/pulls/{number}/files" - PULL_COMMITS = "/repos/{user}/{repo}/pulls/{number}/commits" - PULL_MERGE = "/repos/{user}/{repo}/pulls/{number}/merge" - PULL_REVIEWS = "/repos/{user}/{repo}/pulls/{number}/reviews{/review_id}" - PULL_UPDATE = "/repos/{user}/{repo}/pulls/{number}/update-branch" + + USER = "/user" + USER_APPS = "/user/installations" + USER_ORGS = "/user/orgs" + USER_TEAMS = "/user/teams" + + PULLS = "/repos/{user}/{repo}/pulls{/number}{?head,base,state}" + PULL_FILES = "/repos/{user}/{repo}/pulls/{number}/files" + PULL_COMMITS = "/repos/{user}/{repo}/pulls/{number}/commits" + PULL_MERGE = "/repos/{user}/{repo}/pulls/{number}/merge" + PULL_REVIEWS = "/repos/{user}/{repo}/pulls/{number}/reviews{/review_id}" + PULL_UPDATE = "/repos/{user}/{repo}/pulls/{number}/update-branch" BRANCH_PROTECTION = "/repos/{user}/{repo}/branches/{branch}/protection" - ISSUES = "/repos/{user}/{repo}/issues{/number}" - ISSUE_COMMENTS = "/repos/{user}/{repo}/issues/{number}/comments" - COMMENTS = "/repos/{user}/{repo}/issues/comments{/comment_id}" - CHECK_RUN = "/repos/{user}/{repo}/check-runs{/id}" - GET_CHECK_RUNS = "/repos/{user}/{repo}/commits/{commit}/check-runs" - GET_STATUSES = "/repos/{user}/{repo}/commits/{commit}/statuses" - CONTENTS = "/repos/{user}/{repo}/contents/{path}{?ref}" - GIT_REFERENCE = "/repos/{user}/{repo}/git/refs/{ref}" - ORG_MEMBERS = "/orgs/{user}/members{/username}" - ORG = "/orgs/{user}" - ORG_TEAMS = "/orgs/{user}/teams{/team_slug}" - - PROJECTS_BY_REPO = "/repos/{user}/{repo}/projects" + ISSUES = "/repos/{user}/{repo}/issues{/number}" + ISSUE_COMMENTS = "/repos/{user}/{repo}/issues/{number}/comments" + COMMENTS = "/repos/{user}/{repo}/issues/comments{/comment_id}" + CHECK_RUN = "/repos/{user}/{repo}/check-runs{/id}" + GET_CHECK_RUNS = "/repos/{user}/{repo}/commits/{commit}/check-runs" + GET_STATUSES = "/repos/{user}/{repo}/commits/{commit}/statuses" + CONTENTS = "/repos/{user}/{repo}/contents/{path}{?ref}" + GIT_REFERENCE = "/repos/{user}/{repo}/git/refs/{ref}" + ORG_MEMBERS = "/orgs/{user}/members{/username}" + ORG = "/orgs/{user}" + ORG_TEAMS = "/orgs/{user}/teams{/team_slug}" + + PROJECTS_BY_REPO = "/repos/{user}/{repo}/projects" PROJECT_COL_CARDS = "/projects/columns/{column_id}/cards" - PROJECT_CARDS = "/projects/columns/cards/{card_id}" + PROJECT_CARDS = "/projects/columns/cards/{card_id}" - TEAMS_MEMBERSHIP = "/teams/{team_id}/memberships/{username}" + TEAMS_MEMBERSHIP = "/teams/{team_id}/memberships/{username}" - SEARCH_ISSUES = "/search/issues?q=" + SEARCH_ISSUES = "/search/issues?q=" STATE = IssueState - def __init__(self, token: str = None, - dry_run: bool = False, - to_user: str = "bioconda", - to_repo: str = "bioconda-recipes", - installation: int = None) -> None: + def __init__( + self, + token: str = None, + dry_run: bool = False, + to_user: str = "bioconda", + to_repo: str = "bioconda-recipes", + installation: int = None, + ) -> None: #: API Bearer Token self.token = token #: If set, no actual modifying actions are taken @@ -109,8 +117,7 @@ def __init__(self, token: str = None, #: Name of the Repo self.repo = to_repo #: Default variables for API calls - self.var_default = {'user': to_user, - 'repo': to_repo} + self.var_default = {"user": to_user, "repo": to_repo} # filled in by login(): #: Gidgethub API object @@ -129,12 +136,12 @@ def __repr__(self): def for_json(self): """Return JSON repesentation of object""" return { - '__module__': self.__module__, - '__type__': self.__class__.__qualname__, - 'dry_run': self.dry_run, - 'to_user': self.user, - 'to_repo': self.repo, - 'installation': self.installation + "__module__": self.__module__, + "__type__": self.__class__.__qualname__, + "dry_run": self.dry_run, + "to_user": self.user, + "to_repo": self.repo, + "installation": self.installation, } def __to_json__(self): @@ -157,7 +164,8 @@ def create_api_object(self, *args, **kwargs): def get_file_relurl(self, path: str, branch_name: str = "master") -> str: """Format domain relative url for **path** on **branch_name**""" return "/{user}/{repo}/tree/{branch_name}/{path}".format( - branch_name=branch_name, path=path, **self.var_default) + branch_name=branch_name, path=path, **self.var_default + ) async def login(self, *args, **kwargs) -> bool: """Log into API (fills `username`)""" @@ -195,7 +203,7 @@ async def get_user_orgs(self) -> List[str]: Empty list if the request failed """ try: - return [org['login'] for org in await self.api.getitem(self.USER_ORGS)] + return [org["login"] for org in await self.api.getitem(self.USER_ORGS)] except gidgethub.GitHubException: return [] @@ -209,8 +217,9 @@ async def iter_teams(self) -> AsyncIterator[Dict[str, Any]]: async for team in self.api.getiter(self.ORG_TEAMS, self.var_default): yield team - async def get_team_id(self, team_slug: str = None, - team_name: str = None) -> Optional[int]: + async def get_team_id( + self, team_slug: str = None, team_name: str = None + ) -> Optional[int]: """Get the Team ID from the Team slug If both are set, **team_slug** is tried first. @@ -224,19 +233,19 @@ async def get_team_id(self, team_slug: str = None, """ if team_slug: var_data = copy(self.var_default) - var_data['team_slug'] = team_slug + var_data["team_slug"] = team_slug try: team = await self.api.getitem(self.ORG_TEAMS, var_data) - if team and 'id' in team: - return team['id'] + if team and "id" in team: + return team["id"] except gidgethub.BadRequest as exc: if exc.status_code != 404: raise if team_name: async for team in self.iter_teams(): - if team.get('name') == team_name: - return team.get('id') + if team.get("name") == team_name: + return team.get("id") async def is_team_member(self, username: str, team: Union[str, int]) -> bool: """Check if user is a member of given team @@ -256,20 +265,24 @@ async def is_team_member(self, username: str, team: Union[str, int]) -> bool: return False var_data = { - 'username': username, - 'team_id': team_id, + "username": username, + "team_id": team_id, } accept = "application/vnd.github.hellcat-preview+json" try: - data = await self.api.getitem(self.TEAMS_MEMBERSHIP, var_data, accept=accept) - if data['state'] == "active": + data = await self.api.getitem( + self.TEAMS_MEMBERSHIP, var_data, accept=accept + ) + if data["state"] == "active": return True except gidgethub.BadRequest as exc: if exc.status_code != 404: raise return False - async def is_member(self, username: str, team: Optional[Union[str, int]] = None) -> bool: + async def is_member( + self, username: str, team: Optional[Union[str, int]] = None + ) -> bool: """Check user membership Args: @@ -282,20 +295,21 @@ async def is_member(self, username: str, team: Optional[Union[str, int]] = None) if not username: return False var_data = copy(self.var_default) - var_data['username'] = username + var_data["username"] = username try: await self.api.getitem(self.ORG_MEMBERS, var_data) except gidgethub.BadRequest: - logger.debug("User %s is not a member of %s", username, var_data['user']) + logger.debug("User %s is not a member of %s", username, var_data["user"]) return False - logger.debug("User %s IS a member of %s", username, var_data['user']) + logger.debug("User %s IS a member of %s", username, var_data["user"]) if team: return await self.is_team_member(username, team) return True - async def search_issues(self, author=None, pr=False, issue=False, sha=None, - closed=None): + async def search_issues( + self, author=None, pr=False, issue=False, sha=None, closed=None + ): """Search issues/PRs on our repos Arguments: @@ -323,7 +337,7 @@ async def search_issues(self, author=None, pr=False, issue=False, sha=None, if sha: query += ["sha:" + sha] - return await self.api.getitem(self.SEARCH_ISSUES + '+'.join(query)) + return await self.api.getitem(self.SEARCH_ISSUES + "+".join(query)) async def get_pr_count(self, user) -> int: """Get the number of PRs opened by user @@ -335,7 +349,7 @@ async def get_pr_count(self, user) -> int: Number of PRs that **user** has opened so far """ result = await self.search_issues(pr=True, author=user) - return result.get('total_count', 0) + return result.get("total_count", 0) async def is_org(self) -> bool: """Check if we are operating on an organization""" @@ -355,25 +369,32 @@ async def get_prs_from_sha(self, head_sha: str, only_open=False) -> List[int]: List of PR numbers. """ pr_numbers = [] - result = await self.search_issues(pr=True, sha=head_sha, - closed=False if only_open else None) - for pull in result.get('items', []): - pr_number = int(pull['number']) + result = await self.search_issues( + pr=True, sha=head_sha, closed=False if only_open else None + ) + for pull in result.get("items", []): + pr_number = int(pull["number"]) logger.error("checking %s", pr_number) full_pr = await self.get_prs(number=pr_number) - if full_pr['head']['sha'].startswith(head_sha): + if full_pr["head"]["sha"].startswith(head_sha): pr_numbers.append(pr_number) return pr_numbers # pylint: disable=too-many-arguments - @backoff.on_exception(backoff.fibo, gidgethub.BadRequest, max_tries=10, - giveup=lambda ex: ex.status_code not in [429, 502, 503, 504]) - async def get_prs(self, - from_branch: Optional[str] = None, - from_user: Optional[str] = None, - to_branch: Optional[str] = None, - number: Optional[int] = None, - state: Optional[IssueState] = None) -> List[Dict[Any, Any]]: + @backoff.on_exception( + backoff.fibo, + gidgethub.BadRequest, + max_tries=10, + giveup=lambda ex: ex.status_code not in [429, 502, 503, 504], + ) + async def get_prs( + self, + from_branch: Optional[str] = None, + from_user: Optional[str] = None, + to_branch: Optional[str] = None, + number: Optional[int] = None, + state: Optional[IssueState] = None, + ) -> List[Dict[Any, Any]]: """Retrieve list of PRs matching parameters Arguments: @@ -388,15 +409,15 @@ async def get_prs(self, from_user = self.username if from_branch: if from_user: - var_data['head'] = f"{from_user}:{from_branch}" + var_data["head"] = f"{from_user}:{from_branch}" else: - var_data['head'] = from_branch + var_data["head"] = from_branch if to_branch: - var_data['base'] = to_branch + var_data["base"] = to_branch if number: - var_data['number'] = str(number) + var_data["number"] = str(number) if state: - var_data['state'] = state.name.lower() + var_data["state"] = state.name.lower() accept = "application/vnd.github.shadow-cat-preview" # for draft try: @@ -418,13 +439,13 @@ async def get_issue(self, number: int) -> Dict[str, Any]: if the Issue is a PR. """ var_data = copy(self.var_default) - var_data['number'] = str(number) + var_data["number"] = str(number) return await self.api.getitem(self.ISSUES, var_data) async def iter_pr_commits(self, number: int): """Create iterator over commits in a PR""" var_data = copy(self.var_default) - var_data['number'] = str(number) + var_data["number"] = str(number) try: async for item in self.api.getiter(self.PULL_COMMITS, var_data): yield item @@ -432,13 +453,16 @@ async def iter_pr_commits(self, number: int): return # pylint: disable=too-many-arguments - async def create_pr(self, title: str, - from_branch: str, - from_user: Optional[str] = None, - to_branch: Optional[str] = "master", - body: Optional[str] = None, - maintainer_can_modify: bool = True, - draft: bool = False) -> Dict[Any, Any]: + async def create_pr( + self, + title: str, + from_branch: str, + from_user: Optional[str] = None, + to_branch: Optional[str] = "master", + body: Optional[str] = None, + maintainer_can_modify: bool = True, + draft: bool = False, + ) -> Dict[Any, Any]: """Create new PR Arguments: @@ -454,17 +478,17 @@ async def create_pr(self, title: str, if not from_user: from_user = self.username data: Dict[str, Any] = { - 'title': title, - 'base' : to_branch, - 'body': body or '', - 'maintainer_can_modify': maintainer_can_modify, - 'draft': draft, + "title": title, + "base": to_branch, + "body": body or "", + "maintainer_can_modify": maintainer_can_modify, + "draft": draft, } if from_user and from_user != self.username: - data['head'] = f"{from_user}:{from_branch}" + data["head"] = f"{from_user}:{from_branch}" else: - data['head'] = from_branch + data["head"] = from_branch logger.debug("PR data %s", data) if self.dry_run: @@ -474,14 +498,20 @@ async def create_pr(self, title: str, if body: logger.info(" body:\n%s\n", body) - return {'number': -1} + return {"number": -1} logger.info("Creating PR '%s'", title) accept = "application/vnd.github.shadow-cat-preview" # for draft return await self.api.post(self.PULLS, var_data, data=data, accept=accept) - async def merge_pr(self, number: int, title: str = None, message: str = None, sha: str = None, - method: MergeMethod = MergeMethod.squash) -> Tuple[bool, str]: + async def merge_pr( + self, + number: int, + title: str = None, + message: str = None, + sha: str = None, + method: MergeMethod = MergeMethod.squash, + ) -> Tuple[bool, str]: """Merge a PR Arguments: @@ -495,20 +525,20 @@ async def merge_pr(self, number: int, title: str = None, message: str = None, sh Tuple: True if successful and message """ var_data = copy(self.var_default) - var_data['number'] = str(number) + var_data["number"] = str(number) data = {} if title: - data['commit_title'] = title + data["commit_title"] = title if message: - data['commit_message'] = message + data["commit_message"] = message if sha: - data['sha'] = sha + data["sha"] = sha if method: - data['merge_method'] = method.name + data["merge_method"] = method.name try: res = await self.api.put(self.PULL_MERGE, var_data, data=data) - return True, res['message'] + return True, res["message"] except gidgethub.BadRequest as exc: if exc.status_code in (405, 409): # not allowed / conflict @@ -525,7 +555,7 @@ async def pr_is_merged(self, number) -> bool: True if the PR has been merged. """ var_data = copy(self.var_default) - var_data['number'] = str(number) + var_data["number"] = str(number) try: await self.api.getitem(self.PULL_MERGE, var_data) return True @@ -540,7 +570,7 @@ async def pr_update_branch(self, number) -> bool: Merges changes to "base" into "head" """ var_data = copy(self.var_default) - var_data['number'] = str(number) + var_data["number"] = str(number) accept = "application/vnd.github.lydian-preview+json" try: await self.api.put(self.PULL_UPDATE, var_data, accept=accept, data={}) @@ -552,14 +582,20 @@ async def pr_update_branch(self, number) -> bool: # on anything but 200, 201, 204 (see sansio.py:decipher_response). # So we catch and check the status code. return True - logger.exception("pr_update_branch_failed (2). status_code=%s, args=%s", - exc.status_code, exc.args) + logger.exception( + "pr_update_branch_failed (2). status_code=%s, args=%s", + exc.status_code, + exc.args, + ) return False - async def modify_issue(self, number: int, - labels: Optional[List[str]] = None, - title: Optional[str] = None, - body: Optional[str] = None) -> Dict[Any, Any]: + async def modify_issue( + self, + number: int, + labels: Optional[List[str]] = None, + title: Optional[str] = None, + body: Optional[str] = None, + ) -> Dict[Any, Any]: """Modify existing issue (PRs are issues) Arguments: @@ -571,11 +607,11 @@ async def modify_issue(self, number: int, var_data["number"] = str(number) data: Dict[str, Any] = {} if labels: - data['labels'] = labels + data["labels"] = labels if title: - data['title'] = title + data["title"] = title if body: - data['body'] = body + data["body"] = body if self.dry_run: logger.info("Would modify PR %s", number) @@ -586,7 +622,7 @@ async def modify_issue(self, number: int, if body: logger.info("New Body:\n%s\n", body) - return {'number': number} + return {"number": number} logger.info("Modifying PR %s", number) return await self.api.patch(self.ISSUES, var_data, data=data) @@ -599,15 +635,13 @@ async def create_comment(self, number: int, body: str) -> int: """ var_data = copy(self.var_default) var_data["number"] = str(number) - data = { - 'body': body - } + data = {"body": body} if self.dry_run: logger.info("Would create comment on issue #%i", number) return -1 logger.info("Creating comment on issue #%i", number) res = await self.api.post(self.ISSUE_COMMENTS, var_data, data=data) - return res['id'] + return res["id"] async def iter_comments(self, number: int) -> List[Dict[str, Any]]: """List comments for issue""" @@ -624,15 +658,13 @@ async def update_comment(self, number: int, body: str) -> int: """ var_data = copy(self.var_default) var_data["comment_id"] = str(number) - data = { - 'body': body - } + data = {"body": body} if self.dry_run: - logger.info("Would update comment %i", number) + logger.info("Would update comment %i", number) return -1 logger.info("Updating comment %i", number) res = await self.api.patch(self.COMMENTS, var_data, data=data) - return res['id'] + return res["id"] async def get_pr_modified_files(self, number: int) -> List[Dict[str, Any]]: """Retrieve the list of files modified by the PR @@ -644,8 +676,9 @@ async def get_pr_modified_files(self, number: int) -> List[Dict[str, Any]]: var_data["number"] = str(number) return await self.api.getitem(self.PULL_FILES, var_data) - async def create_check_run(self, name: str, head_sha: str, - details_url: str = None, external_id: str = None) -> int: + async def create_check_run( + self, name: str, head_sha: str, details_url: str = None, external_id: str = None + ) -> int: """Create a check run Arguments: @@ -659,26 +692,29 @@ async def create_check_run(self, name: str, head_sha: str, """ var_data = copy(self.var_default) data = { - 'name': name, - 'head_sha': head_sha, + "name": name, + "head_sha": head_sha, } if details_url: - data['details_url'] = details_url + data["details_url"] = details_url if external_id: - data['external_id'] = external_id + data["external_id"] = external_id accept = "application/vnd.github.antiope-preview+json" result = await self.api.post(self.CHECK_RUN, var_data, data=data, accept=accept) - return int(result.get('id')) - - async def modify_check_run(self, number: int, - status: CheckRunStatus = None, - conclusion: CheckRunConclusion = None, - output_title: str = None, - output_summary: str = None, - output_text: str = None, - output_annotations: List[Dict] = None, - actions: List[Dict] = None) -> Dict['str', Any]: + return int(result.get("id")) + + async def modify_check_run( + self, + number: int, + status: CheckRunStatus = None, + conclusion: CheckRunConclusion = None, + output_title: str = None, + output_summary: str = None, + output_text: str = None, + output_annotations: List[Dict] = None, + actions: List[Dict] = None, + ) -> Dict["str", Any]: """Modify a check runs Arguments: @@ -698,28 +734,33 @@ async def modify_check_run(self, number: int, Returns: Check run "object" as dict. """ - logger.info("Modifying check run %i: status=%s conclusion=%s title=%s", - number, status.name, conclusion.name if conclusion else "N/A", output_title) + logger.info( + "Modifying check run %i: status=%s conclusion=%s title=%s", + number, + status.name, + conclusion.name if conclusion else "N/A", + output_title, + ) var_data = copy(self.var_default) - var_data['id'] = str(number) + var_data["id"] = str(number) data = {} if status is not None: - data['status'] = status.name.lower() + data["status"] = status.name.lower() if status == CheckRunStatus.in_progress: - data['started_at'] = iso_now() + data["started_at"] = iso_now() if conclusion is not None: - data['conclusion'] = conclusion.name.lower() - data['completed_at'] = iso_now() + data["conclusion"] = conclusion.name.lower() + data["completed_at"] = iso_now() if output_title: - data['output'] = { - 'title': output_title, - 'summary': output_summary or "", - 'text': output_text or "", + data["output"] = { + "title": output_title, + "summary": output_summary or "", + "text": output_text or "", } if output_annotations: - data['output']['annotations'] = output_annotations + data["output"]["annotations"] = output_annotations if actions: - data['actions'] = actions + data["actions"] = actions accept = "application/vnd.github.antiope-preview+json" return await self.api.patch(self.CHECK_RUN, var_data, data=data, accept=accept) @@ -732,10 +773,10 @@ async def get_check_runs(self, sha: str) -> List[Dict[str, Any]]: List of check run "objects" """ var_data = copy(self.var_default) - var_data['commit'] = sha + var_data["commit"] = sha accept = "application/vnd.github.antiope-preview+json" res = await self.api.getitem(self.GET_CHECK_RUNS, var_data, accept=accept) - return res['check_runs'] + return res["check_runs"] async def get_statuses(self, sha: str) -> List[Dict[str, Any]]: """List status checks for **sha** @@ -746,7 +787,7 @@ async def get_statuses(self, sha: str) -> List[Dict[str, Any]]: List of status "objects" """ var_data = copy(self.var_default) - var_data['commit'] = sha + var_data["commit"] = sha return await self.api.getitem(self.GET_STATUSES, var_data) async def get_pr_reviews(self, pr_number: int) -> List[Dict[str, Any]]: @@ -759,7 +800,7 @@ async def get_pr_reviews(self, pr_number: int) -> List[Dict[str, Any]]: and ``commit_id`` (SHA, `str`) as well as a ``user`` `dict`. """ var_data = copy(self.var_default) - var_data['number'] = str(pr_number) + var_data["number"] = str(pr_number) return await self.api.getitem(self.PULL_REVIEWS, var_data) async def get_branch_protection(self, branch: str = "master") -> Dict[str, Any]: @@ -805,8 +846,9 @@ async def get_branch_protection(self, branch: str = "master") -> Dict[str, Any]: res = await self.api.getitem(self.BRANCH_PROTECTION, var_data, accept=accept) return res - async def check_protections(self, pr_number: int, - head_sha: str = None) -> Tuple[Optional[bool], str]: + async def check_protections( + self, pr_number: int, head_sha: str = None + ) -> Tuple[Optional[bool], str]: """Check whether PR meets protection requirements Arguments: @@ -817,53 +859,59 @@ async def check_protections(self, pr_number: int, logger.info("Checking protections for PR #%s : %s", pr_number, head_sha) # check that no new commits were made - if head_sha and head_sha != pr['head']['sha']: + if head_sha and head_sha != pr["head"]["sha"]: return False, "Most recent SHA in PR differs" - head_sha = pr['head']['sha'] + head_sha = pr["head"]["sha"] # check whether it's already merged - if pr['merged']: + if pr["merged"]: return False, "PR has already been merged" # check whether it's in draft state - if pr.get('draft'): + if pr.get("draft"): return False, "PR is marked as draft" # check whether it's mergeable - if pr['mergeable'] is None: + if pr["mergeable"] is None: return None, "PR mergability unknown. Retry again later." # get required checks for target branch - protections = await self.get_branch_protection(pr['base']['ref']) + protections = await self.get_branch_protection(pr["base"]["ref"]) # Verify required_checks - required_checks = set(protections.get('required_status_checks', {}).get('contexts', [])) + required_checks = set( + protections.get("required_status_checks", {}).get("contexts", []) + ) if required_checks: logger.info("Verifying %s required checks", len(required_checks)) for status in await self.get_statuses(head_sha): - if status['state'] == 'success': - required_checks.discard(status['context']) + if status["state"] == "success": + required_checks.discard(status["context"]) for check in await self.get_check_runs(head_sha): - if check.get('conclusion') == "success": - required_checks.discard(check['name']) + if check.get("conclusion") == "success": + required_checks.discard(check["name"]) if required_checks: return False, "Not all required checks have passed" logger.info("All status checks passed for #%s", pr_number) # Verify required reviews - required_reviews = protections.get('required_pull_request_reviews', {}) + required_reviews = protections.get("required_pull_request_reviews", {}) if required_reviews: - required_count = required_reviews.get('required_approving_review_count', 1) - logger.info("Checking for %s approvng reviews and no change requests", - required_count) + required_count = required_reviews.get("required_approving_review_count", 1) + logger.info( + "Checking for %s approvng reviews and no change requests", + required_count, + ) approving_count = 0 for review in await self.get_pr_reviews(pr_number): - user = review['user']['login'] - if review['state'] == ReviewState.CHANGES_REQUESTED.name: + user = review["user"]["login"] + if review["state"] == ReviewState.CHANGES_REQUESTED.name: return False, f"Changes have been requested by `@{user}`" - if review['state'] == ReviewState.APPROVED.name: + if review["state"] == ReviewState.APPROVED.name: logger.info("PR #%s was approved by @%s", pr_number, user) approving_count += 1 if approving_count < required_count: - return False, (f"Insufficient number of approving reviews" - f"({approving_count}/{required_count})") + return False, ( + f"Insufficient number of approving reviews" + f"({approving_count}/{required_count})" + ) logger.info("PR #%s is passing configured checks", pr_number) return True, "LGTM" @@ -882,16 +930,16 @@ async def get_contents(self, path: str, ref: str = None) -> str: (Should always be base64) """ var_data = copy(self.var_default) - var_data['path'] = path + var_data["path"] = path if ref: - var_data['ref'] = ref + var_data["ref"] = ref else: - ref = 'master' + ref = "master" result = await self.api.getitem(self.CONTENTS, var_data) - if result['encoding'] != 'base64': + if result["encoding"] != "base64": raise RuntimeError(f"Got unknown encoding for {self}/{path}:{ref}") - content_bytes = base64.b64decode(result['content']) - content = content_bytes.decode('utf-8') + content_bytes = base64.b64decode(result["content"]) + content = content_bytes.decode("utf-8") return content async def delete_branch(self, ref: str) -> None: @@ -901,7 +949,7 @@ async def delete_branch(self, ref: str) -> None: ref: Name of reference/branch """ var_data = copy(self.var_default) - var_data['ref'] = ref + var_data["ref"] = ref await self.api.delete(self.GIT_REFERENCE, var_data) def _deparse_card_pr_number(self, card: Dict[str, Any]) -> Dict[str, Any]: @@ -916,22 +964,25 @@ def _deparse_card_pr_number(self, card: Dict[str, Any]) -> Dict[str, Any]: Results: Card dict with ``issue_number`` field added if card is not a note """ - if 'content_url' not in card: # not content_url to parse + if "content_url" not in card: # not content_url to parse return card - if 'issue_number' in card: # target value already taken + if "issue_number" in card: # target value already taken return card issue_url = gidgethub.sansio.format_url(self.ISSUES, self.var_default) - content_url = card['content_url'] + content_url = card["content_url"] if content_url.startswith(issue_url): try: - card['issue_number'] = int(content_url.lstrip(issue_url)) + card["issue_number"] = int(content_url.lstrip(issue_url)) except ValueError: pass - if 'issue_number' not in card: - logger.error("Failed to deparse content url to issue number.\n" - "content_url=%s\nissue_url=%s\n", - content_url, issue_url) + if "issue_number" not in card: + logger.error( + "Failed to deparse content url to issue number.\n" + "content_url=%s\nissue_url=%s\n", + content_url, + issue_url, + ) return card async def list_project_cards(self, column_id: int) -> List[Dict[str, Any]]: @@ -940,7 +991,7 @@ async def list_project_cards(self, column_id: int) -> List[Dict[str, Any]]: Arguments: column_id: ID number of project column """ - var_data = {'column_id': str(column_id)} + var_data = {"column_id": str(column_id)} accept = "application/vnd.github.inertia-preview+json" res = await self.api.getitem(self.PROJECT_COL_CARDS, var_data, accept=accept) return [self._deparse_card_pr_number(card) for card in res] @@ -953,7 +1004,7 @@ async def get_project_card(self, card_id: int) -> Dict[str, Any]: Returns: Empty dict if the project card was not found """ - var_data = {'card_id': str(card_id)} + var_data = {"card_id": str(card_id)} accept = "application/vnd.github.inertia-preview+json" try: res = await self.api.getitem(self.PROJECT_CARDS, var_data, accept=accept) @@ -962,11 +1013,14 @@ async def get_project_card(self, card_id: int) -> Dict[str, Any]: return {} return self._deparse_card_pr_number(res) - async def create_project_card(self, column_id: int, - note: str = None, - content_id: int = None, - content_type: CardContentType = None, - number: int = None) -> Dict[str, Any]: + async def create_project_card( + self, + column_id: int, + note: str = None, + content_id: int = None, + content_type: CardContentType = None, + number: int = None, + ) -> Dict[str, Any]: """Create a new project card In addition to **column_id**, you must provide *either*: @@ -990,35 +1044,38 @@ async def create_project_card(self, column_id: int, """ var_data = copy(self.var_default) - var_data['column_id'] = str(column_id) + var_data["column_id"] = str(column_id) accept = "application/vnd.github.inertia-preview+json" data = {} if note: if content_id or content_type or number: raise ValueError("Project Cards can only be a note or a Issue/PR") - data['note'] = note + data["note"] = note elif content_id: if number: raise ValueError("Cannot specify pr/issue number AND content_id") if not content_type: raise ValueError("Must specify content_type if giving content_id") - data['content_id'] = content_id - data['content_type'] = content_type.name + data["content_id"] = content_id + data["content_type"] = content_type.name elif number: pullreq = await self.get_prs(number=number) if pullreq: - data['content_id'] = pullreq['id'] - data['content_type'] = CardContentType.PullRequest.name + data["content_id"] = pullreq["id"] + data["content_type"] = CardContentType.PullRequest.name else: issue = await self.get_issue(number=number) - data['content_id'] = issue['id'] - data['content_type'] = CardContentType.Issue.name + data["content_id"] = issue["id"] + data["content_type"] = CardContentType.Issue.name else: - raise ValueError("Must have at least note or content_id/content_type or " - "number parameter") + raise ValueError( + "Must have at least note or content_id/content_type or " + "number parameter" + ) try: - res = await self.api.post(self.PROJECT_COL_CARDS, var_data, data=data, - accept=accept) + res = await self.api.post( + self.PROJECT_COL_CARDS, var_data, data=data, accept=accept + ) return self._deparse_card_pr_number(res) except gidgethub.BadRequest: logger.exception("Failed to create project card with data=%s", data) @@ -1032,7 +1089,7 @@ async def delete_project_card(self, card_id: int) -> bool: Returns: True if the deletion succeeded """ - var_data = {'card_id': str(card_id)} + var_data = {"card_id": str(card_id)} accept = "application/vnd.github.inertia-preview+json" try: await self.api.delete(self.PROJECT_CARDS, var_data, accept=accept) @@ -1041,7 +1098,9 @@ async def delete_project_card(self, card_id: int) -> bool: logger.exception("Failed to delete project cards %s", card_id) return False - async def delete_project_card_from_column(self, column_id: int, number: int) -> bool: + async def delete_project_card_from_column( + self, column_id: int, number: int + ) -> bool: """Deletes a project card identified by PR/Issue number from column Arguments: @@ -1051,8 +1110,8 @@ async def delete_project_card_from_column(self, column_id: int, number: int) -> True if the deleteion succeeded """ for card in await self.list_project_cards(column_id): - if card.get('issue_number') == number: - return await self.delete_project_card(card['id']) + if card.get("issue_number") == number: + return await self.delete_project_card(card["id"]) return False @@ -1063,17 +1122,22 @@ class AiohttpGitHubHandler(GitHubHandler): session: Aiohttp Client Session object requester: Identify self (e.g. user agent) """ - def create_api_object(self, session: aiohttp.ClientSession, - requester: str, *args, **kwargs) -> None: + + def create_api_object( + self, session: aiohttp.ClientSession, requester: str, *args, **kwargs + ) -> None: self.api = gidgethub.aiohttp.GitHubAPI( - session, requester, oauth_token=self.token, - cache=cachetools.LRUCache(maxsize=500) + session, + requester, + oauth_token=self.token, + cache=cachetools.LRUCache(maxsize=500), ) self.session = session class Event(gidgethub.sansio.Event): """Adds **get(path)** method to Github Webhook event""" + def get(self, path: str, altvalue=KeyError) -> str: """Get subkeys from even data using slash separated path""" data = self.data @@ -1104,6 +1168,7 @@ class GitHubAppHandler: of users via OAUTH """ + #: Github API url for creating an access token for a specific installation #: of an app. INSTALLATION_TOKEN = "/app/installations/{installation_id}/access_tokens" @@ -1118,14 +1183,22 @@ class GitHubAppHandler: DOMAIN = "https://github.com" #: URL template for calls to OAUTH authorize - AUTHORIZE = '/login/oauth/authorize{?client_id,client_secret,redirect_uri,state,login}' + AUTHORIZE = ( + "/login/oauth/authorize{?client_id,client_secret,redirect_uri,state,login}" + ) #: URL templete for calls to OAUTH access_token - ACCESS_TOKEN = '/login/oauth/access_token' - - def __init__(self, session: aiohttp.ClientSession, - app_name: str, app_key: str, app_id: str, - client_id: str, client_secret: str) -> None: + ACCESS_TOKEN = "/login/oauth/access_token" + + def __init__( + self, + session: aiohttp.ClientSession, + app_name: str, + app_key: str, + app_id: str, + client_id: str, + client_secret: str, + ) -> None: #: Name of app self.name = app_name #: ID of app @@ -1156,9 +1229,9 @@ def get_app_jwt(self) -> str: if not expires or expires < now + 60: expires = now + self.JWT_RENEW_PERIOD payload = { - 'iat': now, - 'exp': expires, - 'iss': self.app_id, + "iat": now, + "exp": expires, + "iss": self.app_id, } token_utf8 = jwt.encode(payload, self.app_key, algorithm="RS256") token = token_utf8.decode("utf-8") @@ -1167,13 +1240,13 @@ def get_app_jwt(self) -> str: else: msg = "Reusing" - logger.debug("%s JWT valid for %i minutes", msg, (expires - now)/60) + logger.debug("%s JWT valid for %i minutes", msg, (expires - now) / 60) return token @staticmethod def parse_isotime(timestr: str) -> int: """Converts UTC ISO 8601 time stamp to seconds in epoch""" - if timestr[-1] != 'Z': + if timestr[-1] != "Z": raise ValueError(f"Time String '%s' not in UTC") return int(time.mktime(time.strptime(timestr[:-1], "%Y-%m-%dT%H:%M:%S"))) @@ -1182,43 +1255,52 @@ async def get_installation_token(self, installation: str, name: str = None) -> s if name is None: name = installation now = int(time.time()) - expires, token = self._tokens.get(installation, (0, '')) + expires, token = self._tokens.get(installation, (0, "")) if not expires or expires < now + 60: - api = gidgethub.aiohttp.GitHubAPI(self._session, self.name, cache=self._cache) + api = gidgethub.aiohttp.GitHubAPI( + self._session, self.name, cache=self._cache + ) try: res = await api.post( self.INSTALLATION_TOKEN, - {'installation_id': installation}, + {"installation_id": installation}, data=b"", accept="application/vnd.github.machine-man-preview+json", - jwt=self.get_app_jwt() + jwt=self.get_app_jwt(), ) except gidgethub.BadRequest: logger.exception("Failed to get installation token for %s", name) raise - expires = self.parse_isotime(res['expires_at']) - token = res['token'] + expires = self.parse_isotime(res["expires_at"]) + token = res["token"] self._tokens[installation] = (expires, token) msg = "Created new" else: msg = "Reusing" - logger.debug("%s token for %i valid for %i minutes", - msg, installation, (expires - now)/60) + logger.debug( + "%s token for %i valid for %i minutes", + msg, + installation, + (expires - now) / 60, + ) return token async def get_installation_id(self, user, repo): """Retrieve installation ID given user and repo""" api = gidgethub.aiohttp.GitHubAPI(self._session, self.name, cache=self._cache) - res = await api.getitem(self.INSTALLATION, - {'owner': user, 'repo': repo}, - accept="application/vnd.github.machine-man-preview+json", - jwt=self.get_app_jwt()) - return res['id'] - - async def get_github_api(self, dry_run, to_user, to_repo, - installation=None) -> GitHubHandler: + res = await api.getitem( + self.INSTALLATION, + {"owner": user, "repo": repo}, + accept="application/vnd.github.machine-man-preview+json", + jwt=self.get_app_jwt(), + ) + return res["id"] + + async def get_github_api( + self, dry_run, to_user, to_repo, installation=None + ) -> GitHubHandler: """Returns the GitHubHandler for the installation the event came from""" if installation is None: installation = await self.get_installation_id(to_user, to_repo) @@ -1231,8 +1313,10 @@ async def get_github_api(self, dry_run, to_user, to_repo, else: api = AiohttpGitHubHandler( await self.get_installation_token(installation), - to_user=to_user, to_repo=to_repo, dry_run=dry_run, - installation=installation + to_user=to_user, + to_repo=to_repo, + dry_run=dry_run, + installation=installation, ) api.create_api_object(self._session, self.name) self._handlers[handler_key] = api @@ -1256,8 +1340,9 @@ async def get_github_user_api(self, token: str) -> GitHubHandler: last_access = now if len(self._user_handlers) > 50: - lru_keys = sorted(self._user_handlers, - key=lambda k: self._user_handlers[k][0]) + lru_keys = sorted( + self._user_handlers, key=lambda k: self._user_handlers[k][0] + ) for key in lru_keys[:10]: del self._user_handlers[key] @@ -1271,7 +1356,7 @@ def generate_nonce(nbytes=16): Fetches **nbytes** of random data from `os.urandom` and passes them through base64 encoding. """ - return base64.b64encode(os.urandom(nbytes), altchars=b'_-').decode() + return base64.b64encode(os.urandom(nbytes), altchars=b"_-").decode() async def oauth_github_user(self, redirect, session, params): """Acquires `AiohttpGitHubHandler` for user via OAuth @@ -1316,11 +1401,11 @@ async def oauth_github_user(self, redirect, session, params): The API client object with the user logged in. """ - nonce_cookie_name = f'{self.__class__.__name__}::nonce' - token_cookie_name = f'{self.__class__.__name__}::token' + nonce_cookie_name = f"{self.__class__.__name__}::nonce" + token_cookie_name = f"{self.__class__.__name__}::token" - code = params.get('code') - state = params.get('state') + code = params.get("code") + state = params.get("state") nonce = session.get(nonce_cookie_name) # If we have a token already, try to authenticate the user @@ -1336,32 +1421,39 @@ async def oauth_github_user(self, redirect, session, params): if not code or state != nonce: nonce = self.generate_nonce() session[nonce_cookie_name] = nonce - raise aiohttp.web.HTTPFound(uritemplate.expand( - self.DOMAIN + self.AUTHORIZE, { - 'client_id': self.client_id, - 'redirect_uri': redirect, - 'state': nonce, - })) + raise aiohttp.web.HTTPFound( + uritemplate.expand( + self.DOMAIN + self.AUTHORIZE, + { + "client_id": self.client_id, + "redirect_uri": redirect, + "state": nonce, + }, + ) + ) # Second pass. We have a code and the state matched the nonce. # Fetch the OAUTH token from Github using the code and state. - data = {'client_id': self.client_id, - 'client_secret': self.client_secret, - 'code': code, - 'state': nonce} - headers = {'Accept': 'application/json'} - async with self._session.post(self.DOMAIN + self.ACCESS_TOKEN, - json=data, headers=headers) as response: + data = { + "client_id": self.client_id, + "client_secret": self.client_secret, + "code": code, + "state": nonce, + } + headers = {"Accept": "application/json"} + async with self._session.post( + self.DOMAIN + self.ACCESS_TOKEN, json=data, headers=headers + ) as response: response.raise_for_status() result = await response.json() # We should always get a bearer token. This seems to be a future # extension. Check it anyway: - if result.get('token_type') != 'bearer': + if result.get("token_type") != "bearer": raise RuntimeError("Token type not 'bearer'") # Create the client and get user details to verify token validity - handler = await self.get_github_user_api(result.get('access_token')) + handler = await self.get_github_user_api(result.get("access_token")) if not handler: raise RuntimeError("Failed to login") session[token_cookie_name] = handler.token diff --git a/bioconda_utils/gitter.py b/bioconda_utils/gitter.py index 8cbe57d15d..60d25f959f 100644 --- a/bioconda_utils/gitter.py +++ b/bioconda_utils/gitter.py @@ -19,6 +19,7 @@ class User(NamedTuple): """Gitter User""" + @classmethod def from_dict(cls, data): """Create `User` from `dict`""" @@ -48,6 +49,7 @@ def from_dict(cls, data): class Mention(NamedTuple): """Gitter User Mention""" + @classmethod def from_dict(cls, data): """Create `User` from `dict`""" @@ -63,13 +65,14 @@ def from_dict(cls, data): class Message(NamedTuple): """Gitter Chat Message""" + @classmethod def from_dict(cls, data): """Create `Message` from `dict`""" - if 'mentions' in data: - data['mentions'] = [Mention.from_dict(user) for user in data['mentions']] - if 'fromUser' in data: - data['fromUser'] = User.from_dict(data['fromUser']) + if "mentions" in data: + data["mentions"] = [Mention.from_dict(user) for user in data["mentions"]] + if "fromUser" in data: + data["fromUser"] = User.from_dict(data["fromUser"]) return cls(**data) #: Message ID @@ -102,14 +105,14 @@ def from_dict(cls, data): editedAt: str = None - class Room(NamedTuple): """Gitter Chat Room""" + @classmethod def from_dict(cls, data): """Create `Room` from `dict`""" - if 'user' in data: - data['user'] = User.from_dict(data['user']) + if "user" in data: + data["user"] = User.from_dict(data["user"]) return cls(**data) #: Room ID @@ -211,8 +214,9 @@ def __init__(self, token: str) -> None: self.debug_once = False @abc.abstractmethod - async def _request(self, method: str, url: str, headers: Mapping[str, str], - body: bytes = b'') -> Tuple[int, Mapping[str, str], bytes]: + async def _request( + self, method: str, url: str, headers: Mapping[str, str], body: bytes = b"" + ) -> Tuple[int, Mapping[str, str], bytes]: """Execute HTTP request (implemented by IO providing subclass) Args: @@ -226,9 +230,9 @@ async def _request(self, method: str, url: str, headers: Mapping[str, str], """ @abc.abstractmethod - async def _stream_request(self, method: str, url: str, - headers: Mapping[str, str], - body: bytes = b'') -> AsyncIterator[bytes]: + async def _stream_request( + self, method: str, url: str, headers: Mapping[str, str], body: bytes = b"" + ) -> AsyncIterator[bytes]: """Execute streaming HTTP request (implement by IO providing subclass) Args: @@ -241,31 +245,40 @@ async def _stream_request(self, method: str, url: str, Async iterator over data chunks """ - def _prepare_request(self, url: str, var_dict: Mapping[str, str], - data: Any = None, - charset: str = 'utf-8', - accept: str = "application/json") -> Tuple[str, Mapping[str, str], bytes]: + def _prepare_request( + self, + url: str, + var_dict: Mapping[str, str], + data: Any = None, + charset: str = "utf-8", + accept: str = "application/json", + ) -> Tuple[str, Mapping[str, str], bytes]: """Prepare url, headers and json body for request""" url = uritemplate.expand(url, var_dict=var_dict) headers = {} - headers['accept'] = accept - headers['Authorization'] = "Bearer " + self.token + headers["accept"] = accept + headers["Authorization"] = "Bearer " + self.token - body = b'' + body = b"" if isinstance(data, str): body = data.encode(charset) elif isinstance(data, Mapping): body = json.dumps(data).encode(charset) - headers['content-type'] = "application/json; charset=" + charset - headers['content-length'] = str(len(body)) + headers["content-type"] = "application/json; charset=" + charset + headers["content-length"] = str(len(body)) return url, headers, body - async def _make_request(self, method: str, url: str, var_dict: Mapping[str, str], - data: Any = None, - accept: str = "application/json") -> Tuple[str, Any]: + async def _make_request( + self, + method: str, + url: str, + var_dict: Mapping[str, str], + data: Any = None, + accept: str = "application/json", + ) -> Tuple[str, Any]: """Make HTTP request""" - charset = 'utf-8' - url = ''.join((self._GITTER_API, url)) + charset = "utf-8" + url = "".join((self._GITTER_API, url)) url, headers, body = self._prepare_request(url, var_dict, data, charset, accept) status, res_headers, response = await self._request(method, url, headers, body) @@ -282,17 +295,24 @@ async def _make_request(self, method: str, url: str, var_dict: Mapping[str, str] try: return response_text, json.loads(response_text) except json.decoder.JSONDecodeError: - logger.error("Call to '%s' yielded text '%s' - not JSON", - url.replace(self.token, "******"), - response_text.replace(self.token, "******")) + logger.error( + "Call to '%s' yielded text '%s' - not JSON", + url.replace(self.token, "******"), + response_text.replace(self.token, "******"), + ) return response_text, None - async def _make_stream_request(self, method: str, url: str, var_dict: Mapping[str, str], - data: Any = None, - accept: str = "application/json") -> AsyncIterator[Any]: + async def _make_stream_request( + self, + method: str, + url: str, + var_dict: Mapping[str, str], + data: Any = None, + accept: str = "application/json", + ) -> AsyncIterator[Any]: """Make streaming HTTP request""" - charset = 'utf-8' - url = ''.join((self._GITTER_STREAM_API, url)) + charset = "utf-8" + url = "".join((self._GITTER_STREAM_API, url)) url, headers, body = self._prepare_request(url, var_dict, data, charset, accept) async for line_bytes in self._stream_request(method, url, headers, body): line_str = line_bytes.decode(charset) @@ -311,7 +331,7 @@ async def list_rooms(self, name: str = None) -> List[Room]: Args: name: Room name """ - _, data = await self._make_request('GET', self._ROOMS, {}) + _, data = await self._make_request("GET", self._ROOMS, {}) if not data: return [] rooms = [Room.from_dict(item) for item in data] @@ -321,81 +341,91 @@ async def list_rooms(self, name: str = None) -> List[Room]: async def get_room(self, uri: str) -> Room: """Get a room using its URI""" - _, data = await self._make_request('POST', self._ROOMS, {}, {'uri': uri}) + _, data = await self._make_request("POST", self._ROOMS, {}, {"uri": uri}) return Room.from_dict(data) async def join_room(self, user: User, room: Room) -> None: """Add **user** to a **room**""" - await self._make_request('POST', self._USER_ROOMS, {'userId': user.id}, - {'id': room.id}) + await self._make_request( + "POST", self._USER_ROOMS, {"userId": user.id}, {"id": room.id} + ) async def leave_room(self, user: User, room: Room) -> bool: """Remove **user** from **room**""" try: - await self._make_request('DELETE', self._ROOM_USERS, - {'roomId': room.id, 'userId': user.id}) + await self._make_request( + "DELETE", self._ROOM_USERS, {"roomId": room.id, "userId": user.id} + ) except aiohttp.ClientResponseError as exc: if exc.code in (404,): return False return True - async def edit_room(self, room: Room, topic: str = None, tags: str = None, - noindex: bool = None) -> None: + async def edit_room( + self, room: Room, topic: str = None, tags: str = None, noindex: bool = None + ) -> None: """Set **topic**, **tags** or **noindex** for **room**""" data = {} if topic: - data['topic'] = topic + data["topic"] = topic if tags: - data['tags'] = tags + data["tags"] = tags if noindex: - data['noindex'] = str(noindex) - await self._make_request('PUT', self._ROOMS, {'roomId': room.id}, data) + data["noindex"] = str(noindex) + await self._make_request("PUT", self._ROOMS, {"roomId": room.id}, data) - async def list_unread_items(self, user: User, room: Room) -> Tuple[List[str], List[str]]: + async def list_unread_items( + self, user: User, room: Room + ) -> Tuple[List[str], List[str]]: """Get Ids for unread items of **user** in **room** Returns: Two lists of chat IDs are returned. The first are all unread mentions, the second only those in which the user was @Mentioned. """ - _, data = await self._make_request('GET', self._UNREAD, - {'userId': user.id, 'roomId': room.id}) - return data.get('chat', []), data.get('mention', []) + _, data = await self._make_request( + "GET", self._UNREAD, {"userId": user.id, "roomId": room.id} + ) + return data.get("chat", []), data.get("mention", []) async def mark_as_read(self, user: User, room: Room, ids: List[str]) -> None: """Mark chat messages listed in **ids** as read""" - await self._make_request('POST', self._UNREAD, - {'userId': user.id, 'roomId': room.id}, - {'chat': ids}) + await self._make_request( + "POST", self._UNREAD, {"userId": user.id, "roomId": room.id}, {"chat": ids} + ) async def get_message(self, room: Room, msgid: str) -> Message: """Get a single message by its **id**""" - _, data = await self._make_request('GET', self._MESSAGES, - {'roomId': room.id, 'messageId': msgid}) + _, data = await self._make_request( + "GET", self._MESSAGES, {"roomId": room.id, "messageId": msgid} + ) return Message.from_dict(data) async def send_message(self, room: Room, text: str, *args: Any) -> Message: """Send a new message""" - _, data = await self._make_request('POST', self._MESSAGES, - {'roomId': room.id}, - {'text': text % args}) + _, data = await self._make_request( + "POST", self._MESSAGES, {"roomId": room.id}, {"text": text % args} + ) return Message.from_dict(data) async def edit_message(self, room: Room, message: Message, text: str) -> Message: """Edit a message""" - _, data = await self._make_request('PUT', self._MESSAGES, - {'roomId': room.id, 'messageId': message.id}, - {'text': text}) + _, data = await self._make_request( + "PUT", + self._MESSAGES, + {"roomId": room.id, "messageId": message.id}, + {"text": text}, + ) return Message.from_dict(data) async def list_groups(self): """Get list of current user's groups""" - _, data = await self._make_request('GET', self._LIST_GROUPS, {}) + _, data = await self._make_request("GET", self._LIST_GROUPS, {}) return data async def get_user(self) -> User: """Get current user""" - _, data = await self._make_request('GET', self._GET_USER, {}) + _, data = await self._make_request("GET", self._GET_USER, {}) return User.from_dict(data) async def iter_chat(self, room: Room) -> AsyncIterator[Message]: @@ -407,30 +437,36 @@ async def iter_chat(self, room: Room) -> AsyncIterator[Message]: Returns: async iterator over chat messages """ - stream = self._make_stream_request('GET', self._MESSAGES, {'roomId': room.id}) + stream = self._make_stream_request("GET", self._MESSAGES, {"roomId": room.id}) async for data in stream: yield Message.from_dict(data) class AioGitterAPI(GitterAPI): """AioHTTP based implementation of GitterAPI""" - def __init__(self, session: aiohttp.ClientSession, *args: Any, **kwargs: Any) -> None: + + def __init__( + self, session: aiohttp.ClientSession, *args: Any, **kwargs: Any + ) -> None: self._session = session super().__init__(*args, **kwargs) - async def _request(self, method: str, url: str, - headers: Mapping[str, str], - body: bytes = b'') -> Tuple[int, Mapping[str, str], bytes]: - async with self._session.request(method, url, headers=headers, data=body) as response: + async def _request( + self, method: str, url: str, headers: Mapping[str, str], body: bytes = b"" + ) -> Tuple[int, Mapping[str, str], bytes]: + async with self._session.request( + method, url, headers=headers, data=body + ) as response: response.raise_for_status() return response.status, response.headers, await response.read() - async def _stream_request(self, method: str, url: str, - headers: Mapping[str, str], - body: bytes = b'') -> AsyncIterator[bytes]: + async def _stream_request( + self, method: str, url: str, headers: Mapping[str, str], body: bytes = b"" + ) -> AsyncIterator[bytes]: timeout = aiohttp.ClientTimeout(total=3600, sock_read=3600) - async with self._session.request(method, url, headers=headers, data=body, - timeout=timeout) as response: + async with self._session.request( + method, url, headers=headers, data=body, timeout=timeout + ) as response: response.raise_for_status() async for line_bytes in response.content: yield line_bytes diff --git a/bioconda_utils/graph.py b/bioconda_utils/graph.py index 896e74c713..3a44e3d6e9 100644 --- a/bioconda_utils/graph.py +++ b/bioconda_utils/graph.py @@ -46,7 +46,9 @@ def build(recipes, config, blacklist=None, restrict=True): """ logger.info("Generating DAG") recipes = list(recipes) - metadata = list(utils.parallel_iter(utils.load_meta_fast, recipes, "Loading Recipes")) + metadata = list( + utils.parallel_iter(utils.load_meta_fast, recipes, "Loading Recipes") + ) if blacklist is None: blacklist = set() @@ -80,17 +82,18 @@ def get_inner_deps(dependencies): yield dep dag = nx.DiGraph() - dag.add_nodes_from(meta["package"]["name"] - for meta, recipe in metadata) + dag.add_nodes_from(meta["package"]["name"] for meta, recipe in metadata) for meta, recipe in metadata: name = meta["package"]["name"] dag.add_edges_from( (dep, name) - for dep in set(chain( - get_inner_deps(get_deps(meta, "build")), - get_inner_deps(get_deps(meta, "host")), - get_inner_deps(get_deps(meta, "run")), - )) + for dep in set( + chain( + get_inner_deps(get_deps(meta, "build")), + get_inner_deps(get_deps(meta, "host")), + get_inner_deps(get_deps(meta, "run")), + ) + ) ) return dag, name2recipe @@ -115,7 +118,9 @@ def build_from_recipes(recipes): for recipe2 in package2recipes.get(dep, []) ) - logger.info("Building Recipe DAG: done (%i nodes, %i edges)", len(dag), len(dag.edges())) + logger.info( + "Building Recipe DAG: done (%i nodes, %i edges)", len(dag), len(dag.edges()) + ) return dag @@ -123,9 +128,11 @@ def filter_recipe_dag(dag, include, exclude): """Reduces **dag** to packages in **names** and their requirements""" nodes = set() for recipe in dag: - if (recipe not in nodes + if ( + recipe not in nodes and any(fnmatch(recipe.reldir, p) for p in include) - and not any(fnmatch(recipe.reldir, p) for p in exclude)): + and not any(fnmatch(recipe.reldir, p) for p in exclude) + ): nodes.add(recipe) nodes |= nx.ancestors(dag, recipe) return nx.subgraph(dag, nodes) diff --git a/bioconda_utils/hosters.py b/bioconda_utils/hosters.py index 823a089217..4a34fefa97 100644 --- a/bioconda_utils/hosters.py +++ b/bioconda_utils/hosters.py @@ -26,8 +26,19 @@ from distutils.version import LooseVersion from html.parser import HTMLParser from itertools import chain -from typing import (Any, Dict, List, Match, Mapping, Pattern, Set, Tuple, Type, - Optional, TYPE_CHECKING) +from typing import ( + Any, + Dict, + List, + Match, + Mapping, + Pattern, + Set, + Tuple, + Type, + Optional, + TYPE_CHECKING, +) from urllib.parse import urljoin import regex as re @@ -41,9 +52,12 @@ #: This is so complicated because we need to parse matched, not-escaped #: parentheses to determine where the clause ends. #: Requires regex package for recursion. -RE_CAPGROUP = re.compile(r"\(\?P<(\w+)>(?>[^()]+|\\\(|\\\)|(\((?>[^()]+|\\\(|\\\)|(?2))*\)))*\)") +RE_CAPGROUP = re.compile( + r"\(\?P<(\w+)>(?>[^()]+|\\\(|\\\)|(\((?>[^()]+|\\\(|\\\)|(?2))*\)))*\)" +) RE_REFGROUP = re.compile(r"\(\?P=(\w+)\)") + def dedup_named_capture_group(pattern): """Replaces repetitions of capture groups with matches to first instance""" seen: Set[str] = set() @@ -55,17 +69,20 @@ def replace(match): return f"(?P={name})" seen.add(name) return match.group(0) + return re.sub(RE_CAPGROUP, replace, pattern) def replace_named_capture_group(pattern, vals: Dict[str, str]): """Replaces capture groups with values from **vals**""" + def replace(match): "inner replace" name = match.group(1) if name in vals: return vals[name] or "" return match.group(0) + res = re.sub(RE_CAPGROUP, replace, pattern) res = re.sub(RE_REFGROUP, replace, res) return res @@ -80,8 +97,9 @@ class HosterMeta(abc.ABCMeta): hoster_types: List["HosterMeta"] = [] - def __new__(cls, name: str, bases: Tuple[type, ...], - namespace: Dict[str, Any], **kwargs) -> type: + def __new__( + cls, name: str, bases: Tuple[type, ...], namespace: Dict[str, Any], **kwargs + ) -> type: """Creates Hoster classes - expands references among ``{var}_pattern`` attributes @@ -95,8 +113,11 @@ def __new__(cls, name: str, bases: Tuple[type, ...], if not typ.__name__.startswith("Custom"): cls.hoster_types.append(typ) - patterns = {attr.replace("_pattern", ""): getattr(typ, attr) - for attr in dir(typ) if attr.endswith("_pattern")} + patterns = { + attr.replace("_pattern", ""): getattr(typ, attr) + for attr in dir(typ) + if attr.endswith("_pattern") + } for pat in patterns: # expand pattern references: @@ -106,7 +127,8 @@ def __new__(cls, name: str, bases: Tuple[type, ...], pattern = new_pattern new_pattern = re.sub(r"(\{\d+,?\d*\})", r"{\1}", pattern) new_pattern = new_pattern.format_map( - {k: v.rstrip("$") for k, v in patterns.items()}) + {k: v.rstrip("$") for k, v in patterns.items()} + ) patterns[pat] = pattern # repair duplicate capture groups: pattern = dedup_named_capture_group(pattern) @@ -139,13 +161,15 @@ class Hoster(metaclass=HosterMeta): #: - then only numbers, characters or one of -, +, ., :, ~ #: - at most 31 characters length (to avoid matching checksums) #: - accept v or r as prefix if after slash, dot, underscore or dash - version_pattern: str = r"(?:(?<=[/._-])[rv])?(?P\d[\da-zA-Z\-+\.:\~_]{0,30})" + version_pattern: str = ( + r"(?:(?<=[/._-])[rv])?(?P\d[\da-zA-Z\-+\.:\~_]{0,30})" + ) #: matches archive file extensions ext_pattern: str = r"(?P(?i)\.(?:(?:(tar\.|t)(?:xz|bz2|gz))|zip|jar))" #: named patterns that will change with a version upgrade - exclude = ['version'] + exclude = ["version"] @property @abc.abstractmethod @@ -168,22 +192,21 @@ def releases_formats(self) -> List[str]: def __init__(self, url: str, match: Match[str]) -> None: self.vals = {k: v or "" for k, v in match.groupdict().items()} self.releases_urls = [ - template.format_map(self.vals) - for template in self.releases_formats + template.format_map(self.vals) for template in self.releases_formats ] - logger.debug("%s matched %s with %s", - self.__class__.__name__, url, self.vals) + logger.debug("%s matched %s with %s", self.__class__.__name__, url, self.vals) @classmethod - def try_make_hoster(cls: Type["Hoster"], url: str, - config: Dict[str, str]) -> Optional["Hoster"]: + def try_make_hoster( + cls: Type["Hoster"], url: str, config: Dict[str, str] + ) -> Optional["Hoster"]: """Creates hoster if **url** is matched by its **url_pattern**""" if config: try: klass: Type["Hoster"] = type( "Customized" + cls.__name__, (cls,), - {key+"_pattern":val for key, val in config.items()} + {key + "_pattern": val for key, val in config.items()}, ) except KeyError: logger.debug("Overrides invalid for %s - skipping", cls.__name__) @@ -197,12 +220,15 @@ def try_make_hoster(cls: Type["Hoster"], url: str, @classmethod @abc.abstractmethod - def get_versions(cls, req: "AsyncRequests", orig_version: str) -> List[Mapping[str, Any]]: + def get_versions( + cls, req: "AsyncRequests", orig_version: str + ) -> List[Mapping[str, Any]]: "Gets list of versions from upstream hosting site" class HrefParser(HTMLParser): """Extract link targets from HTML""" + def __init__(self, link_re: Pattern[str]) -> None: super().__init__() self.link_re = link_re @@ -237,9 +263,7 @@ class HTMLHoster(Hoster): async def get_versions(self, req, orig_version): exclude = set(self.exclude) - vals = {key: val - for key, val in self.vals.items() - if key not in exclude} + vals = {key: val for key, val in self.vals.items() if key not in exclude} link_pattern = replace_named_capture_group(self.link_pattern_compiled, vals) link_re = re.compile(link_pattern) result = [] @@ -257,11 +281,10 @@ async def get_versions(self, req, orig_version): class FTPHoster(Hoster): """Scans for updates on FTP servers""" + async def get_versions(self, req, orig_version): exclude = set(self.exclude) - vals = {key: val - for key, val in self.vals.items() - if key not in exclude} + vals = {key: val for key, val in self.vals.items() if key not in exclude} link_pattern = replace_named_capture_group(self.link_pattern_compiled, vals) link_re = re.compile(link_pattern) result = [] @@ -271,9 +294,9 @@ async def get_versions(self, req, orig_version): match = link_re.search(fname) if match: data = match.groupdict() - data['fn'] = fname - data['link'] = "ftp://" + vals['host'] + fname - data['releases_url'] = url + data["fn"] = fname + data["link"] = "ftp://" + vals["host"] + fname + data["releases_url"] = url result.append(data) return result @@ -306,17 +329,18 @@ async def get_versions(self, req, orig_version): break if num is None: return matches - return matches[:num + 1] + return matches[: num + 1] class GithubBase(OrderedHTMLHoster): """Base class for software hosted on github.com""" - exclude = ['version', 'fname'] + + exclude = ["version", "fname"] account_pattern = r"(?P[-\w]+)" project_pattern = r"(?P[-.\w]+)" prefix_pattern = r"(?P[-_./\w]+?)" suffix_pattern = r"(?P[-_](lin)?)" - #tag_pattern = "{prefix}??{version}{suffix}??" + # tag_pattern = "{prefix}??{version}{suffix}??" tag_pattern = "{prefix}??{version}" url_pattern = r"github\.com{link}" fname_pattern = r"(?P[^/]+)" @@ -325,40 +349,51 @@ class GithubBase(OrderedHTMLHoster): class GithubRelease(GithubBase): """Matches release artifacts uploaded to Github""" + link_pattern = r"/{account}/{project}/releases/download/{tag}/{fname}{ext}?" class GithubTag(GithubBase): """Matches GitHub repository archives created automatically from tags""" + link_pattern = r"/{account}/{project}/archive(/refs/tags)?/{tag}{ext}" releases_formats = ["https://github.com/{account}/{project}/tags"] class GithubReleaseAttachment(GithubBase): """Matches release artifacts uploaded as attachment to release notes""" + link_pattern = r"/{account}/{project}/files/\d+/{tag}{ext}" class GithubRepoStore(GithubBase): """Matches release artifacts stored in a github repo""" + branch_pattern = r"(master|[\da-f]{40})" subdir_pattern = r"(?P([-._\w]+/)+)" link_pattern = r"/{account}/{project}/blob/master/{subdir}{tag}{ext}" - url_pattern = (r"(?:(?Praw\.githubusercontent)|github)\.com/" - r"{account}/{project}/(?(raw)|(?:(?Pblob/)|raw/))" - r"{branch}/{subdir}?{tag}{ext}(?(blob)\?raw|)") + url_pattern = ( + r"(?:(?Praw\.githubusercontent)|github)\.com/" + r"{account}/{project}/(?(raw)|(?:(?Pblob/)|raw/))" + r"{branch}/{subdir}?{tag}{ext}(?(blob)\?raw|)" + ) releases_formats = ["https://github.com/{account}/{project}/tree/master/{subdir}"] + class Bioconductor(HTMLHoster): """Matches R packages hosted at Bioconductor""" + link_pattern = r"/src/contrib/(?P[^/]+)_{version}{ext}" section_pattern = r"/(bioc|data/annotation|data/experiment)" url_pattern = r"bioconductor.org/packages/(?P[\d\.]+){section}{link}" - releases_formats = ["https://bioconductor.org/packages/{bioc}/bioc/html/{package}.html"] + releases_formats = [ + "https://bioconductor.org/packages/{bioc}/bioc/html/{package}.html" + ] class CargoPort(HTMLHoster): """Matches source backup urls created by cargo-port""" + os_pattern = r"_(?Psrc_all|linux_x86|darwin_x86)" link_pattern = r"(?P[^/]+)_{version}{os}{ext}" url_pattern = r"depot.galaxyproject.org/software/(?P[^/]+)/{link}" @@ -367,9 +402,12 @@ class CargoPort(HTMLHoster): class SourceForge(HTMLHoster): """Matches packages hosted at SourceForge""" + project_pattern = r"(?P[-\w]+)" subproject_pattern = r"((?P[-\w%]+)/)?" - baseurl_pattern = r"sourceforge\.net/project(s)?/{project}/(?(1)files/|){subproject}" + baseurl_pattern = ( + r"sourceforge\.net/project(s)?/{project}/(?(1)files/|){subproject}" + ) package_pattern = r"(?P[-\w_\.+]*?[a-zA-Z+])" type_pattern = r"(?P((linux|x?(64|86)|src|source|all|core|java\d?)[-_.])*)" @@ -384,6 +422,7 @@ class SourceForge(HTMLHoster): class JSONHoster(Hoster): """Base for Hosters handling release listings in JSON format""" + async def get_versions(self, req, orig_version: str): result = [] for url in self.releases_urls: @@ -391,30 +430,33 @@ async def get_versions(self, req, orig_version: str): data = json.loads(text) matches = await self.get_versions_from_json(data, req, orig_version) for match in matches: - match['releases_url'] = url + match["releases_url"] = url result.extend(matches) return result + link_pattern = "https://{url}" @abc.abstractmethod - async def get_versions_from_json(self, data, req, orig_version) -> List[Dict[str, Any]]: - """Extract matches from json data in **data** - """ + async def get_versions_from_json( + self, data, req, orig_version + ) -> List[Dict[str, Any]]: + """Extract matches from json data in **data**""" class PyPi(JSONHoster): """Scans PyPi for updates""" + async def get_versions_from_json(self, data, req, orig_version): latest = data["info"]["version"] result = [] for vers in list(set([latest, orig_version])): - if vers not in data['releases']: + if vers not in data["releases"]: continue - for rel in data['releases'][vers]: + for rel in data["releases"][vers]: if rel["packagetype"] == "sdist": rel["link"] = rel["url"] rel["version"] = vers - rel["info"] = data['info'] + rel["info"] = data["info"] result.append(rel) return result @@ -431,8 +473,16 @@ def _get_requirements(package, fname, url, digest, python_version, build_config) with open("/dev/null", "w") as devnull: with redirect_stdout(devnull), redirect_stderr(devnull): try: - pkg_info = get_pkginfo(package, fname, url, digest, python_version, - [], build_config, []) + pkg_info = get_pkginfo( + package, + fname, + url, + digest, + python_version, + [], + build_config, + [], + ) requirements = get_requirements(package, pkg_info) except SystemExit as exc: raise Exception(exc) from None @@ -443,8 +493,8 @@ def _get_requirements(package, fname, url, digest, python_version, build_config) requirements = requirements[0] requirements_fixed = [] for req in requirements: - if '\n' in req: - requirements_fixed.extend(req.split('\n')) + if "\n" in req: + requirements_fixed.extend(req.split("\n")) else: requirements_fixed.append(req) @@ -453,45 +503,50 @@ def _get_requirements(package, fname, url, digest, python_version, build_config) @staticmethod def _get_python_version(rel): """Try to determine correct python version""" - choose_from = ('3.6', '3.5', '3.7', '2.7') + choose_from = ("3.6", "3.5", "3.7", "2.7") - requires_python = rel.get('requires_python') + requires_python = rel.get("requires_python") if requires_python: requires_python = requires_python.replace(" ", "") checks = [] for check in requires_python.split(","): - for key, func in (('==', lambda x, y: x == y), - ('!=', lambda x, y: x != y), - ('<=', lambda x, y: x <= y), - ('>=', lambda x, y: x >= y), - ('>', lambda x, y: x > y), - ('<', lambda x, y: x > y), - ('~=', lambda x, y: x == y)): + for key, func in ( + ("==", lambda x, y: x == y), + ("!=", lambda x, y: x != y), + ("<=", lambda x, y: x <= y), + (">=", lambda x, y: x >= y), + (">", lambda x, y: x > y), + ("<", lambda x, y: x > y), + ("~=", lambda x, y: x == y), + ): if check.startswith(key): - checks.append((func, check[len(key):])) + checks.append((func, check[len(key) :])) break else: checks.append((lambda x, y: x == y, check)) for vers in choose_from: try: - if all(op(LooseVersion(vers), LooseVersion(check)) - for op, check in checks): + if all( + op(LooseVersion(vers), LooseVersion(check)) + for op, check in checks + ): return vers except TypeError: - logger.exception("Failed to compare %s to %s", vers, requires_python) + logger.exception( + "Failed to compare %s to %s", vers, requires_python + ) python_versions = [ - classifier.split('::')[-1].strip() - for classifier in rel['info'].get('classifiers', []) - if classifier.startswith('Programming Language :: Python ::') + classifier.split("::")[-1].strip() + for classifier in rel["info"].get("classifiers", []) + if classifier.startswith("Programming Language :: Python ::") ] for vers in choose_from: if vers in python_versions: return vers - return '2.7' - + return "2.7" async def get_deps(self, pipeline, build_config, package, rel): """Get dependencies for **package** using version data **rel** @@ -504,10 +559,10 @@ async def get_deps(self, pipeline, build_config, package, rel): """ req = pipeline.req # We download ourselves to get async benefits - target_file = rel['filename'] + target_file = rel["filename"] target_path = os.path.join(build_config.src_cache, target_file) if not os.path.exists(target_path): - await req.get_file_from_url(target_path, rel['link'], target_file) + await req.get_file_from_url(target_path, rel["link"], target_file) python_version = self._get_python_version(rel) @@ -516,12 +571,19 @@ async def get_deps(self, pipeline, build_config, package, rel): try: pkg_info, depends = await pipeline.run_sp( self._get_requirements, - package, target_file, rel['link'], - ('sha256', rel['digests']['sha256']), - python_version, build_config) + package, + target_file, + rel["link"], + ("sha256", rel["digests"]["sha256"]), + python_version, + build_config, + ) except Exception: # pylint: disable=broad-except - logger.info("Failed to get depends for PyPi %s (py=%s)", - target_file, python_version) + logger.info( + "Failed to get depends for PyPi %s (py=%s)", + target_file, + python_version, + ) logger.debug("Exception data", exc_info=True) return @@ -530,36 +592,41 @@ async def get_deps(self, pipeline, build_config, package, rel): # Convert into dict deps = {} for dep in depends: - match = re.search(r'([^<>= ]+)(.*)', dep) + match = re.search(r"([^<>= ]+)(.*)", dep) if match: deps[match.group(1)] = match.group(2) # Write to rel dict for return - rel['depends'] = {'host': deps, 'run': deps} + rel["depends"] = {"host": deps, "run": deps} releases_formats = ["https://pypi.org/pypi/{package}/json"] package_pattern = r"(?P[\w\-\.]+)" source_pattern = r"{package}[-_]{version}{ext}" - hoster_pattern = (r"(?P" - r"files.pythonhosted.org/packages|" - r"pypi.python.org/packages|" - r"pypi.io/packages)") + hoster_pattern = ( + r"(?P" + r"files.pythonhosted.org/packages|" + r"pypi.python.org/packages|" + r"pypi.io/packages)" + ) url_pattern = r"{hoster}/.*/{source}" class Bioarchive(JSONHoster): """Scans for updates to packages hosted on bioarchive.galaxyproject.org""" + async def get_versions_from_json(self, data, req, orig_version): try: latest = data["info"]["Version"] - vals = {key: val - for key, val in self.vals.items() - if key not in self.exclude} - vals['version'] = latest + vals = { + key: val for key, val in self.vals.items() if key not in self.exclude + } + vals["version"] = latest link = replace_named_capture_group(self.link_pattern, vals) - return [{ - "link": link, - "version": latest, - }] + return [ + { + "link": link, + "version": latest, + } + ] except KeyError: return [] @@ -570,53 +637,55 @@ async def get_versions_from_json(self, data, req, orig_version): class CPAN(JSONHoster): """Scans for updates to Perl packages hosted on CPAN""" + @staticmethod def parse_deps(data): """Parse CPAN format dependencies""" run_deps = {} host_deps = {} for dep in data: - if dep['relationship'] != 'requires': + if dep["relationship"] != "requires": continue - if dep['module'] in ('strict', 'warnings'): + if dep["module"] in ("strict", "warnings"): continue - name = dep['module'].lower().replace('::', '-') - if 'version' in dep and dep['version'] not in ('0', None, 'undef'): - version = ">="+str(dep['version']) + name = dep["module"].lower().replace("::", "-") + if "version" in dep and dep["version"] not in ("0", None, "undef"): + version = ">=" + str(dep["version"]) else: - version = '' - if name != 'perl': - name = 'perl-' + name + version = "" + if name != "perl": + name = "perl-" + name else: - version = '' + version = "" - if dep['phase'] == 'runtime': + if dep["phase"] == "runtime": run_deps[name] = version - elif dep['phase'] in ('build', 'configure', 'test'): + elif dep["phase"] in ("build", "configure", "test"): host_deps[name] = version - return {'host': host_deps, 'run': run_deps} + return {"host": host_deps, "run": run_deps} async def get_versions_from_json(self, data, req, orig_version): try: version = { - 'link': data['download_url'], - 'version': str(data['version']), - 'depends': self.parse_deps(data['dependency']) + "link": data["download_url"], + "version": str(data["version"]), + "depends": self.parse_deps(data["dependency"]), } result = [version] - if version['version'] != orig_version: - url = self.orig_release_format.format(vers=orig_version, - dist=data['distribution']) + if version["version"] != orig_version: + url = self.orig_release_format.format( + vers=orig_version, dist=data["distribution"] + ) text = await req.get_text_from_url(url) data2 = json.loads(text) - if data2['hits']['total']: - data = data2['hits']['hits'][0]['_source'] + if data2["hits"]["total"]: + data = data2["hits"]["hits"][0]["_source"] orig_vers = { - 'link': data['download_url'], - 'version': str(data['version']), - 'depends': self.parse_deps(data['dependency']) + "link": data["download_url"], + "version": str(data["version"]), + "depends": self.parse_deps(data["dependency"]), } result.append(orig_vers) return result @@ -625,47 +694,58 @@ async def get_versions_from_json(self, data, req, orig_version): package_pattern = r"(?P[-\w.+]+)" author_pattern = r"(?P[A-Z]+)" - url_pattern = (r"(www.cpan.org|cpan.metacpan.org|search.cpan.org/CPAN)" - r"/authors/id/./../{author}/([^/]+/|){package}-v?{version}{ext}") + url_pattern = ( + r"(www.cpan.org|cpan.metacpan.org|search.cpan.org/CPAN)" + r"/authors/id/./../{author}/([^/]+/|){package}-v?{version}{ext}" + ) releases_formats = ["https://fastapi.metacpan.org/v1/release/{package}"] - orig_release_format = ("https://fastapi.metacpan.org/v1/release/_search" - "?q=distribution:{dist}%20AND%20version:{vers}") + orig_release_format = ( + "https://fastapi.metacpan.org/v1/release/_search" + "?q=distribution:{dist}%20AND%20version:{vers}" + ) class CRAN(JSONHoster): """R packages hosted on r-project.org (CRAN)""" + async def get_versions_from_json(self, data, _, orig_version): res = [] versions = list(set((str(data["latest"]), self.vals["version"], orig_version))) for vers in versions: - if vers not in data['versions']: + if vers not in data["versions"]: continue - vdata = data['versions'][vers] + vdata = data["versions"][vers] depends = { - "r-" + pkg.lower() if pkg != 'R' else 'r-base': - spec.replace(" ", "").replace("\n", "").replace("*", "") - for pkg, spec in chain(vdata.get('Depends', {}).items(), - vdata.get('Imports', {}).items(), - vdata.get('LinkingTo', {}).items()) + "r-" + pkg.lower() + if pkg != "R" + else "r-base": spec.replace(" ", "").replace("\n", "").replace("*", "") + for pkg, spec in chain( + vdata.get("Depends", {}).items(), + vdata.get("Imports", {}).items(), + vdata.get("LinkingTo", {}).items(), + ) } version = { - 'link': '', - 'version': vers, - 'depends': {'host': depends, 'run': depends}, + "link": "", + "version": vers, + "depends": {"host": depends, "run": depends}, } res.append(version) return res package_pattern = r"(?P[\w.]+)" - url_pattern = (r"r-project\.org/src/contrib" - r"(/Archive)?/{package}(?(1)/{package}|)" - r"_{version}{ext}") + url_pattern = ( + r"r-project\.org/src/contrib" + r"(/Archive)?/{package}(?(1)/{package}|)" + r"_{version}{ext}" + ) releases_formats = ["https://crandb.r-pkg.org/{package}/all"] # pylint: disable=abstract-method class BitBucketBase(OrderedHTMLHoster): # abstract """Base class for hosting at bitbucket.org""" + account_pattern = r"(?P[-\w]+)" project_pattern = r"(?P[-.\w]+)" prefix_pattern = r"(?P[-_./\w]+?)??" @@ -674,24 +754,33 @@ class BitBucketBase(OrderedHTMLHoster): # abstract class BitBucketTag(BitBucketBase): """Tag based releases hosted at bitbucket.org""" + link_pattern = "/{account}/{project}/get/{prefix}{version}{ext}" - releases_formats = ["https://bitbucket.org/{account}/{project}/downloads/?tab=tags", - "https://bitbucket.org/{account}/{project}/downloads/?tab=branches"] + releases_formats = [ + "https://bitbucket.org/{account}/{project}/downloads/?tab=tags", + "https://bitbucket.org/{account}/{project}/downloads/?tab=branches", + ] class BitBucketDownload(BitBucketBase): """Uploaded releases hosted at bitbucket.org""" + link_pattern = "/{account}/{project}/downloads/{prefix}{version}{ext}" - releases_formats = ["https://bitbucket.org/{account}/{project}/downloads/?tab=downloads"] + releases_formats = [ + "https://bitbucket.org/{account}/{project}/downloads/?tab=downloads" + ] class GitlabTag(OrderedHTMLHoster): """Tag based releases hosted at gitlab.com""" + account_pattern = r"(?P[-\w]+)" subgroup_pattern = r"(?P(?:/[-\w]+|))" project_pattern = r"(?P[-.\w]+)" - link_pattern = (r"/{account}{subgroup}/{project}/(repository|-/archive)/" - r"{version}/(archive|{project}-{version}){ext}") + link_pattern = ( + r"/{account}{subgroup}/{project}/(repository|-/archive)/" + r"{version}/(archive|{project}-{version}){ext}" + ) url_pattern = r"gitlab\.com{link}" releases_formats = ["https://gitlab.com/{account}{subgroup}/{project}/tags"] diff --git a/bioconda_utils/lint/__init__.py b/bioconda_utils/lint/__init__.py index 3e12499426..2fc4ac4560 100644 --- a/bioconda_utils/lint/__init__.py +++ b/bioconda_utils/lint/__init__.py @@ -116,6 +116,7 @@ class Severity(IntEnum): """Severities for lint checks""" + #: Checks of this severity are purely informational INFO = 10 @@ -125,6 +126,7 @@ class Severity(IntEnum): #: Checks of this severity must be fixed and will fail a recipe. ERROR = 30 + INFO = Severity.INFO WARNING = Severity.WARNING ERROR = Severity.ERROR @@ -137,7 +139,7 @@ class LintMessage(NamedTuple): recipe: _recipe.Recipe #: The check issuing the message - check: 'LintCheck' + check: "LintCheck" #: The severity of the message severity: Severity = ERROR @@ -177,8 +179,9 @@ class LintCheckMeta(abc.ABCMeta): registry: List["LintCheck"] = [] - def __new__(cls, name: str, bases: Tuple[type, ...], - namespace: Dict[str, Any], **kwargs) -> type: + def __new__( + cls, name: str, bases: Tuple[type, ...], namespace: Dict[str, Any], **kwargs + ) -> type: """Creates LintCheck classes""" typ = super().__new__(cls, name, bases, namespace, **kwargs) if name != "LintCheck": # don't register base class @@ -188,14 +191,17 @@ def __new__(cls, name: str, bases: Tuple[type, ...], def __str__(cls): return cls.__name__ + _checks_loaded = False + + def get_checks(): """Loads and returns the available lint checks""" global _checks_loaded if not _checks_loaded: for _loader, name, _ispkg in pkgutil.iter_modules(__path__): - if name.startswith('check_'): - importlib.import_module(__name__ + '.' + name) + if name.startswith("check_"): + importlib.import_module(__name__ + "." + name) _checks_loaded = True return LintCheckMeta.registry @@ -207,9 +213,9 @@ class LintCheck(metaclass=LintCheckMeta): severity: Severity = ERROR #: Checks that must have passed for this check to be executed. - requires: List['LintCheck'] = [] + requires: List["LintCheck"] = [] - def __init__(self, _linter: 'Linter') -> None: + def __init__(self, _linter: "Linter") -> None: #: Messages collected running tests self.messages: List[LintMessage] = [] #: Recipe currently being checked @@ -235,12 +241,12 @@ def run(self, recipe: _recipe.Recipe, fix: bool = False) -> List[LintMessage]: self.check_recipe(recipe) # Run per source checks - source = recipe.get('source', None) + source = recipe.get("source", None) if isinstance(source, dict): - self.check_source(source, 'source') + self.check_source(source, "source") elif isinstance(source, list): for num, src in enumerate(source): - self.check_source(src, f'source/{num}') + self.check_source(src, f"source/{num}") # Run depends checks self.check_deps(recipe.get_deps_dict()) @@ -288,8 +294,9 @@ def check_deps(self, deps: Dict[str, List[str]]) -> None: def fix(self, message, data) -> LintMessage: """Attempt to fix the problem""" - def message(self, section: str = None, fname: str = None, line: int = None, - data: Any = None) -> None: + def message( + self, section: str = None, fname: str = None, line: int = None, data: Any = None + ) -> None: """Add a message to the lint results Also calls `fix` if we are supposed to be fixing. @@ -304,15 +311,20 @@ def message(self, section: str = None, fname: str = None, line: int = None, data: Data to be passed to `fix`. If check can fix, set this to something other than None. """ - message = self.make_message(self.recipe, section, fname, line, - data is not None) + message = self.make_message(self.recipe, section, fname, line, data is not None) if data is not None and self.try_fix and self.fix(message, data): return self.messages.append(message) @classmethod - def make_message(cls, recipe: _recipe.Recipe, section: str = None, - fname: str = None, line=None, canfix: bool=False) -> LintMessage: + def make_message( + cls, + recipe: _recipe.Recipe, + section: str = None, + fname: str = None, + line=None, + canfix: bool = False, + ) -> LintMessage: """Create a LintMessage Args: @@ -324,8 +336,8 @@ def make_message(cls, recipe: _recipe.Recipe, section: str = None, line: If specified, sets the line number for the message directly """ doc = inspect.getdoc(cls) - doc = doc.replace('::', ':').replace('``', '`') - title, _, body = doc.partition('\n') + doc = doc.replace("::", ":").replace("``", "`") + title, _, body = doc.partition("\n") if section: try: sl, sc, el, ec = recipe.get_raw_range(section) @@ -341,15 +353,17 @@ def make_message(cls, recipe: _recipe.Recipe, section: str = None, if not fname: fname = recipe.path - return LintMessage(recipe=recipe, - check=cls, - severity=cls.severity, - title=title.strip(), - body=body, - fname=fname, - start_line=start_line, - end_line=end_line, - canfix=canfix) + return LintMessage( + recipe=recipe, + check=cls, + severity=cls.severity, + title=title.strip(), + body=body, + fname=fname, + start_line=start_line, + end_line=end_line, + canfix=canfix, + ) class linter_failure(LintCheck): @@ -446,8 +460,6 @@ class unknown_check(LintCheck): } - - class Linter: """Lint executor @@ -465,8 +477,14 @@ class Linter: nocatch: Don't catch exceptions in lint checks and turn them into linter_error lint messages. Used by tests. """ - def __init__(self, config: Dict, recipe_folder: str, - exclude: List[str] = None, nocatch: bool=False) ->None: + + def __init__( + self, + config: Dict, + recipe_folder: str, + exclude: List[str] = None, + nocatch: bool = False, + ) -> None: self.config = config self.recipe_folder = recipe_folder self.skip = self.load_skips() @@ -521,17 +539,16 @@ def load_skips(self): skip_dict = defaultdict(list) commit_message = "" - if 'LINT_SKIP' in os.environ: + if "LINT_SKIP" in os.environ: # Allow overwriting of commit message - commit_message = os.environ['LINT_SKIP'] - elif os.path.exists('.git'): + commit_message = os.environ["LINT_SKIP"] + elif os.path.exists(".git"): # Obtain commit message from last commit. commit_message = utils.run( - ['git', 'log', '--format=%B', '-n', '1'], mask=False, loglevel=0 + ["git", "log", "--format=%B", "-n", "1"], mask=False, loglevel=0 ).stdout - skip_re = re.compile( - r'\[\s*lint skip (?P\w+) for (?P.*?)\s*\]') + skip_re = re.compile(r"\[\s*lint skip (?P\w+) for (?P.*?)\s*\]") to_skip = skip_re.findall(commit_message) for func, recipe in to_skip: @@ -563,8 +580,7 @@ def lint(self, recipe_names: List[str], fix: bool = False) -> bool: msgs = [linter_failure.make_message(recipe=recipe)] self._messages.extend(msgs) - return any(message.severity >= ERROR - for message in self._messages) + return any(message.severity >= ERROR for message in self._messages) def lint_one(self, recipe_name: str, fix: bool = False) -> List[LintMessage]: """Run the linter on a single recipe @@ -581,18 +597,15 @@ def lint_one(self, recipe_name: str, fix: bool = False) -> List[LintMessage]: except _recipe.RecipeError as exc: recipe = _recipe.Recipe(recipe_name, self.recipe_folder) check_cls = recipe_error_to_lint_check.get(exc.__class__, linter_failure) - return [check_cls.make_message( - recipe=recipe, - line=getattr(exc, 'line') - )] + return [check_cls.make_message(recipe=recipe, line=getattr(exc, "line"))] # collect checks to skip checks_to_skip = set(self.skip[recipe_name]) checks_to_skip.update(self.exclude) - if isinstance(recipe.get('extra/skip-lints', []), list): + if isinstance(recipe.get("extra/skip-lints", []), list): # If they are not, the extra_skip_lints_not_list check # will be found and issued. - checks_to_skip.update(recipe.get('extra/skip-lints', [])) + checks_to_skip.update(recipe.get("extra/skip-lints", [])) # also skip dependent checks for check in list(checks_to_skip): @@ -601,8 +614,7 @@ def lint_one(self, recipe_name: str, fix: bool = False) -> List[LintMessage]: continue for check_dep in nx.ancestors(self.checks_dag, check): if check_dep not in checks_to_skip: - logger.info("Disabling %s because %s is disabled", - check_dep, check) + logger.info("Disabling %s because %s is disabled", check_dep, check) checks_to_skip.add(check_dep) # run checks @@ -616,11 +628,13 @@ def lint_one(self, recipe_name: str, fix: bool = False) -> List[LintMessage]: if self.nocatch: raise logger.exception("Unexpected exception in lint_one") - res = [LintMessage( - recipe=recipe, - check=check, - severity=ERROR, - title="Check raised an unexpected exception") + res = [ + LintMessage( + recipe=recipe, + check=check, + severity=ERROR, + title="Check raised an unexpected exception", + ) ] if res: # skip checks depending on failed checks @@ -628,7 +642,7 @@ def lint_one(self, recipe_name: str, fix: bool = False) -> List[LintMessage]: messages.extend(res) if fix and recipe.is_modified(): - with open(recipe.path, 'w', encoding='utf-8') as fdes: + with open(recipe.path, "w", encoding="utf-8") as fdes: fdes.write(recipe.dump()) for message in messages: diff --git a/bioconda_utils/lint/check_build_help.py b/bioconda_utils/lint/check_build_help.py index eae342ea85..3820a57d94 100644 --- a/bioconda_utils/lint/check_build_help.py +++ b/bioconda_utils/lint/check_build_help.py @@ -29,8 +29,8 @@ class should_use_compilers(LintCheck): conda-build itself. """ - compilers = ('gcc', 'llvm', 'libgfortran', 'libgcc', 'go', 'cgo', - 'toolchain') + + compilers = ("gcc", "llvm", "libgfortran", "libgcc", "go", "cgo", "toolchain") def check_deps(self, deps): for compiler in self.compilers: @@ -45,15 +45,15 @@ class compilers_must_be_in_build(LintCheck): ``requirements: build:`` section. """ + def check_deps(self, deps): for dep in deps: - if dep.startswith('compiler_'): + if dep.startswith("compiler_"): for location in deps[dep]: - if 'run' in location or 'host' in location: + if "run" in location or "host" in location: self.message(section=location) - class uses_setuptools(LintCheck): """The recipe uses setuptools in run depends @@ -62,10 +62,11 @@ class uses_setuptools(LintCheck): pkg_resources or setuptools console scripts). """ + severity = INFO def check_recipe(self, recipe): - if 'setuptools' in recipe.get_deps('run'): + if "setuptools" in recipe.get_deps("run"): self.message() @@ -81,27 +82,28 @@ class setup_py_install_args(LintCheck): requires defines entrypoints in its ``setup.py``. """ + @staticmethod def _check_line(line: str) -> bool: """Check a line for a broken call to setup.py""" - if 'setup.py install' not in line: + if "setup.py install" not in line: return True - if '--single-version-externally-managed' in line: + if "--single-version-externally-managed" in line: return True return False def check_deps(self, deps): - if 'setuptools' not in deps: + if "setuptools" not in deps: return # no setuptools, no problem - if not self._check_line(self.recipe.get('build/script', '')): - self.message(section='build/script') + if not self._check_line(self.recipe.get("build/script", "")): + self.message(section="build/script") try: - with open(os.path.join(self.recipe.dir, 'build.sh')) as buildsh: + with open(os.path.join(self.recipe.dir, "build.sh")) as buildsh: for num, line in enumerate(buildsh): if not self._check_line(line): - self.message(fname='build.sh', line=num) + self.message(fname="build.sh", line=num) except FileNotFoundError: pass @@ -115,10 +117,10 @@ class cython_must_be_in_host(LintCheck): host: - cython """ + def check_deps(self, deps): - if 'cython' in deps: - if any('host' not in location - for location in deps['cython']): + if "cython" in deps: + if any("host" not in location for location in deps["cython"]): self.message() @@ -132,7 +134,9 @@ class cython_needs_compiler(LintCheck): - {{ compiler('c') }} """ + severity = WARNING + def check_deps(self, deps): - if 'cython' in deps and 'compiler_c' not in deps: + if "cython" in deps and "compiler_c" not in deps: self.message() diff --git a/bioconda_utils/lint/check_completeness.py b/bioconda_utils/lint/check_completeness.py index 146b5be6bb..c5c5e65d3f 100644 --- a/bioconda_utils/lint/check_completeness.py +++ b/bioconda_utils/lint/check_completeness.py @@ -15,9 +15,10 @@ class missing_build_number(LintCheck): build: number: 0 """ + def check_recipe(self, recipe): - if not recipe.get('build/number', ''): - self.message(section='build') + if not recipe.get("build/number", ""): + self.message(section="build") class missing_home(LintCheck): @@ -29,9 +30,10 @@ class missing_home(LintCheck): home: """ + def check_recipe(self, recipe): - if not recipe.get('about/home', ''): - self.message(section='about') + if not recipe.get("about/home", ""): + self.message(section="about") class missing_summary(LintCheck): @@ -43,9 +45,10 @@ class missing_summary(LintCheck): summary: One line briefly describing package """ + def check_recipe(self, recipe): - if not recipe.get('about/summary', ''): - self.message(section='about') + if not recipe.get("about/summary", ""): + self.message(section="about") class missing_license(LintCheck): @@ -57,9 +60,10 @@ class missing_license(LintCheck): license: """ + def check_recipe(self, recipe): - if not recipe.get('about/license', ''): - self.message(section='about') + if not recipe.get("about/license", ""): + self.message(section="about") class missing_tests(LintCheck): @@ -82,16 +86,16 @@ class missing_tests(LintCheck): ``run_test.pl`` executing tests. """ - test_files = ['run_test.py', 'run_test.sh', 'run_test.pl'] + + test_files = ["run_test.py", "run_test.sh", "run_test.pl"] def check_recipe(self, recipe): - if any(os.path.exists(os.path.join(recipe.dir, f)) - for f in self.test_files): + if any(os.path.exists(os.path.join(recipe.dir, f)) for f in self.test_files): return - if recipe.get('test/commands', '') or recipe.get('test/imports', ''): + if recipe.get("test/commands", "") or recipe.get("test/imports", ""): return - if recipe.get('test', False) is not False: - self.message(section='test') + if recipe.get("test", False) is not False: + self.message(section="test") else: self.message() @@ -105,9 +109,9 @@ class missing_hash(LintCheck): sha256: checksum-value """ - checksum_names = ('md5', 'sha1', 'sha256') + + checksum_names = ("md5", "sha1", "sha256") def check_source(self, source, section): if not any(source.get(chk) for chk in self.checksum_names): self.message(section=section) - diff --git a/bioconda_utils/lint/check_deprecation.py b/bioconda_utils/lint/check_deprecation.py index 6741432ddc..df47604f80 100644 --- a/bioconda_utils/lint/check_deprecation.py +++ b/bioconda_utils/lint/check_deprecation.py @@ -11,13 +11,13 @@ class uses_perl_threaded(LintCheck): Please use ``perl`` instead. """ + def check_deps(self, deps): - if 'perl-threaded' in deps: + if "perl-threaded" in deps: self.message(data=True) def fix(self, _message, _data): - self.recipe.replace('perl-threaded', 'perl', - within=('requirements', 'outputs')) + self.recipe.replace("perl-threaded", "perl", within=("requirements", "outputs")) self.recipe.render() return True @@ -28,13 +28,13 @@ class uses_javajdk(LintCheck): Please use ``openjdk`` instead. """ + def check_deps(self, deps): - if 'java-jdk' in deps: + if "java-jdk" in deps: self.message(data=True) def fix(self, _message, _data): - self.recipe.replace('java-jdk', 'openjdk', - within=('requirements', 'outputs')) + self.recipe.replace("java-jdk", "openjdk", within=("requirements", "outputs")) return True @@ -44,17 +44,17 @@ class deprecated_numpy_spec(LintCheck): Please remove the ``x.x`` - pinning is now handled automatically. """ + def check_deps(self, deps): - if 'numpy' not in deps: + if "numpy" not in deps: return - for path in deps['numpy']: - line, _, _ = self.recipe.get_raw(path).partition('#') - if 'x.x' in line: + for path in deps["numpy"]: + line, _, _ = self.recipe.get_raw(path).partition("#") + if "x.x" in line: self.message(section=path, data=True) def fix(self, _message, _data): - self.recipe.replace('numpy x.x', 'numpy', - within=('requirements', 'outputs')) + self.recipe.replace("numpy x.x", "numpy", within=("requirements", "outputs")) return True @@ -65,11 +65,13 @@ class uses_matplotlib(LintCheck): unless the package explicitly needs the PyQt interactive plotting backend. """ + def check_deps(self, deps): - if 'matplotlib' in deps: + if "matplotlib" in deps: self.message(data=True) def fix(self, _message, _data): - self.recipe.replace('matplotlib', 'matplotlib-base', - within=('requirements', 'outputs')) + self.recipe.replace( + "matplotlib", "matplotlib-base", within=("requirements", "outputs") + ) return True diff --git a/bioconda_utils/lint/check_noarch.py b/bioconda_utils/lint/check_noarch.py index a9582bc9c9..94bc1b4095 100644 --- a/bioconda_utils/lint/check_noarch.py +++ b/bioconda_utils/lint/check_noarch.py @@ -22,6 +22,7 @@ # b) Not use ``- python [<>]3``, # but use ``skip: True # [py[23]k]`` + class should_be_noarch_python(LintCheck): """The recipe should be build as ``noarch`` @@ -35,19 +36,20 @@ class should_be_noarch_python(LintCheck): subset of packages. """ + def check_deps(self, deps): - if 'python' not in deps: + if "python" not in deps: return # not a python package - if all('build' not in loc for loc in deps['python']): + if all("build" not in loc for loc in deps["python"]): return # only uses python in run/host - if any(dep.startswith('compiler_') for dep in deps): + if any(dep.startswith("compiler_") for dep in deps): return # not compiled - if self.recipe.get('build/noarch', None) == 'python': + if self.recipe.get("build/noarch", None) == "python": return # already marked noarch: python - self.message(section='build', data=True) + self.message(section="build", data=True) def fix(self, _message, _data): - self.recipe.set('build/noarch', 'python') + self.recipe.set("build/noarch", "python") return True @@ -64,16 +66,18 @@ class should_be_noarch_generic(LintCheck): packages. """ - requires = ['should_be_noarch_python'] + + requires = ["should_be_noarch_python"] + def check_deps(self, deps): - if any(dep.startswith('compiler_') for dep in deps): + if any(dep.startswith("compiler_") for dep in deps): return # not compiled - if self.recipe.get('build/noarch', None): + if self.recipe.get("build/noarch", None): return # already marked noarch - self.message(section='build', data=True) + self.message(section="build", data=True) def fix(self, _message, _data): - self.recipe.set('build/noarch', 'generic') + self.recipe.set("build/noarch", "generic") return True @@ -85,12 +89,13 @@ class should_not_be_noarch_compiler(LintCheck): Please remove the ``build: noarch:`` section. """ + def check_deps(self, deps): - if not any(dep.startswith('compiler_') for dep in deps): + if not any(dep.startswith("compiler_") for dep in deps): return # not compiled - if self.recipe.get('build/noarch', False) is False: + if self.recipe.get("build/noarch", False) is False: return # no noarch, or noarch=False - self.message(section='build/noarch') + self.message(section="build/noarch") class should_not_be_noarch_skip(LintCheck): @@ -99,12 +104,13 @@ class should_not_be_noarch_skip(LintCheck): Recipes marked as ``noarch`` cannot use skip. """ + def check_recipe(self, recipe): - if self.recipe.get('build/noarch', False) is False: + if self.recipe.get("build/noarch", False) is False: return # no noarch, or noarch=False - if self.recipe.get('build/skip', False) is False: + if self.recipe.get("build/skip", False) is False: return # no skip or skip=False - self.message(section='build/noarch') + self.message(section="build/noarch") class should_not_use_skip_python(LintCheck): @@ -124,19 +130,20 @@ class should_not_use_skip_python(LintCheck): skips. """ - bad_skip_terms = ('py2k', 'py3k', 'python') + + bad_skip_terms = ("py2k", "py3k", "python") def check_deps(self, deps): - if 'python' not in deps: + if "python" not in deps: return # not a python package - if any(dep.startswith('compiler_') for dep in deps): + if any(dep.startswith("compiler_") for dep in deps): return # not compiled - if self.recipe.get('build/skip', None) is None: + if self.recipe.get("build/skip", None) is None: return # no build: skip: section - skip_line = self.recipe.get_raw('build/skip') + skip_line = self.recipe.get_raw("build/skip") if not any(term in skip_line for term in self.bad_skip_terms): return # no offending skip terms - self.message(section='build/skip') + self.message(section="build/skip") class should_not_be_noarch_source(LintCheck): @@ -146,10 +153,11 @@ class should_not_be_noarch_source(LintCheck): platform. Remove the noarch section or use just one source for all platforms. """ - _pat = re.compile(r'# +\[.*\]') + + _pat = re.compile(r"# +\[.*\]") def check_source(self, source, section): - if self.recipe.get('build/noarch', False) is False: + if self.recipe.get("build/noarch", False) is False: return # no noarch, or noarch=False # just search the entire source entry for a comment if self._pat.search(self.recipe.get_raw(f"{section}")): diff --git a/bioconda_utils/lint/check_policy.py b/bioconda_utils/lint/check_policy.py index d5aadbe15f..8000b45375 100644 --- a/bioconda_utils/lint/check_policy.py +++ b/bioconda_utils/lint/check_policy.py @@ -11,6 +11,7 @@ from . import LintCheck, ERROR, WARNING, INFO from bioconda_utils import utils + class uses_vcs_url(LintCheck): """The recipe downloads source from a VCS @@ -18,21 +19,24 @@ class uses_vcs_url(LintCheck): ``svn_url`` or ``hg_url`` feature of conda. """ + def check_source(self, source, section): - for vcs in ('git', 'svn', 'hg'): + for vcs in ("git", "svn", "hg"): if f"{vcs}_url" in source: self.message(section=f"{section}/{vcs}_url") + class folder_and_package_name_must_match(LintCheck): """The recipe folder and package name do not match. For clarity, the name of the folder the ``meta.yaml`` resides, in and the name of the toplevel package should match. """ + def check_recipe(self, recipe): - recipe_base_folder, _, _ = recipe.reldir.partition('/') - if recipe.name != recipe_base_folder: - self.message(section='package/name') + recipe_base_folder, _, _ = recipe.reldir.partition("/") + if recipe.name != recipe_base_folder: + self.message(section="package/name") class gpl_requires_license_distributed(LintCheck): @@ -47,13 +51,14 @@ class gpl_requires_license_distributed(LintCheck): If the upstream tar ball does not include a copy, please ask the authors of the software to add it to their distribution archive. """ + severity = WARNING requires = ["missing_license"] def check_recipe(self, recipe): - if 'gpl' in recipe.get('about/license').lower(): - if not recipe.get('about/license_file', ''): - self.message('about/license') + if "gpl" in recipe.get("about/license").lower(): + if not recipe.get("about/license_file", ""): + self.message("about/license") class should_not_use_fn(LintCheck): @@ -62,9 +67,10 @@ class should_not_use_fn(LintCheck): There is no need to specify the filename as the URL should give a name and it will in most cases be unpacked automatically. """ + def check_source(self, source, section): - if 'fn' in source: - self.message(section=section+'/fn') + if "fn" in source: + self.message(section=section + "/fn") class has_windows_bat_file(LintCheck): @@ -78,8 +84,9 @@ class has_windows_bat_file(LintCheck): from the recipe directory. """ + def check_recipe(self, recipe): - for fname in glob.glob(os.path.join(recipe.dir, '*.bat')): + for fname in glob.glob(os.path.join(recipe.dir, "*.bat")): self.message(fname=fname) @@ -101,11 +108,13 @@ class long_summary(LintCheck): description to be one or more paragraphs. """ + severity = WARNING max_length = 120 + def check_recipe(self, recipe): - if len(recipe.get('about/summary', '')) > self.max_length: - self.message('about/summary') + if len(recipe.get("about/summary", "")) > self.max_length: + self.message("about/summary") class cran_packages_to_conda_forge(LintCheck): @@ -115,13 +124,16 @@ class cran_packages_to_conda_forge(LintCheck): from Bioconda. It should therefore be moved to Conda-Forge. """ + def check_deps(self, deps): # must have R in run a run dep - if 'R' in deps and any('run' in dep for dep in deps['R']): + if "R" in deps and any("run" in dep for dep in deps["R"]): # and all deps satisfied in conda-forge - if all(utils.RepoData().get_package_data(name=dep, channels='conda-forge') - for dep in deps): - self.message() + if all( + utils.RepoData().get_package_data(name=dep, channels="conda-forge") + for dep in deps + ): + self.message() class version_starts_with_v(LintCheck): @@ -130,6 +142,7 @@ class version_starts_with_v(LintCheck): Version numbers in Conda recipes need to follow PEP 386 """ + def check_recipe(self, recipe): - if recipe.get('package/version', '').startswith('v'): + if recipe.get("package/version", "").startswith("v"): self.message() diff --git a/bioconda_utils/lint/check_repo.py b/bioconda_utils/lint/check_repo.py index cde5c403d4..3631843542 100644 --- a/bioconda_utils/lint/check_repo.py +++ b/bioconda_utils/lint/check_repo.py @@ -7,6 +7,7 @@ from .. import utils from . import LintCheck, ERROR, WARNING, INFO + class in_other_channels(LintCheck): """A package of the same name already exists in another channel @@ -21,10 +22,11 @@ class in_other_channels(LintCheck): new home at conda-forge. """ + def check_recipe(self, recipe): channels = utils.RepoData().get_package_data(key="channel", name=recipe.name) - if set(channels) - set(('bioconda',)): - self.message(section='package/name') + if set(channels) - set(("bioconda",)): + self.message(section="package/name") class build_number_needs_bump(LintCheck): @@ -35,12 +37,13 @@ class build_number_needs_bump(LintCheck): channel. Please increase the build number. """ + def check_recipe(self, recipe): bldnos = utils.RepoData().get_package_data( - key="build_number", - name=recipe.name, version=recipe.version) + key="build_number", name=recipe.name, version=recipe.version + ) if bldnos and recipe.build_number <= max(bldnos): - self.message('build/number', data=max(bldnos)) + self.message("build/number", data=max(bldnos)) def fix(self, _message, data): self.recipe.reset_buildnumber(data + 1) @@ -53,13 +56,15 @@ class build_number_needs_reset(LintCheck): No previous build of a package of this name and this version exists, the build number should therefore be 0. """ - requires = ['missing_build_number'] + + requires = ["missing_build_number"] + def check_recipe(self, recipe): bldnos = utils.RepoData().get_package_data( - key="build_number", - name=recipe.name, version=recipe.version) + key="build_number", name=recipe.name, version=recipe.version + ) if not bldnos and recipe.build_number > 0: - self.message('build/number', data=0) + self.message("build/number", data=0) def fix(self, _message, data): self.recipe.reset_buildnumber(data) @@ -72,18 +77,19 @@ class recipe_is_blacklisted(LintCheck): If you are intending to repair this recipe, remove it from the build fail blacklist. """ + def __init__(self, linter): super().__init__(linter) self.blacklist = linter.get_blacklist() - self.blacklists = linter.config.get('blacklists') + self.blacklists = linter.config.get("blacklists") def check_recipe(self, recipe): if recipe.name in self.blacklist: - self.message(section='package/name', data=True) + self.message(section="package/name", data=True) def fix(self, _message, _data): for blacklist in self.blacklists: - with open(blacklist, 'r') as fdes: + with open(blacklist, "r") as fdes: data = fdes.readlines() for num, line in enumerate(data): if self.recipe.name in line: @@ -91,8 +97,8 @@ def fix(self, _message, _data): else: continue del data[num] - with open(blacklist, 'w') as fdes: - fdes.write(''.join(data)) + with open(blacklist, "w") as fdes: + fdes.write("".join(data)) break else: return False diff --git a/bioconda_utils/lint/check_syntax.py b/bioconda_utils/lint/check_syntax.py index ea6f76ab2f..a9840bb80c 100644 --- a/bioconda_utils/lint/check_syntax.py +++ b/bioconda_utils/lint/check_syntax.py @@ -18,10 +18,11 @@ class version_constraints_missing_whitespace(LintCheck): python >=3 """ + def check_recipe(self, recipe): check_paths = [] - for section in ('build', 'run', 'host'): - check_paths.append(f'requirements/{section}') + for section in ("build", "run", "host"): + check_paths.append(f"requirements/{section}") constraints = re.compile("(.*?)([<=>].*)") for path in check_paths: @@ -34,8 +35,8 @@ def check_recipe(self, recipe): def fix(self, _message, _data): check_paths = [] - for section in ('build', 'run', 'host'): - check_paths.append(f'requirements/{section}') + for section in ("build", "run", "host"): + check_paths.append(f"requirements/{section}") constraints = re.compile("(.*?)([<=>].*)") for path in check_paths: @@ -45,7 +46,7 @@ def fix(self, _message, _data): space_separated = has_constraints[1].endswith(" ") if not space_separated: dep, ver = has_constraints.groups() - self.recipe.replace(spec, f"{dep} {ver}", within='requirements') + self.recipe.replace(spec, f"{dep} {ver}", within="requirements") return True @@ -59,10 +60,11 @@ class extra_identifiers_not_list(LintCheck): - doi:123 """ + def check_recipe(self, recipe): - identifiers = recipe.get('extra/identifiers', None) + identifiers = recipe.get("extra/identifiers", None) if identifiers and not isinstance(identifiers, list): - self.message(section='extra/identifiers') + self.message(section="extra/identifiers") class extra_identifiers_not_string(LintCheck): @@ -77,13 +79,14 @@ class extra_identifiers_not_string(LintCheck): Note that there is no space around the colon """ + requires = [extra_identifiers_not_list] def check_recipe(self, recipe): - identifiers = recipe.get('extra/identifiers', []) + identifiers = recipe.get("extra/identifiers", []) for n, identifier in enumerate(identifiers): if not isinstance(identifier, str): - self.message(section=f'extra/identifiers/{n}') + self.message(section=f"extra/identifiers/{n}") class extra_identifiers_missing_colon(LintCheck): @@ -96,13 +99,14 @@ class extra_identifiers_missing_colon(LintCheck): - doi:123 """ + requires = [extra_identifiers_not_string] def check_recipe(self, recipe): - identifiers = recipe.get('extra/identifiers', []) + identifiers = recipe.get("extra/identifiers", []) for n, identifier in enumerate(identifiers): - if ':' not in identifier: - self.message(section=f'extra/identifiers/{n}') + if ":" not in identifier: + self.message(section=f"extra/identifiers/{n}") class extra_skip_lints_not_list(LintCheck): @@ -115,7 +119,7 @@ class extra_skip_lints_not_list(LintCheck): - should_use_compilers """ - def check_recipe(self, recipe): - if not isinstance(recipe.get('extra/skip-lints', []), list): - self.message(section='extra/skip-lints') + def check_recipe(self, recipe): + if not isinstance(recipe.get("extra/skip-lints", []), list): + self.message(section="extra/skip-lints") diff --git a/bioconda_utils/pkg_test.py b/bioconda_utils/pkg_test.py index 47736c8f7e..b08861aa13 100644 --- a/bioconda_utils/pkg_test.py +++ b/bioconda_utils/pkg_test.py @@ -26,39 +26,37 @@ def get_tests(path): tmp = tempfile.mkdtemp() t = tarfile.open(path) t.extractall(tmp) - input_dir = os.path.join(tmp, 'info', 'recipe') + input_dir = os.path.join(tmp, "info", "recipe") tests = [ - '/usr/local/env-execute true', - '. /usr/local/env-activate.sh', + "/usr/local/env-execute true", + ". /usr/local/env-activate.sh", ] recipe_meta = MetaData(input_dir) - tests_commands = recipe_meta.get_value('test/commands') - tests_imports = recipe_meta.get_value('test/imports') - requirements = recipe_meta.get_value('requirements/run') + tests_commands = recipe_meta.get_value("test/commands") + tests_imports = recipe_meta.get_value("test/imports") + requirements = recipe_meta.get_value("requirements/run") if tests_imports or tests_commands: if tests_commands: - tests.append(' && '.join(tests_commands)) - if tests_imports and 'python' in requirements: + tests.append(" && ".join(tests_commands)) + if tests_imports and "python" in requirements: tests.append( - ' && '.join('python -c "import %s"' % imp - for imp in tests_imports) + " && ".join('python -c "import %s"' % imp for imp in tests_imports) ) elif tests_imports and ( - 'perl' in requirements or 'perl-threaded' in requirements + "perl" in requirements or "perl-threaded" in requirements ): tests.append( - ' && '.join('''perl -e "use %s;"''' % imp - for imp in tests_imports) + " && ".join('''perl -e "use %s;"''' % imp for imp in tests_imports) ) - tests = ' && '.join(tests) - tests = tests.replace('$R ', 'Rscript ') + tests = " && ".join(tests) + tests = tests.replace("$R ", "Rscript ") # this is specific to involucro, the way how we build our containers - tests = tests.replace('$PREFIX', '/usr/local') - tests = tests.replace('${PREFIX}', '/usr/local') + tests = tests.replace("$PREFIX", "/usr/local") + tests = tests.replace("${PREFIX}", "/usr/local") return f"bash -c {shlex.quote(tests)}" @@ -74,15 +72,15 @@ def get_image_name(path): Path to .tar.by2 package build by conda-build """ - assert path.endswith('.tar.bz2') + assert path.endswith(".tar.bz2") - pkg = os.path.basename(path).replace('.tar.bz2', '') - toks = pkg.split('-') + pkg = os.path.basename(path).replace(".tar.bz2", "") + toks = pkg.split("-") build_string = toks[-1] version = toks[-2] - name = '-'.join(toks[:-2]) + name = "-".join(toks[:-2]) - spec = '%s=%s--%s' % (name, version, build_string) + spec = "%s=%s--%s" % (name, version, build_string) return spec @@ -122,7 +120,7 @@ def test_package( tests. """ - assert path.endswith('.tar.bz2'), "Unrecognized path {0}".format(path) + assert path.endswith(".tar.bz2"), "Unrecognized path {0}".format(path) # assert os.path.exists(path), '{0} does not exist'.format(path) conda_bld_dir = os.path.abspath(os.path.dirname(os.path.dirname(path))) @@ -135,24 +133,26 @@ def test_package( raise ValueError('"local" must be in channel list') channels = [ - 'file://{0}'.format(conda_bld_dir) if channel == 'local' else channel + "file://{0}".format(conda_bld_dir) if channel == "local" else channel for channel in channels ] - channel_args = ['--channels', ','.join(channels)] + channel_args = ["--channels", ",".join(channels)] tests = get_tests(path) - logger.debug('Tests to run: %s', tests) + logger.debug("Tests to run: %s", tests) cmd = [ - 'mulled-build', - 'build-and-test', + "mulled-build", + "build-and-test", spec, - '-n', 'biocontainers', - '--test', tests + "-n", + "biocontainers", + "--test", + tests, ] if name_override: - cmd += ['--name-override', name_override] + cmd += ["--name-override", name_override] cmd += channel_args cmd += shlex.split(mulled_args) @@ -161,12 +161,12 @@ def test_package( # create activation / entrypoint scripts for the container. # We also inject a PREINSTALL to alias conda to mamba so `mamba install` is # used instead of `conda install` in the container builds. - involucro_path = os.path.join(os.path.dirname(__file__), 'involucro') + involucro_path = os.path.join(os.path.dirname(__file__), "involucro") if not os.path.exists(involucro_path): - raise RuntimeError('internal involucro wrapper missing') - cmd += ['--involucro-path', involucro_path] + raise RuntimeError("internal involucro wrapper missing") + cmd += ["--involucro-path", involucro_path] - logger.debug('mulled-build command: %s' % cmd) + logger.debug("mulled-build command: %s" % cmd) env = os.environ.copy() if base_image is not None: diff --git a/bioconda_utils/recipe.py b/bioconda_utils/recipe.py index ca955e6b1f..f108e5c858 100644 --- a/bioconda_utils/recipe.py +++ b/bioconda_utils/recipe.py @@ -45,7 +45,7 @@ # Hack: mirror stringify from conda-build in removing implicit # resolution of numbers -for digit in '0123456789': +for digit in "0123456789": if digit in yaml.resolver.versioned_resolver: del yaml.resolver.versioned_resolver[digit] @@ -75,21 +75,25 @@ class DuplicateKey(RecipeError): For duplicate keys that are a result of ``# [osx]`` style line selectors, `Recipe` attempts to resolve them as a list of dictionaries instead. """ + template = "has duplicate key" class MissingKey(RecipeError): """Raised if a recipe is missing package/version or package/name""" + template = "has missing key" class EmptyRecipe(RecipeError): """Raised if the recipe file is empty""" + template = "is empty" class MissingBuild(RecipeError): """Raised if the recipe is missing the build section""" + template = "is missing build section" @@ -97,6 +101,7 @@ class HasSelector(RecipeError): """Raised when recplacements fail due to ``# [cond]`` line selectors FIXME: This should no longer be an error """ + template = "has selector in line %i (replace failed)" @@ -105,10 +110,13 @@ class MissingMetaYaml(RecipeError): self.item is NOT a Recipe but a str here """ + template = "has missing file `meta.yaml`" + class CondaRenderFailure(RecipeError): """Raised when conda_build.api.render fails""" + template = "could not be rendered by conda-build: %s" @@ -117,10 +125,11 @@ class RenderFailure(RecipeError): May have self.line """ + template = "failed to render in Jinja2. Error was: %s" -class Recipe(): +class Recipe: """Represents a recipe (meta.yaml) in editable form Using conda-build to render recipe is slow and a one-way @@ -139,31 +148,28 @@ class Recipe(): recipe_dir: path to specific recipe """ - #: Variables to pass to Jinja when rendering recipe JINJA_VARS = { "cran_mirror": "https://cloud.r-project.org", "compiler": lambda x: f"compiler_{x}", "pin_compatible": lambda x, max_pin=None, min_pin=None: f"{x}", - "cdt": lambda x: x + "cdt": lambda x: x, } - def __init__(self, recipe_dir, recipe_folder): if not recipe_dir.startswith(recipe_folder): raise RuntimeError(f"'{recipe_dir}' not inside '{recipe_folder}'") - #: path to folder containing recipes self.basedir = recipe_folder #: relative path to recipe dir from folder containing recipes - self.reldir = recipe_dir[len(recipe_folder):].strip("/") + self.reldir = recipe_dir[len(recipe_folder) :].strip("/") # Filled in by render() #: Parsed recipe YAML self.meta: Dict[str, Any] = {} - self.conda_build_config: str = '' + self.conda_build_config: str = "" self.build_scripts: Dict[str, str] = {} # These will be filled in by load_from_string() @@ -213,17 +219,17 @@ def load_from_string(self, data) -> "Recipe": def read_conda_build_config(self): # Cache contents of conda_build_config.yaml for conda_render. - path = Path(self.dir, 'conda_build_config.yaml') + path = Path(self.dir, "conda_build_config.yaml") if path.is_file(): self.conda_build_config = path.read_text() else: - self.conda_build_config = '' + self.conda_build_config = "" def read_build_scripts(self): # Cache contents of build scripts for conda_render since conda-build # inspects build scripts for used variant variables. - scripts = ['build.sh'] + [ - output.get('script') for output in self.meta.get('outputs') or () + scripts = ["build.sh"] + [ + output.get("script") for output in self.meta.get("outputs") or () ] self.build_scripts.clear() for script in scripts: @@ -249,7 +255,7 @@ def from_file(cls, recipe_dir, recipe_fname, return_exceptions=False) -> "Recipe recipe_fname = os.path.dirname(recipe_fname) recipe = cls(recipe_fname, recipe_dir) try: - with open(os.path.join(recipe_fname, 'meta.yaml')) as text: + with open(os.path.join(recipe_fname, "meta.yaml")) as text: recipe.load_from_string(text.read()) except FileNotFoundError: exc = MissingMetaYaml(recipe_fname) @@ -329,12 +335,16 @@ def _rewrite_selector_block(text, block_top, block_left): else: new_lines.append("".join((" " * (block_left + 2), line))) - logger.debug("Replacing: lines %i - %i with %i lines:\n%s\n---\n%s", - block_top, block_top+block_height, len(new_lines), - "\n".join(lines[block_top:block_top+block_height]), - "\n".join(new_lines)) + logger.debug( + "Replacing: lines %i - %i with %i lines:\n%s\n---\n%s", + block_top, + block_top + block_height, + len(new_lines), + "\n".join(lines[block_top : block_top + block_height]), + "\n".join(new_lines), + ) - lines[block_top:block_top+block_height] = new_lines + lines[block_top : block_top + block_height] = new_lines return "\n".join(lines) def get_template(self): @@ -343,9 +353,7 @@ def get_template(self): # Storing it means the recipe cannot be pickled, which in turn # means we cannot pass it to ProcessExecutors. try: - return utils.jinja_silent_undef.from_string( - "\n".join(self.meta_yaml) - ) + return utils.jinja_silent_undef.from_string("\n".join(self.meta_yaml)) except jinja2.exceptions.TemplateSyntaxError as exc: raise RenderFailure(self, message=exc.message, line=exc.lineno) except jinja2.exceptions.TemplateError as exc: @@ -365,7 +373,7 @@ def get_simple_modules(self): attr: getattr(template.module, attr) for attr in dir(template.module) if not attr.startswith("_") - and not hasattr(getattr(template.module, attr), '__call__') + and not hasattr(getattr(template.module, attr), "__call__") } def render(self) -> None: @@ -384,8 +392,9 @@ def render(self) -> None: column = err.problem_mark.column + 1 logger.debug("fixing duplicate key at %i:%i", line, column) # We may have encountered a recipe with linux/osx variants using line selectors - yaml_text = self._rewrite_selector_block(yaml_text, err.context_mark.line, - err.context_mark.column) + yaml_text = self._rewrite_selector_block( + yaml_text, err.context_mark.line, err.context_mark.column + ) if yaml_text: try: self.meta = yaml.load(yaml_text) @@ -394,16 +403,18 @@ def render(self) -> None: else: raise DuplicateKey(self, line=line, column=column) - if "package" not in self.meta \ - or "version" not in self.meta["package"] \ - or "name" not in self.meta["package"]: + if ( + "package" not in self.meta + or "version" not in self.meta["package"] + or "name" not in self.meta["package"] + ): raise MissingKey(self) @property def maintainers(self): """List of recipe maintainers""" - if 'extra' in self.meta and 'recipe-maintainers' in self.meta['extra']: - return utils.ensure_list(self.meta['extra']['recipe-maintainers']) + if "extra" in self.meta and "recipe-maintainers" in self.meta["extra"]: + return utils.ensure_list(self.meta["extra"]["recipe-maintainers"]) return [] @property @@ -427,7 +438,7 @@ def __getitem__(self, key): def _walk(self, path, noraise=False): nodes = [self.meta] keys = [] - for key in path.split('/'): + for key in path.split("/"): last = nodes[-1] if key.isdigit(): number = int(key) @@ -482,7 +493,7 @@ def get_raw_range(self, path): else: node_keys = list(node.keys()) if key != node_keys[-1]: - next_key = node_keys[node_keys.index(key)+1] + next_key = node_keys[node_keys.index(key) + 1] end_row, end_col = node.lc.key(next_key) break else: # reached end of file @@ -520,7 +531,7 @@ def get_raw(self, path): lines.append(self.meta_yaml[end_row][:end_col]) return "\n".join(lines).strip() - def get(self, path: str, default: Any=KeyError) -> Any: + def get(self, path: str, default: Any = KeyError) -> Any: """Get a value or section from the recipe >>> recipe.get('requirements/build') @@ -562,15 +573,15 @@ def set(self, path, value): nodes, keys = self._walk(path, noraise=True) # "mkdir -p" - found_path = '/'.join(str(key) for key in keys) + found_path = "/".join(str(key) for key in keys) if found_path != path: _, col, row, _ = self.get_raw_range(found_path) backup = deepcopy(self.meta_yaml) - for key in path.split('/')[len(keys):]: - self.meta_yaml.insert(row, ' ' * col + key + ':') + for key in path.split("/")[len(keys) :]: + self.meta_yaml.insert(row, " " * col + key + ":") row += 1 col += 2 - self.meta_yaml[row-1] += " marker" + self.meta_yaml[row - 1] += " marker" self.render() # get old content @@ -586,14 +597,18 @@ def package_names(self) -> List[str]: """List of the packages built by this recipe (including outputs)""" packages = [self.name] if "outputs" in self.meta: - packages.extend(output['name'] - for output in self.meta['outputs'] - if output != self.name) + packages.extend( + output["name"] for output in self.meta["outputs"] if output != self.name + ) return packages - def replace(self, before: str, after: str, - within: Sequence[str] = ("package", "source"), - with_fuzz=False) -> int: + def replace( + self, + before: str, + after: str, + within: Sequence[str] = ("package", "source"), + with_fuzz=False, + ) -> int: """Runs string replace on parts of recipe text. - Lines considered are those containing Jinja set statements @@ -650,7 +665,7 @@ def replace(self, before: str, after: str, replacements += 1 return replacements - def reset_buildnumber(self, n: int=0): + def reset_buildnumber(self, n: int = 0): """Resets the build number If the build number is missing, it is added after build. @@ -662,12 +677,12 @@ def reset_buildnumber(self, n: int=0): build = self.meta["build"] first_in_build = next(iter(build)) lineno, colno = build.lc.key(first_in_build) - self.meta_yaml.insert(lineno, " "*colno + "number: 0") + self.meta_yaml.insert(lineno, " " * colno + "number: 0") else: raise MissingBuild(self) line = self.meta_yaml[lineno] - line = re.sub("number: [0-9]+", "number: "+str(n), line) + line = re.sub("number: [0-9]+", "number: " + str(n), line) self.meta_yaml[lineno] = line self.render() @@ -676,30 +691,32 @@ def get_deps(self, sections=None, output=True): def get_deps_dict(self, sections=None, outputs=True): if not sections: - sections = ('build', 'run', 'host') + sections = ("build", "run", "host") else: sections = utils.ensure_list(sections) check_paths = [] for section in sections: - check_paths.append(f'requirements/{section}') + check_paths.append(f"requirements/{section}") if outputs: for section in sections: - for n in range(len(self.get('outputs', []))): - check_paths.append(f'outputs/{n}/requirements/{section}') + for n in range(len(self.get("outputs", []))): + check_paths.append(f"outputs/{n}/requirements/{section}") deps = {} for path in check_paths: for n, spec in enumerate(self.get(path, [])): if spec is None: # Fixme: lint this continue - dep = re.split(r'[\s<=>]', spec)[0] + dep = re.split(r"[\s<=>]", spec)[0] deps.setdefault(dep, []).append(f"{path}/{n}") return deps - def conda_render(self, - bypass_env_check=True, - finalize=True, - permit_unsatisfiable_variants=False, - **kwargs) -> List[Tuple[MetaData, bool, bool]]: + def conda_render( + self, + bypass_env_check=True, + finalize=True, + permit_unsatisfiable_variants=False, + **kwargs, + ) -> List[Tuple[MetaData, bool, bool]]: """Handles calling conda_build.api.render ``conda_build.api.render`` is fragile, loud and slow. Avoid using this @@ -751,8 +768,8 @@ def conda_render(self, self._conda_tempdir = tempfile.TemporaryDirectory() - with open(os.path.join(self._conda_tempdir.name, 'meta.yaml'), 'w') as tmpfile: - tmpfile.write(self.dump()) + with open(os.path.join(self._conda_tempdir.name, "meta.yaml"), "w") as tmpfile: + tmpfile.write(self.dump()) if self.conda_build_config: cbc_path = Path(self._conda_tempdir.name, "conda_build_config.yaml") @@ -764,8 +781,10 @@ def conda_render(self, old_exit = sys.exit if isinstance(sys.exit, types.FunctionType): + def new_exit(args=None): raise SystemExit(args) + sys.exit = new_exit insert_mambabuild() @@ -778,20 +797,24 @@ def new_exit(args=None): finalize=finalize, bypass_env_check=bypass_env_check, permit_unsatisfiable_variants=permit_unsatisfiable_variants, - **kwargs) + **kwargs, + ) except RuntimeError as exc: if exc.args[0].startswith("Couldn't extract raw recipe text"): line = self.meta_yaml[0] - if not line.startswith('package') or line.startswith('build'): - raise CondaRenderFailure(self, "Must start with package or build section") + if not line.startswith("package") or line.startswith("build"): + raise CondaRenderFailure( + self, "Must start with package or build section" + ) raise except SystemExit as exc: msg = exc.args[0] if msg.startswith("Error: Failed to render jinja"): - msg = '; '.join(msg.splitlines()[1:]) if '\n' in msg else msg + msg = "; ".join(msg.splitlines()[1:]) if "\n" in msg else msg raise CondaRenderFailure(self, f"Jinja2 Template Error: '{msg}'") raise CondaRenderFailure( - self, f"Unknown SystemExit raised in Conda-Build Render API: '{msg}'") + self, f"Unknown SystemExit raised in Conda-Build Render API: '{msg}'" + ) finally: sys.exit = old_exit return self._conda_meta @@ -807,12 +830,16 @@ def conda_release(self): def load_parallel_iter(recipe_folder, packages): recipes = list(utils.get_recipes(recipe_folder, packages)) - for recipe in utils.parallel_iter(Recipe.from_file, recipes, "Loading Recipes...", - recipe_folder, return_exceptions=True): + for recipe in utils.parallel_iter( + Recipe.from_file, + recipes, + "Loading Recipes...", + recipe_folder, + return_exceptions=True, + ): if isinstance(recipe, RecipeError): recipe.log() elif isinstance(recipe, Exception): logger.error("Could not load recipe %s", recipe) else: yield recipe - diff --git a/bioconda_utils/update_pinnings.py b/bioconda_utils/update_pinnings.py index 5d5f77cca9..1f0038bf56 100644 --- a/bioconda_utils/update_pinnings.py +++ b/bioconda_utils/update_pinnings.py @@ -9,6 +9,7 @@ import string from .utils import RepoData + # FIXME: trim_build_only_deps is not exported via conda_build.api! # Re-implement it here or ask upstream to export that functionality. from conda_build.metadata import trim_build_only_deps @@ -56,22 +57,26 @@ def will_build_variant(meta: MetaData) -> bool: by the variant MetaData. """ build_numbers = RepoData().get_package_data( - 'build_number', - name=meta.name(), version=meta.version(), - platform=['linux', 'noarch'], + "build_number", + name=meta.name(), + version=meta.version(), + platform=["linux", "noarch"], ) current_num = int(meta.build_number()) res = all(num < current_num for num in build_numbers) if res: - logger.debug("Package %s=%s will be built already because %s < %s)", - meta.name(), meta.version(), - max(build_numbers) if build_numbers else "N/A", - meta.build_number()) + logger.debug( + "Package %s=%s will be built already because %s < %s)", + meta.name(), + meta.version(), + max(build_numbers) if build_numbers else "N/A", + meta.build_number(), + ) return res _legacy_build_string_prefixes = re.compile( - ''' + """ ^ ( (?P np [0-9]{2,9}) | @@ -81,7 +86,7 @@ def will_build_variant(meta: MetaData) -> bool: (?P r [0-9]{2,9}) | (?P mro [0-9]{3,9}) )* - ''', + """, re.X, ) @@ -90,15 +95,17 @@ def will_build_variant(meta: MetaData) -> bool: def _have_partially_matching_build_id(meta): # Stupid legacy special handling: res = RepoData().get_package_data( - 'build', - name=meta.name(), version=meta.version(), + "build", + name=meta.name(), + version=meta.version(), build_number=meta.build_number(), - platform=['linux', 'noarch'], + platform=["linux", "noarch"], ) is_noarch = bool(meta.noarch) current_build_id = meta.build_id() current_matches = _legacy_build_string_prefixes.match(current_build_id) current_prefixes = current_matches.groupdict() + # conda-build add "special" substrings for some packages to the build # string (e.g., "py38", "pl526", ...). When we use `bypass_env_check` then # it does not add those substrings somehow (?). @@ -161,16 +168,18 @@ def is_matching_trimmed_build_id(build_id, current_build_id): # but we might have noarch:generic recipes that use python. # It probably doesn't matter which python is chosen then, so # we also trim the "py*" prefix in that case here. - if not (is_noarch and prefix_key == 'python'): + if not (is_noarch and prefix_key == "python"): return False if prefix: - trimmed_build_id = trimmed_build_id.replace(prefix, '') + trimmed_build_id = trimmed_build_id.replace(prefix, "") if current_prefix: - trimmed_current_build_id = trimmed_current_build_id.replace(current_prefix, '') - if trimmed_build_id.startswith('_'): + trimmed_current_build_id = trimmed_current_build_id.replace( + current_prefix, "" + ) + if trimmed_build_id.startswith("_"): # If we trimmed everything but the number, no '_' is inserted. trimmed_build_id = trimmed_build_id[1:] - if trimmed_current_build_id.startswith('_'): + if trimmed_current_build_id.startswith("_"): # If we trimmed everything but the number, no '_' is inserted. trimmed_current_build_id = trimmed_current_build_id[1:] if trimmed_build_id == trimmed_current_build_id: @@ -179,8 +188,9 @@ def is_matching_trimmed_build_id(build_id, current_build_id): for build_id in res: if is_matching_trimmed_build_id(build_id, current_build_id): - logger.debug("Package %s=%s=%s exists", - meta.name(), meta.version(), build_id) + logger.debug( + "Package %s=%s=%s exists", meta.name(), meta.version(), build_id + ) return True return False @@ -195,12 +205,15 @@ def have_variant(meta: MetaData) -> bool: True if the variant's build string exists already in the repodata """ res = RepoData().get_package_data( - name=meta.name(), version=meta.version(), build=meta.build_id(), - platform=['linux', 'noarch'] + name=meta.name(), + version=meta.version(), + build=meta.build_id(), + platform=["linux", "noarch"], ) if res: - logger.debug("Package %s=%s=%s exists", - meta.name(), meta.version(), meta.build_id()) + logger.debug( + "Package %s=%s=%s exists", meta.name(), meta.version(), meta.build_id() + ) return True return _have_partially_matching_build_id(meta) @@ -214,16 +227,21 @@ def have_noarch_python_build_number(meta: MetaData) -> bool: Returns: True if noarch:python and version+build_number exists already in repodata """ - if meta.get_value('build/noarch') != 'python': + if meta.get_value("build/noarch") != "python": return False res = RepoData().get_package_data( - name=meta.name(), version=meta.version(), + name=meta.name(), + version=meta.version(), build_number=meta.build_number(), - platform=['noarch'], + platform=["noarch"], ) if res: - logger.debug("Package %s=%s[build_number=%s, subdir=noarch] exists", - meta.name(), meta.version(), meta.build_number()) + logger.debug( + "Package %s=%s[build_number=%s, subdir=noarch] exists", + meta.name(), + meta.version(), + meta.build_number(), + ) return res @@ -236,10 +254,7 @@ def will_build_only_missing(metas: List[MetaData]) -> bool: Returns: True if no divergent build strings exist in repodata """ - builds = { - (meta.name(), meta.version(), meta.build_number()) - for meta in metas - } + builds = {(meta.name(), meta.version(), meta.build_number()) for meta in metas} existing_builds = set() for name, version, build_number in builds: existing_builds.update( @@ -247,20 +262,20 @@ def will_build_only_missing(metas: List[MetaData]) -> bool: tuple, RepoData().get_package_data( ["name", "version", "build"], - name=name, version=version, build_number=build_number, - platform=['linux', 'noarch'], + name=name, + version=version, + build_number=build_number, + platform=["linux", "noarch"], ), ), ) - new_builds = { - (meta.name(), meta.version(), meta.build_id()) - for meta in metas - } + new_builds = {(meta.name(), meta.version(), meta.build_id()) for meta in metas} return new_builds.issuperset(existing_builds) class State(enum.Flag): """Recipe Pinning State""" + #: Recipe had a failure rendering FAIL = enum.auto() #: Recipe has a variant that will be skipped @@ -275,24 +290,24 @@ class State(enum.Flag): HAVE_NOARCH_PYTHON = enum.auto() def needs_bump(self) -> bool: - """Checks if the state indicates that the recipe needs to be bumped - """ + """Checks if the state indicates that the recipe needs to be bumped""" return self & self.BUMP - def failed(self) -> bool: """True if the update pinning check failed""" return self & self.FAIL allowed_build_string_characters = frozenset( - string.digits + string.ascii_uppercase + string.ascii_lowercase + '_.' + string.digits + string.ascii_uppercase + string.ascii_lowercase + "_." ) def has_invalid_build_string(meta: MetaData) -> bool: build_string = meta.build_id() - return not (build_string and set(build_string).issubset(allowed_build_string_characters)) + return not ( + build_string and set(build_string).issubset(allowed_build_string_characters) + ) def check( @@ -321,11 +336,15 @@ def check( logger.error(exc) return State.FAIL, recipe except Exception as exc: - logger.exception("update_pinnings.check failed with exception in api.render(%s):", recipe) + logger.exception( + "update_pinnings.check failed with exception in api.render(%s):", recipe + ) return State.FAIL, recipe if maybe_metas is None: - logger.error("Failed to render %s. Got 'None' from recipe.conda_render()", recipe) + logger.error( + "Failed to render %s. Got 'None' from recipe.conda_render()", recipe + ) return State.FAIL, recipe metas = [meta for meta, _, _ in maybe_metas] @@ -349,8 +368,12 @@ def check( elif will_build_variant(meta): flags |= State.BUMPED else: - logger.info("Package %s=%s=%s missing!", - meta.name(), meta.version(), meta.build_id()) + logger.info( + "Package %s=%s=%s missing!", + meta.name(), + meta.version(), + meta.build_id(), + ) maybe_bump = True if maybe_bump: # Skip bump if we only add to the build matrix. diff --git a/bioconda_utils/upload.py b/bioconda_utils/upload.py index 8702c46a0a..61bcba6f27 100644 --- a/bioconda_utils/upload.py +++ b/bioconda_utils/upload.py @@ -6,6 +6,7 @@ import subprocess as sp import logging from . import utils + logger = logging.getLogger(__name__) @@ -27,21 +28,20 @@ def anaconda_upload(package: str, token: str = None, label: str = None) -> bool: """ label_arg = [] if label is not None: - label_arg = ['--label', label] + label_arg = ["--label", label] if not os.path.exists(package): - logger.error("UPLOAD ERROR: package %s cannot be found.", - package) + logger.error("UPLOAD ERROR: package %s cannot be found.", package) return False if token is None: - token = os.environ.get('ANACONDA_TOKEN') + token = os.environ.get("ANACONDA_TOKEN") if token is None: raise ValueError("Env var ANACONDA_TOKEN not found") logger.info("UPLOAD uploading package %s", package) try: - cmds = ["anaconda", "-t", token, 'upload', package] + label_arg + cmds = ["anaconda", "-t", token, "upload", package] + label_arg utils.run(cmds, mask=[token]) logger.info("UPLOAD SUCCESS: uploaded package %s", package) return True @@ -51,15 +51,15 @@ def anaconda_upload(package: str, token: str = None, label: str = None) -> bool: # ignore error assuming that it is caused by # existing package logger.warning( - "UPLOAD WARNING: tried to upload package, got:\n " - "%s", e.stdout) + "UPLOAD WARNING: tried to upload package, got:\n " "%s", e.stdout + ) return True elif "Gateway Timeout" in e.stdout: logger.warning("UPLOAD TEMP FAILURE: Gateway timeout") return False else: - logger.error('UPLOAD ERROR: command: %s', e.cmd) - logger.error('UPLOAD ERROR: stdout+stderr: %s', e.stdout) + logger.error("UPLOAD ERROR: command: %s", e.cmd) + logger.error("UPLOAD ERROR: stdout+stderr: %s", e.stdout) return False @@ -73,18 +73,22 @@ def mulled_upload(image: str, quay_target: str) -> sp.CompletedProcess: image: name of image to push quary_target: name of image on quay """ - cmd = ['mulled-build', 'push', image, '-n', quay_target] + cmd = ["mulled-build", "push", image, "-n", quay_target] mask = [] - if os.environ.get('QUAY_OAUTH_TOKEN', False): - token = os.environ['QUAY_OAUTH_TOKEN'] - cmd.extend(['--oauth-token', token]) + if os.environ.get("QUAY_OAUTH_TOKEN", False): + token = os.environ["QUAY_OAUTH_TOKEN"] + cmd.extend(["--oauth-token", token]) mask = [token] return utils.run(cmd, mask=mask) -def skopeo_upload(image_file: str, target: str, - creds: str, registry: str = "quay.io", - timeout: int = 600) -> bool: +def skopeo_upload( + image_file: str, + target: str, + creds: str, + registry: str = "quay.io", + timeout: int = 600, +) -> bool: """ Upload an image to docker registy @@ -100,15 +104,19 @@ def skopeo_upload(image_file: str, target: str, registry: url of the registry. defaults to "quay.io" timeout: timeout in seconds """ - cmd = ['skopeo', - '--insecure-policy', # disable policy checks - '--command-timeout', str(timeout) + "s", - 'copy', - 'docker-archive:{}'.format(image_file), - 'docker://{}/{}'.format(registry, target), - '--dest-creds', creds] + cmd = [ + "skopeo", + "--insecure-policy", # disable policy checks + "--command-timeout", + str(timeout) + "s", + "copy", + "docker-archive:{}".format(image_file), + "docker://{}/{}".format(registry, target), + "--dest-creds", + creds, + ] try: - utils.run(cmd, mask=creds.split(':')) + utils.run(cmd, mask=creds.split(":")) return True except sp.CalledProcessError as exc: logger.error("Failed to upload %s to %s", image_file, target) diff --git a/bioconda_utils/utils.py b/bioconda_utils/utils.py index 250177f52f..14df1e04cd 100644 --- a/bioconda_utils/utils.py +++ b/bioconda_utils/utils.py @@ -41,6 +41,7 @@ # by conda.core.index.get_index which messes up our logging. # => Prevent custom conda logging init before importing anything conda-related. import conda.gateways.logging + conda.gateways.logging.initialize_logging = lambda: None from conda_build import api @@ -61,10 +62,12 @@ class TqdmHandler(logging.StreamHandler): Passes all log writes through tqdm to allow progress bars and log messages to coexist without clobbering terminal """ + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - # initialise internal tqdm lock so that we can use tqdm.write + # initialise internal tqdm lock so that we can use tqdm.write _tqdm.tqdm(disable=True, total=0) + def emit(self, record): _tqdm.tqdm.write(self.format(record)) @@ -82,16 +85,18 @@ def tqdm(*args, **kwargs): loglevel: logging loglevel (the number, so logging.INFO) logger: local logger (in case it has different effective log level) """ - term_ok = (sys.stderr.isatty() - and os.environ.get("TERM", "") != "dumb" - and os.environ.get("CIRCLECI", "") != "true") - loglevel_ok = (kwargs.get('logger', logger).getEffectiveLevel() - <= kwargs.get('loglevel', logging.INFO)) - kwargs['disable'] = not (term_ok and loglevel_ok) + term_ok = ( + sys.stderr.isatty() + and os.environ.get("TERM", "") != "dumb" + and os.environ.get("CIRCLECI", "") != "true" + ) + loglevel_ok = kwargs.get("logger", logger).getEffectiveLevel() <= kwargs.get( + "loglevel", logging.INFO + ) + kwargs["disable"] = not (term_ok and loglevel_ok) return _tqdm.tqdm(*args, **kwargs) - def ensure_list(obj): """Wraps **obj** in a list if necessary @@ -126,11 +131,12 @@ def wraps(func): """ fb = FunctionBuilder.from_func(func) + def wrapper_wrapper(wrapper_func): fb_wrapper = FunctionBuilder.from_func(wrapper_func) fb.kwonlyargs += fb_wrapper.kwonlyargs fb.kwonlydefaults.update(fb_wrapper.kwonlydefaults) - fb.body = 'return _call(%s)' % fb.get_invocation_str() + fb.body = "return _call(%s)" % fb.get_invocation_str() execdict = dict(_call=wrapper_func, _func=func) fully_wrapped = fb.get_func(execdict) fully_wrapped.__wrapped__ = func @@ -154,8 +160,10 @@ class LogFuncFilter: The implementation assumes that **func** uses a logger initialized with ``getLogger(__name__)``. """ - def __init__(self, func, trunc_msg: str = None, max_lines: int = 0, - consecutive: bool = True) -> None: + + def __init__( + self, func, trunc_msg: str = None, max_lines: int = 0, consecutive: bool = True + ) -> None: self.func = func self.max_lines = max_lines + 1 self.cur_max_lines = max_lines + 1 @@ -163,7 +171,10 @@ def __init__(self, func, trunc_msg: str = None, max_lines: int = 0, self.trunc_msg = trunc_msg def filter(self, record: logging.LogRecord) -> bool: - if record.name == self.func.__module__ and record.funcName == self.func.__name__: + if ( + record.name == self.func.__module__ + and record.funcName == self.func.__name__ + ): if self.cur_max_lines > 1: self.cur_max_lines -= 1 return True @@ -183,22 +194,27 @@ class LoggingSourceRenameFilter: Maps ``bioconda_utils`` to ``BIOCONDA`` and for everything else to just the top level package uppercased. """ + def filter(self, record: logging.LogRecord) -> bool: if record.name.startswith("bioconda_utils"): record.name = "BIOCONDA" else: - record.name = record.name.split('.')[0].upper() + record.name = record.name.split(".")[0].upper() return True -def setup_logger(name: str = 'bioconda_utils', loglevel: Union[str, int] = logging.INFO, - logfile: str = None, logfile_level: Union[str, int] = logging.DEBUG, - log_command_max_lines = None, - prefix: str = "BIOCONDA ", - msgfmt: str = ("%(asctime)s " - "%(log_color)s%(name)s %(levelname)s%(reset)s " - "%(message)s"), - datefmt: str ="%H:%M:%S") -> logging.Logger: +def setup_logger( + name: str = "bioconda_utils", + loglevel: Union[str, int] = logging.INFO, + logfile: str = None, + logfile_level: Union[str, int] = logging.DEBUG, + log_command_max_lines=None, + prefix: str = "BIOCONDA ", + msgfmt: str = ( + "%(asctime)s " "%(log_color)s%(name)s %(levelname)s%(reset)s " "%(message)s" + ), + datefmt: str = "%H:%M:%S", +) -> logging.Logger: """Set up logging for bioconda-utils Args: @@ -224,7 +240,9 @@ def setup_logger(name: str = 'bioconda_utils', loglevel: Union[str, int] = loggi log_file_handler = logging.FileHandler(logfile) log_file_handler.setLevel(logfile_level) log_file_formatter = logging.Formatter( - msgfmt.replace("%(log_color)s", "").replace("%(reset)s", "").format(prefix=prefix), + msgfmt.replace("%(log_color)s", "") + .replace("%(reset)s", "") + .format(prefix=prefix), datefmt=None, ) log_file_handler.setFormatter(log_file_formatter) @@ -244,17 +262,20 @@ def setup_logger(name: str = 'bioconda_utils', loglevel: Union[str, int] = loggi if loglevel: log_stream_handler.setLevel(loglevel) - log_stream_handler.setFormatter(ColoredFormatter( - msgfmt.format(prefix=prefix), - datefmt=datefmt, - reset=True, - log_colors={ - 'DEBUG': 'cyan', - 'INFO': 'green', - 'WARNING': 'yellow', - 'ERROR': 'red', - 'CRITICAL': 'red', - })) + log_stream_handler.setFormatter( + ColoredFormatter( + msgfmt.format(prefix=prefix), + datefmt=datefmt, + reset=True, + log_colors={ + "DEBUG": "cyan", + "INFO": "green", + "WARNING": "yellow", + "ERROR": "red", + "CRITICAL": "red", + }, + ) + ) log_stream_handler.addFilter(LoggingSourceRenameFilter()) root_logger.addHandler(log_stream_handler) @@ -262,14 +283,17 @@ def setup_logger(name: str = 'bioconda_utils', loglevel: Union[str, int] = loggi # We do this here rather than in `utils.run` so that it can be configured # from the CLI more easily if log_command_max_lines is not None: - log_filter = LogFuncFilter(run, "Command output truncated", log_command_max_lines) + log_filter = LogFuncFilter( + run, "Command output truncated", log_command_max_lines + ) log_stream_handler.addFilter(log_filter) return new_logger -def ellipsize_recipes(recipes: Collection[str], recipe_folder: str, - n: int = 5, m: int = 50) -> str: +def ellipsize_recipes( + recipes: Collection[str], recipe_folder: str, n: int = 5, m: int = 50 +) -> str: """Logging helper showing recipe list Args: @@ -290,58 +314,97 @@ def ellipsize_recipes(recipes: Collection[str], recipe_folder: str, append = ", ..." else: append = "" - return ' ('+', '.join(recipe.replace(recipe_folder,'').lstrip('/') - for recipe in recipes) + append + ')' + return ( + " (" + + ", ".join(recipe.replace(recipe_folder, "").lstrip("/") for recipe in recipes) + + append + + ")" + ) class JinjaSilentUndefined(jinja2.Undefined): def _fail_with_undefined_error(self, *args, **kwargs): return "" - __add__ = __radd__ = __mul__ = __rmul__ = __div__ = __rdiv__ = \ - __truediv__ = __rtruediv__ = __floordiv__ = __rfloordiv__ = \ - __mod__ = __rmod__ = __pos__ = __neg__ = __call__ = \ - __getitem__ = __lt__ = __le__ = __gt__ = __ge__ = __int__ = \ - __float__ = __complex__ = __pow__ = __rpow__ = \ - _fail_with_undefined_error + __add__ = ( + __radd__ + ) = ( + __mul__ + ) = ( + __rmul__ + ) = ( + __div__ + ) = ( + __rdiv__ + ) = ( + __truediv__ + ) = ( + __rtruediv__ + ) = ( + __floordiv__ + ) = ( + __rfloordiv__ + ) = ( + __mod__ + ) = ( + __rmod__ + ) = ( + __pos__ + ) = ( + __neg__ + ) = ( + __call__ + ) = ( + __getitem__ + ) = ( + __lt__ + ) = ( + __le__ + ) = ( + __gt__ + ) = ( + __ge__ + ) = ( + __int__ + ) = __float__ = __complex__ = __pow__ = __rpow__ = _fail_with_undefined_error jinja = Environment( - loader=PackageLoader('bioconda_utils', 'templates'), + loader=PackageLoader("bioconda_utils", "templates"), trim_blocks=True, - lstrip_blocks=True + lstrip_blocks=True, ) -jinja_silent_undef = Environment( - undefined=JinjaSilentUndefined -) +jinja_silent_undef = Environment(undefined=JinjaSilentUndefined) # Patterns of allowed environment variables that are allowed to be passed to # conda-build. ENV_VAR_WHITELIST = [ - 'PATH', - 'LC_*', - 'LANG', - 'MACOSX_DEPLOYMENT_TARGET', - 'HTTPS_PROXY','HTTP_PROXY', 'https_proxy', 'http_proxy', + "PATH", + "LC_*", + "LANG", + "MACOSX_DEPLOYMENT_TARGET", + "HTTPS_PROXY", + "HTTP_PROXY", + "https_proxy", + "http_proxy", ] # Of those that make it through the whitelist, remove these specific ones -ENV_VAR_BLACKLIST = [ -] +ENV_VAR_BLACKLIST = [] # Of those, also remove these when we're running in a docker container ENV_VAR_DOCKER_BLACKLIST = [ - 'PATH', + "PATH", ] def get_free_space(): """Return free space in MB on disk""" s = os.statvfs(os.getcwd()) - return s.f_frsize * s.f_bavail / (1024 ** 2) + return s.f_frsize * s.f_bavail / (1024**2) def allowed_env_var(s, docker=False): @@ -357,9 +420,9 @@ def allowed_env_var(s, docker=False): return True -def bin_for(name='conda'): - if 'CONDA_ROOT' in os.environ: - return os.path.join(os.environ['CONDA_ROOT'], 'bin', name) +def bin_for(name="conda"): + if "CONDA_ROOT" in os.environ: + return os.path.join(os.environ["CONDA_ROOT"], "bin", name) return name @@ -456,7 +519,6 @@ def load_all_meta(recipe, config=None, finalize=True): return metas - def load_meta_fast(recipe: str, env=None): """ Given a package name, find the current meta.yaml file, parse it, and return @@ -473,21 +535,21 @@ def load_meta_fast(recipe: str, env=None): env = {} try: - pth = os.path.join(recipe, 'meta.yaml') - template = jinja_silent_undef.from_string(open(pth, 'r', encoding='utf-8').read()) + pth = os.path.join(recipe, "meta.yaml") + template = jinja_silent_undef.from_string( + open(pth, "r", encoding="utf-8").read() + ) meta = yaml.safe_load(template.render(env)) return (meta, recipe) except Exception: - raise ValueError('Problem inspecting {0}'.format(recipe)) + raise ValueError("Problem inspecting {0}".format(recipe)) def load_conda_build_config(platform=None, trim_skip=True): """ Load conda build config while considering global pinnings from conda-forge. """ - config = api.Config( - no_download_source=True, - set_build_id=False) + config = api.Config(no_download_source=True, set_build_id=False) # get environment root env_root = PurePath(shutil.which("bioconda-utils")).parents[1] @@ -495,31 +557,34 @@ def load_conda_build_config(platform=None, trim_skip=True): config.exclusive_config_files = [ os.path.join(env_root, "conda_build_config.yaml"), os.path.join( - os.path.dirname(__file__), - 'bioconda_utils-conda_build_config.yaml'), + os.path.dirname(__file__), "bioconda_utils-conda_build_config.yaml" + ), ] for cfg in chain(config.exclusive_config_files, config.variant_config_files or []): - assert os.path.exists(cfg), ('error: {0} does not exist'.format(cfg)) + assert os.path.exists(cfg), "error: {0} does not exist".format(cfg) if platform: config.platform = platform config.trim_skip = trim_skip return config -CondaBuildConfigFile = namedtuple('CondaBuildConfigFile', ( - 'arg', # either '-e' or '-m' - 'path', -)) +CondaBuildConfigFile = namedtuple( + "CondaBuildConfigFile", + ( + "arg", # either '-e' or '-m' + "path", + ), +) def get_conda_build_config_files(config=None): if config is None: config = load_conda_build_config() # TODO: open PR upstream for conda-build to support multiple exclusive_config_files - for file_path in (config.exclusive_config_files or []): - yield CondaBuildConfigFile('-e', file_path) - for file_path in (config.variant_config_files or []): - yield CondaBuildConfigFile('-m', file_path) + for file_path in config.exclusive_config_files or []: + yield CondaBuildConfigFile("-e", file_path) + for file_path in config.variant_config_files or []: + yield CondaBuildConfigFile("-m", file_path) def load_first_metadata(recipe, config=None, finalize=True): @@ -556,9 +621,15 @@ def temp_os(platform): sys.platform = original -def run(cmds: List[str], env: Dict[str, str]=None, mask: List[str]=None, live: bool=True, - mylogger: logging.Logger=logger, loglevel: int=logging.INFO, - **kwargs: Dict[Any, Any]) -> sp.CompletedProcess: +def run( + cmds: List[str], + env: Dict[str, str] = None, + mask: List[str] = None, + live: bool = True, + mylogger: logging.Logger = logger, + loglevel: int = logging.INFO, + **kwargs: Dict[Any, Any] +) -> sp.CompletedProcess: """ Run a command (with logging, masking, etc) @@ -587,7 +658,7 @@ def run(cmds: List[str], env: Dict[str, str]=None, mask: List[str]=None, live: b def pushqueue(out, pipe): """Reads from a pipe and pushes into a queue, pushing "None" to indicate closed pipe""" - for line in iter(pipe.readline, b''): + for line in iter(pipe.readline, b""): out.put((pipe, line)) out.put(None) # End-of-data-token @@ -596,20 +667,27 @@ def do_mask(arg: str) -> str: if mask is None: # caller has not considered masking, hide the entire command # for security reasons - return '' + return "" if mask is False: # masking has been deactivated return arg for mitem in mask: - arg = arg.replace(mitem, '') + arg = arg.replace(mitem, "") return arg - mylogger.log(loglevel, "(COMMAND) %s", ' '.join(do_mask(arg) for arg in cmds)) + mylogger.log(loglevel, "(COMMAND) %s", " ".join(do_mask(arg) for arg in cmds)) # bufsize=4 result of manual experimentation. Changing it can # drop performance drastically. - with sp.Popen(cmds, stdout=sp.PIPE, stderr=sp.PIPE, - close_fds=True, env=env, bufsize=4, **kwargs) as proc: + with sp.Popen( + cmds, + stdout=sp.PIPE, + stderr=sp.PIPE, + close_fds=True, + env=env, + bufsize=4, + **kwargs + ) as proc: # Start threads reading stdout/stderr and pushing it into queue q out_thread = Thread(target=pushqueue, args=(logq, proc.stdout)) err_thread = Thread(target=pushqueue, args=(logq, proc.stderr)) @@ -622,7 +700,7 @@ def do_mask(arg: str) -> str: try: for _ in range(2): # Run until we've got both `None` tokens for pipe, line in iter(logq.get, None): - line = do_mask(line.decode(errors='replace').rstrip()) + line = do_mask(line.decode(errors="replace").rstrip()) output_lines.append(line) if live: if pipe == proc.stdout: @@ -642,14 +720,20 @@ def do_mask(arg: str) -> str: masked_cmds = [do_mask(c) for c in cmds] if proc.poll() is None: - mylogger.log(loglevel, 'Command closed STDOUT/STDERR but is still running') + mylogger.log(loglevel, "Command closed STDOUT/STDERR but is still running") waitfor = 30 waittimes = 5 for attempt in range(waittimes): - mylogger.log(loglevel, "Waiting %s seconds (%i/%i)", waitfor, attempt+1, waittimes) + mylogger.log( + loglevel, + "Waiting %s seconds (%i/%i)", + waitfor, + attempt + 1, + waittimes, + ) try: proc.wait(timeout=waitfor) - break; + break except sp.TimeoutExpired: pass else: @@ -659,9 +743,11 @@ def do_mask(arg: str) -> str: returncode = proc.poll() if returncode: - logger.error('COMMAND FAILED (exited with %s): %s', returncode, ' '.join(masked_cmds)) + logger.error( + "COMMAND FAILED (exited with %s): %s", returncode, " ".join(masked_cmds) + ) if not live: - logger.error('STDOUT+STDERR:\n%s', output) + logger.error("STDOUT+STDERR:\n%s", output) raise sp.CalledProcessError(returncode, masked_cmds, output=output) return sp.CompletedProcess(returncode, masked_cmds, output) @@ -669,7 +755,7 @@ def do_mask(arg: str) -> str: def envstr(env): env = dict(env) - return ';'.join(['='.join([i, str(j)]) for i, j in sorted(env.items())]) + return ";".join(["=".join([i, str(j)]) for i, j in sorted(env.items())]) def flatten_dict(dict): @@ -713,8 +799,7 @@ def __init__(self, env): self.env = env for key, val in self.env.items(): if key != "CONDA_PY" and not isinstance(val, str): - raise ValueError( - "All versions except CONDA_PY must be strings.") + raise ValueError("All versions except CONDA_PY must be strings.") def __iter__(self): """ @@ -767,9 +852,9 @@ def get_deps(recipe=None, build=True): all_deps = set() for meta in metadata: if build: - deps = meta.get_value('requirements/build', []) + deps = meta.get_value("requirements/build", []) else: - deps = meta.get_value('requirements/run', []) + deps = meta.get_value("requirements/run", []) all_deps.update(dep.split()[0] for dep in deps) return all_deps @@ -784,7 +869,7 @@ def set_max_threads(n): def threads_to_use(): """Returns the number of cores we are allowed to run on""" - if hasattr(os, 'sched_getaffinity'): + if hasattr(os, "sched_getaffinity"): cores = len(os.sched_getaffinity(0)) else: cores = os.cpu_count() @@ -794,13 +879,7 @@ def threads_to_use(): def parallel_iter(func, items, desc, *args, **kwargs): pfunc = partial(func, *args, **kwargs) with Pool(threads_to_use()) as pool: - yield from tqdm( - pool.imap_unordered(pfunc, items), - desc=desc, - total=len(items) - ) - - + yield from tqdm(pool.imap_unordered(pfunc, items), desc=desc, total=len(items)) def get_recipes(recipe_folder, package="*", exclude=None): @@ -824,13 +903,15 @@ def get_recipes(recipe_folder, package="*", exclude=None): if exclude is None: exclude = [] for p in package: - logger.debug("get_recipes(%s, package='%s'): %s", - recipe_folder, package, p) + logger.debug("get_recipes(%s, package='%s'): %s", recipe_folder, package, p) path = os.path.join(recipe_folder, p) for new_dir in glob.glob(path): meta_yaml_found_or_excluded = False for dir_path, dir_names, file_names in os.walk(new_dir): - if any(fnmatch.fnmatch(dir_path[len(recipe_folder):], pat) for pat in exclude): + if any( + fnmatch.fnmatch(dir_path[len(recipe_folder) :], pat) + for pat in exclude + ): meta_yaml_found_or_excluded = True continue if "meta.yaml" in file_names: @@ -840,7 +921,7 @@ def get_recipes(recipe_folder, package="*", exclude=None): logger.warn( "No meta.yaml found in %s." " If you want to ignore this directory, add it to the blacklist.", - new_dir + new_dir, ) yield new_dir @@ -864,8 +945,7 @@ def get_latest_recipes(recipe_folder, config, package="*"): """ def toplevel(x): - return x.replace( - recipe_folder, '').strip(os.path.sep).split(os.path.sep)[0] + return x.replace(recipe_folder, "").strip(os.path.sep).split(os.path.sep)[0] config = load_config(config) recipes = sorted(get_recipes(recipe_folder, package), key=toplevel) @@ -875,11 +955,13 @@ def toplevel(x): if len(group) == 1: yield group[0] else: + def get_version(p): - meta_path = os.path.join(p, 'meta.yaml') + meta_path = os.path.join(p, "meta.yaml") meta = load_first_metadata(meta_path, finalize=False) - version = meta.get_value('package/version') + version = meta.get_value("package/version") return VersionOrder(version) + sorted_versions = sorted(group, key=get_version) if sorted_versions: yield sorted_versions[-1] @@ -926,15 +1008,15 @@ def last_commit_to_master(): """ Identifies the day of the last commit to master branch. """ - if not shutil.which('git'): + if not shutil.which("git"): raise ValueError("git not found") p = sp.run( 'git log master --date=iso | grep "^Date:" | head -n1', - shell=True, stdout=sp.PIPE, check=True + shell=True, + stdout=sp.PIPE, + check=True, ) - date = datetime.datetime.strptime( - p.stdout[:-1].decode().split()[1], - '%Y-%m-%d') + date = datetime.datetime.strptime(p.stdout[:-1].decode().split()[1], "%Y-%m-%d") return date @@ -948,11 +1030,10 @@ def file_from_commit(commit, filename): filename : str """ - if commit == 'HEAD': + if commit == "HEAD": return open(filename).read() - p = run(['git', 'show', '{0}:{1}'.format(commit, filename)], mask=False, - loglevel=0) + p = run(["git", "show", "{0}:{1}".format(commit, filename)], mask=False, loglevel=0) return str(p.stdout) @@ -986,23 +1067,23 @@ def newly_unblacklisted(config_file, recipe_folder, git_range): git_range = [git_range] if len(git_range) == 1: - git_range = ['master', git_range[0]] + git_range = ["master", git_range[0]] # Get the set of previously blacklisted recipes by reading the original # config file and then all the original blacklists it had listed previous = set() orig_config = file_from_commit(git_range[0], config_file) - for bl in yaml.safe_load(orig_config)['blacklists']: - with open('.tmp.blacklist', 'w', encoding='utf8') as fout: + for bl in yaml.safe_load(orig_config)["blacklists"]: + with open(".tmp.blacklist", "w", encoding="utf8") as fout: fout.write(file_from_commit(git_range[0], bl)) - previous.update(get_blacklist({'blacklists': '.tmp.blacklist'}, recipe_folder)) - os.unlink('.tmp.blacklist') + previous.update(get_blacklist({"blacklists": ".tmp.blacklist"}, recipe_folder)) + os.unlink(".tmp.blacklist") current = get_blacklist( - yaml.safe_load(file_from_commit(git_range[1], config_file)), - recipe_folder) + yaml.safe_load(file_from_commit(git_range[1], config_file)), recipe_folder + ) results = previous.difference(current) - logger.info('Recipes newly unblacklisted:\n%s', '\n'.join(list(results))) + logger.info("Recipes newly unblacklisted:\n%s", "\n".join(list(results))) return results @@ -1014,8 +1095,8 @@ def changed_since_master(recipe_folder): repo and have added the main repo as ``upstream``, then you'll have to do a ``git checkout master && git pull upstream master`` to update your fork. """ - p = run(['git', 'fetch', 'origin', 'master'], mask=False, loglevel=0) - p = run(['git', 'diff', 'FETCH_HEAD', '--name-only'], mask=False, loglevel=0) + p = run(["git", "fetch", "origin", "master"], mask=False, loglevel=0) + p = run(["git", "diff", "FETCH_HEAD", "--name-only"], mask=False, loglevel=0) return [ os.path.dirname(os.path.relpath(i, recipe_folder)) for i in p.stdout.splitlines(False) @@ -1025,9 +1106,9 @@ def changed_since_master(recipe_folder): def _load_platform_metas(recipe, finalize=True): # check if package is noarch, if so, build only on linux # with temp_os, we can fool the MetaData if needed. - platform = os.environ.get('OSTYPE', sys.platform) + platform = os.environ.get("OSTYPE", sys.platform) if platform.startswith("darwin"): - platform = 'osx' + platform = "osx" elif platform == "linux-gnu": platform = "linux" @@ -1037,8 +1118,7 @@ def _load_platform_metas(recipe, finalize=True): def _meta_subdir(meta): # logic extracted from conda_build.variants.bldpkg_path - return 'noarch' if meta.noarch or meta.noarch_python else meta.config.host_subdir - + return "noarch" if meta.noarch or meta.noarch_python else meta.config.host_subdir def check_recipe_skippable(recipe, check_channels): @@ -1051,26 +1131,31 @@ def check_recipe_skippable(recipe, check_channels): if not metas: return True # If on CI, handle noarch. - if os.environ.get('CI', None) == 'true': + if os.environ.get("CI", None) == "true": first_meta = metas[0] - if first_meta.get_value('build/noarch'): - if platform != 'linux': - logger.debug('FILTER: only building %s on ' - 'linux because it defines noarch.', - recipe) + if first_meta.get_value("build/noarch"): + if platform != "linux": + logger.debug( + "FILTER: only building %s on " "linux because it defines noarch.", + recipe, + ) return True - packages = set( - (meta.name(), meta.version(), int(meta.build_number() or 0)) - for meta in metas + packages = set( + (meta.name(), meta.version(), int(meta.build_number() or 0)) for meta in metas ) r = RepoData() num_existing_pkg_builds = Counter( (name, version, build_number, subdir) for name, version, build_number in packages - for subdir in r.get_package_data("subdir", name=name, version=version, - build_number=build_number, - channels=check_channels, native=True) + for subdir in r.get_package_data( + "subdir", + name=name, + version=version, + build_number=build_number, + channels=check_channels, + native=True, + ) ) if num_existing_pkg_builds == Counter(): # No packages with same version + build num in channels: no need to skip @@ -1081,9 +1166,10 @@ def check_recipe_skippable(recipe, check_channels): ) if num_new_pkg_builds == num_existing_pkg_builds: logger.info( - 'FILTER: not building recipe %s because ' - 'the same number of builds are in channel(s) and it is not forced.', - recipe) + "FILTER: not building recipe %s because " + "the same number of builds are in channel(s) and it is not forced.", + recipe, + ) return True return False @@ -1101,20 +1187,23 @@ def _filter_existing_packages(metas, check_channels): r = RepoData() for pkg_key, build_meta in key_build_meta.items(): - existing_pkg_builds = set(r.get_package_data(['subdir', 'build'], - name=pkg_key[0], - version=pkg_key[1], - build_number=pkg_key[2], - channels=check_channels, - native=True)) + existing_pkg_builds = set( + r.get_package_data( + ["subdir", "build"], + name=pkg_key[0], + version=pkg_key[1], + build_number=pkg_key[2], + channels=check_channels, + native=True, + ) + ) for pkg_build, meta in build_meta.items(): if pkg_build not in existing_pkg_builds: new_metas.append(meta) else: existing_metas.append(meta) - for divergent_build in (existing_pkg_builds - set(build_meta.keys())): - divergent_builds.add( - '-'.join((pkg_key[0], pkg_key[1], divergent_build[1]))) + for divergent_build in existing_pkg_builds - set(build_meta.keys()): + divergent_builds.add("-".join((pkg_key[0], pkg_key[1], divergent_build[1]))) return new_metas, existing_metas, divergent_builds @@ -1129,34 +1218,38 @@ def get_package_paths(recipe, check_channels, force=False): if not metas: return [] - new_metas, existing_metas, divergent_builds = ( - _filter_existing_packages(metas, check_channels)) + new_metas, existing_metas, divergent_builds = _filter_existing_packages( + metas, check_channels + ) if divergent_builds: raise DivergentBuildsError(*sorted(divergent_builds)) for meta in existing_metas: logger.info( - 'FILTER: not building %s because ' - 'it is in channel(s) and it is not forced.', meta.pkg_fn()) + "FILTER: not building %s because " + "it is in channel(s) and it is not forced.", + meta.pkg_fn(), + ) # yield all pkgs that do not yet exist if force: build_metas = new_metas + existing_metas else: build_metas = new_metas - return list(chain.from_iterable( - api.get_output_file_paths(meta) for meta in build_metas)) + return list( + chain.from_iterable(api.get_output_file_paths(meta) for meta in build_metas) + ) def get_blacklist(config: Dict[str, Any], recipe_folder: str) -> set: "Return list of recipes to skip from blacklists" blacklist = set() - for p in config.get('blacklists', []): + for p in config.get("blacklists", []): blacklist.update( [ os.path.relpath(i.strip(), recipe_folder) - for i in open(p, encoding='utf8') - if not i.startswith('#') and i.strip() + for i in open(p, encoding="utf8") + if not i.startswith("#") and i.strip() ] ) return blacklist @@ -1174,9 +1267,7 @@ def validate_config(config): """ if not isinstance(config, dict): config = yaml.safe_load(open(config)) - fn = pkg_resources.resource_filename( - 'bioconda_utils', 'config.schema.yaml' - ) + fn = pkg_resources.resource_filename("bioconda_utils", "config.schema.yaml") schema = yaml.safe_load(open(fn)) validate(config, schema) @@ -1193,12 +1284,16 @@ def load_config(path): validate_config(path) if isinstance(path, dict): + def relpath(p): return p + config = path else: + def relpath(p): return os.path.join(os.path.dirname(path), p) + config = yaml.safe_load(open(path)) def get_list(key): @@ -1209,15 +1304,15 @@ def get_list(key): return value default_config = { - 'blacklists': [], - 'channels': ['conda-forge', 'bioconda', 'defaults'], - 'requirements': None, - 'upload_channel': 'bioconda' + "blacklists": [], + "channels": ["conda-forge", "bioconda", "defaults"], + "requirements": None, + "upload_channel": "bioconda", } - if 'blacklists' in config: - config['blacklists'] = [relpath(p) for p in get_list('blacklists')] - if 'channels' in config: - config['channels'] = get_list('channels') + if "blacklists" in config: + config["blacklists"] = [relpath(p) for p in get_list("blacklists")] + if "channels" in config: + config["channels"] = get_list("channels") default_config.update(config) @@ -1256,6 +1351,7 @@ class AsyncRequests: This is not really a class, more a name space encapsulating a bunch of calls. """ + #: Identify ourselves USER_AGENT = "bioconda/bioconda-utils" #: Max connections to each server @@ -1311,33 +1407,46 @@ async def async_fetch(cls, urls, descs=None, cb=None, datas=None, fds=None): fds = [] conn = aiohttp.TCPConnector(limit_per_host=cls.CONNECTIONS_PER_HOST) async with aiohttp.ClientSession( - connector=conn, - headers={'User-Agent': cls.USER_AGENT} + connector=conn, headers={"User-Agent": cls.USER_AGENT} ) as session: coros = [ - asyncio.ensure_future(cls._async_fetch_one(session, url, desc, cb, data, fd)) + asyncio.ensure_future( + cls._async_fetch_one(session, url, desc, cb, data, fd) + ) for url, desc, data, fd in zip_longest(urls, descs, datas, fds) ] - with tqdm(asyncio.as_completed(coros), - total=len(coros), - desc="Downloading", unit="files") as t: + with tqdm( + asyncio.as_completed(coros), + total=len(coros), + desc="Downloading", + unit="files", + ) as t: result = [await coro for coro in t] return result @staticmethod - @backoff.on_exception(backoff.fibo, aiohttp.ClientResponseError, max_tries=20, - giveup=lambda ex: ex.code not in [429, 502, 503, 504]) + @backoff.on_exception( + backoff.fibo, + aiohttp.ClientResponseError, + max_tries=20, + giveup=lambda ex: ex.code not in [429, 502, 503, 504], + ) async def _async_fetch_one(session, url, desc, cb=None, data=None, fd=None): result = [] async with session.get(url, timeout=None) as resp: resp.raise_for_status() size = int(resp.headers.get("Content-Length", 0)) - with tqdm(total=size, unit='B', unit_scale=True, unit_divisor=1024, - desc=desc, miniters=1, - disable=logger.getEffectiveLevel() > logging.INFO + with tqdm( + total=size, + unit="B", + unit_scale=True, + unit_divisor=1024, + desc=desc, + miniters=1, + disable=logger.getEffectiveLevel() > logging.INFO, ) as progress: while True: - block = await resp.content.read(1024*16) + block = await resp.content.read(1024 * 16) if not block: break progress.update(len(block)) @@ -1405,16 +1514,18 @@ class RepoData: """ - REPODATA_URL = 'https://conda.anaconda.org/{channel}/{subdir}/repodata.json' - REPODATA_LABELED_URL = 'https://conda.anaconda.org/{channel}/label/{label}/{subdir}/repodata.json' - REPODATA_DEFAULTS_URL = 'https://repo.anaconda.com/pkgs/main/{subdir}/repodata.json' + REPODATA_URL = "https://conda.anaconda.org/{channel}/{subdir}/repodata.json" + REPODATA_LABELED_URL = ( + "https://conda.anaconda.org/{channel}/label/{label}/{subdir}/repodata.json" + ) + REPODATA_DEFAULTS_URL = "https://repo.anaconda.com/pkgs/main/{subdir}/repodata.json" - _load_columns = ['build', 'build_number', 'name', 'version', 'depends'] + _load_columns = ["build", "build_number", "name", "version", "depends"] #: Columns available in internal dataframe - columns = _load_columns + ['channel', 'subdir', 'platform'] + columns = _load_columns + ["channel", "subdir", "platform"] #: Platforms loaded - platforms = ['linux', 'osx', 'noarch'] + platforms = ["linux", "osx", "noarch"] # config object config = None @@ -1423,18 +1534,20 @@ class RepoData: _df_ts = None #: default lifetime for repodata cache - cache_timeout = 60*60*8 + cache_timeout = 60 * 60 * 8 @classmethod def register_config(cls, config): cls.config = config __instance = None + def __new__(cls): """Makes RepoData a singleton""" if RepoData.__instance is None: - assert RepoData.config is not None, ("bug: ensure to load config " - "before instantiating RepoData.") + assert RepoData.config is not None, ( + "bug: ensure to load config " "before instantiating RepoData." + ) RepoData.__instance = object.__new__(cls) return RepoData.__instance @@ -1477,8 +1590,9 @@ def _make_repodata_url(self, channel, platform): else: url_template = self.REPODATA_URL - url = url_template.format(channel=channel, - subdir=self.platform2subdir(platform)) + url = url_template.format( + channel=channel, subdir=self.platform2subdir(platform) + ) return url def _load_channel_dataframe_cached(self): @@ -1505,13 +1619,14 @@ def _load_channel_dataframe(self): def to_dataframe(json_data, meta_data): channel, platform = meta_data repo = json.loads(json_data) - df = pd.DataFrame.from_dict(repo['packages'], 'index', - columns=self._load_columns) + df = pd.DataFrame.from_dict( + repo["packages"], "index", columns=self._load_columns + ) # Ensure that version is always a string. - df['version'] = df['version'].astype(str) - df['channel'] = channel - df['platform'] = platform - df['subdir'] = repo['info']['subdir'] + df["version"] = df["version"].astype(str) + df["channel"] = channel + df["platform"] = platform + df["subdir"] = repo["info"]["subdir"] return df if urls: @@ -1520,8 +1635,8 @@ def to_dataframe(json_data, meta_data): else: res = pd.DataFrame(columns=self.columns) - for col in ('channel', 'platform', 'subdir', 'name', 'version', 'build'): - res[col] = res[col].astype('category') + for col in ("channel", "platform", "subdir", "name", "version", "build"): + res[col] = res[col].astype("category") res = res.reset_index(drop=True) return res @@ -1536,17 +1651,16 @@ def native_platform(): @staticmethod def platform2subdir(platform): - if platform == 'linux': - return 'linux-64' - elif platform == 'osx': - return 'osx-64' - elif platform == 'noarch': - return 'noarch' + if platform == "linux": + return "linux-64" + elif platform == "osx": + return "osx-64" + elif platform == "noarch": + return "noarch" else: raise ValueError( - 'Unsupported platform: bioconda only supports linux, osx and noarch.') - - + "Unsupported platform: bioconda only supports linux, osx and noarch." + ) def get_versions(self, name): """Get versions available for package @@ -1559,20 +1673,31 @@ def get_versions(self, name): e.g. {'0.1': ['linux'], '0.2': ['linux', 'osx'], '0.3': ['noarch']} """ # called from doc generator - packages = self.df[self.df.name == name][['version', 'platform']] - versions = packages.groupby('version').agg(lambda x: list(set(x))) - return versions['platform'].to_dict() + packages = self.df[self.df.name == name][["version", "platform"]] + versions = packages.groupby("version").agg(lambda x: list(set(x))) + return versions["platform"].to_dict() def get_latest_versions(self, channel): """Get the latest version for each package in **channel**""" # called from pypi module - packages = self.df[self.df.channel == channel]['version'] + packages = self.df[self.df.channel == channel]["version"] + def max_vers(x): return max(VersionOrder(v) for v in x) - vers = packages.groupby('name').agg(max_vers) - def get_package_data(self, key=None, channels=None, name=None, version=None, - build_number=None, platform=None, build=None, native=False): + vers = packages.groupby("name").agg(max_vers) + + def get_package_data( + self, + key=None, + channels=None, + name=None, + version=None, + build_number=None, + platform=None, + build=None, + native=False, + ): """Get **key** for each package in **channels** If **key** is not give, returns bool whether there are matches. @@ -1580,7 +1705,7 @@ def get_package_data(self, key=None, channels=None, name=None, version=None, If **key** is a list of string, returns tuple iterator. """ if native: - platform = ['noarch', self.native_platform()] + platform = ["noarch", self.native_platform()] if version is not None: version = str(version) @@ -1591,12 +1716,12 @@ def get_package_data(self, key=None, channels=None, name=None, version=None, # is much faster than executing the comparisons for all values # every time, in particular if we are looking at a specific package. for col, val in ( - ('name', name), # thousands of different values - ('build', build), # build string should vary a lot - ('version', version), # still pretty good variety - ('channel', channels), # 3 values - ('platform', platform), # 3 values - ('build_number', build_number), # most values 0 + ("name", name), # thousands of different values + ("build", build), # build string should vary a lot + ("version", version), # still pretty good variety + ("channel", channels), # 3 values + ("platform", platform), # 3 values + ("build_number", build_number), # most values 0 ): if val is None: continue