diff --git a/bioconda_utils/cli.py b/bioconda_utils/cli.py index 48e4128345..440905a0e6 100644 --- a/bioconda_utils/cli.py +++ b/bioconda_utils/cli.py @@ -632,6 +632,16 @@ def pypi_check(recipe_folder, config, loglevel='info', packages='*', only_out_of print('\t'.join(map(str, result))) +@arg('recipe', nargs="+", help='Path to recipes') +@arg('--loglevel', default='debug', help='Log level') +def bump_build_number(recipes, loglevel="info"): + """ + Bumps the build number in given recipes. + """ + utils.setup_logger('bioconda_utils', loglevel) + utils.bump_build_number(recipes) + + def main(): argh.dispatch_commands([ build, dag, dependent, lint, duplicates, diff --git a/bioconda_utils/lint_functions.py b/bioconda_utils/lint_functions.py index fe973d64c1..f0ea912802 100644 --- a/bioconda_utils/lint_functions.py +++ b/bioconda_utils/lint_functions.py @@ -3,6 +3,7 @@ import os import re +import yaml import pandas import numpy as np @@ -93,6 +94,77 @@ def lint_metas(recipe, metas, df, *args, **kwargs): return lint_metas +JINJA_VAR_DEF = re.compile(r"{%.+?%}") +JINJA_VAR = re.compile(r"{{.*?}}") + + +def load_plain_yaml(recipe): + """Load meta.yaml without applying any jinja templating.""" + with open(os.path.join(recipe, 'meta.yaml')) as content: + def quote_var(match, quotes='"\''): + if (content[match.start() - 1] not in quotes and + content[match.end()] not in quotes): + return '"{}"'.format(match.group()) + else: + return match.group() + + # remove var defs to make yaml happy + content = JINJA_VAR_DEF.sub("", content.read()) + # quote var usages if necessary to make yaml happy + content = JINJA_VAR.sub(quote_var, content) + # load as plain yaml + return yaml.load(content, Loader=yaml.BaseLoader) + + +def _superfluous_jinja_var(recipe, entry): + meta = load_plain_yaml(recipe) + m = meta + xpath = entry.split('/') + for p in xpath: + try: + m = m[p] + except KeyError: + return + + match = JINJA_VAR.search(m) + if match: + return { + 'unwanted_jinja_var': True, + 'fix': 'replace jinja var {} in entry {} in ' + 'meta.yaml with a concrete value'.format(match.group(), entry) + } + + +def jinja_var_name(recipe, meta, df,): + return _superfluous_jinja_var(recipe, 'package/name') + + +def jinja_var_buildnum(recipe, meta, df,): + return _superfluous_jinja_var(recipe, 'build/number') + + +def jinja_var_version(recipe, meta, df): + return _superfluous_jinja_var(recipe, 'package/version') + + +def jinja_var_checksum(recipe, meta, df): + for checksum in ["sha256", "sha1", "md5"]: + ret = _superfluous_jinja_var(recipe, 'source/{}'.format(checksum)) + if ret: + return ret + + +def missing_buildnum(recipe, meta, df): + meta = load_plain_yaml(recipe) + try: + meta['build']['number'] + except KeyError: + return { + 'missing_buildnum': True, + 'fix': 'add build->number to meta.yaml' + } + + @lint_multiple_metas def in_other_channels(recipe, meta, df): """ @@ -405,5 +477,10 @@ def compilers_must_be_in_build(recipe, meta, df): should_not_use_fn, should_use_compilers, compilers_must_be_in_build, + jinja_var_version, + jinja_var_buildnum, + jinja_var_name, + jinja_var_checksum, + missing_buildnum, #bioconductor_37, ) diff --git a/bioconda_utils/utils.py b/bioconda_utils/utils.py index c29df26f88..38a4eb998c 100644 --- a/bioconda_utils/utils.py +++ b/bioconda_utils/utils.py @@ -1082,6 +1082,42 @@ def modified_recipes(git_range, recipe_folder, config_file): return existing +def bump_build_numbers(recipes): + def bump(recipe): + if not recipe.endswith("meta.yaml"): + recipe = os.path.join(recipe, "meta.yaml") + state = 'general' + res = [] + num = None + bumped = None + with open(recipe) as content: + for line in content: + if line.startswith('build'): + state = 'build' + elif state == 'build' and line.lstrip().startswith('number:'): + toks = line.split(':') + if toks == 1: + logger.warning("Failed to parse build number of recipe " + "%s: expected format 'number: value' " + "violated.", recipe) + return + try: + num = int(toks[1].strip()) + except ValueError as e: + logger.warning("Failed to parse build number of " + "recipe %s: %s", recipe, e) + return + bumped = num + 1 + line = '{}: {}\n'.format(toks[0], bumped) + res.append(line) + with open(recipe, "w") as out: + out.writelines(res) + logger.info("Bumped build number of recipe {} from {} to {}".format(recipe, num, bumped)) + + for recipe in recipes: + bump(recipe) + + class Progress: def __init__(self): self.thread = Thread(target=self.progress) diff --git a/docs/source/linting.rst b/docs/source/linting.rst index 0bc0fdde05..7a02591edf 100644 --- a/docs/source/linting.rst +++ b/docs/source/linting.rst @@ -312,6 +312,60 @@ Rational: The compiler tools must not be in ``host:`` or ``run:`` sections. How to resolve: Move ``{{ compiler() }}`` variables to the ``build:`` section. +`jinja_var_version` +~~~~~~~~~~~~~~~~~~~~ +Reason for the failing: The recipe uses a jinja variable for defining the package version + +Rationale: Using a jinja variable for package version is no longer necessary with conda-build 3, because the version can be referenced from all over the ``meta.yaml`` via the jinja expression ``{{ PKG_VERSION }}``. + +How to resolve: Directly specify version, in the corresponding entry in the ``meta.yaml``. +Use the jinja expression ``{{ PKG_VERSION }}`` to refer to the version in other places of the ``meta.yaml``, e.g., in the ``url``. + + +`jinja_var_name` +~~~~~~~~~~~~~~~~ + +Reason for the failing: The recipe uses a jinja variable for package name. + +Rationale: For package name, using a jinja variable has no benefit, because it is not supposed to change with updates. +Therefore, it is better for both human readability and parseability to stick to plain YAML syntax. + +How to resolve: Directly specify name in the corresponding entry in the ``meta.yaml``. + +`jinja_var_buildnum` +~~~~~~~~~~~~~~~~~~~~ + +Reason for the failing: The recipe uses a jinja variable for the build number. + +Rationale: For build number, using a jinja variable has no benefit, because, although it changes during updates, it only appears at one place in the ``meta.yaml``. +Therefore, it is better for both human readability and parseability to stick to plain YAML syntax. + +How to resolve: Directly specify build number in the corresponding entry in the ``meta.yaml``. + +`jinja_var_checksum` +~~~~~~~~~~~~~~~~~~~~ + +Reason for the failing: The recipe uses a jinja variable for the source checksum. + +Rationale: For checksum (sha1, sha256, or md5), using a jinja variable has no benefit, because, although it changes during updates, it only appears at one place in the ``meta.yaml``. +Therefore, it is better for both human readability and parseability to stick to plain YAML syntax. + +How to resolve: Directly specify checksum in the corresponding entry in the ``meta.yaml``. + + +`missing_buildnum` +~~~~~~~~~~~~~~~ +Reason for the failing: The recipe is missing a build number definition. + +Rationale: Build number is crucial for building or updating the package. +Although conda would infer it automatically, it is for operational reasons beneficial +always have it explicitly defined in the recipe. + + + +How to resolve: add ``. + + .. `bioconductor_37` ~~~~~~~~~~~~~~~~~ diff --git a/test/test_linting.py b/test/test_linting.py index 3e95a9fa65..f7fc761a79 100644 --- a/test/test_linting.py +++ b/test/test_linting.py @@ -1019,6 +1019,161 @@ def test_should_not_use_fn(): ) +def test_jinja_var_name(): + run_lint( + func=lint_functions.jinja_var_name, + should_pass=[ + ''' + a: + meta.yaml: | + package: + name: a + version: 0.1 + build: + number: 0 + ''', + ], + should_fail=[ + r''' + a: + meta.yaml: | + {% set name = "a" %} + package: + name: "{{ name|lower }}" + version: 0.1 + build: + number: 0 + ''', + r''' + a: + meta.yaml: | + {% set name = "a" %} + package: + name: {{ name }} + version: 0.1 + build: + number: 0 + ''', + ] + ) + + +def test_jinja_var_version(): + run_lint( + func=lint_functions.jinja_var_version, + should_pass=[ + ''' + a: + meta.yaml: | + package: + name: a + version: 0.1 + build: + number: 0 + ''', + ], + should_fail=[ + r''' + a: + meta.yaml: | + {% set version = "1.0" %} + package: + name: a + version: {{ version }} + build: + number: 0 + ''', + ] + ) + + +def test_jinja_var_buildnum(): + run_lint( + func=lint_functions.jinja_var_buildnum, + should_pass=[ + ''' + a: + meta.yaml: | + package: + name: a + version: 0.1 + build: + number: 0 + ''', + ], + should_fail=[ + r''' + a: + meta.yaml: | + {% set buildnum = 1 %} + package: + name: a + version: 0.1 + build: + number: {{ buildnum }} + ''', + ] + ) + + +def test_jinja_var_checksum(): + run_lint( + func=lint_functions.jinja_var_checksum, + should_pass=[ + ''' + a: + meta.yaml: | + package: + name: a + version: 0.1 + build: + number: 0 + source: + sha256: abc + ''', + ], + should_fail=[ + r''' + a: + meta.yaml: | + {% set sha256 = "abc" %} + package: + name: a + version: 0.1 + build: + number: 0 + source: + sha256: {{ sha256 }} + ''', + ] + ) + + +def test_missing_buildnum(): + run_lint( + func=lint_functions.missing_buildnum, + should_pass=[ + ''' + a: + meta.yaml: | + package: + name: a + version: 0.1 + build: + number: 0 + ''', + ], + should_fail=[ + r''' + a: + meta.yaml: | + package: + name: a + version: 0.1 + ''', + ] + ) + #def test_bioconductor_37(): # run_lint( # func=lint_functions.bioconductor_37, diff --git a/test/test_utils.py b/test/test_utils.py index 8680efc1f8..34a63898fd 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -10,6 +10,7 @@ import tarfile import logging import shutil +import filecmp from textwrap import dedent from bioconda_utils import utils @@ -474,6 +475,66 @@ def test_get_channel_packages(): utils.get_channel_packages('bioconda') +def test_bump_build_numbers(): + r = Recipes( + """ + valid: + meta.yaml: | + package: + name: one + version: "0.1" + build: + number: 1 + + invalid_jinja: + meta.yaml: | + package: + name: two + version: "0.1" + build: + number: {{ CONDA_NCURSES }} + + no_build_num: + meta.yaml: | + package: + name: two + version: "0.1" + """, from_string=True) + r.write_recipes() + + expected = Recipes( + """ + valid: + meta.yaml: | + package: + name: one + version: "0.1" + build: + number: 2 + + invalid_jinja: + meta.yaml: | + package: + name: two + version: "0.1" + build: + number: {{ CONDA_NCURSES }} + + no_build_num: + meta.yaml: | + package: + name: two + version: "0.1" + """, from_string=True) + expected.write_recipes() + + utils.bump_build_numbers(r.recipe_dirs.values()) + get_meta = lambda recipe: os.path.join(recipe, "meta.yaml") + for recipe, path in r.recipe_dirs.items(): + assert filecmp.cmp(get_meta(path), + get_meta(expected.recipe_dirs[recipe])) + + def test_built_package_paths(): r = Recipes( """