dh_python2

#! /usr/bin/python3
# vim: et ts=4 sw=4

# Copyright © 2010-2013 Piotr Ożarowski <piotr@debian.org>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

import logging
import os
import re
import sys
from filecmp import dircmp, cmpfiles, cmp as fcmp
from optparse import OptionParser, SUPPRESS_HELP
from os.path import isabs, isdir, islink, exists, join, normpath, splitext,\
    realpath
from shutil import copy as fcopy
from dhpython.debhelper import DebHelper
from dhpython.depends import Dependencies
from dhpython.interpreter import Interpreter
from dhpython.fs import fix_locations, Scan
from dhpython.version import supported, default, VersionRange, \
    get_requested_versions
from dhpython.pydist import validate as validate_pydist
from dhpython.tools import relative_symlink, so2pyver, parse_ns, remove_ns,\
    pyinstall, pyremove
from dhpython.option import Option

# initialize script
logging.basicConfig(format='%(levelname).1s: dh_python2 '
                           '%(module)s:%(lineno)d: %(message)s')
log = logging.getLogger('dhpython')
os.umask(0o22)
DEFAULT = default('cpython2')
SUPPORTED = supported('cpython2')

fext = lambda fname: splitext(fname)[-1][1:]


class Scanner(Scan):
    def handle_ext(self, fpath):
        so_version = so2pyver(fpath)
        if so_version:
            path, fn = fpath.rsplit('/', 1)
            if self.current_pub_version:
                if self.current_pub_version != so_version:
                    log.error('extension linked to libpython%s '
                              'and shipped in python%s\'s dist-'
                              'packages: %s',
                              so_version, self.current_pub_version, fn)
                log.warn('public extension linked with '
                         'libpython%s: %s', so_version, fn)
            return so_version


### SHARING FILES ##############################################
def share(package, stats, options):
    """Move files to /usr/share/pyshared/ if possible."""
    if package.endswith('-dbg'):
        # nothing to share in debug packages
        return
    interpreter = Interpreter('python')
    pubvers = sorted(v for v in stats['public_vers'] if v.major == 2)
    if len(pubvers) > 1:
        for pos, version1 in enumerate(pubvers):
            dir1 = interpreter.sitedir(package, version1)
            if not exists(dir1):
                continue
            for version2 in pubvers[pos + 1:]:
                dir2 = interpreter.sitedir(package, version2)
                if exists(dir2):
                    dc = dircmp(dir1, dir2)
                    share_2x(dir1, dir2, dc)
#    elif len(pubvers) == 1:
#        move_to_pyshared(interpreter.sitedir(package, pubvers[0]))
#    for version in stats['ext_vers']:
#        create_ext_links(interpreter.sitedir(package, version))

    if options.guess_versions and pubvers:
        for version in get_requested_versions('cpython2', options.vrange):
            if version not in pubvers:
                interpreter.version = version
                log.debug('guessing files for %s', interpreter)
                versions_without_ext = sorted(set(pubvers) -
                                              stats['ext_vers'])
                if not versions_without_ext:
                    log.error('extension for python%s is missing. '
                              'Build extensions for all supported Python '
                              'versions (`pyversions -vr`) or adjust '
                              'X-Python-Version field or pass '
                              '--no-guessing-versions to dh_python2',
                              version)
                    exit(3)
                srcver = versions_without_ext[0]
                if srcver in stats['public_vers']:
                    stats['public_vers'].add(version)
                share_2x(interpreter.sitedir(package, srcver),
                         interpreter.sitedir(package, version))
    # remove duplicates
    stats['requires.txt'] = set(realpath(i) for i in stats['requires.txt'])
    stats['nsp.txt'] = set(realpath(i) for i in stats['nsp.txt'])


# def move_to_pyshared(dir1):
#     # dir1 starts with debian/packagename/usr/lib/pythonX.Y/*-packages/
#     debian, package, path = dir1.split('/', 2)
#     dstdir = join(debian, package, 'usr/share/pyshared/',
#                   '/'.join(dir1.split('/')[6:]))
#
#     for i in os.listdir(dir1):
#         fpath1 = join(dir1, i)
#         if isdir(fpath1) and not islink(fpath1):
#             if any(fn for fn in os.listdir(fpath1) if fext(fn) != 'so'):
#                 # at least one file that is not an extension
#                 move_to_pyshared(join(dir1, i))
#         else:
#             if fext(i) == 'so':
#                 continue
#             fpath2 = join(dstdir, i)
#             if not exists(fpath2):
#                 if not exists(dstdir):
#                     os.makedirs(dstdir)
#                 if islink(fpath1):
#                     fpath1_target = os.readlink(fpath1)
#                     if isabs(fpath1_target):
#                         os.symlink(fpath1_target, fpath2)
#                     else:
#                         fpath1_target = normpath(join(dir1, fpath1_target))
#                         relative_symlink(fpath1_target, fpath2)
#                     os.remove(fpath1)
#                 else:
#                     os.rename(fpath1, fpath2)
#                 relative_symlink(fpath2, fpath1)
#
#
# def create_ext_links(dir1):
#     """Create extension symlinks in /usr/lib/pyshared/pythonX.Y.
#
#     These symlinks are used to let dpkg detect file conflicts with
#     python-support and python-central packages.
#     """
#
#     debian, package, path = dir1.split('/', 2)
#     python, _, module_subpath = path[8:].split('/', 2)
#     dstdir = join(debian, package, 'usr/lib/pyshared/', python, module_subpath)
#
#     for i in os.listdir(dir1):
#         fpath1 = join(dir1, i)
#         if isdir(fpath1):
#             create_ext_links(fpath1)
#         elif fext(i) == 'so':
#             fpath2 = join(dstdir, i)
#             if exists(fpath2):
#                 continue
#             if not exists(dstdir):
#                 os.makedirs(dstdir)
#             relative_symlink(fpath1, join(dstdir, i))


def create_public_links(dir1, vrange, root=''):
    """Create public module symlinks for given directory."""

    debian, package, path = dir1.split('/', 2)
    interpreter = Interpreter('python')
    versions = get_requested_versions('cpython2', vrange)

    for fn in os.listdir(dir1):
        fpath1 = join(dir1, fn)
        if isdir(fpath1):
            create_public_links(fpath1, vrange, join(root, fn))
        else:
            for version in versions:
                dstdir = join(interpreter.sitedir(package, version), root)
                if not exists(dstdir):
                    os.makedirs(dstdir)
                relative_symlink(fpath1, join(dstdir, fn))


def share_2x(dir1, dir2, dc=None):
    """Move common files to pyshared and create symlinks in original
    locations."""
    debian, package, path = dir2.split('/', 2)
    # dir1 starts with debian/packagename/usr/lib/pythonX.Y/*-packages/
    dstdir = join(debian, package, 'usr/share/pyshared/',
                  '/'.join(dir1.split('/')[6:]))
    if not exists(dstdir) and not islink(dir1):
        os.makedirs(dstdir)
    if dc is None:  # guess/copy mode
        if not exists(dir2):
            os.makedirs(dir2)
        common_dirs = []
        common_files = []
        for i in os.listdir(dir1):
            subdir1 = join(dir1, i)
            if isdir(subdir1) and not islink(subdir1):
                common_dirs.append([i, None])
            else:
                # directories with .so files will be blocked earlier
                common_files.append(i)
    elif islink(dir1):
        # skip this symlink in pyshared
        # (dpkg has problems with symlinks anyway)
        common_dirs = []
        common_files = []
    else:
        common_dirs = dc.subdirs.items()
        common_files = dc.common_files
        # dircmp returns common names only, lets check files more carefully...
        common_files = cmpfiles(dir1, dir2, common_files, shallow=False)[0]

    for fn in common_files:
        if 'so' in fn.split('.') and not fn.startswith('so'):
            # foo.so, bar.so.0.1.2, etc.
            # in unlikely case where extensions are exactly the same
            continue
        fpath1 = join(dir1, fn)
        fpath2 = join(dir2, fn)
        fpath3 = join(dstdir, fn)
        # do not touch symlinks created by previous loop or other tools
        if dc and not islink(fpath1):
            # replace with a link to pyshared
            if not exists(fpath3):
                os.rename(fpath1, fpath3)
                relative_symlink(fpath3, fpath1)
            elif fcmp(fpath3, fpath1, shallow=False):
                os.remove(fpath1)
                relative_symlink(fpath3, fpath1)
        if dc is None:  # guess/copy mode
            if islink(fpath1):
                # ralative links will work as well, it's always the same level
                os.symlink(os.readlink(fpath1), fpath2)
            else:
                if exists(fpath3):
                    # cannot share it, pyshared contains another copy
                    fcopy(fpath1, fpath2)
                else:
                    # replace with a link to pyshared
                    os.rename(fpath1, fpath3)
                    relative_symlink(fpath3, fpath1)
                    relative_symlink(fpath3, fpath2)
        elif exists(fpath2) and exists(fpath3) and \
                fcmp(fpath2, fpath3, shallow=False):
            os.remove(fpath2)
            relative_symlink(fpath3, fpath2)
    for dn, dc in common_dirs:
        share_2x(join(dir1, dn), join(dir2, dn), dc)


################################################################
def main():
    usage = '%prog -p PACKAGE [-V [X.Y][-][A.B]] DIR [-X REGEXPR]\n'
    parser = OptionParser(usage, version='%prog DEVELV',
                          option_class=Option)
    parser.add_option('--no-guessing-versions', action='store_false',
                      dest='guess_versions', default=True,
                      help='disable guessing other supported Python versions')
    parser.add_option('--no-guessing-deps', action='store_false',
                      dest='guess_deps', default=True,
                      help='disable guessing dependencies')
    parser.add_option('--skip-private', action='store_true', default=False,
                      help='don\'t check private directories')
    parser.add_option('-v', '--verbose', action='store_true', default=False,
                      help='turn verbose mode on')
    # arch=False->arch:all only, arch=True->arch:any only, None->all of them
    parser.add_option('-i', '--indep', action='store_false',
                      dest='arch', default=None,
                      help='act on architecture independent packages')
    parser.add_option('-a', '-s', '--arch', action='store_true',
                      dest='arch', help='act on architecture dependent packages')
    parser.add_option('-q', '--quiet', action='store_false', dest='verbose',
                      help='be quiet')
    parser.add_option('-p', '--package', action='append',
                      help='act on the package named PACKAGE')
    parser.add_option('-N', '--no-package', action='append',
                      help='do not act on the specified package')
    parser.add_option('--compile-all', action='store_true', default=False,
                      help='compile all files from given private directory '
                           'in postinst, not just the ones provided by the '
                           'package')
    parser.add_option('-V', type='version_range', dest='vrange',
                      help='specify list of supported Python versions. ' +
                           'See pycompile(1) for examples')
    parser.add_option('-X', '--exclude', action='append', dest='regexpr',
                      help='exclude items that match given REGEXPR. You may '
                           'use this option multiple times to build up a list'
                           ' of things to exclude.')
    parser.add_option('--accept-upstream-versions', action='store_true',
                      default=False, help='accept upstream versions while '
                      'translating Python dependencies into Debian ones')
    parser.add_option('--depends', action='append',
                      help='translate given requirements into Debian '
                           'dependencies and add them to ${python:Depends}. '
                           'Use it for missing items in requires.txt.')
    parser.add_option('--depends-section', action='append',
                      help='translate requirements from given section into Debian '
                           'dependencies and add them to ${python3:Depends}')
    parser.add_option('--recommends', action='append',
                      help='translate given requirements into Debian '
                           'dependencies and add them to ${python:Recommends}')
    parser.add_option('--recommends-section', action='append',
                      help='translate requirements from given section into Debian '
                           'dependencies and add them to ${python3:Recommends}')
    parser.add_option('--suggests', action='append',
                      help='translate given requirements into Debian '
                           'dependencies and add them to ${python:Suggests}')
    parser.add_option('--suggests-section', action='append',
                      help='translate requirements from given section into Debian '
                           'dependencies and add them to ${python3:Suggests}')
    parser.add_option('--requires', action='append',
                      help='translate requirements from given file into Debian '
                           'dependencies and add them to ${python:Depends}')
    parser.add_option('--namespace', action='append', dest='namespaces',
                      help='recreate __init__.py files for given namespaces at install time')
    parser.add_option('--clean-pycentral', action='store_true', default=False,
                      help='generate maintainer script that will remove pycentral files')
    parser.add_option('--shebang',
                      help='use given command as shebang in scripts')
    parser.add_option('--ignore-shebangs', action='store_true', default=False,
                      help='do not translate shebangs into Debian dependencies')
    parser.add_option('--ignore-namespace', action='store_true', default=False,
                      help="ignore Egg's namespace_packages.txt file and --namespace option")
    parser.add_option('--no-dbg-cleaning', action='store_false',
                      dest='clean_dbg_pkg', default=True,
                      help='do not remove files from debug packages')
    parser.add_option('--no-ext-rename', action='store_true',
                      default=False, help='do not add magic tags nor multiarch'
                                          ' tuples to extension file names)')
    parser.add_option('--no-shebang-rewrite', action='store_true',
                      default=False, help='do not rewrite shebangs')
    # ignore some debhelper options:
    parser.add_option('-O', help=SUPPRESS_HELP)

    options, args = parser.parse_args(sys.argv[1:] +
                                      os.environ.get('DH_OPTIONS', '').split())
    # regexpr option type is not used so lets check patterns here
    for pattern in options.regexpr or []:
        # fail now rather than at runtime
        try:
            pattern = re.compile(pattern)
        except Exception:
            log.error('regular expression is not valid: %s', pattern)
            exit(1)

    if not options.vrange and exists('debian/pyversions'):
        log.debug('parsing version range from debian/pyversions')
        with open('debian/pyversions', encoding='utf-8') as fp:
            for line in fp:
                line = line.strip()
                if line and not line.startswith('#'):
                    options.vrange = VersionRange(line)
                    break

    # disable PyDist if dh_pydeb is used
    if options.guess_deps:
        try:
            rules = open('debian/rules', 'r', encoding='utf-8').read()
        except IOError:
            log.warning('cannot open debian/rules file')
        else:
            if re.search('\n\s*dh_pydeb', rules) or \
                    re.search('\n\s*dh\s+[^#]*--with[^#]+pydeb', rules):
                log.info('dh_pydeb detected, PyDist feature disabled')
                options.guess_deps = False

    if not args:
        private_dir = None
    else:
        private_dir = args[0]
        if not private_dir.startswith('/'):
            # handle usr/share/foo dirs (without leading slash)
            private_dir = '/' + private_dir
    # TODO: support more than one private dir at the same time (see :meth:scan)
    if options.skip_private:
        private_dir = False

    if options.verbose or os.environ.get('DH_VERBOSE') == '1':
        log.setLevel(logging.DEBUG)
        log.debug('version: DEVELV')
        log.debug('argv: %s', sys.argv)
        log.debug('options: %s', options)
        log.debug('args: %s', args)
        log.debug('supported Python versions: %s (default=%s)',
                  ','.join(str(v) for v in SUPPORTED), DEFAULT)
    else:
        log.setLevel(logging.INFO)

    try:
        dh = DebHelper(options, impl='cpython2')
    except Exception as e:
        log.error('cannot initialize DebHelper: %s', e)
        exit(2)
    if not dh.packages:
        log.error('no package to act on (python-foo or one with ${python:Depends} in Depends)')
        # exit(7)
    if not options.vrange and dh.python_version:
        options.vrange = VersionRange(dh.python_version)

    interpreter = Interpreter('python')
    for package, pdetails in dh.packages.items():
        log.debug('processing package %s...', package)
        interpreter.debug = package.endswith('-dbg')

        if not private_dir:
            try:
                pyinstall(interpreter, package, options.vrange)
            except Exception as err:
                log.error("%s.pyinstall: %s", package, err)
                exit(4)
            try:
                pyremove(interpreter, package, options.vrange)
            except Exception as err:
                log.error("%s.pyremove: %s", package, err)
                exit(5)
            fix_locations(package, interpreter, SUPPORTED, options)
        stats = Scanner(interpreter, package, private_dir, options).result
        if not private_dir:
            share(package, stats, options)
            pyshared_dir = "debian/%s/usr/share/pyshared/" % package
            if not stats['public_vers'] and exists(pyshared_dir):
                create_public_links(pyshared_dir, options.vrange)
                stats = Scanner(interpreter, package, private_dir, options).result

        dependencies = Dependencies(package, 'cpython2', dh.build_depends)
        dependencies.parse(stats, options)

        if stats['public_vers']:
            dh.addsubstvar(package, 'python:Versions',
                           ', '.join(str(i) for i in sorted(stats['public_vers'])))
            ps = package.split('-', 1)
            if len(ps) > 1 and ps[0] == 'python':
                dh.addsubstvar(package, 'python:Provides',
                               ', '.join("python%s-%s" % (i, ps[1])
                                         for i in sorted(stats['public_vers'])))

        pyclean_added = False  # invoke pyclean only once in maintainer script
        if stats['compile']:
            if options.clean_pycentral:
                dh.autoscript(package, 'preinst',
                              'preinst-pycentral-clean', '')
            dh.autoscript(package, 'postinst', 'postinst-pycompile', '')
            dh.autoscript(package, 'prerm', 'prerm-pyclean', '')
            pyclean_added = True

        for pdir, details in sorted(stats['private_dirs'].items()):
            if not details.get('compile'):
                continue
            if not pyclean_added:
                dh.autoscript(package, 'prerm', 'prerm-pyclean', '')
                pyclean_added = True

            args = pdir

            ext_for = details.get('ext_vers')
            ext_no_version = details.get('ext_no_version')
            if ext_for is None and not ext_no_version:  # no extension
                shebang_versions = list(i.version for i in details.get('shebangs', [])
                                        if i.version and i.version.minor)
                if not options.ignore_shebangs and len(shebang_versions) == 1:
                    # only one version from shebang
                    args += " -V %s" % shebang_versions[0]
                elif options.vrange and options.vrange != (None, None):
                    args += " -V %s" % options.vrange
            elif ext_no_version:
                # at least one extension's version not detected
                if options.vrange and '-' not in str(options.vrange):
                    ver = str(options.vrange)
                else:  # try shebang or default Python version
                    ver = (list(i.version for i in details.get('shebangs', [])
                                if i.version and i.version.minor) or [None])[0] or DEFAULT
                dependencies.depend("python%s" % ver)
                args += " -V %s" % ver
            else:
                version = ext_for.pop()
                args += " -V %s" % version
                dependencies.depend("python%s" % version)

            for pattern in options.regexpr or []:
                args += " -X '%s'" % pattern.replace("'", r"'\''")

            dh.autoscript(package, 'postinst', 'postinst-pycompile', args)

        dependencies.export_to(dh)

        pydist_file = join('debian', "%s.pydist" % package)
        if exists(pydist_file):
            if not validate_pydist(pydist_file):
                log.warning("%s.pydist file is invalid", package)
            else:
                dstdir = join('debian', package, 'usr/share/python/dist/')
                if not exists(dstdir):
                    os.makedirs(dstdir)
                fcopy(pydist_file, join(dstdir, package))
        bcep_file = join('debian', "%s.bcep" % package)
        if exists(bcep_file):
            dstdir = join('debian', package, 'usr/share/python/bcep/')
            if not exists(dstdir):
                os.makedirs(dstdir)
            fcopy(bcep_file, join(dstdir, package))

        # namespace feature - recreate __init__.py files at install time
        if options.ignore_namespace:
            nsp = None
        else:
            nsp = parse_ns(stats['nsp.txt'], options.namespaces)
        # note that pycompile/pyclean is already added to maintainer scripts
        # and it should remain there even if __init__.py was the only .py file
        if nsp:
            try:
                nsp = remove_ns(Interpreter('python'), package, nsp,
                                stats['public_vers'])
            except (IOError, OSError) as e:
                log.error('cannot remove __init__.py from package: %s', e)
                exit(6)
        if nsp:
            dstdir = join('debian', package, 'usr/share/python/ns/')
            if not exists(dstdir):
                os.makedirs(dstdir)
            with open(join(dstdir, package), 'a', encoding='utf-8') as fp:
                fp.writelines("%s\n" % i for i in sorted(nsp))

        pyshared = join('debian', package, 'usr/share/pyshared/')
        if isdir(pyshared) and not os.listdir(pyshared):
            # remove empty pyshared directory
            os.rmdir(pyshared)

    dh.save()


if __name__ == '__main__':
    main()