Source code for packman.packman

#!/usr/bin/env python
# #
# ###### Copyright (c) 2014 GigaSpaces Technologies Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
#    * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    * See the License for the specific language governing permissions and
#    * limitations under the License.

# TODO: (FEAT) add http://megastep.org/makeself/ support
# TODO: (FEAT) add http://semver.org/ support
# TODO: (FEAT) add support to download and run from github repos so that "components" repos can be created  # NOQA

import logger
import utils

import python
import yum
import retrieve
import apt
import ruby
import templater
import fpm
import codes
# import importlib

import definitions as defs

import sh
import os
import yaml
import sys

SUPPORTED_DISTROS = ('Ubuntu', 'debian', 'centos')
DEFAULT_PACKAGES_FILE = 'packages.yaml'
PACKAGE_TYPES = {"centos": "rpm", "debian": "deb"}
SUPPORTED_PACKAGE_TYPES = ['deb', 'rpm', 'tar', 'zip', 'tar.gz']


lgr = logger.init()


def _import_packages_dict(config_file=None):
    """returns a configuration object

    :param string config_file: path to config file
    """
    if config_file is None:
        try:
            with open(DEFAULT_PACKAGES_FILE, 'r') as c:
                return yaml.safe_load(c.read())['packages']
        except:
            lgr.error('No config file defines and could not find '
                      'packages.yaml in currect directory.')
            sys.exit(codes.mapping['packages_file_not_found'])
    # get config file path
    lgr.debug('Config file is: {0}'.format(config_file))
    # append to path for importing
    try:
        lgr.info('Importing config...')
        with open(config_file, 'r') as c:
            return yaml.safe_load(c.read())['packages']
    except IOError as ex:
        lgr.error(ex.message)
        lgr.error('Cannot access config file')
        sys.exit(codes.mapping['cannot_access_config_file'])
    except (yaml.parser.ParserError, yaml.scanner.ScannerError) as ex:
        lgr.error(ex.message)
        lgr.error('Invalid yaml file')
        sys.exit(codes.mapping['invalid_yaml_file'])


[docs]def get_package_config(package_name, packages_dict=None, packages_file=None): """returns a package's configuration if `packages_dict` is not supplied, a packages.yaml file in the cwd will be assumed unless `packages_file` is explicitly given. after a `packages_dict` is defined, a `package_config` will be returned for the specified package_name. :param string package: package name to retrieve config for. :param dict packages_dict: dict containing packages configuration :param string packages_file: packages file to search in :rtype: `dict` representing package configuration """ if packages_dict is None: packages_dict = {} lgr.debug('Retrieving configuration for {0}'.format(package_name)) try: if not packages_dict: packages_dict = _import_packages_dict(packages_file) lgr.debug('{0} config retrieved successfully'.format(package_name)) return packages_dict[package_name] except KeyError: lgr.error('Package configuration for' ' {0} was not found, terminating...'.format(package_name)) sys.exit(codes.mapping['no_config_found_for_package'])
[docs]def packman_runner(action, packages_file=None, packages=None, excluded=None, verbose=False): """logic for running packman. mainly called from the cli (pkm.py) if no `packages_file` is supplied, we will assume a local packages.yaml as `packages_file`. if `packages` are supplied, they will be iterated over. if `excluded` are supplied, they will be ignored. if a pack.py or get.py files are present, and an action_package function exists in the files, those functions will be used. else, the base get and pack methods supplied with packman will be used. so for instance, if you have a package named `x`, and you want to write your own `get` function for it. Just write a get_x() function in get.py. :param string action: action to perform (get, pack) :param string packages_file: path to file containing package config :param string packages: comma delimited list of packages to perform `action` on. :param string excluded: comma delimited list of packages to exclude :param bool verbose: determines output verbosity level :rtype: `None` """ def build_excluded_packages_list(excluded_packages): lgr.debug('Building excluded packages list...') return filter(None, (excluded_packages or "").split(',')) def build_packages_list(packages, xcluded_packages_list, packages_dict): lgr.debug('Building packages list...') package_list = [] # if you specified a list of packages if packages: package_list = [p for p in packages.split(',')] # raise if same package appears in both lists if set(package_list) & set(xcluded_packages_list): lgr.error('Your packages list and excluded packages ' 'list contain a similar item.') sys.exit(codes.mapping['excluded_conflict']) # else iterate over all packages in packages file else: package_list = [p for p in packages_dict.keys()] # and rewrite the list after removing excluded packages for xcld in xcluded_packages_list: package_list = [pkg for pkg in package_list if pkg != xcld] return package_list def import_overriding_methods(action): lgr.debug('Importing overriding methods file...') sys.path.append(os.getcwd()) return __import__(action) def rename_package(package): # this is meant to unify package names so that common # dashes, hyphens, dots and case errors do not occur. package_re = package.replace('-', '_') package_re = package_re.replace('.', '') package_re = package_re.lower() return package_re utils.set_global_verbosity_level(verbose) packages_dict = _import_packages_dict(packages_file) xcluded_packages_list = build_excluded_packages_list(excluded) lgr.debug('Excluded packages list: {0}'.format(xcluded_packages_list)) # append packages to list if a list is supplied package_list = build_packages_list( packages, xcluded_packages_list, packages_dict) lgr.debug('Package list: {0}'.format(package_list)) # if at least 1 package exists if package_list: for package in package_list: package_dict = get_package_config( package_name=package, packages_dict=packages_dict, packages_file=packages_file) validate = Validate(package_dict) validate.validate_package_properties() # looks for the overriding methods file in the current path if os.path.isfile(os.path.join( os.getcwd(), '{0}.py'.format(action))): # TODO: allow sending parameters to the overriding methods overr_methods = import_overriding_methods(action) package = rename_package(package) # if the method was found in the overriding file, run it. if hasattr(overr_methods, '{0}_{1}'.format(action, package)): getattr(overr_methods, '{0}_{1}'.format(action, package))() # else run the default action method else: # TODO: check for bad action globals()[action](package_dict) else: globals()[action](package_dict) else: lgr.error('No packages to handle, Verify that your packages file ' 'contains packages and that you did not exclude ' 'all of them.') sys.exit(codes.mapping['no_packages_defined'])
[docs]def get(package): """retrieves resources for packaging .. note:: package params are defined in packages.yaml .. note:: param names in packages.yaml can be overriden by editing definitions.py which also has an explanation on each param. :param dict package: dict representing package config as configured in packages.yaml will be appended to the filename and to the package depending on its type :rtype: `None` """ def handle_sources_path(sources_path, overwrite): if sources_path is None: lgr.error('Sources path key is required under {0} ' 'in packages.yaml.'.format(defs.PARAM_SOURCES_PATH)) sys.exit(codes.mapping['sources_path_required']) u = utils.Handler() # should the source dir be removed before retrieving package contents? if overwrite: lgr.info('Overwrite enabled. removing {0} before retrieval'.format( sources_path)) u.rmdir(sources_path) else: if os.path.isdir(sources_path): lgr.error('The destination directory for this package already ' 'exists and overwrite is disabled.') sys.exit(codes.mapping['path_already_exists_no_overwrite']) # create the directories required for package creation... if not os.path.isdir(sources_path): u.mkdir(sources_path) # you can send the package dict directly, or retrieve it from # the packages.yaml file by sending its name c = package if isinstance(package, dict) else get_package_config(package) repo = yum.Handler() if CENTOS else apt.Handler() if DEBIAN else None retr = retrieve.Handler() py = python.Handler() rb = ruby.Handler() sources_path = c.get(defs.PARAM_SOURCES_PATH, None) handle_sources_path( sources_path, c.get(defs.PARAM_OVERWRITE_SOURCES, True)) # TODO: (TEST) raise on "command not supported by distro" # TODO: (FEAT) add support for building packages from source repo.install(c.get(defs.PARAM_PREREQS, [])) repo.add_src_repos(c.get(defs.PARAM_SOURCE_REPOS, [])) if c.get(defs.PARAM_SOURCE_PPAS, []) and not DEBIAN: lgr.error('ppas not supported by {0}'.format(utils.get_distro())) sys.exit(codes.mapping['ppa_not_supported_by_distro']) repo.add_ppa_repos(c.get(defs.PARAM_SOURCE_PPAS, [])) retr.downloads(c.get(defs.PARAM_SOURCE_KEYS, []), sources_path) repo.add_keys(c.get(defs.PARAM_SOURCE_KEYS, []), sources_path) retr.downloads(c.get(defs.PARAM_SOURCE_URLS, []), sources_path) repo.download(c.get(defs.PARAM_REQS, []), sources_path) if c.get(defs.PARAM_VIRTUALENV): with utils.chdir(os.path.abspath(sources_path)): py.make_venv(c['virtualenv']['path']) py.install(c['virtualenv']['modules'], c['virtualenv']['path'], sources_path) py.get_modules(c.get(defs.PARAM_PYTHON_MODULES, []), sources_path) rb.get_gems(c.get(defs.PARAM_RUBY_GEMS, []), sources_path) # nd.get_packages(c.get(defs.PARAM_NODE_PACKAGES, []), sources_path) lgr.info('Package retrieval completed successfully!')
[docs]def pack(package): """creates a package according to the provided package configuration in packages.yaml uses fpm (https://github.com/jordansissel/fpm/wiki) to create packages. .. note:: package params are defined in packages.yaml but can be passed directly to the pack function as a dict. .. note:: param names in packages.yaml can be overriden by editing definitions.py which also has an explanation on each param. :param string|dict package: string or dict representing package name or params (coorespondingly) as configured in packages.yaml :rtype: `None` """ def handle_package_path(package_path, sources_path, name, overwrite): if not os.path.isdir(package_path): u.mkdir(package_path) if sources_path == package_path: lgr.error('Sources path and package paths must' ' be different to avoid conflicts!') sys.exit(codes.mapping['sources_and_package_paths_identical']) if overwrite: lgr.info('Overwrite enabled. Removing {0}/{1}* ' 'before packaging'.format(package_path, name)) u.rm('{0}/{1}*'.format(package_path, name)) def set_dst_pkg_type(): lgr.debug('Destination package type omitted') if CENTOS: lgr.debug('Assuming default type: {0}'.format( PACKAGE_TYPES['centos'])) return [PACKAGE_TYPES['centos']] elif DEBIAN: lgr.debug('Assuming default type: {0}'.format( PACKAGE_TYPES['debian'])) return [PACKAGE_TYPES['debian']] def convert_tar_to_targz(tar_file): lgr.debug('Converting tar to tar.gz...') sh.gzip(tar_file) # you can send the package dict directly, or retrieve it from # the packages.yaml file by sending its name c = package if isinstance(package, dict) else get_package_config(package) name = c.get(defs.PARAM_NAME) bootstrap_template = c.get(defs.PARAM_BOOTSTRAP_TEMPLATE_PATH, False) bootstrap_script = c.get(defs.PARAM_BOOTSTRAP_SCRIPT_PATH, False) src_pkg_type = c.get(defs.PARAM_SOURCE_PACKAGE_TYPE, False) dst_pkg_types = c.get( defs.PARAM_DESTINATION_PACKAGE_TYPES, set_dst_pkg_type()) try: sources_path = os.path.abspath(c[defs.PARAM_SOURCES_PATH]) except KeyError: lgr.error('Sources path key is required under {0} ' 'in packages.yaml.'.format(defs.PARAM_SOURCES_PATH)) package_path = c.get(defs.PARAM_PACKAGE_PATH, os.getcwd()) u = utils.Handler() templates = templater.Handler() handle_package_path( package_path, sources_path, name, c.get(defs.PARAM_OVERWRITE_OUTPUT, False)) lgr.info('Generating package scripts and config files...') if c.get(defs.PARAM_CONFIG_TEMPLATE_CONFIG, False): templates.generate_configs(c) if bootstrap_script: if bootstrap_template: templates.generate_from_template( c, bootstrap_script, bootstrap_template) for package in dst_pkg_types: # when creating a deb or rpm, it isn't required to chmod the script if package in ('tar', 'tar.gz'): lgr.debug('Granting execution permissions to script.') sh.chmod('+x', bootstrap_script) lgr.debug('Copying bootstrap script to package directory') u.cp(bootstrap_script, sources_path) lgr.info('Packaging: {0}'.format(name)) # this checks if a package needs to be created. If no source package type # is supplied, the assumption is that packages are only being downloaded # so if there's a source package type... if not os.listdir(sources_path) == []: fpm_params = { 'version': c.get(defs.PARAM_VERSION, False), 'force': c.get(defs.PARAM_OVERWRITE_OUTPUT, True), 'depends': c.get(defs.PARAM_DEPENDS, False), 'after_install': False if not bootstrap_script else os.path.abspath(bootstrap_script), 'chdir': False, 'before_install': None } # change the path to the destination path, since fpm doesn't # accept (for now) a dst dir, but rather creates the package in # the cwd. with utils.chdir(os.path.abspath(package_path)): for dst_pkg_type in dst_pkg_types: packager = fpm.Handler( name, src_pkg_type, dst_pkg_type, sources_path) result = packager.execute(**fpm_params) if not result: lgr.error('Failed to create package.') sys.exit(codes.mapping['failed_create_package']) if dst_pkg_type == "tar.gz": tar_file = '{0}.tar'.format(name) targz_file = tar_file + '.gz' if os.path.isfile(targz_file): if fpm_params['force']: u.rm(targz_file) else: lgr.error('{0} already exists and overwrite ' 'is false.'.format(targz_file)) sys.exit(codes.mapping['targz_exists']) convert_tar_to_targz(tar_file) else: lgr.error('Sources directory is empty. Nothing to package.') sys.exit(codes.mapping['sources_empty']) lgr.info('Package creation completed successfully!') if not c.get(defs.PARAM_KEEP_SOURCES, True): lgr.debug('Removing sources...') u.rmdir(sources_path)
[docs]class Validate(): def __init__(self, package): self.package = package
[docs] def validate_package_properties(self): if defs.PARAM_DESTINATION_PACKAGE_TYPES in self.package: self.destination_package_types( self.package[defs.PARAM_DESTINATION_PACKAGE_TYPES])
[docs] def destination_package_types(self, package_types): if not isinstance(package_types, list): lgr.error('{0} key must be of type "list".'.format( defs.PARAM_DESTINATION_PACKAGE_TYPES)) sys.exit(codes.mapping['package_types_must_be_list']) for package_type in package_types: if package_type not in SUPPORTED_PACKAGE_TYPES: lgr.error('{0} key must contain one of: {1}.'.format( defs.PARAM_DESTINATION_PACKAGE_TYPES, SUPPORTED_PACKAGE_TYPES)) sys.exit(codes.mapping['unsupported_package_type'])
def main(): lgr.debug('Running in main...') if __name__ == '__main__': main() # TODO: fail on Windows CENTOS = utils.get_distro() in ('centos') DEBIAN = utils.get_distro() in ('Ubuntu', 'debian')