diff --git a/README.md b/README.md index c8d8594..6525e68 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,13 @@ -Fast Assembler for Bootc -=== +# Fast Assembler for Bootc `fab` is a somewhat opinionated build pipeline for [bootc](https://github.com/containers/bootc), allowing the modularization of Containerfiles/bootc image building. -Fabfiles and module definitions ---- +## Fabfiles and module definitions -`fab` is structured as a main descriptive file for the final image, currently called "Fabfile". A Fabfile example looks like this: +`fab` is structured as a main descriptive file for the final image, currently called "`Fabfile`". A `Fabfile` example looks like this: -``` +```yaml --- metadata: name: fabrules @@ -25,14 +23,14 @@ buildargs: The fields are somewhat self-explanatory: - - `metadata`: just metadata about the bootc image - - `from`: base image (i.e. the first `FROM` in the pipeline) - - `include`: list of modules to include, in order in the bootc image - - `buildargs`: list of buildargs (variables) used in the build process +- `metadata`: just metadata about the bootc image +- `from`: base image (i.e. the first `FROM` in the pipeline) +- `include`: list of modules to include, in order in the bootc image +- `buildargs`: list of buildargs (variables) used in the build process For each module, there is a short descriptive file with the module definition (see `modules/` directory for examples): -``` +```yaml --- metadata: name: dnf-install @@ -45,27 +43,25 @@ buildargs: Like with the top level definition, the fields should be easy to understand: - - `metadata`: module level metadata - - `containerfile`: filename for the Containerfile to use - - `buildargs`: simple list of expected buildargs (without values!) +- `metadata`: module level metadata +- `containerfile`: filename for the `Containerfile` to use +- `buildargs`: simple list of expected buildargs (without values!) -Installation ---- +## Installation -``` +```sh $ git clone git@github.com:kwozyman/fab.git $ cd fab $ python3 -m pip install --requirement requirements.txt ``` -Usage ---- +## Usage A simple `python3 -m fab --help` will show the full command line arguments. In order to build the container from `Fabfile.example`: -``` +```sh $ python3 -m fab build --fabfile Fabfile.example ``` diff --git a/fab/__main__.py b/fab/__main__.py index 1664f4c..10672b1 100755 --- a/fab/__main__.py +++ b/fab/__main__.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 from fab.cli import FabCli -from fab.module import FabModule -if __name__ == '__main__': +if __name__ == "__main__": FabCli() diff --git a/fab/cli.py b/fab/cli.py index 3c12fb0..1bd20bc 100644 --- a/fab/cli.py +++ b/fab/cli.py @@ -1,19 +1,21 @@ import logging from fab.fab import Fab -class FabCli(): + +class FabCli: """ Fab command line """ + def __init__(self, **kwargs): - self.loglevel = 'info' - self.log_format = '%(asctime)19s - %(levelname)8s - %(message)s' - self.log_datefmt = '%d-%m-%Y %H:%M:%S' + self.loglevel = "info" + self.log_format = "%(asctime)19s - %(levelname)8s - %(message)s" + self.log_datefmt = "%d-%m-%Y %H:%M:%S" self.logmap = { - 'info': logging.INFO, - 'warning': logging.WARN, - 'warn': logging.WARN, - 'debug': logging.DEBUG + "info": logging.INFO, + "warning": logging.WARN, + "warn": logging.WARN, + "debug": logging.DEBUG, } config = {**self._cmdargs(), **kwargs} for k in config: @@ -22,85 +24,102 @@ def __init__(self, **kwargs): self.loglevel = self.log_level self.set_loglevel(self.loglevel) - if config['command'] == 'build': - self.fab = Fab(config['fabfile'], - container_tool=config['container_tool'], - tool_args=config['container_tool_extra_args']) + if config["command"] == "build": + self.fab = Fab( + config["fabfile"], + container_tool=config["container_tool"], + tool_args=config["container_tool_extra_args"], + ) self.fab.build() def _basic_logging(self): - logging.basicConfig(level=self.logmap[self.loglevel], - format=self.log_format, - datefmt=self.log_datefmt) + logging.basicConfig( + level=self.logmap[self.loglevel], + format=self.log_format, + datefmt=self.log_datefmt, + ) def set_loglevel(self, level): logging.getLogger().setLevel(self.logmap[level]) - logging.debug('DEBUG mode is enabled') + logging.debug("DEBUG mode is enabled") def _cmdargs(self): """ Parse command line arguments and read config files (if module exists) """ - description = 'Fast Assembly Bootc' + description = "Fast Assembly Bootc" try: import configargparse + parser = configargparse.ArgParser( default_config_files=[ - 'fab.config', - '/etc/fab/config', - '~/.config/fab/config'], - description=description + "fab.config", + "/etc/fab/config", + "~/.config/fab/config", + ], + description=description, ) except ModuleNotFoundError: logging.debug('Could not find module "configparse"') import argparse - parser = argparse.ArgumentParser( - description=description - ) - parser.add_argument('--log-level', '--loglevel', - choices=self.logmap.keys(), - type=str.lower, - default=self.loglevel, - help='Logging level', - ) - parser.add_argument('--log-format', - type=str, - default=self.log_format, - help='Python Logger() compatible format string') - parser.add_argument('--log-datefmt', - type=str, - default=self.log_datefmt, - help='Python Logger() compatible date format str') - parser.add_argument('--container-tool', - help='What container tool to use', - default='/usr/bin/podman') - parser.add_argument('--container-tool-extra-args', - help='container tool extra arguments', - default=[]) + parser = argparse.ArgumentParser(description=description) - subparsers = parser.add_subparsers(dest='command') + parser.add_argument( + "--log-level", + "--loglevel", + choices=self.logmap.keys(), + type=str.lower, + default=self.loglevel, + help="Logging level", + ) + parser.add_argument( + "--log-format", + type=str, + default=self.log_format, + help="Python Logger() compatible format string", + ) + parser.add_argument( + "--log-datefmt", + type=str, + default=self.log_datefmt, + help="Python Logger() compatible date format str", + ) + parser.add_argument( + "--container-tool", + help="What container tool to use", + default="/usr/bin/podman", + ) + parser.add_argument( + "--container-tool-extra-args", + help="container tool extra arguments", + default=[], + ) + + subparsers = parser.add_subparsers(dest="command") subparsers.required = False - parser_build = subparsers.add_parser('build', - help='Build bootc container') - parser_build.add_argument('--fabfile', - help='path to fabfile', - default='Fabfile') + parser_build = subparsers.add_parser("build", help="Build bootc container") + parser_build.add_argument( + "--fabfile", help="path to fabfile", default="Fabfile" + ) try: import argcomplete from os.path import basename - parser.add_argument('--bash-completion', - action='store_true', - help='Dump bash completion file.' - ' Activate with "eval ' - '$({} --bash-completion)"'.format(basename(__file__))) + + parser.add_argument( + "--bash-completion", + action="store_true", + help="Dump bash completion file." + ' Activate with "eval ' + '$({} --bash-completion)"'.format(basename(__file__)), + ) argcomplete.autocomplete(parser) except ModuleNotFoundError: - logging.debug('argcomplete module not found, no bash completion available') + logging.debug("argcomplete module not found, no bash completion available") args = parser.parse_args() - if 'bash_completion' in args: + if "bash_completion" in args: if args.bash_completion: - print(argcomplete.shellcode(basename(__file__), True, 'bash')) + print(argcomplete.shellcode(basename(__file__), True, "bash")) return vars(args) diff --git a/fab/fab.py b/fab/fab.py index 806e3b7..5d43bcd 100644 --- a/fab/fab.py +++ b/fab/fab.py @@ -3,118 +3,132 @@ import subprocess from fab.module import FabModule + class Fab: """ Fabfile definition """ - def __init__(self, source, container_tool='/usr/bin/podman', tool_args=[]): + + def __init__(self, source, container_tool="/usr/bin/podman", tool_args=[]): self.source = source self.name = source self.container_tool = container_tool self.tool_args = tool_args self.includes = [] self._read() - logging.debug('Read Fabfile: {}'.format(self.definition)) + logging.debug("Read Fabfile: {}".format(self.definition)) if not self._validate(): - raise Exception('Fabfile not valid') + raise Exception("Fabfile not valid") self.includes = [] self._load_includes() def _read(self): - f = open(self.source, 'r') + f = open(self.source, "r") self.definition = yaml.load(f, Loader=yaml.Loader) f.close() def _validate(self): is_valid = True - if not 'metadata' in self.definition: - logging.warning('No metadata found for fabfile {}. Generating some.'.format(self.source)) - self.definition['metadata'] = {} - self.definition['metadata']['name'] = self.source - self.definition['metadata']['description'] = 'Autogenerated' + if "metadata" not in self.definition: + logging.warning( + "No metadata found for fabfile {}. Generating some.".format(self.source) + ) + self.definition["metadata"] = {} + self.definition["metadata"]["name"] = self.source + self.definition["metadata"]["description"] = "Autogenerated" else: - if not 'name' in self.definition['metadata']: - logging.warning("No name set in fabfile {} metadata. Autosetting to {}".format(self.name, self.name)) - self.definition['metadata']['name'] = self.name + if "name" not in self.definition["metadata"]: + logging.warning( + "No name set in fabfile {} metadata. Autosetting to {}".format( + self.name, self.name + ) + ) + self.definition["metadata"]["name"] = self.name else: - self.name = self.definition['metadata']['name'] - if not 'description' in self.definition['metadata']: - logging.warning("No description set in fabfile {} metadata. Autosetting".format(self.name)) - self.definition['metadata']['description'] = 'Autogenerated' + self.name = self.definition["metadata"]["name"] + if "description" not in self.definition["metadata"]: + logging.warning( + "No description set in fabfile {} metadata. Autosetting".format( + self.name + ) + ) + self.definition["metadata"]["description"] = "Autogenerated" - if not 'from' in self.definition: + if "from" not in self.definition: logging.error('"from" key is required in fabfile') is_valid = False - if not 'include' in self.definition: - logging.warning('No modules included') - self.definition['include'] = [] + if "include" not in self.definition: + logging.warning("No modules included") + self.definition["include"] = [] - if 'buildargs' in self.definition: - if not isinstance(self.definition['buildargs'],list): + if "buildargs" in self.definition: + if not isinstance(self.definition["buildargs"], list): logging.error("'buildargs' is not a list") is_valid = False else: - self.definition['buildargs'] = [] + self.definition["buildargs"] = [] - return(is_valid) + return is_valid def _load_includes(self): - for include in self.definition['include']: - if isinstance(include,str): + for include in self.definition["include"]: + if isinstance(include, str): _include = include _var_values = {} elif isinstance(include, dict): - _include = include['include'] - if 'buildargs' in include.keys(): - for item in include['buildargs']: + _include = include["include"] + if "buildargs" in include.keys(): + for item in include["buildargs"]: for key in item: _var_values[key] = item[key] else: logging.debug('No "buildargs" set for {}'.format(include)) _var_values = {} - logging.debug('Add new module {} with buildargs {}'.format(_include, _var_values)) + logging.debug( + "Add new module {} with buildargs {}".format(_include, _var_values) + ) self.includes.append(FabModule(source=_include, var_values=_var_values)) def _run(self, command, args, cwd): - logging.debug('{} {}'.format(command, args)) - process = subprocess.Popen([command] + args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=cwd) + logging.debug("{} {}".format(command, args)) + process = subprocess.Popen( + [command] + args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=cwd + ) while True: - output = process.stdout.readline().rstrip().decode('utf-8') - if output == '' and process.poll() is not None: + output = process.stdout.readline().rstrip().decode("utf-8") + if output == "" and process.poll() is not None: break if output: - print(' {}'.format(output.strip())) + print(" {}".format(output.strip())) rc = process.poll() return rc def build(self): - previous_container_image = self.definition['from'] + previous_container_image = self.definition["from"] print(len(self.includes)) for module in self.includes: - tag = '{}-stage-{}'.format( - self.name, - module.name) + tag = "{}-stage-{}".format(self.name, module.name) podman_args = [] podman_args.append(self.tool_args) - podman_args.append('build') - podman_args.append('--from') + podman_args.append("build") + podman_args.append("--from") podman_args.append(previous_container_image) - podman_args.append('--file') + podman_args.append("--file") podman_args.append(module.containerfile) - podman_args.append('--tag') + podman_args.append("--tag") podman_args.append(tag) - for arg in self.definition['buildargs']: + for arg in self.definition["buildargs"]: for key in arg: - podman_args.append('--build-arg') - podman_args.append('{}={}'.format(key, arg[key])) - logging.debug('podman command: {}'.format(podman_args)) - logging.info('Start build of {} stage'.format(tag)) + podman_args.append("--build-arg") + podman_args.append("{}={}".format(key, arg[key])) + logging.debug("podman command: {}".format(podman_args)) + logging.info("Start build of {} stage".format(tag)) self._run(self.container_tool, podman_args, module.working_dir) previous_container_image = tag podman_args = [] podman_args.append(self.tool_args) - podman_args.append('tag') + podman_args.append("tag") podman_args.append(previous_container_image) podman_args.append(self.name) self._run(self.container_tool, podman_args, None) diff --git a/fab/module.py b/fab/module.py index 2df10d6..28a4639 100644 --- a/fab/module.py +++ b/fab/module.py @@ -4,10 +4,12 @@ import urllib.parse import pathlib -class FabModule(): + +class FabModule: """ Definition of a generic Fab module """ + def __init__(self, source, **kwargs): self.source = source self.name = self.source @@ -17,59 +19,77 @@ def __init__(self, source, **kwargs): self._read() logging.debug("Read definition for module: {}".format(self.definition)) if not self._validate(): - logging.error('Definition YAML for {} is not valid.'.format(self.source)) + logging.error("Definition YAML for {} is not valid.".format(self.source)) raise Exception("YAML is not valid") def __str__(self): - return(pprint.pformat(self.definition, indent=4)) - + return pprint.pformat(self.definition, indent=4) + def _read(self): parsed_source = urllib.parse.urlparse(self.source) - logging.debug('Decoded source: {}'.format(parsed_source)) - if parsed_source.scheme == '' or parsed_source.scheme == 'file': + logging.debug("Decoded source: {}".format(parsed_source)) + if parsed_source.scheme == "" or parsed_source.scheme == "file": try: - f = open(parsed_source.path, 'r') + f = open(parsed_source.path, "r") except Exception as err: - logging.error('Could not open file {}: {}'.format(parsed_source.path, err)) + logging.error( + "Could not open file {}: {}".format(parsed_source.path, err) + ) raise self.definition = yaml.load(f, Loader=yaml.Loader) f.close() self.working_dir = pathlib.Path(parsed_source.path).parents[0] else: - logging.error('Unknown scheme "{}" in {}'.format(parsed_source.scheme, self.source)) + logging.error( + 'Unknown scheme "{}" in {}'.format(parsed_source.scheme, self.source) + ) return False return True def _validate(self): is_valid = True - if not 'metadata' in self.definition: - logging.warning('No metadata found for module {}. Generating some.'.format(self.source)) - self.definition['metadata'] = {} - self.definition['metadata']['name'] = self.source - self.definition['metadata']['description'] = 'Autogenerated metadata' + if "metadata" not in self.definition: + logging.warning( + "No metadata found for module {}. Generating some.".format(self.source) + ) + self.definition["metadata"] = {} + self.definition["metadata"]["name"] = self.source + self.definition["metadata"]["description"] = "Autogenerated metadata" else: - if not 'name' in self.definition['metadata']: - logging.warning("No name set in module {}'s metadata. Autosetting to {}".format(self.name, self.name)) - self.definition['metadata']['name'] = self.name + if "name" not in self.definition["metadata"]: + logging.warning( + "No name set in module {}'s metadata. Autosetting to {}".format( + self.name, self.name + ) + ) + self.definition["metadata"]["name"] = self.name else: - self.name = self.definition['metadata']['name'] - if not 'description' in self.definition['metadata']: - logging.warning("No description set in module {}'s metadata. Autosetting".format(self.name)) - self.definition['metadata']['description'] = 'Autogenerated' - - if not 'containerfile' in self.definition: - logging.debug("'containerfile' missing from definition yaml for module {}, using default".format(self.name)) - self.definition['containerfile'] = 'Containerfile' - elif not isinstance(self.definition['containerfile'], str): + self.name = self.definition["metadata"]["name"] + if "description" not in self.definition["metadata"]: + logging.warning( + "No description set in module {}'s metadata. Autosetting".format( + self.name + ) + ) + self.definition["metadata"]["description"] = "Autogenerated" + + if "containerfile" not in self.definition: + logging.debug( + "'containerfile' missing from definition yaml for module {}, using default".format( + self.name + ) + ) + self.definition["containerfile"] = "Containerfile" + elif not isinstance(self.definition["containerfile"], str): logging.error("'containerfile' is not a string") is_valid = False - self.containerfile = self.definition['containerfile'] + self.containerfile = self.definition["containerfile"] - if 'buildargs' in self.definition: - if not isinstance(self.definition['buildargs'],list): + if "buildargs" in self.definition: + if not isinstance(self.definition["buildargs"], list): logging.error("'buildargs' is not a list") is_valid = False else: - self.definition['buildargs'] = [] + self.definition["buildargs"] = [] - return(is_valid) + return is_valid diff --git a/modules/cloud-init/enable.yaml b/modules/cloud-init/enable.yaml index 4e87ba6..11539c2 100644 --- a/modules/cloud-init/enable.yaml +++ b/modules/cloud-init/enable.yaml @@ -4,4 +4,3 @@ metadata: description: | Enable cloud-init containerfile: enable.Containerfile - diff --git a/modules/dnf/upgrade.yaml b/modules/dnf/upgrade.yaml index 2bb7b97..8b8bd90 100644 --- a/modules/dnf/upgrade.yaml +++ b/modules/dnf/upgrade.yaml @@ -4,4 +4,3 @@ metadata: description: | Upgrade all dnf packages containerfile: upgrade.Containerfile - diff --git a/requirements.txt b/requirements.txt index edb1b59..6c0ab97 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ -pyyaml configargparse +pyyaml