diff --git a/dist-git-client/dist_git_client.py b/dist-git-client/dist_git_client.py index bcfb7ca..ea168c6 100644 --- a/dist-git-client/dist_git_client.py +++ b/dist-git-client/dist_git_client.py @@ -12,6 +12,7 @@ import shutil import subprocess import sys +import tempfile from urllib.parse import urlparse try: @@ -217,11 +218,12 @@ def get_spec(distgit_config): return specfile -def sources(args, config): +def parse_sources(args, config): """ Locate the sources, and download them from the appropriate dist-git lookaside cache. """ + # pylint: disable=too-many-locals parsed_url, distgit_config = get_distgit_config(config, args.forked_from) namespace = parsed_url.path.strip('/').split('/') # drop the last {name}.git part @@ -244,7 +246,7 @@ def sources(args, config): return logging.info("Reading sources specification file: %s", sources_file) - with open(sources_file, 'r') as sfd: + with open(sources_file, 'r', encoding="utf8") as sfd: while True: line = sfd.readline() if not line: @@ -273,7 +275,7 @@ def sources(args, config): filename = os.path.basename(source_spec[1]) kwargs["filename"] = filename.strip('()') else: - msg = "Weird sources line: {0}".format(line) + msg = f"Weird sources line: {line}" raise RuntimeError(msg) url_file = '/'.join([ @@ -281,7 +283,15 @@ def sources(args, config): distgit_config["lookaside_uri_pattern"].format(**kwargs) ]) - download_file_and_check(url_file, kwargs, distgit_config) + yield url_file, kwargs, distgit_config + + +def sources(args, config): + """ + Go through all the dist-git files, and download them. + """ + for url, metadata, distgit_config in parse_sources(args, config): + download_file_and_check(url, metadata, distgit_config) def handle_autospec(spec_abspath, spec_basename, args): @@ -308,22 +318,26 @@ def handle_autospec(spec_abspath, spec_basename, args): return result +def _prepare_outputdir(args): + mkdir_p(args.outputdir) + + +def _get_preprocessed_specfile(distgit_config, args): + specs = os.path.join(os.getcwd(), distgit_config["specs"]) + spec = get_spec(distgit_config) + spec_abspath = os.path.join(specs, spec) + return handle_autospec(spec_abspath, spec, args) + + def srpm(args, config): """ Using the appropriate dist-git configuration, generate source RPM file. This requires running 'def sources()' first. """ _, distgit_config = get_distgit_config(config, args.forked_from) - - cwd = os.getcwd() - sources_dir = os.path.join(cwd, distgit_config["sources"]) - specs = os.path.join(cwd, distgit_config["specs"]) - spec = get_spec(distgit_config) - - mkdir_p(args.outputdir) - - spec_abspath = os.path.join(specs, spec) - spec_abspath = handle_autospec(spec_abspath, spec, args) + _prepare_outputdir(args) + spec_abspath = _get_preprocessed_specfile(distgit_config, args) + sources_dir = os.path.join(os.getcwd(), distgit_config["sources"]) if args.mock_chroot: command = [ @@ -348,6 +362,59 @@ def srpm(args, config): check_call(command) +def _list_git_sources(subdir): + directory = os.path.normpath(subdir) + "/" + cmd = ["git", "ls-tree", "HEAD", directory] + git_output = subprocess.check_output(cmd, encoding="utf-8") + for line in git_output.splitlines(): + _, objtype, _, name = line.split() + if objtype == "blob": + yield name + + +def snapshot(args, config): + """ + Generate a tarball snapshot for given checkout. + """ + _, distgit_instance_config = get_distgit_config(config, args.forked_from) + spec = _get_preprocessed_specfile(distgit_instance_config, args) + + with tempfile.TemporaryDirectory(prefix="copr-test-walk") as workdir: + tar_name = os.path.basename(spec)[:-5] + tar_dir = os.path.join(workdir, tar_name) + os.makedirs(tar_dir) + + # We insert every file top-level, while git in general may store them in + # the sources subdir. Bomb early if double should be added. + basenames_added = set() + def _prepare_file(file): + basename = os.path.basename(file) + if basename in basenames_added: + raise RuntimeError(f"{basename} added twice") + basenames_added.add(basename) + shutil.copy(file, tar_dir) + + _prepare_file(spec) + + for _, metadata, _ in parse_sources(args, config): + print("..") + _prepare_file(metadata["filename"]) + + for file in _list_git_sources(distgit_instance_config["sources"]): + if os.path.basename(file) == os.path.basename(spec): + # spec file added separately + continue + # Shall we filter-out files that are not related to RPM + # building? + _prepare_file(file) + + tarball = os.path.join(args.outputdir, tar_name + ".tar.gz") + logging.info("Generating tarball %s", tarball) + tar_cmd = ["tar", "--owner=root", "--group=root", "--format=pax", + "-czf", tarball, "-C", workdir, tar_name] + subprocess.check_call(tar_cmd) + + def clone(args, config): """ Automatically clone a package from a given DistGit instance @@ -454,6 +521,15 @@ def _get_argparser(): "'@copr/copr-dev/copr-cli'."), ) + snapshot_parser = subparsers.add_parser( + "snapshot", + help=("Generate a snapshot for given dist-git checkout"), + ) + snapshot_parser.add_argument( + "--outputdir", + default="/tmp", + help="Where to store the resulting tarball") + return parser @@ -494,6 +570,8 @@ def main(): srpm(args, config) elif args.action == "clone": clone(args, config) + elif args.action == "snapshot": + snapshot(args, config) else: sources(args, config) except RuntimeError as err: