-
Notifications
You must be signed in to change notification settings - Fork 1
/
update_package_cache.py
198 lines (162 loc) · 7.25 KB
/
update_package_cache.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
# update_package_cache.py
# osc plugin to create a link mirror of an OBS project
#
# Written by Aaron Bockover <[email protected]>
#
# Copyright 2006-2009 Novell Inc. All rights reserved.
# This program is free software; it may be used, copied, modified
# and distributed under the terms of the GNU General Public Licence,
# either version 2, or (at your option) any later version
#
@cmdln.option ('-d', '--destdir', metavar='DIR',
help='Destination directory. Default is the package cache dir as '
'defined in osc config or overridden by OSC_PACKAGECACHEDIR. '
'if the package cache is used, additionaly directory structure '
'will be created so the packages can be used for building.')
@cmdln.option ('-e', '--existing', action='store_true',
help='Only download updated versions of packages that have '
'been previously downloaded.')
@cmdln.option ('-p', '--prune', action='store_true',
help='Remove older versions of packages that have '
'been previously downloaded.')
@cmdln.option ('-y', '--dry-run', action='store_true',
help='Do not actually download or prune any files, just report '
'what would happen. Required --v|--verbose.')
@cmdln.option ('-v', '--verbose', action='store_true',
help='Show extra progress and processing details.')
def do_update_package_cache (self, subcmd, opts, project, repository, architecture):
"""${cmd_name}: Fetch all published binaries from a project
${cmd_usage}
${cmd_option_list}
"""
import glob
import fcntl
if not opts.destdir:
opts.destdir = os.path.join (self.get_package_cache_dir (),
project, repository, architecture)
if not os.path.isdir (opts.destdir):
if opts.verbose:
print 'Creating cache directory: %s' % opts.destdir
if not opts.dry_run:
os.makedirs (opts.destdir, 0755)
elif opts.verbose:
print 'Using cache directory: %s' % opts.destdir
try:
lock_file = os.path.join (opts.destdir, 'osc-update-package-cache.lock')
lock_handle = open (lock_file, 'w+')
fcntl.lockf (lock_handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
except:
sys.exit ('Unable to acquire lock. An update may already be in progress for this tree.')
apiurl = conf.config['apiurl']
self.prune_stats = { 'files': [], 'bytes': 0 }
if opts.verbose:
print 'Fetching package list for %s/%s/%s/%s' % (apiurl, project,
repository, architecture)
binaries = get_binarylist (apiurl, project,
repository, architecture, verbose = True)
if binaries == []:
sys.exit ('No binaries found.')
finished_bytes = 0
finished_binaries = 0
total_binaries = len (binaries)
total_bytes = 0
for binary in binaries:
total_bytes += binary.size
for binary in binaries:
finished_binaries += 1
finished_bytes += binary.size
if opts.verbose:
print '==> %d / %d, %d MB / %d MB, %d%%'\
% (finished_binaries,
total_binaries,
finished_bytes / 1024 / 1024,
total_bytes / 1024 / 1024,
(float (finished_bytes) / float (total_bytes)) * 100.0)
print ' Checking for suitable %s' % binary.name
# Look for existing RPMs with same name, modification time, and size.
# We have to do this because OBS doesn't give us properly named RPM
# files for unpublished RPM binaries. After fetching the binary we will
# rename it properly so it can be picked up by osc build.
package, ext = os.path.splitext (binary.name)
local_binaries = glob.glob (os.path.join (opts.destdir, package) + '*')
skip_binary = None
for existing in local_binaries:
st = os.stat (existing)
if st.st_mtime == binary.mtime and st.st_size == binary.size:
if self.get_rpm_package_name (existing) == package:
skip_binary = existing
break
if skip_binary:
if opts.verbose:
print ' Skipping, found %s' % os.path.basename (skip_binary)
if opts.prune:
self.prune (package, local_binaries, skip_binary, opts.verbose, opts.dry_run)
continue
elif opts.existing and local_binaries == []:
if opts.verbose:
print ' Skipping, no older version found locally'
continue
# Download the binary
target_filename = os.path.join (opts.destdir, binary.name)
if opts.verbose:
print ' ',
else:
print '%d/%d %dMB: ' % (finished_binaries, total_binaries,
binary.size / 1024 / 1024),
if opts.dry_run:
print 'Skipping download: dry run'
continue
get_binary_file (apiurl, project, repository, architecture,
binary.name,
target_filename = target_filename,
target_mtime = binary.mtime,
progress_meter = True)
# Update the filename based on RPM data
new_target_filename = self.get_rpm_filename (target_filename)
if opts.verbose:
print ' Renaming %s to %s' % \
(binary.name, os.path.basename (new_target_filename))
os.rename (target_filename, new_target_filename)
if opts.prune and not local_binaries == []:
self.prune (package, local_binaries, new_target_filename, opts.verbose, False)
if not self.prune_stats['files'] == []:
print '%d obsolete RPMs removed (%d MB)' % (len (self.prune_stats['files']),
self.prune_stats['bytes'] / 1024 / 1024)
lock_handle.close ()
os.unlink (lock_file)
def prune (self, package_name, local_binaries, keep_binary, verbose, dry_run):
for binary in local_binaries[:]:
if not binary == keep_binary and self.get_rpm_package_name (binary) == package_name:
self.prune_stats['files'].append (binary)
self.prune_stats['bytes'] += os.stat (binary).st_size
if verbose:
print ' Removed obsolete %s' % os.path.basename (binary)
if not dry_run:
os.remove (binary)
def get_rpm_package_name (self, filename):
return data_from_rpm (filename, 'Name:')['Name:']
def get_rpm_filename (self, filename):
# Adapted from osc's fetch.py
rpm_data = data_from_rpm (filename,
'Name:', 'Version:', 'Release:', 'Arch:',
'SourceRPM:', 'NoSource:', 'NoPatch:')
if not rpm_data:
return filename
arch = rpm_data['Arch:']
if not rpm_data['SourceRPM:']:
if rpm_data['NoSource:'] or rpm_data['NoPatch:']:
arch = 'nosrc'
else:
arch = 'src'
canonname = '%s-%s-%s.%s.rpm' % (rpm_data['Name:'],
rpm_data['Version:'], rpm_data['Release:'], arch)
head, tail = os.path.split (filename)
return os.path.join (head, canonname)
def get_package_cache_dir (self):
val = os.getenv ('OSC_PACKAGECACHEDIR')
if val:
if conf.config.has_key ('packagecachedir'):
print 'Overriding config value for packagecachedir=\'%s\' '
'with \'%s\'' % (conf.config[var], val)
conf.config['packagecachedir'] = val
return conf.config['packagecachedir']