diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c09ee4e --- /dev/null +++ b/.gitignore @@ -0,0 +1,13 @@ +*.py[co] +*.swp +*~ + +*.egg +*.egg-info +.eggs +dist +build +eggs +parts +sdist +develop-eggs diff --git a/README.md b/README.md new file mode 100644 index 0000000..be02126 --- /dev/null +++ b/README.md @@ -0,0 +1,44 @@ +Description +=========== + +Release-helper is a python tool that helps you manage large number of +github repositories by controlling milestones and labels, +and creating a overiew of the release process. + +Configfile +========== + +Username / token are best stored in minimal config file, +and used together with project config file. +This allows you to share the project configfile, +without compromising authentication details. + +Example: + release-helper.py --configs=.secret/credentials.cfg,project.cfg --collect + +Commands +======== + * collect: gather the data from github, creates JSON file (most other steps will use this JSON file) + * render: generate the release notes overview and to discuss + * notes: generate the release notes (there's a *notestemplate* option to select a different template) + Example: + release-helper.py --configs=.secret/credentials.cfg,quattor.cfg --notes --milestone 16.6 + * milestones: configure milestones due date on release data (or generated if milestone is not in release data) + * TODO: Missing the ability to open/close milestones + * labels: configure (add or update; no remove) the labels + * bump: advanced: bump/shift the milestones by `--months` number of months (default 2). + Generates and prints the release section data with updated dates. + +Local usage +=========== +You cannot use `file:///path/to/project/index.html` because some of the css and javascript url use +protocol relative urls, and would require that you install those also in the pooject dir. + +* generate the required html and json using and start webserver using `--render --web` option +* got to url `http://localhost:8000` (and `http://localhost:8000/to_discuss`) + +Example release page +==================== + +Example: + [quattor release overview]:(http://quattor.org/release) diff --git a/bin/release-helper.py b/bin/release-helper.py new file mode 100644 index 0000000..a90e821 --- /dev/null +++ b/bin/release-helper.py @@ -0,0 +1,124 @@ +#!/usr/bin/python +# encoding: utf8 + +import argparse +import BaseHTTPServer +import logging +import os +import SimpleHTTPServer + +from release_helper.config import make_config, get_project, get_repos, get_releases, get_output_filenames, get_labels +from release_helper.collect import collect +from release_helper.labels import configure_labels +from release_helper.milestone import milestones_from_releases, configure_milestones, bump +from release_helper.render import make_html, make_notes + + +def get_args(): + """ + Build up and parse commandline + """ + parser = argparse.ArgumentParser() + parser.add_argument("-d", "--debug", help="set debug loglevel", action="store_true") + parser.add_argument("-C", "--configs", help="comma-separated list of config files") + + parser.add_argument("-c", "--collect", help="collect github repository information", action="store_true") + parser.add_argument("-r", "--render", help="render html release overview and discuss pages", action="store_true") + + parser.add_argument("-n", "--notes", help="Generate the releasenotes", action="store_true") + parser.add_argument("--milestone", help="Milestone to use (for releasenotes)") + parser.add_argument("--notestemplate", help="TT template to use for the releasenotes", default='quattor_releasenotes') + + parser.add_argument("-m", '--milestones', help='Configure milestones (from configured releases)', action='store_true') + + parser.add_argument('-l', '--labels', help='Configure labels', action='store_true') + + parser.add_argument("-w", "--web", help="start simple webserver", action="store_true") + parser.add_argument("-p", "--port", help="webserver port", default=8000) + + parser.add_argument('--bump', help='Bump/shift milestones number of months, print new release section', action='store_true') + parser.add_argument('--months', help='Number of months to bump/shift milestones', action='store_true', default=2) + + args = parser.parse_args() + return args + + +def main(): + logger = logging.getLogger() + lvl = logging.INFO + + args = get_args() + if args.debug: + lvl = logging.DEBUG + + logger.setLevel(lvl) + + cfgs = None + if args.configs: + cfgs = args.configs.split(',') + + use_github = any([getattr(args, x) for x in ('collect', 'milestones', 'bump', 'labels')]) + + # Do not unneccesiraly gather GH data + make_config(cfgs=cfgs, use_github=use_github) + + repos = get_repos() + project = get_project() + releases = get_releases() + + basedir, filenames = get_output_filenames() + + if not os.path.isdir(basedir): + os.mkdir(basedir) + + logging.debug('Release helper start') + + if args.milestones: + logging.info('Configure milestones') + milestones = milestones_from_releases(releases) + for repo in repos: + configure_milestones(repo, milestones) + elif args.bump: + logging.info('Bump milestones with %s months', args.months) + u_release_data = {} + for repo in repos: + u_release_data.update(bump(repo, months=args.months, releases=releases)) + + # Create new release section config + txt = [] + for title, dates in sorted(u_release_data.items()): + txt.append("%s=%s" % (title, ','.join(dates))) + + print "\n".join(txt) + elif args.labels: + labels = get_labels() + logging.info('Configuring labels') + for repo in repos: + configure_labels(repo, labels) + else: + if args.collect: + logging.info('Collect') + collect(repos, filenames) + + if args.render: + logging.info('Render') + make_html(project, releases, filenames) + + if args.notes: + logging.info("Release notes using milestone %s and template %s", args.milestone, args.notestemplate) + make_notes(project, args.milestone, args.notestemplate, filenames) + + logging.debug('Release helper end') + + if args.web: + os.chdir(basedir) + logging.debug('Starting webserver on port %s from basedir %s, running forever', args.port, basedir) + server_address = ('', int(args.port)) + + HandlerClass = SimpleHTTPServer.SimpleHTTPRequestHandler + HandlerClass.protocol_version = "HTTP/1.0" + httpd = BaseHTTPServer.HTTPServer(server_address, HandlerClass) + httpd.serve_forever() + +if __name__ == '__main__': + main() diff --git a/burndown.html b/burndown.html deleted file mode 100644 index 1169393..0000000 --- a/burndown.html +++ /dev/null @@ -1,124 +0,0 @@ - - - - - Quattor Backlog - - - - - - - - -
- - - - -
- - - -
- - diff --git a/github_release_config.py.example b/data/github_release_config.py.example similarity index 100% rename from github_release_config.py.example rename to data/github_release_config.py.example diff --git a/data/quattor.cfg b/data/quattor.cfg new file mode 100644 index 0000000..4b0996a --- /dev/null +++ b/data/quattor.cfg @@ -0,0 +1,26 @@ +[main] +project=quattor + +[releases] +14.8=2014-07-01,2014-08-25,2014-09-01 +14.10=2014-09-01,2014-10-27,2014-11-01 +15.2=2014-11-14,2015-03-07,2015-03-20 +15.4=2015-03-23,2015-05-01,2015-05-08 +15.8=2015-06-03,2015-09-14,2015-09-18 +15.12=2015-10-29,2016-12-07,2015-12-17 +16.2=2016-01-11,2016-02-15,2016-02-19 +16.6=2016-02-29,2016-06-20,2016-07-01 + +[labels] +bug=ef2929 +duplicate=d3d7cf +enhancement=83afdf +invalid=555753 +question=75507b +wontfix=888a85 +backwards incompatible=e46800 +discuss at workshop=fce94f + +[github] +organisation=quattor +white=CAF,CCM diff --git a/data/release_helper.cfg b/data/release_helper.cfg new file mode 100644 index 0000000..a04d126 --- /dev/null +++ b/data/release_helper.cfg @@ -0,0 +1,18 @@ +[main] +github=github,enterprise + +[labels] +bug=ef2929 + +[github] +username=example1 +token=1234567890 +organisation=pubteam +white=abc + +[enterprise] +api=https://enterprise.example.com/some/subtree/v3/ +username=localuser +token=abcdef123456 +organisation=myteam +black=secret diff --git a/github_labels_configure.py b/github_labels_configure.py deleted file mode 100755 index 4a4e51e..0000000 --- a/github_labels_configure.py +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/python - -from github import Github -from json import dump -from datetime import date, datetime, timedelta -from calendar import monthrange -from itertools import ifilter - -from github_release_config import * - -g = Github(USERNAME, OAUTH_TOKEN) -q = g.get_user(ORGANISATION) - -# Requires the dictionary STANDARD_LABELS = { label: hexcolor } to be defined in github_release_config - -# Collect data -for r in REPOS: - print r - repo = q.get_repo(r) - labels = repo.get_labels() - existing_labels = {} - - for label in labels: - existing_labels[label.name] = label - - for label, color in STANDARD_LABELS.iteritems(): - if label in existing_labels: - if existing_labels[label].color != color: - existing_labels[label].edit(label, color) - print " Updated %s : %s" % (label, color) - else: - repo.create_label(label, color) - print " Added %s : %s" % (label, color) - - print # Seperator diff --git a/github_milestones_bump.py b/github_milestones_bump.py deleted file mode 100755 index 30853b6..0000000 --- a/github_milestones_bump.py +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/python - -from github import Github -from json import dump -from datetime import date, datetime, timedelta -from calendar import monthrange -from itertools import ifilter - -from github_release_config import * - -g = Github(USERNAME, OAUTH_TOKEN) -q = g.get_user(ORGANISATION) - -data = {} - -today = date.today() -month = today.month - -def add_months(sourcedate, months): - month = sourcedate.month - 1 + months - year = sourcedate.year + month / 12 - month = month % 12 + 1 - day = min(sourcedate.day, monthrange(year,month)[1]) - return date(year, month, day) - -# Ensure we have an even numbered month! -if month % 2 > 0: - month += 1 - -today = date(today.year, month, 1) - -m_future = [] -for m in range(1, 4): - n = add_months(today, m*2) - m_future.append((n.year, n.month)) -del today - -# Collect data -for r in REPOS: - print r - repo = q.get_repo(r) - data[r] = [] - m_open = repo.get_milestones(state='open', sort='due_date', direction='desc') - m_all = [] - - print ' Open' - for m in m_open: - print ' %s' % m.title - m_all.append(m.title) - release = m.title.split('.') - if len(release) == 2: - year = int('20' + release[0]) - month = int(release[1]) - day = monthrange(year, month)[-1] - - due_original = date(year, month, day) - due_updated = add_months(due_original, 2) - title_updated = '{0:%y}.{0.month}'.format(due_updated) - - print ' New Title %s' % title_updated - print ' Was Due %s' % due_original - print ' Now Due %s' % due_updated - m.edit(title_updated, due_on=due_updated) # DANGER! - print ' Updated' - else: - print ' Point release, will not modify' - - print # Seperator - - -with open('/tmp/github-milestones.json', 'w') as f: - dump(data, f) diff --git a/github_milestones_configure.py b/github_milestones_configure.py deleted file mode 100755 index 74fc5a4..0000000 --- a/github_milestones_configure.py +++ /dev/null @@ -1,100 +0,0 @@ -#!/usr/bin/python - -from github import Github -from json import dump -from datetime import date, datetime, timedelta -from calendar import monthrange -from itertools import ifilter -import argparse - -from github_release_config import * - -g = Github(USERNAME, OAUTH_TOKEN) -q = g.get_user(ORGANISATION) - -data = {} - -parser = argparse.ArgumentParser() -parser.add_argument('--create', metavar='MILESTONE', type=str) -parser.add_argument('--close', metavar='MILESTONE', type=str) -args = parser.parse_args() - -#import sys -#sys.exit() - -today = date.today() -month = today.month - -def add_months(sourcedate, months): - month = sourcedate.month - 1 + months - year = sourcedate.year + month / 12 - month = month % 12 + 1 - day = min(sourcedate.day, monthrange(year,month)[1]) - return date(year, month, day) - -# Ensure we have an even numbered month! -if month % 2 > 0: - month += 1 - -today = date(today.year, month, 1) - -m_future = [] -for m in range(1, 4): - n = add_months(today, m*2) - m_future.append((n.year, n.month)) -del today - -# Collect data -for r in REPOS: - print r - repo = q.get_repo(r) - data[r] = [] - m_open = repo.get_milestones(state='open') - m_closed = repo.get_milestones(state='closed') - m_all = [] - - print ' Closed' - for m in m_closed: - print ' %s' % m.title - m_all.append(m.title) - if m.due_on >= datetime.now() - timedelta(1): - m.edit(m.title, state='open') - print ' Opening' - - print ' Open' - for m in m_open: - print ' %s' % m.title - m_all.append(m.title) - release = m.title.split('.') - if len(release) == 2: - if m.title == '16.4': - m.edit(m.title, due_on=date(2016, 5, 20)) - print ' Pushed back due date for 16.4' - - if m.due_on: - if m.due_on < datetime.now() - timedelta(1): - m.edit(m.title, state='closed') - print ' Closing' - else: - year = int('20' + release[0]) - month = int(release[1]) - day = monthrange(year, month)[-1] - m.edit(m.title, due_on=date(year, month, day)) - print ' Updated due date' - - else: - print ' Point release, will not modify' - - #print ' Checking future milestones' - #for year, month in m_future: - # n = "%d.%d" % (year-2000, month) - # if n not in m_all: - # print ' %s not found - creating' % n - # day = monthrange(year, month)[-1] - # repo.create_milestone(title=n, due_on=date(year, month, day)) - - print # Seperator - - -with open('/tmp/github-milestones.json', 'w') as f: - dump(data, f) diff --git a/github_pulls_collect.py b/github_pulls_collect.py deleted file mode 100755 index b56183c..0000000 --- a/github_pulls_collect.py +++ /dev/null @@ -1,127 +0,0 @@ -#!/usr/bin/python -# encoding: utf8 - -from github import Github, GithubException -from json import dump -from sys import stdout -from collections import namedtuple -from itertools import chain -from ssl import SSLError -from datetime import datetime, timedelta -import socket -from re import compile, IGNORECASE - -RE_DEPENDS = compile(r'((?:depends|based) on|requires)\s+(?P[\w/-]*)#(?P\d+)', IGNORECASE) -RE_FIXES = compile(r'(close[sd]?|fix(?:e[sd])?|resolve[sd]?)', IGNORECASE) - -from github_release_config import * - -g = Github(USERNAME, OAUTH_TOKEN) -g = g.get_user(ORGANISATION) - -data = {} -relationships = [] - -MockMilestone = namedtuple('Milestone', ['title', 'open_issues', 'closed_issues', 'due_on']) -backlog = MockMilestone(title='Backlog', open_issues=0, closed_issues=0, due_on=None) - -# Collect data -for repo_name in REPOS: - repo = g.get_repo(repo_name) - stdout.write(" %-32s" % (repo.name)) - - retries = 3 - while retries: - try: - milestones = repo.get_milestones(state='open') - - for milestone in chain(milestones, [backlog]): - if milestone.title not in data: - data[milestone.title] = {} - - if repo.name not in data[milestone.title]: - data[milestone.title][repo.name] = {'things': [], 'closed': 0, 'open': 0, 'due': None} - - data[milestone.title][repo.name]['open'] += int(milestone.open_issues) - data[milestone.title][repo.name]['closed'] += int(milestone.closed_issues) - - if milestone.due_on: - data[milestone.title][repo.name]['due'] = milestone.due_on.isoformat() - - stdout.write('█') - stdout.flush() - - break - - except (SSLError, socket.error) as e: - stdout.write('R') - retries -= 1 - - try: - retries = 3 - while retries: - try: - # We care about all things that are assigned to a milestone, or things from the last 60 days that are not assigned to a milestone - things_all_milestones = repo.get_issues(state='all', milestone='*') - things_backlog = repo.get_issues(milestone='none', since=datetime.now() - timedelta(days=60)) - things = chain(things_all_milestones, things_backlog) - - for t in things: - stdout.write('▒') - stdout.flush() - milestone_name = 'Backlog' - if t.milestone: - milestone_name = t.milestone.title - - if milestone_name in data: # Skip any issues belonging to milestones that have been closed already - if repo.name in data[milestone_name]: - this_thing = { - 'number' : t.number, - 'url' : t.html_url, - 'title' : t.title, - 'user' : t.user.login, - 'assignee' : t.assignee.login if t.assignee != None else None, - 'created' : t.created_at.isoformat(), - 'updated' : t.updated_at.isoformat(), - 'state' : t.state, - 'comment_count' : t.comments, - 'labels' : [], - } - - this_thing['type'] = 'issue' - if t.pull_request: - this_thing['type'] = 'pull-request' - this_thing['labels'] = [l.name for l in t.labels] - pr = repo.get_pull(t.number) - - this_thing['labels'] = [l.name for l in t.labels] - - if t.closed_at: - this_thing['closed'] = t.closed_at.isoformat() - - data[milestone_name][repo.name]['things'].append(this_thing) - - dependencies = RE_DEPENDS.search(t.body) - if dependencies: - dep_repo = dependencies.group('repository') - if not dep_repo: - dep_repo = repo.name - relationships.append(('%s/%s' % (repo.name, t.number), 'requires', '%s/%s' % (repo.name, dependencies.group('number')))) - else: - print "\nWARNING: Dropped thing %d (%s) from repo %s (missing milestone?)" % (t.number, t.title ,repo.name) - break - - except SSLError: - stdout.write('R') - retries -= 1 - - except GithubException as e: - print 'ERROR: %(message)s' % e - - print - -with open('/tmp/github-pulls.json', 'w') as f: - dump(data, f, indent=4) - -with open('/tmp/github-pulls-relationships.json', 'w') as f: - dump(relationships, f, indent=4) diff --git a/github_pulls_render.py b/github_pulls_render.py deleted file mode 100755 index 5410f69..0000000 --- a/github_pulls_render.py +++ /dev/null @@ -1,216 +0,0 @@ -#!/usr/bin/python -# encoding: utf8 - -HEADER = ''' - - - - Quattor Backlog - - - - - - - - - - -''' - -FOOTER = ''' - - - - - - - -''' - -OUTFILE = 'index.html' - - -from json import load -from cgi import escape -from datetime import datetime - -previous_releases = [ -# '14.8', -# '14.10', -# '15.2', -# '15.4', -# '15.8', -# '15.12', -] - -# Render data -with open('/tmp/github-pulls.json') as f_in: - data = load(f_in) - - # Hacky numerical sort for our release numerbing scheme - milestones = data.keys() - milestones = [[int(i) for i in m.split('.')] for m in milestones if m != 'Backlog'] - milestones.sort() - milestones = previous_releases + [u'.'.join(map(str,m)) for m in milestones] + ['Backlog'] - print milestones - - - with open(OUTFILE, 'w') as f: - f.write(HEADER) - - f.write('
\n') - f.write('''\n''') - - f.write('
    \n') - for milestone in milestones: - tab_class = '' - if milestone in previous_releases: - tab_class = 'thing-closed' - tab_class = ' class="%s"' % tab_class - f.write('
  • %s
    • \n' % (milestone.replace('.', '-'), tab_class, milestone)) - f.write('
\n') - - f.write('
\n') - f.write('
\n') - f.write('
\n') - f.write('
\n') - f.write('

Select a release to view backlog

\n') - f.write('
\n') - f.write('
\n') - f.write('
\n') - - f.write('
\n'); - - for milestone in milestones: - i_progress = 0 - i_closed = 0 - i_open = 0 - m_due = 'never' - - print " %s" % (milestone) - style = 'default' - if milestone == 'Backlog': - style = 'danger' - - repos = [] - if milestone in data: - repos = data[milestone].keys() - repos.sort() - - f.write('
\n' % (style, milestone.replace('.', '-'))) - f.write('\n') - - for repo in repos: - i_closed += data[milestone][repo]['closed'] - i_open += data[milestone][repo]['open'] - - print " " + repo - r = data[milestone][repo] - if r['due']: - m_due = r['due'] - - - things = r['things'] - if things: - f.write('\n') - f.write('\n' % (repo)) - f.write('') - f.write('\n') - f.write('
%s') - f.write('
    \n') - things = sorted(things, key=lambda k: k['title'], reverse=True) - for t in things: -# if not (milestone == 'Backlog' and t['state'] == 'closed'): - t['icon'] = 'issue-opened' - - if t['type'] == 'issue' and t['state'] == 'closed': - t['icon'] = 'issue-closed' - elif t['type'] == 'pull-request' and t['state'] != 'merged': - t['icon'] = 'git-pull-request' - elif t['type'] == 'pull-request' and t['state'] == 'merged': - t['icon'] = 'git-merge' - - f.write('
  • ' % t) - f.write(' ' % t) - f.write('%(title)s ' % t) - if t['comment_count']: - f.write(' %(comment_count)s ' % t) - - f.write('
    ') - f.write('Created by %(user)s %(created)s' % t) - if t['assignee']: - f.write(', Assigned to %(assignee)s' % t) - f.write(', Updated %(updated)s ' % t) - f.write('
    ') - - f.write('
    • \n' % t) - f.write('
\n') - f.write('
\n') - - if i_closed > 0 or i_open > 0: - i_progress = i_closed * 100 / (i_open + i_closed) # Do this as integer maths - - f.write( - '''
- Progress: (%(closed)d closed, %(open)d open, due %(due)s) -
-
%(progress)d%%
-
-
''' % {'progress': i_progress, 'open': i_open, 'closed': i_closed, 'due': m_due} - ) - - f.write('
\n') - - f.write('
\n') - f.write('

Page generated %s

\n' % datetime.utcnow().replace(microsecond=0).isoformat(' ')) - f.write('
\n') - - f.write(FOOTER) diff --git a/github_pulls_render_burndown.py b/github_pulls_render_burndown.py deleted file mode 100755 index fb0f74e..0000000 --- a/github_pulls_render_burndown.py +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/python -# encoding: utf8 - -from json import load, dump -from cgi import escape -from datetime import datetime -import sys - -# Render data -with open('/tmp/github-pulls.json') as f_in: - data = load(f_in) - - # Hacky numerical sort for our release numbering scheme - milestones = data.keys() - milestones = [[int(i) for i in m.split('.')] for m in milestones if m != 'Backlog'] - milestones.sort() - milestones = [u'.'.join(map(str,m)) for m in milestones if m != 'Backlog'] - - print('# Quattor Backlog') - - for milestone in milestones: - print "\n## " + milestone.title() - repos = data[milestone].keys() - to_burn = 0 - burned = [] - - for repo in repos: - sys.stdout.write('R') - things = data[milestone][repo]['things'] - if things: - for t in things: - sys.stdout.write('.') - to_burn += 1 - if 'closed' in t: - burned.append(t['closed']) - print - burned.sort() - - bdata = { - 'to_burn': to_burn, - 'closed': [], - } - for t in burned: - to_burn -= 1 - bdata['closed'].append([t, to_burn]) - - dump(bdata, open('burndown-%s.json' % milestone, 'w')) diff --git a/github_pulls_render_release_notes_markdown.py b/github_pulls_render_release_notes_markdown.py deleted file mode 100755 index 1ce838b..0000000 --- a/github_pulls_render_release_notes_markdown.py +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/python -# encoding: utf8 - -from json import load -from cgi import escape -from datetime import datetime -from web import template -from argparse import ArgumentParser - -TEMPLATE = '''$def with (milestone, notes, issues, pulls, backwards_incompatible) - ---- -layout: article -title: Quattor $milestone.0 released -category: news -author: James Adams ---- - -Packages are available from our [yum repository](http://yum.quattor.org/$milestone.0/), both the RPMs and the repository metadata are signed with [my GPG key](http://yum.quattor.org/GPG/RPM-GPG-KEY-quattor-jrha). - -As always, many thanks to everyone who contributed! We merged $pulls pull requests and resolved $issues issues. - -The next release should be NEXT.0, take a look at the [backlog](http://www.quattor.org/release/) to see what we're working on. - - -Backwards Incompatible Changes ------------------------------- - -$for repo, prs in backwards_incompatible.iteritems(): - $if prs: - ### $repo - $for id, title, labels in prs: - $if ': ' in title: - $code: - title = '**' + title.replace(': ', ':** ', 1) - * [$title](https://github.com/quattor/$repo/pull/$id) - $# Blank line to seperate sections -Changelog ---------- - -$for repo, prs in notes.iteritems(): - ### $repo - $for id, title, labels in prs: - $if ': ' in title: - $code: - title = '**' + title.replace(': ', ':** ', 1) - * [$title](https://github.com/quattor/$repo/pull/$id) - $# Blank line to seperate sections -''' - -RENDER = template.Template(TEMPLATE) - -parser = ArgumentParser(description='Generate release notes.') -parser.add_argument('milestone', metavar='M', help='Milestone associated with this release.') -args = parser.parse_args() - -notes = {} -backwards_incompatible = {} - -# Render data -with open('/tmp/github-pulls.json') as f_in: - data = load(f_in) - - milestone = args.milestone - repos = data[milestone].keys() - repos.sort() - count_issues = 0 - count_pulls = 0 - for repo in repos: - count_issues += len([t for t in data[milestone][repo]['things'] if t['type'] == 'issue']) - things = [t for t in data[milestone][repo]['things'] if t['type'] == 'pull-request'] - count_pulls += len(things) - if things: - notes[repo] = [(t['number'], t['title'], t['labels']) for t in things] - notes[repo].sort(key=lambda t: t[1]) - backwards_incompatible[repo] = [n for n in notes[repo] if 'backwards incompatible' in n[2]] - -print RENDER(milestone, notes, count_issues, count_pulls, backwards_incompatible) diff --git a/github_render_to_discuss.py b/github_render_to_discuss.py deleted file mode 100755 index 8e9693d..0000000 --- a/github_render_to_discuss.py +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/python -# encoding: utf8 -# vi:ts=4:et - -from json import load -from cgi import escape -from datetime import datetime -from web import template -from argparse import ArgumentParser - -render = template.render('templates') - -notes = {} - -# Render data -with open('/tmp/github-pulls.json') as f_in: - data = load(f_in) - - milestones = data.keys() - milestones.sort() - for milestone in milestones: - repos = data[milestone].keys() - repos.sort() - for repo in repos: - things = [t for t in data[milestone][repo]['things']] - if things: - if repo not in notes: - notes[repo] = [] - notes[repo] = notes[repo] + [t for t in things if 'discuss at workshop' in t['labels']] - - -print render.to_discuss(notes) diff --git a/go.sh b/go.sh deleted file mode 100755 index ee00ba0..0000000 --- a/go.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -echo "Collecting..." && ./github_pulls_collect.py && echo "Rendering..." && ./github_pulls_render.py && ./github_pulls_render_burndown.py && echo "Serving..." && python -m SimpleHTTPServer diff --git a/lib/release_helper/__init__.py b/lib/release_helper/__init__.py new file mode 100644 index 0000000..08d3436 --- /dev/null +++ b/lib/release_helper/__init__.py @@ -0,0 +1 @@ +# APL diff --git a/lib/release_helper/collect.py b/lib/release_helper/collect.py new file mode 100644 index 0000000..3ac751e --- /dev/null +++ b/lib/release_helper/collect.py @@ -0,0 +1,219 @@ +""" +Gather the relevant data for a repo, all values must be serialised. +""" + +import logging +import socket +import re +from datetime import datetime, timedelta +from itertools import chain +from json import dump +from ssl import SSLError +from release_helper.milestone import MST_LEGACY, MST_BACKLOG + + +RE_DEPENDS = re.compile(r'((?:depends|based)\s+on|requires)\s+(?P[\w/-]*)#(?P\d+)', re.IGNORECASE) + +TYPE_PR = 'pull-request' +TYPE_ISSUE = 'issue' + +ICON_ISSUE = 'issue-opened' +ICON_ISSUE_CLOSED = 'issue-closed' +ICON_PR_MERGED = 'git-merge' +ICON_PR = 'git-pull-request' + +BACKWARDS_INCOMPATIBLE = 'backwards_incompatible' +TO_DISCUSS = 'to_discuss' + +LABELS_BACKWARDS_INCOMPATIBLE = [BACKWARDS_INCOMPATIBLE, 'backwards incompatible'] +LABELS_TO_DISCUSS = [TO_DISCUSS, 'discuss at workshop'] + +def milestones(repo): + """ + Create milestones dict for repo + - contains all open milestones and Backlog and Legacy + """ + data = {} + + def add_milestone(name): + return data.setdefault(name, {'things': [], 'closed': 0, 'open': 0, 'due': None}) + + add_milestone(MST_BACKLOG) + add_milestone(MST_LEGACY) + + for mst in repo.get_milestones(state='open'): + mst_d = add_milestone(mst.title) + mst_d['open'] = int(mst.open_issues) + mst_d['closed'] = int(mst.closed_issues) + + if mst.due_on: + mst_d['due'] = mst.due_on.isoformat() + + return data + + +def process_issue(issue, msts, backlog_enddate): + """ + Given an issue, extract any relevant data and return as dict. + + Only return issues with a milestone in msts. + + backlog_enddate is used to determine the Backlog/Legacy milestone for unassigned issues + """ + + if issue.milestone: + milestone_name = issue.milestone.title + elif backlog_enddate is not None: + if issue.created_at > backlog_enddate: + milestone_name = MST_BACKLOG + else: + milestone_name = MST_LEGACY + + if milestone_name not in msts: + logging.debug("Ignoring issue %s (milestone %s not in list)", issue.title, milestone_name) + return {} + + this = { + 'milestone': milestone_name, + 'number' : issue.number, + 'url' : issue.html_url, + 'title' : issue.title, + 'user' : issue.user.login, + 'created' : issue.created_at.isoformat(), + 'updated' : issue.updated_at.isoformat(), + 'state' : issue.state, + 'comment_count' : issue.comments, + 'labels' : [l.name for l in issue.labels], + 'type': TYPE_PR if issue.pull_request else TYPE_ISSUE, + 'icon': ICON_ISSUE, + } + + if this['type'] == TYPE_PR: + this['icon'] = ICON_PR_MERGED if this['state'] == 'merged' else ICON_PR + elif this['type'] == TYPE_ISSUE and this['state'] == 'closed': + this['icon'] = ICON_ISSUE_CLOSED + + if issue.assignee: + this['assignee'] = issue.assignee.login + + if issue.closed_at: + this['closed'] = issue.closed_at.isoformat() + + dependencies = RE_DEPENDS.search(issue.body) + if dependencies: + this.update(dependencies.groupdict()) + + # Special labels + if any([l in LABELS_BACKWARDS_INCOMPATIBLE for l in this['labels']]): + this[BACKWARDS_INCOMPATIBLE] = True + + if any([l in LABELS_TO_DISCUSS for l in this['labels']]): + this[TO_DISCUSS] = True + + return this + + +def things(repo, msts, backlog_days=60, legacy=False): + """ + Gather all relevant/intersting "things" from the repo . + + We care about all things that are assigned to + one of the msts milestones + Backlog: things from the last backlog_days days that are not assigned to a milestone + Legacy: all the older non-assigned + + backlog_days sets the number of days in past to take into account for non-milestone isses (as Backlog milestone) + legacy: boolean, if true, also take into account all older issues (as Legacy milestone) + """ + + if backlog_days is None: + backlog_enddate = None + else: + backlog_enddate = datetime.now() - timedelta(days=backlog_days) + + + if legacy: + # Process all issues (open/close, any milestone, any date) + all_issues = repo.get_issues() + msts += [MST_BACKLOG, MST_LEGACY] + else: + mst_issues = repo.get_issues(state='all', milestone='*') + if backlog_enddate is None: + backlog_issues = [] + else: + backlog_issues = repo.get_issues(milestone='none', since=backlog_enddate) + msts.append(MST_BACKLOG) + all_issues = chain(mst_issues, backlog_issues) + + data = dict([(m, []) for m in msts]) + relationships = [] + + for issue in all_issues: + this = process_issue(issue, msts, backlog_enddate) + if this: + mst = this.pop('milestone') + data[mst].append(this) + + dep_number = this.pop('dep_number', None) + if dep_number: + dep_repo = this.pop('dep_repo') or repo.name + relationships.append(('%s/%s' % (repo.name, this['number']), 'requires', '%s/%s' % (dep_repo, dep_number))) + + return data, relationships + + +def retry(fn, args, kwargs, retries=3): + """ + Retry try/except wrapper catching SSLError and socket.error + """ + while retries: + try: + return fn(*args, **kwargs) + except (SSLError, socket.error) as e: + logging.debug("Retry %s failed (retries %s): %s", fn.__name__, retries, e) + retries -= 1 + + # This reraises with last exception + raise Exception("No more retrials left for %s (last %s)" % (fn.__name__, e)) + + +def collect_one(repo): + """ + Gather issues (inlc PRs) for repo + """ + data = retry(milestones, [repo], {}) + + things_data, relations = retry(things, [repo, data.keys()], {}) + + for mst, mst_data in data.items(): + mst_data['things'].extend(things_data[mst]) + + return data, relations + + +def collect(repos, filenames): + """ + collect github repository information + """ + all_pulls = {} + all_relations = [] + for repo in repos: + logging.debug("Collecting %s", repo.name) + + pulls, relations = collect_one(repo) + + for mst, m_pulls in pulls.items(): + all_m_pulls = all_pulls.setdefault(mst, {}) + all_m_pulls[repo.name] = m_pulls + + all_relations.extend(relations) + + pulls_fn = filenames['pulls'] + with open(pulls_fn, 'w') as f: + dump(all_pulls, f, indent=4) + logging.debug('Collect wrote pulls data in %s', pulls_fn) + + relations_fn = filenames['relations'] + with open(relations_fn, 'w') as f: + dump(all_relations, f, indent=4) + logging.debug('Collect wrote relation data in %s', relations_fn) diff --git a/lib/release_helper/config.py b/lib/release_helper/config.py new file mode 100644 index 0000000..fec3b9b --- /dev/null +++ b/lib/release_helper/config.py @@ -0,0 +1,217 @@ +""" +Retrieve configuration information +""" + +import ConfigParser +import logging +import os +import tempfile +import re +from datetime import datetime +from github import Github + +DEFAULT_CONFIG_FILES = ['/etc/release_helper.cfg', os.path.expanduser('~/.release_helper.cfg')] + +# Section names +MAIN = 'main' # is optional, github section is than default github-api section +LABELS = 'labels' +RELEASES = 'releases' # start,rcs,target (YYYY-MM-DD format) + +# main attributes + default github section name +GITHUB = 'github' # comma seperated list of section names with github-api details +PROJECT = 'project' # project name, unique name used to store e.g. the json files + +# github attrbutes +USERNAME = 'username' # username MANDATORY +TOKEN = 'token' # auth token for username MANDATORY +API = 'api' # non-default API (e.g. for github enterprise) +ORGANISATION = 'organisation' # github organisation(/user) to use to query repos MANDATORY +WHITE = 'white' # white filter on repo names (comma separated list of regex; no exact match) +BLACK = 'black' # black filter on repo names (comma separated list of regex; no exact match) + +# Config key +REPOS = 'repos' + +# Config state, set/updated by make_config +CONFIG = { + PROJECT: 'project', + REPOS: [], + LABELS: {}, + RELEASES: {}, +} + +def read_config(cfgs): + """ + Read the config from config files cfgs (or use DEFAULT_CONFIG_FILES if None) + """ + config = ConfigParser.ConfigParser() + + read_cfgs = config.read(cfgs) + if len(cfgs) != len(read_cfgs): + logging.warn("Not all cfgs %s were found: %s", cfgs, read_cfgs) + + return config + + +def make_config(cfgs=None, use_github=True): + """ + Parse the config files and populate the CONFIG dict + """ + if cfgs is None: + cfgs = DEFAULT_CONFIG_FILES + + logging.debug("Reading config from configfiles: %s", cfgs) + config = read_config(cfgs=cfgs) + + if config.has_option(MAIN, PROJECT): + CONFIG[PROJECT] = config.get(MAIN, PROJECT) + + if not use_github: + githubs = [] + elif config.has_option(MAIN, GITHUB): + githubs = map(str.strip, config.get(MAIN, GITHUB).split(',')) + else: + githubs = [GITHUB] + + total = 0 + + logging.debug("Using githubs sections %s (use_github %s)", githubs, use_github) + for gh in githubs: + if config.has_section(gh): + opts = dict(config.items(gh)) + + try: + kwargs = { + 'login_or_token': opts[USERNAME], + 'password': opts[TOKEN], + } + except KeyError: + logging.error('%s and/or %s are mandatory for each github section', USERNAME, TOKEN) + raise + + if API in opts: + kwargs['base_url'] = opts[API] + + g = Github(**kwargs) + + if ORGANISATION in opts: + q = g.get_organization(opts[ORGANISATION]) + logging.debug("%s %s in github section %s", ORGANISATION, opts[ORGANISATION], gh) + else: + logging.debug('Missing %s in github section %s, using current user %s', ORGANISATION, gh, kwargs['login_or_token']) + q = g.get_user() + + # Apply white filter + if WHITE in opts: + white = map(re.compile, map(str.strip, config.get(gh, WHITE).split(','))) + else: + white = None + + # Apply black filter + if BLACK in opts: + black = map(re.compile, map(str.strip, config.get(gh, BLACK).split(','))) + else: + black = None + + # Get all repo names + for repo in q.get_repos(): + total += 1 + if white and not any(map(re.search, white, [repo.name]*len(white))): + # if white and no match, skip + logging.debug("Skipped repo %s in github %s due to white %s", repo.name, gh, [x.pattern for x in white]) + continue + if black and any(map(re.search, black, [repo.name]*len(black))): + # if black and match skip + logging.debug("Skipped repo %s in github %s due to black %s", repo.name, gh, [x.pattern for x in black]) + continue + + logging.debug("Adding repo %s in github %s", repo.name, gh) + CONFIG[REPOS].append(repo) + else: + msg = "No github section %s found" % gh + logging.error(msg) + raise Exception(msg) + + logging.debug("Got %s repos out of %s total: %s", len(CONFIG[REPOS]), total, [x.name for x in CONFIG[REPOS]]) + + # Labels from labels section + if config.has_section(LABELS): + CONFIG[LABELS].update(dict(config.items(LABELS))) + else: + logging.warning("No %s section found" % LABELS) + + # Releases section + # milestone=start,rcs,target + # format YYYY-MM-DD + if config.has_section(RELEASES): + for mst, dates in config.items(RELEASES): + CONFIG[RELEASES][mst] = dict(zip(['start', 'rcs', 'target'], [datetime.strptime(d, '%Y-%m-%d') for d in dates.split(',')[:3]])) + else: + logging.warning("No %s section found" % RELEASES) + + # For unittesting mainly + return CONFIG + +def get_project(): + """ + Return project name + """ + project = CONFIG[PROJECT] + + logging.info("Project names %s", project) + + return project + + +def get_repos(): + """ + Return list of Repository instances + """ + repos = CONFIG[REPOS] + + logging.info("Using %s repositories: %s", len(repos), [r.name for r in repos]) + + return repos + + +def get_labels(): + """ + Return the labels map: name => hexcolor + """ + labels = CONFIG[LABELS] + + logging.info("Using %s labels: %s", len(labels), ','.join(["%s=%s" % (k,v) for k,v in labels.items()])) + + return labels + + +def get_releases(): + """ + Return the labels map: name => hexcolor + """ + releases = CONFIG[RELEASES] + + logging.info("Using %s releases: %s", len(releases), ','.join(sorted(releases.keys()))) + + return releases + + +def get_output_filenames(): + """ + Return the basedir and (abspath) output filenames (e.g. JSON, html) + """ + tmpdir = tempfile.gettempdir() + basedir = os.path.join(tmpdir, CONFIG[PROJECT]) + + names = { + 'pulls': os.path.join(basedir, 'pulls.json'), + 'relations': os.path.join(basedir, 'relations.json'), + 'index': os.path.join(basedir, 'index.html'), + 'discuss': os.path.join(basedir, 'to_discuss.html'), + 'releases': os.path.join(basedir, 'releases.json'), + 'burndown': os.path.join(basedir, 'burndown-%(milestone)s.json'), # filename template + 'javascript': basedir, # directory + 'releasenotes': os.path.join(basedir, 'releasenotes-%(milestone)s'), # filename template + } + + return basedir, names diff --git a/lib/release_helper/javascript/burndown.js b/lib/release_helper/javascript/burndown.js new file mode 100644 index 0000000..df82987 --- /dev/null +++ b/lib/release_helper/javascript/burndown.js @@ -0,0 +1,137 @@ +function burndown(release) { + $.getJSON('releases.json', function(releases) { + if (release in releases) { + start = Date.parse(releases[release]['start']); + rcs = Date.parse(releases[release]['rcs']); + target = Date.parse(releases[release]['target']); + + $.getJSON('burndown-'+release+'.json', function(mydata) { + $.each(mydata.closed, function(i, v) { + v[0] = Date.parse(v[0]); + }); + $('#burndown-container').highcharts({ + title: { + text: 'Progress toward '+release+' release' + }, + xAxis: { + type: 'datetime', + title: { + text: 'Date' + }, + plotLines: [ + { + color: '#ef2929', + value: target, + width: '1', + label: { + text: 'Target', + style: { + color: '#a40000' + } + }, + }, + { + color: '#75507b', + value: Date.now(), + width: '1', + label: { + text: 'Now', + style: { + color: '#5c3566' + } + }, + }, + ], + plotBands: [ + { + color: '#eeeeec', + from: rcs, + to: target, + label: { + text: 'RCs', + style: { + color: '#555753' + } + }, + }, + ], + }, + yAxis: { + title: { + text: 'Open Issues & Pull Requests' + }, + min: 0 + }, + tooltip: { + pointFormat: '{point.y:.0f} open' + }, + plotOptions: { + series: { + marker: { + enabled: false + } + }, + }, + series: [ + { + name: 'Ideal World', + data: [ + [start, mydata.to_burn], + [target, 0] + ], + color: '#73d216', + dashStyle: 'shortdash', + enableMouseTracking: false, + }, + { + name: 'Linear Regression', + type: 'line', + data: (function() { + start = mydata.closed[0][0]; + fit = fitData(mydata.closed); + return [ + [start, fit.y(start)], + [target, fit.y(target)], + ]; + })(), + color: '#ad7fa8', + dashStyle: 'shortdot', + visible: false, + enableMouseTracking: false, + }, + { + name: 'Prediction', + type: 'line', + data: (function() { + secondsinday = 24 * 60 * 60; + start = start / 1000; + end = target / 1000; + days = (end - start) / secondsinday; + coeff_a = mydata.to_burn / (secondsinday * (days / 30)); + coeff_b = mydata.to_burn / (secondsinday / (days / 7)) + points = []; + for (day = 0; day <= days; day++) { + y = -coeff_a * Math.pow(day, 3) + coeff_b * Math.pow(day, 2) - day + mydata.to_burn; + x = mydata.closed[0][0] + day * secondsinday * 1000; + points.push([x, y]); + } + return points; + })(), + color: '#75507b', + dashStyle: 'shortdash', + enableMouseTracking: false, + }, + { + name: 'Reality', + data: mydata.closed, + step: 'left', + color: '#3465a4', + }, + ] + }); + }); + } else { + $('#burndown-container').html('

Cannot draw burndown, no dates for '+release+'!

'); + } + }); +} diff --git a/lib/release_helper/javascript/regression.js b/lib/release_helper/javascript/regression.js new file mode 100644 index 0000000..2e651e5 --- /dev/null +++ b/lib/release_helper/javascript/regression.js @@ -0,0 +1,140 @@ +/** + * Code for regression extracted from jqplot.trendline.js + * + * Version: 1.0.0a_r701 + * + * Copyright (c) 2009-2011 Chris Leonello + * jqPlot is currently available for use in all personal or commercial projects + * under both the MIT (http://www.opensource.org/licenses/mit-license.php) and GPL + * version 2.0 (http://www.gnu.org/licenses/gpl-2.0.html) licenses. This means that you can + * choose the license that best suits your project and use it accordingly. + * + **/ + +function regression(x, y, typ) { + var type = (typ == null) ? 'linear' : typ; + var N = x.length; + var slope; + var intercept; + var SX = 0; + var SY = 0; + var SXX = 0; + var SXY = 0; + var SYY = 0; + var Y = []; + var X = []; + + if (type == 'linear') { + X = x; + Y = y; + } + else if (type == 'exp' || type == 'exponential') { + for (var i = 0; i < y.length; i++) { + // ignore points <= 0, log undefined. + if (y[i] <= 0) { + N--; + } + else { + X.push(x[i]); + Y.push(Math.log(y[i])); + } + } + } + + for (var i = 0; i < N; i++) { + SX = SX + X[i]; + SY = SY + Y[i]; + SXY = SXY + X[i] * Y[i]; + SXX = SXX + X[i] * X[i]; + SYY = SYY + Y[i] * Y[i]; + } + + slope = (N * SXY - SX * SY) / (N * SXX - SX * SX); + intercept = (SY - slope * SX) / N; + + return [slope, intercept]; +} + +function linearRegression(X, Y) { + var ret; + ret = regression(X, Y, 'linear'); + return [ret[0], ret[1]]; +} + +function expRegression(X, Y) { + var ret; + var x = X; + var y = Y; + ret = regression(x, y, 'exp'); + var base = Math.exp(ret[0]); + var coeff = Math.exp(ret[1]); + return [base, coeff]; +} + +/* + TODO: this function is quite inefficient. + Refactor it if there is problem with speed. + */ +function fitData(data, typ) { + var type = (typ == null) ? 'linear' : typ; + var ret; + var res; + var x = []; + var y = []; + var ypred = []; + + for (i = 0; i < data.length; i++) { + if (data[i] != null && Object.prototype.toString.call(data[i]) === '[object Array]') { + if (data[i] != null && data[i][0] != null && data[i][1] != null) { + x.push(data[i][0]); + y.push(data[i][1]); + } + } + else if(data[i] != null && typeof data[i] === 'number' ){//If type of X axis is category + x.push(i); + y.push(data[i]); + } + else if(data[i] != null && Object.prototype.toString.call(data[i]) === '[object Object]'){ + if (data[i] != null && data[i].x != null && data[i].y != null) { + x.push(data[i].x); + y.push(data[i].y); + } + } + } + + if (type == 'linear') { + + ret = linearRegression(x, y); + for (var i = 0; i < x.length; i++) { + res = ret[0] * x[i] + ret[1]; + ypred.push([x[i], res]); + } + + return { + data: ypred, + slope: ret[0], + intercept: ret[1], + y: function(x) { + return (this.slope * x) + this.intercept; + }, + x: function(y) { + return (y - this.intercept) / this.slope; + } + }; + } + else if (type == 'exp' || type == 'exponential') { + + ret = expRegression(x, y); + for (var i = 0; i < x.length; i++) { + res = ret[1] * Math.pow(ret[0], x[i]); + ypred.push([x[i], res]); + } + ypred.sort(); + + return { + data: ypred, + base: ret[0], + coeff: ret[1] + }; + } +} diff --git a/lib/release_helper/labels.py b/lib/release_helper/labels.py new file mode 100644 index 0000000..57aba13 --- /dev/null +++ b/lib/release_helper/labels.py @@ -0,0 +1,25 @@ +""" +Label configuration +""" + +import logging + +def configure_labels(repo, labels): + """ + Configure the labels of repository + + Labels is a mapping label name / color + """ + + existing_labels = dict([(l.name, l) for l in repo.get_labels()]) + + logging.debug("Label configure for repo %s", repo.name) + + for label, color in labels.iteritems(): + if label in existing_labels: + if existing_labels[label].color != color: + existing_labels[label].edit(label, color) + logging.debug(" Updated %s : %s", label, color) + else: + repo.create_label(label, color) + logging.debug(" Added %s : %s", label, color) diff --git a/lib/release_helper/milestone.py b/lib/release_helper/milestone.py new file mode 100644 index 0000000..7c18a79 --- /dev/null +++ b/lib/release_helper/milestone.py @@ -0,0 +1,314 @@ +""" +Configure milestones +""" +import logging +from datetime import date, datetime, timedelta +from calendar import monthrange + + +MST_BACKLOG = 'Backlog' +MST_LEGACY = 'Legacy' + + +class MileStone(object): + def __init__(self, milestone=None, year_month=None, title=None, release=None): + """ + Create a milestone instance from one-of + milestone: another milestone instance + year_month: a tuple (year, month) + title: a title (i.e. a string with title format) + + release is a dictionary with release dates (from CONFIG[RELEASES]) + """ + self.year = None + self.month = None + self.point = None + + self.milestone = None + + self.release = release + + if milestone: + self.milestone = milestone + self.from_title(milestone.title) + elif year_month: + self.from_year_month(year_month) + elif title: + self.from_title(title) + + def from_year_month(self, year_month): + self.year = year_month[0] + self.month = year_month[1] + + def from_title(self, title): + try: + parts = map(int, title.split('.')) + except ValueError as e: + raise("Failed to convert title %s to milestone: %s" % (title, e)) + + self.year = 2000 + parts[0] + self.month = parts[1] + if len(parts) >= 3: + self.point = parts[2] + + def edit(self, *args, **kwargs): + """Edit this milestone""" + if self.milestone is None: + raise('MileStone: cannot edit without milstone attribute') + + # Insert title + args = [self.title()] + list(args) + + self.milestone.edit(*args, **kwargs) + + def set_state(self): + """Open / close milestone""" + now = datetime.now() - timedelta(1) + due_on = self.milestone.due_on + + if due_on >= now: + state = 'open' + msg = 'opening' + else: + state = 'closed' + msg = 'closing' + + self.edit(state=state) + logging.info(" %s milestone %s", msg, self) + + def title(self): + """Make the title""" + if self.year is None: + raise('MileStone: no year set') + if self.month is None: + raise('MileStone: no month set') + txt = '%02d.%02d' % (self.year - 2000, self.month) + if self.point is not None: + txt += ".%d" % self.point + return txt + + def create(self, repo): + """ + Add new milestone to repo and set new instance as milestone attribute + """ + + logging.info('Creating milestone %s in repo %s', self, repo.name) + # Do not pass due_on here, see https://github.com/PyGithub/PyGithub/issues/396 + milestone = repo.create_milestone(title=self.title()) + if self.milestone: + logging.debug("Replacing current milestone %s with newly created one %s", self.milestone, milestone) + self.milestone = milestone + self.set_due_date() + + def get_due(self, from_title=None): + """ + Return the due_on date + + If release is set, use the target date. + If not (or from_title is true), use the year/month (i.e. from the title) + """ + + if self.release and not from_title: + due_on = self.release['target'] + logging.debug('Due date %s from release target for %s', due_on, self) + else: + day = monthrange(self.year, self.month)[-1] + due_on = date(self.year, self.month, day) + + logging.debug("Generated due date %s for %s", due_on, self) + + return due_on + + def set_due_date(self): + """Set the due date""" + if self.milestone: + due = self.get_due() + if self.milestone.due_on != due: + logging.info("Set due date from %s to %s", self.milestone.due_on, due) + self.edit(due_on=due) + else: + logging.debug("No resetting milestone due date to same due %s", due) + else: + logging.warn("No milestone instance set for set_due_date") + + def __str__(self): + return self.title() + + def __eq__(self, other): + return (isinstance(other, self.__class__) + and self.year == other.year + and self.month == other.month + and self.point == other.point) # TODO: issue with None == integer? + + def __ne__(self, other): + return not self.__eq__(other) + + +def add_months(sourcedate, months): + """ + Add months to sourcedate, return new date + """ + month = sourcedate.month - 1 + months + year = sourcedate.year + month / 12 + month = month % 12 + 1 + day = min(sourcedate.day, monthrange(year, month)[1]) + return date(year, month, day) + + +def milestones_from_releases(releases): + """ + Generate a list of milestones based on releases + """ + milestones = [MileStone(title=title, release=release) for title, release in releases.items()] + milestones.sort(key=lambda s: s.release['target']) + + logging.debug("Milestones from releases: %s", ", ".join(map(str, milestones))) + + return milestones + + +def generate_milestones(milestones=4, months=2, today=None): + """ + Generate the next milestones (each spanning months) + """ + if today is None: + today = date.today() + month = today.month + + # Seek till multiple of months + while month % months > 0: + month += 1 + + start = date(today.year, month, 1) + + m_future = [] + for m in range(1, milestones): + n = add_months(start, m*months) + m_future.append(MileStone(year_month=(n.year, n.month))) + + logging.debug("Generated %d milestones (duration %d months) from today %s (start %s): %s", + milestones, months, today, start, m_future) + + return m_future + + +def mkms(repo, state, **kwargs): + """ + Return the milestones in state state (any named args are passed to the get_milestones API method) + """ + kwargs['state'] = state + return [MileStone(milestone=m) for m in repo.get_milestones(**kwargs)] + + +def configure_milestones(repo, milestones): + """ + Open / reopen milestones for repo + Set correct due date from release info from milestones list + """ + + # TODO: change state somehow + + logging.debug("configure milestones for repo %s", repo.name) + + # These MileStone instances have no release data + m_open = mkms(repo, 'open') + m_closed = mkms(repo, 'closed') + m_all = m_open + m_closed + + for m in m_closed: + logging.debug(" Found closed milestone %s", m) + m.set_state() + + for m in m_open: + logging.debug(" Found open milestone %s", m) + + if m.point is None: + if m in milestones: + m.release = milestones[milestones.index(m)].release + logging.debug("Adding release data %s to open milestone %s", m.release, m) + else: + logging.warn("Configuring open milestone %s in repo but not found in releases. Date will be generated", m, repo.name) + m.set_due_date() + m.set_state() + else: + logging.info(' Not modifying open point release %s in repo %s', m, repo.name) + + for m in milestones: + if m in m_all: + logging.debug('Future milestone %s already in open/closed in repo %s', m, repo.name) + else: + m.create(repo) + + +def bump(repo, months=2, releases=None): + """ + Bump the open milestones for the repository. + + This shifts the due date by months and changes the title accordingly (preserves issues/prs). + + If a release is present, it assumes it is the old release data (so do not change it before bump) + and than it also bumps the release start, rcs and target dates. + + The release data is only used for generating new/bumped release config data. + + Return the new releases config data (if any) + """ + + if releases is None: + releases = {} + + # These have to be ordered, such that bumping the milestone + # does not generate an already existing one + # Sort most future one first + m_open = mkms(repo, 'open', sort='due_date', direction='desc') + + u_release_data = {} + for m in m_open: + logging.debug("Found open milestone %s", m) + + if m.point is None: + # get_due from title only + due_from_title = m.get_due(from_title=True) + due_updated = add_months(due_from_title, months) + + # Make a new MileStone + m_u = MileStone(year_month=(due_updated.year, due_updated.month)) + + # Reuse the original milestone instance + m_u.milestone = m.milestone + + # set_due_date will now use the original milestone instance + # to modify the due date and the title + logging.info("Bump milestone %s (due %s) to new %s (due %s) for repo %s", + m, due_from_title, m_u, m_u.get_due(), repo.name) + m_u.set_due_date() + + # Generate bumped release config data + release_u = None + release = releases.get(m.title(), None) + if release: + release_u = [add_months(release[k], months).strftime('%Y-%m-%d') for k in ['start', 'rcs', 'target']] + u_release_data[m_u.title()] = release_u + else: + logging.info('Not modifying open point release %s', m) + + return u_release_data + +def sort_milestones(msts, add_special=True): + """ + Return sorted milestones + """ + # TODO: replace by MileStone instances with ordering support + # special is ordered + special = (MST_BACKLOG, MST_LEGACY,) + + # milestones are sorted keys on date, with Backlog and Legacy last + milestones = [x for x in msts if x not in special] + milestones.sort(key=lambda s: [int(u) for u in s.split('.')]) + + if add_special: + for mst in special: + if mst in msts: + milestones.append(mst) + + return milestones diff --git a/lib/release_helper/render.py b/lib/release_helper/render.py new file mode 100644 index 0000000..a35607f --- /dev/null +++ b/lib/release_helper/render.py @@ -0,0 +1,220 @@ +""" +TT rendering +""" + +import glob +import logging +import os +import shutil +from datetime import datetime +from json import dump, load +from release_helper.collect import TYPE_PR, TYPE_ISSUE, BACKWARDS_INCOMPATIBLE, TO_DISCUSS +from release_helper.milestone import sort_milestones +from template import Template, TemplateException + + +# Distribute the TT files as part of the py files, unzipped +INCLUDEPATH = os.path.join(os.path.dirname(__file__), 'tt') + +JAVASCRIPT = os.path.join(os.path.dirname(__file__), 'javascript') + +def render(tt, data): + """ + Given tt filename, render using data + + (.tt suffix is added if missing) + """ + opts = { + 'INCLUDE_PATH': [INCLUDEPATH], + } + + if not tt.endswith('.tt'): + tt += '.tt' + + try: + t = Template(opts) + return t.process(tt, data) + except TemplateException as e: + msg = "Failed to render TT %s with data %s (TT opts %s): %s" % (tt, data, opts, e) + logging.error(msg) + raise TemplateException('render', msg) + + +def make_html(project, releases, output_filenames): + """ + Generate and write the release + * releases.json + * index.html + * to_discuss.html + * burndown-%(milestone).json + """ + # generate the releases.json data + releases_fn = output_filenames['releases'] + with open(releases_fn, 'w') as f: + dump(releases, f) + logging.info('Wrote releases data in %s', releases_fn) + + index, discuss = index_discuss(project, output_filenames['pulls']) + + index_html = output_filenames['index'] + with open(index_html, 'w') as f: + f.write(index) + logging.info("Wrote index %s", index_html) + + discuss_html = output_filenames['discuss'] + with open(discuss_html, 'w') as f: + f.write(discuss) + logging.info("Wrote discuss %s", discuss_html) + + # generate the burndown json data + make_burndown(output_filenames['pulls'], output_filenames['burndown']) + + # copy the javascript in place + javascript_dir = output_filenames['javascript'] + for fn in glob.glob(os.path.join(JAVASCRIPT, '*.js')): + logging.debug("copying javascript files %s to dir %s", fn, javascript_dir) + shutil.copy(fn, javascript_dir) + + +def index_discuss(project, pulls_filename, previous=None, index='index', discuss='discuss'): + """ + Generate index.html and to_discuss.html + + Load data from pulls_filename json + """ + if previous is None: + previous = [] + + logging.debug("index from %s JSON data", pulls_filename) + with open(pulls_filename) as f_in: + pulls = load(f_in) + + milestones = sort_milestones(pulls.keys()) + + data = { + 'project': project, + 'now': datetime.utcnow().replace(microsecond=0).isoformat(' '), + 'milestones': previous + milestones, # milestones + 'previous_releases': dict([(x, 1) for x in previous]), # dict for easy lookup + 'data': pulls, # dict with milestone key, and dict value, which has repo key + } + + index_html = render(index, data) + + things_discuss = {} + for milestone_data in pulls.values(): + for repo, repo_data in milestone_data.items(): + things = things_discuss.setdefault(repo, []) + things.extend([t for t in repo_data['things'] if t.get(TO_DISCUSS, False) and t['state'] != 'closed']) + + data['discuss'] = things_discuss + discuss_html = render(discuss, data) + + return index_html, discuss_html + + +def make_burndown(pulls_filename, burndownfn_template): + """ + Generate the burndown-%(mst).json + """ + logging.debug("burndown from %s JSON data", pulls_filename) + with open(pulls_filename) as f_in: + pulls = load(f_in) + + # No burndown data from Backlog/Legacy + milestones = sort_milestones(pulls.keys(), add_special=False) + + for mst in milestones: + fn = burndownfn_template % {'milestone': mst} + + repos = pulls[mst].keys() + to_burn = 0 + burned = [] + + for repo in repos: + things = pulls[mst][repo]['things'] + to_burn += len(things) + burned.extend([t['closed'] for t in things if 'closed' in t]) + burned.sort() + + burndown = { + 'to_burn': to_burn, + 'closed': [], + } + for closed in burned: + to_burn -= 1 + burndown['closed'].append([closed, to_burn]) + + with open(fn, 'w') as f: + dump(burndown, f) + logging.info('Wrote burndown data for milestone %s in %s', mst, fn) + + +def make_notes(project, milestone, template, output_filenames): + """ + Create the reelase notes file + """ + notesfn = output_filenames['releasenotes'] % {'milestone': milestone} + notes = release_notes(project, output_filenames['pulls'], milestone, template) + with open(notesfn, 'w') as f: + f.write(notes) + logging.info("Wrote releasenotes %s", notesfn) + + +def release_notes(project, pulls_filename, milestone, template): + """ + Generate the release notes for milestone using release template TT + """ + + logging.debug("burndown from %s JSON data", pulls_filename) + + with open(pulls_filename) as f_in: + all_pulls = load(f_in) + all_milestones = sort_milestones(all_pulls.keys()) + + # Will fail if no current or no next milestone can be determined + try: + next_milestone = all_milestones[all_milestones.index(milestone)+1] + except (IndexError, KeyError): + logging.error("No next milestone for current %s in %s", milestone, all_milestones) + raise + + repos = all_pulls[milestone].keys() + repos.sort() + + # PR notes + pulls = {} + + count_issues = 0 + count_pulls = 0 + for repo in repos: + things = all_pulls[milestone][repo]['things'] + + count_issues += len([t for t in things if t['type'] == TYPE_ISSUE]) + + rpulls = [t for t in things if t['type'] == TYPE_PR] + rpulls.sort(key=lambda t: t['title']) + count_pulls += len(rpulls) + + pulls[repo]= rpulls + + backwards_incompatible = {} + for repo, prs in pulls.items(): + bw_ic = [t for t in prs if t.get(BACKWARDS_INCOMPATIBLE, False)] + if bw_ic: + backwards_incompatible[repo] = bw_ic + + data = { + 'project': project, + 'now': datetime.utcnow().replace(microsecond=0).isoformat(' '), + 'milestone': milestone, + 'next_milestone': next_milestone, + 'count_issues': count_issues, + 'count_pulls': count_pulls, + 'pulls': pulls, # dict with repo key, and list of PRs as value + } + + if backwards_incompatible: + data['backwards'] = backwards_incompatible + + return render(template, data) diff --git a/lib/release_helper/tt/css.tt b/lib/release_helper/tt/css.tt new file mode 100644 index 0000000..34fffe9 --- /dev/null +++ b/lib/release_helper/tt/css.tt @@ -0,0 +1,38 @@ + + + + + + diff --git a/lib/release_helper/tt/discuss.tt b/lib/release_helper/tt/discuss.tt new file mode 100644 index 0000000..a0189b8 --- /dev/null +++ b/lib/release_helper/tt/discuss.tt @@ -0,0 +1,58 @@ + + + + + [% project %] to discuss +[% INCLUDE 'css.tt' %] + + + +
+

To be discussed at the next workshop

+
+ +[% FOREACH repo IN discuss.pairs %] +[% IF repo.value.size == 0; + NEXT; + END; %] + + + + +[% END %] +
[% repo.key %] +
    +[% FOREACH this IN repo.value %] +
  • + + [% this.title %] +[% IF this.comment_count > 0 %] + [% this.comment_count %] +[% END %] +
    + Created by + [% this.user %] + + [% this.created %] + +[% IF this.defined('assignee') %] + , Assigned to + [% this.assignee %] +[% END %] + , Updated [% this.updated %] +
    +
    • +[% END %] +
+
+
+
+ + + diff --git a/lib/release_helper/tt/index.tt b/lib/release_helper/tt/index.tt new file mode 100644 index 0000000..dff5142 --- /dev/null +++ b/lib/release_helper/tt/index.tt @@ -0,0 +1,111 @@ + + + + + [% project %] release + +[% INCLUDE 'css.tt' %] + + + +
+ +
    +[% FOREACH milestone IN milestones %] +
  • + [% milestone %] +
    • +[% END -%] +
+ +
+
+
+
+

Select a release to view backlog

+
+
+
+ +
+
+ +[% FOREACH milestone IN milestones %] +
+ +[% i_closed = 0; i_open = 0; m_due = 'never'; %] +[% FOREACH repo IN data.$milestone.pairs %] +[%- things = repo.value.things; + IF things.size == 0; + NEXT; + END; + i_closed = i_closed + repo.value.closed; + i_open = i_open + repo.value.open; + IF repo.value.defined('due'); + m_due = repo.value.due; + END; %] + + + + +[% END -%] +
[% repo.key %] +
    +[% FOREACH this IN things.sort('title').reverse() %] +
  • + + [% this.title %] +[% IF this.comment_count > 0 %] + [% this.comment_count %] +[% END %] +
    + Created by + [% this.user %] + + [% this.created %] + +[% IF this.defined('assignee') %] + , Assigned to + [% this.assignee %] +[% END %] + , Updated [% this.updated %] +
    +
    • +[% END %] +
+
+ +[% IF i_open + i_closed > 0 -%] +
+ Progress: ([% i_closed %] closed, [% i_open %] open, due [% m_due %]) +
+[% # Do this as integer math + i_progress = i_closed * 100 / (i_open + i_closed) %] +
[% i_progress %]%
+
+
+[% END -%] + +
+[% END -%] +
+

Page generated [% now %]

+
+ + + + + + + diff --git a/lib/release_helper/tt/js_reldate.tt b/lib/release_helper/tt/js_reldate.tt new file mode 100644 index 0000000..dd23484 --- /dev/null +++ b/lib/release_helper/tt/js_reldate.tt @@ -0,0 +1,5 @@ + $(function() { + $('.reldate').each(function(index) { + $(this).text(moment.tz($(this).text(), "YYYY-MM-DDTHH:mm:ss", "UTC").fromNow()); + }); + }); diff --git a/lib/release_helper/tt/quattor_releasenotes.tt b/lib/release_helper/tt/quattor_releasenotes.tt new file mode 100644 index 0000000..fb1e910 --- /dev/null +++ b/lib/release_helper/tt/quattor_releasenotes.tt @@ -0,0 +1,36 @@ +--- +layout: article +title: [% project %] [% milestone %] released +category: news +author: James Adams +--- + +Packages are available from our [yum repository](http://yum.quattor.org/[% milestone %]/), +both the RPMs and the repository metadata are signed with +[my GPG key](http://yum.quattor.org/GPG/RPM-GPG-KEY-quattor-jrha). + +As always, many thanks to everyone who contributed! +We merged [% count_pulls %] pull requests and resolved [% count_issues %] issues. + +The next release should be [% next_milestone %], +take a look at the [backlog](http://www.quattor.org/release/) to see what we're working on. + +[% IF backwards.defined() -%] +Backwards Incompatible Changes +------------------------------ +[% FOREACH repo IN backwards.pairs %] +### [% repo.key %] +[% FOREACH pr IN repo.value -%] +* [[% pr.title.replace('^([^:]+): ', '**$1:** ') %]](https://github.com/quattor/[% repo.key %]/pull/[% pr.number %]) +[% END -%] +[% END %] +[%- END -%] + +Changelog +--------- +[% FOREACH repo IN pulls.pairs %] +### [% repo.key %] +[% FOREACH pr IN repo.value -%] +* [[% pr.title.replace('^([^:]+): ', '**$1:** ') %]](https://github.com/quattor/[% repo.key %]/pull/[% pr.number %]) +[% END -%] +[% END %] diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..e3165d0 --- /dev/null +++ b/setup.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python + +import glob +from setuptools import setup + + +SETUP = { + 'name': 'release-helper', + 'version': '0.1.0', + 'author': 'James Adams', + 'author_email': 'james.adams@stfc.ac.uk', + 'license': 'APL', + 'packages' : ['release_helper'], + 'package_dir': {'': 'lib'}, + 'scripts': glob.glob('bin/*.py') + glob.glob('bin/*.sh'), + 'install_requires': [ + 'PyGithub', + 'Template-Python', # python TT port + # For tests + #'mock', + #'prospector', + ], + 'test_suite': 'test', + 'data_files': [ + # Look for better location + ('release_helper/data', glob.glob('data/*html') + glob.glob('data/*example') + glob.glob('data/*/js')), + ('release_helper/tt', glob.glob('lib/release_helper/tt/*.tt')), + ('release_helper/javascript', glob.glob('lib/release_helper/javascript/*.js')), + ], + 'zip_safe': False, # shipped TT files +} + +if __name__ == '__main__': + setup(**SETUP) diff --git a/test/__init__.py b/test/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/test/__init__.py @@ -0,0 +1 @@ + diff --git a/test/config.py b/test/config.py new file mode 100644 index 0000000..e4f2c34 --- /dev/null +++ b/test/config.py @@ -0,0 +1,69 @@ +import datetime +import os +import unittest +import release_helper.config as cfg + + +# mock.patch really isn't made for this +class Repo(object): + def __init__(self, name): + self.name = name + +class GHU(object): + def __init__(self, org, ghargs): + self.org = org + self.ghargs = ghargs + + def get_repos(self): + if self.ghargs.get('base_url', None) is None: + ts = ['repoxyz', 'repoabc'] + else: + ts = ['repopublic', 'reposecret'] + return map(Repo, ts) + +GHCALLED = [] +class GH(object): + def __init__(self, **kwargs): + GHCALLED.append(kwargs) + + def get_user(self, user): + return GHU(user, GHCALLED[-1]) + + def get_organization(self, org): + return GHU(org, GHCALLED[-1]) + + +# monkey patch Github +cfg.Github = GH + +class UtilsTest(unittest.TestCase): + + def setUp(self): + global GHCALLED + GHCALLED = [] + + def test_make_config(self): + + fn = os.path.join(os.path.dirname(__file__), 'data', 'test1.cfg') + + config = cfg.make_config([fn]) + + self.assertEqual(GHCALLED, [{'login_or_token': 'example1', 'password': '1234567890'}, + {'password': 'abcdef123456', 'login_or_token': 'localuser', 'base_url': 'https://enterprise.example.com/some/subtree/v3/'}]) + self.assertEqual(sorted(config.keys()), ['labels', 'project', 'releases', 'repos']) + + self.assertEqual(config['labels'], {'bug': 'ef2929', 'question': '123456'}) + self.assertEqual(config['project'], 'mytest') + self.assertEqual(config['releases'], { + "14.10" : { + "start" : datetime.datetime(2014, 9, 1, 0, 0), + "rcs" : datetime.datetime(2014, 10, 27, 0, 0), + "target" : datetime.datetime(2014, 11, 1, 0, 0), + }, + "15.2" : { + "start" : datetime.datetime(2014, 11, 14, 0, 0), + "rcs" : datetime.datetime(2015, 3, 7, 0, 0), + "target" : datetime.datetime(2015, 3, 20, 0, 0), + }, + }) + self.assertEqual([r.name for r in config['repos']], ['repoabc', 'repopublic']) diff --git a/test/data/index.html b/test/data/index.html new file mode 100644 index 0000000..2dcd7c6 --- /dev/null +++ b/test/data/index.html @@ -0,0 +1,462 @@ + + + + + myproject release + + + + + + + + + + + +
+ + + +
+
+
+
+

Select a release to view backlog

+
+
+
+ +
+
+ + +
+ + + + + + + + + + + + +
CAF + +
CCM + +
+ +
+ Progress: (36 closed, 0 open, due 2016-06-30T07:00:00) +
+ +
100%
+
+
+ +
+ +
+ + + + + + + + + + + + +
CAF + +
CCM + +
+ +
+ Progress: (0 closed, 14 open, due 2016-08-30T07:00:00) +
+ +
0%
+
+
+ +
+ +
+ + + + + + + + + + + + +
CAF +
    + +
  • + + CAF::Download + +
    + Created by + stdweird + + 2014-10-26T16:07:41 + + + , Updated 2016-05-05T15:07:03 +
    +
    • + +
+
CCM + +
+ +
+ Progress: (0 closed, 2 open, due 2016-10-31T07:00:00) +
+ +
0%
+
+
+ +
+ +
+ + + + + + + + + + + + +
CAF + +
CCM + +
+ + +
+ +
+ + + + + + + +
CAF + +
+ + +
+
+

Page generated 1970-01-02 03:04:05

+
+ + + + + + + diff --git a/test/data/quattor-pulls.json b/test/data/quattor-pulls.json new file mode 100644 index 0000000..f542e2b --- /dev/null +++ b/test/data/quattor-pulls.json @@ -0,0 +1,267 @@ +{ + "16.6": { + "CCM": { + "things": [ + { + "updated": "2016-07-08T13:01:15", + "labels": [], + "number": 111, + "user": "stdweird", + "icon": "git-pull-request", + "title": "ProfileCache: Lock use reporter instance for reporting", + "url": "https://github.com/quattor/CCM/pull/111", + "created": "2016-07-07T12:43:29", + "comment_count": 0, + "state": "closed", + "closed": "2016-07-08T13:01:15", + "type": "pull-request" + }, + { + "updated": "2016-05-17T16:33:26", + "labels": [], + "number": 101, + "user": "stdweird", + "icon": "git-pull-request", + "title": "CCM: fetchProfile should close all CAF::File* instances ", + "url": "https://github.com/quattor/CCM/pull/101", + "created": "2016-05-16T08:56:38", + "comment_count": 4, + "state": "closed", + "closed": "2016-05-17T16:33:26", + "type": "pull-request" + }, + { + "updated": "2016-05-09T10:06:41", + "labels": [], + "number": 99, + "user": "stdweird", + "icon": "git-pull-request", + "title": "bump build-tools to 1.49", + "url": "https://github.com/quattor/CCM/pull/99", + "created": "2016-05-09T09:40:08", + "comment_count": 1, + "state": "closed", + "closed": "2016-05-09T10:06:41", + "type": "pull-request" + } + ], + "open": 0, + "due": "2016-06-30T07:00:00", + "closed": 11 + }, + "CAF": { + "things": [ + { + "updated": "2016-07-07T10:02:43", + "labels": [], + "number": 174, + "user": "jouvin", + "icon": "git-pull-request", + "title": "Add PyCharm and Eclipse files to gitignore", + "url": "https://github.com/quattor/CAF/pull/174", + "created": "2016-07-07T09:49:05", + "comment_count": 0, + "state": "closed", + "closed": "2016-07-07T10:02:43", + "type": "pull-request" + }, + { + "updated": "2016-07-08T13:26:38", + "labels": [ + "discuss at workshop", + "backwards incompatible" + ], + "number": 173, + "user": "stdweird", + "icon": "git-pull-request", + "title": "Lock: handle existing old-style locks", + "url": "https://github.com/quattor/CAF/pull/173", + "created": "2016-07-06T15:54:55", + "comment_count": 16, + "state": "closed", + "closed": "2016-07-08T12:57:56", + "type": "pull-request", + "backwards_incompatible": true, + "to_discuss": true + }, + { + "updated": "2016-07-08T12:57:56", + "labels": [], + "number": 172, + "user": "stdweird", + "icon": "issue-closed", + "title": "Lock not compatible with old lock", + "url": "https://github.com/quattor/CAF/issues/172", + "created": "2016-07-06T13:41:53", + "comment_count": 0, + "state": "closed", + "closed": "2016-07-08T12:57:56", + "type": "issue" + } + ], + "open": 0, + "due": "2016-06-30T07:00:00", + "closed": 25 + } + }, + "Legacy": { + "CCM": { + "things": [], + "open": 0, + "due": null, + "closed": 0 + }, + "CAF": { + "things": [ + { + "updated": "2016-07-06T09:33:14", + "labels": [ + "bug", + "discuss at workshop" + ], + "number": 148, + "user": "ttyS4", + "icon": "issue-opened", + "title": "CAF::FileWriter updates files in-place", + "url": "https://github.com/quattor/CAF/issues/148", + "created": "2016-04-11T13:03:16", + "comment_count": 11, + "state": "open", + "type": "issue", + "to_discuss": true + } + ], + "open": 0, + "due": null, + "closed": 0 + } + }, + "16.10": { + "CCM": { + "things": [ + { + "updated": "2016-06-05T09:55:39", + "labels": [], + "number": 107, + "user": "stdweird", + "icon": "issue-opened", + "title": "remove CCM::Property class", + "url": "https://github.com/quattor/CCM/issues/107", + "created": "2016-06-05T09:52:08", + "comment_count": 1, + "state": "open", + "type": "issue" + } + ], + "open": 1, + "due": "2016-10-31T07:00:00", + "closed": 0 + }, + "CAF": { + "things": [ + { + "updated": "2016-05-05T15:07:03", + "labels": [], + "number": 62, + "user": "stdweird", + "icon": "issue-opened", + "title": "CAF::Download", + "url": "https://github.com/quattor/CAF/issues/62", + "created": "2014-10-26T16:07:41", + "comment_count": 0, + "state": "open", + "type": "issue" + } + ], + "open": 1, + "due": "2016-10-31T07:00:00", + "closed": 0 + } + }, + "Backlog": { + "CCM": { + "things": [ + { + "updated": "2016-07-08T15:25:10", + "labels": [], + "number": 112, + "user": "msmark", + "icon": "issue-opened", + "title": "ccm-fetch to support downloading test profiles", + "url": "https://github.com/quattor/CCM/issues/112", + "created": "2016-07-08T15:19:36", + "comment_count": 2, + "state": "open", + "type": "issue" + } + ], + "open": 0, + "due": null, + "closed": 0 + }, + "CAF": { + "things": [ + { + "updated": "2016-06-03T12:16:39", + "labels": [ + "bug" + ], + "number": 167, + "user": "ned21", + "icon": "issue-opened", + "title": "CAF::Process, $? and NoAction", + "url": "https://github.com/quattor/CAF/issues/167", + "created": "2016-06-02T20:41:26", + "comment_count": 1, + "state": "open", + "type": "issue" + } + ], + "open": 0, + "due": null, + "closed": 0 + } + }, + "16.8": { + "CCM": { + "things": [ + { + "updated": "2016-06-08T11:48:26", + "labels": [], + "number": 109, + "user": "stdweird", + "icon": "issue-opened", + "title": "ugly exception during initial installation", + "url": "https://github.com/quattor/CCM/issues/109", + "created": "2016-06-08T11:48:26", + "comment_count": 0, + "state": "open", + "type": "issue" + } + ], + "open": 7, + "due": "2016-08-30T07:00:00", + "closed": 0 + }, + "CAF": { + "things": [ + { + "updated": "2016-06-04T13:13:01", + "labels": [], + "number": 169, + "user": "stdweird", + "icon": "issue-opened", + "title": "CAF::FileWriter support mtime", + "url": "https://github.com/quattor/CAF/issues/169", + "created": "2016-06-04T13:13:01", + "comment_count": 0, + "state": "open", + "type": "issue" + } + ], + "open": 7, + "due": "2016-08-30T07:00:00", + "closed": 0 + } + } +} diff --git a/test/data/release_notes.md b/test/data/release_notes.md new file mode 100644 index 0000000..a7bb7b5 --- /dev/null +++ b/test/data/release_notes.md @@ -0,0 +1,35 @@ +--- +layout: article +title: myproject 16.6 released +category: news +author: James Adams +--- + +Packages are available from our [yum repository](http://yum.quattor.org/16.6/), +both the RPMs and the repository metadata are signed with +[my GPG key](http://yum.quattor.org/GPG/RPM-GPG-KEY-quattor-jrha). + +As always, many thanks to everyone who contributed! +We merged 5 pull requests and resolved 1 issues. + +The next release should be 16.8, +take a look at the [backlog](http://www.quattor.org/release/) to see what we're working on. + +Backwards Incompatible Changes +------------------------------ + +### CAF +* [**Lock:** handle existing old-style locks](https://github.com/quattor/CAF/pull/173) + +Changelog +--------- + +### CAF +* [Add PyCharm and Eclipse files to gitignore](https://github.com/quattor/CAF/pull/174) +* [**Lock:** handle existing old-style locks](https://github.com/quattor/CAF/pull/173) + +### CCM +* [**CCM:** fetchProfile should close all CAF::File* instances ](https://github.com/quattor/CCM/pull/101) +* [**ProfileCache:** Lock use reporter instance for reporting](https://github.com/quattor/CCM/pull/111) +* [bump build-tools to 1.49](https://github.com/quattor/CCM/pull/99) + diff --git a/test/data/test1.cfg b/test/data/test1.cfg new file mode 100644 index 0000000..511d1a6 --- /dev/null +++ b/test/data/test1.cfg @@ -0,0 +1,24 @@ +[main] +github=github,enterprise +project=mytest + +[releases] +14.10=2014-09-01,2014-10-27,2014-11-01 +15.2=2014-11-14,2015-03-07,2015-03-20 + +[labels] +bug=ef2929 +question=123456 + +[github] +username=example1 +token=1234567890 +organisation=pubteam +white=abc + +[enterprise] +api=https://enterprise.example.com/some/subtree/v3/ +username=localuser +token=abcdef123456 +organisation=myteam +black=secret diff --git a/test/data/to_discuss.html b/test/data/to_discuss.html new file mode 100644 index 0000000..056b392 --- /dev/null +++ b/test/data/to_discuss.html @@ -0,0 +1,97 @@ + + + + + myproject to discuss + + + + + + + + + + +
+

To be discussed at the next workshop

+
+ + + + + + + + + +
CAF + +
+
+
+ + + diff --git a/test/lint.py b/test/lint.py new file mode 100644 index 0000000..8ca4856 --- /dev/null +++ b/test/lint.py @@ -0,0 +1,75 @@ +from prospector.run import Prospector +from prospector.config import ProspectorConfig +import os +import pprint +import re +import sys +import unittest + +class UtilsTest(unittest.TestCase): + # List of regexps patterns applied to code or message of a prospector.message.Message + # Blacklist: if match, skip message, do not check whitelist + # Whitelist: if match, fail test + PROSPECTOR_BLACKLIST = [ + ] + PROSPECTOR_WHITELIST = [ + 'undefined', + 'no-value-for-parameter', + 'dangerous-default-value', + 'redefined-builtin', + 'bare-except', + 'E713', # not 'c' in d: -> 'c' not in d: + 'arguments-differ', + 'unused-argument', + 'unused-variable', + 'reimported', + 'F811', # redefinition of unused name + 'unused-import', + 'syntax-error', + ] + + # Prospector commandline options (positional path is added automatically) + PROSPECTOR_OPTIONS = [ + '--strictness', 'medium', + '--max-line-length', '120', + '--absolute-paths', + ] + + def test_prospector(self): + + if os.environ.get('NO_PROSPECTOR', '0') in ('1', 'y', 'yes'): + self.assertTrue(True, msg='Skipping prospector test') + return + + sys.argv = ['release-helper-prospector'] + + # extra options needed + sys.argv.extend(self.PROSPECTOR_OPTIONS) + # add/set repository path as basepath + sys.argv.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + + config = ProspectorConfig() + prospector = Prospector(config) + + prospector.execute() + + blacklist = map(re.compile, self.PROSPECTOR_BLACKLIST) + whitelist = map(re.compile, self.PROSPECTOR_WHITELIST) + + failures = [] + for msg in prospector.get_messages(): + # example msg.as_dict(): + # {'source': 'pylint', 'message': 'Missing function docstring', 'code': 'missing-docstring', + # 'location': {'function': 'TestHeaders.test_check_header.lgpl', 'path': u'headers.py', + # 'line': 122, 'character': 8, 'module': 'headers'}} + + if any([bool(reg.search(msg.code) or reg.search(msg.message)) for reg in blacklist]): + continue + + if any([bool(reg.search(msg.code) or reg.search(msg.message)) for reg in whitelist]): + failures.append(msg.as_dict()) + + self.assertFalse(failures, "prospector failures: %s" % pprint.pformat(failures)) + +if __name__ == '__main__': + unittest.main() diff --git a/test/render.py b/test/render.py new file mode 100644 index 0000000..c7d6868 --- /dev/null +++ b/test/render.py @@ -0,0 +1,77 @@ +import datetime +import os +import unittest +import release_helper.render +from release_helper.render import render, index_discuss, release_notes +from mock import patch + +ORIG_INCLUDE = release_helper.render.INCLUDEPATH + +class UtilsTest(unittest.TestCase): + def test_render(self): + """ + Test simple rendering example + """ + + # Monkey patch the inlcude path for testing + release_helper.render.INCLUDEPATH = os.path.join(os.path.dirname(__file__), 'tt') + + txt = render('hello', {'world': 'trivial test'}) + + # restore it + release_helper.render.INCLUDEPATH = ORIG_INCLUDE + + self.assertEqual(txt, "Hello trivial test.\n", msg="Trivial test result: %s" % txt) + + + @patch('release_helper.render.datetime') + def test_index(self, mocked_datetime): + """ + Test rendering index using TT + """ + mocked_datetime.utcnow.return_value=datetime.datetime(1970, 1, 2, 3, 4, 5, 6) + + self.maxDiff = None + self.longMessage = True + + fn = os.path.join(os.path.dirname(__file__), 'data', 'index.html') + res_i = open(fn).read() + + fn = os.path.join(os.path.dirname(__file__), 'data', 'to_discuss.html') + res_d = open(fn).read() + + json_filename = os.path.join(os.path.dirname(__file__), 'data', 'quattor-pulls.json') + + index, discuss = index_discuss('myproject', json_filename) + + # Uncomment to update the rendered test/data/index.html once considered ok + # Do not forget to comment it again + #open('/tmp/testrenderindex.html', 'w').write(index) + #open('/tmp/testrenderdiscuss.html', 'w').write(discuss) + + self.assertMultiLineEqual(index, res_i) + self.assertMultiLineEqual(discuss, res_d) + + + @patch('release_helper.render.datetime') + def test_releasenotes(self, mocked_datetime): + """ + Test rendering index using TT + """ + mocked_datetime.utcnow.return_value=datetime.datetime(1970, 1, 2, 3, 4, 5, 6) + + self.maxDiff = None + self.longMessage = True + + fn = os.path.join(os.path.dirname(__file__), 'data', 'release_notes.md') + res = open(fn).read() + + json_filename = os.path.join(os.path.dirname(__file__), 'data', 'quattor-pulls.json') + + md = release_notes('myproject', json_filename, '16.6', 'quattor_releasenotes') + + # Uncomment to update the rendered test/data/index.html once considered ok + # Do not forget to comment it again + #open('/tmp/testrelease_notes.md', 'w').write(md) + + self.assertMultiLineEqual(md, res) diff --git a/test/tt/hello.tt b/test/tt/hello.tt new file mode 100644 index 0000000..db894b1 --- /dev/null +++ b/test/tt/hello.tt @@ -0,0 +1 @@ +Hello [% world %].