relnotes

#!/bin/bash
#
# Copyright 2016 The Kubernetes Authors All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Set PROGram name
PROG=${0##*/}
########################################################################
#+
#+ NAME
#+     $PROG - Generate release notes for K8S releases
#+
#+ SYNOPSIS
#+     $PROG  [--quiet] [[starttag..]endtag] [--htmlize-md] [--full]
#+            [--release-tars=/path/to/release-tars]
#+            [--github-token=<token>] [--branch=<branch>]
#+            [--markdown-file=<file>] [--html-file=<file>]
#+            [--release-bucket=<gs bucket>] [--preview]
#+     $PROG  [--helpshort|--usage|-?]
#+     $PROG  [--help|-man]
#+
#+ DESCRIPTION
#+      $PROG scans 'git log' for 'merge pull's and collects and displays
#+      release notes based on release-note-* labels from PR titles and PR
#+      release-note blocks (in the body).
#+
#+      By default, $PROG produces release notes for the last github release
#+      to the HEAD of current branch and uses a standard git range otherwise.
#+      You may omit the end of the range to terminate the range at the HEAD
#+      of the current branch.
#+
#+      The default output is pure markdown and unless [--markdown-file=] is
#+      specified, the output file is in /tmp/$PROG-release-notes.md.
#+      If [--html-file=] is set, $PROG will also produce a pure html version
#+      of the notes at that location.
#+
#+      If [--quiet] is not specified, the output to stdout will always be
#+      only the markdown version of the output.
#+
#+      [--branch=] is used to specify a branch other than the current one.
#+
#+      Other options detailed in EXAMPLES below.
#+
#+ OPTIONS
#+     --quiet                   - Don't display the notes when done
#+     --htmlize-md              - Output markdown with html for PRs and
#+                                 contributors (for use in CHANGELOG.md)
#+     --full                    - Force 'full' release format to show all
#+                                 sections of release notes. (This is the
#+                                 *default* for new branch X.Y.0 notes)
#+     --release-tars=           - directory of tars to sha256 sum for display
#+     --markdown-file=          - Specify an alt file to use to store notes
#+     --html-file=              - Produce a html version of the notes
#+     --release-bucket=         - Specify gs bucket to point to in
#+                                 generated notes (informational only)
#+     --preview                 - Report additional branch statistics (used for
#+                                 reporting outside of releases)
#+     --github-token=           - Must be specified if GITHUB_TOKEN not set
#+     --branch=                 - Specify a branch other than the current one
#+     [--help | -man]           - Display man page for this script
#+     [--usage | -?]            - Display in-line usage
#+
#+ EXAMPLES
#+     $PROG                     - Notes for last release to HEAD
#+                                 on current branch
#+     $PROG v1.1.4..            - Notes for v1.1.4 to HEAD on current branch
#+     $PROG v1.1.4..v1.1.7      - Notes for v1.1.4..v1.1.7
#+     $PROG v1.1.7              - Notes for last release
#+                                 on current branch to v1.1.7
#+
#+ FILES
#+     /tmp/$PROG-release-htmls.md
#+     /tmp/$PROG-release-htmls.html
#+
#+ SEE ALSO
#+     common.sh                 - Base function definitions
#+     gitlib.sh                 - git-related function definitions
#+     https://stedolan.github.io/jq - JSON CLI
#+
#+ BUGS/TODO
#+
########################################################################
# If NO ARGUMENTS should return *usage*, uncomment the following line:
#usage=${1-yes}

source $(dirname $(readlink -ne $BASH_SOURCE))/lib/common.sh
source $TOOL_LIB_PATH/gitlib.sh

# Validate command-line
common::argc_validate 1
[[ -n $FLAGS_release_tars && ! -d $FLAGS_release_tars ]] \
 && common::exit 1 "--release-tars=$FLAGS_release_tars doesn't exist!  Exiting..."

###############################################################################
# FUNCTIONS
###############################################################################
###############################################################################
# Get titles from a list of PRs
# @param prs - A space separated list of PRs to extract
#
extract_pr_title () {
  local prs="$*"
  local pr
  local content
  local body
  local author
  local pull_json

  for pr in $prs; do
    pull_json="$($GHCURL $K8S_GITHUB_API/pulls/$pr)"
    [[ -z "$pull_json" ]] && return 1
    body="$(echo "$pull_json" |jq -r '.body' |tr -d '\r')"

    # Look for a body release note first and default to title
    # * indent lines >1
    # Try to account for and adjust user-entered formatting
    # * This is somewhat complicated by the fact that we convert this to
    #   html, for things like email, group posts, dashboards, and there
    #   is a disconnect between pandoc and github's markdown for lists.
    #   https://github.com/jgm/pandoc/issues/2210
    content=$(echo "$body" |\
              sed -n '/^``` *release-note/,/^```/{/^```/!p;/^```$/q}' |\
              sed -e '/^$/d' -e '2,$s/^\( *\* \)/        \1/g' \
                  -e '2,$s/^\( *[^ \*]\)/    * \1/g')

    # if the release-note block is empty (unchanged from the template), use title
    if [[ -z "$content" ]]; then
      content=$(echo "$pull_json" | jq -r '.title')
    fi

    author=$(echo "$pull_json" | jq -r '.user.login')
    content=$(echo "$content" |sed -e '1s/^ *\** */* /g' \
                                   -e "1s/$/ (#$pr, @$author)/g")

    logecho -r "$content"
  done
}

###############################################################################
# Get merged PRs that match a specified label
# @param label - A label(string) to use for searching github PRs
get_prs_by_label () {
  local label="$1"
  local matching_prs
  local -a prs
  local total
  local current_page
  local num_pages
  local prs_per_page

  prs_per_page=100

  matching_prs="$($GHCURL ${K8S_GITHUB_SEARCHAPI}label:${label}&per_page=1)"

  total="$(echo -n $matching_prs | jq -r '.total_count')"
  # Calculate number of pages, rounding up
  num_pages=$(((total + prs_per_page - 1) / prs_per_page ))

  for current_page in `seq 1 $num_pages`; do
    prs+=($($GHCURL "${K8S_GITHUB_SEARCHAPI}label:${label}&page=$current_page" | jq -r '.items[] | (.number | tostring)'))
  done
  echo "${prs[@]}"
}

# Create a markdown dtable for the specified tarballs on GCS
# @param heading - level-3 heading to print before the table
# @param @ - local path to tarballs to list
#
create_downloads_table () {
  local heading=$1
  shift
  local url_prefix
  if [[ "$release_bucket" == "kubernetes-release" ]]; then
    url_prefix="https://dl.k8s.io"
  else
    url_prefix="https://storage.googleapis.com/$release_bucket/release"
  fi

  [[ -n "$heading" ]] && echo "### $heading"
  echo
  echo "filename | sha256 hash"
  echo "-------- | -----------"
  for file in $@; do
    echo "[${file##*/}]($url_prefix/$release_tag/${file##*/}) | \`$(common::sha $file 256)\`"
  done
  echo
}

###############################################################################
# Create the release note markdown body
# @param release_tars - A directory containing tarballs to link to on GCS
# @param start_tag - The start tag of range
# @param release_tag - The release tag of range
#
create_body () {
  local release_tars=$1
  local start_tag=$2
  local release_tag=$3
  local release_bucket=${FLAGS_release_bucket:-"kubernetes-release"}
  local title

  ((FLAGS_preview)) && title="Branch "
  # Show a more useful header if release_tag == HEAD
  if [[ "$release_tag" == "HEAD" ]]; then
    title+=$CURRENT_BRANCH
  else
    title+=$release_tag
  fi

  ((FLAGS_preview)) && echo "**Release Note Preview - generated on $(date)**"
  echo
  echo "# $title"
  echo
  echo "[Documentation](https://docs.k8s.io) &" \
       "[Examples](https://releases.k8s.io/$CURRENT_BRANCH/examples)"
  echo
  if [[ -n $release_tars ]]; then
    echo "## Downloads for $title"
    echo
    create_downloads_table "" $release_tars/kubernetes{,-src}.tar.gz
    create_downloads_table "Client Binaries" ${release_tars}/kubernetes-client*.tar.gz
    create_downloads_table "Server Binaries" ${release_tars}/kubernetes-server*.tar.gz
    create_downloads_table "Node Binaries" ${release_tars}/kubernetes-node*.tar.gz
  fi
  cat $PR_NOTES
}

###############################################################################
# CI job status
# Uses global CURRENT_BRANCH
ci_job_status () {
  local content
  local red=${TPUT[RED]}
  local green=${TPUT[GREEN]}
  local off=${TPUT[OFF]}
  local extra_flag

  if ((FLAGS_htmlize_md)); then
    red="<FONT COLOR=RED>"
    green="<FONT COLOR=GREEN>"
    off="</FONT>"
  fi

  # If working on a release branch assume --official for the
  # purpose of displaying find_green_build output
  if [[ $CURRENT_BRANCH =~ release- ]]; then
    extra_flag="--official"
  else
    # For master branch, limit the analysis to 30 primary ci jobs.
    # This is necessary due to the recently expanded blocking test list for
    # master.  The expanded test list is often unable to find a complete
    # passing set and find_green_build runs unbounded for hours.
    extra_flag="--limit=30"
  fi

  # State of tree
  echo
  echo "## State of $CURRENT_BRANCH branch"
  if content=$(find_green_build -v $extra_flag $CURRENT_BRANCH); then
    echo "${green}GOOD TO GO!$off"
  else
    echo "${red}NOT READY$off"
  fi
  echo
  echo "### Details"
  echo '```'
  echo "$content"
  echo '```'
}

###############################################################################
# Scan PRs for release-note-* labels and generate markdown for the actual
# release notes section of the report
# Uses global LAST_RELEASE CURRENT_BRANCH
generate_notes () {
  local branch_head
  local range
  local start_tag
  local release_tag
  local pretty_range
  local labels
  local body
  local tempcss=/tmp/$PROG-ca.$$
  local tempfile=/tmp/$PROG-file-1.$$
  local anchor
  local -a normal_prs
  local -a action_prs
  local -a experimental_prs
  local -a notes_normal
  local -a notes_action
  local -a notes_experimental
  local -a prs

  branch_head=$(git rev-parse refs/remotes/origin/$CURRENT_BRANCH 2>/dev/null)

  # If ${LAST_RELEASE[$CURRENT_BRANCH]} is unset attempt to get the last
  # release from the parent branch and then master
  : ${LAST_RELEASE[$CURRENT_BRANCH]:=${LAST_RELEASE[${CURRENT_BRANCH%.*}]}}
  : ${LAST_RELEASE[$CURRENT_BRANCH]:=${LAST_RELEASE[master]}}

  # Default
  range="${POSITIONAL_ARGV[0]:-"${LAST_RELEASE[$CURRENT_BRANCH]}..$branch_head"}"

  if [[ "${POSITIONAL_ARGV[0]}" =~ ([v0-9.]*-*(alpha|beta|rc)*\.*[0-9]*)\.\.([v0-9.]*-*(alpha|beta|rc)*\.*[0-9]*)$ ]]; then
    start_tag=${BASH_REMATCH[1]}
    release_tag=${BASH_REMATCH[3]}
  else
    start_tag="${LAST_RELEASE[$CURRENT_BRANCH]}"
    release_tag=${POSITIONAL_ARGV[0]}
  fi

  if [[ -z "$start_tag" ]]; then
    common::exit 1 "Unable to set beginning of range automatically." \
                   "Specify on the command-line. Exiting..."
  fi

  range="$start_tag..${release_tag:-$branch_head}"

  # If range is unterminated, finish it with $branch_head
  [[ $range =~ \.\.$ ]] && range+=$branch_head

  # Validate range
  if ! git rev-parse $range &>/dev/null; then
    logecho
    logecho "Invalid tags/range $range !"
    return 1
  fi

  # For pretty printing
  pretty_range=${range/$branch_head/HEAD}

  # Deref all the PRs back to master, paying special attention to 
  # automated cherrypicks that could have multiple sources
  while read line; do
    if [[ "$line" =~ automated-cherry-pick-of-(#[0-9]+-){1,} ]]; then
      prs+=($(echo "${BASH_REMATCH[0]}" | egrep -o "#[0-9]*" |tr -d '#'))
    else
      prs+=($(echo "$line" | awk '{gsub(/#/,"");print $4 }'))
    fi
  done < <(git log $range --format="%s" --grep="Merge pull")

  logecho

  echo "Scanning action required PR labels on the $CURRENT_BRANCH branch..."
  action_prs=($(get_prs_by_label release-note-breaking-change))
  action_prs+=($(get_prs_by_label release-note-action-required))
  echo "Scanning experimental PR label on the $CURRENT_BRANCH branch..."
  experimental_prs=($(get_prs_by_label release-note-experimental))
  echo "Scanning release-note PR label on the $CURRENT_BRANCH branch..."
  normal_prs=($(get_prs_by_label release-note))

  for pr in ${prs[*]}; do
    if [[ " ${action_prs[@]} " =~ " ${pr} " ]]; then
      notes_action+=("$pr")
    elif [[ " ${experimental_prs[@]} " =~ " ${pr} " ]]; then
      notes_experimental+=("$pr")
    elif [[ " ${normal_prs[@]} " =~ " ${pr} " ]]; then
      notes_normal+=("$pr")
    fi
  done
  logecho

  logecho "Generating release notes..."
  # Bootstrap notes for major (new branch) releases
  if ((FLAGS_full)) || [[ $release_tag =~ ${VER_REGEX[dotzero]} ]]; then
    # Check for draft and use it if available
    logecho "Checking if draft release notes exist for $release_tag..."
    $GHCURL \
     $K8S_GITHUB_RAW_ORG/features/master/$CURRENT_BRANCH/release-notes-draft.md > $tempfile

    if [[ -s $tempfile ]]; then
      logecho "Draft found - using for release notes..."
      cat $tempfile >> $PR_NOTES
      logrun rm -f $tempfile
    else
      logecho "No draft found - creating generic template..."
      cat <<EOF+ >> $PR_NOTES
## Major Themes

* TBD

## Other notable improvements

* TBD

## Known Issues

* TBD

## Provider-specific Notes

* TBD

EOF+
    fi

    # Aggregate all previous releases in series
    echo
    echo "### Previous Releases Included in $release_tag"
    while read anchor; do
      echo "$anchor"
    done< <(git show master:CHANGELOG.md | egrep "^- \[${release_tag}-") \
     >> $PR_NOTES
  else
    echo "## Changelog since $start_tag" >> $PR_NOTES
    echo >> $PR_NOTES

    if [[ -n "${notes_experimental[*]}" ]]; then
      echo "### Experimental Features" >> $PR_NOTES
      echo >> $PR_NOTES
      extract_pr_title "${notes_experimental[*]}" >> $PR_NOTES \
       || common::exit 1 "$FAILED: github rate limiting."
      echo >> $PR_NOTES
    fi

    if [[ -n "${notes_action[*]}" ]]; then
      echo "### Action Required" >> $PR_NOTES
      echo >> $PR_NOTES
      extract_pr_title "${notes_action[*]}" >> $PR_NOTES \
       || common::exit 1 "$FAILED: github rate limiting."
      echo >> $PR_NOTES
    fi

    if [[ -n "${notes_normal[*]}" ]]; then
      echo "### Other notable changes" >> $PR_NOTES
      echo >> $PR_NOTES
      extract_pr_title "${notes_normal[*]}" >> $PR_NOTES \
       || common::exit 1 "$FAILED: github rate limiting."
    fi

    if [[ -z "${notes_normal[*]}" && -z "${notes_action[*]}" &&
          -z "${notes_experimental[*]}" ]]; then
      logecho
      logecho "**No notable changes for this release**" >> $PR_NOTES
      logecho
    fi
  fi

  echo >> $PR_NOTES

  logecho "Preparing layout..."
  create_body ${FLAGS_release_tars:-""} $start_tag ${release_tag:-HEAD} \
   > $RELEASE_NOTES_MD

  if ((FLAGS_preview)); then
    # Pending PRS
    logecho "Adding pending PR status..."
    (
    echo "-------"
    echo "## PENDING PRs on the $CURRENT_BRANCH branch"
    gitlib::pending_prs $CURRENT_BRANCH
    ) >> $RELEASE_NOTES_MD
  fi

  if ((FLAGS_htmlize_md)); then
    # Make users and PRs linkable
    # Also, expand anchors (needed for email announce())
    sed -i -e "s,#\([0-9]\{5\,\}\),[#\1]($K8S_GITHUB_URL/pull/\1),g" \
        -e "s,\(#v[0-9]\{3\}-\),$K8S_GITHUB_URL/blob/master/CHANGELOG.md\1,g" \
        -e "s,@\([a-zA-Z0-9-]*\),[@\1](https://github.com/\1),g" \
     $RELEASE_NOTES_MD
  fi

  if ((FLAGS_preview)); then
    # We do this after htmlizing because we don't want to update the
    # issues in the block of this section
    logecho "Adding CI job status (this may take a while)..."
    ci_job_status >> $RELEASE_NOTES_MD
  fi

  if [[ -n "$RELEASE_NOTES_HTML" ]]; then
    echo "<style type=text/css>" \
         "table,th,tr,td {border: 1px solid gray;" \
         "border-collapse: collapse;padding: 5px;}" \
         "</style>" > $tempcss

    pandoc -H $tempcss --from markdown_github --to html \
     $RELEASE_NOTES_MD > $RELEASE_NOTES_HTML

    # Remove temp file
    logrun rm -f $tempcss
  fi
}

##############################################################################
# CONSTANTS
##############################################################################
CURRENT_BRANCH=${FLAGS_branch:-$(gitlib::current_branch)} \
 || common::exit 1

PR_NOTES=/tmp/$PROG-$CURRENT_BRANCH-prnotes
# Initialize new PR_NOTES for session
>$PR_NOTES
RELEASE_NOTES_MD=$(common::absolute_path \
                   ${FLAGS_markdown_file:-/tmp/$PROG-$CURRENT_BRANCH.md})
RELEASE_NOTES_HTML=$(common::absolute_path $FLAGS_html_file)
ANNOUNCEMENT_TEXT=/tmp/$PROG-announcement

###############################################################################
# MAIN
###############################################################################
# Initialize and save up to 10 (rotated logs)
MYLOG=/tmp/$PROG.log
common::logfileinit $MYLOG 10

# BEGIN script
common::timestamp begin

# Check token
gitlib::github_api_token

# Check for packages
common::check_packages jq pandoc

# Build LAST_RELEASE dictionary
gitlib::last_releases

generate_notes || common::exit 1

logecho
if ((FLAGS_quiet)); then
  logecho -n "Notes written to $RELEASE_NOTES_MD"
  if [[ -f $RELEASE_NOTES_HTML ]]; then
    logecho " and $RELEASE_NOTES_HTML"
  else
    logecho
  fi
else
  logecho -r "$HR"
  cat $RELEASE_NOTES_MD
  logecho -r "$HR"
fi

common::timestamp end