diff --git a/bin/generate-issues b/bin/generate-issues new file mode 100755 index 0000000..726e77a --- /dev/null +++ b/bin/generate-issues @@ -0,0 +1,444 @@ +#!/usr/bin/env ruby +# Generate GitHub Issues for tasks and deliverables +# - Each work package gets a WPX/README.md file +# - Each task gets an Issue +# - Each deliverable gets a Milestone +# - Each deliverable gets an Issue associated with its Milestone +# - Each deliverable Issue gets 'deliverable', 'WPX' tags +# - Each task Issue gets 'task', 'WPX' tags + +# Run me from the proposal dir containing proposal.pdata: +# cd ~/OpenDreamKit/Proposal +# ~/path/to/LaTeX-Proposal/bin/generate-issues + +require 'date' +require 'Octokit' +require 'netrc' + +# constants for running the script are for OpenDreamKit +# update these for your own proposal + +START_DATE = Date::new(2015, 9, 1) # start date of the project +HOMEPAGE = "http://opendreamkit.org" +PROPOSAL_URL = "https://github.com/OpenDreamKit/OpenDreamKit/tree/master/Proposal/proposal-www.pdf" +PROJECT = "OpenDreamKit" +REPO = "minrk/odktest" + +# throttle github creation requests to 5 Hz to avoid getting flagged for abuse +THROTTLE_SECONDS = 5 + +NATURES = { + 'R' => 'Report', + 'DEM' => 'Demonstrator', + 'DEC' => 'Websites, patents filing, press & media actions, videos, etc.', + 'OTHER' => 'Other', +} + +DISSEMINATIONS = { + 'PU' => 'Public', + 'CO' => 'Confidential', + 'CI' => 'Classified', +} + +SITES = { + 'PS' => 'Université Paris-Sud', + 'LL' => 'Logilab', + 'UV' => 'Université de Versailles Saint-Quentin', + 'UJF' => 'Université Joseph Fourier', + 'UB' => 'CNRS', + 'UO' => 'University of Oxford', + 'USH' => 'University of Sheffield', + 'USO' => 'University of Southampton', + 'SA' => 'University of St Andrews', + 'UW' => 'University of Warwick', + 'JU' => 'Jacobs University Bremen', + 'UK' => 'University of Kaiserslautern', + 'US' => 'University of Silesia', + 'ZH' => 'Universität Zürich', + 'SR' => 'Simula Research Laboratory', +} + +REVERSE_SITES = {} +SITES.each_pair do |key, name| + REVERSE_SITES[name] = key +end + +#------------------- Parsing proposal.pdata --------------------- + +def split_line(line) + # super primitive state-machine line split (not going to regex this) + parts = [] + level = 0 + buffer = [] + line.each_char do |c| + case c + when '{' + if level > 0 + buffer.push c + end + level += 1 + when '}' + level -= 1 + if level > 0 + buffer.push c + end + if level == 0 + parts.push(scrub_tex(buffer.join(''))) + buffer = [] + end + else + if level > 0 + buffer.push c + end + end + end + return parts +end + + +def scrub_tex(text) + # scrub some latex markup from text + text.gsub!(/\\\w+/, '') + text.gsub!(/[{}]/, '') + text.strip.split.join(' ') +end + + +def transform_value(key, value) + case key + when 'lead' + return SITES[value] + when 'partners' + return value.split(',').map { |v| SITES[v] } + when 'dissem' + return DISSEMINATIONS[value] + when 'nature' + return NATURES[value] + when 'month' + return value.to_i + when 'delivs' + return value.split(',') + else + return scrub_tex(value) + end +end + + +def load_pdata(proposal_dir) + pdata = File.join(proposal_dir, 'proposal.pdata') + deliv_data = File.join(proposal_dir, 'proposal.deliverables') + + workpackages = {} # mapping of workpackage id => wp info + + File.readlines(pdata).each do |line| + key, *args = split_line line + + case key + when 'wp' + name, key, value = args + value = transform_value(key, value) + if not workpackages.include? name + workpackages[name] = { + "tasks" => {}, + "unknown-task" => nil, + "deliverables" => [], + } + end + wp = workpackages[name] + wp[key] = value + + when 'task' + name, key, value = args + value = transform_value(key, value) + # find my workpackage + if name.index('@') + wpkey, short_name = name.split('@') + if short_name.match(/task\d+/) + workpackages[wpkey]['unknown-task'] = short_name + name = short_name + end + else + wpkey = workpackages.keys.select { |wpkey| name.start_with? wpkey }.first + end + # handle workpackage@taskNN weirdness + wp = workpackages[wpkey] + tasks = wp['tasks'] + if not wp['unknown-task'].nil? and wp['unknown-task'] != name + tasks[name] = tasks.delete(wp['unknown-task']) + wp['unknown-task'] = nil + end + if not tasks.include? name + tasks[name] = {} + end + tasks[name][key] = value + + else + # DEBUG: + # puts " Ignored: #{args}" + end + end + + # get deliverable data from proposal.deliverables + File.readlines(deliv_data).each do |line| + args = split_line line + month = args[0].to_i + wpid = scrub_tex(args[7]) + deliverable = { + "month" => month, + # deliverables[deliv_id]['month'] = month + # due date is last day of the given month, so subtract one day + "due_date" => (START_DATE >> month) - 1, + "label" => scrub_tex(args[2]), + "deliv_id" => args[3], + "dissem" => transform_value('dissem', args[4]), + "nature" => transform_value('nature', args[5]), + "title" => scrub_tex(args[6]), + "lead" => SITES[scrub_tex(args[8])], + } + wp = workpackages.values.find {|wp| wp['label'] == wpid} + wp['deliverables'].push(deliverable) + wp['deliverables'].sort_by! {|d| d['label']} + end + + return workpackages.values.sort_by {|wp| wp['label']} +end + + +#--------------- GitHub-related --------------------- + +def check_token + # get GitHub auth token, creating one if we don't find it. + rc = Netrc.read Netrc.default_path + if not rc['api.github.com'].nil? + return + end + puts "We need your password to generate an OAuth token. The password will not be stored." + username = ask "Username: " + password = ask("Password: ") { |q| q.echo = '*' } + client = Octokit::Client.new( + :login => username, + :password => password, + ) + reply = client.create_authorization( + :scopes => ["public_repo"], + :note => "Issue Migration", + ) + token = reply.token + rc['api.github.com'] = username, token + rc.save +end + +$cache = { + 'issues' => {}, + 'milestones' => {}, +} + +def get_issues(github, repo) + # get issues for a repo (cached) + cache = $cache['issues'] + if not cache.include? repo + cache[repo] = github.issues(repo) + end + return cache[repo] +end + +def get_milestones(github, repo) + # get issues for a repo (cached) + cache = $cache['milestones'] + if not cache.include? repo + cache[repo] = github.list_milestones(repo) + end + return cache[repo] +end + +# Templates for making readmes, issues + +README_TPL = <<-END +# %{title} + +Lead institution: %{lead} + +See page %{page} of the [proposal](#{PROPOSAL_URL}) for the full description. +END + +TASK_TPL = <<-END +Work Package %{wptitle} + +Lead Institution: %{lead} + +Partners: %{partners} + +Work phases: %{wphases} +END + +DELIV_TPL = <<-END +Work Package %{wptitle} + +Lead Institution: %{lead} + +Due: %{date} (month %{month}) + +Nature: %{nature} +END + +DELIV_MILESTONE_TPL = "# %{title}\n\n#{DELIV_TPL}" + + +def make_task_issue(github, repo, task, workpackage, options) + title = "#{task['label']}: #{task['title']}" + issues = get_issues(github, repo) + issue = issues.find { |i| i.title.start_with?(task['label'] + ':') } + if issue.nil? + body = TASK_TPL % { + wptitle: "#{workpackage['label']}: #{workpackage['title']}", + lead: task['lead'], + wphases: task['wphases'], + partners: (task['partners'] or ['None']).join(', ') + } + puts "\n\nMaking Issue on #{repo}: #{title}" + puts body + + github.create_issue(repo, title, body, options) + # throttle creation calls to avoid flags for abuse + sleep THROTTLE_SECONDS + else + puts "Found Issue #{repo}##{issue.number}: #{issue.title}" + existing_labels = issue.labels.map { |label| label['name']} + missing_labels = options[:labels].reject { |label| existing_labels.include? label } + if not missing_labels.empty? + puts "Updating milestone, labels on #{repo}##{issue.number}" + github.update_issue(repo, issue.number, options) + sleep THROTTLE_SECONDS + end + end +end + + +def make_deliverable_issue(github, repo, deliverable, workpackage, options) + title = "#{deliverable['label']}: #{deliverable['title']}" + issues = get_issues(github, repo) + issue = issues.find { |i| i.title.start_with?(deliverable['label'] + ':') } + if issue.nil? + body = DELIV_TPL % { + wptitle: "#{workpackage['label']}: #{workpackage['title']}", + lead: deliverable['lead'], + date: deliverable['due_date'], + month: deliverable['month'], + nature: deliverable['nature'], + } + puts "\n\nMaking Issue on #{repo}: #{title}" + puts body + github.create_issue(repo, title, body, options) + # throttle creation calls to avoid flags for abuse + sleep THROTTLE_SECONDS + else + puts "Found Issue #{repo}##{issue.number}: #{issue.title}" + existing_labels = issue.labels.map { |label| label['name']} + missing_labels = options[:labels].reject { |label| existing_labels.include? label } + if issue.milestone.nil? or not missing_labels.empty? + puts "Updating milestone, labels on #{repo}##{issue.number}" + github.update_issue(repo, issue.number, options) + sleep THROTTLE_SECONDS + end + end +end + + +def make_deliverable_milestone(github, repo, deliverable, workpackage) + title = deliverable['label'] + + milestone = get_milestones(github, repo).find { |ms| ms.title == title } + if milestone.nil? + puts "Making milestone on #{repo}: #{title}" + body = DELIV_MILESTONE_TPL % { + wptitle: "#{workpackage['label']}: #{workpackage['title']}", + title: deliverable['title'], + lead: deliverable['lead'], + date: deliverable['due_date'], + month: deliverable['month'], + nature: deliverable['nature'], + } + + milestone = github.create_milestone(repo, title, + :due_on => deliverable['due_date'], + :description => body, + ) + # throttle creation calls to avoid flags for abuse + sleep THROTTLE_SECONDS + else + puts "Milestone #{repo}@#{title} exists" + end + return milestone.number +end + + +def populate_workpackage(github, repo, workpackage) + # populate issues for a given workpackage + label = workpackage['label'] + + readme_path = "#{label}/README.md" + begin + github.contents(repo, :path => readme_path) + rescue Octokit::NotFound + readme = README_TPL % { + title: "#{workpackage['label']}: #{workpackage['title']}", + lead: workpackage['lead'], + page: workpackage['page'], + } + + puts "Creating readme at #{repo}/#{readme_path}" + puts readme + github.create_contents(repo, readme_path, "Creating #{readme_path}", readme) + # throttle creation calls to avoid flags for abuse + sleep THROTTLE_SECONDS + end + + workpackage['tasks'].each_value do |task| + org_labels = [REVERSE_SITES[task['lead']]] + \ + (task['partners'] or []).map {|site| REVERSE_SITES[site]} + + make_task_issue(github, repo, task, workpackage, + :labels => [ + 'task', + workpackage['label'], + ] + org_labels + ) + end + workpackage['deliverables'].each do |deliverable| + milestone = make_deliverable_milestone(github, repo, deliverable, workpackage) + make_deliverable_issue(github, repo, deliverable, workpackage, { + :milestone => milestone, + :labels => [ + 'deliverable', + workpackage['label'], + REVERSE_SITES[deliverable['lead']] + ] + }) + end +end + + +def main(proposal_dir) + # run the whole thing + + # verify that there's a GitHub token in .netrc + check_token + + # create client + github = Octokit::Client.new(:netrc => true) + github.auto_paginate = true + + load_pdata(proposal_dir).each do |wp| + populate_workpackage(github, REPO, wp) + end +end + +if __FILE__ == $0 + puts ARGV + if ARGV.length > 1 + proposal_dir = ARGV[1] + else + proposal_dir = '.' + end + main(proposal_dir) +end \ No newline at end of file