Merge pull request #10 from minrk/generate-issues
add bin/generate-issues, thanks a lot
This commit is contained in:
Michael Kohlhase
commit
a66d7bac1b
428
bin/generate-issues
Executable file
428
bin/generate-issues
Executable file
@ -0,0 +1,428 @@
|
||||
#!/usr/bin/env ruby
|
||||
# Generate GitHub Issues for tasks and deliverables
|
||||
# - Each work package gets a WPX/README.md file
|
||||
# - Each task gets an Issue
|
||||
# - Each deliverable gets a Milestone
|
||||
# - Each deliverable gets an Issue associated with its Milestone
|
||||
# - Each deliverable Issue gets 'deliverable', 'WPX' tags
|
||||
# - Each task Issue gets 'task', 'WPX' tags
|
||||
|
||||
# Run me from the proposal dir containing proposal.pdata:
|
||||
# cd ~/OpenDreamKit/Proposal
|
||||
# ~/path/to/LaTeX-Proposal/bin/generate-issues
|
||||
|
||||
require 'date'
|
||||
require 'Octokit'
|
||||
require 'netrc'
|
||||
|
||||
# constants for running the script are for OpenDreamKit
|
||||
# update these for your own proposal
|
||||
|
||||
START_DATE = Date::new(2015, 9, 1) # start date of the project
|
||||
HOMEPAGE = "http://opendreamkit.org"
|
||||
PROPOSAL_URL = "https://github.com/OpenDreamKit/OpenDreamKit/tree/master/Proposal/proposal-www.pdf"
|
||||
PROJECT = "OpenDreamKit"
|
||||
REPO = "minrk/odktest"
|
||||
|
||||
# throttle github creation requests to 5 Hz to avoid getting flagged for abuse
|
||||
THROTTLE_SECONDS = 5
|
||||
|
||||
NATURES = {
|
||||
'R' => 'Report',
|
||||
'DEM' => 'Demonstrator',
|
||||
'DEC' => 'Websites, patents filing, press & media actions, videos, etc.',
|
||||
'OTHER' => 'Other',
|
||||
}
|
||||
|
||||
DISSEMINATIONS = {
|
||||
'PU' => 'Public',
|
||||
'CO' => 'Confidential',
|
||||
'CI' => 'Classified',
|
||||
}
|
||||
|
||||
SITES = {
|
||||
'PS' => 'Université Paris-Sud',
|
||||
'LL' => 'Logilab',
|
||||
'UV' => 'Université de Versailles Saint-Quentin',
|
||||
'UJF' => 'Université Joseph Fourier',
|
||||
'UB' => 'CNRS',
|
||||
'UO' => 'University of Oxford',
|
||||
'USH' => 'University of Sheffield',
|
||||
'USO' => 'University of Southampton',
|
||||
'SA' => 'University of St Andrews',
|
||||
'UW' => 'University of Warwick',
|
||||
'JU' => 'Jacobs University Bremen',
|
||||
'UK' => 'University of Kaiserslautern',
|
||||
'US' => 'University of Silesia',
|
||||
'ZH' => 'Universität Zürich',
|
||||
'SR' => 'Simula Research Laboratory',
|
||||
}
|
||||
|
||||
#------------------- Parsing proposal.pdata ---------------------
|
||||
|
||||
def split_line(line)
|
||||
# super primitive state-machine line split (not going to regex this)
|
||||
parts = []
|
||||
level = 0
|
||||
buffer = []
|
||||
line.each_char do |c|
|
||||
case c
|
||||
when '{'
|
||||
if level > 0
|
||||
buffer.push c
|
||||
end
|
||||
level += 1
|
||||
when '}'
|
||||
level -= 1
|
||||
if level > 0
|
||||
buffer.push c
|
||||
end
|
||||
if level == 0
|
||||
parts.push(scrub_tex(buffer.join('')))
|
||||
buffer = []
|
||||
end
|
||||
else
|
||||
if level > 0
|
||||
buffer.push c
|
||||
end
|
||||
end
|
||||
end
|
||||
return parts
|
||||
end
|
||||
|
||||
|
||||
def scrub_tex(text)
|
||||
# scrub some latex markup from text
|
||||
text.gsub!(/\\\w+/, '')
|
||||
text.gsub!(/[{}]/, '')
|
||||
text.strip.split.join(' ')
|
||||
end
|
||||
|
||||
|
||||
def transform_value(key, value)
|
||||
case key
|
||||
when 'lead'
|
||||
return SITES[value]
|
||||
when 'partners'
|
||||
return value.split(',').map { |v| SITES[v] }
|
||||
when 'dissem'
|
||||
return DISSEMINATIONS[value]
|
||||
when 'nature'
|
||||
return NATURES[value]
|
||||
when 'month'
|
||||
return value.to_i
|
||||
when 'delivs'
|
||||
return value.split(',')
|
||||
else
|
||||
return scrub_tex(value)
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
def load_pdata(proposal_dir)
|
||||
pdata = File.join(proposal_dir, 'proposal.pdata')
|
||||
deliv_data = File.join(proposal_dir, 'proposal.deliverables')
|
||||
|
||||
workpackages = {} # mapping of workpackage id => wp info
|
||||
|
||||
File.readlines(pdata).each do |line|
|
||||
key, *args = split_line line
|
||||
|
||||
case key
|
||||
when 'wp'
|
||||
name, key, value = args
|
||||
value = transform_value(key, value)
|
||||
if not workpackages.include? name
|
||||
workpackages[name] = {
|
||||
"tasks" => {},
|
||||
"unknown-task" => nil,
|
||||
"deliverables" => [],
|
||||
}
|
||||
end
|
||||
wp = workpackages[name]
|
||||
wp[key] = value
|
||||
|
||||
when 'task'
|
||||
name, key, value = args
|
||||
value = transform_value(key, value)
|
||||
# find my workpackage
|
||||
if name.index('@')
|
||||
wpkey, short_name = name.split('@')
|
||||
if short_name.match(/task\d+/)
|
||||
workpackages[wpkey]['unknown-task'] = short_name
|
||||
name = short_name
|
||||
end
|
||||
else
|
||||
wpkey = workpackages.keys.select { |wpkey| name.start_with? wpkey }.first
|
||||
end
|
||||
# handle workpackage@taskNN weirdness
|
||||
wp = workpackages[wpkey]
|
||||
tasks = wp['tasks']
|
||||
if not wp['unknown-task'].nil? and wp['unknown-task'] != name
|
||||
tasks[name] = tasks.delete(wp['unknown-task'])
|
||||
wp['unknown-task'] = nil
|
||||
end
|
||||
if not tasks.include? name
|
||||
tasks[name] = {}
|
||||
end
|
||||
tasks[name][key] = value
|
||||
|
||||
else
|
||||
# DEBUG:
|
||||
# puts " Ignored: #{args}"
|
||||
end
|
||||
end
|
||||
|
||||
# get deliverable data from proposal.deliverables
|
||||
File.readlines(deliv_data).each do |line|
|
||||
args = split_line line
|
||||
month = args[0].to_i
|
||||
wpid = scrub_tex(args[7])
|
||||
deliverable = {
|
||||
"month" => month,
|
||||
# deliverables[deliv_id]['month'] = month
|
||||
# due date is last day of the given month, so subtract one day
|
||||
"due_date" => (START_DATE >> month) - 1,
|
||||
"label" => scrub_tex(args[2]),
|
||||
"deliv_id" => args[3],
|
||||
"dissem" => transform_value('dissem', args[4]),
|
||||
"nature" => transform_value('nature', args[5]),
|
||||
"title" => scrub_tex(args[6]),
|
||||
"lead" => SITES[scrub_tex(args[8])],
|
||||
}
|
||||
wp = workpackages.values.find {|wp| wp['label'] == wpid}
|
||||
wp['deliverables'].push(deliverable)
|
||||
wp['deliverables'].sort_by! {|d| d['label']}
|
||||
end
|
||||
|
||||
return workpackages.values.sort_by {|wp| wp['label']}
|
||||
end
|
||||
|
||||
|
||||
#--------------- GitHub-related ---------------------
|
||||
|
||||
def check_token
|
||||
# get GitHub auth token, creating one if we don't find it.
|
||||
rc = Netrc.read Netrc.default_path
|
||||
if not rc['api.github.com'].nil?
|
||||
return
|
||||
end
|
||||
puts "We need your password to generate an OAuth token. The password will not be stored."
|
||||
username = ask "Username: "
|
||||
password = ask("Password: ") { |q| q.echo = '*' }
|
||||
client = Octokit::Client.new(
|
||||
:login => username,
|
||||
:password => password,
|
||||
)
|
||||
reply = client.create_authorization(
|
||||
:scopes => ["public_repo"],
|
||||
:note => "Issue Migration",
|
||||
)
|
||||
token = reply.token
|
||||
rc['api.github.com'] = username, token
|
||||
rc.save
|
||||
end
|
||||
|
||||
$cache = {
|
||||
'issues' => {},
|
||||
'milestones' => {},
|
||||
}
|
||||
|
||||
def get_issues(github, repo)
|
||||
# get issues for a repo (cached)
|
||||
cache = $cache['issues']
|
||||
if not cache.include? repo
|
||||
cache[repo] = github.issues(repo)
|
||||
end
|
||||
return cache[repo]
|
||||
end
|
||||
|
||||
def get_milestones(github, repo)
|
||||
# get issues for a repo (cached)
|
||||
cache = $cache['milestones']
|
||||
if not cache.include? repo
|
||||
cache[repo] = github.list_milestones(repo)
|
||||
end
|
||||
return cache[repo]
|
||||
end
|
||||
|
||||
# Templates for making readmes, issues
|
||||
|
||||
README_TPL = <<-END
|
||||
# %{title}
|
||||
|
||||
Lead institution: %{lead}
|
||||
|
||||
See page %{page} of the [proposal](#{PROPOSAL_URL}) for the full description.
|
||||
END
|
||||
|
||||
TASK_TPL = <<-END
|
||||
Lead Institution: %{lead}
|
||||
|
||||
Partners: %{partners}
|
||||
|
||||
Work phases: %{wphases}
|
||||
END
|
||||
|
||||
DELIV_TPL = <<-END
|
||||
Lead Institution: %{lead}
|
||||
|
||||
Due: %{date} (month %{month})
|
||||
|
||||
Nature: %{nature}
|
||||
END
|
||||
|
||||
DELIV_MILESTONE_TPL = "# %{title}\n\n#{DELIV_TPL}"
|
||||
|
||||
|
||||
def make_task_issue(github, repo, task, options)
|
||||
title = "#{task['label']}: #{task['title']}"
|
||||
issues = get_issues(github, repo)
|
||||
issue = issues.find { |i| i.title.start_with?(task['label'] + ':') }
|
||||
if issue.nil?
|
||||
body = TASK_TPL % {
|
||||
lead: task['lead'],
|
||||
wphases: task['wphases'],
|
||||
partners: (task['partners'] or ['None']).join(', ')
|
||||
}
|
||||
puts "\n\nMaking Issue on #{repo}: #{title}"
|
||||
puts body
|
||||
|
||||
github.create_issue(repo, title, body, options)
|
||||
# throttle creation calls to avoid flags for abuse
|
||||
sleep THROTTLE_SECONDS
|
||||
else
|
||||
puts "Found Issue #{repo}##{issue.number}: #{issue.title}"
|
||||
existing_labels = issue.labels.map { |label| label['name']}
|
||||
missing_labels = options[:labels].reject { |label| existing_labels.include? label }
|
||||
if not missing_labels.empty?
|
||||
puts "Updating milestone, labels on #{repo}##{issue.number}"
|
||||
github.update_issue(repo, issue.number, options)
|
||||
sleep THROTTLE_SECONDS
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
def make_deliverable_issue(github, repo, deliverable, options)
|
||||
title = "#{deliverable['label']}: #{deliverable['title']}"
|
||||
issues = get_issues(github, repo)
|
||||
issue = issues.find { |i| i.title.start_with?(deliverable['label'] + ':') }
|
||||
if issue.nil?
|
||||
body = DELIV_TPL % {
|
||||
lead: deliverable['lead'],
|
||||
date: deliverable['due_date'],
|
||||
month: deliverable['month'],
|
||||
nature: deliverable['nature'],
|
||||
}
|
||||
puts "\n\nMaking Issue on #{repo}: #{title}"
|
||||
puts body
|
||||
github.create_issue(repo, title, body, options)
|
||||
# throttle creation calls to avoid flags for abuse
|
||||
sleep THROTTLE_SECONDS
|
||||
else
|
||||
puts "Found Issue #{repo}##{issue.number}: #{issue.title}"
|
||||
existing_labels = issue.labels.map { |label| label['name']}
|
||||
missing_labels = options[:labels].reject { |label| existing_labels.include? label }
|
||||
if issue.milestone.nil? or not missing_labels.empty?
|
||||
puts "Updating milestone, labels on #{repo}##{issue.number}"
|
||||
github.update_issue(repo, issue.number, options)
|
||||
sleep THROTTLE_SECONDS
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
def make_deliverable_milestone(github, repo, deliverable)
|
||||
title = deliverable['label']
|
||||
|
||||
milestone = get_milestones(github, repo).find { |ms| ms.title == title }
|
||||
if milestone.nil?
|
||||
puts "Making milestone on #{repo}: #{title}"
|
||||
body = DELIV_MILESTONE_TPL % {
|
||||
title: deliverable['title'],
|
||||
lead: deliverable['lead'],
|
||||
date: deliverable['due_date'],
|
||||
month: deliverable['month'],
|
||||
nature: deliverable['nature'],
|
||||
}
|
||||
|
||||
milestone = github.create_milestone(repo, title,
|
||||
:due_on => deliverable['due_date'],
|
||||
:description => body,
|
||||
)
|
||||
# throttle creation calls to avoid flags for abuse
|
||||
sleep THROTTLE_SECONDS
|
||||
else
|
||||
puts "Milestone #{repo}@#{title} exists"
|
||||
end
|
||||
return milestone.number
|
||||
end
|
||||
|
||||
|
||||
def populate_workpackage(github, repo, workpackage)
|
||||
# populate issues for a given workpackage
|
||||
label = workpackage['label']
|
||||
|
||||
readme_path = "#{label}/README.md"
|
||||
begin
|
||||
github.contents(repo, :path => readme_path)
|
||||
rescue Octokit::NotFound
|
||||
readme = README_TPL % {
|
||||
title: "#{workpackage['label']}: #{workpackage['title']}",
|
||||
lead: workpackage['lead'],
|
||||
page: workpackage['page'],
|
||||
}
|
||||
|
||||
puts "Creating readme at #{repo}/#{readme_path}"
|
||||
puts readme
|
||||
github.create_contents(repo, readme_path, "Creating #{readme_path}", readme)
|
||||
# throttle creation calls to avoid flags for abuse
|
||||
sleep THROTTLE_SECONDS
|
||||
end
|
||||
|
||||
workpackage['tasks'].each_value do |task|
|
||||
make_task_issue(github, repo, task, {
|
||||
:labels => [
|
||||
'task',
|
||||
workpackage['label'],
|
||||
]
|
||||
})
|
||||
end
|
||||
workpackage['deliverables'].each do |deliverable|
|
||||
milestone = make_deliverable_milestone(github, repo, deliverable)
|
||||
make_deliverable_issue(github, repo, deliverable, {
|
||||
:milestone => milestone,
|
||||
:labels => [
|
||||
'deliverable',
|
||||
workpackage['label'],
|
||||
]
|
||||
})
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
def main(proposal_dir)
|
||||
# run the whole thing
|
||||
|
||||
# verify that there's a GitHub token in .netrc
|
||||
check_token
|
||||
|
||||
# create client
|
||||
github = Octokit::Client.new(:netrc => true)
|
||||
github.auto_paginate = true
|
||||
|
||||
load_pdata(proposal_dir).each do |wp|
|
||||
populate_workpackage(github, REPO, wp)
|
||||
end
|
||||
end
|
||||
|
||||
if __FILE__ == $0
|
||||
puts ARGV
|
||||
if ARGV.length > 1
|
||||
proposal_dir = ARGV[1]
|
||||
else
|
||||
proposal_dir = '.'
|
||||
end
|
||||
main(proposal_dir)
|
||||
end
|
||||
Loading…
Reference in New Issue
Block a user