From e3a90f648eafcc7c27ffe4f127f1427e652b1673 Mon Sep 17 00:00:00 2001
From: Min RK <benjaminrk@gmail.com>
Date: Fri, 4 Sep 2015 16:48:30 +0200
Subject: [PATCH 1/9] initial working version

---
 bin/generate-issues | 431 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 431 insertions(+)
 create mode 100755 bin/generate-issues

diff --git a/bin/generate-issues b/bin/generate-issues
new file mode 100755
index 0000000..85deb18
--- /dev/null
+++ b/bin/generate-issues
@@ -0,0 +1,431 @@
+#!/usr/bin/env ruby
+# Generate GitHub repos and issues for the tasks and deliverables
+# - Each task gets an Issue
+# - Each deliverable gets a Milestone
+# - Each deliverable gets an Issue associated with its Milestone
+
+# Run me from the proposal dir containing proposal.pdata
+
+require 'date'
+require 'Octokit'
+require 'netrc'
+
+# constants for running the script
+proposal_dir = '.'
+
+$start_date = Date::new(2015, 9, 1) # start date of the project
+$github_org = 'minrk-odktest'
+$homepage = "http://opendreamkit.org"
+$proposal_url = "https://github.com/OpenDreamKit/OpenDreamKit"
+$project = "OpenDreamKit"
+
+# throttle github creation requests to 5 Hz to avoid getting flagged for abuse
+THROTTLE_SECONDS = 5
+
+NATURES = {
+  'R'     => 'Report',
+  'DEM'   => 'Demonstrator',
+  'DEC'   => 'Websites, patents filing, press & media actions, videos, etc.',
+  'OTHER' => 'Other',
+}
+
+DISSEMINATIONS = {
+  'PU' => 'Public',
+  'CO' => 'Confidential',
+  'CI' => 'Classified',
+}
+
+SITES = {
+  'PS'  => 'Université Paris-Sud',
+  'LL'  => 'Logilab',
+  'UV'  => 'Université de Versailles Saint-Quentin',
+  'UJF' => 'Université Joseph Fourier',
+  'UB'  => 'CNRS',
+  'UO'  => 'University of Oxford',
+  'USH' => 'University of Sheffield',
+  'USO' => 'University of Southampton',
+  'SA'  => 'University of St Andrews',
+  'UW'  => 'University of Warwick',
+  'JU'  => 'Jacobs University Bremen',
+  'UK'  => 'University of Kaiserslautern',
+  'US'  => 'University of Silesia',
+  'ZH'  => 'Universität Zürich',
+  'SR'  => 'Simula Research Laboratory',
+}
+
+def split_line(line)
+  # super primitive state-machine line split (not going to regex this)
+  parts = []
+  level = 0
+  buffer = []
+  line.each_char do |c|
+    case c
+    when '{'
+      if level > 0
+        buffer.push c
+      end
+      level += 1
+    when '}'
+      level -= 1
+      if level > 0
+        buffer.push c
+      end
+      if level == 0
+        parts.push(scrub_tex(buffer.join('')))
+        buffer = []
+      end
+    else
+      if level > 0
+        buffer.push c
+      end
+    end
+  end
+  return parts
+end
+
+def scrub_tex(text)
+  # scrub some latex markup from text
+  text.gsub!(/\\\w+/, '')
+  text.gsub!(/[{}]/, '')
+  text.strip.split.join(' ')
+end
+
+def transform_value(key, value)
+  case key
+  when 'lead'
+    return SITES[value]
+  when 'partners'
+    return value.split(',').map { |v| SITES[v] }
+  when 'dissem'
+    return DISSEMINATIONS[value]
+  when 'nature'
+    return NATURES[value]
+  when 'month'
+    return value.to_i
+  when 'delivs'
+    return value.split(',')
+  else
+    return scrub_tex(value)
+  end
+end
+
+
+def check_token
+  # get GitHub auth token, creating one if we don't find it.
+  rc = Netrc.read Netrc.default_path
+  if not rc['api.github.com'].nil?
+    return
+  end
+  puts "We need your password to generate an OAuth token. The password will not be stored."
+  username = ask "Username: "
+  password = ask("Password:  ") { |q| q.echo = '*' }
+  client = Octokit::Client.new(
+    :login => username,
+    :password => password,
+  )
+  reply = client.create_authorization(
+    :scopes => ["public_repo"],
+    :note => "Issue Migration",
+  )
+  token = reply.token
+  rc['api.github.com'] = username, token
+  rc.save
+end
+
+
+$readme_tpl = <<-END
+
+# %{title}
+
+This is a Work Package for [#{$project}](#{$homepage}).
+
+Lead institution: %{lead}
+
+Page in the [proposal](#{$proposal_url}) for the full description: %{page}
+END
+
+$task_tpl = <<-END
+Lead Institution: %{lead}
+
+Partners: %{partners}
+
+Work phases: %{wphases}
+END
+
+$deliv_tpl = <<-END
+Lead Institution: %{lead}
+
+Due: %{date} (month %{month})
+
+Nature: %{nature}
+END
+
+$deliv_milestone_tpl = "# %{title}\n\n#{$deliv_tpl}"
+
+def make_task_issue(github, repo, task)
+  title = "#{task['label']}: #{task['title']}"
+  issues = get_issues(github, repo)
+  issue = issues.find { |i| i.title.start_with?(task['label'] + ':') }
+  if issue.nil?
+    body = $task_tpl % {
+      lead: task['lead'],
+      wphases: task['wphases'],
+      partners: (task['partners'] or ['None']).join(' ')
+    }
+    puts "\n\nMaking Issue on #{repo}: #{title}"
+    puts body
+  
+    github.create_issue(repo, title, body)
+    # throttle creation calls to avoid flags for abuse
+    sleep THROTTLE_SECONDS
+  else
+    puts "Found Issue #{repo}##{issue.number}: #{issue.title}"
+  end
+end
+
+def make_deliverable_issue(github, repo, deliverable, milestone)
+  title = "#{deliverable['label']}: #{deliverable['title']}"
+  issues = get_issues(github, repo)
+  issue = issues.find { |i| i.title.start_with?(deliverable['label'] + ':') }
+  if issue.nil?
+    body = $deliv_tpl % {
+      lead: deliverable['lead'],
+      date: deliverable['due_date'],
+      month: deliverable['month'],
+      nature: deliverable['nature'],
+    }
+    puts "\n\nMaking Issue on #{repo}: #{title}"
+    puts body
+    github.create_issue(repo, title, body, :milestone => milestone)
+    # throttle creation calls to avoid flags for abuse
+    sleep THROTTLE_SECONDS
+  else
+    puts "Found Issue #{repo}##{issue.number}: #{issue.title}"
+  end
+end
+
+def make_deliverable_milestone(github, repo, deliverable)
+  title = deliverable['label']
+  
+  milestone = get_milestones(github, repo).find { |ms| ms.title == title }
+  if milestone.nil?
+    puts "Making milestone on #{repo}: #{title}"
+    body = $deliv_milestone_tpl % {
+      title: deliverable['title'],
+      lead: deliverable['lead'],
+      date: deliverable['due_date'],
+      month: deliverable['month'],
+      nature: deliverable['nature'],
+    }
+  
+    milestone = github.create_milestone(repo, title,
+        :due_on => deliverable['due_date'],
+        :description => body,
+    )
+    # throttle creation calls to avoid flags for abuse
+    sleep THROTTLE_SECONDS
+  else
+    puts "Milestone #{repo}@#{title} exists"
+  end
+  return milestone.number
+end
+
+def make_repo(github, workpackage)
+  repo_name = workpackage['label']
+  # repo_name = "WP" # single-repo
+  repo = "#{$github_org}/#{repo_name}"
+  # ensure the repo exists:
+  begin
+    github.repository(repo)
+    puts "Found repo: #{repo}"
+  rescue Octokit::NotFound
+    puts "Creating repo: #{repo}"
+    github.create_repository(repo_name,
+      :organization => $github_org,
+      :description => workpackage['title'],
+      :has_downloads => false,
+      :has_wiki => false,
+    )
+    # throttle creation to avoid flags for abuse
+    sleep THROTTLE_SECONDS
+  end
+  
+  begin
+    github.readme(repo)
+  rescue Octokit::NotFound
+    readme = $readme_tpl % {
+      title: "#{workpackage['label']}: #{workpackage['title']}",
+      lead: workpackage['lead'],
+      page: workpackage['page'],
+    }
+    
+    puts "Creating Readme on #{repo}"
+    puts readme
+    github.create_contents(repo, 'README.md', 'Creating README', readme)
+    # throttle creation calls to avoid flags for abuse
+    sleep THROTTLE_SECONDS
+  end
+  
+  workpackage['tasks'].each_value do |task|
+    make_task_issue(github, repo, task)
+  end
+  workpackage['deliverables'].each do |deliverable|
+    milestone = make_deliverable_milestone(github, repo, deliverable)
+    make_deliverable_issue(github, repo, deliverable, milestone)
+  end
+end
+
+$cache = {
+  'issues' => {},
+  'milestones' => {},
+}
+
+def get_issues(github, repo)
+  # get issues for a repo (cached)
+  cache = $cache['issues']
+  if not cache.include? repo
+    cache[repo] = github.issues(repo)
+  end
+  return cache[repo]
+end
+
+def get_milestones(github, repo)
+  # get issues for a repo (cached)
+  cache = $cache['milestones']
+  if not cache.include? repo
+    cache[repo] = github.list_milestones(repo)
+  end
+  return cache[repo]
+end
+
+def load_pdata(proposal_dir)
+  pdata = File.join(proposal_dir, 'proposal.pdata')
+  deliv_data = File.join(proposal_dir, 'proposal.deliverables')
+  
+  workpackages = {} # mapping of workpackage id => wp info
+  deliverables = {} # mapping of deliv id => deliv info
+  milestones = {} # mapping of milestone id => milestone info
+  
+  File.readlines(pdata).each do |line|
+    key, *args = split_line line
+  
+    case key
+    when 'mile'
+      # milestones are unused
+      name, key, value = args
+      value = transform_value(key, value)
+      # puts "     milestone: #{key} => #{value}"
+      if not milestones.include? name
+        milestones[name] = {}
+      end
+      milestones[name][key] = value
+      if key == 'delivs'
+        value.each do |deliv|
+          if not deliverables.include? deliv
+            deliverables[deliv] = {}
+          end
+          deliverables[deliv]['milestone'] = name
+        end
+      end
+
+    when 'wp'
+      name, key, value = args
+      value = transform_value(key, value)
+      if not workpackages.include? name
+        workpackages[name] = {
+          "tasks" => {},
+          "unknown-task" => nil,
+          "deliverables" => [],
+        }
+      end
+      wp = workpackages[name]
+      wp[key] = value
+
+    when 'task'
+      name, key, value = args
+      value = transform_value(key, value)
+      # find my workpackage
+      if name.index('@')
+        wpkey, short_name = name.split('@')
+        if short_name.match(/task\d+/)
+          workpackages[wpkey]['unknown-task'] = short_name
+          name = short_name
+        end
+      else
+        wpkey = workpackages.keys.select { |wpkey| name.start_with? wpkey }.first
+      end
+      # handle workpackage@taskNN weirdness
+      wp = workpackages[wpkey]
+      tasks = wp['tasks']
+      if not wp['unknown-task'].nil? and wp['unknown-task'] != name
+        tasks[name] = tasks.delete(wp['unknown-task'])
+        wp['unknown-task'] = nil
+      end
+      if not tasks.include? name
+        tasks[name] = {}
+      end
+      tasks[name][key] = value
+
+    when 'deliv'
+      # get deliverable data from proposal.deliverables
+      next
+      name, key, value = args
+      value = transform_value(key, value)
+    
+    
+      if not deliverables.include? name
+        deliverables[name] = {}
+      end
+      deliverables[name][key] = value
+    
+      # find my workpackage
+      if name.index('@')
+        wpkey = name.split('@').first
+      else
+        wpkey = workpackages.keys.select { |wpkey| name.start_with? wpkey }.first
+      end
+      wp = workpackages[wpkey]
+      if not wp['deliverables'].include? name
+        wp['deliverables'][name] = deliverables[name]
+      end
+    else
+      puts "    Ignored: #{args}"
+    end
+  end
+  
+  # get deliverable data from proposal.deliverables
+  File.readlines(deliv_data).each do |line|
+    args = split_line line
+    month = args[0].to_i
+    wpid = scrub_tex(args[7])
+    deliverable = {
+      "month" => month,
+      # deliverables[deliv_id]['month'] = month
+      # due date is last day of the given month, so subtract one day
+      "due_date" => ($start_date >> month) - 1,
+      "label" => scrub_tex(args[2]),
+      "deliv_id" => args[3],
+      "dissem" => transform_value('dissem', args[4]),
+      "nature" => transform_value('nature', args[5]),
+      "title" => scrub_tex(args[6]),
+      "lead" => SITES[scrub_tex(args[8])],
+    }
+    wp = workpackages.values.find {|wp| wp['label'] == wpid}
+    wp['deliverables'].push(deliverable)
+  end
+  
+  return workpackages
+end
+
+
+# verify that there's a GitHub token in .netrc
+check_token
+
+# create client
+github = Octokit::Client.new(:netrc => true)
+github.auto_paginate = true
+
+load_pdata(proposal_dir).each_value do |wp|
+  # puts JSON.pretty_generate(wp)
+  make_repo(github, wp)
+end

From 3d8c47d408fde90215f02424ffc651469f638572 Mon Sep 17 00:00:00 2001
From: Min RK <benjaminrk@gmail.com>
Date: Fri, 4 Sep 2015 17:02:29 +0200
Subject: [PATCH 2/9] move to single-repo model

READMEs go in folders
---
 bin/generate-issues | 88 ++++++++-------------------------------------
 1 file changed, 14 insertions(+), 74 deletions(-)

diff --git a/bin/generate-issues b/bin/generate-issues
index 85deb18..4b4c01e 100755
--- a/bin/generate-issues
+++ b/bin/generate-issues
@@ -14,10 +14,10 @@ require 'netrc'
 proposal_dir = '.'
 
 $start_date = Date::new(2015, 9, 1) # start date of the project
-$github_org = 'minrk-odktest'
 $homepage = "http://opendreamkit.org"
 $proposal_url = "https://github.com/OpenDreamKit/OpenDreamKit"
 $project = "OpenDreamKit"
+$repo = "minrk/odktest"
 
 # throttle github creation requests to 5 Hz to avoid getting flagged for abuse
 THROTTLE_SECONDS = 5
@@ -134,14 +134,11 @@ end
 
 
 $readme_tpl = <<-END
-
 # %{title}
 
-This is a Work Package for [#{$project}](#{$homepage}).
-
 Lead institution: %{lead}
 
-Page in the [proposal](#{$proposal_url}) for the full description: %{page}
+See page %{page} of the [proposal](#{$proposal_url}) for the full description.
 END
 
 $task_tpl = <<-END
@@ -230,28 +227,13 @@ def make_deliverable_milestone(github, repo, deliverable)
   return milestone.number
 end
 
-def make_repo(github, workpackage)
-  repo_name = workpackage['label']
-  # repo_name = "WP" # single-repo
-  repo = "#{$github_org}/#{repo_name}"
-  # ensure the repo exists:
+def populate_workpackage(github, repo, workpackage)
+  # populate issues for a given workpackage
+  label = workpackage['label']
+
+  readme_path = "#{label}/README.md"
   begin
-    github.repository(repo)
-    puts "Found repo: #{repo}"
-  rescue Octokit::NotFound
-    puts "Creating repo: #{repo}"
-    github.create_repository(repo_name,
-      :organization => $github_org,
-      :description => workpackage['title'],
-      :has_downloads => false,
-      :has_wiki => false,
-    )
-    # throttle creation to avoid flags for abuse
-    sleep THROTTLE_SECONDS
-  end
-  
-  begin
-    github.readme(repo)
+    github.contents(repo, :path => readme_path)
   rescue Octokit::NotFound
     readme = $readme_tpl % {
       title: "#{workpackage['label']}: #{workpackage['title']}",
@@ -259,9 +241,9 @@ def make_repo(github, workpackage)
       page: workpackage['page'],
     }
     
-    puts "Creating Readme on #{repo}"
+    puts "Creating readme at #{repo}/#{readme_path}"
     puts readme
-    github.create_contents(repo, 'README.md', 'Creating README', readme)
+    github.create_contents(repo, readme_path, "Creating #{readme_path}", readme)
     # throttle creation calls to avoid flags for abuse
     sleep THROTTLE_SECONDS
   end
@@ -303,31 +285,11 @@ def load_pdata(proposal_dir)
   deliv_data = File.join(proposal_dir, 'proposal.deliverables')
   
   workpackages = {} # mapping of workpackage id => wp info
-  deliverables = {} # mapping of deliv id => deliv info
-  milestones = {} # mapping of milestone id => milestone info
   
   File.readlines(pdata).each do |line|
     key, *args = split_line line
   
     case key
-    when 'mile'
-      # milestones are unused
-      name, key, value = args
-      value = transform_value(key, value)
-      # puts "     milestone: #{key} => #{value}"
-      if not milestones.include? name
-        milestones[name] = {}
-      end
-      milestones[name][key] = value
-      if key == 'delivs'
-        value.each do |deliv|
-          if not deliverables.include? deliv
-            deliverables[deliv] = {}
-          end
-          deliverables[deliv]['milestone'] = name
-        end
-      end
-
     when 'wp'
       name, key, value = args
       value = transform_value(key, value)
@@ -366,28 +328,6 @@ def load_pdata(proposal_dir)
       end
       tasks[name][key] = value
 
-    when 'deliv'
-      # get deliverable data from proposal.deliverables
-      next
-      name, key, value = args
-      value = transform_value(key, value)
-    
-    
-      if not deliverables.include? name
-        deliverables[name] = {}
-      end
-      deliverables[name][key] = value
-    
-      # find my workpackage
-      if name.index('@')
-        wpkey = name.split('@').first
-      else
-        wpkey = workpackages.keys.select { |wpkey| name.start_with? wpkey }.first
-      end
-      wp = workpackages[wpkey]
-      if not wp['deliverables'].include? name
-        wp['deliverables'][name] = deliverables[name]
-      end
     else
       puts "    Ignored: #{args}"
     end
@@ -412,9 +352,10 @@ def load_pdata(proposal_dir)
     }
     wp = workpackages.values.find {|wp| wp['label'] == wpid}
     wp['deliverables'].push(deliverable)
+    wp['deliverables'].sort_by! {|d| d['label']}
   end
   
-  return workpackages
+  return workpackages.values.sort_by {|wp| wp['label']}
 end
 
 
@@ -425,7 +366,6 @@ check_token
 github = Octokit::Client.new(:netrc => true)
 github.auto_paginate = true
 
-load_pdata(proposal_dir).each_value do |wp|
-  # puts JSON.pretty_generate(wp)
-  make_repo(github, wp)
+load_pdata(proposal_dir).each do |wp|
+  populate_workpackage(github, $repo, wp)
 end

From 5b1ce461af53052deaf9f8fb77d55b04ea44cad4 Mon Sep 17 00:00:00 2001
From: Min RK <benjaminrk@gmail.com>
Date: Fri, 4 Sep 2015 17:16:08 +0200
Subject: [PATCH 3/9] update proposal URL

---
 bin/generate-issues | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/generate-issues b/bin/generate-issues
index 4b4c01e..680a013 100755
--- a/bin/generate-issues
+++ b/bin/generate-issues
@@ -15,7 +15,7 @@ proposal_dir = '.'
 
 $start_date = Date::new(2015, 9, 1) # start date of the project
 $homepage = "http://opendreamkit.org"
-$proposal_url = "https://github.com/OpenDreamKit/OpenDreamKit"
+$proposal_url = "https://github.com/OpenDreamKit/OpenDreamKit/tree/master/Proposal/proposal-www.pdf"
 $project = "OpenDreamKit"
 $repo = "minrk/odktest"
 

From 0526cdee5677ba40489da08cf17b8c88b1804965 Mon Sep 17 00:00:00 2001
From: Min RK <benjaminrk@gmail.com>
Date: Fri, 4 Sep 2015 17:17:10 +0200
Subject: [PATCH 4/9] add WPX and deliverable/task labels to issues

---
 bin/generate-issues | 23 +++++++++++++++++------
 1 file changed, 17 insertions(+), 6 deletions(-)

diff --git a/bin/generate-issues b/bin/generate-issues
index 680a013..8b96043 100755
--- a/bin/generate-issues
+++ b/bin/generate-issues
@@ -159,7 +159,7 @@ END
 
 $deliv_milestone_tpl = "# %{title}\n\n#{$deliv_tpl}"
 
-def make_task_issue(github, repo, task)
+def make_task_issue(github, repo, task, options)
   title = "#{task['label']}: #{task['title']}"
   issues = get_issues(github, repo)
   issue = issues.find { |i| i.title.start_with?(task['label'] + ':') }
@@ -172,7 +172,7 @@ def make_task_issue(github, repo, task)
     puts "\n\nMaking Issue on #{repo}: #{title}"
     puts body
   
-    github.create_issue(repo, title, body)
+    github.create_issue(repo, title, body, options)
     # throttle creation calls to avoid flags for abuse
     sleep THROTTLE_SECONDS
   else
@@ -180,7 +180,7 @@ def make_task_issue(github, repo, task)
   end
 end
 
-def make_deliverable_issue(github, repo, deliverable, milestone)
+def make_deliverable_issue(github, repo, deliverable, options)
   title = "#{deliverable['label']}: #{deliverable['title']}"
   issues = get_issues(github, repo)
   issue = issues.find { |i| i.title.start_with?(deliverable['label'] + ':') }
@@ -193,7 +193,7 @@ def make_deliverable_issue(github, repo, deliverable, milestone)
     }
     puts "\n\nMaking Issue on #{repo}: #{title}"
     puts body
-    github.create_issue(repo, title, body, :milestone => milestone)
+    github.create_issue(repo, title, body, options)
     # throttle creation calls to avoid flags for abuse
     sleep THROTTLE_SECONDS
   else
@@ -249,11 +249,22 @@ def populate_workpackage(github, repo, workpackage)
   end
   
   workpackage['tasks'].each_value do |task|
-    make_task_issue(github, repo, task)
+    make_task_issue(github, repo, task, {
+      :labels => [
+        'task',
+        workpackage['label'],
+      ].join(',')
+    })
   end
   workpackage['deliverables'].each do |deliverable|
     milestone = make_deliverable_milestone(github, repo, deliverable)
-    make_deliverable_issue(github, repo, deliverable, milestone)
+    make_deliverable_issue(github, repo, deliverable, {
+      :milestone => milestone,
+      :labels => [
+        'deliverable',
+        workpackage['label'],
+      ].join(',')
+    })
   end
 end
 

From 3be074051068459c940f10e3bd7e56754c8b1bd3 Mon Sep 17 00:00:00 2001
From: Min RK <benjaminrk@gmail.com>
Date: Fri, 4 Sep 2015 17:32:20 +0200
Subject: [PATCH 5/9] update labels on existing issues

in case the issue has been created prior to the label specification
---
 bin/generate-issues | 20 +++++++++++++++++---
 1 file changed, 17 insertions(+), 3 deletions(-)

diff --git a/bin/generate-issues b/bin/generate-issues
index 8b96043..5cd7e98 100755
--- a/bin/generate-issues
+++ b/bin/generate-issues
@@ -177,6 +177,13 @@ def make_task_issue(github, repo, task, options)
     sleep THROTTLE_SECONDS
   else
     puts "Found Issue #{repo}##{issue.number}: #{issue.title}"
+    existing_labels = issue.labels.map { |label| label['name']}
+    missing_labels = options[:labels].reject { |label| existing_labels.include? label }
+    if not missing_labels.empty?
+      puts "Updating milestone, labels on #{repo}##{issue.number}"
+      github.update_issue(repo, issue.number, options)
+      sleep THROTTLE_SECONDS
+    end
   end
 end
 
@@ -198,6 +205,13 @@ def make_deliverable_issue(github, repo, deliverable, options)
     sleep THROTTLE_SECONDS
   else
     puts "Found Issue #{repo}##{issue.number}: #{issue.title}"
+    existing_labels = issue.labels.map { |label| label['name']}
+    missing_labels = options[:labels].reject { |label| existing_labels.include? label }
+    if issue.milestone.nil? or not missing_labels.empty?
+      puts "Updating milestone, labels on #{repo}##{issue.number}"
+      github.update_issue(repo, issue.number, options)
+      sleep THROTTLE_SECONDS
+    end
   end
 end
 
@@ -253,7 +267,7 @@ def populate_workpackage(github, repo, workpackage)
       :labels => [
         'task',
         workpackage['label'],
-      ].join(',')
+      ]
     })
   end
   workpackage['deliverables'].each do |deliverable|
@@ -263,7 +277,7 @@ def populate_workpackage(github, repo, workpackage)
       :labels => [
         'deliverable',
         workpackage['label'],
-      ].join(',')
+      ]
     })
   end
 end
@@ -340,7 +354,7 @@ def load_pdata(proposal_dir)
       tasks[name][key] = value
 
     else
-      puts "    Ignored: #{args}"
+      # puts "    Ignored: #{args}"
     end
   end
   

From a0f218291cc798716a045cffba34ef8b5802855d Mon Sep 17 00:00:00 2001
From: Min RK <benjaminrk@gmail.com>
Date: Fri, 4 Sep 2015 17:52:06 +0200
Subject: [PATCH 6/9] cleanup structure a bit

- caps constants
- group related functions
---
 bin/generate-issues | 458 ++++++++++++++++++++++++--------------------
 1 file changed, 245 insertions(+), 213 deletions(-)

diff --git a/bin/generate-issues b/bin/generate-issues
index 5cd7e98..5eaa3e1 100755
--- a/bin/generate-issues
+++ b/bin/generate-issues
@@ -1,23 +1,28 @@
 #!/usr/bin/env ruby
-# Generate GitHub repos and issues for the tasks and deliverables
+# Generate GitHub Issues for tasks and deliverables
+# - Each work package gets a WPX/README.md file
 # - Each task gets an Issue
 # - Each deliverable gets a Milestone
 # - Each deliverable gets an Issue associated with its Milestone
+# - Each deliverable Issue gets 'deliverable', 'WPX' tags
+# - Each task Issue gets 'task', 'WPX' tags
 
-# Run me from the proposal dir containing proposal.pdata
+# Run me from the proposal dir containing proposal.pdata:
+# cd ~/OpenDreamKit/Proposal
+# ~/path/to/LaTeX-Proposal/bin/generate-issues
 
 require 'date'
 require 'Octokit'
 require 'netrc'
 
-# constants for running the script
-proposal_dir = '.'
+# constants for running the script are for OpenDreamKit
+# update these for your own proposal
 
-$start_date = Date::new(2015, 9, 1) # start date of the project
-$homepage = "http://opendreamkit.org"
-$proposal_url = "https://github.com/OpenDreamKit/OpenDreamKit/tree/master/Proposal/proposal-www.pdf"
-$project = "OpenDreamKit"
-$repo = "minrk/odktest"
+START_DATE = Date::new(2015, 9, 1) # start date of the project
+HOMEPAGE = "http://opendreamkit.org"
+PROPOSAL_URL = "https://github.com/OpenDreamKit/OpenDreamKit/tree/master/Proposal/proposal-www.pdf"
+PROJECT = "OpenDreamKit"
+REPO = "minrk/odktest"
 
 # throttle github creation requests to 5 Hz to avoid getting flagged for abuse
 THROTTLE_SECONDS = 5
@@ -53,6 +58,8 @@ SITES = {
   'SR'  => 'Simula Research Laboratory',
 }
 
+#------------------- Parsing proposal.pdata ---------------------
+
 def split_line(line)
   # super primitive state-machine line split (not going to regex this)
   parts = []
@@ -83,6 +90,7 @@ def split_line(line)
   return parts
 end
 
+
 def scrub_tex(text)
   # scrub some latex markup from text
   text.gsub!(/\\\w+/, '')
@@ -90,6 +98,7 @@ def scrub_tex(text)
   text.strip.split.join(' ')
 end
 
+
 def transform_value(key, value)
   case key
   when 'lead'
@@ -110,201 +119,6 @@ def transform_value(key, value)
 end
 
 
-def check_token
-  # get GitHub auth token, creating one if we don't find it.
-  rc = Netrc.read Netrc.default_path
-  if not rc['api.github.com'].nil?
-    return
-  end
-  puts "We need your password to generate an OAuth token. The password will not be stored."
-  username = ask "Username: "
-  password = ask("Password:  ") { |q| q.echo = '*' }
-  client = Octokit::Client.new(
-    :login => username,
-    :password => password,
-  )
-  reply = client.create_authorization(
-    :scopes => ["public_repo"],
-    :note => "Issue Migration",
-  )
-  token = reply.token
-  rc['api.github.com'] = username, token
-  rc.save
-end
-
-
-$readme_tpl = <<-END
-# %{title}
-
-Lead institution: %{lead}
-
-See page %{page} of the [proposal](#{$proposal_url}) for the full description.
-END
-
-$task_tpl = <<-END
-Lead Institution: %{lead}
-
-Partners: %{partners}
-
-Work phases: %{wphases}
-END
-
-$deliv_tpl = <<-END
-Lead Institution: %{lead}
-
-Due: %{date} (month %{month})
-
-Nature: %{nature}
-END
-
-$deliv_milestone_tpl = "# %{title}\n\n#{$deliv_tpl}"
-
-def make_task_issue(github, repo, task, options)
-  title = "#{task['label']}: #{task['title']}"
-  issues = get_issues(github, repo)
-  issue = issues.find { |i| i.title.start_with?(task['label'] + ':') }
-  if issue.nil?
-    body = $task_tpl % {
-      lead: task['lead'],
-      wphases: task['wphases'],
-      partners: (task['partners'] or ['None']).join(' ')
-    }
-    puts "\n\nMaking Issue on #{repo}: #{title}"
-    puts body
-  
-    github.create_issue(repo, title, body, options)
-    # throttle creation calls to avoid flags for abuse
-    sleep THROTTLE_SECONDS
-  else
-    puts "Found Issue #{repo}##{issue.number}: #{issue.title}"
-    existing_labels = issue.labels.map { |label| label['name']}
-    missing_labels = options[:labels].reject { |label| existing_labels.include? label }
-    if not missing_labels.empty?
-      puts "Updating milestone, labels on #{repo}##{issue.number}"
-      github.update_issue(repo, issue.number, options)
-      sleep THROTTLE_SECONDS
-    end
-  end
-end
-
-def make_deliverable_issue(github, repo, deliverable, options)
-  title = "#{deliverable['label']}: #{deliverable['title']}"
-  issues = get_issues(github, repo)
-  issue = issues.find { |i| i.title.start_with?(deliverable['label'] + ':') }
-  if issue.nil?
-    body = $deliv_tpl % {
-      lead: deliverable['lead'],
-      date: deliverable['due_date'],
-      month: deliverable['month'],
-      nature: deliverable['nature'],
-    }
-    puts "\n\nMaking Issue on #{repo}: #{title}"
-    puts body
-    github.create_issue(repo, title, body, options)
-    # throttle creation calls to avoid flags for abuse
-    sleep THROTTLE_SECONDS
-  else
-    puts "Found Issue #{repo}##{issue.number}: #{issue.title}"
-    existing_labels = issue.labels.map { |label| label['name']}
-    missing_labels = options[:labels].reject { |label| existing_labels.include? label }
-    if issue.milestone.nil? or not missing_labels.empty?
-      puts "Updating milestone, labels on #{repo}##{issue.number}"
-      github.update_issue(repo, issue.number, options)
-      sleep THROTTLE_SECONDS
-    end
-  end
-end
-
-def make_deliverable_milestone(github, repo, deliverable)
-  title = deliverable['label']
-  
-  milestone = get_milestones(github, repo).find { |ms| ms.title == title }
-  if milestone.nil?
-    puts "Making milestone on #{repo}: #{title}"
-    body = $deliv_milestone_tpl % {
-      title: deliverable['title'],
-      lead: deliverable['lead'],
-      date: deliverable['due_date'],
-      month: deliverable['month'],
-      nature: deliverable['nature'],
-    }
-  
-    milestone = github.create_milestone(repo, title,
-        :due_on => deliverable['due_date'],
-        :description => body,
-    )
-    # throttle creation calls to avoid flags for abuse
-    sleep THROTTLE_SECONDS
-  else
-    puts "Milestone #{repo}@#{title} exists"
-  end
-  return milestone.number
-end
-
-def populate_workpackage(github, repo, workpackage)
-  # populate issues for a given workpackage
-  label = workpackage['label']
-
-  readme_path = "#{label}/README.md"
-  begin
-    github.contents(repo, :path => readme_path)
-  rescue Octokit::NotFound
-    readme = $readme_tpl % {
-      title: "#{workpackage['label']}: #{workpackage['title']}",
-      lead: workpackage['lead'],
-      page: workpackage['page'],
-    }
-    
-    puts "Creating readme at #{repo}/#{readme_path}"
-    puts readme
-    github.create_contents(repo, readme_path, "Creating #{readme_path}", readme)
-    # throttle creation calls to avoid flags for abuse
-    sleep THROTTLE_SECONDS
-  end
-  
-  workpackage['tasks'].each_value do |task|
-    make_task_issue(github, repo, task, {
-      :labels => [
-        'task',
-        workpackage['label'],
-      ]
-    })
-  end
-  workpackage['deliverables'].each do |deliverable|
-    milestone = make_deliverable_milestone(github, repo, deliverable)
-    make_deliverable_issue(github, repo, deliverable, {
-      :milestone => milestone,
-      :labels => [
-        'deliverable',
-        workpackage['label'],
-      ]
-    })
-  end
-end
-
-$cache = {
-  'issues' => {},
-  'milestones' => {},
-}
-
-def get_issues(github, repo)
-  # get issues for a repo (cached)
-  cache = $cache['issues']
-  if not cache.include? repo
-    cache[repo] = github.issues(repo)
-  end
-  return cache[repo]
-end
-
-def get_milestones(github, repo)
-  # get issues for a repo (cached)
-  cache = $cache['milestones']
-  if not cache.include? repo
-    cache[repo] = github.list_milestones(repo)
-  end
-  return cache[repo]
-end
-
 def load_pdata(proposal_dir)
   pdata = File.join(proposal_dir, 'proposal.pdata')
   deliv_data = File.join(proposal_dir, 'proposal.deliverables')
@@ -354,6 +168,7 @@ def load_pdata(proposal_dir)
       tasks[name][key] = value
 
     else
+      # DEBUG:
       # puts "    Ignored: #{args}"
     end
   end
@@ -367,7 +182,7 @@ def load_pdata(proposal_dir)
       "month" => month,
       # deliverables[deliv_id]['month'] = month
       # due date is last day of the given month, so subtract one day
-      "due_date" => ($start_date >> month) - 1,
+      "due_date" => (START_DATE >> month) - 1,
       "label" => scrub_tex(args[2]),
       "deliv_id" => args[3],
       "dissem" => transform_value('dissem', args[4]),
@@ -384,13 +199,230 @@ def load_pdata(proposal_dir)
 end
 
 
-# verify that there's a GitHub token in .netrc
-check_token
+#--------------- GitHub-related ---------------------
 
-# create client
-github = Octokit::Client.new(:netrc => true)
-github.auto_paginate = true
-
-load_pdata(proposal_dir).each do |wp|
-  populate_workpackage(github, $repo, wp)
+def check_token
+  # get GitHub auth token, creating one if we don't find it.
+  rc = Netrc.read Netrc.default_path
+  if not rc['api.github.com'].nil?
+    return
+  end
+  puts "We need your password to generate an OAuth token. The password will not be stored."
+  username = ask "Username: "
+  password = ask("Password:  ") { |q| q.echo = '*' }
+  client = Octokit::Client.new(
+    :login => username,
+    :password => password,
+  )
+  reply = client.create_authorization(
+    :scopes => ["public_repo"],
+    :note => "Issue Migration",
+  )
+  token = reply.token
+  rc['api.github.com'] = username, token
+  rc.save
 end
+
+$cache = {
+  'issues' => {},
+  'milestones' => {},
+}
+
+def get_issues(github, repo)
+  # get issues for a repo (cached)
+  cache = $cache['issues']
+  if not cache.include? repo
+    cache[repo] = github.issues(repo)
+  end
+  return cache[repo]
+end
+
+def get_milestones(github, repo)
+  # get issues for a repo (cached)
+  cache = $cache['milestones']
+  if not cache.include? repo
+    cache[repo] = github.list_milestones(repo)
+  end
+  return cache[repo]
+end
+
+# Templates for making readmes, issues
+
+README_TPL = <<-END
+# %{title}
+
+Lead institution: %{lead}
+
+See page %{page} of the [proposal](#{PROPOSAL_URL}) for the full description.
+END
+
+TASK_TPL = <<-END
+Lead Institution: %{lead}
+
+Partners: %{partners}
+
+Work phases: %{wphases}
+END
+
+DELIV_TPL = <<-END
+Lead Institution: %{lead}
+
+Due: %{date} (month %{month})
+
+Nature: %{nature}
+END
+
+DELIV_MILESTONE_TPL = "# %{title}\n\n#{DELIV_TPL}"
+
+
+def make_task_issue(github, repo, task, options)
+  title = "#{task['label']}: #{task['title']}"
+  issues = get_issues(github, repo)
+  issue = issues.find { |i| i.title.start_with?(task['label'] + ':') }
+  if issue.nil?
+    body = TASK_TPL % {
+      lead: task['lead'],
+      wphases: task['wphases'],
+      partners: (task['partners'] or ['None']).join(' ')
+    }
+    puts "\n\nMaking Issue on #{repo}: #{title}"
+    puts body
+  
+    github.create_issue(repo, title, body, options)
+    # throttle creation calls to avoid flags for abuse
+    sleep THROTTLE_SECONDS
+  else
+    puts "Found Issue #{repo}##{issue.number}: #{issue.title}"
+    existing_labels = issue.labels.map { |label| label['name']}
+    missing_labels = options[:labels].reject { |label| existing_labels.include? label }
+    if not missing_labels.empty?
+      puts "Updating milestone, labels on #{repo}##{issue.number}"
+      github.update_issue(repo, issue.number, options)
+      sleep THROTTLE_SECONDS
+    end
+  end
+end
+
+
+def make_deliverable_issue(github, repo, deliverable, options)
+  title = "#{deliverable['label']}: #{deliverable['title']}"
+  issues = get_issues(github, repo)
+  issue = issues.find { |i| i.title.start_with?(deliverable['label'] + ':') }
+  if issue.nil?
+    body = DELIV_TPL % {
+      lead: deliverable['lead'],
+      date: deliverable['due_date'],
+      month: deliverable['month'],
+      nature: deliverable['nature'],
+    }
+    puts "\n\nMaking Issue on #{repo}: #{title}"
+    puts body
+    github.create_issue(repo, title, body, options)
+    # throttle creation calls to avoid flags for abuse
+    sleep THROTTLE_SECONDS
+  else
+    puts "Found Issue #{repo}##{issue.number}: #{issue.title}"
+    existing_labels = issue.labels.map { |label| label['name']}
+    missing_labels = options[:labels].reject { |label| existing_labels.include? label }
+    if issue.milestone.nil? or not missing_labels.empty?
+      puts "Updating milestone, labels on #{repo}##{issue.number}"
+      github.update_issue(repo, issue.number, options)
+      sleep THROTTLE_SECONDS
+    end
+  end
+end
+
+
+def make_deliverable_milestone(github, repo, deliverable)
+  title = deliverable['label']
+  
+  milestone = get_milestones(github, repo).find { |ms| ms.title == title }
+  if milestone.nil?
+    puts "Making milestone on #{repo}: #{title}"
+    body = DELIV_MILESTONE_TPL % {
+      title: deliverable['title'],
+      lead: deliverable['lead'],
+      date: deliverable['due_date'],
+      month: deliverable['month'],
+      nature: deliverable['nature'],
+    }
+  
+    milestone = github.create_milestone(repo, title,
+        :due_on => deliverable['due_date'],
+        :description => body,
+    )
+    # throttle creation calls to avoid flags for abuse
+    sleep THROTTLE_SECONDS
+  else
+    puts "Milestone #{repo}@#{title} exists"
+  end
+  return milestone.number
+end
+
+
+def populate_workpackage(github, repo, workpackage)
+  # populate issues for a given workpackage
+  label = workpackage['label']
+
+  readme_path = "#{label}/README.md"
+  begin
+    github.contents(repo, :path => readme_path)
+  rescue Octokit::NotFound
+    readme = README_TPL % {
+      title: "#{workpackage['label']}: #{workpackage['title']}",
+      lead: workpackage['lead'],
+      page: workpackage['page'],
+    }
+    
+    puts "Creating readme at #{repo}/#{readme_path}"
+    puts readme
+    github.create_contents(repo, readme_path, "Creating #{readme_path}", readme)
+    # throttle creation calls to avoid flags for abuse
+    sleep THROTTLE_SECONDS
+  end
+  
+  workpackage['tasks'].each_value do |task|
+    make_task_issue(github, repo, task, {
+      :labels => [
+        'task',
+        workpackage['label'],
+      ]
+    })
+  end
+  workpackage['deliverables'].each do |deliverable|
+    milestone = make_deliverable_milestone(github, repo, deliverable)
+    make_deliverable_issue(github, repo, deliverable, {
+      :milestone => milestone,
+      :labels => [
+        'deliverable',
+        workpackage['label'],
+      ]
+    })
+  end
+end
+
+
+def main(proposal_dir)
+  # run the whole thing
+
+  # verify that there's a GitHub token in .netrc
+  check_token
+
+  # create client
+  github = Octokit::Client.new(:netrc => true)
+  github.auto_paginate = true
+
+  load_pdata(proposal_dir).each do |wp|
+    populate_workpackage(github, REPO, wp)
+  end
+end
+
+if __FILE__ == $0
+  puts ARGV
+  if ARGV.length > 1
+    proposal_dir = ARGV[1]
+  else
+    proposal_dir = '.'
+  end
+  main(proposal_dir)
+end
\ No newline at end of file

From fdd33829825a12a319a54c481aae38e2f895399a Mon Sep 17 00:00:00 2001
From: Min RK <benjaminrk@gmail.com>
Date: Fri, 4 Sep 2015 18:08:36 +0200
Subject: [PATCH 7/9] join partners with commas

---
 bin/generate-issues | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/generate-issues b/bin/generate-issues
index 5eaa3e1..8954120 100755
--- a/bin/generate-issues
+++ b/bin/generate-issues
@@ -283,7 +283,7 @@ def make_task_issue(github, repo, task, options)
     body = TASK_TPL % {
       lead: task['lead'],
       wphases: task['wphases'],
-      partners: (task['partners'] or ['None']).join(' ')
+      partners: (task['partners'] or ['None']).join(', ')
     }
     puts "\n\nMaking Issue on #{repo}: #{title}"
     puts body

From 8bd49e05cce0eabd4a7875be670b94234a32b973 Mon Sep 17 00:00:00 2001
From: Min RK <benjaminrk@gmail.com>
Date: Fri, 4 Sep 2015 18:37:32 +0200
Subject: [PATCH 8/9] add site labels to issues

---
 bin/generate-issues | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/bin/generate-issues b/bin/generate-issues
index 8954120..af7a3a7 100755
--- a/bin/generate-issues
+++ b/bin/generate-issues
@@ -58,6 +58,11 @@ SITES = {
   'SR'  => 'Simula Research Laboratory',
 }
 
+REVERSE_SITES = {}
+SITES.each_pair do |key, name|
+  REVERSE_SITES[name] = key
+end
+
 #------------------- Parsing proposal.pdata ---------------------
 
 def split_line(line)
@@ -382,11 +387,14 @@ def populate_workpackage(github, repo, workpackage)
   end
   
   workpackage['tasks'].each_value do |task|
+    org_labels = [REVERSE_SITES[task['lead']]] + \
+      (task['partners'] or []).map {|site| REVERSE_SITES[site]}
+    
     make_task_issue(github, repo, task, {
       :labels => [
         'task',
         workpackage['label'],
-      ]
+      ] + org_labels
     })
   end
   workpackage['deliverables'].each do |deliverable|
@@ -396,6 +404,7 @@ def populate_workpackage(github, repo, workpackage)
       :labels => [
         'deliverable',
         workpackage['label'],
+        REVERSE_SITES[deliverable['lead']]
       ]
     })
   end

From d25c4f180a7ec20fa82d614b631b67b0c3c5c558 Mon Sep 17 00:00:00 2001
From: Min RK <benjaminrk@gmail.com>
Date: Fri, 4 Sep 2015 18:42:53 +0200
Subject: [PATCH 9/9] add work package title to issues

---
 bin/generate-issues | 21 ++++++++++++++-------
 1 file changed, 14 insertions(+), 7 deletions(-)

diff --git a/bin/generate-issues b/bin/generate-issues
index af7a3a7..726e77a 100755
--- a/bin/generate-issues
+++ b/bin/generate-issues
@@ -262,6 +262,8 @@ See page %{page} of the [proposal](#{PROPOSAL_URL}) for the full description.
 END
 
 TASK_TPL = <<-END
+Work Package %{wptitle}
+
 Lead Institution: %{lead}
 
 Partners: %{partners}
@@ -270,6 +272,8 @@ Work phases: %{wphases}
 END
 
 DELIV_TPL = <<-END
+Work Package %{wptitle}
+
 Lead Institution: %{lead}
 
 Due: %{date} (month %{month})
@@ -280,12 +284,13 @@ END
 DELIV_MILESTONE_TPL = "# %{title}\n\n#{DELIV_TPL}"
 
 
-def make_task_issue(github, repo, task, options)
+def make_task_issue(github, repo, task, workpackage, options)
   title = "#{task['label']}: #{task['title']}"
   issues = get_issues(github, repo)
   issue = issues.find { |i| i.title.start_with?(task['label'] + ':') }
   if issue.nil?
     body = TASK_TPL % {
+      wptitle: "#{workpackage['label']}: #{workpackage['title']}",
       lead: task['lead'],
       wphases: task['wphases'],
       partners: (task['partners'] or ['None']).join(', ')
@@ -309,12 +314,13 @@ def make_task_issue(github, repo, task, options)
 end
 
 
-def make_deliverable_issue(github, repo, deliverable, options)
+def make_deliverable_issue(github, repo, deliverable, workpackage, options)
   title = "#{deliverable['label']}: #{deliverable['title']}"
   issues = get_issues(github, repo)
   issue = issues.find { |i| i.title.start_with?(deliverable['label'] + ':') }
   if issue.nil?
     body = DELIV_TPL % {
+      wptitle: "#{workpackage['label']}: #{workpackage['title']}",
       lead: deliverable['lead'],
       date: deliverable['due_date'],
       month: deliverable['month'],
@@ -338,13 +344,14 @@ def make_deliverable_issue(github, repo, deliverable, options)
 end
 
 
-def make_deliverable_milestone(github, repo, deliverable)
+def make_deliverable_milestone(github, repo, deliverable, workpackage)
   title = deliverable['label']
   
   milestone = get_milestones(github, repo).find { |ms| ms.title == title }
   if milestone.nil?
     puts "Making milestone on #{repo}: #{title}"
     body = DELIV_MILESTONE_TPL % {
+      wptitle: "#{workpackage['label']}: #{workpackage['title']}",
       title: deliverable['title'],
       lead: deliverable['lead'],
       date: deliverable['due_date'],
@@ -390,16 +397,16 @@ def populate_workpackage(github, repo, workpackage)
     org_labels = [REVERSE_SITES[task['lead']]] + \
       (task['partners'] or []).map {|site| REVERSE_SITES[site]}
     
-    make_task_issue(github, repo, task, {
+    make_task_issue(github, repo, task, workpackage,
       :labels => [
         'task',
         workpackage['label'],
       ] + org_labels
-    })
+    )
   end
   workpackage['deliverables'].each do |deliverable|
-    milestone = make_deliverable_milestone(github, repo, deliverable)
-    make_deliverable_issue(github, repo, deliverable, {
+    milestone = make_deliverable_milestone(github, repo, deliverable, workpackage)
+    make_deliverable_issue(github, repo, deliverable, workpackage, {
       :milestone => milestone,
       :labels => [
         'deliverable',