Working via GUI, added README

README.md

@@ -1,51 +1,48 @@
-redmine_repository_fetch
+redmine_git_remote
-========================
+==================
 
-Redmine plugin to automatically clone and fetch referenced repositories.
+Redmine plugin to automatically clone and remote git repositories.
 
 ## Installation
 
-Currently the plugin hardcodes this config, change it for your use-case:
+Install the plugin as usual:
 
 ```
-   PATTERNS = [
+cd REDMINE_ROOT/plugins
-     {  :pattern => "/redmine_git_fetch/github.com/",
+git clone https://github.com/dergachev/redmine_git_remote
-        :uri_prefix => "git@github.com:",
-        :host => "github.com",
-        :key => "/home/redmine/data/keys/id_rsa"
-     },
-     {  :pattern => "/redmine_git_fetch/gitlab.com/",
-        :uri_prefix => "git@gitlab.com:",
-        :host => "gitlab.com",
-        :key => "/home/redmine/data/keys/id_rsa"
-     },
-     {  :pattern => "/redmine_git_fetch/git.ewdev.ca/",
-        :uri_prefix => "git@git.ewdev.ca:",
-        :host => "git.ewdev.ca",
-        :key => "/home/redmine/data/keys/id_rsa"
-     }
-   ]
 ```
 
-Be sure to populate the appropriate keys for your redmine user (www-data, redmine, etc),
+Be sure to install the appropriate SSH keys to `~/.ssh/id_rsa` (for your redmine user).
-either in `~/.ssh` or in the place specified by the `PATTERNS[x][:key]` property.
+I recommend creating a dedicated deployment user on github/gitlab for this purpose.
 
 ## Usage
 
-Add `/redmine_git_fetch/github.com/evolvingweb/sitediff.git` to a repo.  The
+This plugin defines a new repository type, GitFetch, which allows you to associate
-plugin will automatically detect the prefix `/redmine_git_fetch/github.com/`
+a remote repository with your Redmine project. First create a new repository of type
-and figure out it needs to clone `git@github.com:evolvingweb/sitediff.git`.
+GitFetch, enter the clone URL. The identifier and path will be auto-generated, but can be overriden.
-If it's already cloned it will fetch instead. In all cases you need to specify
-a path to a private key to use, since all clones happen over SSH.
 
-Note that `/redmine_git_fetch` folder will get auto-created.
+![](https://dl.dropbox.com/u/29440342/screenshots/ATIAQXHG-2014.11.27-15-03-51.png)
 
-The plugin currently doesn't fetch any repos outside its purview.
+On submitting the repository creation form, the identifier and `url`
+(filesystem path) fields will be auto-generated (if not explicitly provided) as follows:
 
-It also needs to be run as follows, probably from cron:
+Clone URL: `https://github.com/dergachev/vagrant-vbox-snapshot`
+URL (filesystem path): `REDMINE_PLUGINS_PATH/redmine_git_remote/repos/github.com/dergachev/vagrant-vbox-snapshot`
+Identifier: `vagrant-vbox-snapshot`
+
+Once the remote URL is validated, the plugin creates an "empty clone" at the specified path.
+
+This plugin hooks into the core `Repository.fetch_changesets` to automatically
+run `git fetch --all` on all GitRemote managed repositories, before those
+commits are imported into Redmine. To avoid slowing down the GUI, we recommend
+unchecking the "Fetch commits automatically" setting at
+[http://redmine-root/settings?tab=repositories](http://redmine-root/settings?tab=repositories)
+and relying on the following cron job as per [Redmine Wiki Instructions](http://www.redmine.org/projects/redmine/wiki/RedmineRepositories):
 
 ```
-bundle exec rails runner "RepositoryFetch.fetch" -e production
+cd /home/redmine/redmine && ./script/rails runner \"Repository.fetch_changesets\" -e production >> log/cron_rake.log 2>&1
 ```
 
+Note GitRemote doesn't delete the cloned repos when the associated record is deleted from Redmine.
+
 Tested on Redmine 2.6.

app/models/repository/git_fetch.rb

@@ -0,0 +1,142 @@
+require 'redmine/scm/adapters/git_adapter'
+require 'pathname'
+require 'fileutils'
+
+class Repository::GitFetch < Repository::Git
+
+  PLUGIN_ROOT = Pathname.new(__FILE__).join("../../../..").realpath.to_s
+  PATH_PREFIX = PLUGIN_ROOT + "/repos/"
+
+  before_validation :initialize_clone
+
+  # TODO: figureo ut how to do this safely (if at all)
+  # before_deletion :rm_removed_repo
+  # def rm_removed_repo
+  #   if Repository.find_all_by_url(repo.url).length <= 1
+  #     system "rm -Rf #{self.clone_url}"
+  #   end
+  # end
+
+  def extra_clone_url
+    return nil unless extra_info
+    extra_info["extra_clone_url"]
+  end
+
+  def clone_url
+    self.extra_clone_url
+  end
+
+  def clone_path
+    self.url
+  end
+
+  def clone_host
+    p = parse(clone_url)
+    return p[:host]
+  end
+
+  # hook into Repository.fetch_changesets to also run 'git fetch'
+  def fetch_changesets
+    puts "Calling fetch changesets on #{clone_path}"
+    # runs git fetch
+    self.fetch
+    super
+  end
+
+  # called in before_validate handler, sets form errors
+  def initialize_clone
+    # avoids crash in RepositoriesController#destroy
+    return unless attributes["extra_info"]["extra_clone_url"]
+    
+    p = parse(attributes["extra_info"]["extra_clone_url"])
+    self.identifier = p[:identifier] if identifier.empty?
+    self.url = PATH_PREFIX + p[:path] if url.empty?
+
+    err = clone_empty
+    errors.add :extra_clone_url, err if err 
+  end
+
+  # equality check ignoring trailing whitespace and slashes
+  def two_remotes_equal(a,b)
+    a.chomp.gsub(/\/$/,'') == b.chomp.gsub(/\/$/,'')
+  end
+
+  def clone_empty
+    Repository::GitFetch.add_known_host(clone_host)
+
+    unless system "git ls-remote -h #{clone_url}"
+      return "#{clone_url} is not a valid remote."
+    end
+
+    if Dir.exists? clone_path
+      existing_repo_remote = `git -C #{clone_path} config --get remote.origin.url`
+      unless two_remotes_equal(existing_repo_remote, clone_url)
+        return "Clone path '#{clone_path}' already exits, unmatching clone url: #{existing_repo_remote}"
+      end
+    else
+      unless system "git init --bare #{clone_path}"
+        return  "Unable to run git init at #{clone_path}"
+      end
+
+      unless system "git -C #{clone_path} remote add --tags --mirror=fetch origin #{clone_url}"
+        return  "Unable to run: git -C #{clone_path} remote add #{clone_url}"
+      end
+    end
+  end
+
+  unloadable
+  def self.scm_name
+    'GitFetch'
+  end
+
+  def parse(url)
+    ret = {}
+    # start with http://github.com/evolvingweb/git_remote or git@git.ewdev.ca:some/repo.git
+    ret[:url] = url
+    # path is github.com/evolvingweb/muhc-ci
+    ret[:path] = url
+                    .gsub(/^.*:\/\//, '')    # Remove anything before ://
+                    .gsub(/:/, '/')          # convert ":" to "/"
+                    .gsub(/^.*@/, '')        # Remove anything before @
+                    .gsub(/\.git$/, '')      # Remove trailing .git
+    ret[:host] = ret[:path].split('/').first
+    #TODO: handle project uniqueness automatically or prompt
+    ret[:identifier] =   ret[:path].split('/').last.downcase
+    return ret
+  end
+
+  def fetch
+    puts "Fetching repo #{clone_path}"
+    Repository::GitFetch.add_known_host(clone_host)
+
+    err = clone_empty
+    Rails.logger.warn err if err
+
+    # If dir exists and non-empty, should be safe to 'git fetch'
+    unless system "git -C #{clone_path} fetch --all"
+      Rails.logger.warn "Unable to run 'git -c #{clone_path} fetch --all'"
+    end
+  end
+
+  def self.fetch_all
+    Repository::GitFetch.all.each do |x|
+      x.fetch
+    end
+  end
+
+  # Checks if host is in ~/.ssh/known_hosts, adds it if not present
+  def self.add_known_host(host)
+    # if not found...
+    if `ssh-keygen -F #{host} | grep 'found'` == ""
+      # hack to work with 'docker exec' where HOME isn't set (or set to /)
+      ssh_dir = (ENV['HOME'] == "/" || ENV['HOME'] == nil ? "/root" : ENV['HOME']) + "/.ssh"
+      ssh_known_hosts = ssh_dir + "/known_hosts"
+      FileUtils.mkdir_p ssh_dir
+      puts "Adding #{host} to #{ssh_known_hosts}"
+      unless system `ssh-keyscan #{host} >> #{ssh_known_hosts}`
+        Rails.logger.warn "Unable to add known host #{host} to #{ssh_known_hosts}"
+      end
+    end
+  end
+
+end

config/locales/en.yml

@@ -0,0 +1,4 @@
+en:
+  field_extra_clone_url: Clone URL
+  text_git_fetch_repository_note: The URL to clone from.
+  text_git_fetch_repository_path_note: The absolute filesystem path to clone to. Leave blank to auto-populate from URL.

init.rb

@@ -1,10 +1,19 @@
 require 'redmine'
-require_dependency "repository_fetch/fetch"
 
-Redmine::Plugin.register :repository_fetch do
+# TODO: why isn't this autoloaded? 
+# NB: at this point, $PATH only contains {PLUGINS}/lib and app/models, app/controllers
+#     but not {PLUGINS}/app/models. Maybe those get added later?
+require File.dirname(__FILE__) + '/app/models/repository/git_fetch'
+
+require_dependency "repository_fetch/repositories_helper_patch"
+
+Redmine::Scm::Base.add "GitFetch"
+
+Redmine::Plugin.register :redmine_repository_fetch do
   name 'Repository Fetch'
   author 'Alex Dergachev'
   url 'https://github.com/dergachev/redmine_repository_fetch'
   description 'Automatically clone and fetch referenced repositories'
   version '0.0.1'
 end
+

lib/repository_fetch/fetch.rb

@@ -1,79 +0,0 @@
-module RepositoryFetch
-
-  def self.logger
-    ::Rails.logger
-  end
-
-   PATTERNS = [ 
-     {  :pattern => "/redmine_git_fetch/github.com/",
-        :uri_prefix => "git@github.com:",
-        :host => "github.com",
-        :key => "/home/redmine/data/keys/id_rsa"
-     },
-     {  :pattern => "/redmine_git_fetch/gitlab.com/",
-        :uri_prefix => "git@gitlab.com:",
-        :host => "gitlab.com",
-        :key => "/home/redmine/data/keys/id_rsa"
-     },
-     {  :pattern => "/redmine_git_fetch/git.ewdev.ca/",
-        :uri_prefix => "git@git.ewdev.ca:",
-        :host => "git.ewdev.ca",
-        :key => "/home/redmine/data/keys/id_rsa"
-     }
-   ]
-
-  def self.clone_or_fetch(repository)
-    return unless repository.scm_name == "Git"
-
-    path = repository.url
-
-    p = PATTERNS.find { |p| path.starts_with? p[:pattern] }
-    unless p
-      # TODO: figure out how to handle non-matching repos.
-      # eg. skip them, try fetching them, throw warning or not?
-      proj = repository.project.identifier
-      logger.warn "repository_fetch: no match for '#{path}' in project '#{proj}'"
-      return
-    end
-
-    add_known_host(p[:host])
-
-    # If dir exists and non-empty, should be safe to 'git fetch'
-    if Dir.exists?(path) && Dir.entries(path) != [".", ".."]
-      puts "Running git fetch on #{path}"
-      puts exec_with_key "git -C #{path} fetch --all", p[:key]
-    else
-      # try cloning the repo
-      url = path.sub( p[:pattern], p[:uri_prefix])
-      puts "Matched new URL, trying to clone: " + url
-      puts exec_with_key "git clone --mirror #{url} #{path}", p[:key]
-    end
-  end
-
-  def self.exec_with_key(cmd, keyfile)
-    return `ssh-agent bash -c 'ssh-add #{keyfile}; #{cmd}'`
-  end
-
-  def self.fetch
-    Project.active.has_module(:repository).all.each do |project|
-      project.repositories.each do |repository|
-        clone_or_fetch(repository)
-      end
-    end
-  end
-
-  # Checks if host is in ~/.ssh/known_hosts, adds it if not present
-  def self.add_known_host(host)
-    # if not found...
-    if `ssh-keygen -F #{host} | grep 'found'` == ""
-      # hack to work with 'docker exec' where HOME isn't set (or set to /)
-      ssh_known_hosts = (ENV['HOME'] == "/" or ENV['HOME'] == nil ? "/root" : ENV['HOME']) + "/.ssh/known_hosts"
-      puts "Authorizing #{host}"
-      puts `ssh-keyscan #{host} >> #{ssh_known_hosts}`
-    end
-  end
-
-
-  class Fetch
-  end
-end

lib/repository_fetch/repositories_helper_patch.rb

@@ -0,0 +1,26 @@
+module RepositoryFetch
+  module RepositoriesHelperPatch
+    def self.included(base) # :nodoc:
+      base.send(:include, InstanceMethods)
+    end
+
+    module InstanceMethods
+      def git_fetch_field_tags(form, repository)
+        #TODO: change URL label to "Path"
+        content_tag('p', form.text_field(:url, :size => 60, :required => true,
+                           :disabled => !repository.safe_attribute?('url'), :required => false) +
+                          content_tag('em', 
+                            l(:text_git_fetch_repository_path_note),
+                            :class => 'info') +
+                          form.text_field(:extra_clone_url, :size => 60, :required => true,
+                           :disabled => !repository.safe_attribute?('url')) +
+                          content_tag('em', 
+                            l(:text_git_fetch_repository_note),
+                            :class => 'info')
+                   )
+      end
+    end
+  end
+
+  RepositoriesHelper.send(:include, RepositoriesHelperPatch)
+end