From 4b51250389454f0ce6032fa990f8859179203ecb Mon Sep 17 00:00:00 2001 From: Alex Dergachev Date: Thu, 27 Nov 2014 20:48:29 +0000 Subject: [PATCH] Working via GUI, added README --- README.md | 61 ++++---- app/models/repository/git_fetch.rb | 142 ++++++++++++++++++ config/locales/en.yml | 4 + init.rb | 13 +- lib/repository_fetch/fetch.rb | 79 ---------- .../repositories_helper_patch.rb | 26 ++++ 6 files changed, 212 insertions(+), 113 deletions(-) create mode 100644 app/models/repository/git_fetch.rb create mode 100644 config/locales/en.yml delete mode 100644 lib/repository_fetch/fetch.rb create mode 100644 lib/repository_fetch/repositories_helper_patch.rb diff --git a/README.md b/README.md index 0882135..0a66a71 100644 --- a/README.md +++ b/README.md @@ -1,51 +1,48 @@ -redmine_repository_fetch -======================== +redmine_git_remote +================== -Redmine plugin to automatically clone and fetch referenced repositories. +Redmine plugin to automatically clone and remote git repositories. ## Installation -Currently the plugin hardcodes this config, change it for your use-case: +Install the plugin as usual: ``` - PATTERNS = [ - { :pattern => "/redmine_git_fetch/github.com/", - :uri_prefix => "git@github.com:", - :host => "github.com", - :key => "/home/redmine/data/keys/id_rsa" - }, - { :pattern => "/redmine_git_fetch/gitlab.com/", - :uri_prefix => "git@gitlab.com:", - :host => "gitlab.com", - :key => "/home/redmine/data/keys/id_rsa" - }, - { :pattern => "/redmine_git_fetch/git.ewdev.ca/", - :uri_prefix => "git@git.ewdev.ca:", - :host => "git.ewdev.ca", - :key => "/home/redmine/data/keys/id_rsa" - } - ] +cd REDMINE_ROOT/plugins +git clone https://github.com/dergachev/redmine_git_remote ``` -Be sure to populate the appropriate keys for your redmine user (www-data, redmine, etc), -either in `~/.ssh` or in the place specified by the `PATTERNS[x][:key]` property. +Be sure to install the appropriate SSH keys to `~/.ssh/id_rsa` (for your redmine user). +I recommend creating a dedicated deployment user on github/gitlab for this purpose. ## Usage -Add `/redmine_git_fetch/github.com/evolvingweb/sitediff.git` to a repo. The -plugin will automatically detect the prefix `/redmine_git_fetch/github.com/` -and figure out it needs to clone `git@github.com:evolvingweb/sitediff.git`. -If it's already cloned it will fetch instead. In all cases you need to specify -a path to a private key to use, since all clones happen over SSH. +This plugin defines a new repository type, GitFetch, which allows you to associate +a remote repository with your Redmine project. First create a new repository of type +GitFetch, enter the clone URL. The identifier and path will be auto-generated, but can be overriden. -Note that `/redmine_git_fetch` folder will get auto-created. +![](https://dl.dropbox.com/u/29440342/screenshots/ATIAQXHG-2014.11.27-15-03-51.png) -The plugin currently doesn't fetch any repos outside its purview. +On submitting the repository creation form, the identifier and `url` +(filesystem path) fields will be auto-generated (if not explicitly provided) as follows: -It also needs to be run as follows, probably from cron: +Clone URL: `https://github.com/dergachev/vagrant-vbox-snapshot` +URL (filesystem path): `REDMINE_PLUGINS_PATH/redmine_git_remote/repos/github.com/dergachev/vagrant-vbox-snapshot` +Identifier: `vagrant-vbox-snapshot` + +Once the remote URL is validated, the plugin creates an "empty clone" at the specified path. + +This plugin hooks into the core `Repository.fetch_changesets` to automatically +run `git fetch --all` on all GitRemote managed repositories, before those +commits are imported into Redmine. To avoid slowing down the GUI, we recommend +unchecking the "Fetch commits automatically" setting at +[http://redmine-root/settings?tab=repositories](http://redmine-root/settings?tab=repositories) +and relying on the following cron job as per [Redmine Wiki Instructions](http://www.redmine.org/projects/redmine/wiki/RedmineRepositories): ``` -bundle exec rails runner "RepositoryFetch.fetch" -e production +cd /home/redmine/redmine && ./script/rails runner \"Repository.fetch_changesets\" -e production >> log/cron_rake.log 2>&1 ``` +Note GitRemote doesn't delete the cloned repos when the associated record is deleted from Redmine. + Tested on Redmine 2.6. diff --git a/app/models/repository/git_fetch.rb b/app/models/repository/git_fetch.rb new file mode 100644 index 0000000..652fd17 --- /dev/null +++ b/app/models/repository/git_fetch.rb @@ -0,0 +1,142 @@ +require 'redmine/scm/adapters/git_adapter' +require 'pathname' +require 'fileutils' + +class Repository::GitFetch < Repository::Git + + PLUGIN_ROOT = Pathname.new(__FILE__).join("../../../..").realpath.to_s + PATH_PREFIX = PLUGIN_ROOT + "/repos/" + + before_validation :initialize_clone + + # TODO: figureo ut how to do this safely (if at all) + # before_deletion :rm_removed_repo + # def rm_removed_repo + # if Repository.find_all_by_url(repo.url).length <= 1 + # system "rm -Rf #{self.clone_url}" + # end + # end + + def extra_clone_url + return nil unless extra_info + extra_info["extra_clone_url"] + end + + def clone_url + self.extra_clone_url + end + + def clone_path + self.url + end + + def clone_host + p = parse(clone_url) + return p[:host] + end + + # hook into Repository.fetch_changesets to also run 'git fetch' + def fetch_changesets + puts "Calling fetch changesets on #{clone_path}" + # runs git fetch + self.fetch + super + end + + # called in before_validate handler, sets form errors + def initialize_clone + # avoids crash in RepositoriesController#destroy + return unless attributes["extra_info"]["extra_clone_url"] + + p = parse(attributes["extra_info"]["extra_clone_url"]) + self.identifier = p[:identifier] if identifier.empty? + self.url = PATH_PREFIX + p[:path] if url.empty? + + err = clone_empty + errors.add :extra_clone_url, err if err + end + + # equality check ignoring trailing whitespace and slashes + def two_remotes_equal(a,b) + a.chomp.gsub(/\/$/,'') == b.chomp.gsub(/\/$/,'') + end + + def clone_empty + Repository::GitFetch.add_known_host(clone_host) + + unless system "git ls-remote -h #{clone_url}" + return "#{clone_url} is not a valid remote." + end + + if Dir.exists? clone_path + existing_repo_remote = `git -C #{clone_path} config --get remote.origin.url` + unless two_remotes_equal(existing_repo_remote, clone_url) + return "Clone path '#{clone_path}' already exits, unmatching clone url: #{existing_repo_remote}" + end + else + unless system "git init --bare #{clone_path}" + return "Unable to run git init at #{clone_path}" + end + + unless system "git -C #{clone_path} remote add --tags --mirror=fetch origin #{clone_url}" + return "Unable to run: git -C #{clone_path} remote add #{clone_url}" + end + end + end + + unloadable + def self.scm_name + 'GitFetch' + end + + def parse(url) + ret = {} + # start with http://github.com/evolvingweb/git_remote or git@git.ewdev.ca:some/repo.git + ret[:url] = url + # path is github.com/evolvingweb/muhc-ci + ret[:path] = url + .gsub(/^.*:\/\//, '') # Remove anything before :// + .gsub(/:/, '/') # convert ":" to "/" + .gsub(/^.*@/, '') # Remove anything before @ + .gsub(/\.git$/, '') # Remove trailing .git + ret[:host] = ret[:path].split('/').first + #TODO: handle project uniqueness automatically or prompt + ret[:identifier] = ret[:path].split('/').last.downcase + return ret + end + + def fetch + puts "Fetching repo #{clone_path}" + Repository::GitFetch.add_known_host(clone_host) + + err = clone_empty + Rails.logger.warn err if err + + # If dir exists and non-empty, should be safe to 'git fetch' + unless system "git -C #{clone_path} fetch --all" + Rails.logger.warn "Unable to run 'git -c #{clone_path} fetch --all'" + end + end + + def self.fetch_all + Repository::GitFetch.all.each do |x| + x.fetch + end + end + + # Checks if host is in ~/.ssh/known_hosts, adds it if not present + def self.add_known_host(host) + # if not found... + if `ssh-keygen -F #{host} | grep 'found'` == "" + # hack to work with 'docker exec' where HOME isn't set (or set to /) + ssh_dir = (ENV['HOME'] == "/" || ENV['HOME'] == nil ? "/root" : ENV['HOME']) + "/.ssh" + ssh_known_hosts = ssh_dir + "/known_hosts" + FileUtils.mkdir_p ssh_dir + puts "Adding #{host} to #{ssh_known_hosts}" + unless system `ssh-keyscan #{host} >> #{ssh_known_hosts}` + Rails.logger.warn "Unable to add known host #{host} to #{ssh_known_hosts}" + end + end + end + +end diff --git a/config/locales/en.yml b/config/locales/en.yml new file mode 100644 index 0000000..a500cd9 --- /dev/null +++ b/config/locales/en.yml @@ -0,0 +1,4 @@ +en: + field_extra_clone_url: Clone URL + text_git_fetch_repository_note: The URL to clone from. + text_git_fetch_repository_path_note: The absolute filesystem path to clone to. Leave blank to auto-populate from URL. diff --git a/init.rb b/init.rb index 280c44b..d8811f7 100644 --- a/init.rb +++ b/init.rb @@ -1,10 +1,19 @@ require 'redmine' -require_dependency "repository_fetch/fetch" -Redmine::Plugin.register :repository_fetch do +# TODO: why isn't this autoloaded? +# NB: at this point, $PATH only contains {PLUGINS}/lib and app/models, app/controllers +# but not {PLUGINS}/app/models. Maybe those get added later? +require File.dirname(__FILE__) + '/app/models/repository/git_fetch' + +require_dependency "repository_fetch/repositories_helper_patch" + +Redmine::Scm::Base.add "GitFetch" + +Redmine::Plugin.register :redmine_repository_fetch do name 'Repository Fetch' author 'Alex Dergachev' url 'https://github.com/dergachev/redmine_repository_fetch' description 'Automatically clone and fetch referenced repositories' version '0.0.1' end + diff --git a/lib/repository_fetch/fetch.rb b/lib/repository_fetch/fetch.rb deleted file mode 100644 index ac5a838..0000000 --- a/lib/repository_fetch/fetch.rb +++ /dev/null @@ -1,79 +0,0 @@ -module RepositoryFetch - - def self.logger - ::Rails.logger - end - - PATTERNS = [ - { :pattern => "/redmine_git_fetch/github.com/", - :uri_prefix => "git@github.com:", - :host => "github.com", - :key => "/home/redmine/data/keys/id_rsa" - }, - { :pattern => "/redmine_git_fetch/gitlab.com/", - :uri_prefix => "git@gitlab.com:", - :host => "gitlab.com", - :key => "/home/redmine/data/keys/id_rsa" - }, - { :pattern => "/redmine_git_fetch/git.ewdev.ca/", - :uri_prefix => "git@git.ewdev.ca:", - :host => "git.ewdev.ca", - :key => "/home/redmine/data/keys/id_rsa" - } - ] - - def self.clone_or_fetch(repository) - return unless repository.scm_name == "Git" - - path = repository.url - - p = PATTERNS.find { |p| path.starts_with? p[:pattern] } - unless p - # TODO: figure out how to handle non-matching repos. - # eg. skip them, try fetching them, throw warning or not? - proj = repository.project.identifier - logger.warn "repository_fetch: no match for '#{path}' in project '#{proj}'" - return - end - - add_known_host(p[:host]) - - # If dir exists and non-empty, should be safe to 'git fetch' - if Dir.exists?(path) && Dir.entries(path) != [".", ".."] - puts "Running git fetch on #{path}" - puts exec_with_key "git -C #{path} fetch --all", p[:key] - else - # try cloning the repo - url = path.sub( p[:pattern], p[:uri_prefix]) - puts "Matched new URL, trying to clone: " + url - puts exec_with_key "git clone --mirror #{url} #{path}", p[:key] - end - end - - def self.exec_with_key(cmd, keyfile) - return `ssh-agent bash -c 'ssh-add #{keyfile}; #{cmd}'` - end - - def self.fetch - Project.active.has_module(:repository).all.each do |project| - project.repositories.each do |repository| - clone_or_fetch(repository) - end - end - end - - # Checks if host is in ~/.ssh/known_hosts, adds it if not present - def self.add_known_host(host) - # if not found... - if `ssh-keygen -F #{host} | grep 'found'` == "" - # hack to work with 'docker exec' where HOME isn't set (or set to /) - ssh_known_hosts = (ENV['HOME'] == "/" or ENV['HOME'] == nil ? "/root" : ENV['HOME']) + "/.ssh/known_hosts" - puts "Authorizing #{host}" - puts `ssh-keyscan #{host} >> #{ssh_known_hosts}` - end - end - - - class Fetch - end -end diff --git a/lib/repository_fetch/repositories_helper_patch.rb b/lib/repository_fetch/repositories_helper_patch.rb new file mode 100644 index 0000000..b604874 --- /dev/null +++ b/lib/repository_fetch/repositories_helper_patch.rb @@ -0,0 +1,26 @@ +module RepositoryFetch + module RepositoriesHelperPatch + def self.included(base) # :nodoc: + base.send(:include, InstanceMethods) + end + + module InstanceMethods + def git_fetch_field_tags(form, repository) + #TODO: change URL label to "Path" + content_tag('p', form.text_field(:url, :size => 60, :required => true, + :disabled => !repository.safe_attribute?('url'), :required => false) + + content_tag('em', + l(:text_git_fetch_repository_path_note), + :class => 'info') + + form.text_field(:extra_clone_url, :size => 60, :required => true, + :disabled => !repository.safe_attribute?('url')) + + content_tag('em', + l(:text_git_fetch_repository_note), + :class => 'info') + ) + end + end + end + + RepositoriesHelper.send(:include, RepositoriesHelperPatch) +end