redmine_plugins/redmine_dmsf
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #633 from carlolars/WebDAV_Cache

Browse Source 
WebDav cache
This commit is contained in:
Karel Picman 2017-01-16 08:53:37 +01:00 committed by GitHub
commit eb87b50159
17 changed files with 415 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Gemfile
View File

@ -27,6 +27,7 @@ gem 'zip-zip'
gem 'simple_enum'
gem 'uuidtools'
gem 'dav4rack'
gem 'dalli'
group :xapian do
gem 'xapian-full-alaveteli', :require => false

11
README.md
View File

@ -251,6 +251,17 @@ Example of cron job (once per hour at 8th minute):
See redmine_dmsf/extra/xapian_indexer.rb for help.
### WebDAV caching (optional, experimental!)
Creating the file lists for the WebDAV takes a lot of resources, for folders with many files it can take several seconds
and for clients that don't cache the lists (Windows WebClient!) a new list must be created every time you browse into that folder, even if nothing has changed in the folder so browsing a WebDAV share in Windows is not a pleasant experience.
By enabling caching the response time can be significantly reduced from several seconds for folders with hundreds of items down to a few milliseconds.
To enable caching you must have a memcached server installed.
Follow the installation instructions at <https://github.com/memcached/memcached/wiki>, write the address/ip to the memcached server in the DMSF plugin configuration and then restart Redmine.
If you installed your memcached server on the same machine as your Redmine installation then you can use 'localhost' as the memcached server address.
Only one server is supported, and it has only been tested using 'localhost'.
To disable caching just clear the memcached server address and restart Redmine.
Uninstalling DMSF
-----------------
Before uninstalling the DMSF plugin, please ensure that the Redmine instance is stopped.

29
app/models/dmsf_file.rb
View File

@ -478,4 +478,33 @@ class DmsfFile < ActiveRecord::Base
nil
end
def save(*args)
RedmineDmsf::Webdav::Cache.invalidate_item(propfind_cache_key)
super(*args)
end
def save!(*args)
RedmineDmsf::Webdav::Cache.invalidate_item(propfind_cache_key)
super(*args)
end
def destroy
RedmineDmsf::Webdav::Cache.invalidate_item(propfind_cache_key)
super
end
def destroy!
RedmineDmsf::Webdav::Cache.invalidate_item(propfind_cache_key)
super
end
def propfind_cache_key
if dmsf_folder_id.nil?
# File is in project root
return "PROPFIND/#{project_id}"
else
return "PROPFIND/#{project_id}/#{dmsf_folder_id}"
end
end
end

17
app/models/dmsf_file_revision.rb
View File

@ -109,6 +109,7 @@ class DmsfFileRevision < ActiveRecord::Base
dependencies = DmsfFileRevision.where(:disk_filename => self.disk_filename).all.count
File.delete(self.disk_file) if dependencies <= 1 && File.exist?(self.disk_file)
end
RedmineDmsf::Webdav::Cache.invalidate_item(propfind_cache_key)
super
end
@ -290,5 +291,19 @@ class DmsfFileRevision < ActiveRecord::Base
end
ActionView::Base.full_sanitizer.sanitize(text)
end
def save(*args)
RedmineDmsf::Webdav::Cache.invalidate_item(propfind_cache_key)
super(*args)
end
end
def save!(*args)
RedmineDmsf::Webdav::Cache.invalidate_item(propfind_cache_key)
super(*args)
end
def propfind_cache_key
dmsf_file.propfind_cache_key
end
end

33
app/models/dmsf_folder.rb
View File

@ -295,7 +295,7 @@ class DmsfFolder < ActiveRecord::Base
file_links.visible.count +
url_links.visible.count
end
def self.is_column_on?(column)
columns = Setting.plugin_redmine_dmsf['dmsf_columns']
columns = DmsfFolder::DEFAULT_COLUMNS unless columns
@ -395,6 +395,35 @@ class DmsfFolder < ActiveRecord::Base
nil
end
def save(*args)
RedmineDmsf::Webdav::Cache.invalidate_item(propfind_cache_key)
super(*args)
end
def save!(*args)
RedmineDmsf::Webdav::Cache.invalidate_item(propfind_cache_key)
super(*args)
end
def destroy
RedmineDmsf::Webdav::Cache.invalidate_item(propfind_cache_key)
super
end
def destroy!
RedmineDmsf::Webdav::Cache.invalidate_item(propfind_cache_key)
super
end
def propfind_cache_key
if dmsf_folder_id.nil?
# Folder is in project root
return "PROPFIND/#{project_id}"
else
return "PROPFIND/#{project_id}/#{dmsf_folder_id}"
end
end
private
def self.directory_subtree(tree, folder, level, current_folder)
@ -406,4 +435,4 @@ class DmsfFolder < ActiveRecord::Base
end
end
end
end

7
app/views/settings/_dmsf_settings.html.erb
View File

@ -176,6 +176,13 @@
<%= l(:note_webdav_strategy).html_safe %> <%= l(:label_default) %>: <%= l(:select_option_webdav_readonly) %>
</em>
</p>
<p>
<%= content_tag(:label, l(:label_memcached_servers)) %>
<%= text_field_tag 'settings[dmsf_memcached_servers]', @settings['dmsf_memcached_servers'], :size => 50 %>
<em class="info">
<%= l(:text_memcached_servers) %>
</em>
</p>
<% end %>
<hr/>

6
config/locales/en.yml
View File

@ -338,4 +338,8 @@ en:
label_webdav: WebDAV
label_full_text: Full-text search
link_extension: Ext
link_extension: Ext
label_memcached_servers: "memcached server address"
text_memcached_servers: "Address to memcached server. Only a single server is supported, if empty then caching is disabled. After changing this the server must be restarted!"

3
init.rb
View File

@ -47,7 +47,8 @@ Redmine::Plugin.register :redmine_dmsf do
'dmsf_webdav' => '1',
'dmsf_display_notified_recipients' => 0,
'dmsf_global_title_format' => '',
'dmsf_columns' => %w(title size modified version workflow author)
'dmsf_columns' => %w(title size modified version workflow author),
'dmsf_memcached_servers' => ''
}
menu :project_menu, :dmsf, { :controller => 'dmsf', :action => 'show' }, :caption => :menu_dmsf, :before => :documents, :param => :id

1
lib/redmine_dmsf.rb
View File

@ -35,6 +35,7 @@ require 'redmine_dmsf/patches/user_patch'
# Load up classes that make up our WebDAV solution ontop of DAV4Rack
require 'redmine_dmsf/webdav/base_resource'
require 'redmine_dmsf/webdav/controller'
require 'redmine_dmsf/webdav/cache'
require 'redmine_dmsf/webdav/dmsf_resource'
require 'redmine_dmsf/webdav/download'
require 'redmine_dmsf/webdav/index_resource'

83
lib/redmine_dmsf/webdav/cache.rb Normal file
View File

@ -0,0 +1,83 @@
# encoding: utf-8
#
# Redmine plugin for Document Management System "Features"
#
# Copyright (C) 2012 Daniel Munn <dan.munn@munnster.co.uk>
# Copyright (C) 2011-16 Karel Picman <karel.picman@kontron.com>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
module RedmineDmsf
module Webdav
class Cache
def self.read(name, options = nil)
init unless defined?(@@WebDAVCache)
@@WebDAVCache.read(name, options)
end
def self.write(name, value, options = nil)
init unless defined?(@@WebDAVCache)
@@WebDAVCache.write(name, value, options)
end
def self.delete(name, options = nil)
init unless defined?(@@WebDAVCache)
@@WebDAVCache.delete(name, options)
end
def self.exist?(name, options = nil)
init unless defined?(@@WebDAVCache)
@@WebDAVCache.exist?(name, options)
end
def self.invalidate_item(key)
init unless defined?(@@WebDAVCache)
# Write an .invalid entry to notify anyone that is currently creating a response
# that that response is invalid and should not be cached
@@WebDAVCache.write("#{key}.invalid", expires_in: 60.seconds)
# Delete any existing entry in the cache
@@WebDAVCache.delete(key)
end
def self.cache
@@WebDAVCache
end
def self.init_testcache
puts "Webdav::Cache: Enable MemoryStore cache."
@@WebDAVCache = ActiveSupport::Cache::MemoryStore.new(options={:namespace => "RedmineDmsfWebDAV"})
end
def self.init_nullcache
puts "Webdav::Cache: Disable cache."
@@WebDAVCache = ActiveSupport::Cache::NullStore.new
end
private
def self.init
if Setting.plugin_redmine_dmsf['dmsf_memcached_servers'].nil? || Setting.plugin_redmine_dmsf['dmsf_memcached_servers'].empty?
# Disable caching by using a null cache
Rails.logger.info "Webdav::Cache: Cache disabled!"
@@WebDAVCache = ActiveSupport::Cache::NullStore.new
else
# Create cache using the provided server address
Rails.logger.info "Webdav::Cache: Cache enabled, using memcached server '#{Setting.plugin_redmine_dmsf['dmsf_memcached_servers']}'"
@@WebDAVCache = ActiveSupport::Cache::MemCacheStore.new(Setting.plugin_redmine_dmsf['dmsf_memcached_servers'], options={:namespace => "RedmineDmsfWebDAV"})
end
end
end
end
end

97
lib/redmine_dmsf/webdav/controller.rb
View File

@ -107,18 +107,44 @@ module RedmineDmsf
raise BadRequest
end
end
multistatus do |xml|
find_resources.each do |resource|
xml.response do
unless(resource.propstat_relative_path)
xml.href "#{scheme}://#{host}:#{port}#{url_format(resource)}"
else
xml.href url_format(resource)
end
propstats(xml, get_properties(resource, properties.empty? ? resource.properties : properties))
if depth != 0
# Only use cache for requests with a depth>0, depth=0 responses are already fast.
pinfo = resource.path.split('/').drop(1)
if (pinfo.length == 0) # If this is the base_path, we're at root
# Don't know when projects are added/removed from the visibility list for this user,
# so don't cache root.
elsif (pinfo.length == 1) #This is first level, and as such, project path
propfind_key = "PROPFIND/#{resource.resource.project_id}"
else # We made it all the way to DMSF Data
if resource.collection?
# Only store collections in the cache since responses to files are simple and fast already.
propfind_key = "PROPFIND/#{resource.resource.project_id}/#{resource.resource.folder.id}"
end
end
end
end
if propfind_key.nil?
# This PROPFIND is never cached so always create a new response
create_propfind_response(properties)
else
response.body = RedmineDmsf::Webdav::Cache.read(propfind_key)
if !response.body.nil?
# Found cached PROPFIND, fill in Content-Type and Content-Length
response["Content-Type"] = 'text/xml; charset="utf-8"'
response["Content-Length"] = response.body.size.to_s
else
# No cached PROPFIND found
# Remove .invalid entry for this propfind since we are now creating a new valid propfind
RedmineDmsf::Webdav::Cache.delete("#{propfind_key}.invalid")
create_propfind_response(properties)
# Cache response.body, but only if no .invalid entry was stored while creating the propfind
RedmineDmsf::Webdav::Cache.write(propfind_key, response.body) unless RedmineDmsf::Webdav::Cache.exist?("#{propfind_key}.invalid")
end
end
# Return HTTP code.
MultiStatus
end
end
@ -179,6 +205,57 @@ module RedmineDmsf
Addressable::URI.unescape str
end
private
def create_propfind_response(properties)
# Generate response, is stored in response.body
render_xml(:multistatus) do |xml|
find_resources.each do |resource|
if resource.collection?
# Index, Project or Folder
# path is unique enough for the key and is available for all three, and the path doesn't change
# for this path as long as it stays. On its path. The path does not stray from its path without
# changing its path.
propstats_key = "PROPSTATS/#{resource.path}"
else
# File
# Use file.id & file.last_revision.id as key
# When revision changes then the key will change and the old cached item will eventually be evicted
propstats_key = "PROPSTATS/#{resource.resource.file.id}-#{resource.resource.file.last_revision.id}"
end
xml_str = RedmineDmsf::Webdav::Cache.read(propstats_key)
if xml_str.nil?
# Create the complete PROPSTATS response
propstats_builder = Nokogiri::XML::Builder.new do |propstats_xml|
propstats_xml.send('propstat', {'xmlns:D' => 'DAV:'}.merge(resource.root_xml_attributes)) do
propstats_xml.parent.namespace = propstats_xml.parent.namespace_definitions.first
xml2 = propstats_xml['D']
xml2.response do
unless(resource.propstat_relative_path)
xml2.href "#{scheme}://#{host}:#{port}#{url_format(resource)}"
else
xml2.href url_format(resource)
end
propstats(xml2, get_properties(resource, properties.empty? ? resource.properties : properties))
end
end
end
# Just want to add the <:D:response> so extract it.
# Q: Is there a better/faster way to do this?
xml_str = Nokogiri::XML.parse(propstats_builder.to_xml).xpath('//D:response').first.to_xml
# Add PROPSTATS to cache
# Caching the PROPSTATS response as xml text string.
RedmineDmsf::Webdav::Cache.write(propstats_key, xml_str)
end
xml << xml_str
end
end
end
end
end
end

4
lib/redmine_dmsf/webdav/dmsf_resource.rb
View File

@ -79,6 +79,10 @@ module RedmineDmsf
def really_exist?
return project && project.module_enabled?('dmsf') && (folder || file)
end
def project_id
project.id unless project.nil?
end
# Is this entity a folder?
def collection?

4
lib/redmine_dmsf/webdav/project_resource.rb
View File

@ -78,6 +78,10 @@ module RedmineDmsf
def long_name
project.name unless project.nil?
end
def project_id
project.id unless project.nil?
end
def content_type
'inode/directory'

4
lib/redmine_dmsf/webdav/resource_proxy.rb
View File

@ -84,6 +84,10 @@ module RedmineDmsf
def really_exist?
@resource_c.really_exist?
end
def project_id
@resource_c.project_id
end
def creation_date
@resource_c.creation_date

41
test/unit/dmsf_file_revision_test.rb
View File

@ -25,11 +25,15 @@ class DmsfFileRevisionTest < RedmineDmsf::Test::UnitTest
:member_roles, :enabled_modules, :enumerations, :dmsf_locks
def setup
@revision5 = DmsfFileRevision.find_by_id 5
@revision1 = DmsfFileRevision.find_by_id 1
@revision2 = DmsfFileRevision.find_by_id 2
@revision5 = DmsfFileRevision.find_by_id 5
end
def test_truth
assert_kind_of DmsfFileRevision, @revision5
assert_kind_of DmsfFileRevision, @revision1
assert_kind_of DmsfFileRevision, @revision2
assert_kind_of DmsfFileRevision, @revision5
end
def test_delete_restore
@ -49,5 +53,38 @@ class DmsfFileRevisionTest < RedmineDmsf::Test::UnitTest
def test_create_digest
assert_equal @revision5.create_digest, 0, "MD5 should be 0, if the file is missing"
end
def test_save_and_destroy_with_cache
RedmineDmsf::Webdav::Cache.init_testcache
# save
cache_key = @revision1.propfind_cache_key
RedmineDmsf::Webdav::Cache.write(cache_key, "")
assert RedmineDmsf::Webdav::Cache.exist?(cache_key)
assert !RedmineDmsf::Webdav::Cache.exist?("#{cache_key}.invalid")
@revision1.save
assert !RedmineDmsf::Webdav::Cache.exist?(cache_key)
assert RedmineDmsf::Webdav::Cache.exist?("#{cache_key}.invalid")
RedmineDmsf::Webdav::Cache.delete("#{cache_key}.invalid")
# destroy
RedmineDmsf::Webdav::Cache.write(cache_key, "")
assert RedmineDmsf::Webdav::Cache.exist?(cache_key)
assert !RedmineDmsf::Webdav::Cache.exist?("#{cache_key}.invalid")
@revision1.destroy
assert !RedmineDmsf::Webdav::Cache.exist?(cache_key)
assert RedmineDmsf::Webdav::Cache.exist?("#{cache_key}.invalid")
# save!
cache_key = @revision2.propfind_cache_key
RedmineDmsf::Webdav::Cache.write(cache_key, "")
assert RedmineDmsf::Webdav::Cache.exist?(cache_key)
assert !RedmineDmsf::Webdav::Cache.exist?("#{cache_key}.invalid")
@revision2.save!
assert !RedmineDmsf::Webdav::Cache.exist?(cache_key)
assert RedmineDmsf::Webdav::Cache.exist?("#{cache_key}.invalid")
RedmineDmsf::Webdav::Cache.init_nullcache
end
end

44
test/unit/dmsf_file_test.rb
View File

@ -35,6 +35,7 @@ class DmsfFileTest < RedmineDmsf::Test::UnitTest
@file3 = DmsfFile.find_by_id 3
@file4 = DmsfFile.find_by_id 4
@file5 = DmsfFile.find_by_id 5
@file6 = DmsfFile.find_by_id 6
User.current = nil
end
@ -47,6 +48,7 @@ class DmsfFileTest < RedmineDmsf::Test::UnitTest
assert_kind_of DmsfFile, @file3
assert_kind_of DmsfFile, @file4
assert_kind_of DmsfFile, @file5
assert_kind_of DmsfFile, @file6
end
def test_project_file_count_differs_from_project_visibility_count
@ -134,5 +136,47 @@ class DmsfFileTest < RedmineDmsf::Test::UnitTest
assert_equal 0, @file4.referenced_links.count
@file4.dmsf_folder.lock!
end
def test_save_and_destroy_with_cache
RedmineDmsf::Webdav::Cache.init_testcache
# save
cache_key = @file5.propfind_cache_key
RedmineDmsf::Webdav::Cache.write(cache_key, "")
assert RedmineDmsf::Webdav::Cache.exist?(cache_key)
assert !RedmineDmsf::Webdav::Cache.exist?("#{cache_key}.invalid")
@file5.save
assert !RedmineDmsf::Webdav::Cache.exist?(cache_key)
assert RedmineDmsf::Webdav::Cache.exist?("#{cache_key}.invalid")
RedmineDmsf::Webdav::Cache.delete("#{cache_key}.invalid")
# destroy
RedmineDmsf::Webdav::Cache.write(cache_key, "")
assert RedmineDmsf::Webdav::Cache.exist?(cache_key)
assert !RedmineDmsf::Webdav::Cache.exist?("#{cache_key}.invalid")
@file5.destroy
assert !RedmineDmsf::Webdav::Cache.exist?(cache_key)
assert RedmineDmsf::Webdav::Cache.exist?("#{cache_key}.invalid")
# save!
cache_key = @file6.propfind_cache_key
RedmineDmsf::Webdav::Cache.write(cache_key, "")
assert RedmineDmsf::Webdav::Cache.exist?(cache_key)
assert !RedmineDmsf::Webdav::Cache.exist?("#{cache_key}.invalid")
@file6.save!
assert !RedmineDmsf::Webdav::Cache.exist?(cache_key)
assert RedmineDmsf::Webdav::Cache.exist?("#{cache_key}.invalid")
RedmineDmsf::Webdav::Cache.delete("#{cache_key}.invalid")
# destroy!
RedmineDmsf::Webdav::Cache.write(cache_key, "")
assert RedmineDmsf::Webdav::Cache.exist?(cache_key)
assert !RedmineDmsf::Webdav::Cache.exist?("#{cache_key}.invalid")
@file6.destroy!
assert !RedmineDmsf::Webdav::Cache.exist?(cache_key)
assert RedmineDmsf::Webdav::Cache.exist?("#{cache_key}.invalid")
RedmineDmsf::Webdav::Cache.init_nullcache
end
end

48
test/unit/dmsf_folder_test.rb
View File

@ -25,10 +25,14 @@ class DmsfFolderTest < RedmineDmsf::Test::UnitTest
fixtures :projects, :users, :email_addresses, :dmsf_folders, :roles, :members, :member_roles
def setup
@folder4 = DmsfFolder.find_by_id 4
@folder5 = DmsfFolder.find_by_id 5
@folder6 = DmsfFolder.find_by_id 6
end
def test_truth
assert_kind_of DmsfFolder, @folder4
assert_kind_of DmsfFolder, @folder5
assert_kind_of DmsfFolder, @folder6
end
@ -96,5 +100,47 @@ class DmsfFolderTest < RedmineDmsf::Test::UnitTest
assert_equal DmsfFolder.get_column_position('version_calculated'), 11,
"The expected position of the 'version_calculated' column is 14"
end
def test_save_and_destroy_with_cache
RedmineDmsf::Webdav::Cache.init_testcache
# save
cache_key = @folder4.propfind_cache_key
RedmineDmsf::Webdav::Cache.write(cache_key, "")
assert RedmineDmsf::Webdav::Cache.exist?(cache_key)
assert !RedmineDmsf::Webdav::Cache.exist?("#{cache_key}.invalid")
@folder4.save
assert !RedmineDmsf::Webdav::Cache.exist?(cache_key)
assert RedmineDmsf::Webdav::Cache.exist?("#{cache_key}.invalid")
RedmineDmsf::Webdav::Cache.delete("#{cache_key}.invalid")
# destroy
RedmineDmsf::Webdav::Cache.write(cache_key, "")
assert RedmineDmsf::Webdav::Cache.exist?(cache_key)
assert !RedmineDmsf::Webdav::Cache.exist?("#{cache_key}.invalid")
@folder4.destroy
assert !RedmineDmsf::Webdav::Cache.exist?(cache_key)
assert RedmineDmsf::Webdav::Cache.exist?("#{cache_key}.invalid")
RedmineDmsf::Webdav::Cache.cache.clear
# save!
cache_key = @folder5.propfind_cache_key
RedmineDmsf::Webdav::Cache.write(cache_key, "")
assert RedmineDmsf::Webdav::Cache.exist?(cache_key)
assert !RedmineDmsf::Webdav::Cache.exist?("#{cache_key}.invalid")
@folder5.save!
assert !RedmineDmsf::Webdav::Cache.exist?(cache_key)
assert RedmineDmsf::Webdav::Cache.exist?("#{cache_key}.invalid")
RedmineDmsf::Webdav::Cache.delete("#{cache_key}.invalid")
# destroy!
RedmineDmsf::Webdav::Cache.write(cache_key, "")
assert RedmineDmsf::Webdav::Cache.exist?(cache_key)
assert !RedmineDmsf::Webdav::Cache.exist?("#{cache_key}.invalid")
@folder5.destroy!
assert !RedmineDmsf::Webdav::Cache.exist?(cache_key)
assert RedmineDmsf::Webdav::Cache.exist?("#{cache_key}.invalid")
RedmineDmsf::Webdav::Cache.init_nullcache
end
end