redmine_plugins/redmine_dmsf
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

#1150 Uploading big files

Browse Source
This commit is contained in:
karel.picman@lbcfree.net 2020-07-16 13:48:33 +02:00
parent eaf55b35f6
commit a334dd7a9e
14 changed files with 45 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
after_init.rb
View File

@ -67,8 +67,8 @@ def dmsf_init
pmap.permission :file_manipulation,
{ dmsf_files: [:create_revision, :lock, :unlock, :delete_revision, :obsolete_revision,
:notify_activate, :notify_deactivate, :restore],
dmsf_upload: [:upload_files, :upload_file, :upload, :commit_files, :commit,
:delete_dmsf_attachment, :delete_dmsf_link_attachment, :multi_upload],
dmsf_upload: [:upload_files, :upload, :commit_files, :commit, :delete_dmsf_attachment,
:delete_dmsf_link_attachment, :multi_upload],
dmsf_links: [:new, :create, :destroy, :restore, :autocomplete_for_project,
:autocomplete_for_folder],
dmsf_files_copy: [:new, :copy, :move],

6
app/controllers/dmsf_files_controller.rb
View File

@ -128,17 +128,11 @@ class DmsfFilesController < ApplicationController
revision.size = upload.size
revision.disk_filename = revision.new_storage_filename
revision.mime_type = upload.mime_type
revision.digest = DmsfFileRevision.create_digest upload.tempfile_path
end
else
revision.size = last_revision.size
revision.disk_filename = last_revision.disk_filename
revision.mime_type = last_revision.mime_type
if last_revision.digest.blank?
revision.digest = DmsfFileRevision.create_digest last_revision.disk_file
else
revision.digest = last_revision.digest
end
end
# Custom fields

28
app/controllers/dmsf_upload_controller.rb
View File

@ -27,10 +27,8 @@ class DmsfUploadController < ApplicationController
before_action :find_project, except: [:upload, :delete_dmsf_attachment, :delete_dmsf_link_attachment]
before_action :authorize, except: [:upload, :delete_dmsf_attachment, :delete_dmsf_link_attachment]
before_action :authorize_global, only: [:upload, :delete_dmsf_attachment, :delete_dmsf_link_attachment]
before_action :find_folder, except: [:upload_file, :upload, :commit, :delete_dmsf_attachment,
:delete_dmsf_link_attachment]
before_action :permissions, except: [:upload_file, :upload, :commit, :delete_dmsf_attachment,
:delete_dmsf_link_attachment]
before_action :find_folder, except: [:upload, :commit, :delete_dmsf_attachment, :delete_dmsf_link_attachment]
before_action :permissions, except: [:upload, :commit, :delete_dmsf_attachment, :delete_dmsf_link_attachment]
helper :all
helper :dmsf_workflows
@ -57,27 +55,6 @@ class DmsfUploadController < ApplicationController
end
end
# async single file upload handling
def upload_file
@tempfile = params[:file]
unless @tempfile.original_filename
render_404
return
end
@disk_filename = DmsfHelper.temp_filename(@tempfile.original_filename)
@tempfile_path = DmsfHelper.temp_dir.join(@disk_filename).to_s
File.open(@tempfile_path, 'wb') do |f|
if params[:file].respond_to?(:read)
while (buffer = @tempfile.read(8192))
f.write(buffer)
end
else
f.write(@tempfile)
end
end
render layout: false
end
# REST API and Redmine attachment form
def upload
unless request.content_type == 'application/octet-stream'
@ -124,6 +101,7 @@ class DmsfUploadController < ApplicationController
uploaded_file[:disk_filename] = upload.disk_filename
uploaded_file[:tempfile_path] = upload.tempfile_path
uploaded_file[:size] = upload.size
uploaded_file[:digest] = upload.digest
end
end
commit_files_internal uploaded_files

6
app/helpers/dmsf_upload_helper.rb
View File

@ -80,7 +80,7 @@ module DmsfUploadHelper
end
new_revision.mime_type = commited_file[:mime_type]
new_revision.size = commited_file[:size]
new_revision.digest = DmsfFileRevision.create_digest commited_file[:tempfile_path]
new_revision.digest = commited_file[:digest]
if commited_file[:custom_field_values].present?
i = 0
@ -96,12 +96,12 @@ module DmsfUploadHelper
new_revision.disk_filename = new_revision.new_storage_filename
else
Rails.logger.error (new_revision.errors.full_messages + file.errors.full_messages).to_sentence
failed_uploads.push(commited_file)
failed_uploads.push commited_file
next
end
if new_revision.save
new_revision.assign_workflow(commited_file[:dmsf_workflow_id])
new_revision.assign_workflow commited_file[:dmsf_workflow_id]
begin
FileUtils.mv commited_file[:tempfile_path], new_revision.disk_file(false)
FileUtils.chmod 'u=wr,g=r', new_revision.disk_file(false)

23
app/models/dmsf_file_revision.rb
View File

@ -268,11 +268,19 @@ class DmsfFileRevision < ActiveRecord::Base
end
def copy_file_content(open_file)
sha = Digest::SHA256.new
File.open(disk_file(false), 'wb') do |f|
if open_file.respond_to?(:read)
while (buffer = open_file.read(8192))
f.write(buffer)
f.write buffer
sha.update buffer
end
else
f.write open_file
sha.update open_file
end
end
self.digest = sha.hexdigest
end
# Overrides Redmine::Acts::Customizable::InstanceMethods#available_custom_fields
@ -304,19 +312,6 @@ class DmsfFileRevision < ActiveRecord::Base
format2
end
def self.create_digest(path)
begin
Digest::SHA256.file(path).hexdigest
rescue => e
Rails.logger.error e.message
0
end
end
def create_digest
self.digest = DmsfFileRevision.create_digest(disk_file)
end
# Returns either MD5 or SHA256 depending on the way self.digest was computed
def digest_type
digest.size < 64 ? 'MD5' : 'SHA256' if digest.present?

5
app/models/dmsf_upload.rb
View File

@ -35,6 +35,7 @@ class DmsfUpload
attr_accessor :workflow
attr_accessor :custom_values
attr_accessor :tempfile_path
attr_accessor :digest
def disk_file
DmsfHelper.temp_dir.join(disk_filename).to_s
@ -48,7 +49,8 @@ class DmsfUpload
content_type: a.content_type,
original_filename: a.filename,
comment: uploaded_file[:description],
tempfile_path: a.diskfile
tempfile_path: a.diskfile,
digest: a.digest
}
DmsfUpload.new project, folder, uploaded
else
@ -74,6 +76,7 @@ class DmsfUpload
Rails.logger.error "Cannot find #{uploaded[:tempfile_path]}"
end
@tempfile_path = uploaded[:tempfile_path]
@digest = uploaded[:digest]
if file.nil? || file.last_revision.nil?
@title = DmsfFileRevision.filename_to_title(@name)

1
app/views/dmsf_upload/_upload_file.html.erb
View File

@ -27,6 +27,7 @@
<%= hidden_field_tag "commited_files[#{i}][tempfile_path]", upload.tempfile_path %>
<%= hidden_field_tag "commited_files[#{i}][size]", upload.size %>
<%= hidden_field_tag "commited_files[#{i}][mime_type]", upload.mime_type %>
<%= hidden_field_tag "commited_files[#{i}][digest]", upload.digest %>
<div class="clear">
<div class="splitcontentleft">
<p>

1
app/views/dmsf_upload/_upload_file_locked.html.erb
View File

@ -28,6 +28,7 @@
<%= hidden_field_tag "commited_files[#{i}][size]", upload.size %>
<%= hidden_field_tag "commited_files[#{i}][mime_type]", upload.mime_type %>
<%= hidden_field_tag "commited_files[#{i}][name]", upload.name %>
<%= hidden_field_tag "commited_files[#{i}][digest]", upload.digest %>
<p class="warning"><%= l(:info_file_locked) %></p>
<div class="clear">
<div class="splitcontentleft">

1
app/views/dmsf_upload/upload_file.html.erb
View File

@ -27,4 +27,5 @@
"content_type": "<%= @tempfile.content_type.gsub('"', '').html_safe %>",
"disk_filename": "<%= @disk_filename.html_safe %>",
"tempfile_path": "<%= @tempfile_path.html_safe %>"
"digest": "<%= @digest.html_safe %>"
}

1
config/routes.rb
View File

@ -73,7 +73,6 @@ if Redmine::Plugin.installed? :redmine_dmsf
get '/projects/:id/dmsf/upload/multi_upload', controller: 'dmsf_upload', action: 'multi_upload', as: 'multi_dmsf_upload'
post '/projects/:id/dmsf/upload/files', controller: 'dmsf_upload', action: 'upload_files'
get '/projects/:id/dmsf/upload/files', controller: 'dmsf_upload', action: 'upload_files'
post '/projects/:id/dmsf/upload/file', controller: 'dmsf_upload', action: 'upload_file'
post '/projects/:id/dmsf/upload', controller: 'dmsf_upload', action: 'upload'
post '/projects/:id/dmsf/upload/commit', controller: 'dmsf_upload', action: 'commit_files'
post '/projects/:id/dmsf/commit', controller: 'dmsf_upload', action: 'commit'

3
extra/api/api_client.sh
View File

@ -1,3 +1,4 @@
#!/bin/bash
# encoding: utf-8
#
# Redmine plugin for Document Management System "Features"
@ -18,8 +19,6 @@
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#!/bin/bash
# Authentication as input parameters either as login + password or the API key
# -u ${1}:${2}
# -H "X-Redmine-API-Key: ${1}"

1
lib/redmine_dmsf/webdav/dmsf_resource.rb
View File

@ -578,7 +578,6 @@ module RedmineDmsf
if new_revision.save
new_revision.copy_file_content(request.body)
new_revision.create_digest
new_revision.save
# Notifications
DmsfMailer.deliver_files_updated(project, [f])

23
lib/tasks/dmsf_create_digests.rake
View File

@ -35,25 +35,38 @@ END_DESC
namespace :redmine do
task :dmsf_create_digests => :environment do
m = DmsfDigest.new
m.create_digests
m = DmsfCreateDigest.new
m.dmsf_create_digests
end
end
class DmsfDigest
class DmsfCreateDigest
def initialize
@dry_run = ENV['dry_run']
@force_sha256 = ENV['forceSHA256']
end
def create_digests
def dmsf_create_digests
revisions = DmsfFileRevision.where(['digest IS NULL OR length(digest) < ?', @force_sha256 ? 64 : 32])
count = revisions.count
n = 0
revisions.each_with_index do |rev, i|
rev.create_digest
if File.exist?(rev.disk_file)
sha = Digest::SHA256.new
file = File.new rev.disk_file, 'r'
if file.respond_to?(:read)
while (buffer = file.read(8192))
sha.update buffer
end
else
sha.update @temp_file
end
rev.digest = sha.hexdigest
rev.save unless @dry_run
else
puts "#{rev.disk_file} not found"
end
n += 1
# Progress bar
print "\r#{i * 100 / count}%"

9
test/unit/dmsf_file_revision_test.rb
View File

@ -77,18 +77,9 @@ class DmsfFileRevisionTest < RedmineDmsf::Test::UnitTest
assert_nil DmsfFileRevision.find_by(id: @revision5.id)
end
def test_create_digest
@revision1.create_digest
assert_equal 'SHA256', @revision1.digest_type
assert_equal @revision8.create_digest, 0, 'Digest should be 0, if the file is missing'
end
def test_digest_type
# Old type MD5
assert_equal 'MD5', @revision1.digest_type
# New type SHA256
@revision1.create_digest
assert_equal 'SHA256', @revision1.digest_type
end
def test_new_storage_filename