From a334dd7a9ef3ef65ab7d0952e3e6c5bf8f0b6e3c Mon Sep 17 00:00:00 2001 From: "karel.picman@lbcfree.net" Date: Thu, 16 Jul 2020 13:48:33 +0200 Subject: [PATCH] #1150 Uploading big files --- after_init.rb | 4 +-- app/controllers/dmsf_files_controller.rb | 6 ---- app/controllers/dmsf_upload_controller.rb | 28 ++----------------- app/helpers/dmsf_upload_helper.rb | 6 ++-- app/models/dmsf_file_revision.rb | 25 +++++++---------- app/models/dmsf_upload.rb | 5 +++- app/views/dmsf_upload/_upload_file.html.erb | 1 + .../dmsf_upload/_upload_file_locked.html.erb | 1 + app/views/dmsf_upload/upload_file.html.erb | 1 + config/routes.rb | 1 - extra/api/api_client.sh | 3 +- lib/redmine_dmsf/webdav/dmsf_resource.rb | 1 - lib/tasks/dmsf_create_digests.rake | 25 +++++++++++++---- test/unit/dmsf_file_revision_test.rb | 9 ------ 14 files changed, 45 insertions(+), 71 deletions(-) diff --git a/after_init.rb b/after_init.rb index f4f5dd5e..aa58ed38 100644 --- a/after_init.rb +++ b/after_init.rb @@ -67,8 +67,8 @@ def dmsf_init pmap.permission :file_manipulation, { dmsf_files: [:create_revision, :lock, :unlock, :delete_revision, :obsolete_revision, :notify_activate, :notify_deactivate, :restore], - dmsf_upload: [:upload_files, :upload_file, :upload, :commit_files, :commit, - :delete_dmsf_attachment, :delete_dmsf_link_attachment, :multi_upload], + dmsf_upload: [:upload_files, :upload, :commit_files, :commit, :delete_dmsf_attachment, + :delete_dmsf_link_attachment, :multi_upload], dmsf_links: [:new, :create, :destroy, :restore, :autocomplete_for_project, :autocomplete_for_folder], dmsf_files_copy: [:new, :copy, :move], diff --git a/app/controllers/dmsf_files_controller.rb b/app/controllers/dmsf_files_controller.rb index b12e138a..8591ab60 100644 --- a/app/controllers/dmsf_files_controller.rb +++ b/app/controllers/dmsf_files_controller.rb @@ -128,17 +128,11 @@ class DmsfFilesController < ApplicationController revision.size = upload.size revision.disk_filename = revision.new_storage_filename revision.mime_type = upload.mime_type - revision.digest = DmsfFileRevision.create_digest upload.tempfile_path end else revision.size = last_revision.size revision.disk_filename = last_revision.disk_filename revision.mime_type = last_revision.mime_type - if last_revision.digest.blank? - revision.digest = DmsfFileRevision.create_digest last_revision.disk_file - else - revision.digest = last_revision.digest - end end # Custom fields diff --git a/app/controllers/dmsf_upload_controller.rb b/app/controllers/dmsf_upload_controller.rb index 0e4e1144..50bac58b 100644 --- a/app/controllers/dmsf_upload_controller.rb +++ b/app/controllers/dmsf_upload_controller.rb @@ -27,10 +27,8 @@ class DmsfUploadController < ApplicationController before_action :find_project, except: [:upload, :delete_dmsf_attachment, :delete_dmsf_link_attachment] before_action :authorize, except: [:upload, :delete_dmsf_attachment, :delete_dmsf_link_attachment] before_action :authorize_global, only: [:upload, :delete_dmsf_attachment, :delete_dmsf_link_attachment] - before_action :find_folder, except: [:upload_file, :upload, :commit, :delete_dmsf_attachment, - :delete_dmsf_link_attachment] - before_action :permissions, except: [:upload_file, :upload, :commit, :delete_dmsf_attachment, - :delete_dmsf_link_attachment] + before_action :find_folder, except: [:upload, :commit, :delete_dmsf_attachment, :delete_dmsf_link_attachment] + before_action :permissions, except: [:upload, :commit, :delete_dmsf_attachment, :delete_dmsf_link_attachment] helper :all helper :dmsf_workflows @@ -57,27 +55,6 @@ class DmsfUploadController < ApplicationController end end - # async single file upload handling - def upload_file - @tempfile = params[:file] - unless @tempfile.original_filename - render_404 - return - end - @disk_filename = DmsfHelper.temp_filename(@tempfile.original_filename) - @tempfile_path = DmsfHelper.temp_dir.join(@disk_filename).to_s - File.open(@tempfile_path, 'wb') do |f| - if params[:file].respond_to?(:read) - while (buffer = @tempfile.read(8192)) - f.write(buffer) - end - else - f.write(@tempfile) - end - end - render layout: false - end - # REST API and Redmine attachment form def upload unless request.content_type == 'application/octet-stream' @@ -124,6 +101,7 @@ class DmsfUploadController < ApplicationController uploaded_file[:disk_filename] = upload.disk_filename uploaded_file[:tempfile_path] = upload.tempfile_path uploaded_file[:size] = upload.size + uploaded_file[:digest] = upload.digest end end commit_files_internal uploaded_files diff --git a/app/helpers/dmsf_upload_helper.rb b/app/helpers/dmsf_upload_helper.rb index 37a56fb7..aaf48f35 100644 --- a/app/helpers/dmsf_upload_helper.rb +++ b/app/helpers/dmsf_upload_helper.rb @@ -80,7 +80,7 @@ module DmsfUploadHelper end new_revision.mime_type = commited_file[:mime_type] new_revision.size = commited_file[:size] - new_revision.digest = DmsfFileRevision.create_digest commited_file[:tempfile_path] + new_revision.digest = commited_file[:digest] if commited_file[:custom_field_values].present? i = 0 @@ -96,12 +96,12 @@ module DmsfUploadHelper new_revision.disk_filename = new_revision.new_storage_filename else Rails.logger.error (new_revision.errors.full_messages + file.errors.full_messages).to_sentence - failed_uploads.push(commited_file) + failed_uploads.push commited_file next end if new_revision.save - new_revision.assign_workflow(commited_file[:dmsf_workflow_id]) + new_revision.assign_workflow commited_file[:dmsf_workflow_id] begin FileUtils.mv commited_file[:tempfile_path], new_revision.disk_file(false) FileUtils.chmod 'u=wr,g=r', new_revision.disk_file(false) diff --git a/app/models/dmsf_file_revision.rb b/app/models/dmsf_file_revision.rb index 2915f575..3dee65b0 100644 --- a/app/models/dmsf_file_revision.rb +++ b/app/models/dmsf_file_revision.rb @@ -268,11 +268,19 @@ class DmsfFileRevision < ActiveRecord::Base end def copy_file_content(open_file) + sha = Digest::SHA256.new File.open(disk_file(false), 'wb') do |f| - while (buffer = open_file.read(8192)) - f.write(buffer) + if open_file.respond_to?(:read) + while (buffer = open_file.read(8192)) + f.write buffer + sha.update buffer + end + else + f.write open_file + sha.update open_file end end + self.digest = sha.hexdigest end # Overrides Redmine::Acts::Customizable::InstanceMethods#available_custom_fields @@ -304,19 +312,6 @@ class DmsfFileRevision < ActiveRecord::Base format2 end - def self.create_digest(path) - begin - Digest::SHA256.file(path).hexdigest - rescue => e - Rails.logger.error e.message - 0 - end - end - - def create_digest - self.digest = DmsfFileRevision.create_digest(disk_file) - end - # Returns either MD5 or SHA256 depending on the way self.digest was computed def digest_type digest.size < 64 ? 'MD5' : 'SHA256' if digest.present? diff --git a/app/models/dmsf_upload.rb b/app/models/dmsf_upload.rb index 9b851ffb..aedccda0 100644 --- a/app/models/dmsf_upload.rb +++ b/app/models/dmsf_upload.rb @@ -35,6 +35,7 @@ class DmsfUpload attr_accessor :workflow attr_accessor :custom_values attr_accessor :tempfile_path + attr_accessor :digest def disk_file DmsfHelper.temp_dir.join(disk_filename).to_s @@ -48,7 +49,8 @@ class DmsfUpload content_type: a.content_type, original_filename: a.filename, comment: uploaded_file[:description], - tempfile_path: a.diskfile + tempfile_path: a.diskfile, + digest: a.digest } DmsfUpload.new project, folder, uploaded else @@ -74,6 +76,7 @@ class DmsfUpload Rails.logger.error "Cannot find #{uploaded[:tempfile_path]}" end @tempfile_path = uploaded[:tempfile_path] + @digest = uploaded[:digest] if file.nil? || file.last_revision.nil? @title = DmsfFileRevision.filename_to_title(@name) diff --git a/app/views/dmsf_upload/_upload_file.html.erb b/app/views/dmsf_upload/_upload_file.html.erb index 4df66b33..306113c7 100644 --- a/app/views/dmsf_upload/_upload_file.html.erb +++ b/app/views/dmsf_upload/_upload_file.html.erb @@ -27,6 +27,7 @@ <%= hidden_field_tag "commited_files[#{i}][tempfile_path]", upload.tempfile_path %> <%= hidden_field_tag "commited_files[#{i}][size]", upload.size %> <%= hidden_field_tag "commited_files[#{i}][mime_type]", upload.mime_type %> + <%= hidden_field_tag "commited_files[#{i}][digest]", upload.digest %>

diff --git a/app/views/dmsf_upload/_upload_file_locked.html.erb b/app/views/dmsf_upload/_upload_file_locked.html.erb index 41908b91..2c7fd992 100644 --- a/app/views/dmsf_upload/_upload_file_locked.html.erb +++ b/app/views/dmsf_upload/_upload_file_locked.html.erb @@ -28,6 +28,7 @@ <%= hidden_field_tag "commited_files[#{i}][size]", upload.size %> <%= hidden_field_tag "commited_files[#{i}][mime_type]", upload.mime_type %> <%= hidden_field_tag "commited_files[#{i}][name]", upload.name %> + <%= hidden_field_tag "commited_files[#{i}][digest]", upload.digest %>

<%= l(:info_file_locked) %>

diff --git a/app/views/dmsf_upload/upload_file.html.erb b/app/views/dmsf_upload/upload_file.html.erb index ae21ea9c..03b476e0 100644 --- a/app/views/dmsf_upload/upload_file.html.erb +++ b/app/views/dmsf_upload/upload_file.html.erb @@ -27,4 +27,5 @@ "content_type": "<%= @tempfile.content_type.gsub('"', '').html_safe %>", "disk_filename": "<%= @disk_filename.html_safe %>", "tempfile_path": "<%= @tempfile_path.html_safe %>" + "digest": "<%= @digest.html_safe %>" } diff --git a/config/routes.rb b/config/routes.rb index 234dbec4..4f09fedf 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -73,7 +73,6 @@ if Redmine::Plugin.installed? :redmine_dmsf get '/projects/:id/dmsf/upload/multi_upload', controller: 'dmsf_upload', action: 'multi_upload', as: 'multi_dmsf_upload' post '/projects/:id/dmsf/upload/files', controller: 'dmsf_upload', action: 'upload_files' get '/projects/:id/dmsf/upload/files', controller: 'dmsf_upload', action: 'upload_files' - post '/projects/:id/dmsf/upload/file', controller: 'dmsf_upload', action: 'upload_file' post '/projects/:id/dmsf/upload', controller: 'dmsf_upload', action: 'upload' post '/projects/:id/dmsf/upload/commit', controller: 'dmsf_upload', action: 'commit_files' post '/projects/:id/dmsf/commit', controller: 'dmsf_upload', action: 'commit' diff --git a/extra/api/api_client.sh b/extra/api/api_client.sh index 1aad5539..b6a46d22 100644 --- a/extra/api/api_client.sh +++ b/extra/api/api_client.sh @@ -1,3 +1,4 @@ +#!/bin/bash # encoding: utf-8 # # Redmine plugin for Document Management System "Features" @@ -18,8 +19,6 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -#!/bin/bash - # Authentication as input parameters either as login + password or the API key # -u ${1}:${2} # -H "X-Redmine-API-Key: ${1}" diff --git a/lib/redmine_dmsf/webdav/dmsf_resource.rb b/lib/redmine_dmsf/webdav/dmsf_resource.rb index 63f24460..27673594 100644 --- a/lib/redmine_dmsf/webdav/dmsf_resource.rb +++ b/lib/redmine_dmsf/webdav/dmsf_resource.rb @@ -578,7 +578,6 @@ module RedmineDmsf if new_revision.save new_revision.copy_file_content(request.body) - new_revision.create_digest new_revision.save # Notifications DmsfMailer.deliver_files_updated(project, [f]) diff --git a/lib/tasks/dmsf_create_digests.rake b/lib/tasks/dmsf_create_digests.rake index 97eb6336..a89799ec 100644 --- a/lib/tasks/dmsf_create_digests.rake +++ b/lib/tasks/dmsf_create_digests.rake @@ -35,25 +35,38 @@ END_DESC namespace :redmine do task :dmsf_create_digests => :environment do - m = DmsfDigest.new - m.create_digests + m = DmsfCreateDigest.new + m.dmsf_create_digests end end -class DmsfDigest +class DmsfCreateDigest def initialize @dry_run = ENV['dry_run'] @force_sha256 = ENV['forceSHA256'] end - def create_digests + def dmsf_create_digests revisions = DmsfFileRevision.where(['digest IS NULL OR length(digest) < ?', @force_sha256 ? 64 : 32]) count = revisions.count n = 0 revisions.each_with_index do |rev, i| - rev.create_digest - rev.save unless @dry_run + if File.exist?(rev.disk_file) + sha = Digest::SHA256.new + file = File.new rev.disk_file, 'r' + if file.respond_to?(:read) + while (buffer = file.read(8192)) + sha.update buffer + end + else + sha.update @temp_file + end + rev.digest = sha.hexdigest + rev.save unless @dry_run + else + puts "#{rev.disk_file} not found" + end n += 1 # Progress bar print "\r#{i * 100 / count}%" diff --git a/test/unit/dmsf_file_revision_test.rb b/test/unit/dmsf_file_revision_test.rb index 3487b3af..a349de3a 100644 --- a/test/unit/dmsf_file_revision_test.rb +++ b/test/unit/dmsf_file_revision_test.rb @@ -77,18 +77,9 @@ class DmsfFileRevisionTest < RedmineDmsf::Test::UnitTest assert_nil DmsfFileRevision.find_by(id: @revision5.id) end - def test_create_digest - @revision1.create_digest - assert_equal 'SHA256', @revision1.digest_type - assert_equal @revision8.create_digest, 0, 'Digest should be 0, if the file is missing' - end - def test_digest_type # Old type MD5 assert_equal 'MD5', @revision1.digest_type - # New type SHA256 - @revision1.create_digest - assert_equal 'SHA256', @revision1.digest_type end def test_new_storage_filename