#1150 Uploading big files

after_init.rb

@@ -67,8 +67,8 @@ def dmsf_init
       pmap.permission :file_manipulation,
                       { dmsf_files: [:create_revision, :lock, :unlock, :delete_revision, :obsolete_revision,
                                        :notify_activate, :notify_deactivate, :restore],
-                       dmsf_upload: [:upload_files, :upload_file, :upload, :commit_files, :commit,
+                       dmsf_upload: [:upload_files, :upload, :commit_files, :commit, :delete_dmsf_attachment,
-                                        :delete_dmsf_attachment, :delete_dmsf_link_attachment, :multi_upload],
+                                     :delete_dmsf_link_attachment, :multi_upload],
                        dmsf_links: [:new, :create, :destroy, :restore, :autocomplete_for_project,
                                        :autocomplete_for_folder],
                        dmsf_files_copy: [:new, :copy, :move],

app/controllers/dmsf_files_controller.rb

@@ -128,17 +128,11 @@ class DmsfFilesController < ApplicationController
             revision.size = upload.size
             revision.disk_filename = revision.new_storage_filename
             revision.mime_type = upload.mime_type
-            revision.digest = DmsfFileRevision.create_digest upload.tempfile_path
           end
         else
           revision.size = last_revision.size
           revision.disk_filename = last_revision.disk_filename
           revision.mime_type = last_revision.mime_type
-          if last_revision.digest.blank?
-            revision.digest = DmsfFileRevision.create_digest last_revision.disk_file
-          else
-            revision.digest = last_revision.digest
-          end
         end
 
         # Custom fields

app/controllers/dmsf_upload_controller.rb

@@ -27,10 +27,8 @@ class DmsfUploadController < ApplicationController
   before_action :find_project, except: [:upload, :delete_dmsf_attachment, :delete_dmsf_link_attachment]
   before_action :authorize, except: [:upload, :delete_dmsf_attachment, :delete_dmsf_link_attachment]
   before_action :authorize_global, only: [:upload, :delete_dmsf_attachment, :delete_dmsf_link_attachment]
-  before_action :find_folder, except: [:upload_file, :upload, :commit, :delete_dmsf_attachment,
+  before_action :find_folder, except: [:upload, :commit, :delete_dmsf_attachment, :delete_dmsf_link_attachment]
-                                       :delete_dmsf_link_attachment]
+  before_action :permissions, except: [:upload, :commit, :delete_dmsf_attachment, :delete_dmsf_link_attachment]
-  before_action :permissions, except: [:upload_file, :upload, :commit, :delete_dmsf_attachment,
-                                       :delete_dmsf_link_attachment]
 
   helper :all
   helper :dmsf_workflows
@@ -57,27 +55,6 @@ class DmsfUploadController < ApplicationController
     end
   end
 
-  # async single file upload handling
-  def upload_file
-    @tempfile = params[:file]
-    unless @tempfile.original_filename
-      render_404
-      return
-    end
-    @disk_filename = DmsfHelper.temp_filename(@tempfile.original_filename)
-    @tempfile_path = DmsfHelper.temp_dir.join(@disk_filename).to_s
-    File.open(@tempfile_path, 'wb') do |f|
-      if params[:file].respond_to?(:read)
-        while (buffer = @tempfile.read(8192))
-          f.write(buffer)
-        end
-      else
-        f.write(@tempfile)
-      end
-    end
-    render layout: false
-  end
-
   # REST API and Redmine attachment form
   def upload
     unless request.content_type == 'application/octet-stream'
@@ -124,6 +101,7 @@ class DmsfUploadController < ApplicationController
           uploaded_file[:disk_filename] = upload.disk_filename
           uploaded_file[:tempfile_path] = upload.tempfile_path
           uploaded_file[:size] = upload.size
+          uploaded_file[:digest] = upload.digest
         end
       end
       commit_files_internal uploaded_files

app/helpers/dmsf_upload_helper.rb

@@ -80,7 +80,7 @@ module DmsfUploadHelper
         end
         new_revision.mime_type = commited_file[:mime_type]
         new_revision.size = commited_file[:size]
-        new_revision.digest = DmsfFileRevision.create_digest commited_file[:tempfile_path]
+        new_revision.digest = commited_file[:digest]
 
         if commited_file[:custom_field_values].present?
           i = 0
@@ -96,12 +96,12 @@ module DmsfUploadHelper
           new_revision.disk_filename = new_revision.new_storage_filename
         else
           Rails.logger.error (new_revision.errors.full_messages + file.errors.full_messages).to_sentence
-          failed_uploads.push(commited_file)
+          failed_uploads.push commited_file
           next
         end
 
         if new_revision.save
-          new_revision.assign_workflow(commited_file[:dmsf_workflow_id])
+          new_revision.assign_workflow commited_file[:dmsf_workflow_id]
           begin
             FileUtils.mv commited_file[:tempfile_path], new_revision.disk_file(false)
             FileUtils.chmod 'u=wr,g=r', new_revision.disk_file(false)

app/models/dmsf_file_revision.rb

@@ -268,11 +268,19 @@ class DmsfFileRevision < ActiveRecord::Base
   end
 
   def copy_file_content(open_file)
+    sha = Digest::SHA256.new
     File.open(disk_file(false), 'wb') do |f|
-      while (buffer = open_file.read(8192))
+      if open_file.respond_to?(:read)
-        f.write(buffer)
+        while (buffer = open_file.read(8192))
+          f.write buffer
+          sha.update buffer
+        end
+      else
+        f.write open_file
+        sha.update open_file
       end
     end
+    self.digest = sha.hexdigest
   end
 
   # Overrides Redmine::Acts::Customizable::InstanceMethods#available_custom_fields
@@ -304,19 +312,6 @@ class DmsfFileRevision < ActiveRecord::Base
     format2
   end
 
-  def self.create_digest(path)
-    begin
-      Digest::SHA256.file(path).hexdigest
-    rescue => e
-      Rails.logger.error e.message
-      0
-    end
-  end
-
-  def create_digest
-    self.digest = DmsfFileRevision.create_digest(disk_file)
-  end
-
   # Returns either MD5 or SHA256 depending on the way self.digest was computed
   def digest_type
     digest.size < 64 ? 'MD5' : 'SHA256' if digest.present?

app/models/dmsf_upload.rb

@@ -35,6 +35,7 @@ class DmsfUpload
   attr_accessor :workflow
   attr_accessor :custom_values
   attr_accessor :tempfile_path
+  attr_accessor :digest
 
   def disk_file
     DmsfHelper.temp_dir.join(disk_filename).to_s
@@ -48,7 +49,8 @@ class DmsfUpload
         content_type: a.content_type,
         original_filename: a.filename,
         comment: uploaded_file[:description],
-        tempfile_path: a.diskfile
+        tempfile_path: a.diskfile,
+        digest: a.digest
       }
       DmsfUpload.new project, folder, uploaded
     else
@@ -74,6 +76,7 @@ class DmsfUpload
       Rails.logger.error "Cannot find #{uploaded[:tempfile_path]}"
     end
     @tempfile_path = uploaded[:tempfile_path]
+    @digest = uploaded[:digest]
 
     if file.nil? || file.last_revision.nil?
       @title = DmsfFileRevision.filename_to_title(@name)

app/views/dmsf_upload/_upload_file.html.erb

@@ -27,6 +27,7 @@
   <%= hidden_field_tag "commited_files[#{i}][tempfile_path]", upload.tempfile_path %>
   <%= hidden_field_tag "commited_files[#{i}][size]", upload.size %>
   <%= hidden_field_tag "commited_files[#{i}][mime_type]", upload.mime_type %>
+  <%= hidden_field_tag "commited_files[#{i}][digest]", upload.digest %>
   <div class="clear">
     <div class="splitcontentleft">
       <p>

app/views/dmsf_upload/_upload_file_locked.html.erb

@@ -28,6 +28,7 @@
   <%= hidden_field_tag "commited_files[#{i}][size]", upload.size %>
   <%= hidden_field_tag "commited_files[#{i}][mime_type]", upload.mime_type %>
   <%= hidden_field_tag "commited_files[#{i}][name]", upload.name %>
+  <%= hidden_field_tag "commited_files[#{i}][digest]", upload.digest %>
   <p class="warning"><%= l(:info_file_locked) %></p>
   <div class="clear">
     <div class="splitcontentleft">

app/views/dmsf_upload/upload_file.html.erb

@@ -27,4 +27,5 @@
   "content_type":     "<%= @tempfile.content_type.gsub('"', '').html_safe %>",
   "disk_filename":    "<%= @disk_filename.html_safe %>",
   "tempfile_path":    "<%= @tempfile_path.html_safe %>"
+  "digest":           "<%= @digest.html_safe %>"
 }

config/routes.rb

@@ -73,7 +73,6 @@ if Redmine::Plugin.installed? :redmine_dmsf
     get '/projects/:id/dmsf/upload/multi_upload', controller: 'dmsf_upload', action: 'multi_upload', as: 'multi_dmsf_upload'
     post '/projects/:id/dmsf/upload/files', controller: 'dmsf_upload', action: 'upload_files'
     get '/projects/:id/dmsf/upload/files', controller: 'dmsf_upload', action: 'upload_files'
-    post '/projects/:id/dmsf/upload/file', controller: 'dmsf_upload', action: 'upload_file'
     post '/projects/:id/dmsf/upload', controller: 'dmsf_upload', action: 'upload'
     post '/projects/:id/dmsf/upload/commit', controller: 'dmsf_upload', action: 'commit_files'
     post '/projects/:id/dmsf/commit', controller: 'dmsf_upload', action: 'commit'

extra/api/api_client.sh

@@ -1,3 +1,4 @@
+#!/bin/bash
 # encoding: utf-8
 #
 # Redmine plugin for Document Management System "Features"
@@ -18,8 +19,6 @@
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 
-#!/bin/bash
-
 # Authentication as input parameters either as login + password or the API key
 # -u ${1}:${2}
 # -H "X-Redmine-API-Key: ${1}"

lib/redmine_dmsf/webdav/dmsf_resource.rb

@@ -578,7 +578,6 @@ module RedmineDmsf
 
         if new_revision.save
           new_revision.copy_file_content(request.body)
-          new_revision.create_digest
           new_revision.save
           # Notifications
           DmsfMailer.deliver_files_updated(project, [f])

lib/tasks/dmsf_create_digests.rake

@@ -35,25 +35,38 @@ END_DESC
 
 namespace :redmine do
   task :dmsf_create_digests => :environment do
-    m = DmsfDigest.new
+    m = DmsfCreateDigest.new
-    m.create_digests
+    m.dmsf_create_digests
   end
 end
 
-class DmsfDigest
+class DmsfCreateDigest
 
   def initialize
     @dry_run = ENV['dry_run']
     @force_sha256 = ENV['forceSHA256']
   end
 
-  def create_digests
+  def dmsf_create_digests
     revisions = DmsfFileRevision.where(['digest IS NULL OR length(digest) < ?', @force_sha256 ? 64 : 32])
     count = revisions.count
     n = 0
     revisions.each_with_index do |rev, i|
-      rev.create_digest
+      if File.exist?(rev.disk_file)
-      rev.save unless @dry_run
+        sha = Digest::SHA256.new
+        file = File.new rev.disk_file, 'r'
+        if file.respond_to?(:read)
+          while (buffer = file.read(8192))
+            sha.update buffer
+          end
+        else
+          sha.update @temp_file
+        end
+        rev.digest = sha.hexdigest
+        rev.save unless @dry_run
+      else
+        puts "#{rev.disk_file} not found"
+      end
       n += 1
       # Progress bar
       print "\r#{i * 100 / count}%"

test/unit/dmsf_file_revision_test.rb

@@ -77,18 +77,9 @@ class DmsfFileRevisionTest < RedmineDmsf::Test::UnitTest
     assert_nil DmsfFileRevision.find_by(id: @revision5.id)
   end
 
-  def test_create_digest
-    @revision1.create_digest
-    assert_equal 'SHA256', @revision1.digest_type
-    assert_equal @revision8.create_digest, 0, 'Digest should be 0, if the file is missing'
-  end
-
   def test_digest_type
     # Old type MD5
     assert_equal 'MD5', @revision1.digest_type
-    # New type SHA256
-    @revision1.create_digest
-    assert_equal 'SHA256', @revision1.digest_type
   end
 
   def test_new_storage_filename