#9 Active Storage - basic functionality
This commit is contained in:
Karel Pičman
parent
b69451807a
commit
da8456bc74
7
Gemfile
7
Gemfile
@ -18,14 +18,13 @@
|
||||
# <https://www.gnu.org/licenses/>.
|
||||
|
||||
source 'https://rubygems.org' do
|
||||
gem 'active_record_union'
|
||||
gem 'activestorage'
|
||||
gem 'ox' # Dav4Rack
|
||||
gem 'rake' unless Dir.exist?(File.expand_path('../../redmine_dashboard', __FILE__))
|
||||
gem 'simple_enum'
|
||||
gem 'uuidtools'
|
||||
gem 'zip-zip' unless Dir.exist?(File.expand_path('../../vault', __FILE__))
|
||||
|
||||
# Redmine extensions
|
||||
gem 'active_record_union'
|
||||
gem 'simple_enum'
|
||||
group :xapian do
|
||||
gem 'xapian-ruby'
|
||||
end
|
||||
|
||||
43
README.md
43
README.md
@ -1,4 +1,4 @@
|
||||
# Redmine DMSF Plugin 4.2.4 devel
|
||||
# Redmine DMSF Plugin 5.0.0 devel
|
||||
|
||||
[](https://github.com/picman/redmine_dmsf/actions/workflows/rubyonrails.yml)
|
||||
[](https://github.com/support-ukraine/support-ukraine)
|
||||
@ -13,7 +13,7 @@ The development has been supported by [Kontron](https://www.kontron.com) and has
|
||||
Project home: <https://github.com/picman/redmine_dmsf>
|
||||
|
||||
Redmine Document Management System "Features" plugin is distributed under GNU General Public License v3 (GPL).
|
||||
33Redmine is a flexible project management web application, released under the terms of the GNU General Public License v2 (GPL) at <https://www.redmine.org/>
|
||||
Redmine is a flexible project management web application, released under the terms of the GNU General Public License v2 (GPL) at <https://www.redmine.org/>
|
||||
|
||||
Further information about the GPL license can be found at
|
||||
<https://www.gnu.org/licenses/gpl-3.0.html>
|
||||
@ -296,7 +296,8 @@ instance is stopped.
|
||||
|
||||
### WebDAV
|
||||
|
||||
In order to enable WebDAV module, it is necessary to put the following code into yor `config/additional_environment.rb`
|
||||
In order to enable WebDAV module, it is necessary to put the following code into your
|
||||
`config/additional_environment.rb`:
|
||||
|
||||
```ruby
|
||||
# Redmine DMSF's WebDAV
|
||||
@ -304,6 +305,42 @@ require Rails.root.join('plugins', 'redmine_dmsf', 'lib', 'redmine_dmsf', 'webda
|
||||
config.middleware.insert_before ActionDispatch::Cookies, RedmineDmsf::Webdav::CustomMiddleware
|
||||
```
|
||||
|
||||
### Active Storage
|
||||
|
||||
Documents are stored using Active Storage. It requires the following lines to be added into
|
||||
`config/additional_environment.rb`:
|
||||
|
||||
```ruby
|
||||
# Active storage
|
||||
require 'active_storage/engine'
|
||||
require Rails.root.join('plugins', 'redmine_dmsf', 'lib', 'redmine_dmsf', 'xapian_analyzer').to_s
|
||||
config.active_storage.service = :local # Store files locally
|
||||
#config.active_storage.service = :amazon # Store files on Amazon S3
|
||||
config.active_storage.analyzers.append RedmineDmsf::XapianAnalyzer # Index uploaded files for Xapian full-text search
|
||||
```
|
||||
|
||||
Then install Active Storage with the following commands:
|
||||
|
||||
```shell
|
||||
bin/rails active_storage:install RAILS_ENV=production
|
||||
bin/rails db:migrate RAILS_ENV=production
|
||||
```
|
||||
|
||||
Configure your DMS files storage in config/storage.yml:
|
||||
|
||||
```yml
|
||||
local:
|
||||
service: Disk
|
||||
root: <%= Rails.root.join('dmsf_as') %>
|
||||
## Use bin/rails credentials:edit to set the AWS secrets (as aws:access_key_id|secret_access_key)
|
||||
#amazon:
|
||||
# service: S3
|
||||
# access_key_id: <%= Rails.application.credentials.dig(:aws, :access_key_id) %>
|
||||
# secret_access_key: <%= Rails.application.credentials.dig(:aws, :secret_access_key) %>
|
||||
# bucket: your_own_bucket-<%= Rails.env %>
|
||||
# region: "" # e.g. 'us-east-1'
|
||||
```
|
||||
|
||||
### Installation in a sub-uri
|
||||
|
||||
In order to documents and folders are available via WebDAV in case that the Redmine is configured to be run in a sub-uri
|
||||
|
||||
@ -510,7 +510,7 @@ class DmsfController < ApplicationController
|
||||
raise DmsfAccessError unless User.current.allowed_to?(:email_documents, @project)
|
||||
|
||||
zip = Zip.new
|
||||
zip_entries(zip, selected_folders, selected_files)
|
||||
zip_entries zip, selected_folders, selected_files
|
||||
zipped_content = zip.finish
|
||||
|
||||
max_filesize = RedmineDmsf.dmsf_max_email_filesize
|
||||
@ -548,7 +548,7 @@ class DmsfController < ApplicationController
|
||||
|
||||
def download_entries(selected_folders, selected_files)
|
||||
zip = Zip.new
|
||||
zip_entries(zip, selected_folders, selected_files)
|
||||
zip_entries zip, selected_folders, selected_files
|
||||
zip.dmsf_files.each do |f|
|
||||
# Action
|
||||
audit = DmsfFileRevisionAccess.new
|
||||
|
||||
@ -52,7 +52,7 @@ class DmsfFilesController < ApplicationController
|
||||
end
|
||||
|
||||
check_project @revision.dmsf_file
|
||||
raise ActionController::MissingFile if @file.deleted?
|
||||
raise ActionController::MissingFile if @file.deleted? || !@revision.file.attached?
|
||||
|
||||
# Action
|
||||
access = DmsfFileRevisionAccess.new
|
||||
@ -82,12 +82,12 @@ class DmsfFilesController < ApplicationController
|
||||
# Text preview
|
||||
elsif !api_request? && params[:download].blank? && (@file.size <= Setting.file_max_size_displayed.to_i.kilobyte) &&
|
||||
(@file.text? || @file.markdown? || @file.textile?) && !@file.html? && formats.include?(:html)
|
||||
@content = File.read(@revision.disk_file, mode: 'rb')
|
||||
@content = @revision.file.download
|
||||
render action: 'document'
|
||||
# Offer the file for download
|
||||
else
|
||||
params[:disposition] = 'attachment' if params[:filename].present?
|
||||
send_file @revision.disk_file,
|
||||
send_data @revision.file.download,
|
||||
filename: filename,
|
||||
type: @revision.detect_content_type,
|
||||
disposition: params[:disposition].presence || @revision.dmsf_file.disposition
|
||||
@ -142,6 +142,11 @@ class DmsfFilesController < ApplicationController
|
||||
revision.disk_filename = revision.new_storage_filename
|
||||
revision.mime_type = upload.mime_type
|
||||
revision.digest = upload.digest
|
||||
revision.file.attach(
|
||||
io: File.open(upload.tempfile_path),
|
||||
filename: revision.disk_filename,
|
||||
content_type: revision.mime_type
|
||||
)
|
||||
end
|
||||
else
|
||||
revision.size = last_revision.size
|
||||
@ -155,17 +160,7 @@ class DmsfFilesController < ApplicationController
|
||||
ok = true
|
||||
if revision.save
|
||||
revision.assign_workflow params[:dmsf_workflow_id]
|
||||
if upload
|
||||
begin
|
||||
FileUtils.mv upload.tempfile_path, revision.disk_file(search_if_not_exists: false)
|
||||
rescue StandardError => e
|
||||
Rails.logger.error e.message
|
||||
flash[:error] = e.message
|
||||
revision.destroy
|
||||
ok = false
|
||||
end
|
||||
end
|
||||
if ok && @file.locked? && !@file.locks.empty?
|
||||
if @file.locked? && !@file.locks.empty?
|
||||
begin
|
||||
@file.unlock!
|
||||
flash[:notice] = "#{l(:notice_file_unlocked)}, "
|
||||
|
||||
@ -96,8 +96,11 @@ module DmsfUploadHelper
|
||||
a = Attachment.find_by_token(committed_file[:token])
|
||||
committed_file[:tempfile_path] = a.diskfile if a
|
||||
end
|
||||
FileUtils.mv committed_file[:tempfile_path], new_revision.disk_file(search_if_not_exists: false)
|
||||
FileUtils.chmod 'u=wr,g=r', new_revision.disk_file(search_if_not_exists: false)
|
||||
new_revision.file.attach(
|
||||
io: File.open(committed_file[:tempfile_path]),
|
||||
filename: new_revision.name,
|
||||
content_type: new_revision.mime_type
|
||||
)
|
||||
file.last_revision = new_revision
|
||||
files.push file
|
||||
container.dmsf_file_added file if container && !new_object
|
||||
|
||||
47
app/jobs/remove_from_index_job.rb
Normal file
47
app/jobs/remove_from_index_job.rb
Normal file
@ -0,0 +1,47 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
# Redmine plugin for Document Management System "Features"
|
||||
#
|
||||
# Vít Jonáš <vit.jonas@gmail.com>, Karel Pičman <karel.picman@kontron.com>
|
||||
#
|
||||
# This file is part of Redmine DMSF plugin.
|
||||
#
|
||||
# Redmine DMSF plugin is free software: you can redistribute it and/or modify it under the terms of the GNU General
|
||||
# Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any
|
||||
# later version.
|
||||
#
|
||||
# Redmine DMSF plugin is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even
|
||||
# the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
||||
# more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License along with Redmine DMSF plugin. If not, see
|
||||
# <https://www.gnu.org/licenses/>.
|
||||
|
||||
# An asynchronous job to remove file from Xapian full-text search index
|
||||
class RemoveFromIndexJob < ApplicationJob
|
||||
def self.schedule(key)
|
||||
perform_later key
|
||||
end
|
||||
|
||||
def perform(key)
|
||||
url = File.join(key[0..1], key[2..3], key)
|
||||
stem_lang = RedmineDmsf.dmsf_stemming_lang
|
||||
db_path = File.join RedmineDmsf.dmsf_index_database, stem_lang
|
||||
db = Xapian::WritableDatabase.new(db_path, Xapian::DB_OPEN)
|
||||
found = false
|
||||
db.postlist('').each do |it|
|
||||
doc = db.document(it.docid)
|
||||
dochash = Hash[*doc.data.scan(%r{(url|sample|modtime|author|type|size)=/?([^\n\]]+)}).flatten]
|
||||
next unless url == dochash['url']
|
||||
|
||||
db.delete_document it.docid
|
||||
found = true
|
||||
break
|
||||
end
|
||||
Rails.logger.warn "Document's URL '#{url}' not found in the index" unless found
|
||||
rescue StandardError => e
|
||||
Rails.logger.error e.message
|
||||
ensure
|
||||
db&.close
|
||||
end
|
||||
end
|
||||
@ -177,21 +177,21 @@ class DmsfFile < ApplicationRecord
|
||||
if locked_for_user? && !User.current.allowed_to?(:force_file_unlock, project)
|
||||
Rails.logger.info l(:error_file_is_locked)
|
||||
if lock.reverse[0].user
|
||||
errors.add(:base, l(:title_locked_by_user, user: lock.reverse[0].user))
|
||||
errors.add :base, l(:title_locked_by_user, user: lock.reverse[0].user)
|
||||
else
|
||||
errors.add(:base, l(:error_file_is_locked))
|
||||
errors.add :base, l(:error_file_is_locked)
|
||||
end
|
||||
return false
|
||||
end
|
||||
begin
|
||||
# Revisions and links of a deleted file SHOULD be deleted too
|
||||
dmsf_file_revisions.each { |r| r.delete(commit: commit, force: true) }
|
||||
if commit
|
||||
destroy
|
||||
else
|
||||
self.deleted = STATUS_DELETED
|
||||
self.deleted_by_user = User.current
|
||||
save
|
||||
# Associated revisions should be marked as deleted too
|
||||
dmsf_file_revisions.each { |r| r.delete(commit: commit, force: true) }
|
||||
end
|
||||
rescue StandardError => e
|
||||
Rails.logger.error e.message
|
||||
@ -444,25 +444,24 @@ class DmsfFile < ApplicationRecord
|
||||
matchset = enquire.mset(0, 1000)
|
||||
|
||||
matchset&.matches&.each do |m|
|
||||
docdata = m.document.data { url }
|
||||
docdata = m.document.data
|
||||
dochash = Hash[*docdata.scan(%r{(url|sample|modtime|author|type|size)=/?([^\n\]]+)}).flatten]
|
||||
filename = dochash['url']
|
||||
next unless filename
|
||||
next unless dochash['url'] =~ %r{^\w{2}/\w{2}/(\w+)$} # /76/df/76dfsp2ubbgq4yvq90zrfoyxt012
|
||||
|
||||
dmsf_attrs = filename.scan(%r{^\d{4}/\d{2}/(\d{12}_(\d+)_.*)$})
|
||||
id_attribute = 0
|
||||
id_attribute = dmsf_attrs[0][1] if dmsf_attrs.length.positive?
|
||||
next if dmsf_attrs.empty? || id_attribute.to_i.zero?
|
||||
key = Regexp.last_match(1)
|
||||
blob = ActiveStorage::Blob.find_by(key: key)
|
||||
attachment = blob&.attachments&.first
|
||||
dmsf_file_revision = attachment&.record
|
||||
|
||||
dmsf_file = DmsfFile.visible.where(limit_options).find_by(id: id_attribute)
|
||||
next unless dmsf_file_revision
|
||||
|
||||
dmsf_file = dmsf_file_revision.dmsf_file
|
||||
next unless dmsf_file && DmsfFolder.permissions?(dmsf_file.dmsf_folder) &&
|
||||
user.allowed_to?(:view_dmsf_files, dmsf_file.project) &&
|
||||
(project_ids.blank? || project_ids.include?(dmsf_file.project_id))
|
||||
|
||||
rev_id = DmsfFileRevision.where(dmsf_file_id: dmsf_file.id, disk_filename: dmsf_attrs[0][0])
|
||||
.pick(:id)
|
||||
if dochash['sample']
|
||||
Redmine::Search.cache_store.write("DmsfFile-#{dmsf_file.id}-#{rev_id}",
|
||||
Redmine::Search.cache_store.write("DmsfFile-#{dmsf_file.id}-#{dmsf_file_revision.id}",
|
||||
dochash['sample'].force_encoding('UTF-8'))
|
||||
end
|
||||
break if options[:limit].present? && results.count >= options[:limit]
|
||||
@ -551,12 +550,14 @@ class DmsfFile < ApplicationRecord
|
||||
def pdf_preview
|
||||
return '' unless previewable?
|
||||
|
||||
target = File.join(DmsfFile.previews_storage_path, "#{File.basename(last_revision&.disk_file.to_s, '.*')}.pdf")
|
||||
target = File.join(DmsfFile.previews_storage_path, "#{last_revision.file.blob.key}.pdf")
|
||||
begin
|
||||
RedmineDmsf::Preview.generate last_revision&.disk_file.to_s, target
|
||||
last_revision.file.open do |f|
|
||||
RedmineDmsf::Preview.generate f.path, target
|
||||
end
|
||||
rescue StandardError => e
|
||||
Rails.logger.error do
|
||||
%(An error occurred while generating preview for #{last_revision&.disk_file} to #{target}\n
|
||||
%(An error occurred while generating preview for #{last_revision.file.name} to #{target}\n
|
||||
Exception was: #{e.message})
|
||||
end
|
||||
''
|
||||
@ -567,15 +568,16 @@ class DmsfFile < ApplicationRecord
|
||||
result = +'No preview available'
|
||||
if text?
|
||||
begin
|
||||
f = File.new(last_revision.disk_file)
|
||||
f.each_line do |line|
|
||||
case f.lineno
|
||||
when 1
|
||||
result = line
|
||||
when limit.to_i + 1
|
||||
break
|
||||
else
|
||||
result << line
|
||||
last_revision.file.open do |f|
|
||||
f.each_line do |line|
|
||||
case f.lineno
|
||||
when 1
|
||||
result = line
|
||||
when limit.to_i + 1
|
||||
break
|
||||
else
|
||||
result << line
|
||||
end
|
||||
end
|
||||
end
|
||||
rescue StandardError => e
|
||||
|
||||
@ -30,6 +30,8 @@ class DmsfFileRevision < ApplicationRecord
|
||||
belongs_to :dmsf_workflow_assigned_by_user, class_name: 'User'
|
||||
belongs_to :dmsf_workflow
|
||||
|
||||
has_one_attached :shared_file
|
||||
|
||||
has_many :dmsf_file_revision_access, dependent: :destroy
|
||||
has_many :dmsf_workflow_step_assignment, dependent: :destroy
|
||||
|
||||
@ -98,6 +100,20 @@ class DmsfFileRevision < ApplicationRecord
|
||||
validates :description, length: { maximum: 1.kilobyte }
|
||||
validates :size, dmsf_max_file_size: true
|
||||
|
||||
def file
|
||||
unless shared_file.attached?
|
||||
# If no file is attached, look at the source revision
|
||||
# This way we prevent the same file from being attached to multiple revisions
|
||||
sr = source_revision
|
||||
while sr
|
||||
return sr.shared_file if sr.shared_file.attached?
|
||||
|
||||
sr = sr.source_revision
|
||||
end
|
||||
end
|
||||
shared_file
|
||||
end
|
||||
|
||||
def visible?(_user = nil)
|
||||
deleted == STATUS_ACTIVE
|
||||
end
|
||||
@ -300,19 +316,8 @@ class DmsfFileRevision < ApplicationRecord
|
||||
end
|
||||
|
||||
def copy_file_content(open_file)
|
||||
sha = Digest::SHA256.new
|
||||
File.open(disk_file(search_if_not_exists: false), 'wb') do |f|
|
||||
if open_file.respond_to?(:read)
|
||||
while (buffer = open_file.read(8192))
|
||||
f.write buffer
|
||||
sha.update buffer
|
||||
end
|
||||
else
|
||||
f.write open_file
|
||||
sha.update open_file
|
||||
end
|
||||
end
|
||||
self.digest = sha.hexdigest
|
||||
file.attach io: open_file, filename: dmsf_file.name
|
||||
self.digest = file.blob.checksum
|
||||
end
|
||||
|
||||
# Overrides Redmine::Acts::Customizable::InstanceMethods#available_custom_fields
|
||||
@ -400,14 +405,27 @@ class DmsfFileRevision < ApplicationRecord
|
||||
end
|
||||
|
||||
def delete_source_revision
|
||||
derived_revisions = []
|
||||
DmsfFileRevision.where(source_dmsf_file_revision_id: id).find_each do |d|
|
||||
# Derived revision without its own file
|
||||
derived_revisions << d unless d.shared_file.attached?
|
||||
# Replace the source revision
|
||||
d.source_revision = source_revision
|
||||
d.save!
|
||||
end
|
||||
return unless RedmineDmsf.physical_file_delete?
|
||||
return unless shared_file.attached?
|
||||
|
||||
dependencies = DmsfFileRevision.where(disk_filename: disk_filename).all.size
|
||||
FileUtils.rm_f(disk_file) if dependencies <= 1
|
||||
if derived_revisions.empty?
|
||||
# Remove the file from Xapian index
|
||||
RemoveFromIndexJob.schedule file.blob.key if RedmineDmsf::Plugin.lib_available?('xapian')
|
||||
# Remove the file
|
||||
shared_file.purge_later if RedmineDmsf.physical_file_delete?
|
||||
else
|
||||
# Move the shared file to an unattached derived revision
|
||||
d = derived_revisions.first
|
||||
d.shared_file.attach shared_file.blob
|
||||
shared_file.detach
|
||||
end
|
||||
end
|
||||
|
||||
def copy_custom_field_values(values, source_revision = nil)
|
||||
|
||||
@ -0,0 +1,25 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
# Redmine plugin for Document Management System "Features"
|
||||
#
|
||||
# Karel Pičman <karel.picman@kontron.com>
|
||||
#
|
||||
# This file is part of Redmine DMSF plugin.
|
||||
#
|
||||
# Redmine DMSF plugin is free software: you can redistribute it and/or modify it under the terms of the GNU General
|
||||
# Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any
|
||||
# later version.
|
||||
#
|
||||
# Redmine DMSF plugin is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even
|
||||
# the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
||||
# more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License along with Redmine DMSF plugin. If not, see
|
||||
# <https://www.gnu.org/licenses/>.
|
||||
|
||||
# Add index
|
||||
class AddIndexOnSourceDmsfFileRevisionId < ActiveRecord::Migration[7.0]
|
||||
def change
|
||||
add_index :dmsf_file_revisions, :source_dmsf_file_revision_id
|
||||
end
|
||||
end
|
||||
@ -1,168 +0,0 @@
|
||||
#!/usr/bin/ruby -W0
|
||||
|
||||
# frozen_string_literal: true
|
||||
|
||||
# Redmine plugin for Document Management System "Features"
|
||||
#
|
||||
# Xabier Elkano, Karel Pičman <karel.picman@kontron.com>
|
||||
#
|
||||
# This file is part of Redmine DMSF plugin.
|
||||
#
|
||||
# Redmine DMSF plugin is free software: you can redistribute it and/or modify it under the terms of the GNU General
|
||||
# Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any
|
||||
# later version.
|
||||
#
|
||||
# Redmine DMSF plugin is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even
|
||||
# the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
||||
# more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License along with Redmine DMSF plugin. If not, see
|
||||
# <https://www.gnu.org/licenses/>.
|
||||
|
||||
require 'optparse'
|
||||
|
||||
########################################################################################################################
|
||||
# BEGIN Configuration parameters
|
||||
# Configure the following parameters (most of them can be configured through the command line):
|
||||
########################################################################################################################
|
||||
|
||||
# Redmine installation directory
|
||||
REDMINE_ROOT = File.expand_path('../../../', __dir__)
|
||||
|
||||
# DMSF document location REDMINE_ROOT/FILES
|
||||
FILES = 'dmsf'
|
||||
|
||||
# omindex binary path
|
||||
# To index "non-text" files, use omindex filters
|
||||
# e.g.: tesseract OCR engine as a filter for PNG files
|
||||
OMINDEX = '/usr/bin/omindex'
|
||||
# OMINDEX += " --filter=image/png:'tesseract -l chi_sim+chi_tra %f -'"
|
||||
# OMINDEX += " --filter=image/jpeg:'tesseract -l chi_sim+chi_tra %f -'"
|
||||
|
||||
# Directory containing Xapian databases for omindex (Attachments indexing)
|
||||
db_root_path = File.expand_path('dmsf_index', REDMINE_ROOT)
|
||||
|
||||
# Verbose output, false/true
|
||||
verbose = false
|
||||
|
||||
# Define stemmed languages to index attachments Eg. [ 'english', 'italian', 'spanish' ]
|
||||
# Available languages are danish, dutch, english, finnish, french, german, german2, hungarian, italian, kraaij_pohlmann,
|
||||
# lovins, norwegian, porter, portuguese, romanian, russian, spanish, swedish and turkish.
|
||||
stem_langs = ['english']
|
||||
|
||||
ENVIRONMENT = File.join(REDMINE_ROOT, 'config/environment.rb')
|
||||
env = 'production'
|
||||
|
||||
########################################################################################################################
|
||||
# END Configuration parameters
|
||||
########################################################################################################################
|
||||
|
||||
retry_failed = false
|
||||
no_delete = false
|
||||
max_size = ''
|
||||
overwrite = false
|
||||
|
||||
VERSION = '0.3'
|
||||
|
||||
optparse = OptionParser.new do |opts|
|
||||
opts.banner = 'Usage: xapian_indexer.rb [OPTIONS...]'
|
||||
opts.separator('')
|
||||
opts.separator("Index Redmine's DMS documents")
|
||||
opts.separator('')
|
||||
opts.separator('')
|
||||
opts.separator('Options:')
|
||||
opts.on('-d', '--index_db DB_PATH', 'Absolute path to index database according plugin settings in UI') do |db|
|
||||
db_root_path = db
|
||||
end
|
||||
opts.on('-s', '--stemming_lang a,b,c', Array, 'Comma separated list of stemming languages for indexing') do |s|
|
||||
stem_langs = s
|
||||
end
|
||||
opts.on('-v', '--verbose', 'verbose') do
|
||||
verbose = true
|
||||
end
|
||||
opts.on('-e', '--environment ENV', 'Rails ENVIRONMENT(development, testing or production), default production') do |e|
|
||||
env = e
|
||||
end
|
||||
opts.on('-V', '--version', 'show version and exit') do
|
||||
$stdout.puts VERSION
|
||||
exit
|
||||
end
|
||||
opts.on('-h', '--help', 'show help and exit') do
|
||||
$stdout.puts opts
|
||||
exit
|
||||
end
|
||||
opts.on('-R', '--retry-failed', 'retry files which omindex failed to extract text') do
|
||||
retry_failed = true
|
||||
end
|
||||
opts.on('-p', '--no-delete', 'skip the deletion of records corresponding to deleted files') do
|
||||
no_delete = true
|
||||
end
|
||||
opts.on('-m', '--max-size SIZE', "maximum size of file to index(e.g.: '5M', '1G',...)") do |m|
|
||||
max_size = m
|
||||
end
|
||||
opts.on('', '--overwrite', 'create the database anew instead of updating') do
|
||||
overwrite = true
|
||||
end
|
||||
opts.separator('')
|
||||
opts.separator('Examples:')
|
||||
opts.separator(' xapian_indexer.rb -s english,italian -v')
|
||||
opts.separator(' xapian_indexer.rb -d $HOME/index_db -s english,italian -v')
|
||||
opts.separator('')
|
||||
opts.summary_width = 25
|
||||
end
|
||||
|
||||
optparse.parse!
|
||||
|
||||
ENV['RAILS_ENV'] = env
|
||||
|
||||
def log(text, verbose, error: false)
|
||||
if error
|
||||
warn text
|
||||
elsif verbose
|
||||
$stdout.puts text
|
||||
end
|
||||
end
|
||||
|
||||
def system_or_raise(command, verbose)
|
||||
if verbose
|
||||
system command, exception: true
|
||||
else
|
||||
system command, out: '/dev/null', exception: true
|
||||
end
|
||||
end
|
||||
|
||||
log "Trying to load Redmine environment <<#{ENVIRONMENT}>>...", verbose
|
||||
|
||||
begin
|
||||
require ENVIRONMENT
|
||||
|
||||
log "Redmine environment [RAILS_ENV=#{env}] correctly loaded ...", verbose
|
||||
|
||||
# Indexing documents
|
||||
stem_langs.each do |lang|
|
||||
filespath = RedmineDmsf.dmsf_storage_directory
|
||||
unless File.directory?(filespath)
|
||||
warn "'#{filespath}' doesn't exist."
|
||||
exit 1
|
||||
end
|
||||
databasepath = File.join(db_root_path, lang)
|
||||
unless File.directory?(databasepath)
|
||||
log "#{databasepath} does not exist, creating ...", verbose
|
||||
FileUtils.mkdir_p databasepath
|
||||
end
|
||||
cmd = "#{OMINDEX} -s #{lang} --db #{databasepath} #{filespath} --url / --depth-limit=0"
|
||||
cmd << ' -v' if verbose
|
||||
cmd << ' --retry-failed' if retry_failed
|
||||
cmd << ' -p' if no_delete
|
||||
cmd << " -m #{max_size}" if max_size.present?
|
||||
cmd << ' --overwrite' if overwrite
|
||||
log cmd, verbose
|
||||
system_or_raise cmd, verbose
|
||||
end
|
||||
log 'Redmine DMS documents indexed', verbose
|
||||
rescue LoadError => e
|
||||
warn e.message
|
||||
exit 1
|
||||
end
|
||||
|
||||
exit 0
|
||||
2
init.rb
2
init.rb
@ -27,7 +27,7 @@ Redmine::Plugin.register :redmine_dmsf do
|
||||
author_url 'https://github.com/picman/redmine_dmsf/graphs/contributors'
|
||||
author 'Vít Jonáš / Daniel Munn / Karel Pičman'
|
||||
description 'Document Management System Features'
|
||||
version '4.2.4 devel'
|
||||
version '5.0.0 devel'
|
||||
|
||||
requires_redmine version_or_higher: '6.1.0'
|
||||
|
||||
|
||||
@ -47,7 +47,9 @@ module RedmineDmsf
|
||||
end
|
||||
|
||||
def add_dmsf_file(dmsf_file, member = nil, root_path = nil, path = nil)
|
||||
raise DmsfFileNotFoundError unless dmsf_file&.last_revision && File.exist?(dmsf_file.last_revision.disk_file)
|
||||
raise DmsfFileNotFoundError unless dmsf_file&.last_revision
|
||||
|
||||
raise DmsfFileNotFoundError unless dmsf_file.last_revision.file.attached?
|
||||
|
||||
if path
|
||||
string_path = path
|
||||
@ -62,7 +64,7 @@ module RedmineDmsf
|
||||
zip_entry = ::Zip::Entry.new(@zip_file, string_path, nil, nil, nil, nil, nil, nil,
|
||||
::Zip::DOSTime.at(dmsf_file.last_revision.updated_at))
|
||||
@zip_file.put_next_entry zip_entry
|
||||
File.open(dmsf_file.last_revision.disk_file, 'rb') do |f|
|
||||
dmsf_file.last_revision.file.open do |f|
|
||||
while (buffer = f.read(8192))
|
||||
@zip_file.write buffer
|
||||
end
|
||||
@ -71,31 +73,6 @@ module RedmineDmsf
|
||||
@dmsf_files << dmsf_file
|
||||
end
|
||||
|
||||
def add_attachment(attachment, path)
|
||||
return if @files.include?(path)
|
||||
|
||||
raise DmsfFileNotFoundError unless File.exist?(attachment.diskfile)
|
||||
|
||||
zip_entry = ::Zip::Entry.new(@zip_file, path, nil, nil, nil, nil, nil, nil,
|
||||
::Zip::DOSTime.at(attachment.created_on))
|
||||
@zip_file.put_next_entry zip_entry
|
||||
File.open(attachment.diskfile, 'rb') do |f|
|
||||
while (buffer = f.read(8192))
|
||||
@zip_file.write buffer
|
||||
end
|
||||
end
|
||||
@files << path
|
||||
end
|
||||
|
||||
def add_raw_file(filename, data)
|
||||
return if @files.include?(filename)
|
||||
|
||||
zip_entry = ::Zip::Entry.new(@zip_file, filename, nil, nil, nil, nil, nil, nil, ::Zip::DOSTime.now)
|
||||
@zip_file.put_next_entry zip_entry
|
||||
@zip_file.write data
|
||||
@files << filename
|
||||
end
|
||||
|
||||
def add_dmsf_folder(dmsf_folder, member, root_path = nil)
|
||||
string_path = dmsf_folder.dmsf_path_str + File::SEPARATOR
|
||||
string_path = string_path[(root_path.length + 1)..string_path.length] if root_path
|
||||
|
||||
@ -46,6 +46,8 @@ module RedmineDmsf
|
||||
office_bin = RedmineDmsf.office_bin.presence || 'libreoffice'
|
||||
cmd = "#{shell_quote(office_bin)} --convert-to pdf --headless --outdir #{shell_quote(dir)} #{shell_quote(source)}"
|
||||
if system(cmd)
|
||||
filename = "#{File.basename(source, '.*')}.pdf"
|
||||
FileUtils.mv File.join(dir, filename), target
|
||||
target
|
||||
else
|
||||
Rails.logger.error "Creating preview failed (#{$CHILD_STATUS}):\nCommand: #{cmd}"
|
||||
|
||||
@ -697,10 +697,7 @@ module RedmineDmsf
|
||||
# implementation of service for request, which allows for us to pipe a single file through
|
||||
# also best-utilising Dav4rack's implementation.
|
||||
def download
|
||||
raise NotFound unless file&.last_revision
|
||||
|
||||
disk_file = file.last_revision.disk_file
|
||||
raise NotFound unless disk_file && File.exist?(disk_file)
|
||||
raise NotFound unless file.last_revision&.file&.attached?
|
||||
raise Forbidden unless !parent.exist? || !parent.folder || DmsfFolder.permissions?(parent.folder)
|
||||
|
||||
# If there is no range (start of ranged download, or direct download) then we log the
|
||||
@ -719,7 +716,9 @@ module RedmineDmsf
|
||||
Rails.logger.error "Could not send email notifications: #{e.message}"
|
||||
end
|
||||
end
|
||||
File.new disk_file
|
||||
file.last_revision.file.open do |f|
|
||||
File.new f.path
|
||||
end
|
||||
end
|
||||
|
||||
def reuse_version_for_locked_file?(file)
|
||||
|
||||
53
lib/redmine_dmsf/xapian_analyzer.rb
Normal file
53
lib/redmine_dmsf/xapian_analyzer.rb
Normal file
@ -0,0 +1,53 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
# Redmine plugin for Document Management System "Features"
|
||||
#
|
||||
# Vít Jonáš <vit.jonas@gmail.com>, Karel Pičman <karel.picman@kontron.com>
|
||||
#
|
||||
# This file is part of Redmine DMSF plugin.
|
||||
#
|
||||
# Redmine DMSF plugin is free software: you can redistribute it and/or modify it under the terms of the GNU General
|
||||
# Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any
|
||||
# later version.
|
||||
#
|
||||
# Redmine DMSF plugin is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even
|
||||
# the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
||||
# more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License along with Redmine DMSF plugin. If not, see
|
||||
# <https://www.gnu.org/licenses/>.
|
||||
|
||||
module RedmineDmsf
|
||||
# ActiveRecord Analyzer for Xapian
|
||||
class XapianAnalyzer < ActiveStorage::Analyzer
|
||||
def self.accept?(blob)
|
||||
return false unless RedmineDmsf::Plugin.lib_available?('xapian') && blob.byte_size < 1_024 * 1_024 * 3 # 3MB
|
||||
|
||||
@blob = blob
|
||||
true
|
||||
end
|
||||
|
||||
def metadata
|
||||
index
|
||||
{}
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def index
|
||||
stem_lang = RedmineDmsf.dmsf_stemming_lang
|
||||
db_path = File.join RedmineDmsf.dmsf_index_database, stem_lang
|
||||
url = File.join(@blob.key[0..1], @blob.key[2..3])
|
||||
dir = File.join(Dir.tmpdir, @blob.key)
|
||||
FileUtils.mkdir dir
|
||||
@blob.open do |file|
|
||||
FileUtils.mv file.path, File.join(dir, @blob.key)
|
||||
system "omindex -s \"#{stem_lang}\" -D \"#{db_path}\" --url=/#{url} \"#{dir}\" -p", exception: true
|
||||
end
|
||||
rescue StandardError => e
|
||||
Rails.logger.error e.message
|
||||
ensure
|
||||
FileUtils.rm_f dir
|
||||
end
|
||||
end
|
||||
end
|
||||
@ -51,32 +51,6 @@ class DmsfZipTest < RedmineDmsf::Test::HelperTest
|
||||
end
|
||||
end
|
||||
|
||||
def test_add_attachment
|
||||
assert File.exist?(@attachment6.diskfile), @attachment6.diskfile
|
||||
@zip.add_attachment @attachment6, @attachment6.filename
|
||||
assert_equal 0, @zip.dmsf_files.size
|
||||
zip_file = @zip.finish
|
||||
Zip::File.open(zip_file) do |file|
|
||||
file.each do |entry|
|
||||
assert_equal @attachment6.filename, entry.name
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def test_add_raw_file
|
||||
filename = 'data.txt'
|
||||
content = '1,2,3'
|
||||
@zip.add_raw_file filename, content
|
||||
assert_equal 0, @zip.dmsf_files.size
|
||||
zip_file = @zip.finish
|
||||
Zip::File.open(zip_file) do |file|
|
||||
file.each do |entry|
|
||||
assert_equal filename, entry.name
|
||||
assert_equal content, entry.get_input_stream.read
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def test_read
|
||||
@zip.add_dmsf_file @dmsf_file1
|
||||
assert_not_empty @zip.read
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user