redmine_plugins/redmine_dmsf
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Xapian Easy search

Browse Source
This commit is contained in:
Karel Pičman 2023-07-12 12:07:49 +02:00
parent c1476dcf2d
commit f31534ce58
27 changed files with 380 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.rubocop.yml
View File

@ -1,6 +1,5 @@
# Redmine plugin for Custom Workflows
# Redmine plugin for Document Management System "Features"
#
# Copyright © 2015-19 Anton Argirov
# Copyright © 2019-23 Karel Pičman <karel.picman@kontron.com>
#
# This program is free software; you can redistribute it and/or
@ -21,6 +20,8 @@ AllCops:
TargetRubyVersion: 2.7
TargetRailsVersion: 6.1
SuggestExtensions: false
NewCops: enable
Exclude:
@ -78,6 +79,7 @@ Rails/DynamicFindBy:
Rails/SkipsModelValidations:
Exclude:
- app/helpers/dmsf_upload_helper.rb # touch is Okay
- app/models/dmsf_workflow.rb # update doesn't work here
- lib/redmine_dmsf/patches/user_patch.rb
- lib/redmine_dmsf/patches/role_patch.rb

4
app/helpers/dmsf_upload_helper.rb
View File

@ -101,6 +101,10 @@ module DmsfUploadHelper
begin
FileUtils.mv commited_file[:tempfile_path], new_revision.disk_file(search_if_not_exists: false)
FileUtils.chmod 'u=wr,g=r', new_revision.disk_file(search_if_not_exists: false)
if defined?(EasyExtensions)
# We need to trigger Xapian indexing after the file is moved to its target destination
file.touch
end
file.last_revision = new_revision
files.push file
container.dmsf_file_added file if container && !new_object

58
app/models/dmsf_file.rb
View File

@ -21,6 +21,7 @@
require 'xapian'
require "#{File.dirname(__FILE__)}/../../lib/redmine_dmsf/lockable"
require 'English'
# File
class DmsfFile < ApplicationRecord
@ -624,4 +625,61 @@ class DmsfFile < ApplicationRecord
parent = dmsf_folder.dmsf_folder
Regexp.last_match(1).constantize.visible.find_by(id: issue_id) if parent&.title&.match(/^\.(.+)s/)
end
if defined?(EasyExtensions)
include Redmine::Utils::Shell
def sheet?
case File.extname(last_revision&.disk_filename)
when '.ods', # LibreOffice
'.xls', '.xlsx', '.xlsm' # MS Office
true
else
false
end
end
def content
if File.exist?(last_revision.disk_file)
if File.size?(last_revision.disk_file) < 5.megabytes
tmp = Rails.root.join('tmp')
if sheet?
cmd = "#{shell_quote(RedmineDmsf::Preview::OFFICE_BIN)} --convert-to 'csv' \
--outdir #{shell_quote(tmp.to_s)} #{shell_quote(last_revision.disk_file)}"
text_file = tmp.join(last_revision.disk_filename).sub_ext('.csv')
elsif office_doc?
cmd = "#{shell_quote(RedmineDmsf::Preview::OFFICE_BIN)} --convert-to 'txt:Text (encoded):UTF8' \
--outdir #{shell_quote(tmp.to_s)} #{shell_quote(last_revision.disk_file)}"
text_file = tmp.join(last_revision.disk_filename).sub_ext('.txt')
elsif pdf?
text_file = tmp.join(last_revision.disk_filename).sub_ext('.txt')
cmd = "pdftotext -q #{shell_quote(last_revision.disk_file)} #{shell_quote(text_file.to_s)}"
elsif text?
return File.read(last_revision.disk_file)
end
if cmd
if system(cmd) && File.exist?(text_file)
text = File.read(text_file)
FileUtils.rm_f text_file
return text
else
Rails.logger.error "Conversion to text failed (#{$CHILD_STATUS}):\nCommand: #{cmd}"
end
end
else
Rails.logger.warn "File #{last_revision.disk_file} is to big to be indexed (>5MB)"
end
end
description
rescue StandardError => e
Rails.logger.warn e.message
''
ensure
FileUtils.rm_f(text_file) if text_file.present?
end
end
def to_s
name
end
end

4
app/models/dmsf_folder.rb
View File

@ -597,6 +597,10 @@ class DmsfFolder < ApplicationRecord
!(dmsf_folders.visible.exists? || dmsf_files.visible.exists? || dmsf_links.visible.exists?)
end
def to_s
title
end
class << self
def directory_subtree(tree, folder, level, current_folder)
folders = folder.dmsf_folders.visible.to_a

84
app/views/settings/_dmsf_settings.html.erb
View File

@ -336,45 +336,53 @@
<%= l(:label_full_text) %>
</em>
<p>
<%= content_tag :label, l(:label_index_database) %>
<%= text_field_tag 'settings[dmsf_index_database]', @settings['dmsf_index_database'], size: 50 %>
<em class="<%= klass %>">
<%= l(:label_default) %>: <%= File.expand_path('dmsf_index', Rails.root) %>
</em>
</p>
<% if defined?(EasyExtensions) %>
<p>
<em class="<%= klass %>">
<%= l(:text_fulltext_search, cmd1: 'libreoffice', cmd2: 'pdftotext') %>
</em>
</p>
<% else %>
<p>
<%= content_tag :label, l(:label_index_database) %>
<%= text_field_tag 'settings[dmsf_index_database]', @settings['dmsf_index_database'], size: 50 %>
<em class="<%= klass %>">
<%= l(:label_default) %>: <%= File.expand_path('dmsf_index', Rails.root) %>
</em>
</p>
<% stem_langs = %w(danish dutch english finnish french german hungarian italian norwegian portuguese romanian russian
spanish swedish turkish) %>
<% stem_langs = %w(danish dutch english finnish french german hungarian italian norwegian portuguese romanian russian
spanish swedish turkish) %>
<p>
<%= content_tag :label, l(:label_stemming_language) %>
<%= select_tag 'settings[dmsf_stemming_lang]', options_for_select(stem_langs, @settings['dmsf_stemming_lang']) %>
<em class="<%= klass %>">
<%= l(:note_possible_values) %>: <%= stem_langs.join(', ') %>. <%= "#{l(:label_default)}: #{stem_langs[2]}" %>
</em>
</p>
<p>
<%= content_tag :label, l(:label_stemming_language) %>
<%= select_tag 'settings[dmsf_stemming_lang]', options_for_select(stem_langs, @settings['dmsf_stemming_lang']) %>
<em class="<%= klass %>">
<%= l(:note_possible_values) %>: <%= stem_langs.join(', ') %>. <%= "#{l(:label_default)}: #{stem_langs[2]}" %>
</em>
</p>
<p>
<%= content_tag :label, l(:label_stem_strategy) %>
<%= radio_button_tag 'settings[dmsf_stemming_strategy]', 'STEM_NONE',
@settings['dmsf_stemming_strategy'] == 'STEM_NONE', checked: true %> <%= l(:option_stem_none) %>
<br>
<%= radio_button_tag 'settings[dmsf_stemming_strategy]', 'STEM_SOME',
@settings['dmsf_stemming_strategy'] == 'STEM_SOME' %> <%= l(:option_stem_some) %>
<br>
<%= radio_button_tag 'settings[dmsf_stemming_strategy]', 'STEM_ALL',
@settings['dmsf_stemming_strategy'] == 'STEM_ALL' %> <%= l(:option_stem_all) %>
<br>
<em class="<%= klass %>">
<%= l(:text_stemming_info) %>
</em>
</p>
<p>
<%= content_tag :label, l(:label_stem_strategy) %>
<%= radio_button_tag 'settings[dmsf_stemming_strategy]', 'STEM_NONE',
@settings['dmsf_stemming_strategy'] == 'STEM_NONE', checked: true %> <%= l(:option_stem_none) %>
<br>
<%= radio_button_tag 'settings[dmsf_stemming_strategy]', 'STEM_SOME',
@settings['dmsf_stemming_strategy'] == 'STEM_SOME' %> <%= l(:option_stem_some) %>
<br>
<%= radio_button_tag 'settings[dmsf_stemming_strategy]', 'STEM_ALL',
@settings['dmsf_stemming_strategy'] == 'STEM_ALL' %> <%= l(:option_stem_all) %>
<br>
<em class="<%= klass %>">
<%= l(:text_stemming_info) %>
</em>
</p>
<p>
<%= content_tag :label, l(:label_enable_cjk_ngrams) %>
<%= check_box_tag 'settings[dmsf_enable_cjk_ngrams]', true, @settings['dmsf_enable_cjk_ngrams'] %>
<em class="<%= klass %>">
<%= l(:text_enable_cjk_ngrams) %>
</em>
</p>
<p>
<%= content_tag :label, l(:label_enable_cjk_ngrams) %>
<%= check_box_tag 'settings[dmsf_enable_cjk_ngrams]', true, @settings['dmsf_enable_cjk_ngrams'] %>
<em class="<%= klass %>">
<%= l(:text_enable_cjk_ngrams) %>
</em>
</p>
<% end %>

2
config/locales/cs.yml
View File

@ -470,6 +470,8 @@ cs:
label_remove_original_documents_module: Odstranit původní modul Dokumenty
text_fulltext_search: 'Full-textové vyhledávání v dokumentech vyžaduje přítomnost %{cmd1} and %{cmd2} na serveru.'
easy_pages:
modules:
dmsf_locked_documents: My locked documents

2
config/locales/de.yml
View File

@ -466,6 +466,8 @@ de:
label_remove_original_documents_module: Entfernen originelles Modul Dokumente
text_fulltext_search: 'Full-text Suche in Dokumente fordert die Existenz %{cmd1} and %{cmd2} auf dem Server.'
easy_pages:
modules:
dmsf_locked_documents: Von mir gesperrte Dokumente

2
config/locales/en.yml
View File

@ -470,6 +470,8 @@ en:
label_remove_original_documents_module: Remove the original Documents module
text_fulltext_search: 'Full-text search in documents requires presence of %{cmd1} and %{cmd2} commands on the server.'
easy_pages:
modules:
dmsf_locked_documents: My locked documents

2
config/locales/es.yml
View File

@ -470,6 +470,8 @@ es:
label_remove_original_documents_module: Remove the original Documents module
text_fulltext_search: 'Full-text search in documents requires presence of %{cmd1} and %{cmd2} commands on the server.'
easy_pages:
modules:
dmsf_locked_documents: My locked documents

2
config/locales/fa.yml
View File

@ -449,6 +449,8 @@ fa:
label_remove_original_documents_module: Remove the original Documents module
text_fulltext_search: 'Full-text search in documents requires presence of %{cmd1} and %{cmd2} commands on the server.'
easy_pages:
modules:
dmsf_locked_documents: اسناد قفل شده‌ی من

2
config/locales/fr.yml
View File

@ -470,6 +470,8 @@ fr:
label_remove_original_documents_module: Remove the original Documents module
text_fulltext_search: 'Full-text search in documents requires presence of %{cmd1} and %{cmd2} commands on the server.'
easy_pages:
modules:
dmsf_locked_documents: My locked documents

2
config/locales/hu.yml
View File

@ -469,6 +469,8 @@ hu:
label_remove_original_documents_module: Remove the original Documents module
text_fulltext_search: 'Full-text search in documents requires presence of %{cmd1} and %{cmd2} commands on the server.'
easy_pages:
modules:
dmsf_locked_documents: My locked documents

2
config/locales/it.yml
View File

@ -470,6 +470,8 @@ it: # Italian strings thx 2 Matteo Arceci!
label_remove_original_documents_module: Remove the original Documents module
text_fulltext_search: 'Full-text search in documents requires presence of %{cmd1} and %{cmd2} commands on the server.'
easy_pages:
modules:
dmsf_locked_documents: My locked documents

2
config/locales/ja.yml
View File

@ -471,6 +471,8 @@ ja:
label_remove_original_documents_module: Remove the original Documents module
text_fulltext_search: 'Full-text search in documents requires presence of %{cmd1} and %{cmd2} commands on the server.'
easy_pages:
modules:
dmsf_locked_documents: 自分がロック中の文書

2
config/locales/ko.yml
View File

@ -470,6 +470,8 @@ ko:
label_remove_original_documents_module: Remove the original Documents module
text_fulltext_search: 'Full-text search in documents requires presence of %{cmd1} and %{cmd2} commands on the server.'
easy_pages:
modules:
dmsf_locked_documents: 내 잠긴 파일

2
config/locales/nl.yml
View File

@ -470,6 +470,8 @@ nl:
label_remove_original_documents_module: Remove the original Documents module
text_fulltext_search: 'Full-text search in documents requires presence of %{cmd1} and %{cmd2} commands on the server.'
easy_pages:
modules:
dmsf_locked_documents: My locked documents

2
config/locales/pl.yml
View File

@ -470,6 +470,8 @@ pl:
label_remove_original_documents_module: Remove the original Documents module
text_fulltext_search: 'Full-text search in documents requires presence of %{cmd1} and %{cmd2} commands on the server.'
easy_pages:
modules:
dmsf_locked_documents: My locked documents

2
config/locales/pt-BR.yml
View File

@ -470,6 +470,8 @@ pt-BR:
label_remove_original_documents_module: Remove the original Documents module
text_fulltext_search: 'Full-text search in documents requires presence of %{cmd1} and %{cmd2} commands on the server.'
easy_pages:
modules:
dmsf_locked_documents: My locked documents

2
config/locales/sl.yml
View File

@ -470,6 +470,8 @@ sl:
label_remove_original_documents_module: Remove the original Documents module
text_fulltext_search: 'Full-text search in documents requires presence of %{cmd1} and %{cmd2} commands on the server.'
easy_pages:
modules:
dmsf_locked_documents: My locked documents

2
config/locales/zh-TW.yml
View File

@ -469,6 +469,8 @@ zh-TW:
label_remove_original_documents_module: Remove the original Documents module
text_fulltext_search: 'Full-text search in documents requires presence of %{cmd1} and %{cmd2} commands on the server.'
easy_pages:
modules:
dmsf_locked_documents: My locked documents

2
config/locales/zh.yml
View File

@ -470,6 +470,8 @@ zh:
label_remove_original_documents_module: Remove the original Documents module
text_fulltext_search: 'Full-text search in documents requires presence of %{cmd1} and %{cmd2} commands on the server.'
easy_pages:
modules:
dmsf_locked_documents: My locked documents

7
lib/redmine_dmsf.rb
View File

@ -41,6 +41,7 @@ require "#{File.dirname(__FILE__)}/redmine_dmsf/patches/queries_controller_patch
require "#{File.dirname(__FILE__)}/redmine_dmsf/patches/pdf_patch"
require "#{File.dirname(__FILE__)}/redmine_dmsf/patches/access_control_patch"
require "#{File.dirname(__FILE__)}/redmine_dmsf/patches/search_patch"
require "#{File.dirname(__FILE__)}/redmine_dmsf/patches/search_controller_patch"
# A workaround for obsolete 'alias_method' usage in RedmineUp's plugins
if RedmineDmsf::Plugin.an_obsolete_plugin_present?
@ -53,6 +54,12 @@ if defined?(EasyExtensions)
require "#{File.dirname(__FILE__)}/redmine_dmsf/patches/easy_crm_case_patch"
require "#{File.dirname(__FILE__)}/redmine_dmsf/patches/attachable_patch"
require "#{File.dirname(__FILE__)}/redmine_dmsf/patches/easy_crm_cases_controller_patch.rb"
require "#{File.dirname(__FILE__)}/redmine_dmsf/patches/xapian_easy_search_helper_patch.rb"
require "#{File.dirname(__FILE__)}/redmine_dmsf/patches/application_helper_patch.rb"
# Mappers
require "#{File.dirname(__FILE__)}/xapian_easy_search/dmsf_file_mapper.rb"
require "#{File.dirname(__FILE__)}/xapian_easy_search/dmsf_folder_mapper.rb"
end
# Load up classes that make up our WebDAV solution ontop of Dav4rack

43
lib/redmine_dmsf/patches/application_helper_patch.rb Normal file
View File

@ -0,0 +1,43 @@
# frozen_string_literal: true
# Redmine plugin for Document Management System "Features"
#
# Copyright © 2011-23 Karel Pičman <karel.picman@kontron.com>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
module RedmineDmsf
module Patches
# ApplicationHelper patch
module ApplicationHelperPatch
##################################################################################################################
# Overridden methods
def xapian_link_to_entity(entity, html_options = {})
case entity.is_a?
when DmsfFolder
link_to h(entity.to_s), dmsf_folder_path(id: entity.project_id, folder_id: entity), class: 'icon icon-folder'
when DmsfFile
link_to h(entity.to_s), dmsf_file_path(id: entity), class: 'icon icon-file'
else
super
end
end
end
end
end
# Apply the patch
ApplicationHelper.prepend RedmineDmsf::Patches::ApplicationHelperPatch

45
lib/redmine_dmsf/patches/search_controller_patch.rb Normal file
View File

@ -0,0 +1,45 @@
# frozen_string_literal: true
# Redmine plugin for Document Management System "Features"
#
# Copyright © 2011-23 Karel Pičman <karel.picman@kontron.com>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#
module RedmineDmsf
module Patches
# Search controller patch
module SearchControllerPatch
##################################################################################################################
# New methods
private
def query_params
p = super
p[:dmsf_files] = params[:dmsf_files].present?
p[:dmsf_folders] = params[:dmsf_folders].present?
p
end
end
end
end
# Apply the patch
if Redmine::Plugin.installed?(:easy_extensions)
RedmineExtensions::PatchManager.register_controller_patch 'SearchController',
'RedmineDmsf::Patches::SearchControllerPatch',
prepend: true
end

40
lib/redmine_dmsf/patches/xapian_easy_search_helper_patch.rb Normal file
View File

@ -0,0 +1,40 @@
# frozen_string_literal: true
# Redmine plugin for Document Management System "Features"
#
# Copyright © 2011-23 Karel Pičman <karel.picman@kontron.com>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
module RedmineDmsf
module Patches
# XapianEasySearchHelper patch
module XapianEasySearchHelperPatch
##################################################################################################################
# Overridden methods
def xapian_entity_path(xapian_doc)
if xapian_doc.values[:source_type] == 'dmsf_folder'
dmsf_folder_path id: xapian_doc.values[:project_id], folder_id: xapian_doc.values[:source_id]
else
super
end
end
end
end
end
# Apply the patch
XapianEasySearchHelper.prepend RedmineDmsf::Patches::XapianEasySearchHelperPatch

53
lib/xapian_easy_search/dmsf_file_mapper.rb Normal file
View File

@ -0,0 +1,53 @@
# frozen_string_literal: true
# Redmine plugin for Document Management System "Features"
#
# Copyright © 2011 Vít Jonáš <vit.jonas@gmail.com>
# Copyright © 2012 Daniel Munn <dan.munn@munnster.co.uk>
# Copyright © 2011-23 Karel Pičman <karel.picman@kontron.com>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
require 'xapian_easy_search/base_mapper'
module XapianEasySearch
# DmsfFile mapper
class DmsfFileMapper < XapianEasySearch::BaseMapper
class << self
def default_index_options
super.merge(
title: :name,
updated_at: ->(dmsf_file) { dmsf_file&.last_revision&.updated_at },
content: ->(dmsf_file) { dmsf_file&.content }
)
end
def extend_query_filter
proc do |query|
# TODO: we filter the results with all files visible for the user. It's the worst way how to filter them.
files = DmsfFile.visible
.joins('JOIN dmsf_file_revisions ON dmsf_file_revisions.dmsf_file_id = dmsf_files.id')
.joins(:project)
.where(Project.allowed_to_condition(User.current, :view_dmsf_files)).to_a
files.delete_if { |f| !DmsfFolder.permissions?(f.dmsf_folder) }
ids = files.map(&:id)
Xapian::Query.new Xapian::Query::OP_AND, query, boolean_filter_query(:source_id, ids)
end
end
end
end
end
XapianEasySearch::DmsfFileMapper.attach if Redmine::Plugin.installed?('easy_extensions')

44
lib/xapian_easy_search/dmsf_folder_mapper.rb Normal file
View File

@ -0,0 +1,44 @@
# frozen_string_literal: true
# Redmine plugin for Document Management System "Features"
#
# Copyright © 2011 Vít Jonáš <vit.jonas@gmail.com>
# Copyright © 2012 Daniel Munn <dan.munn@munnster.co.uk>
# Copyright © 2011-23 Karel Pičman <karel.picman@kontron.com>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
require 'xapian_easy_search/base_mapper'
module XapianEasySearch
# DmsfFolder mapper
class DmsfFolderMapper < XapianEasySearch::BaseMapper
class << self
def default_index_options
super.merge title: :title, updated_at: :updated_at, content: :description
end
def extend_query_filter
proc do |query|
# TODO: we filter the results with all folders visible for the user. It's the worst way how to filter them.
ids = DmsfFolder.visible.ids
Xapian::Query.new Xapian::Query::OP_AND, query, boolean_filter_query(:source_id, ids)
end
end
end
end
end
XapianEasySearch::DmsfFolderMapper.attach if Redmine::Plugin.installed?('easy_extensions')