diff options
author | 2019-09-05 15:51:34 +0200 | |
---|---|---|
committer | 2019-09-05 15:51:34 +0200 | |
commit | c62bd4d5644a77d1224f188d092f39e0438062ec (patch) | |
tree | 8dc0d218c0dae45add04b1324111318048fa1105 | |
parent | Added docker-compose.override.yml for development purposes (diff) | |
download | packages-5-c62bd4d5644a77d1224f188d092f39e0438062ec.tar.gz packages-5-c62bd4d5644a77d1224f188d092f39e0438062ec.tar.bz2 packages-5-c62bd4d5644a77d1224f188d092f39e0438062ec.zip |
Migrate to ES 7.3 and the repository pattern
Elasticsearch-persistence is used as the persistence layer for Ruby
domain objects in Elasticsearch in this application. So far, the
ActiveRecord pattern has been used here. However, this pattern
has been deprecated as of version 6 of the gem and was removed in
version 7. That's why the application has been migrated to use the
repository pattern instead.
For further information, please see:
https://www.elastic.co/blog/activerecord-to-repository-changing-
persistence-patterns-with-the-elasticsearch-rails-gem
Note: The old Elasticsearch index won't be compatible with this
version anymore. That's why a fresh index should be populated.
Signed-off-by: Max Magorsch <max@magorsch.de>
44 files changed, 819 insertions, 505 deletions
@@ -23,8 +23,8 @@ gem 'jbuilder', '~> 2.0' gem 'sdoc', '~> 1.0', group: :doc # packages stuff -gem 'elasticsearch-rails', '~> 5.0' -gem 'elasticsearch-persistence', '~> 5.0' +gem 'elasticsearch-rails', '~> 7.0.0' +gem 'elasticsearch-persistence', '~> 7.0.0' gem 'nokogiri' gem 'thin' diff --git a/Gemfile.lock b/Gemfile.lock index 78774c0..14ab1e1 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -44,42 +44,32 @@ GEM tzinfo (~> 1.1) arel (9.0.0) ast (2.4.0) - axiom-types (0.1.1) - descendants_tracker (~> 0.0.4) - ice_nine (~> 0.11.0) - thread_safe (~> 0.3, >= 0.3.1) bindex (0.8.1) builder (3.2.3) byebug (11.0.1) - coercible (1.0.0) - descendants_tracker (~> 0.0.1) concurrent-ruby (1.1.5) connection_pool (2.2.2) crass (1.0.4) daemons (1.3.1) - descendants_tracker (0.0.4) - thread_safe (~> 0.3, >= 0.3.1) - elasticsearch (5.0.5) - elasticsearch-api (= 5.0.5) - elasticsearch-transport (= 5.0.5) - elasticsearch-api (5.0.5) + elasticsearch (7.3.0) + elasticsearch-api (= 7.3.0) + elasticsearch-transport (= 7.3.0) + elasticsearch-api (7.3.0) multi_json - elasticsearch-model (5.1.0) + elasticsearch-model (7.0.0) activesupport (> 3) - elasticsearch (~> 5) + elasticsearch (> 1) hashie - elasticsearch-persistence (5.1.0) + elasticsearch-persistence (7.0.0) activemodel (> 4) activesupport (> 4) - elasticsearch (~> 5) - elasticsearch-model (~> 5) + elasticsearch (~> 7) + elasticsearch-model (= 7.0.0) hashie - virtus - elasticsearch-rails (5.1.0) - elasticsearch-transport (5.0.5) + elasticsearch-rails (7.0.0) + elasticsearch-transport (7.3.0) faraday multi_json - equalizer (0.0.11) erubi (1.8.0) eventmachine (1.2.7) execjs (2.7.0) @@ -91,7 +81,6 @@ GEM hashie (3.6.0) i18n (1.6.0) concurrent-ruby (~> 1.0) - ice_nine (0.11.2) jaro_winkler (1.5.3) jbuilder (2.9.1) activesupport (>= 4.2.0) @@ -222,11 +211,6 @@ GEM uglifier (4.1.20) execjs (>= 0.3.0, < 3) unicode-display_width (1.6.0) - virtus (1.0.5) - axiom-types (~> 0.1) - coercible (~> 1.0) - descendants_tracker (~> 0.0, >= 0.0.3) - equalizer (~> 0.0, >= 0.0.9) web-console (3.7.0) actionview (>= 5.0) activemodel (>= 5.0) @@ -241,8 +225,8 @@ PLATFORMS DEPENDENCIES byebug - elasticsearch-persistence (~> 5.0) - elasticsearch-rails (~> 5.0) + elasticsearch-persistence (~> 7.0.0) + elasticsearch-rails (~> 7.0.0) jbuilder (~> 2.0) jquery-rails (~> 4.3.5) listen diff --git a/app/controllers/arches_controller.rb b/app/controllers/arches_controller.rb index cbbcb65..c72e378 100644 --- a/app/controllers/arches_controller.rb +++ b/app/controllers/arches_controller.rb @@ -43,9 +43,9 @@ class ArchesController < ApplicationController def keyworded_packages(arch) Rails.cache.fetch("keyworded_packages/#{arch}", expires_in: 10.minutes) do - Change.filter_all({ change_type: 'keyword', arches: arch }, - size: 50, - sort: { created_at: { order: 'desc' } }).map do |change| + ChangeRepository.filter_all({ change_type: 'keyword', arches: arch }, + size: 50, + sort: { created_at: { order: 'desc' } }).map do |change| change.to_os(:change_type, :package, :category, :version, :arches, :created_at) end end @@ -53,9 +53,9 @@ class ArchesController < ApplicationController def stabled_packages(arch) Rails.cache.fetch("stabled_packages/#{arch}", expires_in: 10.minutes) do - Change.filter_all({ change_type: 'stable', arches: arch }, - size: 50, - sort: { created_at: { order: 'desc' } }).map do |change| + ChangeRepository.filter_all({ change_type: 'stable', arches: arch }, + size: 50, + sort: { created_at: { order: 'desc' } }).map do |change| change.to_os(:change_type, :package, :category, :version, :arches, :created_at) end end diff --git a/app/controllers/categories_controller.rb b/app/controllers/categories_controller.rb index 33817aa..a9c9b06 100644 --- a/app/controllers/categories_controller.rb +++ b/app/controllers/categories_controller.rb @@ -3,15 +3,15 @@ class CategoriesController < ApplicationController before_action :set_nav def index - @categories = Category.all_sorted_by(:name, :asc) + @categories = CategoryRepository.all_sorted_by(:id, :asc) end def show @packages = Rails.cache.fetch("category/#{@category.name}/packages", expires_in: 10.minutes) do - Package.find_all_by(:category, - @category.name, - sort: { name_sort: { order: 'asc' } }).map do |pkg| + PackageRepository.find_all_by(:category, + @category.name, + sort: { name_sort: { order: 'asc' } }).map do |pkg| pkg.to_os(:name, :atom, :description) end end @@ -24,7 +24,7 @@ class CategoriesController < ApplicationController private def set_category - @category = Category.find_by(:name, params[:id]) + @category = CategoryRepository.find_by(:name, params[:id]) fail ActionController::RoutingError, 'No such category' unless @category @title = @category.name diff --git a/app/controllers/concerns/package_update_feeds.rb b/app/controllers/concerns/package_update_feeds.rb index 2d20672..28a951b 100644 --- a/app/controllers/concerns/package_update_feeds.rb +++ b/app/controllers/concerns/package_update_feeds.rb @@ -3,7 +3,7 @@ module PackageUpdateFeeds def new_packages Rails.cache.fetch('new_packages', expires_in: 10.minutes) do - Change.find_all_by(:change_type, 'new_package', { size: 50, sort: { created_at: { order: 'desc' } } }).map do |change| + ChangeRepository.find_all_by(:change_type, 'new_package', { size: 50, sort: { created_at: { order: 'desc' } } }).map do |change| change.to_os(:change_type, :package, :category, :created_at) end end @@ -11,7 +11,7 @@ module PackageUpdateFeeds def version_bumps Rails.cache.fetch('version_bumps', expires_in: 10.minutes) do - Change.find_all_by(:change_type, 'version_bump', { size: 50, sort: { created_at: { order: 'desc' } } }).map do |change| + ChangeRepository.find_all_by(:change_type, 'version_bump', { size: 50, sort: { created_at: { order: 'desc' } } }).map do |change| change.to_os(:change_type, :package, :category, :version, :created_at) end end @@ -19,7 +19,7 @@ module PackageUpdateFeeds def keyworded_packages Rails.cache.fetch('keyworded_packages', expires_in: 10.minutes) do - Change.find_all_by(:change_type, 'keyword', { size: 50, sort: { created_at: { order: 'desc' } } }).map do |change| + ChangeRepository.find_all_by(:change_type, 'keyword', { size: 50, sort: { created_at: { order: 'desc' } } }).map do |change| change.to_os(:change_type, :package, :category, :version, :arches, :created_at) end end @@ -27,7 +27,7 @@ module PackageUpdateFeeds def stabled_packages Rails.cache.fetch('stabled_packages', expires_in: 10.minutes) do - Change.find_all_by(:change_type, 'stable', { size: 50, sort: { created_at: { order: 'desc' } } }).map do |change| + ChangeRepository.find_all_by(:change_type, 'stable', { size: 50, sort: { created_at: { order: 'desc' } } }).map do |change| change.to_os(:change_type, :package, :category, :version, :arches, :created_at) end end diff --git a/app/controllers/packages_controller.rb b/app/controllers/packages_controller.rb index 64cb289..67cc86f 100644 --- a/app/controllers/packages_controller.rb +++ b/app/controllers/packages_controller.rb @@ -8,24 +8,24 @@ class PackagesController < ApplicationController def search @offset = params[:o].to_i || 0 - @packages = Package.default_search(params[:q], @offset) + @packages = PackageRepository.default_search(params[:q], @offset) redirect_to package_path(@packages.first).gsub('%2F', '/') if @packages.size == 1 end def suggest - @packages = Package.suggest(params[:q]) + @packages = PackageRepository.suggest(params[:q]) end def resolve - @packages = Package.resolve(params[:atom]) + @packages = PackageRepository.resolve(params[:atom]) end def show - @package = Package.find_by(:atom, params[:id]) + @package = PackageRepository.find_by(:atom, params[:id]) fail ActionController::RoutingError, 'No such package' unless @package - fresh_when etag: @package.updated_at, last_modified: @package.updated_at, public: true + fresh_when etag: Time.parse(@package.updated_at), last_modified: Time.parse(@package.updated_at), public: true # Enable this in 2024 (when we have full-color emojis on a Linux desktop) # @title = ' 📦 %s' % @package.atom @@ -34,10 +34,10 @@ class PackagesController < ApplicationController end def changelog - @package = Package.find_by(:atom, params[:id]) + @package = PackageRepository.find_by(:atom, params[:id]) fail ActionController::RoutingError, 'No such package' unless @package - if stale?(etag: @package.updated_at, last_modified: @package.updated_at, public: true) + if stale?(etag: Time.parse(@package.updated_at), last_modified: Time.parse(@package.updated_at), public: true) @changelog = Rails.cache.fetch("changelog/#{@package.atom}") do Portage::Util::History.for(@package.category, @package.name, 5) end diff --git a/app/controllers/useflags_controller.rb b/app/controllers/useflags_controller.rb index 0fa74f4..9802b78 100644 --- a/app/controllers/useflags_controller.rb +++ b/app/controllers/useflags_controller.rb @@ -6,18 +6,18 @@ class UseflagsController < ApplicationController end def show - @useflags = Useflag.get_flags(params[:id]) + @useflags = UseflagRepository.get_flags(params[:id]) if @useflags.empty? || (@useflags[:use_expand].empty? && @useflags[:local].empty? && @useflags[:global].empty?) fail ActionController::RoutingError, 'No such useflag' end - @packages = Package.find_atoms_by_useflag(params[:id]) + @packages = PackageRepository.find_atoms_by_useflag(params[:id]) @title = '%s – %s' % [params[:id], t(:use_flags)] unless @useflags[:use_expand].empty? @useflag = @useflags[:use_expand].first - @use_expand_flags = Useflag.find_all_by(:use_expand_prefix, @useflag.use_expand_prefix) + @use_expand_flags = UseflagRepository.find_all_by(:use_expand_prefix, @useflag.use_expand_prefix) @use_expand_flag_name = @useflag.use_expand_prefix.upcase render template: 'useflags/show_use_expand' @@ -29,16 +29,16 @@ class UseflagsController < ApplicationController def search # TODO: Different search? - @flags = Useflag.suggest(params[:q]) + @flags = UseflagRepository.suggest(params[:q]) end def suggest - @flags = Useflag.suggest(params[:q]) + @flags = UseflagRepository.suggest(params[:q]) end def popular @popular_useflags = Rails.cache.fetch('popular_useflags', expires_in: 24.hours) do - Version.get_popular_useflags(100) + VersionRepository.get_popular_useflags(100) end end diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rb index 619582c..8405e59 100644 --- a/app/helpers/application_helper.rb +++ b/app/helpers/application_helper.rb @@ -37,6 +37,9 @@ module ApplicationHelper end def i18n_date(date, format = '%a, %e %b %Y %H:%M') + + date = Time.parse(date).utc if date.is_a? String + content_tag :span, l(date, format: format), class: 'kk-i18n-date', diff --git a/app/jobs/category_update_job.rb b/app/jobs/category_update_job.rb index 7443099..e764ad8 100644 --- a/app/jobs/category_update_job.rb +++ b/app/jobs/category_update_job.rb @@ -5,8 +5,8 @@ class CategoryUpdateJob < ApplicationJob category_path, options = args model = Portage::Repository::Category.new(category_path) - category = Category.find_by(:name, model.name) || Category.new - idx_packages = Package.find_all_by(:category, model.name) || [] + category = CategoryRepository.find_by(:name, model.name) || Category.new + idx_packages = PackageRepository.find_all_by(:category, model.name) || [] if category.needs_import? model category.import! model diff --git a/app/jobs/package_removal_job.rb b/app/jobs/package_removal_job.rb index 877ed07..e625b96 100644 --- a/app/jobs/package_removal_job.rb +++ b/app/jobs/package_removal_job.rb @@ -4,11 +4,11 @@ class PackageRemovalJob < ApplicationJob def perform(*args) atom, _options = args - package_doc = Package.find_by(:atom, atom) + package_doc = PackageRepository.find_by(:atom, atom) return if package_doc.nil? - package_doc.versions.each(&:delete) - package_doc.delete + package_doc.versions.each { |v| VersionRepository.delete(v) } + PackageRepository.delete(package_doc) Rails.logger.warn { "Package deleted: #{atom}" } # USE flags are cleaned up by the UseflagsUpdateJob diff --git a/app/jobs/package_update_job.rb b/app/jobs/package_update_job.rb index 55e278f..53a352c 100644 --- a/app/jobs/package_update_job.rb +++ b/app/jobs/package_update_job.rb @@ -4,7 +4,7 @@ class PackageUpdateJob < ApplicationJob def perform(*args) path, options = args package_model = Portage::Repository::Package.new(path) - package_doc = Package.find_by(:atom, package_model.to_cp) || Package.new + package_doc = PackageRepository.find_by(:atom, package_model.to_cp) || Package.new if package_doc.needs_import? package_model package_doc.import!(package_model, options) diff --git a/app/jobs/record_change_job.rb b/app/jobs/record_change_job.rb index 0e6a011..ed5dd5e 100644 --- a/app/jobs/record_change_job.rb +++ b/app/jobs/record_change_job.rb @@ -25,6 +25,6 @@ class RecordChangeJob < ApplicationJob c.change_type = 'removal' end - c.save + ChangeRepository.save(c) end end diff --git a/app/jobs/useflags_update_job.rb b/app/jobs/useflags_update_job.rb index 21145c3..5558d47 100644 --- a/app/jobs/useflags_update_job.rb +++ b/app/jobs/useflags_update_job.rb @@ -10,7 +10,7 @@ class UseflagsUpdateJob < ApplicationJob def update_global(repo) model_flags = repo.global_useflags - index_flags = Useflag.global + index_flags = UseflagRepository.global new_flags = model_flags.keys - index_flags.keys del_flags = index_flags.keys - model_flags.keys @@ -21,24 +21,24 @@ class UseflagsUpdateJob < ApplicationJob flag_doc.name = flag flag_doc.description = model_flags[flag] flag_doc.scope = 'global' - flag_doc.save + UseflagRepository.save(flag_doc) end eql_flags.each do |flag| unless index_flags[flag].description == model_flags[flag] index_flags[flag].description = model_flags[flag] - index_flags[flag].save + UseflagRepository.save(index_flags[flag]) end end del_flags.each do |flag| - index_flags[flag].delete + UseflagRepository.delete(index_flags[flag]) end end def update_use_expand(repo) model_flags = repo.use_expand_flags - index_flags = Useflag.use_expand + index_flags = UseflagRepository.use_expand # Calculate keys only once index_flag_keys = index_flags.keys @@ -55,7 +55,7 @@ class UseflagsUpdateJob < ApplicationJob if index_flag_keys.include? _flag unless index_flags[_flag].description == desc index_flags[_flag].description = desc - index_flags[_flag].save + UseflagRepository.save(index_flags[_flag]) end else # New flag @@ -64,14 +64,14 @@ class UseflagsUpdateJob < ApplicationJob flag_doc.description = desc flag_doc.scope = 'use_expand' flag_doc.use_expand_prefix = variable - flag_doc.save + UseflagRepository.save(flag_doc) end end end # Find and process removed flags flag_status.each_pair do |flag, status| - index_flags[flag].delete unless status + UseflagRepository.delete(index_flags[flag]) unless status end end diff --git a/app/models/category.rb b/app/models/category.rb index f629bde..4e361c1 100644 --- a/app/models/category.rb +++ b/app/models/category.rb @@ -1,12 +1,39 @@ class Category - include Elasticsearch::Persistence::Model - include Kkuleomi::Store::Model + include ActiveModel::Model + include ActiveModel::Validations - index_name "categories-#{Rails.env}" + ATTRIBUTES = [:id, + :created_at, + :updated_at, + :name, + :description, + :metadata_hash] + attr_accessor(*ATTRIBUTES) + attr_reader :attributes + + validates :name, presence: true + + def initialize(attr={}) + attr.each do |k,v| + if ATTRIBUTES.include?(k.to_sym) + send("#{k}=", v) + end + end + end + + def attributes + @id = @name + @created_at ||= DateTime.now + @updated_at = DateTime.now + ATTRIBUTES.inject({}) do |hash, attr| + if value = send(attr) + hash[attr] = value + end + hash + end + end + alias :to_hash :attributes - attribute :name, String, mapping: { type: 'keyword' } - attribute :description, String, mapping: { type: 'text' } - attribute :metadata_hash, String, mapping: { type: 'text' } # Determines if the document model needs an update from the repository model # @@ -29,7 +56,7 @@ class Category # @param [Portage::Repository::Category] category_model Input category model def import!(category_model) import(category_model) - save + CategoryRepository.save(self) end # Returns the URL parameter for referencing this package (Rails internal stuff) diff --git a/app/models/change.rb b/app/models/change.rb index 6eaf00c..1793da4 100644 --- a/app/models/change.rb +++ b/app/models/change.rb @@ -1,13 +1,48 @@ class Change - include Elasticsearch::Persistence::Model - include Kkuleomi::Store::Model + include ActiveModel::Model + include ActiveModel::Validations - index_name "change-#{Rails.env}" + ATTRIBUTES = [:_id, + :created_at, + :updated_at, + :package, + :category, + :change_type, + :version, + :arches, + :commit] + attr_accessor(*ATTRIBUTES) + attr_reader :attributes + + validates :package, presence: true + + def initialize(attr={}) + attr.each do |k,v| + if ATTRIBUTES.include?(k.to_sym) + send("#{k}=", v) + end + end + end + + def attributes + @created_at ||= DateTime.now + @updated_at = DateTime.now + ATTRIBUTES.inject({}) do |hash, attr| + if value = send(attr) + hash[attr] = value + end + hash + end + end + alias :to_hash :attributes + + # Converts the model to an OpenStruct instance + # + # @param [Array<Symbol>] fields Fields to export into the OpenStruct, or all fields if nil + # @return [OpenStruct] OpenStruct containing the selected fields + def to_os(*fields) + fields = all_fields if fields.empty? + OpenStruct.new(Hash[fields.map { |field| [field, send(field)] }]) + end - attribute :package, String, mapping: { type: 'keyword' } - attribute :category, String, mapping: { type: 'keyword' } - attribute :change_type, String, mapping: { type: 'keyword' } - attribute :version, String, mapping: { type: 'keyword' } - attribute :arches, String, mapping: { type: 'keyword' } - attribute :commit, Hash, default: {}, mapping: { type: 'object' } end diff --git a/app/models/package.rb b/app/models/package.rb index 7ad3cbe..11ef135 100644 --- a/app/models/package.rb +++ b/app/models/package.rb @@ -1,31 +1,52 @@ class Package - include Elasticsearch::Persistence::Model - include Kkuleomi::Store::Model + include ActiveModel::Model + include ActiveModel::Validations include Kkuleomi::Store::Models::PackageImport - include Kkuleomi::Store::Models::PackageSearch - - index_name "packages-#{Rails.env}" - - raw_fields = { - type: 'keyword' - } - - attribute :category, String, mapping: raw_fields - attribute :name, String, mapping: raw_fields - attribute :name_sort, String, mapping: raw_fields - attribute :atom, String, mapping: raw_fields - attribute :description, String, mapping: { type: 'text' } - attribute :longdescription, String, mapping: { type: 'text' } - attribute :homepage, String, default: [], mapping: raw_fields - attribute :license, String, mapping: raw_fields - attribute :licenses, String, default: [], mapping: raw_fields - attribute :herds, String, default: [], mapping: raw_fields - attribute :maintainers, Array, default: [], mapping: { type: 'object' } - attribute :useflags, Hash, default: {}, mapping: { type: 'object' } - attribute :metadata_hash, String, mapping: raw_fields + + ATTRIBUTES = [:id, + :created_at, + :updated_at, + :category, + :name, + :name_sort, + :atom, + :description, + :longdescription, + :homepage, + :license, + :licenses, + :herds, + :maintainers, + :useflags, + :metadata_hash] + attr_accessor(*ATTRIBUTES) + attr_reader :attributes + + validates :name, presence: true + + def initialize(attr={}) + attr.each do |k,v| + if ATTRIBUTES.include?(k.to_sym) + send("#{k}=", v) + end + end + end + + def attributes + @id = @atom + @created_at ||= DateTime.now + @updated_at = DateTime.now + ATTRIBUTES.inject({}) do |hash, attr| + if value = send(attr) + hash[attr] = value + end + hash + end + end + alias :to_hash :attributes def category_model - @category_model ||= Category.find_by(:name, category) + @category_model ||= CategoryRepository.find_by(:name, category) end def to_param @@ -44,7 +65,7 @@ class Package end def versions - @versions ||= Version.find_all_by(:package, atom, sort: { sort_key: { order: 'asc' } }) + @versions ||= VersionRepository.find_all_by(:package, atom, sort: { sort_key: { order: 'asc' } }) end def latest_version @@ -65,6 +86,15 @@ class Package maintainers.empty? && herds.empty? end + # Converts the model to an OpenStruct instance + # + # @param [Array<Symbol>] fields Fields to export into the OpenStruct, or all fields if nil + # @return [OpenStruct] OpenStruct containing the selected fields + def to_os(*fields) + fields = all_fields if fields.empty? + OpenStruct.new(Hash[fields.map { |field| [field, send(field)] }]) + end + private # Splits a license string into single licenses, stripping the permitted logic constructs diff --git a/app/models/useflag.rb b/app/models/useflag.rb index 131a89c..12758cb 100644 --- a/app/models/useflag.rb +++ b/app/models/useflag.rb @@ -1,14 +1,41 @@ class Useflag - include Elasticsearch::Persistence::Model - include Kkuleomi::Store::Model - - index_name "useflags-#{Rails.env}" + include ActiveModel::Model + include ActiveModel::Validations + + ATTRIBUTES = [:id, + :created_at, + :updated_at, + :name, + :description, + :atom, + :scope, + :use_expand_prefix] + attr_accessor(*ATTRIBUTES) + attr_reader :attributes + + validates :name, presence: true + + + def initialize(attr={}) + attr.each do |k,v| + if ATTRIBUTES.include?(k.to_sym) + send("#{k}=", v) + end + end + end - attribute :name, String, mapping: { type: 'keyword' } - attribute :description, String, mapping: { type: 'text' } - attribute :atom, String, mapping: { type: 'keyword' } - attribute :scope, String, mapping: { type: 'keyword' } - attribute :use_expand_prefix, String, mapping: { type: 'keyword' } + def attributes + @id = @name + '-' + (@atom || 'global' ) + '-' + @scope + @created_at ||= DateTime.now + @updated_at = DateTime.now + ATTRIBUTES.inject({}) do |hash, attr| + if value = send(attr) + hash[attr] = value + end + hash + end + end + alias :to_hash :attributes def all_fields [:name, :description, :atom, :scope, :use_expand_prefix] @@ -22,78 +49,14 @@ class Useflag name.gsub(use_expand_prefix + '_', '') end - class << self - # Retrieves all flags sorted by their state - def get_flags(name) - result = { local: {}, global: [], use_expand: [] } - - find_all_by(:name, name).each do |flag| - case flag.scope - when 'local' - result[:local][flag.atom] = flag - when 'global' - result[:global] << flag - when 'use_expand' - result[:use_expand] << flag - end - end - - result - end - - def suggest(q) - results = Useflag.search( - size: 20, - query: { match_phrase_prefix: { name: q } } - ) - - processed_results = {} - results.each do |result| - if processed_results.key? result.name - processed_results[result.name] = { - name: result.name, - description: '(multiple definitions)', - scope: 'multi' - } - else - processed_results[result.name] = result - end - end - - processed_results.values.sort { |a, b| a[:name].length <=> b[:name].length } - end - - # Loads the local USE flags for a given package in a name -> model hash - # - # @param [String] atom Package to find flags for - # @return [Hash] - def local_for(atom) - map_by_name find_all_by(:atom, atom) - end - - # Maps the global USE flags in the index by their name - # This is expensive! - # - def global - map_by_name find_all_by(:scope, 'global') - end - - # Maps the USE_EXPAND variables in the index by their name - # - def use_expand - map_by_name find_all_by(:scope, 'use_expand') - end - - private + # Converts the model to a Hash + # + # @param [Array<Symbol>] fields Fields to export into the Hash, or all fields if nil + # @return [Hash] Hash containing the selected fields + def to_hsh(*fields) + fields = all_fields if fields.empty? + Hash[fields.map { |field| [field, send(field)] }] + end - def map_by_name(collection) - map = {} - collection.each do |item| - map[item.name] = item - end - - map - end - end end diff --git a/app/models/version.rb b/app/models/version.rb index 62c72f8..3629b98 100644 --- a/app/models/version.rb +++ b/app/models/version.rb @@ -1,23 +1,52 @@ +require 'date' + class Version - include Elasticsearch::Persistence::Model - include Kkuleomi::Store::Model + include ActiveModel::Model + include ActiveModel::Validations include Kkuleomi::Store::Models::VersionImport - index_name "versions-#{Rails.env}" - - attribute :version, String, mapping: { type: 'keyword' } - attribute :package, String, mapping: { type: 'keyword' } - attribute :atom, String, mapping: { type: 'keyword' } - attribute :sort_key, Integer, mapping: { type: 'integer' } - attribute :slot, String, mapping: { type: 'keyword' } - attribute :subslot, String, mapping: { type: 'keyword' } - attribute :eapi, String, mapping: { type: 'keyword' } - attribute :keywords, String, mapping: { type: 'keyword' } - attribute :masks, Array, default: [], mapping: { type: 'object' } - attribute :use, String, default: [], mapping: { type: 'keyword' } - attribute :restrict, String, default: [], mapping: { type: 'keyword' } - attribute :properties, String, default: [], mapping: { type: 'keyword' } - attribute :metadata_hash, String, mapping: { type: 'keyword' } + ATTRIBUTES = [:id, + :created_at, + :updated_at, + :version, + :package, + :atom, + :sort_key, + :slot, + :subslot, + :eapi, + :keywords, + :masks, + :use, + :restrict, + :properties, + :metadata_hash] + attr_accessor(*ATTRIBUTES) + attr_reader :attributes + + validates :version, presence: true + + def initialize(attr={}) + attr.each do |k,v| + if ATTRIBUTES.include?(k.to_sym) + send("#{k}=", v) + end + end + end + + def attributes + @id = @atom + @created_at ||= DateTime.now + @updated_at = DateTime.now + + ATTRIBUTES.inject({}) do |hash, attr| + if value = send(attr) + hash[attr] = value + end + hash + end + end + alias :to_hash :attributes # Returns the keywording state on a given architecture # @@ -138,14 +167,14 @@ class Version def calc_useflags result = { local: {}, global: {}, use_expand: {} } - local_flag_map = Useflag.local_for(atom.gsub("-#{version}", '')) + local_flag_map = UseflagRepository.local_for(atom.gsub("-#{version}", '')) local_flags = local_flag_map.keys use.sort.each do |flag| if local_flags.include? flag result[:local][flag] = local_flag_map[flag].to_hsh else - useflag = Useflag.find_by(:name, flag) + useflag = UseflagRepository.find_by(:name, flag) # This should not happen, but let's be sure next unless useflag diff --git a/app/repositories/base_repository.rb b/app/repositories/base_repository.rb new file mode 100644 index 0000000..7154691 --- /dev/null +++ b/app/repositories/base_repository.rb @@ -0,0 +1,88 @@ +require 'forwardable' +require 'singleton' + +class BaseRepository + include Elasticsearch::Persistence::Repository + include Elasticsearch::Persistence::Repository::DSL + include Singleton + + client ElasticsearchClient.default + + class << self + extend Forwardable + def_delegators :instance, :find_all_by, :filter_all, :find_by, :find_all_by_parent, :all_sorted_by + def_delegators :instance, :count, :search, :delete, :save, :refresh_index!, :create_index + end + + # Finds instances by exact IDs using the 'term' filter + def find_all_by(field, value, opts = {}) + search({ + size: 10_000, + query: { match: { field => value } } + }.merge(opts)) + end + + # Filter all instances by the given parameters + def filter_all(filters, opts = {}) + filter_args = [] + filters.each_pair { |field, value| filter_args << { term: { field => value } } } + + search({ + query: { + bool: { filter: { bool: { must: filter_args } } } + }, + size: 10_000 + }.merge(opts)) + end + + def find_by(field, value, opts = {}) + find_all_by(field, value, opts).first + end + + def find_all_by_parent(parent, opts = {}) + search(opts.merge( + size: 10_000, + query: { + bool: { + filter: { + has_parent: { + parent_type: parent.class.document_type, + query: { term: { _id: parent.id } } + } + }, + must: { + match_all: {} + } + } + }) + ) + end + + # Returns all (by default 10k) records of this class sorted by a field. + def all_sorted_by(field, order, options = {}) + search({ + size: 10_000, + query: { match_all: {} }, + sort: { field => { order: order } } + }.merge(options)) + end + + # Converts the model to an OpenStruct instance + # + # @param [Array<Symbol>] fields Fields to export into the OpenStruct, or all fields if nil + # @return [OpenStruct] OpenStruct containing the selected fields + def to_os(*fields) + fields = all_fields if fields.empty? + OpenStruct.new(Hash[fields.map { |field| [field, send(field)] }]) + end + + # Converts the model to a Hash + # + # @param [Array<Symbol>] fields Fields to export into the Hash, or all fields if nil + # @return [Hash] Hash containing the selected fields + def to_hsh(*fields) + fields = all_fields if fields.empty? + Hash[fields.map { |field| [field, send(field)] }] + end + +end
\ No newline at end of file diff --git a/app/repositories/category_repository.rb b/app/repositories/category_repository.rb new file mode 100644 index 0000000..e9cf033 --- /dev/null +++ b/app/repositories/category_repository.rb @@ -0,0 +1,21 @@ +require 'singleton' + +class CategoryRepository < BaseRepository + include Singleton + + client ElasticsearchClient.default + + index_name "categories-#{Rails.env}" + + klass Category + + mapping do + indexes :id, type: 'keyword' + indexes :name, type: 'text' + indexes :description, type: 'text' + indexes :metadata_hash, type: 'keyword' + indexes :created_at, type: 'date' + indexes :updated_at, type: 'date' + end + +end diff --git a/app/repositories/change_repository.rb b/app/repositories/change_repository.rb new file mode 100644 index 0000000..e5cc2f2 --- /dev/null +++ b/app/repositories/change_repository.rb @@ -0,0 +1,23 @@ +require 'singleton' + +class ChangeRepository < BaseRepository + include Singleton + + client ElasticsearchClient.default + + index_name "change-#{Rails.env}" + + klass Change + + mapping do + indexes :package, type: 'keyword' + indexes :category, type: 'keyword' + indexes :change_type, type: 'keyword' + indexes :version, type: 'keyword' + indexes :arches, type: 'keyword' + indexes :commit, type: 'object' + indexes :created_at, type: 'date' + indexes :updated_at, type: 'date' + end + +end diff --git a/app/repositories/elasticsearch_client.rb b/app/repositories/elasticsearch_client.rb new file mode 100644 index 0000000..88de0c8 --- /dev/null +++ b/app/repositories/elasticsearch_client.rb @@ -0,0 +1,13 @@ +class ElasticsearchClient + + def self.default + @default ||= Elasticsearch::Client.new host: ENV['ELASTICSEARCH_URL'] || 'localhost:9200' + end + + private + + def initialize(*) + raise "Should not be initialiazed" + end + +end
\ No newline at end of file diff --git a/app/repositories/package_repository.rb b/app/repositories/package_repository.rb new file mode 100644 index 0000000..ed77afb --- /dev/null +++ b/app/repositories/package_repository.rb @@ -0,0 +1,188 @@ +require 'forwardable' +require 'singleton' + +class PackageRepository < BaseRepository + include Singleton + + class << self + extend Forwardable + def_delegators :instance, :suggest, :resolve, :find_atoms_by_useflag, :default_search_size, :default_search, + :build_query, :match_wildcard, :match_phrase, :match_description, :match_category, :scoring_functions + end + + index_name "packages-#{Rails.env}" + + klass Package + + mapping do + indexes :category, type: 'keyword' + indexes :name, type: 'keyword' + indexes :name_sort, type: 'keyword' + indexes :atom, type: 'keyword' + indexes :description, type: 'text' + indexes :longdescription, type: 'text' + indexes :homepage, type: 'keyword' + indexes :license, type: 'keyword' + indexes :licenses, type: 'keyword' + indexes :herds, type: 'keyword' + indexes :maintainers, type: 'object' + indexes :useflags, type: 'object' + indexes :metadata_hash, type: 'keyword' + indexes :created_at, type: 'date' + indexes :updated_at, type: 'date' + end + + def suggest(q) + PackageRepository.search( + size: 20, + query: { + wildcard: { + name_sort: { + wildcard: q.downcase + '*' + } + } + } + ) + end + + # Tries to resolve a query atom to one or more packages + def resolve(atom) + [] if atom.nil? || atom.empty? + + PackageRepository.find_all_by(:atom, atom) + PackageRepository.find_all_by(:name, atom) + end + + # Searches the versions index for versions using a certain USE flag. + # Results are aggregated by package atoms. + def find_atoms_by_useflag(useflag) + VersionRepository.search( + size: 0, # collect all packages. + query: { + bool: { + must: { match_all: {} }, + filter: { term: { use: useflag } } + } + }, + aggs: { + group_by_package: { + terms: { + field: 'package', + order: { '_key' => 'asc' }, + # https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-bucket-terms-aggregation.html + # ES actually dislikes large sizes like this (it defines 10k buckets basically) and it will be *very* expensive but lets try it and see. + # Other limits in this app are also 10k mostly to 'make things fit kinda'. + size: 10000, + } + } + }, + ).response.aggregations['group_by_package'].buckets + end + + def default_search_size + 25 + end + + def default_search(q, offset) + return [] if q.nil? || q.empty? + + part1, part2 = q.split('/', 2) + + if part2.nil? + search(build_query(part1, nil, default_search_size, offset)) + else + search(build_query(part2, part1, default_search_size, offset)) + end + end + + def build_query(q, category, size, offset) + { + size: size, + from: offset, + query: { + function_score: { + query: { bool: bool_query_parts(q, category) }, + functions: scoring_functions + } + } + } + end + + def bool_query_parts(q, category = nil) + q_dwncsd = q.downcase + + query = { + must: [ + match_wildcard(q_dwncsd) + ], + should: [ + match_phrase(q_dwncsd), + match_description(q) + ] + } + + query[:must] << [match_category(category)] if category + + query + end + + def match_wildcard(q) + q = ('*' + q + '*') unless q.include? '*' + q.tr!(' ', '*') + + { + wildcard: { + name_sort: { + wildcard: q, + boost: 4 + } + } + } + end + + def match_phrase(q) + { + match_phrase: { + name: { + query: q, + boost: 5 + } + } + } + end + + def match_description(q) + { + match: { + description: { + query: q, + boost: 0.1 + } + } + } + end + + def match_category(cat) + { + match: { + category: { + query: cat, + boost: 2 + } + } + } + end + + def scoring_functions + [ + { + filter: { + term: { + category: 'virtual' + } + }, + weight: 0.6 + } + ] + end + +end diff --git a/app/repositories/useflag_repository.rb b/app/repositories/useflag_repository.rb new file mode 100644 index 0000000..26328f4 --- /dev/null +++ b/app/repositories/useflag_repository.rb @@ -0,0 +1,99 @@ +require 'singleton' + +class UseflagRepository < BaseRepository + include Singleton + + class << self + extend Forwardable + def_delegators :instance, :get_flags, :suggest, :local_for, :global, :use_expand + end + + index_name "useflags-#{Rails.env}" + + klass Useflag + + mapping do + indexes :name, type: 'text' + indexes :description, type: 'text' + indexes :atom, type: 'keyword' + indexes :scope, type: 'keyword' + indexes :use_expand_prefix, type: 'keyword' + indexes :created_at, type: 'date' + indexes :updated_at, type: 'date' + end + + + # Retrieves all flags sorted by their state + def get_flags(name) + result = { local: {}, global: [], use_expand: [] } + + find_all_by(:name, name).each do |flag| + case flag.scope + when 'local' + result[:local][flag.atom] = flag + when 'global' + result[:global] << flag + when 'use_expand' + result[:use_expand] << flag + end + end + + result + end + + def suggest(q) + results = search( + size: 20, + query: { match_phrase_prefix: { name: q } } + ) + + processed_results = {} + results.each do |result| + if processed_results.key? result.name + processed_results[result.name] = { + name: result.name, + description: '(multiple definitions)', + scope: 'multi' + } + else + processed_results[result.name] = result + end + end + + processed_results.values.sort { |a, b| a.name.length <=> b.name.length } + end + + # Loads the local USE flags for a given package in a name -> model hash + # + # @param [String] atom Package to find flags for + # @return [Hash] + def local_for(atom) + map_by_name find_all_by(:atom, atom) + end + + # Maps the global USE flags in the index by their name + # This is expensive! + # + def global + map_by_name find_all_by(:scope, 'global') + end + + # Maps the USE_EXPAND variables in the index by their name + # + def use_expand + map_by_name find_all_by(:scope, 'use_expand') + end + + private + + def map_by_name(collection) + map = {} + + collection.each do |item| + map[item.name] = item + end + + map + end + +end diff --git a/app/repositories/version_repository.rb b/app/repositories/version_repository.rb new file mode 100644 index 0000000..43168d2 --- /dev/null +++ b/app/repositories/version_repository.rb @@ -0,0 +1,56 @@ +require 'singleton' + +class VersionRepository < BaseRepository + include Singleton + + class << self + extend Forwardable + def_delegators :instance, :get_popular_useflags + end + + index_name "versions-#{Rails.env}" + + klass Version + + mapping do + indexes :version, type: 'keyword' + indexes :package, type: 'keyword' + indexes :atom, type: 'keyword' + indexes :sort_key, type: 'integer' + indexes :slot, type: 'keyword' + indexes :subslot, type: 'keyword' + indexes :eapi, type: 'keyword' + indexes :keywords, type: 'keyword' + indexes :masks do + indexes :arches, type: 'keyword' + indexes :atoms, type: 'keyword' + indexes :author, type: 'keyword' + indexes :date, type: 'keyword' + indexes :reason, type: 'text' + end + indexes :use, type: 'keyword' + indexes :restrict, type: 'keyword' + indexes :properties, type: 'keyword' + indexes :metadata_hash, type: 'keyword' + indexes :created_at, type: 'date' + indexes :updated_at, type: 'date' + end + + # Retrieves the most widely used USE flags by all versions + # Note that packages with many versions are over-represented + def get_popular_useflags(n = 50) + search( + query: { match_all: {} }, + aggs: { + group_by_flag: { + terms: { + field: 'use', + size: n + } + } + }, + size: 0 + ).response.aggregations['group_by_flag'].buckets + end + +end diff --git a/app/views/arches/keyworded.html.erb b/app/views/arches/keyworded.html.erb index b7ae03d..ae1df29 100644 --- a/app/views/arches/keyworded.html.erb +++ b/app/views/arches/keyworded.html.erb @@ -12,7 +12,7 @@ <% cache("keyworded-full-#{@arch}-#{@changes.hash}") do %> <ul class="list-group"> <% @changes.each do |change| - _package = Package.find_by(:atom, cp_to_atom(change.category, change.package)) %> + _package = PackageRepository.find_by(:atom, cp_to_atom(change.category, change.package)) %> <%= render partial: 'packages/changed_package', object: change, as: 'change', locals: { package: _package, version: _package.version(change.version) } %> <% end %> </ul> diff --git a/app/views/arches/stable.html.erb b/app/views/arches/stable.html.erb index b1a4548..eb66245 100644 --- a/app/views/arches/stable.html.erb +++ b/app/views/arches/stable.html.erb @@ -12,7 +12,7 @@ <% cache("stable-full-#{@arch}-#{@changes.hash}") do %> <ul class="list-group"> <% @changes.each do |change| - _package = Package.find_by(:atom, cp_to_atom(change.category, change.package)) %> + _package = PackageRepository.find_by(:atom, cp_to_atom(change.category, change.package)) %> <%= render partial: 'packages/changed_package', object: change, as: 'change', locals: { package: _package, version: _package.version(change.version) } %> <% end %> </ul> diff --git a/app/views/feeds/changes.atom.builder b/app/views/feeds/changes.atom.builder index 5991f45..a8af3df 100644 --- a/app/views/feeds/changes.atom.builder +++ b/app/views/feeds/changes.atom.builder @@ -10,7 +10,7 @@ atom_feed(id: atom_id(@feed_type, @feed_id, 'feed')) do |feed| @changes.each do |change| atom = cp_to_atom change.category, change.package - package = Package.find_by :atom, atom + package = PackageRepository.find_by :atom, atom if package.nil? logger.warn "Package for change (#{change}) nil!" next diff --git a/app/views/index/_package.html.erb b/app/views/index/_package.html.erb index eeb3109..a364209 100644 --- a/app/views/index/_package.html.erb +++ b/app/views/index/_package.html.erb @@ -1,4 +1,4 @@ -<%- package = Package.find_by(:atom, cp_to_atom(change.category, change.package)); unless package.nil? -%> +<%- package = PackageRepository.find_by(:atom, cp_to_atom(change.category, change.package)); unless package.nil? -%> <tr> <td> <a href="<%= slf(package_path(cp_to_atom(change.category, change.package))) %>"> diff --git a/app/views/index/index.html.erb b/app/views/index/index.html.erb index 890a5f3..af86c9e 100644 --- a/app/views/index/index.html.erb +++ b/app/views/index/index.html.erb @@ -1,5 +1,5 @@ <div class="jumbotron"> - <h2 class="site-welcome stick-top">Welcome to the Home of <span class="text-primary"><%= number_with_delimiter Package.count %></span> Gentoo Packages</h2> + <h2 class="site-welcome stick-top">Welcome to the Home of <span class="text-primary"><%= number_with_delimiter PackageRepository.count %></span> Gentoo Packages</h2> <form action="<%= search_packages_path %>" method="get"> <div class="typeahead-container"> @@ -43,7 +43,7 @@ </div> <ul class="list-group"> <% @version_bumps.each do |change| - _package = Package.find_by(:atom, cp_to_atom(change.category, change.package)) %> + _package = PackageRepository.find_by(:atom, cp_to_atom(change.category, change.package)) %> <%= render partial: 'packages/changed_package', object: change, as: 'change', locals: { package: _package, version: _package.version(change.version) } %> <% end %> </ul> diff --git a/app/views/packages/_metadata.html.erb b/app/views/packages/_metadata.html.erb index 426afd9..5568c08 100644 --- a/app/views/packages/_metadata.html.erb +++ b/app/views/packages/_metadata.html.erb @@ -3,7 +3,7 @@ <h3 class="panel-title"><%= t :box_metadata %></h3> </div> <ul class="list-group kk-metadata-list"> - <% if package.homepage.size > 1 %> + <% if !package.homepage.nil? && package.homepage.size > 1 %> <li class="kk-metadata-item list-group-item"> <div class="row"> <div class="col-xs-12 col-md-3 kk-metadata-key"> diff --git a/app/views/packages/_package_header.html.erb b/app/views/packages/_package_header.html.erb index 1b7876b..8c611da 100644 --- a/app/views/packages/_package_header.html.erb +++ b/app/views/packages/_package_header.html.erb @@ -25,7 +25,7 @@ <%= package.description %> </p> - <% unless package.homepage.empty? || package.homepage.first.nil? || package.homepage.first.empty? %> + <% unless package.homepage.nil? || package.homepage.first.nil? || package.homepage.first.empty? %> <p class="kk-package-homepage"> <%= content_tag :a, package.homepage.first, href: package.homepage.first, rel: 'nofollow' %> </p> diff --git a/app/views/packages/added.html.erb b/app/views/packages/added.html.erb index 97d5cb6..589226a 100644 --- a/app/views/packages/added.html.erb +++ b/app/views/packages/added.html.erb @@ -12,7 +12,7 @@ <% cache("added-full-#{@changes.hash}") do %> <ul class="list-group"> <% @changes.each do |change| - _package = Package.find_by(:atom, cp_to_atom(change.category, change.package)) %> + _package = PackageRepository.find_by(:atom, cp_to_atom(change.category, change.package)) %> <%= render partial: 'changed_package', object: change, as: 'change', locals: { package: _package, version: _package.latest_version } %> <% end %> </ul> diff --git a/app/views/packages/keyworded.html.erb b/app/views/packages/keyworded.html.erb index ff5b60c..a83a558 100644 --- a/app/views/packages/keyworded.html.erb +++ b/app/views/packages/keyworded.html.erb @@ -12,7 +12,7 @@ <% cache("keyworded-full-#{@changes.hash}") do %> <ul class="list-group"> <% @changes.each do |change| - _package = Package.find_by(:atom, cp_to_atom(change.category, change.package)) %> + _package = PackageRepository.find_by(:atom, cp_to_atom(change.category, change.package)) %> <%= render partial: 'changed_package', object: change, as: 'change', locals: { package: _package, version: _package.version(change.version) } %> <% end %> </ul> diff --git a/app/views/packages/search.html.erb b/app/views/packages/search.html.erb index fe77dd3..c4b5a75 100644 --- a/app/views/packages/search.html.erb +++ b/app/views/packages/search.html.erb @@ -3,15 +3,15 @@ <% if @packages.size > 0 %> <div class="panel panel-default"> <div class="panel-heading"> - Results <%= @offset + 1 %>—<%= [@offset + Package.default_search_size, @packages.total].min %> of <%= @packages.total %> + Results <%= @offset + 1 %>—<%= [@offset + PackageRepository.default_search_size, @packages.total].min %> of <%= @packages.total %> </div> <div class="list-group"> <%= render partial: 'package_result_row', collection: @packages, as: 'package' %> </div> <div class="panel-footer"> <div class="btn-group" role="group" aria-label="Result navigation"> - <%= link_to '< Prev', search_packages_path(q: params[:q], o: [@offset - Package.default_search_size, 0].max), class: 'btn btn-default' + (@offset > 0 ? '' : ' disabled') %> - <%= link_to 'Next >', search_packages_path(q: params[:q], o: @offset + Package.default_search_size), class: 'btn btn-default ' + ((@offset + Package.default_search_size) > @packages.total ? 'disabled' : '') %> + <%= link_to '< Prev', search_packages_path(q: params[:q], o: [@offset - PackageRepository.default_search_size, 0].max), class: 'btn btn-default' + (@offset > 0 ? '' : ' disabled') %> + <%= link_to 'Next >', search_packages_path(q: params[:q], o: @offset + PackageRepository.default_search_size), class: 'btn btn-default ' + ((@offset + PackageRepository.default_search_size) > @packages.total ? 'disabled' : '') %> </div> </div> </div> diff --git a/app/views/packages/stable.html.erb b/app/views/packages/stable.html.erb index 7b230fe..d9654de 100644 --- a/app/views/packages/stable.html.erb +++ b/app/views/packages/stable.html.erb @@ -12,7 +12,7 @@ <% cache("stable-full-#{@changes.hash}") do %> <ul class="list-group"> <% @changes.each do |change| - _package = Package.find_by(:atom, cp_to_atom(change.category, change.package)) %> + _package = PackageRepository.find_by(:atom, cp_to_atom(change.category, change.package)) %> <%= render partial: 'changed_package', object: change, as: 'change', locals: { package: _package, version: _package.version(change.version) } %> <% end %> </ul> diff --git a/app/views/packages/updated.html.erb b/app/views/packages/updated.html.erb index b774c58..af54ce1 100644 --- a/app/views/packages/updated.html.erb +++ b/app/views/packages/updated.html.erb @@ -12,7 +12,7 @@ <% cache("updated-full-#{@changes.hash}") do %> <ul class="list-group"> <% @changes.each do |change| - _package = Package.find_by(:atom, cp_to_atom(change.category, change.package)) %> + _package = PackageRepository.find_by(:atom, cp_to_atom(change.category, change.package)) %> <%= render partial: 'changed_package', object: change, as: 'change', locals: { package: _package, version: _package.version(change.version) } %> <% end %> </ul> diff --git a/app/views/useflags/_useflag_result_row.html.erb b/app/views/useflags/_useflag_result_row.html.erb index 084669f..3bdcd30 100644 --- a/app/views/useflags/_useflag_result_row.html.erb +++ b/app/views/useflags/_useflag_result_row.html.erb @@ -1,4 +1,4 @@ -<a class="list-group-item" href="<%= slf useflag_path useflag[:name] %>"> - <h3 class="kk-search-result-header"><%= useflag[:name] %></h3> - <%= useflag[:description] %> +<a class="list-group-item" href="<%= slf useflag_path useflag.name %>"> + <h3 class="kk-search-result-header"><%= useflag.name %></h3> + <%= useflag.description %> </a> diff --git a/config/initializers/elasticsearch.rb b/config/initializers/elasticsearch.rb index 4ced5b5..1037b1f 100644 --- a/config/initializers/elasticsearch.rb +++ b/config/initializers/elasticsearch.rb @@ -1,9 +1,10 @@ -require 'elasticsearch/persistence/model' +require 'elasticsearch/persistence' + +DEFAULT_CLIENT = Elasticsearch::Client.new host: ENV['ELASTICSEARCH_URL'] || 'localhost:9200' -Elasticsearch::Persistence.client = Elasticsearch::Client.new host: ENV['ELASTICSEARCH_URL'] || 'localhost:9200' if Rails.env.development? or ENV['RAILS_DEBUG'] logger = ActiveSupport::Logger.new(STDERR) logger.level = Logger::DEBUG logger.formatter = proc { |s, d, p, m| "\e[2m#{m}\n\e[0m" } - Elasticsearch::Persistence.client.transport.logger = logger + DEFAULT_CLIENT.transport.logger = logger end diff --git a/lib/kkuleomi/store.rb b/lib/kkuleomi/store.rb index a1a2d93..a36f0ca 100644 --- a/lib/kkuleomi/store.rb +++ b/lib/kkuleomi/store.rb @@ -1,15 +1,12 @@ module Kkuleomi::Store - def self.refresh_index - Category.gateway.refresh_index! - end def self.create_index(force = false) - types = [ - Category, - Package, - Version, - Change, - Useflag, + repositories = [ + CategoryRepository, + PackageRepository, + VersionRepository, + ChangeRepository, + UseflagRepository, ] base_settings = { @@ -33,15 +30,11 @@ module Kkuleomi::Store mapping: { total_fields: { limit: 50000 } } } + settings = JSON.parse('{ "mapping": { "total_fields": { "limit": 50000 } } }') + # In ES 1.5, we could use 1 mega-index. But in ES6, each model needs its own. - types.each { |type| - client = type.gateway.client - client.indices.delete(index: type.index_name) rescue nil if force - body = { - settings: type.settings.to_hash.merge(base_settings), - mappings: type.mappings.to_hash - } - client.indices.create(index: type.index_name, body: body) + repositories.each { |repository| + repository.instance.create_index!(force: true, settings: settings) } end end diff --git a/lib/kkuleomi/store/model.rb b/lib/kkuleomi/store/model.rb deleted file mode 100644 index 653884b..0000000 --- a/lib/kkuleomi/store/model.rb +++ /dev/null @@ -1,78 +0,0 @@ -module Kkuleomi::Store::Model - def self.included(base) - base.send :include, InstanceMethods - base.extend ClassMethods - end - - module ClassMethods - # Finds instances by exact IDs using the 'term' filter - def find_all_by(field, value, opts = {}) - search({ - size: 10_000, - query: { bool: { filter: { term: { field => value } } } } - }.merge(opts)) - end - - # Filter all instances by the given parameters - def filter_all(filters, opts = {}) - filter_args = [] - filters.each_pair { |field, value| filter_args << { term: { field => value } } } - - search({ - query: { - bool: { filter: { bool: { must: filter_args } } } - }, - size: 10_000 - }.merge(opts)) - end - - def find_by(field, value, opts = {}) - find_all_by(field, value, opts).first - end - - def find_all_by_parent(parent, opts = {}) - search(opts.merge( - size: 10_000, - query: { - bool: { - filter: { - has_parent: { - parent_type: parent.class.document_type, - query: { term: { _id: parent.id } } - } - }, - must: { match_all: {} } - } - } - )) - end - - # Returns all (by default 10k) records of this class sorted by a field. - def all_sorted_by(field, order, options = {}) - all({ - query: { match_all: {} }, - sort: { field => { order: order } } - }, options) - end - end - - module InstanceMethods - # Converts the model to an OpenStruct instance - # - # @param [Array<Symbol>] fields Fields to export into the OpenStruct, or all fields if nil - # @return [OpenStruct] OpenStruct containing the selected fields - def to_os(*fields) - fields = all_fields if fields.empty? - OpenStruct.new(Hash[fields.map { |field| [field, send(field)] }]) - end - - # Converts the model to a Hash - # - # @param [Array<Symbol>] fields Fields to export into the Hash, or all fields if nil - # @return [Hash] Hash containing the selected fields - def to_hsh(*fields) - fields = all_fields if fields.empty? - Hash[fields.map { |field| [field, send(field)] }] - end - end -end diff --git a/lib/kkuleomi/store/models/package_import.rb b/lib/kkuleomi/store/models/package_import.rb index 99ab433..8ae1e5d 100644 --- a/lib/kkuleomi/store/models/package_import.rb +++ b/lib/kkuleomi/store/models/package_import.rb @@ -30,15 +30,15 @@ module Kkuleomi::Store::Models::PackageImport set_basic_metadata(package_model, latest_ebuild) # Be sure to have an ID now - save + PackageRepository.save(self) import_useflags!(package_model) - Kkuleomi::Store.refresh_index + CategoryRepository.refresh_index! import_versions!(package_model, ebuilds, options) # Do this last, so that any exceptions before this point skip this step self.metadata_hash = package_model.metadata_hash - save + PackageRepository.save(self) if options[:package_state] == 'new' && !options[:suppress_change_objects] RecordChangeJob.perform_later( @@ -73,7 +73,7 @@ module Kkuleomi::Store::Models::PackageImport end def import_useflags!(package_model) - index_flags = Useflag.local_for(package_model.to_cp) + index_flags = UseflagRepository.local_for(package_model.to_cp) model_flags = package_model.metadata[:use] new_flags = model_flags.keys - index_flags.keys @@ -87,23 +87,23 @@ module Kkuleomi::Store::Models::PackageImport flag_doc.description = model_flags[flag] flag_doc.atom = package_model.to_cp flag_doc.scope = 'local' - flag_doc.save + UseflagRepository.save(flag_doc) end eql_flags.each do |flag| unless index_flags[flag].description == model_flags[flag] index_flags[flag].description = model_flags[flag] - index_flags[flag].save + UseflagRepository.save(index_flags[flag]) end end del_flags.each do |flag| - index_flags[flag].delete + UseflagRepository.delete(index_flags[flag]) end end def import_versions!(package_model, ebuilds, options) - index_v = Hash[Version.find_all_by(:package, package_model.to_cp).map { |v| [v.version, v] }] + index_v = Hash[VersionRepository.find_all_by(:package, package_model.to_cp).map { |v| [v.version, v] }] model_v = Hash[ebuilds.map { |v| [v.version, v] }] index_keys = index_v.keys @@ -128,7 +128,7 @@ module Kkuleomi::Store::Models::PackageImport if sort_key == 0 self.useflags = version_doc.useflags - save + VersionRepository.save(version_doc) end end @@ -144,12 +144,12 @@ module Kkuleomi::Store::Models::PackageImport if sort_key == 0 self.useflags = version_doc.useflags - save + VersionRepository.save(version_doc) end end del_v.each do |v| - index_v[v].delete + VersionRepository.delete(index_v[v]) end end end diff --git a/lib/kkuleomi/store/models/package_search.rb b/lib/kkuleomi/store/models/package_search.rb deleted file mode 100644 index ec0268c..0000000 --- a/lib/kkuleomi/store/models/package_search.rb +++ /dev/null @@ -1,161 +0,0 @@ -# Contains the search logic for packages -module Kkuleomi::Store::Models::PackageSearch - def self.included(base) - base.send :include, InstanceMethods - base.extend ClassMethods - end - - module ClassMethods - def suggest(q) - Package.search( - size: 20, - query: { - wildcard: { - name_sort: { - wildcard: q.downcase + '*' - } - } - } - ) - end - - # Tries to resolve a query atom to one or more packages - def resolve(atom) - [] if atom.nil? || atom.empty? - - Package.find_all_by(:atom, atom) + Package.find_all_by(:name, atom) - end - - # Searches the versions index for versions using a certain USE flag. - # Results are aggregated by package atoms. - def find_atoms_by_useflag(useflag) - Version.search( - size: 0, # collect all packages. - query: { - bool: { - must: { match_all: {} }, - filter: { term: { use: useflag } } - } - }, - aggs: { - group_by_package: { - terms: { - field: 'package', - order: { '_key' => 'asc' }, - # https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-bucket-terms-aggregation.html - # ES actually dislikes large sizes like this (it defines 10k buckets basically) and it will be *very* expensive but lets try it and see. - # Other limits in this app are also 10k mostly to 'make things fit kinda'. - size: 10000, - } - } - }, - ).response.aggregations['group_by_package'].buckets - end - - def default_search_size - 25 - end - - def default_search(q, offset) - return [] if q.nil? || q.empty? - - part1, part2 = q.split('/', 2) - - if part2.nil? - search(build_query(part1, nil, default_search_size, offset)) - else - search(build_query(part2, part1, default_search_size, offset)) - end - end - - def build_query(q, category, size, offset) - { - size: size, - from: offset, - query: { - function_score: { - query: { bool: bool_query_parts(q, category) }, - functions: scoring_functions - } - } - } - end - - def bool_query_parts(q, category = nil) - q_dwncsd = q.downcase - - query = { - must: [ - match_wildcard(q_dwncsd) - ], - should: [ - match_phrase(q_dwncsd), - match_description(q) - ] - } - - query[:must] << [match_category(category)] if category - - query - end - - def match_wildcard(q) - q = ('*' + q + '*') unless q.include? '*' - q.tr!(' ', '*') - - { - wildcard: { - name_sort: { - wildcard: q, - boost: 4 - } - } - } - end - - def match_phrase(q) - { - match_phrase: { - name: { - query: q, - boost: 5 - } - } - } - end - - def match_description(q) - { - match: { - description: { - query: q, - boost: 0.1 - } - } - } - end - - def match_category(cat) - { - match: { - category: { - query: cat, - boost: 2 - } - } - } - end - - def scoring_functions - [ - { - filter: { term: { category: 'virtual' } }, - weight: 0.6 - } - ] - end - end - - module InstanceMethods - end -end diff --git a/lib/kkuleomi/store/models/version_import.rb b/lib/kkuleomi/store/models/version_import.rb index b65b683..6ee6b64 100644 --- a/lib/kkuleomi/store/models/version_import.rb +++ b/lib/kkuleomi/store/models/version_import.rb @@ -38,7 +38,7 @@ module Kkuleomi::Store::Models::VersionImport self.masks = Portage::Util::Masks.for(ebuild_model) self.metadata_hash = ebuild_model.metadata_hash - save() + VersionRepository.save(self) # If keywords changed, calculate changes and record as needed (but only do that if we should) unless options[:suppress_change_objects] @@ -60,7 +60,7 @@ module Kkuleomi::Store::Models::VersionImport # @param [Package] parent Parent package model def set_sort_key!(key, parent) self.sort_key = key - save() + VersionRepository.save(self) end def strip_useflag_defaults(flags) |