diff --git a/cpanfile b/cpanfile index afd00cb20..88c5711fa 100644 --- a/cpanfile +++ b/cpanfile @@ -99,7 +99,7 @@ requires 'MooseX::Getopt::Dashes'; requires 'MooseX::Getopt::OptionTypeMap'; requires 'MooseX::StrictConstructor'; requires 'MooseX::Types'; -requires 'MooseX::Types::ElasticSearch', '== 0.0.4'; +requires 'MooseX::Types::ElasticSearch', '0.0.4'; requires 'MooseX::Types::Moose'; requires 'Mozilla::CA', '20211001'; requires 'namespace::autoclean'; @@ -122,7 +122,9 @@ requires 'Pod::Text', '4.14'; requires 'Ref::Util'; requires 'Safe', '2.35'; # bug fixes (used by Parse::PMFile) requires 'Scalar::Util', '1.62'; # Moose -requires 'Search::Elasticsearch', '== 2.03'; +requires 'Search::Elasticsearch' => '8.12'; +requires 'Search::Elasticsearch::Client::2_0' => '6.81'; +requires 'Search::Elasticsearch::Client::5_0' => '6.81'; requires 'Term::Choose', '1.754'; # Git::Helpers requires 'Throwable::Error'; requires 'Term::Size::Any'; # for Catalyst diff --git a/cpanfile.snapshot b/cpanfile.snapshot index 96007b5df..0f89f415a 100644 --- a/cpanfile.snapshot +++ b/cpanfile.snapshot @@ -5041,6 +5041,12 @@ DISTRIBUTIONS perl 5.006002 strict 0 warnings 0 + Net-IP-1.26 + pathname: M/MA/MANU/Net-IP-1.26.tar.gz + provides: + Net::IP 1.26 + requirements: + ExtUtils::MakeMaker 0 Net-OAuth-0.28 pathname: K/KG/KGRENNAN/Net-OAuth-0.28.tar.gz provides: @@ -6324,65 +6330,87 @@ DISTRIBUTIONS ExtUtils::MakeMaker 0 Test::More 0 perl 5.006001 - Search-Elasticsearch-2.03 - pathname: D/DR/DRTECH/Search-Elasticsearch-2.03.tar.gz - provides: - Search::Elasticsearch 2.03 - Search::Elasticsearch::Bulk 2.03 - Search::Elasticsearch::Client::0_90::Direct 2.03 - Search::Elasticsearch::Client::0_90::Direct::Cluster 2.03 - Search::Elasticsearch::Client::0_90::Direct::Indices 2.03 - Search::Elasticsearch::Client::1_0::Direct 2.03 - Search::Elasticsearch::Client::1_0::Direct::Cat 2.03 - Search::Elasticsearch::Client::1_0::Direct::Cluster 2.03 - Search::Elasticsearch::Client::1_0::Direct::Indices 2.03 - Search::Elasticsearch::Client::1_0::Direct::Nodes 2.03 - Search::Elasticsearch::Client::1_0::Direct::Snapshot 2.03 - Search::Elasticsearch::Client::2_0::Direct 2.03 - Search::Elasticsearch::Client::2_0::Direct::Cat 2.03 - Search::Elasticsearch::Client::2_0::Direct::Cluster 2.03 - Search::Elasticsearch::Client::2_0::Direct::Indices 2.03 - Search::Elasticsearch::Client::2_0::Direct::Nodes 2.03 - Search::Elasticsearch::Client::2_0::Direct::Snapshot 2.03 - Search::Elasticsearch::Client::2_0::Direct::Tasks 2.03 - Search::Elasticsearch::Cxn::Factory 2.03 - Search::Elasticsearch::Cxn::HTTPTiny 2.03 - Search::Elasticsearch::Cxn::Hijk 2.03 - Search::Elasticsearch::Cxn::LWP 2.03 - Search::Elasticsearch::CxnPool::Sniff 2.03 - Search::Elasticsearch::CxnPool::Static 2.03 - Search::Elasticsearch::CxnPool::Static::NoPing 2.03 - Search::Elasticsearch::Error 2.03 - Search::Elasticsearch::Logger::LogAny 2.03 - Search::Elasticsearch::Role::API::0_90 2.03 - Search::Elasticsearch::Role::API::1_0 2.03 - Search::Elasticsearch::Role::API::2_0 2.03 - Search::Elasticsearch::Role::Bulk 2.03 - Search::Elasticsearch::Role::Client 2.03 - Search::Elasticsearch::Role::Client::Direct 2.03 - Search::Elasticsearch::Role::Client::Direct::Main 2.03 - Search::Elasticsearch::Role::Cxn 2.03 - Search::Elasticsearch::Role::Cxn::HTTP 2.03 - Search::Elasticsearch::Role::CxnPool 2.03 - Search::Elasticsearch::Role::CxnPool::Sniff 2.03 - Search::Elasticsearch::Role::CxnPool::Static 2.03 - Search::Elasticsearch::Role::CxnPool::Static::NoPing 2.03 - Search::Elasticsearch::Role::Is_Sync 2.03 - Search::Elasticsearch::Role::Logger 2.03 - Search::Elasticsearch::Role::Scroll 2.03 - Search::Elasticsearch::Role::Serializer 2.03 - Search::Elasticsearch::Role::Serializer::JSON 2.03 - Search::Elasticsearch::Role::Transport 2.03 - Search::Elasticsearch::Scroll 2.03 - Search::Elasticsearch::Serializer::JSON 2.03 - Search::Elasticsearch::Serializer::JSON::Cpanel 2.03 - Search::Elasticsearch::Serializer::JSON::PP 2.03 - Search::Elasticsearch::Serializer::JSON::XS 2.03 - Search::Elasticsearch::TestServer 2.03 - Search::Elasticsearch::Transport 2.03 - Search::Elasticsearch::Util 2.03 - Search::Elasticsearch::Util::API::Path 2.03 - Search::Elasticsearch::Util::API::QS 2.03 + Search-Elasticsearch-8.12 + pathname: E/EZ/EZIMUEL/Search-Elasticsearch-8.12.tar.gz + provides: + Search::Elasticsearch 8.12 + Search::Elasticsearch::Client::8_0 8.12 + Search::Elasticsearch::Client::8_0::Bulk 8.12 + Search::Elasticsearch::Client::8_0::Direct 8.12 + Search::Elasticsearch::Client::8_0::Direct::Autoscaling 8.12 + Search::Elasticsearch::Client::8_0::Direct::CCR 8.12 + Search::Elasticsearch::Client::8_0::Direct::Cat 8.12 + Search::Elasticsearch::Client::8_0::Direct::Cluster 8.12 + Search::Elasticsearch::Client::8_0::Direct::Connector 8.12 + Search::Elasticsearch::Client::8_0::Direct::ConnectorSyncJob 8.12 + Search::Elasticsearch::Client::8_0::Direct::DanglingIndices 8.12 + Search::Elasticsearch::Client::8_0::Direct::Enrich 8.12 + Search::Elasticsearch::Client::8_0::Direct::Eql 8.12 + Search::Elasticsearch::Client::8_0::Direct::Esql 8.12 + Search::Elasticsearch::Client::8_0::Direct::Features 8.12 + Search::Elasticsearch::Client::8_0::Direct::Fleet 8.12 + Search::Elasticsearch::Client::8_0::Direct::Graph 8.12 + Search::Elasticsearch::Client::8_0::Direct::ILM 8.12 + Search::Elasticsearch::Client::8_0::Direct::Indices 8.12 + Search::Elasticsearch::Client::8_0::Direct::Inference 8.12 + Search::Elasticsearch::Client::8_0::Direct::Ingest 8.12 + Search::Elasticsearch::Client::8_0::Direct::License 8.12 + Search::Elasticsearch::Client::8_0::Direct::Logstash 8.12 + Search::Elasticsearch::Client::8_0::Direct::ML 8.12 + Search::Elasticsearch::Client::8_0::Direct::Migration 8.12 + Search::Elasticsearch::Client::8_0::Direct::Monitoring 8.12 + Search::Elasticsearch::Client::8_0::Direct::Nodes 8.12 + Search::Elasticsearch::Client::8_0::Direct::Profiling 8.12 + Search::Elasticsearch::Client::8_0::Direct::QueryRuleset 8.12 + Search::Elasticsearch::Client::8_0::Direct::Rollup 8.12 + Search::Elasticsearch::Client::8_0::Direct::SQL 8.12 + Search::Elasticsearch::Client::8_0::Direct::SSL 8.12 + Search::Elasticsearch::Client::8_0::Direct::SearchApplication 8.12 + Search::Elasticsearch::Client::8_0::Direct::SearchableSnapshots 8.12 + Search::Elasticsearch::Client::8_0::Direct::Security 8.12 + Search::Elasticsearch::Client::8_0::Direct::Shutdown 8.12 + Search::Elasticsearch::Client::8_0::Direct::Simulate 8.12 + Search::Elasticsearch::Client::8_0::Direct::Slm 8.12 + Search::Elasticsearch::Client::8_0::Direct::Snapshot 8.12 + Search::Elasticsearch::Client::8_0::Direct::Synonyms 8.12 + Search::Elasticsearch::Client::8_0::Direct::Tasks 8.12 + Search::Elasticsearch::Client::8_0::Direct::TextStructure 8.12 + Search::Elasticsearch::Client::8_0::Direct::Transform 8.12 + Search::Elasticsearch::Client::8_0::Direct::Watcher 8.12 + Search::Elasticsearch::Client::8_0::Direct::XPack 8.12 + Search::Elasticsearch::Client::8_0::Role::API 8.12 + Search::Elasticsearch::Client::8_0::Role::Bulk 8.12 + Search::Elasticsearch::Client::8_0::Role::Scroll 8.12 + Search::Elasticsearch::Client::8_0::Scroll 8.12 + Search::Elasticsearch::Client::8_0::TestServer 8.12 + Search::Elasticsearch::Cxn::Factory 8.12 + Search::Elasticsearch::Cxn::HTTPTiny 8.12 + Search::Elasticsearch::Cxn::LWP 8.12 + Search::Elasticsearch::CxnPool::Sniff 8.12 + Search::Elasticsearch::CxnPool::Static 8.12 + Search::Elasticsearch::CxnPool::Static::NoPing 8.12 + Search::Elasticsearch::Error 8.12 + Search::Elasticsearch::Logger::LogAny 8.12 + Search::Elasticsearch::Role::API 8.12 + Search::Elasticsearch::Role::Client 8.12 + Search::Elasticsearch::Role::Client::Direct 8.12 + Search::Elasticsearch::Role::Cxn 8.12 + Search::Elasticsearch::Role::CxnPool 8.12 + Search::Elasticsearch::Role::CxnPool::Sniff 8.12 + Search::Elasticsearch::Role::CxnPool::Static 8.12 + Search::Elasticsearch::Role::CxnPool::Static::NoPing 8.12 + Search::Elasticsearch::Role::Is_Sync 8.12 + Search::Elasticsearch::Role::Logger 8.12 + Search::Elasticsearch::Role::Serializer 8.12 + Search::Elasticsearch::Role::Serializer::JSON 8.12 + Search::Elasticsearch::Role::Transport 8.12 + Search::Elasticsearch::Serializer::JSON 8.12 + Search::Elasticsearch::Serializer::JSON::Cpanel 8.12 + Search::Elasticsearch::Serializer::JSON::PP 8.12 + Search::Elasticsearch::Serializer::JSON::XS 8.12 + Search::Elasticsearch::TestServer 8.12 + Search::Elasticsearch::Transport 8.12 + Search::Elasticsearch::Util 8.12 requirements: Any::URI::Escape 0 Data::Dumper 0 @@ -6392,9 +6420,12 @@ DISTRIBUTIONS File::Temp 0 HTTP::Headers 0 HTTP::Request 0 - HTTP::Tiny 0.043 + HTTP::Tiny 0.076 + IO::Compress::Deflate 0 + IO::Compress::Gzip 0 IO::Select 0 IO::Socket 0 + IO::Uncompress::Gunzip 0 IO::Uncompress::Inflate 0 JSON::MaybeXS 1.002002 JSON::PP 0 @@ -6404,8 +6435,9 @@ DISTRIBUTIONS Log::Any::Adapter 0 MIME::Base64 0 Module::Runtime 0 - Moo 1.003 + Moo 2.001000 Moo::Role 0 + Net::IP 0 POSIX 0 Package::Stash 0.34 Scalar::Util 0 @@ -6417,6 +6449,69 @@ DISTRIBUTIONS overload 0 strict 0 warnings 0 + Search-Elasticsearch-Client-2_0-6.81 + pathname: E/EZ/EZIMUEL/Search-Elasticsearch-Client-2_0-6.81.tar.gz + provides: + Search::Elasticsearch::Client::2_0 6.81 + Search::Elasticsearch::Client::2_0::Bulk 6.81 + Search::Elasticsearch::Client::2_0::Direct 6.81 + Search::Elasticsearch::Client::2_0::Direct::Cat 6.81 + Search::Elasticsearch::Client::2_0::Direct::Cluster 6.81 + Search::Elasticsearch::Client::2_0::Direct::Indices 6.81 + Search::Elasticsearch::Client::2_0::Direct::Nodes 6.81 + Search::Elasticsearch::Client::2_0::Direct::Snapshot 6.81 + Search::Elasticsearch::Client::2_0::Direct::Tasks 6.81 + Search::Elasticsearch::Client::2_0::Role::API 6.81 + Search::Elasticsearch::Client::2_0::Role::Bulk 6.81 + Search::Elasticsearch::Client::2_0::Role::Scroll 6.81 + Search::Elasticsearch::Client::2_0::Scroll 6.81 + Search::Elasticsearch::Client::2_0::TestServer 6.81 + requirements: + Devel::GlobalDestruction 0 + ExtUtils::MakeMaker 0 + Moo 0 + Moo::Role 0 + Search::Elasticsearch 6.00 + Search::Elasticsearch::Role::API 0 + Search::Elasticsearch::Role::Client::Direct 0 + Search::Elasticsearch::Role::Is_Sync 0 + Search::Elasticsearch::Util 0 + Try::Tiny 0 + namespace::clean 0 + strict 0 + warnings 0 + Search-Elasticsearch-Client-5_0-6.81 + pathname: E/EZ/EZIMUEL/Search-Elasticsearch-Client-5_0-6.81.tar.gz + provides: + Search::Elasticsearch::Client::5_0 6.81 + Search::Elasticsearch::Client::5_0::Bulk 6.81 + Search::Elasticsearch::Client::5_0::Direct 6.81 + Search::Elasticsearch::Client::5_0::Direct::Cat 6.81 + Search::Elasticsearch::Client::5_0::Direct::Cluster 6.81 + Search::Elasticsearch::Client::5_0::Direct::Indices 6.81 + Search::Elasticsearch::Client::5_0::Direct::Ingest 6.81 + Search::Elasticsearch::Client::5_0::Direct::Nodes 6.81 + Search::Elasticsearch::Client::5_0::Direct::Snapshot 6.81 + Search::Elasticsearch::Client::5_0::Direct::Tasks 6.81 + Search::Elasticsearch::Client::5_0::Role::API 6.81 + Search::Elasticsearch::Client::5_0::Role::Bulk 6.81 + Search::Elasticsearch::Client::5_0::Role::Scroll 6.81 + Search::Elasticsearch::Client::5_0::Scroll 6.81 + Search::Elasticsearch::Client::5_0::TestServer 6.81 + requirements: + Devel::GlobalDestruction 0 + ExtUtils::MakeMaker 0 + Moo 0 + Moo::Role 0 + Search::Elasticsearch 6.00 + Search::Elasticsearch::Role::API 0 + Search::Elasticsearch::Role::Client::Direct 0 + Search::Elasticsearch::Role::Is_Sync 0 + Search::Elasticsearch::Util 0 + Try::Tiny 0 + namespace::clean 0 + strict 0 + warnings 0 Sereal-Decoder-5.004 pathname: Y/YV/YVES/Sereal-Decoder-5.004.tar.gz provides: diff --git a/lib/Catalyst/Plugin/Session/Store/ElasticSearch.pm b/lib/Catalyst/Plugin/Session/Store/ElasticSearch.pm index 761fb529a..b02e4b718 100644 --- a/lib/Catalyst/Plugin/Session/Store/ElasticSearch.pm +++ b/lib/Catalyst/Plugin/Session/Store/ElasticSearch.pm @@ -7,6 +7,7 @@ extends 'Catalyst::Plugin::Session::Store'; use MooseX::Types::ElasticSearch qw( ES ); use MetaCPAN::Server::Config (); +use MetaCPAN::Util qw( true false ); has _session_es => ( is => 'ro', @@ -55,7 +56,7 @@ sub store_session_data { type => $self->_session_es_type, id => $sid, body => $session, - refresh => 1, + refresh => true, ); } } @@ -68,7 +69,7 @@ sub delete_session_data { index => $self->_session_es_index, type => $self->_session_es_type, id => $sid, - refresh => 1, + refresh => true, ); }; } diff --git a/lib/ElasticSearchX/Model/Document/Set.pm b/lib/ElasticSearchX/Model/Document/Set.pm new file mode 100644 index 000000000..c2424d14b --- /dev/null +++ b/lib/ElasticSearchX/Model/Document/Set.pm @@ -0,0 +1,47 @@ +package ElasticSearchX::Model::Document::Set; +use strict; +use warnings; + +use MetaCPAN::Model::Hacks; + +no warnings 'redefine'; + +our %query_override; +my $_build_query = \&_build_query; +*_build_query = sub { + my $query = $_build_query->(@_); + %$query = ( %$query, %query_override, ); + return $query; +}; + +our %qs_override; +my $_build_qs = \&_build_qs; +*_build_qs = sub { + my $qs = $_build_qs->(@_); + %$qs = ( %$qs, %qs_override, ); + return $qs; +}; + +my $delete = \&delete; +*delete = sub { + local %qs_override = ( search_type => 'query_then_fetch' ); + local %query_override = ( sort => '_doc' ); + return $delete->(@_); +}; + +my $get = \&get; +*get = sub { + my ( $self, $args, $qs ) = @_; + if ( $self->es->api_version eq '2_0' ) { + goto &$get; + } + my %qs = %{ $qs || {} }; + if ( my $fields = $self->fields ) { + $qs{_source} = $fields; + local $self->{fields}; + return $get->( $self, $args, \%qs ); + } + goto &$get; +}; + +1; diff --git a/lib/MetaCPAN/API/Model/Cover.pm b/lib/MetaCPAN/API/Model/Cover.pm index 43134d3a9..7a9dfa1e4 100644 --- a/lib/MetaCPAN/API/Model/Cover.pm +++ b/lib/MetaCPAN/API/Model/Cover.pm @@ -2,6 +2,8 @@ package MetaCPAN::API::Model::Cover; use MetaCPAN::Moose; +use MetaCPAN::Util qw(hit_total); + with 'MetaCPAN::API::Model::Role::ES'; sub find_release_coverage { @@ -17,7 +19,7 @@ sub find_release_coverage { size => 999, } ); - $res->{hits}{total} or return {}; + hit_total($res) or return {}; return +{ %{ $res->{hits}{hits}[0]{_source} }, diff --git a/lib/MetaCPAN/Document/File.pm b/lib/MetaCPAN/Document/File.pm index f5c3c9f6a..4cf2b88c6 100644 --- a/lib/MetaCPAN/Document/File.pm +++ b/lib/MetaCPAN/Document/File.pm @@ -414,9 +414,8 @@ sub _build_suggest { $weight = 0 if $weight < 0; return +{ - input => [$doc], - payload => { doc_name => $doc }, - weight => $weight, + input => [$doc], + weight => $weight, }; } diff --git a/lib/MetaCPAN/Document/File/Set.pm b/lib/MetaCPAN/Document/File/Set.pm index f4f4fe7b4..28b90349a 100644 --- a/lib/MetaCPAN/Document/File/Set.pm +++ b/lib/MetaCPAN/Document/File/Set.pm @@ -6,7 +6,7 @@ use List::Util qw( max ); use MetaCPAN::Query::Favorite (); use MetaCPAN::Query::File (); use MetaCPAN::Query::Release (); -use MetaCPAN::Util qw( single_valued_arrayref_to_scalar true false ); +use MetaCPAN::Util qw( true false ); extends 'ElasticSearchX::Model::Document::Set'; @@ -75,7 +75,8 @@ my @ROGUE_DISTRIBUTIONS = qw( sub find { my ( $self, $module ) = @_; - my @candidates = $self->index->type('file')->query( { + + my $query = { bool => { must => [ { term => { indexed => true } }, @@ -119,22 +120,38 @@ sub find { }, ], }, - } )->sort( [ - '_score', - { 'version_numified' => { order => 'desc' } }, - { 'date' => { order => 'desc' } }, - { 'mime' => { order => 'asc' } }, - { 'stat.mtime' => { order => 'desc' } } - ] )->search_type('dfs_query_then_fetch')->size(100)->all; + }; + + my $res = $self->es->search( + index => $self->index->name, + type => 'file', + search_type => 'dfs_query_then_fetch', + body => { + query => $query, + sort => [ + '_score', + { 'version_numified' => { order => 'desc' } }, + { 'date' => { order => 'desc' } }, + { 'mime' => { order => 'asc' } }, + { 'stat.mtime' => { order => 'desc' } } + ], + _source => [ + qw( documentation module.indexed module.authoried module.name ) + ], + size => 100, + }, + ); + + my @candidates = @{ $res->{hits}{hits} }; my ($file) = grep { - grep { $_->indexed && $_->authorized && $_->name eq $module } - @{ $_->module || [] } - } grep { !$_->documentation || $_->documentation eq $module } + grep { $_->{indexed} && $_->{authorized} && $_->{name} eq $module } + @{ $_->{module} || [] } + } grep { !$_->{documentation} || $_->{documentation} eq $module } @candidates; $file ||= shift @candidates; - return $file ? $self->get( $file->id ) : undef; + return $file ? $self->get( $file->{_id} ) : undef; } sub find_pod { @@ -265,15 +282,13 @@ sub history { sub autocomplete { my ( $self, @terms ) = @_; - my $query = join( q{ }, @terms ); - return $self unless $query; - my $data = $self->search_type('dfs_query_then_fetch')->query( { + my $query = { bool => { must => [ { multi_match => { - query => $query, + query => join( q{ }, @terms ), type => 'most_fields', fields => [ 'documentation', 'documentation.*' ], analyzer => 'camelcase', @@ -288,15 +303,21 @@ sub autocomplete { must_not => [ { terms => { distribution => \@ROGUE_DISTRIBUTIONS } }, ], }, - } )->sort( [ '_score', 'documentation' ] ); - - $data = $data->fields( [qw(documentation release author distribution)] ) - unless $self->fields; - - $data = $data->source(0)->raw->all; + }; + + my $data = $self->es->search( + search_type => 'dfs_query_then_fetch', + index => $self->index->name, + type => 'file', + body => { + query => $query, + sort => [ '_score', 'documentation' ], + _source => [qw( documentation release author distribution )], + }, + ); - single_valued_arrayref_to_scalar( $_->{fields} ) - for @{ $data->{hits}{hits} }; + # this is backcompat. we don't use this end point. + $_->{fields} = delete $_->{_source} for @{ $data->{hits}{hits} }; return $data; } @@ -307,8 +328,7 @@ sub autocomplete_suggester { my $search_size = 100; - my $suggestions - = $self->search_type('dfs_query_then_fetch')->es->suggest( { + my $suggestions = $self->es->suggest( { index => $self->index->name, body => { documentation => { @@ -319,7 +339,7 @@ sub autocomplete_suggester { } } }, - } ); + } ); my %docs; @@ -328,14 +348,6 @@ sub autocomplete_suggester { ( $docs{ $suggest->{text} }, $suggest->{score} ); } - my @fields = qw( - author - date - deprecated - distribution - documentation - release - ); my $data = $self->es->search( { index => $self->index->name, type => 'file', @@ -355,17 +367,22 @@ sub autocomplete_suggester { ], } }, + _source => [ qw( + author + date + deprecated + distribution + documentation + release + ) ], + size => $search_size, }, - fields => \@fields, - size => $search_size, } ); my %valid = map { - my $got = $_->{fields}; - my %record; - @record{@fields} = map { $got->{$_}[0] } @fields; - $record{name} = delete $record{documentation}; # rename - ( $_->{fields}{documentation}[0] => \%record ); + my %record = %{ $_->{_source} }; + $record{name} = delete $record{documentation}; # rename + ( $record{name} => \%record ); } @{ $data->{hits}{hits} }; # remove any exact match, it will be added later @@ -393,7 +410,7 @@ sub autocomplete_suggester { sub find_changes_files { my ( $self, $author, $release ) = @_; my $result = $self->files_by_category( $author, $release, ['changelog'], - { fields => true } ); + { _source => true } ); my ($file) = @{ $result->{categories}{changelog} || [] }; return $file; } diff --git a/lib/MetaCPAN/Document/Mirror.pm b/lib/MetaCPAN/Document/Mirror.pm index 0526285f0..7b3b04e94 100644 --- a/lib/MetaCPAN/Document/Mirror.pm +++ b/lib/MetaCPAN/Document/Mirror.pm @@ -4,7 +4,7 @@ use strict; use warnings; use Moose; -use ElasticSearchX::Model::Document::Types qw( Location ); +use MooseX::Types::ElasticSearch qw( Location ); use ElasticSearchX::Model::Document; use MetaCPAN::Types::TypeTiny qw( Dict Str ); diff --git a/lib/MetaCPAN/Document/Release/Set.pm b/lib/MetaCPAN/Document/Release/Set.pm index 08a2c462b..8adae0f94 100644 --- a/lib/MetaCPAN/Document/Release/Set.pm +++ b/lib/MetaCPAN/Document/Release/Set.pm @@ -2,8 +2,6 @@ package MetaCPAN::Document::Release::Set; use Moose; -use MetaCPAN::Util qw( single_valued_arrayref_to_scalar ); - use MetaCPAN::Query::Release (); extends 'ElasticSearchX::Model::Document::Set'; diff --git a/lib/MetaCPAN/Model/Hacks.pm b/lib/MetaCPAN/Model/Hacks.pm new file mode 100644 index 000000000..66ad655c3 --- /dev/null +++ b/lib/MetaCPAN/Model/Hacks.pm @@ -0,0 +1,23 @@ +package MetaCPAN::Model::Hacks; +use strict; +use warnings; + +sub import { + my ( $caller, $caller_file ) = caller; + + my $file = $caller =~ s{::}{/}gr . '.pm'; + my $dir = $caller_file =~ s{/\Q$file\E\z}{}r; + local @INC = grep $_ ne $dir, @INC; + my $inc; + { + local $INC{$file}; + delete $INC{$file}; + require $file; + $inc = $INC{$file}; + } + delete $INC{$file}; + $INC{$file} = $inc; + return; +} + +1; diff --git a/lib/MetaCPAN/Model/Release.pm b/lib/MetaCPAN/Model/Release.pm index e2ce4cfe7..c5f06fff6 100644 --- a/lib/MetaCPAN/Model/Release.pm +++ b/lib/MetaCPAN/Model/Release.pm @@ -222,7 +222,8 @@ sub _build_document { || $document->{abstract} eq 'null' ); $document - = $self->index->type('release')->put( $document, { refresh => 1 } ); + = $self->index->type('release') + ->put( $document, { refresh => true } ); # create distribution if doesn't exist my $dist_count = $self->es->count( diff --git a/lib/MetaCPAN/Query/Author.pm b/lib/MetaCPAN/Query/Author.pm index ea52e58fe..c6179c5b0 100644 --- a/lib/MetaCPAN/Query/Author.pm +++ b/lib/MetaCPAN/Query/Author.pm @@ -2,7 +2,7 @@ package MetaCPAN::Query::Author; use MetaCPAN::Moose; -use MetaCPAN::Util qw( single_valued_arrayref_to_scalar ); +use MetaCPAN::Util qw(hit_total); use Ref::Util qw( is_arrayref ); with 'MetaCPAN::Query::Role::Common'; @@ -23,15 +23,12 @@ sub by_ids { body => $body, ); - my @authors = map { - single_valued_arrayref_to_scalar( $_->{_source} ); - $_->{_source} - } @{ $authors->{hits}{hits} }; + my @authors = map $_->{_source}, @{ $authors->{hits}{hits} }; return { authors => \@authors, took => $authors->{took}, - total => $authors->{hits}{total}, + total => hit_total($authors), }; } @@ -48,15 +45,12 @@ sub by_user { } ); - my @authors = map { - single_valued_arrayref_to_scalar( $_->{_source} ); - $_->{_source} - } @{ $authors->{hits}{hits} }; + my @authors = map $_->{_source}, @{ $authors->{hits}{hits} }; return { authors => \@authors, took => $authors->{took}, - total => $authors->{hits}{total}, + total => hit_total($authors), }; } @@ -94,15 +88,13 @@ sub search { body => $body, ); - my @authors = map { - single_valued_arrayref_to_scalar( $_->{_source} ); - +{ %{ $_->{_source} }, id => $_->{_id} } - } @{ $ret->{hits}{hits} }; + my @authors = map { +{ %{ $_->{_source} }, id => $_->{_id} } } + @{ $ret->{hits}{hits} }; return +{ authors => \@authors, took => $ret->{took}, - total => $ret->{hits}{total}, + total => hit_total($ret), }; } @@ -127,15 +119,13 @@ sub prefix_search { body => $body, ); - my @authors = map { - single_valued_arrayref_to_scalar( $_->{_source} ); - +{ %{ $_->{_source} }, id => $_->{_id} } - } @{ $ret->{hits}{hits} }; + my @authors = map { +{ %{ $_->{_source} }, id => $_->{_id} } } + @{ $ret->{hits}{hits} }; return +{ authors => \@authors, took => $ret->{took}, - total => $ret->{hits}{total}, + total => hit_total($ret), }; } diff --git a/lib/MetaCPAN/Query/Contributor.pm b/lib/MetaCPAN/Query/Contributor.pm index 0599420fd..30b53774f 100644 --- a/lib/MetaCPAN/Query/Contributor.pm +++ b/lib/MetaCPAN/Query/Contributor.pm @@ -2,6 +2,8 @@ package MetaCPAN::Query::Contributor; use MetaCPAN::Moose; +use MetaCPAN::Util qw(hit_total); + with 'MetaCPAN::Query::Role::Common'; sub find_release_contributors { @@ -24,7 +26,7 @@ sub find_release_contributors { size => 999, } ); - $res->{hits}{total} or return {}; + hit_total($res) or return {}; return +{ contributors => [ map { $_->{_source} } @{ $res->{hits}{hits} } ] }; diff --git a/lib/MetaCPAN/Query/Cover.pm b/lib/MetaCPAN/Query/Cover.pm index bd4ad47c1..63738c206 100644 --- a/lib/MetaCPAN/Query/Cover.pm +++ b/lib/MetaCPAN/Query/Cover.pm @@ -2,6 +2,8 @@ package MetaCPAN::Query::Cover; use MetaCPAN::Moose; +use MetaCPAN::Util qw(hit_total); + with 'MetaCPAN::Query::Role::Common'; sub find_release_coverage { @@ -17,7 +19,7 @@ sub find_release_coverage { size => 999, } ); - $res->{hits}{total} or return {}; + hit_total($res) or return {}; return +{ %{ $res->{hits}{hits}[0]{_source} }, diff --git a/lib/MetaCPAN/Query/Distribution.pm b/lib/MetaCPAN/Query/Distribution.pm index 241bb1f6e..63fd0559c 100644 --- a/lib/MetaCPAN/Query/Distribution.pm +++ b/lib/MetaCPAN/Query/Distribution.pm @@ -2,6 +2,8 @@ package MetaCPAN::Query::Distribution; use MetaCPAN::Moose; +use MetaCPAN::Util qw(hit_total); + with 'MetaCPAN::Query::Role::Common'; sub get_river_data_by_dist { @@ -21,7 +23,7 @@ sub get_river_data_by_dist { size => 999, } ); - $res->{hits}{total} or return {}; + hit_total($res) or return {}; return +{ river => +{ $dist => $res->{hits}{hits}[0]{_source}{river} } }; } @@ -43,7 +45,7 @@ sub get_river_data_by_dists { size => 999, } ); - $res->{hits}{total} or return {}; + hit_total($res) or return {}; return +{ river => +{ diff --git a/lib/MetaCPAN/Query/Favorite.pm b/lib/MetaCPAN/Query/Favorite.pm index da2d0987f..0d1c2447c 100644 --- a/lib/MetaCPAN/Query/Favorite.pm +++ b/lib/MetaCPAN/Query/Favorite.pm @@ -2,7 +2,7 @@ package MetaCPAN::Query::Favorite; use MetaCPAN::Moose; -use MetaCPAN::Util qw( single_valued_arrayref_to_scalar ); +use MetaCPAN::Util qw(hit_total); with 'MetaCPAN::Query::Role::Common'; @@ -72,18 +72,16 @@ sub by_user { index => $self->index_name, type => 'favorite', body => { - query => { term => { user => $user } }, - fields => [qw( author date distribution )], - sort => ['distribution'], - size => $size, + query => { term => { user => $user } }, + _source => [qw( author date distribution )], + sort => ['distribution'], + size => $size, } ); - return {} unless $favs->{hits}{total}; + return {} unless hit_total($favs); my $took = $favs->{took}; - my @favs = map { $_->{fields} } @{ $favs->{hits}{hits} }; - - single_valued_arrayref_to_scalar( \@favs ); + my @favs = map { $_->{_source} } @{ $favs->{hits}{hits} }; # filter out backpan only distributions @@ -104,14 +102,14 @@ sub by_user { ] } }, - fields => ['distribution'], - size => scalar(@favs), + _source => ['distribution'], + size => scalar(@favs), } ); $took += $no_backpan->{took}; - if ( $no_backpan->{hits}{total} ) { - my %has_no_backpan = map { $_->{fields}{distribution}[0] => 1 } + if ( hit_total($no_backpan) ) { + my %has_no_backpan = map { $_->{_source}{distribution} => 1 } @{ $no_backpan->{hits}{hits} }; @favs = grep { exists $has_no_backpan{ $_->{distribution} } } @favs; @@ -175,7 +173,7 @@ sub recent { return +{ favorites => \@favs, took => $favs->{took}, - total => $favs->{hits}{total} + total => hit_total($favs), }; } @@ -191,12 +189,10 @@ sub users_by_distribution { size => 1000, } ); - return {} unless $favs->{hits}{total}; + return {} unless hit_total($favs); my @plusser_users = map { $_->{_source}{user} } @{ $favs->{hits}{hits} }; - single_valued_arrayref_to_scalar( \@plusser_users ); - return { users => \@plusser_users }; } diff --git a/lib/MetaCPAN/Query/File.pm b/lib/MetaCPAN/Query/File.pm index ae00db15c..ace8334a9 100644 --- a/lib/MetaCPAN/Query/File.pm +++ b/lib/MetaCPAN/Query/File.pm @@ -2,7 +2,7 @@ package MetaCPAN::Query::File; use MetaCPAN::Moose; -use MetaCPAN::Util qw( single_valued_arrayref_to_scalar true false ); +use MetaCPAN::Util qw( hit_total true false ); with 'MetaCPAN::Query::Role::Common'; @@ -24,8 +24,8 @@ sub dir { ] }, }, - size => 999, - fields => [ + size => 999, + _source => [ qw(name stat.mtime path stat.size directory slop documentation mime) ], }; @@ -36,8 +36,7 @@ sub dir { body => $body, } ); - my $dir = [ map { $_->{fields} } @{ $data->{hits}{hits} } ]; - single_valued_arrayref_to_scalar($dir); + my $dir = [ map { $_->{_source} } @{ $data->{hits}{hits} } ]; return { dir => $dir }; } @@ -259,7 +258,7 @@ sub interesting_files { return $return unless @clauses; - my $source = $options->{fields} || [ qw( + $options->{_source} //= [ qw( author distribution documentation @@ -269,6 +268,7 @@ sub interesting_files { release status ) ]; + $options->{size} //= 250; my $body = { query => { @@ -291,9 +291,7 @@ sub interesting_files { ], }, }, - _source => $source, - - size => $options->{size} || 250, + %$options, }; my $data = $self->es->search( { @@ -303,7 +301,7 @@ sub interesting_files { } ); $return->{took} = $data->{took}; - $return->{total} = $data->{hits}{total}; + $return->{total} = hit_total($data); return $return unless $return->{total}; diff --git a/lib/MetaCPAN/Query/Mirror.pm b/lib/MetaCPAN/Query/Mirror.pm index dbd8764c1..b6c281c52 100644 --- a/lib/MetaCPAN/Query/Mirror.pm +++ b/lib/MetaCPAN/Query/Mirror.pm @@ -1,6 +1,7 @@ package MetaCPAN::Query::Mirror; use MetaCPAN::Moose; +use MetaCPAN::Util qw( hit_total ); with 'MetaCPAN::Query::Role::Common'; @@ -57,7 +58,7 @@ sub search { return { mirrors => $data, - total => $ret->{hits}{total}, + total => hit_total($ret), took => $ret->{took} }; } diff --git a/lib/MetaCPAN/Query/Package.pm b/lib/MetaCPAN/Query/Package.pm index 9ea50d8a6..cf3addd5c 100644 --- a/lib/MetaCPAN/Query/Package.pm +++ b/lib/MetaCPAN/Query/Package.pm @@ -8,13 +8,11 @@ sub get_modules { my ( $self, $dist, $ver ) = @_; my $query = +{ - query => { - bool => { - must => [ - { term => { distribution => $dist } }, - { term => { dist_version => $ver } }, - ], - } + bool => { + must => [ + { term => { distribution => $dist } }, + { term => { dist_version => $ver } }, + ], } }; diff --git a/lib/MetaCPAN/Query/Release.pm b/lib/MetaCPAN/Query/Release.pm index 1c832116b..3e688fe59 100644 --- a/lib/MetaCPAN/Query/Release.pm +++ b/lib/MetaCPAN/Query/Release.pm @@ -2,7 +2,8 @@ package MetaCPAN::Query::Release; use MetaCPAN::Moose; -use MetaCPAN::Util qw( single_valued_arrayref_to_scalar true false ); +use MetaCPAN::Util + qw( hit_total single_valued_arrayref_to_scalar true false ); with 'MetaCPAN::Query::Role::Common'; @@ -63,14 +64,12 @@ sub get_contributors { my ( $self, $author_name, $release_name ) = @_; my $query = +{ - query => { - bool => { - must => [ - { term => { name => $release_name } }, - { term => { author => $author_name } }, - ], - }, - } + bool => { + must => [ + { term => { name => $release_name } }, + { term => { author => $author_name } }, + ], + }, }; my $res = $self->es->search( @@ -186,7 +185,7 @@ sub get_contributors { } ); - if ( $check_author->{hits}{total} ) { + if ( hit_total($check_author) ) { $contrib->{pauseid} = uc $check_author->{hits}{hits}[0]{_source}{pauseid}; } @@ -195,11 +194,9 @@ sub get_contributors { } my $contrib_query = +{ - query => { - terms => { - pauseid => - [ map { $_->{pauseid} ? $_->{pauseid} : () } @contribs ] - } + terms => { + pauseid => + [ map { $_->{pauseid} ? $_->{pauseid} : () } @contribs ] } }; @@ -256,7 +253,7 @@ sub get_files { sub get_checksums { my ( $self, $release ) = @_; - my $query = +{ query => { term => { name => $release } } }; + my $query = { term => { name => $release } }; my $ret = $self->es->search( index => $self->index_name, @@ -365,12 +362,11 @@ sub by_author_and_name { ); my $data = $ret->{hits}{hits}[0]{_source}; - single_valued_arrayref_to_scalar($data); return { took => $ret->{took}, release => $data, - total => $ret->{hits}{total} + total => hit_total($ret), }; } @@ -386,17 +382,15 @@ sub by_author_and_names { should => [ map { +{ - query => { - bool => { - must => [ - { - term => { - author => uc( $_->{author} ) - } - }, - { term => { 'name' => $_->{name} } }, - ] - } + bool => { + must => [ + { + term => { + author => uc( $_->{author} ) + } + }, + { term => { 'name' => $_->{name} } }, + ] } } } @$releases @@ -414,13 +408,12 @@ sub by_author_and_names { my @releases; for my $hit ( @{ $ret->{hits}{hits} } ) { my $src = $hit->{_source}; - single_valued_arrayref_to_scalar($src); push @releases, $src; } return { took => $ret->{took}, - total => $ret->{hits}{total}, + total => hit_total($ret), releases => \@releases, }; } @@ -455,11 +448,10 @@ sub by_author { ); my $data = [ map { $_->{_source} } @{ $ret->{hits}{hits} } ]; - single_valued_arrayref_to_scalar($data); return { releases => $data, - total => $ret->{hits}{total}, + total => hit_total($ret), took => $ret->{took} }; } @@ -491,12 +483,11 @@ sub latest_by_distribution { ); my $data = $ret->{hits}{hits}[0]{_source}; - single_valued_arrayref_to_scalar($data); return { release => $data, took => $ret->{took}, - total => $ret->{hits}{total} + total => hit_total($ret), }; } @@ -514,7 +505,7 @@ sub latest_by_author { }, sort => [ 'distribution', { 'version_numified' => { reverse => 1 } } ], - fields => [ + _source => [ qw(author distribution name status abstract date download_url version authorized maturity) ], size => 1000, @@ -526,8 +517,7 @@ sub latest_by_author { body => $body, ); - my $data = [ map { $_->{fields} } @{ $ret->{hits}{hits} } ]; - single_valued_arrayref_to_scalar($data); + my $data = [ map { $_->{_source} } @{ $ret->{hits}{hits} } ]; return { took => $ret->{took}, releases => $data }; } @@ -538,9 +528,9 @@ sub all_by_author { $page //= 1; my $body = { - query => { term => { author => uc($author) } }, - sort => [ { date => 'desc' } ], - fields => [ + query => { term => { author => uc($author) } }, + sort => [ { date => 'desc' } ], + _source => [ qw(author distribution name status abstract date download_url version authorized maturity) ], size => $size, @@ -552,13 +542,12 @@ sub all_by_author { body => $body, ); - my $data = [ map { $_->{fields} } @{ $ret->{hits}{hits} } ]; - single_valued_arrayref_to_scalar($data); + my $data = [ map { $_->{_source} } @{ $ret->{hits}{hits} } ]; return { took => $ret->{took}, releases => $data, - total => $ret->{hits}{total} + total => hit_total($ret), }; } @@ -611,10 +600,10 @@ sub versions { } my $body = { - query => $query, - size => $size, - sort => [ { date => 'desc' } ], - fields => [ + query => $query, + size => $size, + sort => [ { date => 'desc' } ], + _source => [ qw( name date author version status maturity authorized download_url) ], }; @@ -625,12 +614,11 @@ sub versions { body => $body, ); - my $data = [ map { $_->{fields} } @{ $ret->{hits}{hits} } ]; - single_valued_arrayref_to_scalar($data); + my $data = [ map { $_->{_source} } @{ $ret->{hits}{hits} } ]; return { releases => $data, - total => $ret->{hits}{total}, + total => hit_total($ret), took => $ret->{took} }; } @@ -716,12 +704,11 @@ sub _get_latest_release { ] }, }, - fields => [qw< name author >], + _source => [qw< name author >], }, ); - my ($release_info) = map { $_->{fields} } @{ $release->{hits}{hits} }; - single_valued_arrayref_to_scalar($release_info); + my ($release_info) = map { $_->{_source} } @{ $release->{hits}{hits} }; return $release_info->{name} && $release_info->{author} ? +{ @@ -826,7 +813,7 @@ sub _get_depended_releases { return +{ data => [ map { $_->{_source} } @{ $depended->{hits}{hits} } ], - total => $depended->{hits}{total}, + total => hit_total($depended), took => $depended->{took}, }; } @@ -876,10 +863,10 @@ sub recent { } my $body = { - size => $page_size, - from => $from, - query => $query, - fields => + size => $page_size, + from => $from, + query => $query, + _source => [qw(name author status abstract date distribution maturity)], sort => [ { 'date' => { order => 'desc' } } ] }; @@ -890,12 +877,11 @@ sub recent { body => $body, ); - my $data = [ map { $_->{fields} } @{ $ret->{hits}{hits} } ]; - single_valued_arrayref_to_scalar($data); + my $data = [ map { $_->{_source} } @{ $ret->{hits}{hits} } ]; return { releases => $data, - total => $ret->{hits}{total}, + total => hit_total($ret), took => $ret->{took} }; } @@ -959,9 +945,9 @@ sub modules { # Sort by documentation name; if there isn't one, sort by path. sort => [ 'documentation', 'path' ], - _source => [ "module", "abstract" ], - - fields => [ qw( + _source => [ qw( + module + abstract author authorized distribution @@ -980,15 +966,11 @@ sub modules { body => $body, ); - my @files = map +{ - %{ ( single_valued_arrayref_to_scalar( $_->{fields} ) )[0] }, - %{ $_->{_source} } - }, - @{ $ret->{hits}{hits} }; + my @files = map $_->{_source}, @{ $ret->{hits}{hits} }; return { files => \@files, - total => $ret->{hits}{total}, + total => hit_total($ret), took => $ret->{took} }; } @@ -1044,7 +1026,9 @@ sub find_download_url { my $module_filter = $type eq 'module'; if ( !$explicit_version ) { - push @filters, { not => { term => { status => 'backpan' } } }; + push @filters, + { bool => { must_not => [ { term => { status => 'backpan' } } ] } + }; if ( !$dev ) { push @filters, { term => { maturity => 'released' } }; } @@ -1086,9 +1070,8 @@ sub find_download_url { push @filters, { nested => { - path => 'module', - inner_hits => { _source => 'version' }, - filter => $entity_filter, + path => 'module', + query => $entity_filter, } }; } @@ -1124,13 +1107,13 @@ sub find_download_url { my $query; if ($dev) { - $query = { filtered => { filter => $filter } }; + $query = $filter; } else { # if not dev, then prefer latest > cpan > backpan $query = { function_score => { - filter => $filter, + query => $filter, score_mode => 'first', boost_mode => 'replace', functions => [ @@ -1171,7 +1154,7 @@ sub find_download_url { search_type => 'dfs_query_then_fetch', ); - return unless $res->{hits}{total}; + return unless hit_total($res); my @checksums; diff --git a/lib/MetaCPAN/Query/Search.pm b/lib/MetaCPAN/Query/Search.pm index f9640a84b..55d7e8364 100644 --- a/lib/MetaCPAN/Query/Search.pm +++ b/lib/MetaCPAN/Query/Search.pm @@ -7,7 +7,7 @@ use Hash::Merge qw( merge ); use List::Util qw( min uniq ); use Log::Contextual qw( :log :dlog ); use MetaCPAN::Types::TypeTiny qw( Object Str ); -use MetaCPAN::Util qw( single_valued_arrayref_to_scalar true false ); +use MetaCPAN::Util qw( hit_total true false ); use MooseX::StrictConstructor; with 'MetaCPAN::Query::Role::Common'; @@ -31,8 +31,7 @@ sub search_for_first_result { my $es_results = $self->run_query( file => $es_query ); my $data = $es_results->{hits}{hits}[0]; - single_valued_arrayref_to_scalar( $data->{fields} ); - return $data->{fields}; + return $data->{_source}; } =head2 search_web @@ -114,7 +113,7 @@ sub _search_expanded { my $return = { results => $results, - total => $es_results->{hits}->{total}, + total => hit_total($es_results), took => $es_results->{took}, collapsed => false, }; @@ -129,13 +128,12 @@ sub _search_collapsed { my $total_size = $from + $page_size; my $es_query_opts = { - size => 0, - fields => [ qw( + size => 0, + _source => [ qw( ) ], }; my $es_query = $self->build_query( $search_term, $es_query_opts ); - my $fields = delete $es_query->{fields}; my $source = delete $es_query->{_source}; $es_query->{aggregations} = { @@ -150,15 +148,16 @@ sub _search_collapsed { aggregations => { top_files => { top_hits => { - fields => $fields, _source => $source, size => $max_collapsed_hits, }, }, max_score => { max => { - lang => "expression", - script => "_score", + script => { + lang => "expression", + inline => "_score", + }, }, }, }, @@ -328,9 +327,9 @@ sub build_query { $params, { query => $query, - _source => [ "module", ], - fields => [ qw( - abstract.analyzed + _source => [ qw( + module + abstract author authorized date @@ -350,7 +349,7 @@ sub build_query { # Ensure our requested fields are unique so that Elasticsearch doesn't # return us the same value multiple times in an unexpected arrayref. - $search->{fields} = [ uniq @{ $search->{fields} || [] } ]; + $search->{_source} = [ uniq @{ $search->{_source} || [] } ]; return $search; } @@ -371,11 +370,8 @@ sub _extract_results { return [ map { my $res = $_; - single_valued_arrayref_to_scalar( $res->{fields} ); +{ - abstract => delete $res->{fields}->{'abstract.analyzed'}, - favorites => delete $res->{fields}->{dist_fav_count}, - %{ $res->{fields} }, + favorites => delete $res->{_source}->{dist_fav_count}, %{ $res->{_source} }, score => $res->{_score}, } diff --git a/lib/MetaCPAN/Role/Script.pm b/lib/MetaCPAN/Role/Script.pm index b27f82514..f5cab3bfa 100644 --- a/lib/MetaCPAN/Role/Script.pm +++ b/lib/MetaCPAN/Role/Script.pm @@ -2,16 +2,16 @@ package MetaCPAN::Role::Script; use Moose::Role; -use Carp (); -use ElasticSearchX::Model::Document::Types qw( ES ); -use File::Path (); -use IO::Prompt::Tiny qw( prompt ); -use Log::Contextual qw( :log :dlog ); -use MetaCPAN::Model (); -use MetaCPAN::Types::TypeTiny qw( Bool HashRef Int Path Str ); -use MetaCPAN::Util qw( root_dir ); -use Mojo::Server (); -use Term::ANSIColor qw( colored ); +use Carp (); +use MooseX::Types::ElasticSearch qw( ES ); +use File::Path (); +use IO::Prompt::Tiny qw( prompt ); +use Log::Contextual qw( :log :dlog ); +use MetaCPAN::Model (); +use MetaCPAN::Types::TypeTiny qw( Bool HashRef Int Path Str ); +use MetaCPAN::Util qw( root_dir ); +use Mojo::Server (); +use Term::ANSIColor qw( colored ); with( 'MetaCPAN::Role::HasConfig', 'MetaCPAN::Role::Fastly', 'MetaCPAN::Role::Logger' ); diff --git a/lib/MetaCPAN/Script/Author.pm b/lib/MetaCPAN/Script/Author.pm index f2708bd7e..7abf621fa 100644 --- a/lib/MetaCPAN/Script/Author.pm +++ b/lib/MetaCPAN/Script/Author.pm @@ -282,7 +282,7 @@ sub update_author { $bulk->update( { id => $pauseid, doc => $data, - doc_as_upsert => 1, + doc_as_upsert => true, } ); $self->purge_author_key($pauseid); diff --git a/lib/MetaCPAN/Script/Backpan.pm b/lib/MetaCPAN/Script/Backpan.pm index d7fd0fd4a..44b30b45a 100644 --- a/lib/MetaCPAN/Script/Backpan.pm +++ b/lib/MetaCPAN/Script/Backpan.pm @@ -62,18 +62,20 @@ sub build_release_status_map { log_info {"find_releases"}; my $scroll = $self->es->scroll_helper( - size => 500, scroll => '5m', index => $self->index->name, type => 'release', - fields => [ 'author', 'archive', 'name' ], - body => $self->_get_release_query, + body => { + %{ $self->_get_release_query }, + size => 500, + _source => [ 'author', 'archive', 'name' ], + }, ); while ( my $release = $scroll->next ) { - my $author = $release->{fields}{author}[0]; - my $archive = $release->{fields}{archive}[0]; - my $name = $release->{fields}{name}[0]; + my $author = $release->{_source}{author}; + my $archive = $release->{_source}{archive}; + my $name = $release->{_source}{name}; next unless $name; # bypass some broken releases $self->_release_status->{$author}{$name} = [ @@ -94,7 +96,8 @@ sub _get_release_query { unless ( $self->undo ) { return +{ query => { - not => { term => { status => 'backpan' } } + bool => + { must_not => [ { term => { status => 'backpan' } } ] } } }; } @@ -162,11 +165,9 @@ sub update_files_author { log_info { "update_files: " . $author }; my $scroll = $self->es->scroll_helper( - size => 500, scroll => '5m', index => $self->index->name, type => 'file', - fields => ['release'], body => { query => { bool => { @@ -175,7 +176,9 @@ sub update_files_author { { terms => { release => $author_releases } } ] } - } + }, + size => 500, + _source => ['release'], }, ); @@ -188,7 +191,7 @@ sub update_files_author { my $bulk = $self->_bulk->{file}; while ( my $file = $scroll->next ) { - my $release = $file->{fields}{release}[0]; + my $release = $file->{_source}{release}; $bulk->update( { id => $file->{_id}, doc => { diff --git a/lib/MetaCPAN/Script/Backup.pm b/lib/MetaCPAN/Script/Backup.pm index 700150f8a..32d6fd54b 100644 --- a/lib/MetaCPAN/Script/Backup.pm +++ b/lib/MetaCPAN/Script/Backup.pm @@ -9,6 +9,7 @@ use DateTime (); use IO::Zlib (); use Log::Contextual qw( :log :dlog ); use MetaCPAN::Types::TypeTiny qw( Bool Int Path Str CommaSepOption ); +use MetaCPAN::Util qw( true false ); use Moose; use Try::Tiny qw( catch try ); @@ -87,11 +88,11 @@ sub run { my $scroll = $es->scroll_helper( index => $index, $self->type ? ( type => $self->type ) : (), - size => $self->size, - fields => [qw(_parent _source)], scroll => '1m', body => { - sort => '_doc', + _source => true, + size => $self->size, + sort => '_doc', }, ); @@ -143,7 +144,7 @@ sub run_restore { # Fetch relevant bulk helper my $bulk = $bulk_store{$bulk_key}; - my $parent = $raw->{fields}->{_parent}; + my $parent = $raw->{_parent}; if ( $raw->{_type} eq 'author' ) { @@ -176,7 +177,7 @@ sub run_restore { $bulk->update( { id => $raw->{_id}, doc => $raw->{_source}, - doc_as_upsert => 1, + doc_as_upsert => true, } ); } diff --git a/lib/MetaCPAN/Script/CPANTesters.pm b/lib/MetaCPAN/Script/CPANTesters.pm index bf2fa3ad4..3a6547290 100644 --- a/lib/MetaCPAN/Script/CPANTesters.pm +++ b/lib/MetaCPAN/Script/CPANTesters.pm @@ -8,6 +8,7 @@ use File::stat qw( stat ); use IO::Uncompress::Bunzip2 qw( bunzip2 ); use Log::Contextual qw( :log :dlog ); use MetaCPAN::Types::TypeTiny qw( Bool Path Uri ); +use MetaCPAN::Util qw( true false ); with 'MetaCPAN::Role::Script', 'MooseX::Getopt::Dashes'; @@ -157,7 +158,7 @@ sub index_reports { my %tests = map { $_ => $row_from_db->{$_} } qw(fail pass na unknown); $self->_bulk->update( { doc => { tests => \%tests }, - doc_as_upsert => 1, + doc_as_upsert => true, id => $release_doc->{id}, } ); } diff --git a/lib/MetaCPAN/Script/CPANTestersAPI.pm b/lib/MetaCPAN/Script/CPANTestersAPI.pm index b690c621a..959c909a7 100644 --- a/lib/MetaCPAN/Script/CPANTestersAPI.pm +++ b/lib/MetaCPAN/Script/CPANTestersAPI.pm @@ -7,6 +7,7 @@ use Cpanel::JSON::XS qw( decode_json ); use ElasticSearchX::Model::Document::Types qw( ESBulk ); use Log::Contextual qw( :log :dlog ); use MetaCPAN::Types::TypeTiny qw( Uri ); +use MetaCPAN::Util qw( true false ); use Moose; with 'MetaCPAN::Role::Script', 'MooseX::Getopt::Dashes'; @@ -127,7 +128,7 @@ sub index_reports { my %tests = map { $_ => $row->{$_} } qw(fail pass na unknown); $self->_bulk->update( { doc => { tests => \%tests }, - doc_as_upsert => 1, + doc_as_upsert => true, id => $release_doc->{id}, } ); } diff --git a/lib/MetaCPAN/Script/CVE.pm b/lib/MetaCPAN/Script/CVE.pm index 1ea2246c5..3980f6461 100644 --- a/lib/MetaCPAN/Script/CVE.pm +++ b/lib/MetaCPAN/Script/CVE.pm @@ -6,7 +6,7 @@ use namespace::autoclean; use Cpanel::JSON::XS qw( decode_json ); use Log::Contextual qw( :log :dlog ); use MetaCPAN::Types::TypeTiny qw( Bool Str Uri ); -use MetaCPAN::Util qw( numify_version ); +use MetaCPAN::Util qw( hit_total numify_version true false ); use Path::Tiny qw( path ); use Ref::Util qw( is_arrayref ); @@ -158,33 +158,34 @@ sub index_cve_data { } if (@filters) { - my $query = { - query => { - bool => { - must => [ - { term => { distribution => $dist } }, - @filters, - ] - } - }, - }; + my $query = {}; my $releases = $self->es->search( - index => 'cpan', - type => 'release', - body => $query, - fields => [ "version", "name", "author", ], - size => 2000, + index => 'cpan', + type => 'release', + body => { + query => { + bool => { + must => [ + { term => { distribution => $dist } }, + @filters, + ] + } + }, + _source => [ "version", "name", "author", ], + size => 2000, + }, ); - if ( $releases->{hits}{total} ) { + if ( hit_total($releases) ) { ## no critic (ControlStructures::ProhibitMutatingListFunctions) @matches = map { $_->[0] } sort { $a->[1] <=> $b->[1] } map { - my %fields = %{ $_->{fields} }; - ref $_ and $_ = $_->[0] for values %fields; - [ \%fields, numify_version( $fields{version} ) ]; + [ + $_->{_source}, + numify_version( $_->{_source}{version} ) + ]; } @{ $releases->{hits}{hits} }; } else { @@ -215,7 +216,7 @@ sub index_cve_data { $bulk->update( { id => $cpansa->{cpansa_id}, doc => $doc_data, - doc_as_upsert => 1, + doc_as_upsert => true, } ); } } diff --git a/lib/MetaCPAN/Script/Check.pm b/lib/MetaCPAN/Script/Check.pm index c1a660c01..712896ac5 100644 --- a/lib/MetaCPAN/Script/Check.pm +++ b/lib/MetaCPAN/Script/Check.pm @@ -87,10 +87,6 @@ sub check_modules { my $results = $es->search( index => $self->index->name, type => 'file', - size => 100, # shouldn't get more than this - fields => [ - qw(name release author distribution version authorized indexed maturity date) - ], query => { bool => { must => [ @@ -99,6 +95,18 @@ sub check_modules { { term => { 'maturity' => 'released' } }, ], }, + size => 100, # shouldn't get more than this + _source => [ qw( + name + release + author + distribution + version + authorized + indexed + maturity + date + ) ], }, ); my @files = @{ $results->{hits}->{hits} }; @@ -106,22 +114,23 @@ sub check_modules { # now find the first latest releases for these files foreach my $file (@files) { my $release_results = $es->search( - index => $self->index->name, - type => 'release', - size => 1, - fields => - [qw(name status authorized version id date)], + index => $self->index->name, + type => 'release', query => { bool => { must => [ { term => { - name => $file->{fields}->{release} + name => + $file->{_source}->{release} } }, { term => { status => 'latest' } }, ], }, + size => 1, + _source => + [qw(name status authorized version id date)], }, ); @@ -136,22 +145,23 @@ sub check_modules { if ( !@releases ) { foreach my $file (@files) { my $release_results = $es->search( - index => $self->index->name, - type => 'release', - size => 1, - fields => - [qw(name status authorized version id date)], + index => $self->index->name, + type => 'release', query => { bool => { must => [ { term => { - name => - $file->{fields}->{release} + name => $file->{_source} + ->{release} } }, ], }, + size => 1, + _source => [ + qw(name status authorized version id date) + ], }, ); @@ -163,10 +173,10 @@ sub check_modules { # if we found the releases tell them about it if (@releases) { if ( @releases == 1 - && $releases[0]->{fields}->{status} eq 'latest' ) + && $releases[0]->{_source}->{status} eq 'latest' ) { log_info { - "Found latest release $releases[0]->{fields}->{name} for $pkg"; + "Found latest release $releases[0]->{_source}->{name} for $pkg"; } unless $self->errors_only; } @@ -174,16 +184,16 @@ sub check_modules { log_error {"Could not find latest release for $pkg"}; foreach my $rel (@releases) { log_warn { - " Found release $rel->{fields}->{name}"; + " Found release $rel->{_source}->{name}"; }; log_warn { - " STATUS : $rel->{fields}->{status}"; + " STATUS : $rel->{_source}->{status}"; }; log_warn { - " AUTORIZED : $rel->{fields}->{authorized}"; + " AUTORIZED : $rel->{_source}->{authorized}"; }; log_warn { - " DATE : $rel->{fields}->{date}"; + " DATE : $rel->{_source}->{date}"; }; } $self->_set_error_count( $self->error_count + 1 ); @@ -194,17 +204,17 @@ sub check_modules { "Module $pkg doesn't have any releases in ElasticSearch!"; }; foreach my $file (@files) { - log_warn {" Found file $file->{fields}->{name}"}; + log_warn {" Found file $file->{_source}->{name}"}; log_warn { - " RELEASE : $file->{fields}->{release}"; + " RELEASE : $file->{_source}->{release}"; }; log_warn { - " AUTHOR : $file->{fields}->{author}"; + " AUTHOR : $file->{_source}->{author}"; }; log_warn { - " AUTHORIZED : $file->{fields}->{authorized}"; + " AUTHORIZED : $file->{_source}->{authorized}"; }; - log_warn {" DATE : $file->{fields}->{date}"}; + log_warn {" DATE : $file->{_source}->{date}"}; } $self->_set_error_count( $self->error_count + 1 ); } diff --git a/lib/MetaCPAN/Script/Checksum.pm b/lib/MetaCPAN/Script/Checksum.pm index 8766a1258..52a22ccf9 100644 --- a/lib/MetaCPAN/Script/Checksum.pm +++ b/lib/MetaCPAN/Script/Checksum.pm @@ -4,6 +4,7 @@ use Moose; use Log::Contextual qw( :log ); use MetaCPAN::Types::TypeTiny qw( Bool Int ); +use MetaCPAN::Util qw( true false ); use Digest::file qw( digest_file_hex ); @@ -49,14 +50,18 @@ sub run { scroll => '10m', body => { query => { - not => { - exists => { - field => "checksum_md5" - } - } - } + bool => { + must_not => [ + { + exists => { + field => "checksum_md5" + } + }, + ], + }, + }, + _source => [qw( name download_url )], }, - fields => [qw( id name download_url )], ); log_warn { "Found " . $scroll->total . " releases" }; @@ -70,12 +75,11 @@ sub run { last; } - log_info { "Adding checksums for " . $p->{fields}{name}->[0] }; + log_info { "Adding checksums for " . $p->{_source}{name} }; - if ( my $download_url = $p->{fields}{download_url} ) { + if ( my $download_url = $p->{_source}{download_url} ) { my $file - = $self->cpan . "/authors" . $p->{fields}{download_url}->[0] - =~ s/^.*authors//r; + = $self->cpan . "/authors" . $download_url =~ s/^.*authors//r; my $checksum_md5 = digest_file_hex( $file, 'MD5' ); my $checksum_sha256 = digest_file_hex( $file, 'SHA-256' ); @@ -90,13 +94,13 @@ sub run { checksum_md5 => $checksum_md5, checksum_sha256 => $checksum_sha256 }, - doc_as_upsert => 1, + doc_as_upsert => true, } ); } } else { log_info { - $p->{fields}{name}->[0] . " is missing a download_url" + $p->{_source}{name} . " is missing a download_url" }; } } diff --git a/lib/MetaCPAN/Script/Contributor.pm b/lib/MetaCPAN/Script/Contributor.pm index 09404f9f8..2239e3fb3 100644 --- a/lib/MetaCPAN/Script/Contributor.pm +++ b/lib/MetaCPAN/Script/Contributor.pm @@ -82,24 +82,26 @@ sub run { my $timeout = $self->all ? '720m' : '5m'; my $scroll = $self->es->scroll_helper( - size => 500, scroll => $timeout, index => $self->index->name, type => 'release', - body => { query => $query }, - fields => [qw( author distribution name )], + body => { + query => $query, + size => 500, + _source => [qw( author distribution name )], + }, ); my @data; while ( my $r = $scroll->next ) { my $contrib_data = $self->get_cpan_author_contributors( - $r->{fields}{author}[0], - $r->{fields}{name}[0], - $r->{fields}{distribution}[0], + $r->{_source}{author}, + $r->{_source}{name}, + $r->{_source}{distribution}, ); next unless is_arrayref($contrib_data); - log_debug { 'adding release ' . $r->{fields}{name}[0] }; + log_debug { 'adding release ' . $r->{_source}{name} }; push @data => @{$contrib_data}; } diff --git a/lib/MetaCPAN/Script/Cover.pm b/lib/MetaCPAN/Script/Cover.pm index ee067118a..46dd3dab5 100644 --- a/lib/MetaCPAN/Script/Cover.pm +++ b/lib/MetaCPAN/Script/Cover.pm @@ -7,6 +7,7 @@ use Cpanel::JSON::XS qw( decode_json ); use Log::Contextual qw( :log :dlog ); use MetaCPAN::Types::TypeTiny qw( Bool Str Uri ); use Path::Tiny qw( path ); +use MetaCPAN::Util qw( hit_total true false ); with 'MetaCPAN::Role::Script', 'MooseX::Getopt'; @@ -70,7 +71,7 @@ sub index_cover_data { query => { term => { name => $release } }, }, ); - if ( $rel_check->{hits}{total} ) { + if ( hit_total($rel_check) ) { log_info { "Adding release info for '" . $release . "'" }; } else { @@ -92,7 +93,7 @@ sub index_cover_data { release => $release, criteria => \%doc_data, }, - doc_as_upsert => 1, + doc_as_upsert => true, } ); } } diff --git a/lib/MetaCPAN/Script/External.pm b/lib/MetaCPAN/Script/External.pm index de73be63e..c74de55c1 100644 --- a/lib/MetaCPAN/Script/External.pm +++ b/lib/MetaCPAN/Script/External.pm @@ -8,6 +8,7 @@ use Email::Simple (); use Log::Contextual qw( :log ); use MetaCPAN::Types::TypeTiny qw( Str ); +use MetaCPAN::Util qw( true false ); with( 'MetaCPAN::Role::Script', @@ -103,7 +104,7 @@ sub update { $external_source => $dist->{$d} } }, - doc_as_upsert => 1, + doc_as_upsert => true, } ); } diff --git a/lib/MetaCPAN/Script/Favorite.pm b/lib/MetaCPAN/Script/Favorite.pm index 13457a6a7..d59f2401f 100644 --- a/lib/MetaCPAN/Script/Favorite.pm +++ b/lib/MetaCPAN/Script/Favorite.pm @@ -5,6 +5,7 @@ use Moose; use Log::Contextual qw( :log ); use MetaCPAN::Types::TypeTiny qw( Bool Int Str ); +use MetaCPAN::Util qw( true false ); with 'MooseX::Getopt', 'MetaCPAN::Role::Script'; @@ -94,19 +95,18 @@ sub index_favorites { index => $self->index->name, type => 'favorite', scroll => '5m', - fields => [qw< distribution >], - size => 500, body => { - query => $age_filter, - ( $self->limit ? ( size => $self->limit ) : () ), - sort => '_doc', + query => $age_filter, + _source => [qw< distribution >], + size => $self->limit || 500, + sort => '_doc', } ); my %recent_dists; while ( my $fav = $favs->next ) { - my $dist = $fav->{fields}{distribution}[0]; + my $dist = $fav->{_source}{distribution}; $recent_dists{$dist}++ if $dist; } @@ -128,16 +128,16 @@ sub index_favorites { index => $self->index->name, type => 'favorite', scroll => '30s', - fields => [qw< distribution >], - size => 500, body => { - query => $query, - sort => '_doc', + query => $query, + _source => [qw< distribution >], + size => 500, + sort => '_doc', }, ); while ( my $fav = $favs->next ) { - my $dist = $fav->{fields}{distribution}[0]; + my $dist = $fav->{_source}{distribution}; $dist_fav_count{$dist}++ if $dist; } @@ -157,8 +157,6 @@ sub index_favorites { index => $self->index->name, type => 'file', scroll => '15m', - fields => [qw< id distribution >], - size => 500, body => { query => { bool => { @@ -168,12 +166,14 @@ sub index_favorites { @age_filter, } }, - sort => '_doc', + _source => [qw< distribution >], + size => 500, + sort => '_doc', }, ); while ( my $file = $files->next ) { - my $dist = $file->{fields}{distribution}[0]; + my $dist = $file->{_source}{distribution}; next unless $dist; next if exists $missing{$dist} or exists $dist_fav_count{$dist}; @@ -233,22 +233,22 @@ sub index_favorites { index => $self->index->name, type => 'file', scroll => '15s', - fields => [qw< id >], - size => 500, body => { - query => { term => { distribution => $dist } }, - sort => '_doc', + query => { term => { distribution => $dist } }, + _source => false, + size => 500, + sort => '_doc', }, ); while ( my $file = $files->next ) { - my $id = $file->{fields}{id}[0]; + my $id = $file->{_id}; my $cnt = $dist_fav_count{$dist}; log_debug {"Updating file id $id with fav_count $cnt"}; $bulk->update( { - id => $file->{fields}{id}[0], + id => $file->{_id}, doc => { dist_fav_count => $cnt }, } ); } diff --git a/lib/MetaCPAN/Script/Latest.pm b/lib/MetaCPAN/Script/Latest.pm index 42f6e9e34..345785d0e 100644 --- a/lib/MetaCPAN/Script/Latest.pm +++ b/lib/MetaCPAN/Script/Latest.pm @@ -123,8 +123,8 @@ sub run { must => [ { nested => { - path => 'module', - filter => { bool => { must => $filter } } + path => 'module', + query => { bool => { must => $filter } } } }, { term => { 'maturity' => 'released' } }, diff --git a/lib/MetaCPAN/Script/Mapping.pm b/lib/MetaCPAN/Script/Mapping.pm index a838a4a39..f1da40695 100644 --- a/lib/MetaCPAN/Script/Mapping.pm +++ b/lib/MetaCPAN/Script/Mapping.pm @@ -688,7 +688,24 @@ sub _compare_mapping { } } else { # Scalar Value - if ( $deploy_value ne $model_value ) { + if ( + $sfield eq 'type' + && $model_value eq 'string' + && ( $deploy_value eq 'text' + || $deploy_value eq 'keyword' ) + ) + { + # ES5 automatically converts string types to text + # or keyword. once we upgrade to ES5 and update + # our mappings, this special case can be removed. + } + elsif ($sfield eq 'index' + && $model_value eq 'no' + && $deploy_value eq 'false' ) + { + # another ES5 string automatic conversion + } + elsif ( $deploy_value ne $model_value ) { log_error { 'Mismatch field: ' . $sname . '.' @@ -708,11 +725,24 @@ sub _compare_mapping { $imatch = 0; } + unless ( defined $rmodel->{$sfield} ) { - log_error { - 'Missing definition: ' . $sname . '.' . $sfield - }; - $imatch = 0; + if ( $sfield eq 'payloads' + && $rmodel->{type} + && $rmodel->{type} eq 'completion' + && !$rdeploy->{$sfield} ) + { + # ES5 doesn't allow payloads option. we've removed + # it from our mapping. but it gets a default + # value. ignore the default. + } + else { + log_error { + 'Missing definition: ' . $sname . '.' + . $sfield + }; + $imatch = 0; + } } } } diff --git a/lib/MetaCPAN/Script/Mapping/CPAN/File.pm b/lib/MetaCPAN/Script/Mapping/CPAN/File.pm index fbd99d9ba..7c38501bc 100644 --- a/lib/MetaCPAN/Script/Mapping/CPAN/File.pm +++ b/lib/MetaCPAN/Script/Mapping/CPAN/File.pm @@ -206,15 +206,15 @@ sub mapping { }, "pod" : { "type" : "string", - "index" : "no", + "index" : "analyzed", + "analyzer" : "standard", + "doc_values" : false, "fields" : { "analyzed" : { "type" : "string", - "term_vector" : "with_positions_offsets", - "fielddata" : { - "format" : "disabled" - }, - "analyzer" : "standard" + "index" : "analyzed", + "analyzer" : "standard", + "doc_values" : false } } }, @@ -283,7 +283,6 @@ sub mapping { "suggest" : { "type" : "completion", "analyzer" : "simple", - "payloads" : true, "preserve_separators" : true, "preserve_position_increments" : true, "max_input_length" : 50 diff --git a/lib/MetaCPAN/Script/Package.pm b/lib/MetaCPAN/Script/Package.pm index afed7c5e6..708f986db 100644 --- a/lib/MetaCPAN/Script/Package.pm +++ b/lib/MetaCPAN/Script/Package.pm @@ -6,6 +6,7 @@ use CPAN::DistnameInfo (); use IO::Uncompress::Gunzip (); use Log::Contextual qw( :log ); use MetaCPAN::Types::TypeTiny qw( Bool ); +use MetaCPAN::Util qw( true false ); with 'MooseX::Getopt', 'MetaCPAN::Role::Script'; @@ -79,7 +80,7 @@ sub index_packages { $bulk->update( { id => $name, doc => $doc, - doc_as_upsert => 1, + doc_as_upsert => true, } ); $seen{$name} = 1; diff --git a/lib/MetaCPAN/Script/Permission.pm b/lib/MetaCPAN/Script/Permission.pm index 90d1e139d..b2cc0a925 100644 --- a/lib/MetaCPAN/Script/Permission.pm +++ b/lib/MetaCPAN/Script/Permission.pm @@ -4,6 +4,7 @@ use Moose; use Log::Contextual qw( :log ); use MetaCPAN::Types::TypeTiny qw( Bool ); +use MetaCPAN::Util qw( true false ); use PAUSE::Permissions (); with 'MooseX::Getopt', 'MetaCPAN::Role::Script'; @@ -64,7 +65,7 @@ sub index_permissions { $bulk->update( { id => $name, doc => $doc, - doc_as_upsert => 1, + doc_as_upsert => true, } ); $seen{$name} = 1; diff --git a/lib/MetaCPAN/Script/Purge.pm b/lib/MetaCPAN/Script/Purge.pm index e87b86f7e..afa80d5b7 100644 --- a/lib/MetaCPAN/Script/Purge.pm +++ b/lib/MetaCPAN/Script/Purge.pm @@ -4,7 +4,7 @@ use Moose; use Log::Contextual qw( :log ); use MetaCPAN::Types::TypeTiny qw( Bool HashRef Str ); -use MetaCPAN::Util qw( author_dir ); +use MetaCPAN::Util qw( author_dir true false ); with 'MooseX::Getopt', 'MetaCPAN::Role::Script'; @@ -55,47 +55,56 @@ sub _build_bulk { sub _get_scroller_release { my ( $self, $query ) = @_; return $self->es->scroll_helper( - size => 500, scroll => '10m', index => $self->index->name, type => 'release', - body => { query => $query }, - fields => [qw( archive name )], + body => { + query => $query, + size => 500, + _source => [qw( archive name )], + }, ); } sub _get_scroller_file { my ( $self, $query ) = @_; return $self->es->scroll_helper( - size => 500, scroll => '10m', index => $self->index->name, type => 'file', - body => { query => $query }, - fields => [qw( name )], + body => { + query => $query, + size => 500, + _source => [qw( name )], + }, ); } sub _get_scroller_favorite { my ( $self, $query ) = @_; return $self->es->scroll_helper( - size => 500, scroll => '10m', index => $self->index->name, type => 'favorite', - body => { query => $query }, + body => { + query => $query, + size => 500, + _source => false, + }, ); } sub _get_scroller_contributor { my ( $self, $query ) = @_; return $self->es->scroll_helper( - size => 500, scroll => '10m', index => 'contributor', type => 'contributor', - body => { query => $query }, - fields => [qw( release_name )], + body => { + query => $query, + size => 500, + _source => [qw( release_name )], + }, ); } @@ -169,10 +178,10 @@ sub _purge_release { my @remove_release_archives; while ( my $r = $scroll->next ) { - log_debug { 'Removing release ' . $r->{fields}{name}[0] }; + log_debug { 'Removing release ' . $r->{_source}{name} }; push @remove_ids, $r->{_id}; - push @remove_release_files, $r->{fields}{name}[0]; - push @remove_release_archives, $r->{fields}{archive}[0]; + push @remove_release_files, $r->{_source}{name}; + push @remove_release_archives, $r->{_source}{archive}; } if (@remove_ids) { @@ -215,10 +224,7 @@ sub _purge_files { while ( my $f = $scroll->next ) { log_debug { - 'Removing file ' - . $f->{fields}{name}[0] - . ' of release ' - . $release + 'Removing file ' . $f->{_source}{name} . ' of release ' . $release }; push @remove, $f->{_id}; } diff --git a/lib/MetaCPAN/Script/River.pm b/lib/MetaCPAN/Script/River.pm index 975154368..d12c8bd88 100644 --- a/lib/MetaCPAN/Script/River.pm +++ b/lib/MetaCPAN/Script/River.pm @@ -6,6 +6,7 @@ use namespace::autoclean; use Cpanel::JSON::XS qw( decode_json ); use Log::Contextual qw( :log :dlog ); use MetaCPAN::Types::TypeTiny qw( Uri ); +use MetaCPAN::Util qw( true false ); with 'MetaCPAN::Role::Script', 'MooseX::Getopt'; @@ -42,7 +43,7 @@ sub index_river_summaries { name => $dist, river => $summary, }, - doc_as_upsert => 1, + doc_as_upsert => true, } ); } $bulk->flush; diff --git a/lib/MetaCPAN/Script/Role/Contributor.pm b/lib/MetaCPAN/Script/Role/Contributor.pm index aacab70c1..3802eff9e 100644 --- a/lib/MetaCPAN/Script/Role/Contributor.pm +++ b/lib/MetaCPAN/Script/Role/Contributor.pm @@ -2,7 +2,7 @@ package MetaCPAN::Script::Role::Contributor; use Moose::Role; -use MetaCPAN::Util qw( digest ); +use MetaCPAN::Util qw( digest true false ); use Ref::Util qw( is_arrayref ); sub get_cpan_author_contributors { @@ -58,7 +58,7 @@ sub update_release_contirbutors { release_author => $d->{release_author}, distribution => $d->{distribution}, }, - doc_as_upsert => 1, + doc_as_upsert => true, } ); } diff --git a/lib/MetaCPAN/Script/Suggest.pm b/lib/MetaCPAN/Script/Suggest.pm index 8a80ffb3b..2edb77eef 100644 --- a/lib/MetaCPAN/Script/Suggest.pm +++ b/lib/MetaCPAN/Script/Suggest.pm @@ -66,8 +66,6 @@ sub _update_slice { index => $self->index->name, type => 'file', scroll => '5m', - fields => [qw< id documentation >], - size => 500, body => { query => { bool => { @@ -76,7 +74,9 @@ sub _update_slice { ], }, }, - sort => '_doc', + _source => [qw< documentation >], + size => 500, + sort => '_doc', }, ); @@ -88,17 +88,16 @@ sub _update_slice { ); while ( my $file = $files->next ) { - my $documentation = $file->{fields}{documentation}[0]; + my $documentation = $file->{_source}{documentation}; my $weight = 1000 - length($documentation); $weight = 0 if $weight < 0; $bulk->update( { - id => $file->{fields}{id}[0], + id => $file->{_id}, doc => { suggest => { - input => [$documentation], - payload => { doc_name => $documentation }, - weight => $weight, + input => [$documentation], + weight => $weight, } }, } ); diff --git a/lib/MetaCPAN/Script/Tickets.pm b/lib/MetaCPAN/Script/Tickets.pm index 776a0905d..7318cd07b 100644 --- a/lib/MetaCPAN/Script/Tickets.pm +++ b/lib/MetaCPAN/Script/Tickets.pm @@ -8,6 +8,7 @@ $ENV{PERL_LWP_SSL_VERIFY_HOSTNAME} = 0; use HTTP::Request::Common qw( GET ); use Log::Contextual qw( :log :dlog ); +use MetaCPAN::Util qw( true false ); use Net::GitHub::V4 (); use Ref::Util qw( is_hashref is_ref ); use Text::CSV_XS (); @@ -76,22 +77,23 @@ sub check_all_distributions { # first: make sure all distributions have an entry my $scroll = $self->es->scroll_helper( - size => 500, scroll => '5m', index => $self->index->name, type => 'release', - fields => ['distribution'], body => { query => { - not => { term => { status => 'backpan' } } - } + bool => + { must_not => [ { term => { status => 'backpan' } } ] } + }, + size => 500, + _source => ['distribution'], }, ); my $dists = {}; while ( my $release = $scroll->next ) { - my $distribution = $release->{'fields'}{'distribution'}[0]; + my $distribution = $release->{_source}{'distribution'}; $distribution or next; $dists->{$distribution} = { name => $distribution }; } @@ -275,7 +277,7 @@ sub _bulk_update { $self->_bulk->update( { id => $distribution, doc => $summary->{$distribution}, - doc_as_upsert => 1, + doc_as_upsert => true, } ); } diff --git a/lib/MetaCPAN/Script/Watcher.pm b/lib/MetaCPAN/Script/Watcher.pm index bd943a506..2caf4aa5c 100644 --- a/lib/MetaCPAN/Script/Watcher.pm +++ b/lib/MetaCPAN/Script/Watcher.pm @@ -8,7 +8,7 @@ use CPAN::DistnameInfo (); use Cpanel::JSON::XS qw( decode_json ); use Log::Contextual qw( :log ); use MetaCPAN::Types::TypeTiny qw( Bool ); -use MetaCPAN::Util; +use MetaCPAN::Util qw( true false ); with 'MetaCPAN::Role::Script', 'MooseX::Getopt'; @@ -95,23 +95,23 @@ sub changes { sub backpan_changes { my $self = shift; my $scroll = $self->es->scroll_helper( { - size => 1000, scroll => '1m', index => $self->index->name, type => 'release', - fields => [qw(author archive)], body => { query => { bool => { must_not => [ { term => { status => 'backpan' } }, ], }, }, - sort => '_doc', + size => 1000, + _source => [qw(author archive)], + sort => '_doc', } } ); my @changes; while ( my $release = $scroll->next ) { - my $data = $release->{fields}; + my $data = $release->{_source}; my $path = $self->cpan->child( 'authors', MetaCPAN::Util::author_dir( $data->{author} ), @@ -173,8 +173,6 @@ sub reindex_release { index => $self->index->name, type => 'file', scroll => '1m', - size => 1000, - fields => [ '_parent', '_source' ], body => { query => { bool => { @@ -192,7 +190,9 @@ sub reindex_release { ], }, }, - sort => '_doc', + size => 1000, + _source => true, + sort => '_doc', }, } ); return if ( $self->dry_run ); @@ -210,11 +210,7 @@ sub reindex_release { $bulk_helper{file}->index( { id => $row->{_id}, source => { - $row->{fields}->{_parent} - ? ( parent => $row->{fields}->{_parent} ) - : (), - %$source, - status => 'backpan', + %$source, status => 'backpan', } } ); } diff --git a/lib/MetaCPAN/Server/Controller/Author.pm b/lib/MetaCPAN/Server/Controller/Author.pm index 1129c9ac6..581176d93 100644 --- a/lib/MetaCPAN/Server/Controller/Author.pm +++ b/lib/MetaCPAN/Server/Controller/Author.pm @@ -4,7 +4,6 @@ use strict; use warnings; use Moose; -use MetaCPAN::Util qw( single_valued_arrayref_to_scalar ); BEGIN { extends 'MetaCPAN::Server::Controller' } diff --git a/lib/MetaCPAN/Server/Controller/Login.pm b/lib/MetaCPAN/Server/Controller/Login.pm index 24555eeed..9d8770d77 100644 --- a/lib/MetaCPAN/Server/Controller/Login.pm +++ b/lib/MetaCPAN/Server/Controller/Login.pm @@ -5,6 +5,7 @@ use warnings; use Cpanel::JSON::XS qw( decode_json encode_json ); use Moose; +use MetaCPAN::Util qw( true false ); BEGIN { extends 'Catalyst::Controller' } @@ -44,7 +45,7 @@ sub update_user { $user ||= $model->new_document; $user->add_identity( { name => $type, key => $id, extra => $data } ); $user->clear_looks_human; # rebuild - $user->put( { refresh => 1 } ); + $user->put( { refresh => true } ); } $c->authenticate( { user => $user } ); diff --git a/lib/MetaCPAN/Server/Controller/OAuth2.pm b/lib/MetaCPAN/Server/Controller/OAuth2.pm index 51c9ff2bb..1ae22701b 100644 --- a/lib/MetaCPAN/Server/Controller/OAuth2.pm +++ b/lib/MetaCPAN/Server/Controller/OAuth2.pm @@ -5,6 +5,7 @@ BEGIN { extends 'Catalyst::Controller' } use Cpanel::JSON::XS qw( decode_json encode_json ); use Digest::SHA (); +use MetaCPAN::Util qw( true false ); use URI (); has login => ( is => 'ro' ); @@ -50,7 +51,7 @@ sub authorize : Local { my $code = $self->_build_code; $uri->query_form( { code => $code, $state ? ( state => $state ) : () } ); $c->user->_set_code($code); - $c->user->put( { refresh => 1 } ); + $c->user->put( { refresh => true } ); $c->res->redirect($uri); } @@ -89,7 +90,7 @@ sub access_token : Local { { token => $access_token, client => $client_id } ); } $user->clear_token; - $user->put( { refresh => 1 } ); + $user->put( { refresh => true } ); $c->res->content_type('application/json'); $c->res->body( encode_json( diff --git a/lib/MetaCPAN/Server/Controller/Release.pm b/lib/MetaCPAN/Server/Controller/Release.pm index 16ebb6d63..8d5f1660d 100644 --- a/lib/MetaCPAN/Server/Controller/Release.pm +++ b/lib/MetaCPAN/Server/Controller/Release.pm @@ -4,7 +4,6 @@ use strict; use warnings; use Moose; -use MetaCPAN::Util qw( single_valued_arrayref_to_scalar ); BEGIN { extends 'MetaCPAN::Server::Controller' } diff --git a/lib/MetaCPAN/Server/Controller/Search/Autocomplete.pm b/lib/MetaCPAN/Server/Controller/Search/Autocomplete.pm index 8f7faa7c6..7005c7dfa 100644 --- a/lib/MetaCPAN/Server/Controller/Search/Autocomplete.pm +++ b/lib/MetaCPAN/Server/Controller/Search/Autocomplete.pm @@ -4,7 +4,6 @@ use strict; use warnings; use Moose; -use MetaCPAN::Util qw( single_valued_arrayref_to_scalar ); BEGIN { extends 'MetaCPAN::Server::Controller' } diff --git a/lib/MetaCPAN/Server/Controller/User.pm b/lib/MetaCPAN/Server/Controller/User.pm index 1d3ae9bb7..fd0a0febf 100644 --- a/lib/MetaCPAN/Server/Controller/User.pm +++ b/lib/MetaCPAN/Server/Controller/User.pm @@ -3,7 +3,8 @@ package MetaCPAN::Server::Controller::User; use strict; use warnings; -use DateTime (); +use DateTime (); +use MetaCPAN::Util qw( true false ); use Moose; use Log::Log4perl::MDC (); @@ -57,7 +58,7 @@ sub identity_DELETE { my $user = $c->user; if ( $user->has_identity($identity) ) { my $id = $user->remove_identity($identity); - $user->put( { refresh => 1 } ); + $user->put( { refresh => true } ); $self->status_ok( $c, entity => $id ); } else { @@ -102,7 +103,7 @@ sub profile_PUT { } else { $profile - = $c->model('CPAN::Author')->put( $profile, { refresh => 1 } ); + = $c->model('CPAN::Author')->put( $profile, { refresh => true } ); $self->status_created( $c, location => $c->uri_for( '/author/' . $profile->{pauseid} ), diff --git a/lib/MetaCPAN/Server/Controller/User/Favorite.pm b/lib/MetaCPAN/Server/Controller/User/Favorite.pm index 619864b49..62024c268 100644 --- a/lib/MetaCPAN/Server/Controller/User/Favorite.pm +++ b/lib/MetaCPAN/Server/Controller/User/Favorite.pm @@ -4,6 +4,7 @@ use strict; use warnings; use Moose; +use MetaCPAN::Util qw( true false ); BEGIN { extends 'Catalyst::Controller::REST' } @@ -33,7 +34,7 @@ sub index_POST { release => $data->{release}, distribution => $data->{distribution}, }, - { refresh => 1 } + { refresh => true } ); $c->purge_author_key( $data->{author} ) if $data->{author}; $c->purge_dist_key( $data->{distribution} ) if $data->{distribution}; @@ -50,7 +51,7 @@ sub index_DELETE { my $favorite = $c->model('CPAN::Favorite') ->get( { user => $c->user->id, distribution => $distribution } ); if ($favorite) { - $favorite->delete( { refresh => 1 } ); + $favorite->delete( { refresh => true } ); $c->purge_author_key( $favorite->author ) if $favorite->author; $c->purge_dist_key($distribution); diff --git a/lib/MetaCPAN/Server/Controller/User/Turing.pm b/lib/MetaCPAN/Server/Controller/User/Turing.pm index 539c450ca..790a69544 100644 --- a/lib/MetaCPAN/Server/Controller/User/Turing.pm +++ b/lib/MetaCPAN/Server/Controller/User/Turing.pm @@ -5,6 +5,7 @@ use warnings; use DateTime (); use Moose; +use MetaCPAN::Util qw( true false ); BEGIN { extends 'Catalyst::Controller::REST' } @@ -33,7 +34,7 @@ sub index_POST { if ( $result->{is_valid} ) { $user->_set_passed_captcha( DateTime->now ); $user->clear_looks_human; # rebuild - $user->put( { refresh => 1 } ); + $user->put( { refresh => true } ); $self->status_ok( $c, entity => $user->meta->get_data($user) ); } else { diff --git a/lib/MetaCPAN/Util.pm b/lib/MetaCPAN/Util.pm index f250734ca..4bf2f0ed9 100644 --- a/lib/MetaCPAN/Util.pm +++ b/lib/MetaCPAN/Util.pm @@ -29,6 +29,7 @@ use Sub::Exporter -setup => { fix_pod fix_version generate_sid + hit_total numify_version pod_lines strip_pod @@ -97,6 +98,15 @@ sub author_dir { return $dir; } +sub hit_total { + my $res = shift; + my $total = $res && $res->{hits} && $res->{hits}{total}; + if ( ref $total ) { + return $total->{value}; + } + return $total; +} + # TODO: E sub strip_pod { my $pod = shift; diff --git a/t/lib/MetaCPAN/TestServer.pm b/t/lib/MetaCPAN/TestServer.pm index bdc051aa5..2af4f76e9 100644 --- a/t/lib/MetaCPAN/TestServer.pm +++ b/t/lib/MetaCPAN/TestServer.pm @@ -18,6 +18,7 @@ use MetaCPAN::Server (); use MetaCPAN::Server::Config (); use MetaCPAN::TestHelpers qw( fakecpan_dir ); use MetaCPAN::Types::TypeTiny qw( HashRef Path ); +use MetaCPAN::Util qw( true false ); use MooseX::Types::ElasticSearch qw( ES ); use Search::Elasticsearch (); use Test::More; @@ -262,12 +263,12 @@ sub prepare_user_test_data { ); ok( $user->add_identity( { name => 'pause', key => 'MO' } ), 'add pause identity' ); - ok( $user->put( { refresh => 1 } ), 'put user' ); + ok( $user->put( { refresh => true } ), 'put user' ); ok( MetaCPAN::Server->model('User::Account')->put( { access_token => [ { client => 'testing', token => 'bot' } ] }, - { refresh => 1 } + { refresh => true } ), 'put bot user' ); diff --git a/t/query/release.t b/t/query/release.t index acc114005..30367d999 100644 --- a/t/query/release.t +++ b/t/query/release.t @@ -7,9 +7,10 @@ use MetaCPAN::Query::Release (); use MetaCPAN::Server::Test (); use Test::More; -my $query - = MetaCPAN::Query::Release->new( - es => MetaCPAN::Server::Test::model->es() ); +my $query = MetaCPAN::Query::Release->new( + es => MetaCPAN::Server::Test::model->es(), + index_name => 'cpan', +); is( $query->_get_latest_release('DoesNotExist'), undef, '_get_latest_release returns undef when release does not exist' ); diff --git a/t/release/moose.t b/t/release/moose.t index 0b9811510..1454b41a9 100644 --- a/t/release/moose.t +++ b/t/release/moose.t @@ -3,6 +3,7 @@ use warnings; use lib 't/lib'; use MetaCPAN::Server::Test qw( model ); +use MetaCPAN::Util qw( true false hit_total ); use Test::More; my $model = model(); @@ -84,7 +85,7 @@ $signature = $idx->type('file')->query( { must => [ { term => { name => 'SIGNATURE' } }, { exists => { field => 'documentation' } }, - { term => { indexed => 1 } }, + { term => { indexed => true } }, ], }, } )->first; @@ -97,10 +98,10 @@ ok( !$signature, 'SIGNATURE is not pod' ); is_deeply( $module->{hits}, $file->{hits}, 'history of Moose and lib/Moose.pm match' ); - is( $module->{hits}->{total}, 2, 'two hits' ); + is( hit_total($module), 2, 'two hits' ); my $pod = $files->history( documentation => 'Moose::FAQ' )->raw->all; - is( $pod->{hits}->{total}, 1, 'one hit' ); + is( hit_total($pod), 1, 'one hit' ); } done_testing; diff --git a/t/server/controller/distribution.t b/t/server/controller/distribution.t index c4af70c69..491cf62b0 100644 --- a/t/server/controller/distribution.t +++ b/t/server/controller/distribution.t @@ -4,6 +4,7 @@ use lib 't/lib'; use MetaCPAN::Server::Test qw( app GET test_psgi ); use MetaCPAN::TestHelpers qw( decode_json_ok test_cache_headers ); +use MetaCPAN::Util qw( hit_total ); use Test::More; my @tests = ( @@ -53,7 +54,7 @@ test_psgi app, sub { my $json = decode_json_ok($res); if ( $k eq '/distribution' ) { - ok( $json->{hits}->{total}, 'got total count' ); + ok( hit_total($json), 'got total count' ); } elsif ( $v eq 200 ) { diff --git a/t/server/controller/file.t b/t/server/controller/file.t index d5d33062c..db5c5087b 100644 --- a/t/server/controller/file.t +++ b/t/server/controller/file.t @@ -4,6 +4,7 @@ use lib 't/lib'; use MetaCPAN::Server::Test qw( app GET test_psgi ); use MetaCPAN::TestHelpers qw( decode_json_ok test_cache_headers ); +use MetaCPAN::Util qw( hit_total ); use Test::More; my %tests = ( @@ -59,7 +60,7 @@ test_psgi app, sub { my $json = decode_json_ok($res); if ( $k eq '/file' ) { - ok( $json->{hits}->{total}, 'got total count' ); + ok( hit_total($json), 'got total count' ); } elsif ( $v eq 200 ) { ok( $json->{name} eq 'Moose.pm', 'Moose.pm' ); diff --git a/t/server/controller/module.t b/t/server/controller/module.t index 3db2505b0..5faf6ea79 100644 --- a/t/server/controller/module.t +++ b/t/server/controller/module.t @@ -4,6 +4,7 @@ use lib 't/lib'; use MetaCPAN::Server::Test qw( app GET test_psgi ); use MetaCPAN::TestHelpers qw( decode_json_ok test_cache_headers ); +use MetaCPAN::Util qw( hit_total ); use Test::More; my %tests = ( @@ -69,7 +70,7 @@ test_psgi app, sub { my $json = decode_json_ok($res); if ( $k eq '/module' ) { - ok( $json->{hits}->{total}, 'got total count' ); + ok( hit_total($json), 'got total count' ); } elsif ( $k =~ /fields/ ) { is_deeply(