← Index
NYTProf Performance Profile   « line view »
For ./view
  Run on Fri Jul 31 19:05:14 2015
Reported on Fri Jul 31 19:08:11 2015

Filename/var/www/foswiki11/lib/Foswiki/Store/SearchAlgorithms/Forking.pm
StatementsExecuted 204571 statements in 673ms
Subroutines
Calls P F Exclusive
Time
Inclusive
Time
Subroutine
40111.17s43.0sFoswiki::Store::SearchAlgorithms::Forking::::searchFoswiki::Store::SearchAlgorithms::Forking::search
401121.0ms43.4sFoswiki::Store::SearchAlgorithms::Forking::::_webQueryFoswiki::Store::SearchAlgorithms::Forking::_webQuery
40114.80ms43.5sFoswiki::Store::SearchAlgorithms::Forking::::queryFoswiki::Store::SearchAlgorithms::Forking::query
11114µs28µsFoswiki::Store::SearchAlgorithms::Forking::::BEGIN@5Foswiki::Store::SearchAlgorithms::Forking::BEGIN@5
11110µs15µsFoswiki::Store::SearchAlgorithms::Forking::::BEGIN@6Foswiki::Store::SearchAlgorithms::Forking::BEGIN@6
1119µs24µsFoswiki::Store::SearchAlgorithms::Forking::::BEGIN@7Foswiki::Store::SearchAlgorithms::Forking::BEGIN@7
1118µs8µsFoswiki::Store::SearchAlgorithms::Forking::::BEGIN@9Foswiki::Store::SearchAlgorithms::Forking::BEGIN@9
1116µs6µsFoswiki::Store::SearchAlgorithms::Forking::::BEGIN@8Foswiki::Store::SearchAlgorithms::Forking::BEGIN@8
Call graph for these subroutines as a Graphviz dot language file.
Line State
ments
Time
on line
Calls Time
in subs
Code
1# See bottom of file for license and copyright information
2
3package Foswiki::Store::SearchAlgorithms::Forking;
4
5229µs242µs
# spent 28µs (14+14) within Foswiki::Store::SearchAlgorithms::Forking::BEGIN@5 which was called: # once (14µs+14µs) by Foswiki::Store::VC::Store::query at line 5
use strict;
# spent 28µs making 1 call to Foswiki::Store::SearchAlgorithms::Forking::BEGIN@5 # spent 14µs making 1 call to strict::import
6225µs220µs
# spent 15µs (10+6) within Foswiki::Store::SearchAlgorithms::Forking::BEGIN@6 which was called: # once (10µs+6µs) by Foswiki::Store::VC::Store::query at line 6
use warnings;
# spent 15µs making 1 call to Foswiki::Store::SearchAlgorithms::Forking::BEGIN@6 # spent 6µs making 1 call to warnings::import
7223µs238µs
# spent 24µs (9+15) within Foswiki::Store::SearchAlgorithms::Forking::BEGIN@7 which was called: # once (9µs+15µs) by Foswiki::Store::VC::Store::query at line 7
use Assert;
# spent 24µs making 1 call to Foswiki::Store::SearchAlgorithms::Forking::BEGIN@7 # spent 15µs making 1 call to Assert::import
8228µs16µs
# spent 6µs within Foswiki::Store::SearchAlgorithms::Forking::BEGIN@8 which was called: # once (6µs+0s) by Foswiki::Store::VC::Store::query at line 8
use Foswiki::Search::InfoCache;
921.04ms18µs
# spent 8µs within Foswiki::Store::SearchAlgorithms::Forking::BEGIN@9 which was called: # once (8µs+0s) by Foswiki::Store::VC::Store::query at line 9
use Foswiki::Search::ResultSet;
10
11=begin TML
12
13---+ package Foswiki::Store::SearchAlgorithms::Forking
14
15Forking implementation of the RCS cache search.
16
17---++ search($searchString, $web, $inputTopicSet, $session, $options) -> \%seen
18Search .txt files in $dir for $searchString. See RcsFile::searchInWebContent
19for details.
20
21DEPRECATED
22
23=cut
24
25
# spent 43.0s (1.17+41.8) within Foswiki::Store::SearchAlgorithms::Forking::search which was called 40 times, avg 1.07s/call: # 40 times (1.17s+41.8s) by Foswiki::Store::SearchAlgorithms::Forking::_webQuery at line 267, avg 1.07s/call
sub search {
264057µs my ( $searchString, $web, $inputTopicSet, $session, $options ) = @_;
27
28 # Default (Forking) search
29
30 # SMELL: I18N: 'grep' must use locales if needed,
31 # for case-insensitive searching.
324025µs my $program = '';
33
3440129µs if ( $options->{type}
35 && ( $options->{type} eq 'regex' || $options->{wordboundaries} ) )
36 {
37 $program = $Foswiki::cfg{Store}{EgrepCmd};
38 }
39 else {
40 $program = $Foswiki::cfg{Store}{FgrepCmd};
41 }
42
434046µs if ( $options->{casesensitive} ) {
4440280µs $program =~ s/%CS{(.*?)\|.*?}%/$1/g;
45 }
46 else {
47 $program =~ s/%CS{.*?\|(.*?)}%/$1/g;
48 }
494051µs if ( $options->{files_without_match} ) {
5040175µs $program =~ s/%DET{.*?\|(.*?)}%/$1/g;
51 }
52 else {
53 $program =~ s/%DET{(.*?)\|.*?}%/$1/g;
54 }
554029µs if ( $options->{wordboundaries} ) {
56
57# Item5529: Can't use quotemeta because $searchString may be UTF8 encoded
58# TODO when testing UTF-8 code, try quotemeta. It should work with a decent perl
59 $searchString =~ s#([][|/\\\$\^*()+{};@?.{}])#\\$1#g;
60 $searchString = '\b' . $searchString . '\b';
61 }
62
634060µs if ( $Foswiki::cfg{DetailedOS} eq 'MSWin32' ) {
64
65 #try to escape the ^ and "" for native windows grep and apache
66 $searchString =~ s/\[\^/[^^/g;
67
68 # Fix escaping and quoting for Windows
69 $searchString =~ s#\\#\\\\#g;
70 $searchString =~ s#"#\\"#g;
71 $searchString = q(") . $searchString . q(");
72 }
73
744019µs my $matches = '';
75
76 #SMELL, TODO, replace with Store call.
774041µs my $sDir = $Foswiki::cfg{DataDir} . '/' . $web . '/';
78
79 # process topics in sets, fix for Codev.ArgumentListIsTooLongForSearch
804011µs my $maxTopicsInSet = 512; # max number of topics for a grep call
81 #TODO: the number is actually dependant on the length of the path to each file
82 #SMELL: the following while loop should probably be made by sysCommand, as this is a leaky abstraction.
83 ##heck, on pre WinXP its only 2048, post XP its 8191 - http://support.microsoft.com/kb/830473
844023µs if ( $Foswiki::cfg{DetailedOS} eq 'MSWin32' ) {
85
86 #tune the number based on the length of "$sDir/WebSearchAdvanced.txt"
87 #30 is a guess - wotamess
88 $maxTopicsInSet =
89 ( ( 8191 - ( length($program) + length($searchString) + 30 ) ) /
90 ( length("$sDir/LongWebSearchAdvanced.txt") + 10 ) );
91
92 #print STDERR "++++++++++++ $maxTopicsInSet \n";
93 }
94
95 # while (my @set = splice( @take, 0, $maxTopicsInSet )) {
96 # @set = map { "$sDir/$_.txt" } @set;
974015µs my @set;
984060µs40278µs $inputTopicSet->reset();
# spent 278µs making 40 calls to Foswiki::Iterator::FilterIterator::reset, avg 7µs/call
994082µs4012.6ms while ( $inputTopicSet->hasNext() ) {
# spent 12.6ms making 40 calls to Foswiki::Iterator::FilterIterator::hasNext, avg 315µs/call
1004604070.9ms46040329ms my $webtopic = $inputTopicSet->next();
# spent 329ms making 46040 calls to Foswiki::Iterator::FilterIterator::next, avg 7µs/call
1014604094.8ms46040972ms my ( $Iweb, $tn ) =
# spent 972ms making 46040 calls to Foswiki::Func::normalizeWebTopicName, avg 21µs/call
102 Foswiki::Func::normalizeWebTopicName( $web, $webtopic );
1034604052.2ms push( @set, "$sDir/$tn.txt" );
10446040200ms920005.56s if (
# spent 5.56s making 92000 calls to Foswiki::Iterator::FilterIterator::hasNext, avg 60µs/call
105 ( $#set >= $maxTopicsInSet ) #replace with character count..
106 || !( $inputTopicSet->hasNext() )
107 )
108 {
1091202.13ms12035.0s my ( $m, $exit ) = Foswiki::Sandbox->sysCommand(
# spent 35.0s making 120 calls to Foswiki::Sandbox::sysCommand, avg 291ms/call
110 $program,
111 TOKEN => $searchString,
112 FILES => \@set
113 );
11412020.3ms @set = ();
115
116 # man grep: "Normally, exit status is 0 if selected lines are found
117 # and 1 otherwise. But the exit status is 2 if an error occurred,
118 # unless the -q or --quiet or --silent option is used and a selected
119 # line is found."
120120156µs if ( $exit > 1 ) {
121
122 #TODO: need to work out a way to alert the admin there is a problem, without
123 # filling up the log files with repeated SEARCH's
124
125# NOTE: we ignore the error, because grep returns an error if it comes across a broken file link
126# or a file it does not have permission to open, so throwing here gives wrong search results.
127# throw Error::Simple("$program Grep for '$searchString' returned error")
128 }
1291201.31ms $matches .= $m;
130 }
131 }
1324036µs my %seen;
133
134 # Note use of / and \ as dir separators, to support Winblows
135 $matches =~
13617560203ms s/([^\/\\]*?)\.txt(:(.*))?$/push( @{$seen{$1}}, ($3||'') ); ''/gem;
137
138 # Implicit untaint OK; data from grep
139
14040904µs return \%seen;
141}
142
143=begin TML
144
145this is the new way -
146
147=cut
148
149
# spent 43.5s (4.80ms+43.5) within Foswiki::Store::SearchAlgorithms::Forking::query which was called 40 times, avg 1.09s/call: # 40 times (4.80ms+43.5s) by Foswiki::Store::VC::Store::query at line 518 of /var/www/foswiki11/lib/Foswiki/Store/VC/Store.pm, avg 1.09s/call
sub query {
1504033µs my ( $query, $inputTopicSet, $session, $options ) = @_;
151
1524040µs if ( ( @{ $query->{tokens} } ) == 0 ) {
153 return new Foswiki::Search::InfoCache( $session, '' );
154 }
155
1564066µs my $webNames = $options->{web} || '';
1574061µs my $recurse = $options->{'recurse'} || '';
1584089µs4094µs my $isAdmin = $session->{users}->isAdmin( $session->{user} );
# spent 94µs making 40 calls to Foswiki::Users::isAdmin, avg 2µs/call
159
16040212µs my $searchAllFlag = ( $webNames =~ /(^|[\,\s])(all|on)([\,\s]|$)/i );
1614099µs401.97ms my @webs = Foswiki::Search::InfoCache::_getListOfWebs( $webNames, $recurse,
# spent 1.97ms making 40 calls to Foswiki::Search::InfoCache::_getListOfWebs, avg 49µs/call
162 $searchAllFlag );
163
164409µs my @resultCacheList;
1654044µs foreach my $web (@webs) {
166
167 # can't process what ain't thar
1684086µs401.83ms next unless $session->webExists($web);
# spent 1.83ms making 40 calls to Foswiki::webExists, avg 46µs/call
169
17040140µs40651µs my $webObject = Foswiki::Meta->new( $session, $web );
# spent 651µs making 40 calls to Foswiki::Meta::new, avg 16µs/call
17140145µs802.16ms my $thisWebNoSearchAll =
# spent 1.84ms making 40 calls to Foswiki::Meta::getPreference, avg 46µs/call # spent 320µs making 40 calls to Foswiki::isTrue, avg 8µs/call
172 Foswiki::isTrue( $webObject->getPreference('NOSEARCHALL') );
173
174 # make sure we can report this web on an 'all' search
175 # DON'T filter out unless it's part of an 'all' search.
176 next
1774017µs if ( $searchAllFlag
178 && !$isAdmin
179 && ( $thisWebNoSearchAll || $web =~ /^[\.\_]/ )
180 && $web ne $session->{webName} );
181
18240153µs4043.4s my $infoCache =
# spent 43.4s making 40 calls to Foswiki::Store::SearchAlgorithms::Forking::_webQuery, avg 1.08s/call
183 _webQuery( $query, $web, $inputTopicSet, $session, $options );
18440290µs40129ms $infoCache->sortResults($options);
# spent 129ms making 40 calls to Foswiki::Search::InfoCache::sortResults, avg 3.22ms/call
18540763µs push( @resultCacheList, $infoCache );
186 }
18740946µs801.20ms my $resultset =
# spent 983µs making 40 calls to Foswiki::Search::ResultSet::new, avg 25µs/call # spent 215µs making 40 calls to Foswiki::isTrue, avg 5µs/call
188 new Foswiki::Search::ResultSet( \@resultCacheList, $options->{groupby},
189 $options->{order}, Foswiki::isTrue( $options->{reverse} ) );
190
191 # first filter, then sort the remaining
19240124µs my $date = $options->{'date'} || '';
1934013µs if ($date) {
194 $resultset->filterByDate($date);
195 }
196
197 #TODO: $options should become redundant
19840321µs40920µs $resultset->sortResults($options);
# spent 920µs making 40 calls to Foswiki::Search::ResultSet::sortResults, avg 23µs/call
19940418µs return $resultset;
200}
201
202#ok, for initial validation, naively call the code with a web.
203
# spent 43.4s (21.0ms+43.3) within Foswiki::Store::SearchAlgorithms::Forking::_webQuery which was called 40 times, avg 1.08s/call: # 40 times (21.0ms+43.3s) by Foswiki::Store::SearchAlgorithms::Forking::query at line 182, avg 1.08s/call
sub _webQuery {
2044065µs my ( $query, $web, $inputTopicSet, $session, $options ) = @_;
2054057µs4037µs ASSERT( scalar( @{ $query->{tokens} } ) > 0 ) if DEBUG;
# spent 37µs making 40 calls to Assert::ASSERTS_OFF, avg 925ns/call
206
207 #print STDERR "ForkingSEARCH(".join(', ', @{ $query->{tokens} }).")\n";
208 # default scope is 'text'
2094078µs $options->{'scope'} = 'text'
210 unless ( defined( $options->{'scope'} )
211 && $options->{'scope'} =~ /^(topic|all)$/ );
212
2134020µs my $topicSet = $inputTopicSet;
2144014µs if ( !defined($topicSet) ) {
215
216 #then we start with the whole web
217 #TODO: i'm sure that is a flawed assumption
218 my $webObject = Foswiki::Meta->new( $session, $web );
219 $topicSet =
220 Foswiki::Search::InfoCache::getTopicListIterator( $webObject,
221 $options );
222 }
2234047µs4027µs ASSERT( UNIVERSAL::isa( $topicSet, 'Foswiki::Iterator' ) ) if DEBUG;
# spent 27µs making 40 calls to Assert::ASSERTS_OFF, avg 668ns/call
224
225#print STDERR "######## Forking search ($web) tokens ".scalar(@{$query->{tokens}})." : ".join(',', @{$query->{tokens}})."\n";
226# AND search - search once for each token, ANDing result together
2274078µs foreach my $token ( @{ $query->{tokens} } ) {
228
2294055µs my $tokenCopy = $token;
230
231 # flag for AND NOT search
2324010µs my $invertSearch = 0;
2334083µs $invertSearch = ( $tokenCopy =~ s/^\!//o );
234
235 # scope can be 'topic' (default), 'text' or "all"
236 # scope='topic', e.g. Perl search on topic name:
2374010µs my %topicMatches;
2384029µs unless ( $options->{'scope'} eq 'text' ) {
239 my $qtoken = $tokenCopy;
240
241# FIXME I18N
242# http://foswiki.org/Tasks/Item1646 this causes us to use/leak huge amounts of memory if called too often
243 $qtoken = quotemeta($qtoken) if ( $options->{'type'} ne 'regex' );
244
245 $topicSet->reset();
246 while ( $topicSet->hasNext() ) {
247 my $webtopic = $topicSet->next();
248 my ( $itrWeb, $topic ) =
249 Foswiki::Func::normalizeWebTopicName( $web, $webtopic );
250
251 if ( $options->{'casesensitive'} ) {
252
253 # fix for Codev.SearchWithNoPipe
254 #push(@scopeTopicList, $topic) if ( $topic =~ /$qtoken/ );
255 $topicMatches{$topic} = 1 if ( $topic =~ /$qtoken/ );
256 }
257 else {
258
259 #push(@scopeTopicList, $topic) if ( $topic =~ /$qtoken/i );
260 $topicMatches{$topic} = 1 if ( $topic =~ /$qtoken/i );
261 }
262 }
263 }
264
265 # scope='text', e.g. grep search on topic text:
266401.08ms unless ( $options->{'scope'} eq 'topic' ) {
26740163µs4043.0s my $textMatches =
# spent 43.0s making 40 calls to Foswiki::Store::SearchAlgorithms::Forking::search, avg 1.07s/call
268 search( $tokenCopy, $web, $topicSet, $session, $options );
269
270 #bring the text matches into the topicMatch hash
2714010.5ms if ($textMatches) {
272 @topicMatches{ keys %$textMatches } = values %$textMatches;
273 }
274 }
275
27640101µs my @scopeTextList = ();
27740596µs if ($invertSearch) {
278 $topicSet->reset();
279 while ( $topicSet->hasNext() ) {
280 my $webtopic = $topicSet->next();
281 my ( $Iweb, $topic ) =
282 Foswiki::Func::normalizeWebTopicName( $web, $webtopic );
283
284 if ( $topicMatches{$topic} ) {
285 }
286 else {
287 push( @scopeTextList, $topic );
288 }
289 }
290 }
291 else {
292
293 #TODO: the sad thing about this is we lose info
294402.67ms @scopeTextList = keys(%topicMatches);
295 }
296
297 # reduced topic list for next token
298404.88ms40353ms $topicSet =
# spent 353ms making 40 calls to Foswiki::Search::InfoCache::new, avg 8.81ms/call
299 new Foswiki::Search::InfoCache( $Foswiki::Plugins::SESSION, $web,
300 \@scopeTextList );
301 }
302
30340397µs return $topicSet;
304}
305
30612µs1;
307__END__