Filename | /var/www/foswiki11/lib/Foswiki/Store/SearchAlgorithms/Forking.pm |
Statements | Executed 204571 statements in 673ms |
Calls | P | F | Exclusive Time |
Inclusive Time |
Subroutine |
---|---|---|---|---|---|
40 | 1 | 1 | 1.17s | 43.0s | search | Foswiki::Store::SearchAlgorithms::Forking::
40 | 1 | 1 | 21.0ms | 43.4s | _webQuery | Foswiki::Store::SearchAlgorithms::Forking::
40 | 1 | 1 | 4.80ms | 43.5s | query | Foswiki::Store::SearchAlgorithms::Forking::
1 | 1 | 1 | 14µs | 28µs | BEGIN@5 | Foswiki::Store::SearchAlgorithms::Forking::
1 | 1 | 1 | 10µs | 15µs | BEGIN@6 | Foswiki::Store::SearchAlgorithms::Forking::
1 | 1 | 1 | 9µs | 24µs | BEGIN@7 | Foswiki::Store::SearchAlgorithms::Forking::
1 | 1 | 1 | 8µs | 8µs | BEGIN@9 | Foswiki::Store::SearchAlgorithms::Forking::
1 | 1 | 1 | 6µs | 6µs | BEGIN@8 | Foswiki::Store::SearchAlgorithms::Forking::
Line | State ments |
Time on line |
Calls | Time in subs |
Code |
---|---|---|---|---|---|
1 | # See bottom of file for license and copyright information | ||||
2 | |||||
3 | package Foswiki::Store::SearchAlgorithms::Forking; | ||||
4 | |||||
5 | 2 | 29µs | 2 | 42µs | # spent 28µs (14+14) within Foswiki::Store::SearchAlgorithms::Forking::BEGIN@5 which was called:
# once (14µs+14µs) by Foswiki::Store::VC::Store::query at line 5 # spent 28µs making 1 call to Foswiki::Store::SearchAlgorithms::Forking::BEGIN@5
# spent 14µs making 1 call to strict::import |
6 | 2 | 25µs | 2 | 20µs | # spent 15µs (10+6) within Foswiki::Store::SearchAlgorithms::Forking::BEGIN@6 which was called:
# once (10µs+6µs) by Foswiki::Store::VC::Store::query at line 6 # spent 15µs making 1 call to Foswiki::Store::SearchAlgorithms::Forking::BEGIN@6
# spent 6µs making 1 call to warnings::import |
7 | 2 | 23µs | 2 | 38µs | # spent 24µs (9+15) within Foswiki::Store::SearchAlgorithms::Forking::BEGIN@7 which was called:
# once (9µs+15µs) by Foswiki::Store::VC::Store::query at line 7 # spent 24µs making 1 call to Foswiki::Store::SearchAlgorithms::Forking::BEGIN@7
# spent 15µs making 1 call to Assert::import |
8 | 2 | 28µs | 1 | 6µs | # spent 6µs within Foswiki::Store::SearchAlgorithms::Forking::BEGIN@8 which was called:
# once (6µs+0s) by Foswiki::Store::VC::Store::query at line 8 # spent 6µs making 1 call to Foswiki::Store::SearchAlgorithms::Forking::BEGIN@8 |
9 | 2 | 1.04ms | 1 | 8µs | # spent 8µs within Foswiki::Store::SearchAlgorithms::Forking::BEGIN@9 which was called:
# once (8µs+0s) by Foswiki::Store::VC::Store::query at line 9 # spent 8µs making 1 call to Foswiki::Store::SearchAlgorithms::Forking::BEGIN@9 |
10 | |||||
11 | =begin TML | ||||
12 | |||||
13 | ---+ package Foswiki::Store::SearchAlgorithms::Forking | ||||
14 | |||||
15 | Forking implementation of the RCS cache search. | ||||
16 | |||||
17 | ---++ search($searchString, $web, $inputTopicSet, $session, $options) -> \%seen | ||||
18 | Search .txt files in $dir for $searchString. See RcsFile::searchInWebContent | ||||
19 | for details. | ||||
20 | |||||
21 | DEPRECATED | ||||
22 | |||||
23 | =cut | ||||
24 | |||||
25 | # spent 43.0s (1.17+41.8) within Foswiki::Store::SearchAlgorithms::Forking::search which was called 40 times, avg 1.07s/call:
# 40 times (1.17s+41.8s) by Foswiki::Store::SearchAlgorithms::Forking::_webQuery at line 267, avg 1.07s/call | ||||
26 | 40 | 57µs | my ( $searchString, $web, $inputTopicSet, $session, $options ) = @_; | ||
27 | |||||
28 | # Default (Forking) search | ||||
29 | |||||
30 | # SMELL: I18N: 'grep' must use locales if needed, | ||||
31 | # for case-insensitive searching. | ||||
32 | 40 | 25µs | my $program = ''; | ||
33 | |||||
34 | 40 | 129µs | if ( $options->{type} | ||
35 | && ( $options->{type} eq 'regex' || $options->{wordboundaries} ) ) | ||||
36 | { | ||||
37 | $program = $Foswiki::cfg{Store}{EgrepCmd}; | ||||
38 | } | ||||
39 | else { | ||||
40 | $program = $Foswiki::cfg{Store}{FgrepCmd}; | ||||
41 | } | ||||
42 | |||||
43 | 40 | 46µs | if ( $options->{casesensitive} ) { | ||
44 | 40 | 280µs | $program =~ s/%CS{(.*?)\|.*?}%/$1/g; | ||
45 | } | ||||
46 | else { | ||||
47 | $program =~ s/%CS{.*?\|(.*?)}%/$1/g; | ||||
48 | } | ||||
49 | 40 | 51µs | if ( $options->{files_without_match} ) { | ||
50 | 40 | 175µs | $program =~ s/%DET{.*?\|(.*?)}%/$1/g; | ||
51 | } | ||||
52 | else { | ||||
53 | $program =~ s/%DET{(.*?)\|.*?}%/$1/g; | ||||
54 | } | ||||
55 | 40 | 29µs | if ( $options->{wordboundaries} ) { | ||
56 | |||||
57 | # Item5529: Can't use quotemeta because $searchString may be UTF8 encoded | ||||
58 | # TODO when testing UTF-8 code, try quotemeta. It should work with a decent perl | ||||
59 | $searchString =~ s#([][|/\\\$\^*()+{};@?.{}])#\\$1#g; | ||||
60 | $searchString = '\b' . $searchString . '\b'; | ||||
61 | } | ||||
62 | |||||
63 | 40 | 60µs | if ( $Foswiki::cfg{DetailedOS} eq 'MSWin32' ) { | ||
64 | |||||
65 | #try to escape the ^ and "" for native windows grep and apache | ||||
66 | $searchString =~ s/\[\^/[^^/g; | ||||
67 | |||||
68 | # Fix escaping and quoting for Windows | ||||
69 | $searchString =~ s#\\#\\\\#g; | ||||
70 | $searchString =~ s#"#\\"#g; | ||||
71 | $searchString = q(") . $searchString . q("); | ||||
72 | } | ||||
73 | |||||
74 | 40 | 19µs | my $matches = ''; | ||
75 | |||||
76 | #SMELL, TODO, replace with Store call. | ||||
77 | 40 | 41µs | my $sDir = $Foswiki::cfg{DataDir} . '/' . $web . '/'; | ||
78 | |||||
79 | # process topics in sets, fix for Codev.ArgumentListIsTooLongForSearch | ||||
80 | 40 | 11µs | my $maxTopicsInSet = 512; # max number of topics for a grep call | ||
81 | #TODO: the number is actually dependant on the length of the path to each file | ||||
82 | #SMELL: the following while loop should probably be made by sysCommand, as this is a leaky abstraction. | ||||
83 | ##heck, on pre WinXP its only 2048, post XP its 8191 - http://support.microsoft.com/kb/830473 | ||||
84 | 40 | 23µs | if ( $Foswiki::cfg{DetailedOS} eq 'MSWin32' ) { | ||
85 | |||||
86 | #tune the number based on the length of "$sDir/WebSearchAdvanced.txt" | ||||
87 | #30 is a guess - wotamess | ||||
88 | $maxTopicsInSet = | ||||
89 | ( ( 8191 - ( length($program) + length($searchString) + 30 ) ) / | ||||
90 | ( length("$sDir/LongWebSearchAdvanced.txt") + 10 ) ); | ||||
91 | |||||
92 | #print STDERR "++++++++++++ $maxTopicsInSet \n"; | ||||
93 | } | ||||
94 | |||||
95 | # while (my @set = splice( @take, 0, $maxTopicsInSet )) { | ||||
96 | # @set = map { "$sDir/$_.txt" } @set; | ||||
97 | 40 | 15µs | my @set; | ||
98 | 40 | 60µs | 40 | 278µs | $inputTopicSet->reset(); # spent 278µs making 40 calls to Foswiki::Iterator::FilterIterator::reset, avg 7µs/call |
99 | 40 | 82µs | 40 | 12.6ms | while ( $inputTopicSet->hasNext() ) { # spent 12.6ms making 40 calls to Foswiki::Iterator::FilterIterator::hasNext, avg 315µs/call |
100 | 46040 | 70.9ms | 46040 | 329ms | my $webtopic = $inputTopicSet->next(); # spent 329ms making 46040 calls to Foswiki::Iterator::FilterIterator::next, avg 7µs/call |
101 | 46040 | 94.8ms | 46040 | 972ms | my ( $Iweb, $tn ) = # spent 972ms making 46040 calls to Foswiki::Func::normalizeWebTopicName, avg 21µs/call |
102 | Foswiki::Func::normalizeWebTopicName( $web, $webtopic ); | ||||
103 | 46040 | 52.2ms | push( @set, "$sDir/$tn.txt" ); | ||
104 | 46040 | 200ms | 92000 | 5.56s | if ( # spent 5.56s making 92000 calls to Foswiki::Iterator::FilterIterator::hasNext, avg 60µs/call |
105 | ( $#set >= $maxTopicsInSet ) #replace with character count.. | ||||
106 | || !( $inputTopicSet->hasNext() ) | ||||
107 | ) | ||||
108 | { | ||||
109 | 120 | 2.13ms | 120 | 35.0s | my ( $m, $exit ) = Foswiki::Sandbox->sysCommand( # spent 35.0s making 120 calls to Foswiki::Sandbox::sysCommand, avg 291ms/call |
110 | $program, | ||||
111 | TOKEN => $searchString, | ||||
112 | FILES => \@set | ||||
113 | ); | ||||
114 | 120 | 20.3ms | @set = (); | ||
115 | |||||
116 | # man grep: "Normally, exit status is 0 if selected lines are found | ||||
117 | # and 1 otherwise. But the exit status is 2 if an error occurred, | ||||
118 | # unless the -q or --quiet or --silent option is used and a selected | ||||
119 | # line is found." | ||||
120 | 120 | 156µs | if ( $exit > 1 ) { | ||
121 | |||||
122 | #TODO: need to work out a way to alert the admin there is a problem, without | ||||
123 | # filling up the log files with repeated SEARCH's | ||||
124 | |||||
125 | # NOTE: we ignore the error, because grep returns an error if it comes across a broken file link | ||||
126 | # or a file it does not have permission to open, so throwing here gives wrong search results. | ||||
127 | # throw Error::Simple("$program Grep for '$searchString' returned error") | ||||
128 | } | ||||
129 | 120 | 1.31ms | $matches .= $m; | ||
130 | } | ||||
131 | } | ||||
132 | 40 | 36µs | my %seen; | ||
133 | |||||
134 | # Note use of / and \ as dir separators, to support Winblows | ||||
135 | $matches =~ | ||||
136 | 17560 | 203ms | s/([^\/\\]*?)\.txt(:(.*))?$/push( @{$seen{$1}}, ($3||'') ); ''/gem; | ||
137 | |||||
138 | # Implicit untaint OK; data from grep | ||||
139 | |||||
140 | 40 | 904µs | return \%seen; | ||
141 | } | ||||
142 | |||||
143 | =begin TML | ||||
144 | |||||
145 | this is the new way - | ||||
146 | |||||
147 | =cut | ||||
148 | |||||
149 | # spent 43.5s (4.80ms+43.5) within Foswiki::Store::SearchAlgorithms::Forking::query which was called 40 times, avg 1.09s/call:
# 40 times (4.80ms+43.5s) by Foswiki::Store::VC::Store::query at line 518 of /var/www/foswiki11/lib/Foswiki/Store/VC/Store.pm, avg 1.09s/call | ||||
150 | 40 | 33µs | my ( $query, $inputTopicSet, $session, $options ) = @_; | ||
151 | |||||
152 | 40 | 40µs | if ( ( @{ $query->{tokens} } ) == 0 ) { | ||
153 | return new Foswiki::Search::InfoCache( $session, '' ); | ||||
154 | } | ||||
155 | |||||
156 | 40 | 66µs | my $webNames = $options->{web} || ''; | ||
157 | 40 | 61µs | my $recurse = $options->{'recurse'} || ''; | ||
158 | 40 | 89µs | 40 | 94µs | my $isAdmin = $session->{users}->isAdmin( $session->{user} ); # spent 94µs making 40 calls to Foswiki::Users::isAdmin, avg 2µs/call |
159 | |||||
160 | 40 | 212µs | my $searchAllFlag = ( $webNames =~ /(^|[\,\s])(all|on)([\,\s]|$)/i ); | ||
161 | 40 | 99µs | 40 | 1.97ms | my @webs = Foswiki::Search::InfoCache::_getListOfWebs( $webNames, $recurse, # spent 1.97ms making 40 calls to Foswiki::Search::InfoCache::_getListOfWebs, avg 49µs/call |
162 | $searchAllFlag ); | ||||
163 | |||||
164 | 40 | 9µs | my @resultCacheList; | ||
165 | 40 | 44µs | foreach my $web (@webs) { | ||
166 | |||||
167 | # can't process what ain't thar | ||||
168 | 40 | 86µs | 40 | 1.83ms | next unless $session->webExists($web); # spent 1.83ms making 40 calls to Foswiki::webExists, avg 46µs/call |
169 | |||||
170 | 40 | 140µs | 40 | 651µs | my $webObject = Foswiki::Meta->new( $session, $web ); # spent 651µs making 40 calls to Foswiki::Meta::new, avg 16µs/call |
171 | 40 | 145µs | 80 | 2.16ms | my $thisWebNoSearchAll = # spent 1.84ms making 40 calls to Foswiki::Meta::getPreference, avg 46µs/call
# spent 320µs making 40 calls to Foswiki::isTrue, avg 8µs/call |
172 | Foswiki::isTrue( $webObject->getPreference('NOSEARCHALL') ); | ||||
173 | |||||
174 | # make sure we can report this web on an 'all' search | ||||
175 | # DON'T filter out unless it's part of an 'all' search. | ||||
176 | next | ||||
177 | 40 | 17µs | if ( $searchAllFlag | ||
178 | && !$isAdmin | ||||
179 | && ( $thisWebNoSearchAll || $web =~ /^[\.\_]/ ) | ||||
180 | && $web ne $session->{webName} ); | ||||
181 | |||||
182 | 40 | 153µs | 40 | 43.4s | my $infoCache = # spent 43.4s making 40 calls to Foswiki::Store::SearchAlgorithms::Forking::_webQuery, avg 1.08s/call |
183 | _webQuery( $query, $web, $inputTopicSet, $session, $options ); | ||||
184 | 40 | 290µs | 40 | 129ms | $infoCache->sortResults($options); # spent 129ms making 40 calls to Foswiki::Search::InfoCache::sortResults, avg 3.22ms/call |
185 | 40 | 763µs | push( @resultCacheList, $infoCache ); | ||
186 | } | ||||
187 | 40 | 946µs | 80 | 1.20ms | my $resultset = # spent 983µs making 40 calls to Foswiki::Search::ResultSet::new, avg 25µs/call
# spent 215µs making 40 calls to Foswiki::isTrue, avg 5µs/call |
188 | new Foswiki::Search::ResultSet( \@resultCacheList, $options->{groupby}, | ||||
189 | $options->{order}, Foswiki::isTrue( $options->{reverse} ) ); | ||||
190 | |||||
191 | # first filter, then sort the remaining | ||||
192 | 40 | 124µs | my $date = $options->{'date'} || ''; | ||
193 | 40 | 13µs | if ($date) { | ||
194 | $resultset->filterByDate($date); | ||||
195 | } | ||||
196 | |||||
197 | #TODO: $options should become redundant | ||||
198 | 40 | 321µs | 40 | 920µs | $resultset->sortResults($options); # spent 920µs making 40 calls to Foswiki::Search::ResultSet::sortResults, avg 23µs/call |
199 | 40 | 418µs | return $resultset; | ||
200 | } | ||||
201 | |||||
202 | #ok, for initial validation, naively call the code with a web. | ||||
203 | # spent 43.4s (21.0ms+43.3) within Foswiki::Store::SearchAlgorithms::Forking::_webQuery which was called 40 times, avg 1.08s/call:
# 40 times (21.0ms+43.3s) by Foswiki::Store::SearchAlgorithms::Forking::query at line 182, avg 1.08s/call | ||||
204 | 40 | 65µs | my ( $query, $web, $inputTopicSet, $session, $options ) = @_; | ||
205 | 40 | 57µs | 40 | 37µs | ASSERT( scalar( @{ $query->{tokens} } ) > 0 ) if DEBUG; # spent 37µs making 40 calls to Assert::ASSERTS_OFF, avg 925ns/call |
206 | |||||
207 | #print STDERR "ForkingSEARCH(".join(', ', @{ $query->{tokens} }).")\n"; | ||||
208 | # default scope is 'text' | ||||
209 | 40 | 78µs | $options->{'scope'} = 'text' | ||
210 | unless ( defined( $options->{'scope'} ) | ||||
211 | && $options->{'scope'} =~ /^(topic|all)$/ ); | ||||
212 | |||||
213 | 40 | 20µs | my $topicSet = $inputTopicSet; | ||
214 | 40 | 14µs | if ( !defined($topicSet) ) { | ||
215 | |||||
216 | #then we start with the whole web | ||||
217 | #TODO: i'm sure that is a flawed assumption | ||||
218 | my $webObject = Foswiki::Meta->new( $session, $web ); | ||||
219 | $topicSet = | ||||
220 | Foswiki::Search::InfoCache::getTopicListIterator( $webObject, | ||||
221 | $options ); | ||||
222 | } | ||||
223 | 40 | 47µs | 40 | 27µs | ASSERT( UNIVERSAL::isa( $topicSet, 'Foswiki::Iterator' ) ) if DEBUG; # spent 27µs making 40 calls to Assert::ASSERTS_OFF, avg 668ns/call |
224 | |||||
225 | #print STDERR "######## Forking search ($web) tokens ".scalar(@{$query->{tokens}})." : ".join(',', @{$query->{tokens}})."\n"; | ||||
226 | # AND search - search once for each token, ANDing result together | ||||
227 | 40 | 78µs | foreach my $token ( @{ $query->{tokens} } ) { | ||
228 | |||||
229 | 40 | 55µs | my $tokenCopy = $token; | ||
230 | |||||
231 | # flag for AND NOT search | ||||
232 | 40 | 10µs | my $invertSearch = 0; | ||
233 | 40 | 83µs | $invertSearch = ( $tokenCopy =~ s/^\!//o ); | ||
234 | |||||
235 | # scope can be 'topic' (default), 'text' or "all" | ||||
236 | # scope='topic', e.g. Perl search on topic name: | ||||
237 | 40 | 10µs | my %topicMatches; | ||
238 | 40 | 29µs | unless ( $options->{'scope'} eq 'text' ) { | ||
239 | my $qtoken = $tokenCopy; | ||||
240 | |||||
241 | # FIXME I18N | ||||
242 | # http://foswiki.org/Tasks/Item1646 this causes us to use/leak huge amounts of memory if called too often | ||||
243 | $qtoken = quotemeta($qtoken) if ( $options->{'type'} ne 'regex' ); | ||||
244 | |||||
245 | $topicSet->reset(); | ||||
246 | while ( $topicSet->hasNext() ) { | ||||
247 | my $webtopic = $topicSet->next(); | ||||
248 | my ( $itrWeb, $topic ) = | ||||
249 | Foswiki::Func::normalizeWebTopicName( $web, $webtopic ); | ||||
250 | |||||
251 | if ( $options->{'casesensitive'} ) { | ||||
252 | |||||
253 | # fix for Codev.SearchWithNoPipe | ||||
254 | #push(@scopeTopicList, $topic) if ( $topic =~ /$qtoken/ ); | ||||
255 | $topicMatches{$topic} = 1 if ( $topic =~ /$qtoken/ ); | ||||
256 | } | ||||
257 | else { | ||||
258 | |||||
259 | #push(@scopeTopicList, $topic) if ( $topic =~ /$qtoken/i ); | ||||
260 | $topicMatches{$topic} = 1 if ( $topic =~ /$qtoken/i ); | ||||
261 | } | ||||
262 | } | ||||
263 | } | ||||
264 | |||||
265 | # scope='text', e.g. grep search on topic text: | ||||
266 | 40 | 1.08ms | unless ( $options->{'scope'} eq 'topic' ) { | ||
267 | 40 | 163µs | 40 | 43.0s | my $textMatches = # spent 43.0s making 40 calls to Foswiki::Store::SearchAlgorithms::Forking::search, avg 1.07s/call |
268 | search( $tokenCopy, $web, $topicSet, $session, $options ); | ||||
269 | |||||
270 | #bring the text matches into the topicMatch hash | ||||
271 | 40 | 10.5ms | if ($textMatches) { | ||
272 | @topicMatches{ keys %$textMatches } = values %$textMatches; | ||||
273 | } | ||||
274 | } | ||||
275 | |||||
276 | 40 | 101µs | my @scopeTextList = (); | ||
277 | 40 | 596µs | if ($invertSearch) { | ||
278 | $topicSet->reset(); | ||||
279 | while ( $topicSet->hasNext() ) { | ||||
280 | my $webtopic = $topicSet->next(); | ||||
281 | my ( $Iweb, $topic ) = | ||||
282 | Foswiki::Func::normalizeWebTopicName( $web, $webtopic ); | ||||
283 | |||||
284 | if ( $topicMatches{$topic} ) { | ||||
285 | } | ||||
286 | else { | ||||
287 | push( @scopeTextList, $topic ); | ||||
288 | } | ||||
289 | } | ||||
290 | } | ||||
291 | else { | ||||
292 | |||||
293 | #TODO: the sad thing about this is we lose info | ||||
294 | 40 | 2.67ms | @scopeTextList = keys(%topicMatches); | ||
295 | } | ||||
296 | |||||
297 | # reduced topic list for next token | ||||
298 | 40 | 4.88ms | 40 | 353ms | $topicSet = # spent 353ms making 40 calls to Foswiki::Search::InfoCache::new, avg 8.81ms/call |
299 | new Foswiki::Search::InfoCache( $Foswiki::Plugins::SESSION, $web, | ||||
300 | \@scopeTextList ); | ||||
301 | } | ||||
302 | |||||
303 | 40 | 397µs | return $topicSet; | ||
304 | } | ||||
305 | |||||
306 | 1 | 2µs | 1; | ||
307 | __END__ |