1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package no.sesat.search.run;
20
21
22 import java.util.ArrayList;
23 import java.util.Collection;
24 import java.util.Collections;
25 import java.util.HashMap;
26 import java.util.List;
27 import java.util.Map;
28 import java.util.Map.Entry;
29 import java.util.Properties;
30 import java.util.concurrent.ExecutionException;
31 import java.util.concurrent.Future;
32 import java.util.concurrent.TimeoutException;
33 import javax.xml.parsers.DocumentBuilder;
34 import no.schibstedsok.commons.ioc.BaseContext;
35 import no.schibstedsok.commons.ioc.ContextWrapper;
36 import no.sesat.search.InfrastructureException;
37 import no.sesat.search.datamodel.DataModel;
38 import no.sesat.search.datamodel.DataModelFactory;
39 import no.sesat.search.datamodel.access.ControlLevel;
40 import no.sesat.search.datamodel.generic.DataObject;
41 import no.sesat.search.datamodel.generic.MapDataObject;
42 import no.sesat.search.datamodel.generic.MapDataObjectSupport;
43 import no.sesat.search.datamodel.generic.StringDataObject;
44 import no.sesat.search.datamodel.navigation.NavigationDataObject;
45 import no.sesat.search.datamodel.query.QueryDataObject;
46 import no.sesat.search.query.analyser.AnalysisRule;
47 import no.sesat.search.query.analyser.AnalysisRuleFactory;
48 import no.sesat.search.query.QueryStringContext;
49 import no.sesat.search.query.token.TokenEvaluationEngine;
50 import no.sesat.search.query.token.TokenEvaluationEngineImpl;
51 import no.sesat.search.mode.command.SearchCommand;
52 import no.sesat.search.mode.SearchCommandFactory;
53 import no.sesat.search.mode.config.SearchConfiguration;
54 import no.sesat.search.mode.executor.SearchCommandExecutor;
55 import no.sesat.search.mode.executor.SearchCommandExecutorFactory;
56 import no.sesat.search.query.parser.AbstractQueryParserContext;
57 import no.sesat.search.query.Query;
58 import no.sesat.search.query.parser.QueryParser;
59 import no.sesat.search.query.parser.QueryParserImpl;
60 import no.sesat.search.query.token.TokenEvaluationEngineContext;
61 import no.sesat.search.result.NavigationItem;
62 import no.sesat.search.result.ResultItem;
63 import no.sesat.search.result.ResultList;
64 import no.sesat.search.run.handler.NavigationRunHandlerConfig;
65 import no.sesat.search.run.handler.RunHandler;
66 import no.sesat.search.run.handler.RunHandlerConfig;
67 import no.sesat.search.run.handler.RunHandlerFactory;
68 import no.sesat.search.run.transform.RunTransformer;
69 import no.sesat.search.run.transform.RunTransformerConfig;
70 import no.sesat.search.run.transform.RunTransformerFactory;
71 import no.sesat.search.site.Site;
72 import no.sesat.search.site.SiteContext;
73 import no.sesat.search.site.SiteKeyedFactoryInstantiationException;
74 import no.sesat.search.site.config.BytecodeLoader;
75 import no.sesat.search.site.config.DocumentLoader;
76 import no.sesat.search.site.config.PropertiesLoader;
77 import no.sesat.search.view.config.SearchTab.EnrichmentHint;
78 import org.apache.log4j.Level;
79 import org.apache.log4j.Logger;
80
81
82
83
84
85
86
87
88
89 public class RunningQueryImpl extends AbstractRunningQuery implements RunningQuery {
90
91
92
93 private static final int TIMEOUT = Logger.getRootLogger().getLevel().isGreaterOrEqual(Level.INFO)
94 ? 10000
95 : Integer.MAX_VALUE;
96
97
98 public static final String PARAM_LAYOUT = "layout";
99
100 private static final String PARAM_COMMANDS = "commands";
101
102 private static final String PARAM_WAITFOR = "waitFor";
103
104 private static final String PARAM_OUTPUT = "output";
105
106 private static final Logger LOG = Logger.getLogger(RunningQueryImpl.class);
107 private static final Logger ANALYSIS_LOG = Logger.getLogger("no.sesat.search.analyzer.Analysis");
108 private static final Logger PRODUCT_LOG = Logger.getLogger("no.sesat.Product");
109
110 private static final String ERR_RUN_QUERY = "Failure to run query";
111 private static final String ERR_EXECUTION_ERROR = "Failure in a search command.";
112 private static final String ERR_MODE_TIMEOUT = "Timeout running all search commands.";
113 private static final String INFO_COMMAND_COUNT = "Commands to invoke ";
114
115
116
117 private final AnalysisRuleFactory rules;
118
119
120
121 protected boolean allCancelled = false;
122
123 protected final DataModel datamodel;
124
125 protected final TokenEvaluationEngine engine;
126 private final Map<String,Integer> hits = new HashMap<String,Integer>();
127 private final Map<String,Integer> scores = new HashMap<String,Integer>();
128 private final Map<String,Integer> scoresByRule = new HashMap<String,Integer>();
129
130
131
132
133
134
135
136
137
138
139
140
141 public RunningQueryImpl(
142 final Context cxt,
143 final String query) throws SiteKeyedFactoryInstantiationException {
144
145 super(cxt);
146 this.datamodel = cxt.getDataModel();
147
148 LOG.trace("RunningQuery(cxt," + query + ')');
149
150 final String queryStr = trimDuplicateSpaces(query);
151
152 final SiteContext siteCxt = new SiteContext(){
153 public Site getSite() {
154 return datamodel.getSite().getSite();
155 }
156 };
157
158 final TokenEvaluationEngine.Context tokenEvalFactoryCxt =
159 ContextWrapper.wrap(
160 TokenEvaluationEngine.Context.class,
161 context,
162 new QueryStringContext() {
163 public String getQueryString() {
164 return queryStr;
165 }
166 },
167 siteCxt);
168
169
170
171 engine = new TokenEvaluationEngineImpl(tokenEvalFactoryCxt);
172
173
174 final QueryParser parser = new QueryParserImpl(new AbstractQueryParserContext() {
175 public TokenEvaluationEngine getTokenEvaluationEngine() {
176 return engine;
177 }
178 });
179
180 final DataModelFactory factory
181 = DataModelFactory.instanceOf(ContextWrapper.wrap(DataModelFactory.Context.class, cxt, siteCxt));
182
183 final QueryDataObject queryDO = factory.instantiate(
184 QueryDataObject.class,
185 datamodel,
186 new DataObject.Property("string", queryStr),
187 new DataObject.Property("query", parser.getQuery()));
188
189 final MapDataObject<NavigationItem> navigations
190 = new MapDataObjectSupport<NavigationItem>(Collections.<String, NavigationItem>emptyMap());
191 final NavigationDataObject navDO = factory.instantiate(
192 NavigationDataObject.class,
193 datamodel,
194 new DataObject.Property("configuration", context.getSearchTab().getNavigationConfiguration()),
195 new DataObject.Property("navigation",navigations),
196 new DataObject.Property("navigations", navigations));
197
198 datamodel.setQuery(queryDO);
199 datamodel.setNavigation(navDO);
200
201 rules = AnalysisRuleFactory.instanceOf(ContextWrapper.wrap(AnalysisRuleFactory.Context.class, context, siteCxt));
202
203 }
204
205
206
207
208
209
210
211
212
213 public void run() throws InterruptedException {
214
215 LOG.debug("run()");
216 final StringBuilder analysisReport
217 = new StringBuilder(" <analyse><query>" + datamodel.getQuery().getXmlEscaped() + "</query>\n");
218
219 final Map<String,StringDataObject> parameters = datamodel.getParameters().getValues();
220
221 try {
222
223 final DataModelFactory dataModelFactory = DataModelFactory
224 .instanceOf(ContextWrapper.wrap(DataModelFactory.Context.class, context, new SiteContext(){
225 public Site getSite(){
226 return datamodel.getSite().getSite();
227 }
228 }));
229
230
231
232 dataModelFactory.assignControlLevel(datamodel, ControlLevel.SEARCH_COMMAND_CONSTRUCTION);
233
234 final Collection<SearchCommand> commands = new ArrayList<SearchCommand>();
235
236 final SearchCommandFactory.Context scfContext = new SearchCommandFactory.Context() {
237 public Site getSite() {
238 return context.getDataModel().getSite().getSite();
239 }
240 public BytecodeLoader newBytecodeLoader(final SiteContext site, final String name, final String jar) {
241 return context.newBytecodeLoader(site, name, jar);
242 }
243 };
244
245 final SearchCommandFactory searchCommandFactory = new SearchCommandFactory(scfContext);
246
247 for (SearchConfiguration searchConfiguration : applicableSearchConfigurations()) {
248
249 final SearchConfiguration config = searchConfiguration;
250 final String confName = config.getId();
251
252 try{
253
254 hits.put(confName, Integer.valueOf(0));
255
256 final SearchCommand.Context searchCmdCxt = ContextWrapper.wrap(
257 SearchCommand.Context.class,
258 context,
259 new BaseContext() {
260 public SearchConfiguration getSearchConfiguration() {
261 return config;
262 }
263 public RunningQuery getRunningQuery() {
264 return RunningQueryImpl.this;
265 }
266 public Query getQuery() {
267 return datamodel.getQuery().getQuery();
268 }
269 public TokenEvaluationEngine getTokenEvaluationEngine() {
270 return engine;
271 }
272 }
273 );
274
275 final EnrichmentHint eHint = context.getSearchTab().getEnrichmentByCommand(confName);
276 if (eHint != null && !datamodel.getQuery().getQuery().isBlank()) {
277
278
279 if(useEnrichment(eHint, config, searchCmdCxt, analysisReport)){
280 commands.add(searchCommandFactory.getController(searchCmdCxt));
281 }
282
283 }else{
284
285
286 commands.add(searchCommandFactory.getController(searchCmdCxt));
287 }
288 }catch(RuntimeException re){
289 LOG.error("Failed to add command " + confName, re);
290 }
291 }
292 ANALYSIS_LOG.info(analysisReport.toString() + " </analyse>");
293
294 LOG.info(INFO_COMMAND_COUNT + commands.size());
295
296
297 allCancelled = commands.size() > 0;
298 boolean hitsToShow = false;
299
300
301
302
303 dataModelFactory.assignControlLevel(datamodel, ControlLevel.SEARCH_COMMAND_EXECUTION);
304
305 final Map<Future<ResultList<? extends ResultItem>>,SearchCommand> results
306 = executeSearchCommands(commands);
307
308
309
310 dataModelFactory.assignControlLevel(datamodel, ControlLevel.RUNNING_QUERY_RESULT_HANDLING);
311
312 if( !allCancelled ){
313
314 final StringBuilder noHitsOutput = new StringBuilder();
315
316 for (Future<ResultList<? extends ResultItem>> task : results.keySet()) {
317
318 if (task.isDone() && !task.isCancelled()) {
319
320 try{
321 final ResultList<? extends ResultItem> searchResult = task.get();
322 if (searchResult != null) {
323
324
325 final SearchCommand command = results.get(task);
326 final SearchConfiguration config = command.getSearchConfiguration();
327
328 final String name = config.getId();
329 final EnrichmentHint eHint = context.getSearchTab().getEnrichmentByCommand(name);
330
331 final float score = scores.get(name) != null
332 ? scores.get(name) * eHint.getWeight()
333 : 0;
334
335
336 hitsToShow |= searchResult.getHitCount() > 0;
337 hits.put(name, searchResult.getHitCount());
338
339 if( searchResult.getHitCount() <= 0 && command.isPaginated() ){
340 noHitsOutput.append("<command id=\"" + config.getId()
341 + "\" name=\"" + config.getStatisticalName()
342 + "\" type=\"" + config.getClass().getSimpleName()
343 + "\"/>");
344 }
345
346
347 if(eHint != null && searchResult.getHitCount() > 0 && score >= eHint.getThreshold()) {
348
349 searchResult.addField(EnrichmentHint.NAME_KEY, name);
350 searchResult.addObjectField(EnrichmentHint.SCORE_KEY, score);
351 searchResult.addObjectField(EnrichmentHint.HINT_KEY, eHint);
352 for(Map.Entry<String,String> property : eHint.getProperties().entrySet()){
353 searchResult.addObjectField(property.getKey(), property.getValue());
354 }
355 }
356 }
357 }catch(ExecutionException ee){
358 LOG.error(ERR_EXECUTION_ERROR, ee);
359 }
360 }
361 }
362
363 performHandlers();
364
365 if (!hitsToShow) {
366 handleNoHits(noHitsOutput, parameters);
367 }
368 }
369
370 } catch (Exception e) {
371 LOG.error(ERR_RUN_QUERY, e);
372 throw new InfrastructureException(e);
373
374 }
375 }
376
377
378
379
380
381
382
383
384
385 protected Map<String,Integer> getHits(){
386 return Collections.unmodifiableMap(hits);
387 }
388
389
390
391
392
393 protected Collection<SearchConfiguration> applicableSearchConfigurations(){
394
395 final Collection<SearchConfiguration> applicableSearchConfigurations = new ArrayList<SearchConfiguration>();
396
397 final String[] explicitCommands = null != datamodel.getParameters().getValue(PARAM_COMMANDS)
398 ? datamodel.getParameters().getValue(PARAM_COMMANDS).getString().split(",")
399 : new String[0];
400
401 for (SearchConfiguration conf : context.getSearchMode().getSearchConfigurations()) {
402
403
404 boolean applicable = (0 == explicitCommands.length);
405
406
407 for(String explicitCommand : explicitCommands){
408 applicable |= explicitCommand.equalsIgnoreCase(conf.getId());
409 }
410
411
412 applicable &= !isRss() || context.getSearchTab().getRssResultName().equals(conf.getId());
413
414
415 applicable &= conf.isAlwaysRun() ||
416 (null != context.getSearchTab().getEnrichmentByCommand(conf.getId())
417 && !datamodel.getQuery().getQuery().isBlank());
418
419
420 if(applicable){
421 applicableSearchConfigurations.add(conf);
422 }
423 }
424
425 return performTransformers(applicableSearchConfigurations);
426 }
427
428
429
430 private boolean useEnrichment(
431 final EnrichmentHint eHint,
432 final SearchConfiguration config,
433 final TokenEvaluationEngineContext tokenEvaluationEngineContext,
434 final StringBuilder analysisReport){
435
436 boolean result = false;
437
438 final Map<String,StringDataObject> parameters = datamodel.getParameters().getValues();
439
440
441 final boolean collapse = null == parameters.get("collapse")
442 || "".equals(parameters.get("collapse").getString());
443
444 if (context.getSearchMode().isAnalysis() && collapse && eHint.getWeight() > 0){
445
446 int score = eHint.getBaseScore();
447
448 if(null != eHint.getRule()){
449
450 final AnalysisRule rule = rules.getRule(eHint.getRule());
451
452 if (null == scoresByRule.get(eHint.getRule())) {
453
454 final StringBuilder analysisRuleReport = new StringBuilder();
455
456 score += rule.evaluate(datamodel.getQuery().getQuery(),
457 ContextWrapper.wrap(
458 AnalysisRule.Context.class,
459 new BaseContext(){
460 public String getRuleName(){
461 return eHint.getRule();
462 }
463 public Appendable getReportBuffer(){
464 return analysisRuleReport;
465 }
466 },
467 tokenEvaluationEngineContext));
468
469 scoresByRule.put(eHint.getRule(), score);
470 analysisReport.append(analysisRuleReport);
471
472 LOG.debug("Score for " + config.getId() + " is " + score);
473
474 } else {
475 score = scoresByRule.get(eHint.getRule());
476 }
477 }
478
479 scores.put(config.getId(), score);
480
481 result = score >= eHint.getThreshold();
482
483 }
484
485 return config.isAlwaysRun() || result;
486 }
487
488 @SuppressWarnings("unchecked")
489 private Map<Future<ResultList<? extends ResultItem>>,SearchCommand> executeSearchCommands(
490 final Collection<SearchCommand> commands) throws InterruptedException, TimeoutException, ExecutionException{
491
492 Map<Future<ResultList<? extends ResultItem>>,SearchCommand> results = Collections.EMPTY_MAP;
493
494 try{
495 final SearchCommandExecutor executor = SearchCommandExecutorFactory
496 .getController(context.getSearchMode().getExecutor());
497
498 try{
499 results = executor.invokeAll(commands);
500
501 }finally{
502
503 final Map<Future<ResultList<? extends ResultItem>>,SearchCommand> waitFor;
504
505 if(null != datamodel.getParameters().getValue(PARAM_WAITFOR)){
506
507 waitFor = new HashMap<Future<ResultList<? extends ResultItem>>,SearchCommand>();
508
509 final String[] waitForArr
510 = datamodel.getParameters().getValue(PARAM_WAITFOR).getString().split(",");
511
512 for(String waitForStr : waitForArr){
513
514 for(Entry
515 : results.entrySet()){
516
517 final String entryName
518 = ((SearchCommand)entry.getValue()).getSearchConfiguration().getId();
519 if(waitForStr.equalsIgnoreCase(entryName)){
520
521 waitFor.put(
522 (Future<ResultList<? extends ResultItem>>)entry.getKey(),
523 (SearchCommand)entry.getValue());
524 break;
525 }
526 }
527 }
528
529 }else if(null != datamodel.getParameters().getValue(PARAM_COMMANDS)){
530
531
532 waitFor = results;
533
534 }else{
535
536
537 waitFor = new HashMap<Future<ResultList<? extends ResultItem>>,SearchCommand>();
538
539 for(Entry
540 if(!((SearchCommand)entry.getValue()).getSearchConfiguration().isAsynchronous()){
541
542 waitFor.put(
543 (Future<ResultList<? extends ResultItem>>)entry.getKey(),
544 (SearchCommand)entry.getValue());
545 }
546 }
547 }
548 executor.waitForAll(waitFor, TIMEOUT);
549 }
550 }catch(TimeoutException te){
551 LOG.error(ERR_MODE_TIMEOUT + te.getMessage());
552 }
553
554
555 for(SearchCommand command : commands){
556 allCancelled &= (null != datamodel.getParameters().getValue(PARAM_COMMANDS)
557 || !command.getSearchConfiguration().isAsynchronous());
558 allCancelled &= command.isCancelled();
559 }
560
561 return results;
562 }
563
564 private Collection<SearchConfiguration> performTransformers(final Collection<SearchConfiguration> applicableSearchConfigurations) {
565 final RunTransformer.Context transformerContext = new RunTransformer.Context() {
566 public Collection<SearchConfiguration>getApplicableSearchConfigurations() {
567 return applicableSearchConfigurations;
568 }
569
570 public DataModel getDataModel() {
571 return datamodel;
572 }
573
574 public DocumentLoader newDocumentLoader(final SiteContext siteContext, final String resource, final DocumentBuilder builder) {
575 return context.newDocumentLoader(siteContext, resource, builder);
576 }
577
578 public PropertiesLoader newPropertiesLoader(final SiteContext siteContext, final String resource, final Properties properties) {
579 return context.newPropertiesLoader(siteContext, resource, properties);
580 }
581
582 public BytecodeLoader newBytecodeLoader(final SiteContext siteContext, final String className, final String jarFileName) {
583 return context.newBytecodeLoader(siteContext, className, jarFileName);
584 }
585
586 public Site getSite() {
587 return datamodel.getSite().getSite();
588 }
589 };
590
591 final List<RunTransformerConfig> rtcList = context.getSearchMode().getRunTransformers();
592
593 for (final RunTransformerConfig rtc : rtcList) {
594 final RunTransformer rt = RunTransformerFactory.getController(transformerContext, rtc);
595 rt.transform(transformerContext);
596 }
597
598 return applicableSearchConfigurations;
599 }
600
601 private void performHandlers(){
602
603 final RunHandler.Context handlerContext = ContextWrapper.wrap(
604 RunHandler.Context.class,
605 new SiteContext(){
606 public Site getSite() {
607 return datamodel.getSite().getSite();
608 }
609 },
610 context);
611
612 final List<RunHandlerConfig> rhcList
613 = new ArrayList<RunHandlerConfig>(context.getSearchMode().getRunHandlers());
614
615
616 rhcList.add(new NavigationRunHandlerConfig());
617
618 for (final RunHandlerConfig rhc : rhcList) {
619 final RunHandler rh = RunHandlerFactory.getController(handlerContext, rhc);
620 rh.handleRunningQuery(handlerContext);
621 }
622 }
623
624 private boolean isRss() {
625
626 final StringDataObject outputParam = datamodel.getParameters().getValue(PARAM_OUTPUT);
627 return null != outputParam && "rss".equals(outputParam.getString());
628 }
629
630 private void handleNoHits(final StringBuilder noHitsOutput, final Map<String,StringDataObject> parameters)
631 throws SiteKeyedFactoryInstantiationException, InterruptedException{
632
633
634 noHitsOutput.append("<absolute/>");
635
636 if( noHitsOutput.length() >0 && datamodel.getQuery().getString().length() >0
637 && !"NOCOUNT".equals(parameters.get("IGNORE"))){
638
639 final String output = null != parameters.get("output")
640 ? parameters.get("output").getString()
641 : null;
642
643 noHitsOutput.insert(0, "<no-hits mode=\"" + context.getSearchTab().getKey()
644 + (null != output ? "\" output=\"" + output : "") + "\">"
645 + "<query>" + datamodel.getQuery().getXmlEscaped() + "</query>");
646 noHitsOutput.append("</no-hits>");
647 PRODUCT_LOG.info(noHitsOutput.toString());
648 }
649
650
651
652
653 final String queryStr = datamodel.getQuery().getString();
654
655 if (!queryStr.startsWith("(") && !queryStr.endsWith(")")
656 && datamodel.getQuery().getQuery().getTermCount() > 1) {
657
658
659
660 final DataModelFactory dataModelFactory = DataModelFactory
661 .instanceOf(ContextWrapper.wrap(DataModelFactory.Context.class, context, new SiteContext(){
662 public Site getSite(){
663 return datamodel.getSite().getSite();
664 }
665 }));
666 dataModelFactory.assignControlLevel(datamodel, ControlLevel.RUNNING_QUERY_CONSTRUCTION);
667
668
669 new RunningQueryImpl(context, '(' + queryStr + ')').run();
670
671
672 }
673
674 }
675
676
677 }