--- mnogosearch-3.3.12.old/src/indexer.c +++ mnogosearch-3.3.12/src/indexer.c @@ -1924,11 +1924,14 @@ if (strncasecmp(UDM_NULL2EMPTY(Doc->CurURL.schema), "http", 4) && !UseRobotsTxtURL) return UDM_OK; - + if (!Doc->Spider.use_robots) { +#ifdef NOTDEF // Disable UdmLog call to speed things up UdmLog(Indexer,UDM_LOG_WARN, "robots.txt support is disallowed for '%s'", UDM_NULL2EMPTY(Doc->CurURL.hostinfo)); +#endif // Endif NOTDEF + UDM_GETLOCK(Indexer,UDM_LOCK_CONF); result= UdmRobotParse(Indexer, NULL, NULL, NULL, UDM_NULL2EMPTY(Doc->CurURL.hostinfo)); UDM_RELEASELOCK(Indexer,UDM_LOCK_CONF);