From b0914d58f557c5e7cd840396ca4af8b7ae6622a6 Mon Sep 17 00:00:00 2001 From: jar3b Date: Sat, 12 Mar 2016 18:24:02 +0300 Subject: [PATCH 01/10] Add bottle class proto (and simple class method routing) --- aore/miscutils/bottlecl.py | 24 +++++++++++++++ aore/phias.py | 62 ++++++++++++++++++++------------------ passenger_wsgi.py | 31 ++++++++++--------- 3 files changed, 73 insertions(+), 44 deletions(-) create mode 100644 aore/miscutils/bottlecl.py diff --git a/aore/miscutils/bottlecl.py b/aore/miscutils/bottlecl.py new file mode 100644 index 0000000..210d8b5 --- /dev/null +++ b/aore/miscutils/bottlecl.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- + +from bottle import Bottle + + +class BottleCL(object): + def __init__(self): + self._app = Bottle() + self.init_routes() + + def init_routes(self): + pass + + def add_route(self, route_path, handler): + self._app.route(route_path, callback=handler) + + def add_error(self, error_code, handler): + if not self._app.error_handler: + self._app.error_handler = {error_code: handler} + else: + self._app.error_handler[error_code] = handler + + def start(self, host, port): + self._app.run(host=host, port=port) diff --git a/aore/phias.py b/aore/phias.py index 0e0d80c..6b2bd77 100644 --- a/aore/phias.py +++ b/aore/phias.py @@ -2,44 +2,48 @@ import json import logging -from bottle import Bottle, response - -from aore.search.fiasfactory import FiasFactory - -logging.basicConfig(format='%(asctime)s %(message)s', level=logging.INFO) -app = Bottle() -fias_factory = FiasFactory() +from bottle import response +from miscutils.bottlecl import BottleCL -@app.route(r'/expand/') -def expand(aoid): - response.content_type = 'application/json' - response.set_header('Access-Control-Allow-Origin', '*') +class App(BottleCL): + def __init__(self, config): + super(App, self).__init__() + logging.basicConfig(format='%(asctime)s %(message)s', level=logging.INFO) - return json.dumps(fias_factory.expand(aoid)) + self._config = config + # self._factory = FiasFactory() + def init_routes(self): + self.add_route(r'/expand/', self.__expand) + self.add_route(r'/normalize/', self.__normalize) + self.add_route(r'/find/', self.__find) + self.add_route(r'/find//', self.__find) + self.add_error(404, self.basic_error_handler) + self.add_error(500, self.basic_error_handler) -@app.route(r'/normalize/') -def normalize(aoid): - response.content_type = 'application/json' - response.set_header('Access-Control-Allow-Origin', '*') + def __expand(self, aoid): + response.content_type = 'application/json' + response.set_header('Access-Control-Allow-Origin', '*') - return json.dumps(fias_factory.normalize(aoid)) + return json.dumps(self._factory.expand(aoid)) + def __normalize(self, aoid): + response.content_type = 'application/json' + response.set_header('Access-Control-Allow-Origin', '*') -@app.route(r'/find/') -@app.route(r'/find//') -def find(text, strong=False): - strong = (strong == "strong") - response.content_type = 'application/json' - response.set_header('Access-Control-Allow-Origin', '*') + return json.dumps(self._factory.normalize(aoid)) - return json.dumps(fias_factory.find(text, strong)) + def __find(self, text, strong=False): + strong = (strong == "strong") + response.content_type = 'application/json' + response.set_header('Access-Control-Allow-Origin', '*') + return json.dumps(self._factory.find(text, strong)) -@app.error(404) -def error404(error): - response.content_type = 'application/json' - response.set_header('Access-Control-Allow-Origin', '*') + @staticmethod + def basic_error_handler(error): + response.content_type = 'application/json' + response.set_header('Access-Control-Allow-Origin', '*') - return json.dumps(dict(error="Page not found")) + return json.dumps(dict(error=error.status)) diff --git a/passenger_wsgi.py b/passenger_wsgi.py index 1f5a062..c0845cb 100644 --- a/passenger_wsgi.py +++ b/passenger_wsgi.py @@ -1,25 +1,26 @@ # -*- coding: utf-8 -*- from aore import phias, config +from bottle import run # Config section -config.sphinx_conf.listen = "192.168.0.37:9312" -config.sphinx_conf.var_dir = "C:\\Sphinx" - -config.db_conf.database = "pyfias" -config.db_conf.host = "192.168.0.37" -config.db_conf.port = 5432 -config.db_conf.user = "postgres" -config.db_conf.password = "intercon" - -config.unrar_config.path = "C:\\Program Files (x86)\\WinRAR\\unrar.exe" -config.folders.temp = "E:\\!TEMP" - -config.basic.logging = True +# config.sphinx_conf.listen = "192.168.0.37:9312" +# config.sphinx_conf.var_dir = "C:\\Sphinx" +# +# config.db_conf.database = "pyfias" +# config.db_conf.host = "192.168.0.37" +# config.db_conf.port = 5432 +# config.db_conf.user = "postgres" +# config.db_conf.password = "intercon" +# +# config.unrar_config.path = "C:\\Program Files (x86)\\WinRAR\\unrar.exe" +# config.folders.temp = "E:\\!TEMP" +# +# config.basic.logging = True # Define main app -application = phias.app +application = phias.App('test-config-fname') # Run bottle WSGI server if no external if __name__ == '__main__': - application.run(host='0.0.0.0', port=8087, debug=True) + application.start('0.0.0.0', 8087) From 19d80f5811ce4eb5878915a4735494fb37af6c94 Mon Sep 17 00:00:00 2001 From: jar3b Date: Sat, 12 Mar 2016 19:54:24 +0300 Subject: [PATCH 02/10] Small config rework. Add .gitignore --- .gitignore | 1 + aore/phias.py | 7 ++++--- config.example.py | 17 +++++++++++++++++ passenger_wsgi.py | 24 +++++++----------------- 4 files changed, 29 insertions(+), 20 deletions(-) create mode 100644 .gitignore create mode 100644 config.example.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f85c6b1 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +config.py \ No newline at end of file diff --git a/aore/phias.py b/aore/phias.py index 6b2bd77..eda1d97 100644 --- a/aore/phias.py +++ b/aore/phias.py @@ -3,16 +3,17 @@ import json import logging from bottle import response + +from aore.search.fiasfactory import FiasFactory from miscutils.bottlecl import BottleCL class App(BottleCL): - def __init__(self, config): + def __init__(self): super(App, self).__init__() logging.basicConfig(format='%(asctime)s %(message)s', level=logging.INFO) - self._config = config - # self._factory = FiasFactory() + self._factory = FiasFactory() def init_routes(self): self.add_route(r'/expand/', self.__expand) diff --git a/config.example.py b/config.example.py new file mode 100644 index 0000000..8f32d5d --- /dev/null +++ b/config.example.py @@ -0,0 +1,17 @@ +# -*- coding: utf-8 -*- +from aore import config + +# Config section +config.sphinx_conf.listen = "192.168.0.37:9312" +config.sphinx_conf.var_dir = "C:\\Sphinx" + +config.db_conf.database = "pyfias" +config.db_conf.host = "192.168.0.37" +config.db_conf.port = 5432 +config.db_conf.user = "postgres" +config.db_conf.password = "intercon" + +config.unrar_config.path = "C:\\Program Files (x86)\\WinRAR\\unrar.exe" +config.folders.temp = "E:\\!TEMP" + +config.basic.logging = True diff --git a/passenger_wsgi.py b/passenger_wsgi.py index c0845cb..4dfa1b9 100644 --- a/passenger_wsgi.py +++ b/passenger_wsgi.py @@ -1,25 +1,15 @@ # -*- coding: utf-8 -*- -from aore import phias, config -from bottle import run +from aore import phias -# Config section -# config.sphinx_conf.listen = "192.168.0.37:9312" -# config.sphinx_conf.var_dir = "C:\\Sphinx" -# -# config.db_conf.database = "pyfias" -# config.db_conf.host = "192.168.0.37" -# config.db_conf.port = 5432 -# config.db_conf.user = "postgres" -# config.db_conf.password = "intercon" -# -# config.unrar_config.path = "C:\\Program Files (x86)\\WinRAR\\unrar.exe" -# config.folders.temp = "E:\\!TEMP" -# -# config.basic.logging = True +# Load config +try: + from config import * +except ImportError: + assert "No config" # Define main app -application = phias.App('test-config-fname') +application = phias.App() # Run bottle WSGI server if no external if __name__ == '__main__': From ad70ef77426ed99f90c90b6700a43b90993be92e Mon Sep 17 00:00:00 2001 From: jar3b Date: Sat, 12 Mar 2016 20:14:28 +0300 Subject: [PATCH 03/10] Fixes #7 --- aore/miscutils/bottlecl.py | 4 ++-- aore/phias.py | 4 ++-- aore/updater/updater.py | 20 ++++++++++++++++++-- config.example.py | 8 ++++---- passenger_wsgi.py | 4 ++-- 5 files changed, 28 insertions(+), 12 deletions(-) diff --git a/aore/miscutils/bottlecl.py b/aore/miscutils/bottlecl.py index 210d8b5..371338a 100644 --- a/aore/miscutils/bottlecl.py +++ b/aore/miscutils/bottlecl.py @@ -20,5 +20,5 @@ class BottleCL(object): else: self._app.error_handler[error_code] = handler - def start(self, host, port): - self._app.run(host=host, port=port) + def start(self, **kwargs): + self._app.run(**kwargs) diff --git a/aore/phias.py b/aore/phias.py index eda1d97..b195816 100644 --- a/aore/phias.py +++ b/aore/phias.py @@ -9,9 +9,9 @@ from miscutils.bottlecl import BottleCL class App(BottleCL): - def __init__(self): + def __init__(self, log_filename): super(App, self).__init__() - logging.basicConfig(format='%(asctime)s %(message)s', level=logging.INFO) + logging.basicConfig(format='%(asctime)s %(message)s', level=logging.INFO, filename=log_filename) self._factory = FiasFactory() diff --git a/aore/updater/updater.py b/aore/updater/updater.py index a4b12b0..0568819 100644 --- a/aore/updater/updater.py +++ b/aore/updater/updater.py @@ -55,11 +55,27 @@ class Updater: finally: db.close() + # Получает верию ФИАС с клавиатуры (если мы берем базу из папки или локального архива и не можем определить, + # что это за версия + @staticmethod + def __get_update_version_from_console(): + mode = None + while not mode: + try: + mode = int(raw_input('Enter FIAS update version (3 digit):')) + except ValueError: + print "Not a valid fias version, try again." + + return mode + def __get_updates_from_folder(self, foldername): # TODO: Вычислять версию, если берем данные из каталога - yield dict(intver=0, textver="Unknown", delta_url=foldername, complete_url=foldername) + yield dict(intver=self.__get_update_version_from_console(), + textver="Unknown", delta_url=foldername, + complete_url=foldername) - def __get_updates_from_rar(self, url): + @staticmethod + def __get_updates_from_rar(url): aorar = AoRar() if url.startswith("http://") or url.startswith("https://"): diff --git a/config.example.py b/config.example.py index 8f32d5d..cc0d46c 100644 --- a/config.example.py +++ b/config.example.py @@ -2,16 +2,16 @@ from aore import config # Config section -config.sphinx_conf.listen = "192.168.0.37:9312" +config.sphinx_conf.listen = "127.0.0.1:9312" config.sphinx_conf.var_dir = "C:\\Sphinx" config.db_conf.database = "pyfias" -config.db_conf.host = "192.168.0.37" +config.db_conf.host = "192.168.0.1" config.db_conf.port = 5432 config.db_conf.user = "postgres" -config.db_conf.password = "intercon" +config.db_conf.password = "postgres" -config.unrar_config.path = "C:\\Program Files (x86)\\WinRAR\\unrar.exe" +config.unrar_config.path = "C:\\Program Files\\WinRAR\\unrar.exe" config.folders.temp = "E:\\!TEMP" config.basic.logging = True diff --git a/passenger_wsgi.py b/passenger_wsgi.py index 4dfa1b9..48a287f 100644 --- a/passenger_wsgi.py +++ b/passenger_wsgi.py @@ -9,8 +9,8 @@ except ImportError: assert "No config" # Define main app -application = phias.App() +application = phias.App("pyphias.log") # Run bottle WSGI server if no external if __name__ == '__main__': - application.start('0.0.0.0', 8087) + application.start(host='0.0.0.0', port=8087, debug=True) From a5f4344ecb9a81cf961a4c04662a2dc7fb0e9029 Mon Sep 17 00:00:00 2001 From: jar3b Date: Sat, 12 Mar 2016 20:22:35 +0300 Subject: [PATCH 04/10] Fix manage.py config. Fix warning in xmlparser.py (null check) --- aore/updater/xmlparser.py | 2 +- manage.py | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/aore/updater/xmlparser.py b/aore/updater/xmlparser.py index a6bddb3..e33bb2f 100644 --- a/aore/updater/xmlparser.py +++ b/aore/updater/xmlparser.py @@ -15,7 +15,7 @@ class XMLParser: elem.clear() # Also eliminate now-empty references from the root node to elem for ancestor in elem.xpath('ancestor-or-self::*'): - while ancestor.getprevious(): + while ancestor.getprevious() is not None: del ancestor.getparent()[0] del context diff --git a/manage.py b/manage.py index 9a2c1f3..f17ef3f 100644 --- a/manage.py +++ b/manage.py @@ -7,6 +7,13 @@ from aore.miscutils.sphinx import SphinxHelper from aore.updater.soapreceiver import SoapReceiver from aore.updater.updater import Updater +# Load config +try: + from config import * +except ImportError: + assert "No config" + +# Initialize logging logging.basicConfig(format='%(asctime)s %(message)s', level=logging.INFO) From 0d9cfbfd3b08330a74939c1c13be641a41373c42 Mon Sep 17 00:00:00 2001 From: jar3b Date: Sat, 12 Mar 2016 21:00:30 +0300 Subject: [PATCH 05/10] Auto-create necessary folders for Sphinx. Mod Readme.md --- README.md | 12 ++++++------ aore/miscutils/sphinx.py | 8 ++++++++ 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index d8e19d5..538c068 100644 --- a/README.md +++ b/README.md @@ -135,8 +135,8 @@ _Внимание_! Только Python 2.7+ (на 3+ не тестировал) ## Настройка ### Первоначальная настройка базы данных 1. Настроим конфиг, для этого необходимо изменить параметры в Вашем wsgi-entrypoint (в моем случае _passenger_wsgi.py_): -прописать параметры доступа к базе, демону Sphinx и путь, куда будут сохраняться данные Sphinx; по этому пути -дополнительно необходимо создать 3 папки: log, run и data. Все доступные настройки можно увидеть в _aore/config/common.py_ +в строке `from config import *` измените _config_ на имя Вашего конфигурационного файла (создается рядом с wsgi app), +пример конфига находится в файле _config.example.py_. 2. Создадим базу: - из архива `sudo -u phias python manage.py -b create -s /tmp/fias_xml.rar` - из директории `sudo -u phias python manage.py -b create -s /tmp/fias_xml_unpacked` @@ -144,14 +144,14 @@ _Внимание_! Только Python 2.7+ (на 3+ не тестировал) - Также, можно указать конкретную версию ФИАС _только_ при http загрузке, с ключом `--update-version `, где num - номер версии ФИАС, все доступные версии можно получить, выполнив `manage.py -v`. - **Внимание**! Если Вы инициализируете БД из архива или директории, для последующего корректного обновления необходимо - прописать номер версии ФИАС, которую Вы только что установили, в базе приложения, таблица CONFIG + Внимание! Если Вы инициализируете БД из архива или директории, при создании или обновлении базы у Вас будет + запрошен номер устанавливаемой версии ФИАС. 3. Проиндексируем Sphinx: - - Windows: `python manage.py -c -i C://sphinx//indexer.exe -o C://sphinx//sphinx.conf` + - Windows: `python manage.py -c -i C:\sphinx\bin\indexer.exe -o C:\sphinx\sphinx.conf` - Debian: `sudo python manage.py -c -i indexer -o /usr/local/sphinx/etc/sphinx.conf` 4. Затем запустим searchd: - Windows: - - Устанавливаем службу: `C:\Sphinx\bin\searchd --install --config C:\Sphinx\sphinx.conf.in --servicename sphinxsearch` + - Устанавливаем службу: `C:\Sphinx\bin\searchd --install --config C:\Sphinx\sphinx.conf --servicename sphinxsearch` - и запускаем: `net start sphinxsearch` - Debian: `sudo searchd --config /usr/local/sphinx/etc/sphinx.conf` 5. Настроим WSGI server, я использую nginx + passenger (см. файл passenger_wsgi.py). Вы можете использовать любое diff --git a/aore/miscutils/sphinx.py b/aore/miscutils/sphinx.py index 04a66e7..011dd82 100644 --- a/aore/miscutils/sphinx.py +++ b/aore/miscutils/sphinx.py @@ -21,6 +21,14 @@ class SphinxHelper: if not os.path.exists(folders.temp): os.makedirs(folders.temp) + # оздаем 3 папки для Сфинкса + if not os.path.exists(sphinx_conf.var_dir+'/run'): + os.makedirs(sphinx_conf.var_dir+'/run') + if not os.path.exists(sphinx_conf.var_dir+'/log'): + os.makedirs(sphinx_conf.var_dir+'/log') + if not os.path.exists(sphinx_conf.var_dir+'/data'): + os.makedirs(sphinx_conf.var_dir+'/data') + def configure_indexer(self, indexer_binary, config_filename): logging.info("Start configuring Sphinx...") self.index_binary = indexer_binary From 7462e50e3710fa05feed5149c8e64a573e4fb726 Mon Sep 17 00:00:00 2001 From: jar3b Date: Sat, 12 Mar 2016 21:09:55 +0300 Subject: [PATCH 06/10] Add logging for sphinx query time (if logging == True in config) --- .gitignore | 1 + aore/phias.py | 2 +- aore/search/search.py | 5 +++-- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index f85c6b1..95149c7 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ +*.log config.py \ No newline at end of file diff --git a/aore/phias.py b/aore/phias.py index b195816..d6b7f6d 100644 --- a/aore/phias.py +++ b/aore/phias.py @@ -11,7 +11,7 @@ from miscutils.bottlecl import BottleCL class App(BottleCL): def __init__(self, log_filename): super(App, self).__init__() - logging.basicConfig(format='%(asctime)s %(message)s', level=logging.INFO, filename=log_filename) + logging.basicConfig(format='%(asctime)s %(message)s', level=logging.DEBUG, filename=log_filename) self._factory = FiasFactory() diff --git a/aore/search/search.py b/aore/search/search.py index 6083868..ec69a0f 100644 --- a/aore/search/search.py +++ b/aore/search/search.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +import logging import re import time @@ -7,9 +8,9 @@ import sphinxapi from aore.config import basic from aore.config import sphinx_conf +from aore.miscutils.trigram import trigram from aore.search.wordentry import WordEntry from aore.search.wordvariation import VariationType -from aore.miscutils.trigram import trigram class SphinxSearch: @@ -140,7 +141,7 @@ class SphinxSearch: elapsed_t = time.time() - start_t if basic.logging: - print(elapsed_t) + logging.info("Sphinx time for {} = {}".format(text, elapsed_t)) results = [] parsed_ids = [] From f03cd63faa57577c17de0727717237c94b9e788e Mon Sep 17 00:00:00 2001 From: jar3b Date: Sun, 13 Mar 2016 01:44:22 +0300 Subject: [PATCH 07/10] Fix program won't exit if rar file doesn't exists. Mod readme.md. --- README.md | 18 +++++++++++------- aore/updater/aorar.py | 1 + 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 538c068..461df47 100644 --- a/README.md +++ b/README.md @@ -134,9 +134,10 @@ _Внимание_! Только Python 2.7+ (на 3+ не тестировал) ## Настройка ### Первоначальная настройка базы данных -1. Настроим конфиг, для этого необходимо изменить параметры в Вашем wsgi-entrypoint (в моем случае _passenger_wsgi.py_): -в строке `from config import *` измените _config_ на имя Вашего конфигурационного файла (создается рядом с wsgi app), -пример конфига находится в файле _config.example.py_. +1. Настроим конфиг, для этого необходимо изменить параметры в Вашем wsgi-entrypoint (в моем случае +[passenger_wsgi.py](passenger_wsgi.py)): в строке `from config import *` измените _config_ на имя Вашего +конфигурационного файла (создается рядом с wsgi app), пример конфига находится в файле +[config.example.py](config.example.py). 2. Создадим базу: - из архива `sudo -u phias python manage.py -b create -s /tmp/fias_xml.rar` - из директории `sudo -u phias python manage.py -b create -s /tmp/fias_xml_unpacked` @@ -144,8 +145,11 @@ _Внимание_! Только Python 2.7+ (на 3+ не тестировал) - Также, можно указать конкретную версию ФИАС _только_ при http загрузке, с ключом `--update-version `, где num - номер версии ФИАС, все доступные версии можно получить, выполнив `manage.py -v`. - Внимание! Если Вы инициализируете БД из архива или директории, при создании или обновлении базы у Вас будет - запрошен номер устанавливаемой версии ФИАС. + Примечание 1: Если Вы инициализируете БД из архива или директории, при создании или обновлении базы у Вас будет + запрошен номер устанавливаемой версии ФИАС. + + Примечание 2: У пользователя PostgreSql (postgres, либо созданного Вами) должны быть права на чтение из директории, + указанной в `config.folders.temp`, иначе будет Permission denied при попытке bulk-import. 3. Проиндексируем Sphinx: - Windows: `python manage.py -c -i C:\sphinx\bin\indexer.exe -o C:\sphinx\sphinx.conf` - Debian: `sudo python manage.py -c -i indexer -o /usr/local/sphinx/etc/sphinx.conf` @@ -154,5 +158,5 @@ _Внимание_! Только Python 2.7+ (на 3+ не тестировал) - Устанавливаем службу: `C:\Sphinx\bin\searchd --install --config C:\Sphinx\sphinx.conf --servicename sphinxsearch` - и запускаем: `net start sphinxsearch` - Debian: `sudo searchd --config /usr/local/sphinx/etc/sphinx.conf` -5. Настроим WSGI server, я использую nginx + passenger (см. файл passenger_wsgi.py). Вы можете использовать любое -приемлемое сочетание. \ No newline at end of file +5. Настроим WSGI server, я использую nginx + passenger (см. файл [passenger_wsgi.py](passenger_wsgi.py)). Вы можете +использовать любое приемлемое сочетание. \ No newline at end of file diff --git a/aore/updater/aorar.py b/aore/updater/aorar.py index a028024..b8b095d 100644 --- a/aore/updater/aorar.py +++ b/aore/updater/aorar.py @@ -61,3 +61,4 @@ class AoRar: logging.warning("Cannot delete %s, do it manually", self.fname) else: logging.error("No file specified or not exists") + raise FiasException("No DB archive specified.") From 7076a13c534000d54392347c586a039202fe8c38 Mon Sep 17 00:00:00 2001 From: jar3b Date: Mon, 14 Mar 2016 12:52:03 +0300 Subject: [PATCH 08/10] Refactor: using short import names --- aore/search/fiasfactory.py | 2 +- aore/search/search.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/aore/search/fiasfactory.py b/aore/search/fiasfactory.py index 175620e..a5445fc 100644 --- a/aore/search/fiasfactory.py +++ b/aore/search/fiasfactory.py @@ -8,7 +8,7 @@ from bottle import template from aore.config import db_conf from aore.dbutils.dbimpl import DBImpl -from aore.search.search import SphinxSearch +from search import SphinxSearch class FiasFactory: diff --git a/aore/search/search.py b/aore/search/search.py index ec69a0f..0bc8aa6 100644 --- a/aore/search/search.py +++ b/aore/search/search.py @@ -9,8 +9,8 @@ import sphinxapi from aore.config import basic from aore.config import sphinx_conf from aore.miscutils.trigram import trigram -from aore.search.wordentry import WordEntry -from aore.search.wordvariation import VariationType +from wordentry import WordEntry +from wordvariation import VariationType class SphinxSearch: From 5e310df926698daacfaba043b79a2ebb3a1c6852 Mon Sep 17 00:00:00 2001 From: jar3b Date: Mon, 14 Mar 2016 13:21:45 +0300 Subject: [PATCH 09/10] Added error logging on HTTP requests (if config.basic.logging is enabled) --- aore/search/fiasfactory.py | 10 +++++++++- aore/search/search.py | 4 ++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/aore/search/fiasfactory.py b/aore/search/fiasfactory.py index a5445fc..0f63c89 100644 --- a/aore/search/fiasfactory.py +++ b/aore/search/fiasfactory.py @@ -1,12 +1,14 @@ # -*- coding: utf-8 -*- +import logging import re import urllib from uuid import UUID import psycopg2 +import traceback from bottle import template -from aore.config import db_conf +from aore.config import db_conf, basic from aore.dbutils.dbimpl import DBImpl from search import SphinxSearch @@ -54,6 +56,8 @@ class FiasFactory: results = self.searcher.find(text, strong) except Exception, err: + if basic.logging: + logging.error(traceback.format_exc(err)) return dict(error=err.args[0]) return results @@ -66,6 +70,8 @@ class FiasFactory: sql_query = self.normalize_templ.replace("//aoid", aoid_guid) rows = self.db.get_rows(sql_query, True) except Exception, err: + if basic.logging: + logging.error(traceback.format_exc(err)) return dict(error=err.args[0]) if len(rows) == 0: @@ -85,6 +91,8 @@ class FiasFactory: sql_query = self.expand_templ.replace("//aoid", normalized_id) rows = self.db.get_rows(sql_query, True) except Exception, err: + if basic.logging: + logging.error(traceback.format_exc(err)) return dict(error=err.args[0]) return rows diff --git a/aore/search/search.py b/aore/search/search.py index 0bc8aa6..fe596c7 100644 --- a/aore/search/search.py +++ b/aore/search/search.py @@ -8,6 +8,7 @@ import sphinxapi from aore.config import basic from aore.config import sphinx_conf +from aore.miscutils.exceptions import FiasException from aore.miscutils.trigram import trigram from wordentry import WordEntry from wordvariation import VariationType @@ -143,6 +144,9 @@ class SphinxSearch: if basic.logging: logging.info("Sphinx time for {} = {}".format(text, elapsed_t)) + if rs is None: + raise FiasException("Cannot find sentence.") + results = [] parsed_ids = [] From 8d0a54618ea151f9d0abc20c12fcc9d465463358 Mon Sep 17 00:00:00 2001 From: jar3b Date: Mon, 14 Mar 2016 17:39:43 +0300 Subject: [PATCH 10/10] Small refactoring (static method and comments) --- aore/search/fiasfactory.py | 3 ++- aore/search/search.py | 10 +++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/aore/search/fiasfactory.py b/aore/search/fiasfactory.py index 0f63c89..b8cdae8 100644 --- a/aore/search/fiasfactory.py +++ b/aore/search/fiasfactory.py @@ -21,7 +21,8 @@ class FiasFactory: self.normalize_templ = template('aore/templates/postgre/normalize_query.sql', aoid="//aoid") # Проверка, что строка является действительым UUID v4 - def __check_uuid(self, guid): + @staticmethod + def __check_uuid(guid): try: UUID(guid) except ValueError: diff --git a/aore/search/search.py b/aore/search/search.py index fe596c7..f54535b 100644 --- a/aore/search/search.py +++ b/aore/search/search.py @@ -30,15 +30,19 @@ class SphinxSearch: sphinx_host = sphinx_conf.listen sphinx_port = None + + # Получаем строку подключения для Sphinx if ":" in sphinx_conf.listen and "unix:/" not in sphinx_conf.listen: sphinx_host, sphinx_port = sphinx_conf.listen.split(":") sphinx_port = int(sphinx_port) + # Настраиваем подключение для подсказок self.client_sugg = sphinxapi.SphinxClient() self.client_sugg.SetServer(sphinx_host, sphinx_port) self.client_sugg.SetLimits(0, self.max_result) self.client_sugg.SetConnectTimeout(3.0) + # Настраиваем подключение для поиска адреса self.client_show = sphinxapi.SphinxClient() self.client_show.SetServer(sphinx_host, sphinx_port) self.client_show.SetLimits(0, self.max_result) @@ -141,12 +145,12 @@ class SphinxSearch: rs = self.client_show.RunQueries() elapsed_t = time.time() - start_t - if basic.logging: - logging.info("Sphinx time for {} = {}".format(text, elapsed_t)) - if rs is None: raise FiasException("Cannot find sentence.") + if basic.logging: + logging.info("Sphinx time for {} = {}".format(text, elapsed_t)) + results = [] parsed_ids = []