diff --git a/aore/__init__.py b/aore/__init__.py index 2a488d7..eeecfb6 100644 --- a/aore/__init__.py +++ b/aore/__init__.py @@ -2,6 +2,6 @@ import os import sys reload(sys) -cwd = os.getcwd() -sys.path.append(cwd) +cwd_dir = os.getcwd() +sys.path.append(cwd_dir) sys.setdefaultencoding("utf-8") diff --git a/aore/dbutils/dbimpl.py b/aore/dbutils/dbimpl.py index 3b6ed03..a977d0e 100644 --- a/aore/dbutils/dbimpl.py +++ b/aore/dbutils/dbimpl.py @@ -31,6 +31,7 @@ class DBImpl: cur = self.get_cursor() cur.execute(sql_query) cur.close() + del cur self.transaction_commit() except: self.transaction_rollback() @@ -47,6 +48,7 @@ class DBImpl: rows = cur.fetchall() cur.close() + del cur self.transaction_commit() except: self.transaction_rollback() diff --git a/aore/fias/fiasfactory.py b/aore/fias/fiasfactory.py index cc22055..1e14987 100644 --- a/aore/fias/fiasfactory.py +++ b/aore/fias/fiasfactory.py @@ -1,12 +1,13 @@ # -*- coding: utf-8 -*- +import re +from uuid import UUID + import psycopg2 from bottle import template -import sys + +from aore.config import db_conf from aore.dbutils.dbimpl import DBImpl from aore.fias.search import SphinxSearch -from aore.config import db_conf -from uuid import UUID -import re class FiasFactory: @@ -30,18 +31,20 @@ class FiasFactory: # rule - "boolean", "uuid", "text" def __check_param(self, param, rule): if rule == "boolean": - assert type(param) is bool, "Invalid parameter type" + assert isinstance(param, bool), "Invalid parameter type" if rule == "uuid": - assert (type(param) is str or type(param) is unicode) and self.__check_uuid(param), "Invalid parameter value" - if rule == "text": - assert type(param) is str or type(param) is unicode, "Invalid parameter type" + assert (isinstance(param, str) or isinstance(param, unicode)) and self.__check_uuid( + param), "Invalid parameter value" + if rule == "text": + assert isinstance(param, str) or isinstance(param, unicode), "Invalid parameter type" assert len(param) > 3, "Text too short" - pattern = re.compile("[A-za-zА-Яа-я \-,.#№]+") + pattern = re.compile(r"[A-za-zА-Яа-я \-,.#№]+") assert pattern.match(param), "Invalid parameter value" - # text - строка поиска - # strong - строгий поиск (True) или "мягкий" (False) (с допущением ошибок, опечаток) - # Строгий используется при импорте из внешних систем (автоматически), где ошибка критична + # text - строка поиска + # strong - строгий поиск (True) или "мягкий" (False) (с допущением ошибок, опечаток) + # Строгий используется при импорте из внешних систем (автоматически), где ошибка критична + def find(self, text, strong=False): try: self.__check_param(text, "text") diff --git a/aore/fias/search.py b/aore/fias/search.py index 9e89761..1b61a54 100644 --- a/aore/fias/search.py +++ b/aore/fias/search.py @@ -74,15 +74,12 @@ class SphinxSearch: maxleven = jaro_rating - jaro_rating * self.regression_coef if jaro_rating >= rating_limit and jaro_rating >= maxleven: outlist.append([match['attrs']['word'], jaro_rating]) + del jaro_rating outlist.sort(key=lambda x: x[1], reverse=True) return outlist - def __split_phrase(self, phrase): - phrase = unicode(phrase).replace('-', '').replace('@', '').lower() - return re.split(r"[ ,:.#$]+", phrase) - def __add_word_variations(self, word_entry, strong): if word_entry.MT_MANY_SUGG and not strong: suggs = self.__get_suggest(word_entry.word, self.rating_limit_soft, self.rating_limit_soft_count) @@ -111,7 +108,11 @@ class SphinxSearch: return we_list def find(self, text, strong): - words = self.__split_phrase(text) + def split_phrase(phrase): + phrase = unicode(phrase).replace('-', '').replace('@', '').lower() + return re.split(r"[ ,:.#$]+", phrase) + + words = split_phrase(text) word_entries = self.__get_word_entries(words, strong) word_count = len(word_entries) for x in range(word_count, max(0, word_count - 3), -1): diff --git a/aore/manage.py b/aore/manage.py index 820aceb..41044b3 100644 --- a/aore/manage.py +++ b/aore/manage.py @@ -1,9 +1,7 @@ # -*- coding: utf-8 -*- -import json import logging import optparse -from aore.fias.fiasfactory import FiasFactory from aore.miscutils.sphinx import SphinxHelper from aore.updater.soapreceiver import SoapReceiver from aore.updater.updater import Updater @@ -117,5 +115,6 @@ def main(): sphinxh = SphinxHelper() sphinxh.configure_indexer(options.indexer_path, options.output_conf) + if __name__ == '__main__': main() diff --git a/aore/miscutils/sphinx.py b/aore/miscutils/sphinx.py index 7943919..5fc3b7f 100644 --- a/aore/miscutils/sphinx.py +++ b/aore/miscutils/sphinx.py @@ -6,9 +6,9 @@ import os from bottle import template from aore.config import folders, db_conf, sphinx_conf +from aore.miscutils.trigram import trigram from aore.updater.aoxmltableentry import AoXmlTableEntry from aore.updater.dbhandler import DbHandler -from trigram import trigram class SphinxHelper: @@ -37,22 +37,22 @@ class SphinxHelper: # Indexing both configs run_index_cmd = "{} -c {} --all --rotate".format(self.index_binary, out_fname) - logging.info("Indexing main ({})...".format(out_fname)) + logging.info("Indexing main (%s)...", out_fname) os.system(run_index_cmd) - logging.info("All indexes were created.".format(out_fname)) + logging.info("All indexes were created.") # remove temp files for fname, fpath in self.files.iteritems(): try: os.remove(fpath) except: - logging.warning("Cannot delete {}. Not accessible.".format(fpath)) + logging.warning("Cannot delete %s. Not accessible.", fpath) logging.info("Temporary files removed.") logging.info("Successfully configured. Please restart searchd.") def __create_sugg_index_config(self): fname = os.path.abspath(folders.temp + "/suggest.conf") - logging.info("Creating config {}".format(fname)) + logging.info("Creating config %s", fname) conf_data = template('aore/templates/sphinx/idx_suggest.conf', db_host=db_conf.host, db_user=db_conf.user, @@ -70,7 +70,7 @@ class SphinxHelper: return fname def __dbexport_sugg_dict(self): - logging.info("Place suggestion dict to DB {}...".format(self.files['dict.txt'])) + logging.info("Place suggestion dict to DB %s...", self.files['dict.txt']) dict_dat_fname = os.path.abspath(folders.temp + "/suggdict.csv") csv_counter = 0 @@ -104,7 +104,7 @@ class SphinxHelper: def __create_ao_index_config(self): fname = os.path.abspath(folders.temp + "/addrobj.conf") - logging.info("Creating config {}".format(fname)) + logging.info("Creating config %s", fname) conf_data = template('aore/templates/sphinx/idx_addrobj.conf', db_host=db_conf.host, db_user=db_conf.user, @@ -124,7 +124,7 @@ class SphinxHelper: def __create_suggestion_dict(self): fname = os.path.abspath(folders.temp + "/suggdict.txt") - logging.info("Make suggestion dict ({})...".format(fname)) + logging.info("Make suggestion dict (%s)...", fname) run_builddict_cmd = "{} {} -c {} --buildstops {} 200000 --buildfreqs".format(self.index_binary, sphinx_conf.index_addjobj, @@ -136,7 +136,7 @@ class SphinxHelper: def __create_main_config(self, config_fname): out_filename = os.path.abspath(config_fname) - logging.info("Creating main config {}...".format(out_filename)) + logging.info("Creating main config %s...", out_filename) conf_data = template('aore/templates/sphinx/sphinx.conf', sphinx_var_path=sphinx_conf.var_dir) diff --git a/aore/phias.py b/aore/phias.py index 5eea907..5adf1e9 100644 --- a/aore/phias.py +++ b/aore/phias.py @@ -33,6 +33,6 @@ def find(text, strong=False): @app.error(404) -def error404(error): +def error404(): response.content_type = 'application/json' return json.dumps(dict(error="Page not found")) diff --git a/aore/updater/aodataparser.py b/aore/updater/aodataparser.py index 046aa0d..5fe773e 100644 --- a/aore/updater/aodataparser.py +++ b/aore/updater/aodataparser.py @@ -4,7 +4,7 @@ import os from aore.config import folders from aore.dbutils.dbschemas import db_shemas from aore.miscutils.exceptions import FiasException -from xmlparser import XMLParser +from aore.updater.xmlparser import XMLParser class AoDataParser: diff --git a/aore/updater/aorar.py b/aore/updater/aorar.py index a8c3fb7..ba95561 100644 --- a/aore/updater/aorar.py +++ b/aore/updater/aorar.py @@ -17,7 +17,7 @@ class AoRar: rarfile.UNRAR_TOOL = unrar_config.path def download(self, url): - logging.info("Downloading {}".format(url)) + logging.info("Downloading %s", url) try: local_filename = os.path.abspath(folders.temp + "/" + url.split('/')[-1]) if os.path.isfile(local_filename): @@ -32,7 +32,7 @@ class AoRar: except: raise FiasException("Error downloading. Reason : {}".format(format_exc())) - logging.info("Downloaded {} bytes".format(request.headers['Content-length'])) + logging.info("Downloaded %d bytes", request.headers['Content-length']) return local_filename def get_table_entries(self, file_name, allowed_tables): diff --git a/aore/updater/dbhandler.py b/aore/updater/dbhandler.py index 5f46e71..5e99337 100644 --- a/aore/updater/dbhandler.py +++ b/aore/updater/dbhandler.py @@ -44,7 +44,7 @@ class DbHandler: assert sql_query, "Invalid operation type: {}".format(operation_type) self.db.execute(sql_query) - logging.info("Processed {} queries FROM {}".format(processed_count - 1, csv_file_name)) + logging.info("Processed %d queries FROM %s", processed_count - 1, csv_file_name) def create_structure(self): logging.info("Prepare to create DB structure...") diff --git a/aore/updater/updater.py b/aore/updater/updater.py index 26d8afe..35bc40e 100644 --- a/aore/updater/updater.py +++ b/aore/updater/updater.py @@ -3,6 +3,8 @@ import logging from os import walk, path +import psycopg2 + from aore.config import db_conf from aore.dbutils.dbimpl import DBImpl from aore.dbutils.dbschemas import allowed_tables, db_shemas @@ -10,7 +12,6 @@ from aore.updater.aodataparser import AoDataParser from aore.updater.aorar import AoRar from aore.updater.aoxmltableentry import AoXmlTableEntry from aore.updater.dbhandler import DbHandler -import psycopg2 class Updater: @@ -42,15 +43,14 @@ class Updater: db.close() @classmethod - def __set__update_version(cls, updver = 0): + def __set__update_version(cls, updver=0): db = DBImpl(psycopg2, db_conf) try: - assert type(updver) is int, "Update version must be of int type." + assert isinstance(updver, int), "Update version must be of int type." db.execute('UPDATE "CONFIG" SET version={} WHERE id=0'.format(updver)) finally: db.close() - def __get_updates_from_folder(self, foldername): # TODO: Вычислять версию, если берем данные из каталога yield dict(intver=0, textver="Unknown", delta_url=foldername, complete_url=foldername) @@ -80,7 +80,7 @@ class Updater: self.db_handler.create_structure() for update_entry in self.updalist_generator: - logging.info("Processing DB #{}".format(update_entry['intver'])) + logging.info("Processing DB #%d", update_entry['intver']) for table_entry in self.tablelist_generator(update_entry['complete_url']): if table_entry.operation_type == AoXmlTableEntry.OperationType.update: table_entry.operation_type = AoXmlTableEntry.OperationType.create @@ -103,7 +103,7 @@ class Updater: if not indexes_dropped: self.db_handler.drop_indexes(allowed_tables) indexes_dropped = True - logging.info("Processing update #{}".format(update_entry['intver'])) + logging.info("Processing update #%d", update_entry['intver']) for table_entry in self.tablelist_generator(update_entry['delta_url']): self.process_single_entry(table_entry.operation_type, table_entry) Updater.__set__update_version(update_entry['intver'])