diff --git a/aore/config/common.py b/aore/config/common.py index 34bee2e..9553d79 100644 --- a/aore/config/common.py +++ b/aore/config/common.py @@ -1,29 +1,32 @@ # -*- coding: utf-8 -*- -class basic: - logging = False - logfile = "" + +class BasicConfig: + def __init__(self): + self.logging = False + self.logfile = "" + +class SphinxConfig: + def __init__(self): + self.listen = "127.0.0.1:9312" + self.index_addjobj = "idx_fias_addrobj" + self.index_sugg = "idx_fias_sugg" + self.var_dir = None + self.min_length_to_star = 3 + +class DatabaseConfig: + def __init__(self): + self.host = None + self.user = None + self.password = None + self.database = None + self.port = None + +class UnrarConfig: + def __init__(self): + self.path = None -class sphinx_conf: - listen = "127.0.0.1:9312" - index_addjobj = "idx_fias_addrobj" - index_sugg = "idx_fias_sugg" - var_dir = None - min_length_to_star = 3 - - -class db_conf: - host = None - user = None - password = None - database = None - port = None - - -class unrar_config: - path = None - - -class folders: - temp = None +class Folders: + def __init__(self): + self.temp = None diff --git a/aore/miscutils/sphinx.py b/aore/miscutils/sphinx.py index 011dd82..22c1542 100644 --- a/aore/miscutils/sphinx.py +++ b/aore/miscutils/sphinx.py @@ -5,7 +5,7 @@ import os from bottle import template -from aore.config import folders, db_conf, sphinx_conf +from aore.config import Folders, DatabaseConfig, SphinxConfig from aore.miscutils.trigram import trigram from aore.updater.aoxmltableentry import AoXmlTableEntry from aore.updater.dbhandler import DbHandler @@ -18,16 +18,16 @@ class SphinxHelper: self.aodp = DbHandler() # Создаем временную папку, если ее нет - if not os.path.exists(folders.temp): - os.makedirs(folders.temp) + if not os.path.exists(Folders.temp): + os.makedirs(Folders.temp) # оздаем 3 папки для Сфинкса - if not os.path.exists(sphinx_conf.var_dir+'/run'): - os.makedirs(sphinx_conf.var_dir+'/run') - if not os.path.exists(sphinx_conf.var_dir+'/log'): - os.makedirs(sphinx_conf.var_dir+'/log') - if not os.path.exists(sphinx_conf.var_dir+'/data'): - os.makedirs(sphinx_conf.var_dir+'/data') + if not os.path.exists(SphinxConfig.var_dir+ '/run'): + os.makedirs(SphinxConfig.var_dir + '/run') + if not os.path.exists(SphinxConfig.var_dir+ '/log'): + os.makedirs(SphinxConfig.var_dir + '/log') + if not os.path.exists(SphinxConfig.var_dir+ '/data'): + os.makedirs(SphinxConfig.var_dir + '/data') def configure_indexer(self, indexer_binary, config_filename): logging.info("Start configuring Sphinx...") @@ -64,15 +64,15 @@ class SphinxHelper: logging.info("Successfully configured. Please restart searchd.") def __create_sugg_index_config(self): - fname = os.path.abspath(folders.temp + "/suggest.conf") + fname = os.path.abspath(Folders.temp + "/suggest.conf") logging.info("Creating config %s", fname) - conf_data = template('aore/templates/sphinx/idx_suggest.conf', db_host=db_conf.host, - db_user=db_conf.user, - db_password=db_conf.password, - db_name=db_conf.database, db_port=db_conf.port, - index_name=sphinx_conf.index_sugg, - sphinx_var_path=sphinx_conf.var_dir) + conf_data = template('aore/templates/sphinx/idx_suggest.conf', db_host=DatabaseConfig.host, + db_user=DatabaseConfig.user, + db_password=DatabaseConfig.password, + db_name=DatabaseConfig.database, db_port=DatabaseConfig.port, + index_name=SphinxConfig.index_sugg, + sphinx_var_path=SphinxConfig.var_dir) f = open(fname, "w") f.write(conf_data) @@ -84,7 +84,7 @@ class SphinxHelper: def __dbexport_sugg_dict(self): logging.info("Place suggestion dict to DB %s...", self.files['dict.txt']) - dict_dat_fname = os.path.abspath(folders.temp + "/suggdict.csv") + dict_dat_fname = os.path.abspath(Folders.temp + "/suggdict.csv") csv_counter = 0 with open(self.files['dict.txt'], "r") as dict_file, open(dict_dat_fname, "w") as exit_file: @@ -115,17 +115,17 @@ class SphinxHelper: logging.info("Done.") def __create_ao_index_config(self): - fname = os.path.abspath(folders.temp + "/addrobj.conf") + fname = os.path.abspath(Folders.temp + "/addrobj.conf") logging.info("Creating config %s", fname) - conf_data = template('aore/templates/sphinx/idx_addrobj.conf', db_host=db_conf.host, - db_user=db_conf.user, - db_password=db_conf.password, - db_name=db_conf.database, db_port=db_conf.port, + conf_data = template('aore/templates/sphinx/idx_addrobj.conf', db_host=DatabaseConfig.host, + db_user=DatabaseConfig.user, + db_password=DatabaseConfig.password, + db_name=DatabaseConfig.database, db_port=DatabaseConfig.port, sql_query=template('aore/templates/postgre/sphinx_query.sql').replace("\n", " \\\n"), - index_name=sphinx_conf.index_addjobj, - sphinx_var_path=sphinx_conf.var_dir, - min_length_to_star=sphinx_conf.min_length_to_star) + index_name=SphinxConfig.index_addjobj, + sphinx_var_path=SphinxConfig.var_dir, + min_length_to_star=SphinxConfig.min_length_to_star) f = open(fname, "w") f.write(conf_data) @@ -136,11 +136,11 @@ class SphinxHelper: return fname def __create_suggestion_dict(self): - fname = os.path.abspath(folders.temp + "/suggdict.txt") + fname = os.path.abspath(Folders.temp + "/suggdict.txt") logging.info("Make suggestion dict (%s)...", fname) run_builddict_cmd = "{} {} -c {} --buildstops {} 200000 --buildfreqs".format(self.index_binary, - sphinx_conf.index_addjobj, + SphinxConfig.index_addjobj, self.files['addrobj.conf'], fname) os.system(run_builddict_cmd) logging.info("Done.") @@ -152,8 +152,8 @@ class SphinxHelper: logging.info("Creating main config %s...", out_filename) conf_data = template('aore/templates/sphinx/sphinx.conf', - sphinx_listen=sphinx_conf.listen.replace("unix://", ''), - sphinx_var_path=sphinx_conf.var_dir) + sphinx_listen=SphinxConfig.listen.replace("unix://", ''), + sphinx_var_path=SphinxConfig.var_dir) f = open(out_filename, "w") for fname, fpath in self.files.iteritems(): diff --git a/aore/search/fiasfactory.py b/aore/search/fiasfactory.py index b8cdae8..6fbbffc 100644 --- a/aore/search/fiasfactory.py +++ b/aore/search/fiasfactory.py @@ -8,14 +8,14 @@ import psycopg2 import traceback from bottle import template -from aore.config import db_conf, basic +from aore.config import DatabaseConfig, BasicConfig from aore.dbutils.dbimpl import DBImpl from search import SphinxSearch class FiasFactory: def __init__(self): - self.db = DBImpl(psycopg2, db_conf) + self.db = DBImpl(psycopg2, DatabaseConfig) self.searcher = SphinxSearch(self.db) self.expand_templ = template('aore/templates/postgre/expand_query.sql', aoid="//aoid") self.normalize_templ = template('aore/templates/postgre/normalize_query.sql', aoid="//aoid") @@ -71,7 +71,7 @@ class FiasFactory: sql_query = self.normalize_templ.replace("//aoid", aoid_guid) rows = self.db.get_rows(sql_query, True) except Exception, err: - if basic.logging: + if BasicConfig.logging: logging.error(traceback.format_exc(err)) return dict(error=err.args[0]) @@ -92,7 +92,7 @@ class FiasFactory: sql_query = self.expand_templ.replace("//aoid", normalized_id) rows = self.db.get_rows(sql_query, True) except Exception, err: - if basic.logging: + if BasicConfig.logging: logging.error(traceback.format_exc(err)) return dict(error=err.args[0]) diff --git a/aore/search/search.py b/aore/search/search.py index a1efeac..4e3c384 100644 --- a/aore/search/search.py +++ b/aore/search/search.py @@ -6,10 +6,10 @@ import time import Levenshtein import sphinxapi +from aore.config import BasicConfig +from aore.config import SphinxConfig from fuzzywuzzy import fuzz -from aore.config import basic -from aore.config import sphinx_conf from aore.miscutils.exceptions import FiasException from aore.miscutils.fysearch import violet_ratio from aore.miscutils.trigram import trigram @@ -31,7 +31,7 @@ class SphinxSearch: def __init__(self, db): self.db = db - sphinx_host = sphinx_conf.listen + sphinx_host = SphinxConfig.listen sphinx_port = None # Получаем строку подключения для Sphinx @@ -53,7 +53,7 @@ class SphinxSearch: def __configure(self, index_name, word_len): self.client_sugg.ResetFilters() - if index_name == sphinx_conf.index_sugg: + if index_name == SphinxConfig.index_sugg: self.client_sugg.SetRankingMode(sphinxapi.SPH_RANK_WORDCOUNT) self.client_sugg.SetFilterRange("len", int(word_len) - self.delta_len, int(word_len) + self.delta_len) self.client_sugg.SetSelect("word, len, @weight+{}-abs(len-{}) AS krank".format(self.delta_len, word_len)) @@ -67,8 +67,8 @@ class SphinxSearch: word_len = str(len(word) / 2) trigrammed_word = '"{}"/1'.format(trigram(word)) - self.__configure(sphinx_conf.index_sugg, word_len) - result = self.client_sugg.Query(trigrammed_word, sphinx_conf.index_sugg) + self.__configure(SphinxConfig.index_sugg, word_len) + result = self.client_sugg.Query(trigrammed_word, SphinxConfig.index_sugg) # Если по данному слову не найдено подсказок (а такое бывает?) # возвращаем [] @@ -138,26 +138,27 @@ class SphinxSearch: good_vars_word_count = len(set([v.parent for v in good_vars])) freq_vars_word_count = len(set([v.parent for v in freq_vars])) - self.__configure(sphinx_conf.index_addjobj, word_count) + self.__configure(SphinxConfig.index_addjobj, word_count) # формируем строки для поиска в Сфинксе for i in range(good_vars_word_count, max(0, good_vars_word_count - 3), -1): first_q = "@fullname \"{}\"/{}".format(" ".join(good_var.text for good_var in good_vars), i) if self.search_freq_words and freq_vars_word_count: second_q = " @sname {}".format(" ".join(freq_var.text for freq_var in freq_vars)) - self.client_show.AddQuery(first_q + second_q, sphinx_conf.index_addjobj) + self.client_show.AddQuery(first_q + second_q, SphinxConfig.index_addjobj) del second_q - self.client_show.AddQuery(first_q, sphinx_conf.index_addjobj) + self.client_show.AddQuery(first_q, SphinxConfig.index_addjobj) del first_q start_t = time.time() rs = self.client_show.RunQueries() elapsed_t = time.time() - start_t + if rs is None: raise FiasException("Cannot find sentence.") - if basic.logging: + if BasicConfig.logging: logging.info("Sphinx time for {} = {}".format(text, elapsed_t)) results = [] diff --git a/aore/search/wordentry.py b/aore/search/wordentry.py index 300008b..99a3b07 100644 --- a/aore/search/wordentry.py +++ b/aore/search/wordentry.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- import re -from aore.config import sphinx_conf +from aore.config import SphinxConfig from aore.search.wordvariation import WordVariation, VariationType @@ -68,7 +68,7 @@ class WordEntry: self.MT_AS_IS = False # Строка слишком котроткая, то по лайку не ищем, сфинкс такого не прожует - if self.MT_LAST_STAR and self.word_len < sphinx_conf.min_length_to_star: + if self.MT_LAST_STAR and self.word_len < SphinxConfig.min_length_to_star: self.MT_LAST_STAR = False self.MT_AS_IS = True diff --git a/aore/updater/aodataparser.py b/aore/updater/aodataparser.py index 32c6ff8..e0e2522 100644 --- a/aore/updater/aodataparser.py +++ b/aore/updater/aodataparser.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- import os -from aore.config import folders +from aore.config import Folders from aore.dbutils.dbschemas import db_shemas from aore.miscutils.exceptions import FiasException from aore.updater.xmlparser import XMLParser @@ -16,8 +16,8 @@ class AoDataParser: self.allowed_fields = db_shemas[self.datasource.table_name].fields # Создаем временную папку, если ее нет - if not os.path.exists(folders.temp): - os.makedirs(folders.temp) + if not os.path.exists(Folders.temp): + os.makedirs(Folders.temp) self.pagesize = pagesize self.currentpage = 0 @@ -56,7 +56,7 @@ class AoDataParser: self.data_bereit_callback = data_callback self.currentpage = 0 self.base_filename = \ - folders.temp + "/fd_" + \ + Folders.temp + "/fd_" + \ str(self.datasource.operation_type) + "_" + \ self.datasource.table_name + ".csv.part{}" self.counter = self.pagesize + 1 diff --git a/aore/updater/aorar.py b/aore/updater/aorar.py index b8b095d..05c1d36 100644 --- a/aore/updater/aorar.py +++ b/aore/updater/aorar.py @@ -7,14 +7,14 @@ from traceback import format_exc import rarfile import requests -from aore.config import folders, unrar_config +from aore.config import Folders, UnrarConfig from aore.miscutils.exceptions import FiasException from aoxmltableentry import AoXmlTableEntry class AoRar: def __init__(self): - rarfile.UNRAR_TOOL = unrar_config.path + rarfile.UNRAR_TOOL = UnrarConfig.path self.fname = None self.mode = None @@ -25,12 +25,12 @@ class AoRar: def download(self, url): logging.info("Downloading %s", url) try: - local_filename = os.path.abspath(folders.temp + "/" + url.split('/')[-1]) + local_filename = os.path.abspath(Folders.temp + "/" + url.split('/')[-1]) if os.path.isfile(local_filename): os.remove(local_filename) else: - if not os.path.exists(folders.temp): - os.makedirs(folders.temp) + if not os.path.exists(Folders.temp): + os.makedirs(Folders.temp) request = requests.get(url, stream=True) with open(local_filename, 'wb') as f: diff --git a/aore/updater/dbhandler.py b/aore/updater/dbhandler.py index 5e99337..96e00f6 100644 --- a/aore/updater/dbhandler.py +++ b/aore/updater/dbhandler.py @@ -5,7 +5,7 @@ import logging import psycopg2 from bottle import template -from aore.config import db_conf +from aore.config import DatabaseConfig from aore.dbutils.dbimpl import DBImpl from aore.dbutils.dbschemas import db_shemas from aore.updater.aoxmltableentry import AoXmlTableEntry @@ -13,7 +13,7 @@ from aore.updater.aoxmltableentry import AoXmlTableEntry class DbHandler: def __init__(self): - self.db = DBImpl(psycopg2, db_conf) + self.db = DBImpl(psycopg2, DatabaseConfig) def bulk_csv(self, operation_type, table_name, processed_count, csv_file_name): sql_query = None diff --git a/aore/updater/updater.py b/aore/updater/updater.py index 0568819..5babd7f 100644 --- a/aore/updater/updater.py +++ b/aore/updater/updater.py @@ -5,7 +5,7 @@ from os import walk, path import psycopg2 -from aore.config import db_conf +from aore.config import DatabaseConfig from aore.dbutils.dbimpl import DBImpl from aore.dbutils.dbschemas import allowed_tables, db_shemas from aore.updater.aodataparser import AoDataParser @@ -36,7 +36,7 @@ class Updater: def get_current_fias_version(cls): db = None try: - db = DBImpl(psycopg2, db_conf) + db = DBImpl(psycopg2, DatabaseConfig) rows = db.get_rows('SELECT version FROM "CONFIG" WHERE id=0', True) assert len(rows) > 0, "Cannot get a version" return rows[0]['version'] @@ -48,7 +48,7 @@ class Updater: @classmethod def __set__update_version(cls, updver=0): - db = DBImpl(psycopg2, db_conf) + db = DBImpl(psycopg2, DatabaseConfig) try: assert isinstance(updver, int), "Update version must be of int type." db.execute('UPDATE "CONFIG" SET version={} WHERE id=0'.format(updver)) @@ -69,9 +69,10 @@ class Updater: return mode def __get_updates_from_folder(self, foldername): - # TODO: Вычислять версию, если берем данные из каталога - yield dict(intver=self.__get_update_version_from_console(), - textver="Unknown", delta_url=foldername, + fias_db_version = self.__get_update_version_from_console() + yield dict(intver=fias_db_version, + textver="Version {}".format(fias_db_version), + delta_url=foldername, complete_url=foldername) @staticmethod diff --git a/config.example.py b/config.example.py index 23c87b6..8c43376 100644 --- a/config.example.py +++ b/config.example.py @@ -2,17 +2,24 @@ from aore import config # Config section -config.sphinx_conf.listen = "127.0.0.1:9312" -config.sphinx_conf.var_dir = "C:\\Sphinx" -config.db_conf.database = "pyfias" -config.db_conf.host = "192.168.0.1" -config.db_conf.port = 5432 -config.db_conf.user = "postgres" -config.db_conf.password = "postgres" +# Address and port where sphinx was listening, +# may be a unix socket like 'unix://tmp/pyphias.sock' +config.SphinxConfig.listen = "127.0.0.1:9312" +# Base sphinx folder +config.SphinxConfig.var_dir = "C:\\Sphinx" -config.unrar_config.path = "C:\\Program Files\\WinRAR\\unrar.exe" -config.folders.temp = "E:\\!TEMP" +# DB config +config.DatabaseConfig.database = "fias_db" +config.DatabaseConfig.host = "192.168.0.1" +config.DatabaseConfig.port = 5432 +config.DatabaseConfig.user = "postgres" +config.DatabaseConfig.password = "postgres" -config.basic.logging = True -config.basic.logfile = "pyphias.log" +# Path to unrar, in Linux may be 'unrar' +config.UnrarConfig.path = "C:\\Program Files\\WinRAR\\unrar.exe" +# Temp folder, in Linux may be '/tmp/myfolder' +config.Folders.temp = "E:\\!TEMP" + +config.BasicConfig.logging = True +config.BasicConfig.logfile = "pyphias.log" \ No newline at end of file