stale/py-phias
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Lint рефактор (кое-что поправлено)

Browse Source
This commit is contained in:
Jack Stdin 2016-02-03 14:33:01 +03:00
parent 16a144205b
commit 29a26132e1
11 changed files with 46 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
aore/__init__.py
View File

@ -2,6 +2,6 @@ import os
import sys
reload(sys)
cwd = os.getcwd()
sys.path.append(cwd)
cwd_dir = os.getcwd()
sys.path.append(cwd_dir)
sys.setdefaultencoding("utf-8")

2
aore/dbutils/dbimpl.py
View File

@ -31,6 +31,7 @@ class DBImpl:
cur = self.get_cursor()
cur.execute(sql_query)
cur.close()
del cur
self.transaction_commit()
except:
self.transaction_rollback()
@ -47,6 +48,7 @@ class DBImpl:
rows = cur.fetchall()
cur.close()
del cur
self.transaction_commit()
except:
self.transaction_rollback()

27
aore/fias/fiasfactory.py
View File

@ -1,12 +1,13 @@
# -*- coding: utf-8 -*-
import re
from uuid import UUID
import psycopg2
from bottle import template
import sys
from aore.config import db_conf
from aore.dbutils.dbimpl import DBImpl
from aore.fias.search import SphinxSearch
from aore.config import db_conf
from uuid import UUID
import re
class FiasFactory:
@ -30,18 +31,20 @@ class FiasFactory:
# rule - "boolean", "uuid", "text"
def __check_param(self, param, rule):
if rule == "boolean":
assert type(param) is bool, "Invalid parameter type"
assert isinstance(param, bool), "Invalid parameter type"
if rule == "uuid":
assert (type(param) is str or type(param) is unicode) and self.__check_uuid(param), "Invalid parameter value"
if rule == "text":
assert type(param) is str or type(param) is unicode, "Invalid parameter type"
assert (isinstance(param, str) or isinstance(param, unicode)) and self.__check_uuid(
param), "Invalid parameter value"
if rule == "text":
assert isinstance(param, str) or isinstance(param, unicode), "Invalid parameter type"
assert len(param) > 3, "Text too short"
pattern = re.compile("[A-za-zА-Яа-я \-,.#№]+")
pattern = re.compile(r"[A-za-zА-Яа-я \-,.#№]+")
assert pattern.match(param), "Invalid parameter value"
# text - строка поиска
# strong - строгий поиск (True) или "мягкий" (False) (с допущением ошибок, опечаток)
# Строгий используется при импорте из внешних систем (автоматически), где ошибка критична
# text - строка поиска
# strong - строгий поиск (True) или "мягкий" (False) (с допущением ошибок, опечаток)
# Строгий используется при импорте из внешних систем (автоматически), где ошибка критична
def find(self, text, strong=False):
try:
self.__check_param(text, "text")

11
aore/fias/search.py
View File

@ -74,15 +74,12 @@ class SphinxSearch:
maxleven = jaro_rating - jaro_rating * self.regression_coef
if jaro_rating >= rating_limit and jaro_rating >= maxleven:
outlist.append([match['attrs']['word'], jaro_rating])
del jaro_rating
outlist.sort(key=lambda x: x[1], reverse=True)
return outlist
def __split_phrase(self, phrase):
phrase = unicode(phrase).replace('-', '').replace('@', '').lower()
return re.split(r"[ ,:.#$]+", phrase)
def __add_word_variations(self, word_entry, strong):
if word_entry.MT_MANY_SUGG and not strong:
suggs = self.__get_suggest(word_entry.word, self.rating_limit_soft, self.rating_limit_soft_count)
@ -111,7 +108,11 @@ class SphinxSearch:
return we_list
def find(self, text, strong):
words = self.__split_phrase(text)
def split_phrase(phrase):
phrase = unicode(phrase).replace('-', '').replace('@', '').lower()
return re.split(r"[ ,:.#$]+", phrase)
words = split_phrase(text)
word_entries = self.__get_word_entries(words, strong)
word_count = len(word_entries)
for x in range(word_count, max(0, word_count - 3), -1):

3
aore/manage.py
View File

@ -1,9 +1,7 @@
# -*- coding: utf-8 -*-
import json
import logging
import optparse
from aore.fias.fiasfactory import FiasFactory
from aore.miscutils.sphinx import SphinxHelper
from aore.updater.soapreceiver import SoapReceiver
from aore.updater.updater import Updater
@ -117,5 +115,6 @@ def main():
sphinxh = SphinxHelper()
sphinxh.configure_indexer(options.indexer_path, options.output_conf)
if __name__ == '__main__':
main()

18
aore/miscutils/sphinx.py
View File

@ -6,9 +6,9 @@ import os
from bottle import template
from aore.config import folders, db_conf, sphinx_conf
from aore.miscutils.trigram import trigram
from aore.updater.aoxmltableentry import AoXmlTableEntry
from aore.updater.dbhandler import DbHandler
from trigram import trigram
class SphinxHelper:
@ -37,22 +37,22 @@ class SphinxHelper:
# Indexing both configs
run_index_cmd = "{} -c {} --all --rotate".format(self.index_binary, out_fname)
logging.info("Indexing main ({})...".format(out_fname))
logging.info("Indexing main (%s)...", out_fname)
os.system(run_index_cmd)
logging.info("All indexes were created.".format(out_fname))
logging.info("All indexes were created.")
# remove temp files
for fname, fpath in self.files.iteritems():
try:
os.remove(fpath)
except:
logging.warning("Cannot delete {}. Not accessible.".format(fpath))
logging.warning("Cannot delete %s. Not accessible.", fpath)
logging.info("Temporary files removed.")
logging.info("Successfully configured. Please restart searchd.")
def __create_sugg_index_config(self):
fname = os.path.abspath(folders.temp + "/suggest.conf")
logging.info("Creating config {}".format(fname))
logging.info("Creating config %s", fname)
conf_data = template('aore/templates/sphinx/idx_suggest.conf', db_host=db_conf.host,
db_user=db_conf.user,
@ -70,7 +70,7 @@ class SphinxHelper:
return fname
def __dbexport_sugg_dict(self):
logging.info("Place suggestion dict to DB {}...".format(self.files['dict.txt']))
logging.info("Place suggestion dict to DB %s...", self.files['dict.txt'])
dict_dat_fname = os.path.abspath(folders.temp + "/suggdict.csv")
csv_counter = 0
@ -104,7 +104,7 @@ class SphinxHelper:
def __create_ao_index_config(self):
fname = os.path.abspath(folders.temp + "/addrobj.conf")
logging.info("Creating config {}".format(fname))
logging.info("Creating config %s", fname)
conf_data = template('aore/templates/sphinx/idx_addrobj.conf', db_host=db_conf.host,
db_user=db_conf.user,
@ -124,7 +124,7 @@ class SphinxHelper:
def __create_suggestion_dict(self):
fname = os.path.abspath(folders.temp + "/suggdict.txt")
logging.info("Make suggestion dict ({})...".format(fname))
logging.info("Make suggestion dict (%s)...", fname)
run_builddict_cmd = "{} {} -c {} --buildstops {} 200000 --buildfreqs".format(self.index_binary,
sphinx_conf.index_addjobj,
@ -136,7 +136,7 @@ class SphinxHelper:
def __create_main_config(self, config_fname):
out_filename = os.path.abspath(config_fname)
logging.info("Creating main config {}...".format(out_filename))
logging.info("Creating main config %s...", out_filename)
conf_data = template('aore/templates/sphinx/sphinx.conf', sphinx_var_path=sphinx_conf.var_dir)

2
aore/phias.py
View File

@ -33,6 +33,6 @@ def find(text, strong=False):
@app.error(404)
def error404(error):
def error404():
response.content_type = 'application/json'
return json.dumps(dict(error="Page not found"))

2
aore/updater/aodataparser.py
View File

@ -4,7 +4,7 @@ import os
from aore.config import folders
from aore.dbutils.dbschemas import db_shemas
from aore.miscutils.exceptions import FiasException
from xmlparser import XMLParser
from aore.updater.xmlparser import XMLParser
class AoDataParser:

4
aore/updater/aorar.py
View File

@ -17,7 +17,7 @@ class AoRar:
rarfile.UNRAR_TOOL = unrar_config.path
def download(self, url):
logging.info("Downloading {}".format(url))
logging.info("Downloading %s", url)
try:
local_filename = os.path.abspath(folders.temp + "/" + url.split('/')[-1])
if os.path.isfile(local_filename):
@ -32,7 +32,7 @@ class AoRar:
except:
raise FiasException("Error downloading. Reason : {}".format(format_exc()))
logging.info("Downloaded {} bytes".format(request.headers['Content-length']))
logging.info("Downloaded %d bytes", request.headers['Content-length'])
return local_filename
def get_table_entries(self, file_name, allowed_tables):

2
aore/updater/dbhandler.py
View File

@ -44,7 +44,7 @@ class DbHandler:
assert sql_query, "Invalid operation type: {}".format(operation_type)
self.db.execute(sql_query)
logging.info("Processed {} queries FROM {}".format(processed_count - 1, csv_file_name))
logging.info("Processed %d queries FROM %s", processed_count - 1, csv_file_name)
def create_structure(self):
logging.info("Prepare to create DB structure...")

12
aore/updater/updater.py
View File

@ -3,6 +3,8 @@
import logging
from os import walk, path
import psycopg2
from aore.config import db_conf
from aore.dbutils.dbimpl import DBImpl
from aore.dbutils.dbschemas import allowed_tables, db_shemas
@ -10,7 +12,6 @@ from aore.updater.aodataparser import AoDataParser
from aore.updater.aorar import AoRar
from aore.updater.aoxmltableentry import AoXmlTableEntry
from aore.updater.dbhandler import DbHandler
import psycopg2
class Updater:
@ -42,15 +43,14 @@ class Updater:
db.close()
@classmethod
def __set__update_version(cls, updver = 0):
def __set__update_version(cls, updver=0):
db = DBImpl(psycopg2, db_conf)
try:
assert type(updver) is int, "Update version must be of int type."
assert isinstance(updver, int), "Update version must be of int type."
db.execute('UPDATE "CONFIG" SET version={} WHERE id=0'.format(updver))
finally:
db.close()
def __get_updates_from_folder(self, foldername):
# TODO: Вычислять версию, если берем данные из каталога
yield dict(intver=0, textver="Unknown", delta_url=foldername, complete_url=foldername)
@ -80,7 +80,7 @@ class Updater:
self.db_handler.create_structure()
for update_entry in self.updalist_generator:
logging.info("Processing DB #{}".format(update_entry['intver']))
logging.info("Processing DB #%d", update_entry['intver'])
for table_entry in self.tablelist_generator(update_entry['complete_url']):
if table_entry.operation_type == AoXmlTableEntry.OperationType.update:
table_entry.operation_type = AoXmlTableEntry.OperationType.create
@ -103,7 +103,7 @@ class Updater:
if not indexes_dropped:
self.db_handler.drop_indexes(allowed_tables)
indexes_dropped = True
logging.info("Processing update #{}".format(update_entry['intver']))
logging.info("Processing update #%d", update_entry['intver'])
for table_entry in self.tablelist_generator(update_entry['delta_url']):
self.process_single_entry(table_entry.operation_type, table_entry)
Updater.__set__update_version(update_entry['intver'])