stale/py-phias
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Big commit for py3

Browse Source
This commit is contained in:
jar3b
2016-04-22 14:30:05 +03:00
parent 9ed372857e
commit c7f0de74a5
15 changed files with 103 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
aore/__init__.py
View File

@@ -1,7 +1,5 @@
import os
import sys
reload(sys)
cwd_dir = os.getcwd()
sys.path.append(cwd_dir)
sys.setdefaultencoding("utf-8")

1
aore/config/common.py
View File

@@ -3,6 +3,7 @@
class BasicConfig:
logging = False
debug_print = False
logfile = ""
def __init__(self):

4
aore/miscutils/fysearch.py
View File

@@ -13,7 +13,7 @@ def violet_ratio(pattern, candidate):
for i in range(len(arr_pattern) - 1, -1, -1):
max_j = -1
max_ratio = -1
allowed_nums = range(len(arr_candidate) - 1, -1, -1)
allowed_nums = list(range(len(arr_candidate) - 1, -1, -1))
for j in allowed_nums:
ratio = Levenshtein.ratio(arr_pattern[i], arr_candidate[j])
@@ -24,7 +24,7 @@ def violet_ratio(pattern, candidate):
result.append(max_j*abs(max_ratio))
if max_j > -1:
allowed_nums.remove(max_j)
del allowed_nums[max_j]
del arr_candidate[max_j]
return sum(result) - len(arr_candidate)

18
aore/miscutils/sphinx.py
View File

@@ -22,11 +22,11 @@ class SphinxHelper:
os.makedirs(Folders.temp)
# оздаем 3 папки для Сфинкса
if not os.path.exists(SphinxConfig.var_dir+ '/run'):
if not os.path.exists(SphinxConfig.var_dir + '/run'):
os.makedirs(SphinxConfig.var_dir + '/run')
if not os.path.exists(SphinxConfig.var_dir+ '/log'):
if not os.path.exists(SphinxConfig.var_dir + '/log'):
os.makedirs(SphinxConfig.var_dir + '/log')
if not os.path.exists(SphinxConfig.var_dir+ '/data'):
if not os.path.exists(SphinxConfig.var_dir + '/data'):
os.makedirs(SphinxConfig.var_dir + '/data')
def configure_indexer(self, indexer_binary, config_filename):
@@ -55,7 +55,7 @@ class SphinxHelper:
logging.info("All indexes were created.")
# remove temp files
for fname, fpath in self.files.iteritems():
for fname, fpath in self.files.items():
try:
os.remove(fpath)
except:
@@ -84,10 +84,10 @@ class SphinxHelper:
def __dbexport_sugg_dict(self):
logging.info("Place suggestion dict to DB %s...", self.files['dict.txt'])
dict_dat_fname = os.path.abspath(Folders.temp + "/suggdict.csv")
fname = os.path.abspath(Folders.temp + "/suggdict.csv")
csv_counter = 0
with open(self.files['dict.txt'], "r") as dict_file, open(dict_dat_fname, "w") as exit_file:
with open(self.files['dict.txt'], "r") as dict_file, open(fname, "w") as exit_file:
line = None
while line != '':
nodes = []
@@ -111,9 +111,11 @@ class SphinxHelper:
except:
pass
self.aodp.bulk_csv(AoXmlTableEntry.OperationType.update, "AOTRIG", csv_counter, dict_dat_fname)
self.aodp.bulk_csv(AoXmlTableEntry.OperationType.update, "AOTRIG", csv_counter, fname)
logging.info("Done.")
return fname
def __create_ao_index_config(self):
fname = os.path.abspath(Folders.temp + "/addrobj.conf")
logging.info("Creating config %s", fname)
@@ -156,7 +158,7 @@ class SphinxHelper:
sphinx_var_path=SphinxConfig.var_dir)
f = open(out_filename, "w")
for fname, fpath in self.files.iteritems():
for fname, fpath in self.files.items():
if ".conf" in fname:
with open(fpath, "r") as conff:
for line in conff:

2
aore/phias.py
View File

@@ -5,7 +5,7 @@ import logging
from bottle import response
from aore.search.fiasfactory import FiasFactory
from miscutils.bottlecl import BottleCL
from .miscutils.bottlecl import BottleCL
class App(BottleCL):

44
aore/search/fiasfactory.py
View File

@@ -1,16 +1,16 @@
# -*- coding: utf-8 -*-
import logging
import re
import urllib
import traceback
import urllib.parse
from uuid import UUID
import psycopg2
import traceback
from bottle import template
from aore.config import DatabaseConfig, BasicConfig
from aore.dbutils.dbimpl import DBImpl
from search import SphinxSearch
from .search import SphinxSearch
class FiasFactory:
@@ -38,10 +38,10 @@ class FiasFactory:
if rule == "boolean":
assert isinstance(param, bool), "Invalid parameter type"
if rule == "uuid":
assert (isinstance(param, str) or isinstance(param, unicode)) and self.__check_uuid(
assert isinstance(param, str) and self.__check_uuid(
param), "Invalid parameter value"
if rule == "text":
assert isinstance(param, str) or isinstance(param, unicode), "Invalid parameter type"
assert isinstance(param, str), "Invalid parameter type"
assert len(param) > 3, "Text too short"
pattern = re.compile(r"[A-za-zА-Яа-я \-,.#№]+")
assert pattern.match(param), "Invalid parameter value"
@@ -52,15 +52,17 @@ class FiasFactory:
def find(self, text, strong=False):
try:
text = urllib.unquote(text).decode('utf8')
text = urllib.parse.unquote(str(text))
self.__check_param(text, "text")
self.__check_param(strong, "boolean")
results = self.searcher.find(text, strong)
except Exception, err:
except Exception as err:
if BasicConfig.logging:
logging.error(traceback.format_exc(err))
return dict(error=err.args[0])
logging.error(traceback.format_exc())
if BasicConfig.debug_print:
traceback.print_exc()
return dict(error=str(err))
return results
@@ -73,10 +75,12 @@ class FiasFactory:
rows = self.db.get_rows(sql_query, True)
assert len(rows), "Record with this AOID not found in DB"
except Exception, err:
except Exception as err:
if BasicConfig.logging:
logging.error(traceback.format_exc(err))
return dict(error=err.args[0])
logging.error(traceback.format_exc())
if BasicConfig.debug_print:
traceback.print_exc()
return dict(error=str(err))
if len(rows) == 0:
return []
@@ -94,10 +98,12 @@ class FiasFactory:
sql_query = self.expand_templ.replace("//aoid", normalized_id)
rows = self.db.get_rows(sql_query, True)
except Exception, err:
except Exception as err:
if BasicConfig.logging:
logging.error(traceback.format_exc(err))
return dict(error=err.args[0])
logging.error(traceback.format_exc())
if BasicConfig.debug_print:
traceback.print_exc()
return dict(error=str(err))
return rows
@@ -111,9 +117,11 @@ class FiasFactory:
rows = self.db.get_rows(sql_query, True)
assert len(rows), "Record with this AOID not found in DB"
except Exception, err:
except Exception as err:
if BasicConfig.logging:
logging.error(traceback.format_exc(err))
return dict(error=err.args[0])
logging.error(traceback.format_exc())
if BasicConfig.debug_print:
traceback.print_exc()
return dict(error=str(err))
return rows

14
aore/search/search.py
View File

@@ -11,8 +11,8 @@ from aore.config import SphinxConfig
from aore.miscutils.exceptions import FiasException
from aore.miscutils.fysearch import violet_ratio
from aore.miscutils.trigram import trigram
from wordentry import WordEntry
from wordvariation import VariationType
from .wordentry import WordEntry
from .wordvariation import VariationType
class SphinxSearch:
@@ -62,7 +62,7 @@ class SphinxSearch:
self.client_show.SetSortMode(sphinxapi.SPH_SORT_EXTENDED, "krank DESC")
def __get_suggest(self, word, rating_limit, count):
word_len = str(len(word) / 2)
word_len = len(word)
trigrammed_word = '"{}"/1'.format(trigram(word))
self.__configure(SphinxConfig.index_sugg, word_len)
@@ -95,7 +95,7 @@ class SphinxSearch:
return outlist
# Получает список объектов (слово)
def __get_word_entries(self, words, strong):
def __get_word_entries(self, words):
we_list = []
for word in words:
if word != '':
@@ -111,14 +111,14 @@ class SphinxSearch:
def find(self, text, strong):
def split_phrase(phrase):
phrase = unicode(phrase).lower()
phrase = phrase.lower()
return re.split(r"[ ,:.#$]+", phrase)
# сплитим текст на слова
words = split_phrase(text)
# получаем список объектов (слов)
word_entries = self.__get_word_entries(words, strong)
word_entries = self.__get_word_entries(words)
word_count = len(word_entries)
# проверяем, есть ли вообще что-либо в списке объектов слов (или же все убрали как частое)
@@ -169,7 +169,7 @@ class SphinxSearch:
parsed_ids.append(match['attrs']['aoid'])
results.append(
dict(aoid=match['attrs']['aoid'],
text=unicode(match['attrs']['fullname']),
text=str(match['attrs']['fullname']),
ratio=match['attrs']['krank'],
cort=i))

20
aore/search/wordentry.py
View File

@@ -5,6 +5,10 @@ from aore.config import SphinxConfig
from aore.search.wordvariation import WordVariation, VariationType
def cleanup_string(word):
return word.replace('-', '').replace('@', '').replace('#', '')
class WordEntry:
# Варианты распеределния для слов с первыми двумя символами, где:
# 0 - не найдено, 1 - найдено одно, x - найдено много (>1)
@@ -51,14 +55,14 @@ class WordEntry:
def __init__(self, db, word):
self.db = db
self.bare_word = str(word)
self.word = self.__cleanify(self.bare_word)
self.word_len = len(unicode(self.word))
self.bare_word = word
self.word = cleanup_string(self.bare_word)
self.word_len = len(self.word)
self.parameters = dict(IS_FREQ=False, SOCR_WORD=None)
self.ranks = self.__init_ranks()
# Заполняем параметры слова
for mt_name, mt_values in self.match_types.iteritems():
for mt_name, mt_values in self.match_types.items():
self.__dict__[mt_name] = False
for mt_value in mt_values:
self.__dict__[mt_name] = self.__dict__[mt_name] or re.search(mt_value, self.ranks)
@@ -72,9 +76,6 @@ class WordEntry:
self.MT_LAST_STAR = False
self.MT_AS_IS = True
def __cleanify(self, word):
return word.replace('-', '').replace('@', '')
def variations_generator(self, strong, suggestion_func):
default_var_type = VariationType.normal
# Если слово встречается часто, ставим у всех вариантов тип VariationType.freq
@@ -115,8 +116,9 @@ class WordEntry:
"UNION ALL SELECT COUNT(*), NULL FROM \"AOTRIG\" WHERE word='{}' " \
"UNION ALL SELECT COUNT(*), MAX(scname) FROM \"SOCRBASE\" WHERE socrname ILIKE '{}'" \
"UNION ALL SELECT COUNT(*), NULL FROM \"SOCRBASE\" WHERE scname ILIKE '{}'" \
"UNION ALL SELECT frequency, NULL FROM \"AOTRIG\" WHERE word='{}';".format(
self.word, self.word_len, self.word, self.bare_word, self.bare_word, self.word)
"UNION ALL SELECT frequency, NULL FROM \"AOTRIG\" WHERE word='{}';".format(self.word, self.word_len,
self.word, self.bare_word,
self.bare_word, self.word)
result = self.db.get_rows(sql_qry)

2
aore/updater/aorar.py
View File

@@ -9,7 +9,7 @@ import requests
from aore.config import Folders, UnrarConfig
from aore.miscutils.exceptions import FiasException
from aoxmltableentry import AoXmlTableEntry
from .aoxmltableentry import AoXmlTableEntry
class AoRar:

1
aore/updater/soapreceiver.py
View File

@@ -2,6 +2,7 @@
from pysimplesoap.client import SoapClient
class SoapReceiver:
def __init__(self):
self.client = SoapClient(

4
aore/updater/updater.py
View File

@@ -62,9 +62,9 @@ class Updater:
mode = None
while not mode:
try:
mode = int(raw_input('Enter FIAS update version (3 digit):'))
mode = int(input('Enter FIAS update version (3 digit):'))
except ValueError:
print "Not a valid fias version, try again."
print("Not a valid fias version, try again.")
return mode

3
aore/updater/xmlparser.py
View File

@@ -7,7 +7,8 @@ class XMLParser:
def __init__(self, parse_function):
self.parse_function = parse_function
def fast_iter(self, context, func, *args, **kwargs):
@staticmethod
def fast_iter(context, func, *args, **kwargs):
for event, elem in context:
# print event
func(elem, *args, **kwargs)