stale/py-phias
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Изменена генерация индексов, начало обработки "частых" слов.

Browse Source
This commit is contained in:
Jack Stdin
2016-02-03 17:30:47 +03:00
parent 29a26132e1
commit 897ea66046
7 changed files with 42 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
aore/fias/search.py
View File

@@ -11,21 +11,21 @@ from aore.miscutils.trigram import trigram
class SphinxSearch:
# Config's
delta_len = 2
rating_limit_soft = 0.41
rating_limit_soft_count = 6
word_length_soft = 3
rating_limit_hard = 0.82
rating_limit_hard_count = 3
default_rating_delta = 2
regression_coef = 0.08
max_result = 10
def __init__(self, db):
self.delta_len = 2
self.rating_limit_soft = 0.41
self.rating_limit_soft_count = 6
self.word_length_soft = 3
self.rating_limit_hard = 0.82
self.rating_limit_hard_count = 3
self.default_rating_delta = 2
self.regression_coef = 0.08
self.max_result = 10
self.db = db
self.client_sugg = sphinxapi.SphinxClient()
self.client_sugg.SetServer(sphinx_conf.host_name, sphinx_conf.port)
@@ -38,6 +38,7 @@ class SphinxSearch:
self.client_show.SetConnectTimeout(3.0)
def __configure(self, index_name, wlen=None):
self.client_sugg.ResetFilters()
if index_name == sphinx_conf.index_sugg and wlen:
self.client_sugg.SetRankingMode(sphinxapi.SPH_RANK_WORDCOUNT)
self.client_sugg.SetFilterRange("len", int(wlen) - self.delta_len, int(wlen) + self.delta_len)
@@ -116,6 +117,7 @@ class SphinxSearch:
word_entries = self.__get_word_entries(words, strong)
word_count = len(word_entries)
for x in range(word_count, max(0, word_count - 3), -1):
logging.info("\"{}\"/{}".format(" ".join(x.get_variations() for x in word_entries), x))
self.client_show.AddQuery("\"{}\"/{}".format(" ".join(x.get_variations() for x in word_entries), x),
sphinx_conf.index_addjobj)

21
aore/fias/wordentry.py
View File

@@ -1,6 +1,8 @@
# -*- coding: utf-8 -*-
import re
from aore.config import sphinx_conf
class WordEntry:
# Варианты распеределния для слов с первыми двумя символами, где:
@@ -39,14 +41,13 @@ class WordEntry:
MT_ADD_SOCR=['..10', '..x0']
)
min_word_len_to_star = 4
def __init__(self, db, word):
self.db = db
self.word = str(word)
self.word_len = len(unicode(self.word))
self.variations = []
self.scname = None
self.is_freq_word = False
self.ranks = self.__get_ranks()
for x, y in self.match_types.iteritems():
@@ -59,7 +60,7 @@ class WordEntry:
self.MT_AS_IS = False
# Строка слишком котроткая, то по лайку не ищем, будет очень долго
if self.MT_LAST_STAR and self.word_len < self.min_word_len_to_star:
if self.MT_LAST_STAR and self.word_len < sphinx_conf.min_length_to_star:
self.MT_LAST_STAR = False
self.MT_AS_IS = True
@@ -79,8 +80,9 @@ class WordEntry:
sql_qry = "SELECT COUNT(*), NULL FROM \"AOTRIG\" WHERE word LIKE '{}%' AND LENGTH(word) > {} " \
"UNION ALL SELECT COUNT(*), NULL FROM \"AOTRIG\" WHERE word='{}' " \
"UNION ALL SELECT COUNT(*), MAX(scname) FROM \"SOCRBASE\" WHERE socrname ILIKE '{}'" \
"UNION ALL SELECT COUNT(*), NULL FROM \"SOCRBASE\" WHERE scname ILIKE '{}';".format(
self.word, self.word_len, self.word, self.word, self.word)
"UNION ALL SELECT COUNT(*), NULL FROM \"SOCRBASE\" WHERE scname ILIKE '{}'" \
"UNION ALL SELECT frequency, NULL FROM \"AOTRIG\" WHERE word='{}';".format(
self.word, self.word_len, self.word, self.word, self.word, self.word)
result = self.db.get_rows(sql_qry)
@@ -88,6 +90,9 @@ class WordEntry:
if not self.scname:
self.scname = result[2][1]
if len(result) == 5 and result[4][0] > 30000:
self.is_freq_word = True
# Формируем список найденных величин совпадений:
# result[x]
# x = 0, поиск по неполному совпадению (лайк*), и по длине строки больше исходной
@@ -95,11 +100,11 @@ class WordEntry:
# x = 2, поиск по базе сокращений (по полному)
# x = 3, то же, но по краткому
out_mask_list = []
for ra in result:
if ra[0] > 1:
for i in range(0, 4):
if result[i][0] > 1:
out_mask_list.append('x')
else:
out_mask_list.append(str(ra[0]))
out_mask_list.append(str(result[i][0]))
return ''.join(out_mask_list)