171 lines
6.3 KiB
Python
171 lines
6.3 KiB
Python
# -*- coding: utf-8 -*-
|
||
|
||
import logging
|
||
import os
|
||
|
||
from bottle import template
|
||
|
||
from aore.config import Folders, DatabaseConfig, SphinxConfig
|
||
from aore.miscutils.trigram import trigram
|
||
from aore.updater.aoxmltableentry import AoXmlTableEntry
|
||
from aore.updater.dbhandler import DbHandler
|
||
|
||
|
||
class SphinxHelper:
|
||
def __init__(self):
|
||
self.index_binary = None
|
||
self.files = dict()
|
||
self.aodp = DbHandler()
|
||
|
||
# Создаем временную папку, если ее нет
|
||
if not os.path.exists(Folders.temp):
|
||
os.makedirs(Folders.temp)
|
||
|
||
# оздаем 3 папки для Сфинкса
|
||
if not os.path.exists(SphinxConfig.var_dir+ '/run'):
|
||
os.makedirs(SphinxConfig.var_dir + '/run')
|
||
if not os.path.exists(SphinxConfig.var_dir+ '/log'):
|
||
os.makedirs(SphinxConfig.var_dir + '/log')
|
||
if not os.path.exists(SphinxConfig.var_dir+ '/data'):
|
||
os.makedirs(SphinxConfig.var_dir + '/data')
|
||
|
||
def configure_indexer(self, indexer_binary, config_filename):
|
||
logging.info("Start configuring Sphinx...")
|
||
self.index_binary = indexer_binary
|
||
|
||
# Create ADDROBJ config
|
||
self.files['addrobj.conf'] = self.__create_ao_index_config()
|
||
|
||
# Produce dict file
|
||
self.files['dict.txt'] = self.__create_suggestion_dict()
|
||
|
||
# Put dict into db
|
||
self.files['dict.csv'] = self.__dbexport_sugg_dict()
|
||
|
||
# Create SUGGEST config
|
||
self.files['suggest.conf'] = self.__create_sugg_index_config()
|
||
|
||
# Create main config (sphinx.conf)
|
||
out_fname = self.__create_main_config(config_filename)
|
||
|
||
# Indexing both configs
|
||
run_index_cmd = "{} -c {} --all --rotate".format(self.index_binary, out_fname)
|
||
logging.info("Indexing main (%s)...", out_fname)
|
||
os.system(run_index_cmd)
|
||
logging.info("All indexes were created.")
|
||
|
||
# remove temp files
|
||
for fname, fpath in self.files.iteritems():
|
||
try:
|
||
os.remove(fpath)
|
||
except:
|
||
logging.warning("Cannot delete %s. Not accessible.", fpath)
|
||
logging.info("Temporary files removed.")
|
||
logging.info("Successfully configured. Please restart searchd.")
|
||
|
||
def __create_sugg_index_config(self):
|
||
fname = os.path.abspath(Folders.temp + "/suggest.conf")
|
||
logging.info("Creating config %s", fname)
|
||
|
||
conf_data = template('aore/templates/sphinx/idx_suggest.conf', db_host=DatabaseConfig.host,
|
||
db_user=DatabaseConfig.user,
|
||
db_password=DatabaseConfig.password,
|
||
db_name=DatabaseConfig.database, db_port=DatabaseConfig.port,
|
||
index_name=SphinxConfig.index_sugg,
|
||
sphinx_var_path=SphinxConfig.var_dir)
|
||
|
||
f = open(fname, "w")
|
||
f.write(conf_data)
|
||
f.close()
|
||
|
||
logging.info("Done.")
|
||
|
||
return fname
|
||
|
||
def __dbexport_sugg_dict(self):
|
||
logging.info("Place suggestion dict to DB %s...", self.files['dict.txt'])
|
||
dict_dat_fname = os.path.abspath(Folders.temp + "/suggdict.csv")
|
||
|
||
csv_counter = 0
|
||
with open(self.files['dict.txt'], "r") as dict_file, open(dict_dat_fname, "w") as exit_file:
|
||
line = None
|
||
while line != '':
|
||
nodes = []
|
||
line = dict_file.readline()
|
||
if line == '':
|
||
break
|
||
csv_counter += 1
|
||
splitting_seq = line.split(' ')
|
||
keyword = splitting_seq[0]
|
||
freq = splitting_seq[1].rstrip('\n')
|
||
assert keyword and freq, "Cannot process {}".format(self.files['dict.txt'])
|
||
|
||
nodes.append(keyword)
|
||
nodes.append(trigram(keyword))
|
||
nodes.append(freq)
|
||
|
||
exit_file.write("\t".join(nodes) + "\n")
|
||
try:
|
||
dict_file.close()
|
||
exit_file.close()
|
||
except:
|
||
pass
|
||
|
||
self.aodp.bulk_csv(AoXmlTableEntry.OperationType.update, "AOTRIG", csv_counter, dict_dat_fname)
|
||
logging.info("Done.")
|
||
|
||
def __create_ao_index_config(self):
|
||
fname = os.path.abspath(Folders.temp + "/addrobj.conf")
|
||
logging.info("Creating config %s", fname)
|
||
|
||
conf_data = template('aore/templates/sphinx/idx_addrobj.conf', db_host=DatabaseConfig.host,
|
||
db_user=DatabaseConfig.user,
|
||
db_password=DatabaseConfig.password,
|
||
db_name=DatabaseConfig.database, db_port=DatabaseConfig.port,
|
||
sql_query=template('aore/templates/postgre/sphinx_query.sql').replace("\n", " \\\n"),
|
||
index_name=SphinxConfig.index_addjobj,
|
||
sphinx_var_path=SphinxConfig.var_dir,
|
||
min_length_to_star=SphinxConfig.min_length_to_star)
|
||
|
||
f = open(fname, "w")
|
||
f.write(conf_data)
|
||
f.close()
|
||
|
||
logging.info("Done.")
|
||
|
||
return fname
|
||
|
||
def __create_suggestion_dict(self):
|
||
fname = os.path.abspath(Folders.temp + "/suggdict.txt")
|
||
logging.info("Make suggestion dict (%s)...", fname)
|
||
|
||
run_builddict_cmd = "{} {} -c {} --buildstops {} 200000 --buildfreqs".format(self.index_binary,
|
||
SphinxConfig.index_addjobj,
|
||
self.files['addrobj.conf'], fname)
|
||
os.system(run_builddict_cmd)
|
||
logging.info("Done.")
|
||
|
||
return fname
|
||
|
||
def __create_main_config(self, config_fname):
|
||
out_filename = os.path.abspath(config_fname)
|
||
logging.info("Creating main config %s...", out_filename)
|
||
|
||
conf_data = template('aore/templates/sphinx/sphinx.conf',
|
||
sphinx_listen=SphinxConfig.listen.replace("unix://", ''),
|
||
sphinx_var_path=SphinxConfig.var_dir)
|
||
|
||
f = open(out_filename, "w")
|
||
for fname, fpath in self.files.iteritems():
|
||
if ".conf" in fname:
|
||
with open(fpath, "r") as conff:
|
||
for line in conff:
|
||
f.write(line)
|
||
f.write('\n')
|
||
f.write(conf_data)
|
||
f.close()
|
||
|
||
logging.info("Done.")
|
||
|
||
return out_filename
|