125 lines
4.6 KiB
Python
125 lines
4.6 KiB
Python
# -*- coding: utf-8 -*-
|
|
|
|
import logging
|
|
import os
|
|
|
|
from bottle import template
|
|
|
|
from aore.aoutils.aoxmltableentry import AoXmlTableEntry
|
|
from aore.dbutils.dbhandler import DbHandler
|
|
from trigram import trigram
|
|
|
|
from aore.config import db as dbconfig, sphinx_index_addjobj, sphinx_var_dir, trashfolder, sphinx_index_sugg
|
|
|
|
|
|
class SphinxHelper:
|
|
def __init__(self, ):
|
|
self.index_binary = None
|
|
self.files = dict()
|
|
|
|
def configure_indexer(self, indexer_binary):
|
|
logging.info("Start configuring Sphinx...")
|
|
self.index_binary = indexer_binary
|
|
|
|
# Create ADDROBJ config
|
|
self.files['addrobj.conf'] = self.__create_ao_index_config()
|
|
|
|
# Indexing ADDROBJ config
|
|
run_index_cmd = "{} -c {} --all".format(self.index_binary, self.files['addrobj.conf'])
|
|
logging.info("Indexing main ({})...".format(sphinx_index_addjobj))
|
|
#os.system(run_index_cmd)
|
|
logging.info("{} index was created.".format(sphinx_index_addjobj))
|
|
|
|
# Produce dict file
|
|
self.files['dict.txt'] = self.__create_suggestion_dict()
|
|
|
|
# Put dict into db
|
|
#self.files['dict.csv'] = self.__dbexport_sugg_dict()
|
|
|
|
# Create SUGGEST config
|
|
self.files['suggest.conf'] = self.__create_sugg_index_config()
|
|
run_index_cmd = "{} -c {} --all".format(self.index_binary, self.files['suggest.conf'])
|
|
logging.info("Indexing main ({})...".format(sphinx_index_sugg))
|
|
os.system(run_index_cmd)
|
|
logging.info("{} index was created.".format(sphinx_index_sugg))
|
|
|
|
|
|
def __create_sugg_index_config(self):
|
|
fname = os.path.abspath(trashfolder + "suggest.conf")
|
|
logging.info("Creating config {}".format(fname))
|
|
|
|
conf_data = template('aore/templates/sphinx/idx_suggest.conf', db_host=dbconfig['host'],
|
|
db_user=dbconfig['user'],
|
|
db_password=dbconfig['password'],
|
|
db_name=dbconfig['database'], db_port=dbconfig['port'],
|
|
index_name=sphinx_index_sugg,
|
|
sphinx_var_path=sphinx_var_dir)
|
|
|
|
f = open(fname, "w")
|
|
f.write(conf_data)
|
|
f.close()
|
|
|
|
logging.info("Done.")
|
|
|
|
return fname
|
|
|
|
def __dbexport_sugg_dict(self):
|
|
logging.info("Place suggestion dict to DB {}...".format(self.files['dict.txt']))
|
|
dict_dat_fname = os.path.abspath(trashfolder + "suggdict.csv")
|
|
|
|
with open(self.files['dict.txt'], "r") as dict_file, open(dict_dat_fname, "w") as exit_file:
|
|
line = None
|
|
while line != '':
|
|
nodes = []
|
|
line = dict_file.readline()
|
|
if line == '':
|
|
break
|
|
|
|
keyword = line.split(' ')[0]
|
|
if not keyword:
|
|
raise BaseException("Cannot process {}".format(self.files['dict.txt']))
|
|
|
|
nodes.append(keyword)
|
|
nodes.append(trigram(keyword))
|
|
|
|
exit_file.write("\t".join(nodes) + "\n")
|
|
|
|
aodp = DbHandler()
|
|
aodp.bulk_csv(AoXmlTableEntry.OperationType.update, "AOTRIG", 8, dict_dat_fname)
|
|
logging.info("Done.")
|
|
|
|
def __create_ao_index_config(self):
|
|
fname = os.path.abspath(trashfolder + "addrobj.conf")
|
|
logging.info("Creating config {}".format(fname))
|
|
|
|
conf_data = template('aore/templates/sphinx/idx_addrobj.conf', db_host=dbconfig['host'],
|
|
db_user=dbconfig['user'],
|
|
db_password=dbconfig['password'],
|
|
db_name=dbconfig['database'], db_port=dbconfig['port'],
|
|
sql_query=template('aore/templates/postgre/sphinx_query.sql').replace("\n", " \\\n"),
|
|
index_name=sphinx_index_addjobj,
|
|
sphinx_var_path=sphinx_var_dir)
|
|
|
|
f = open(fname, "w")
|
|
f.write(conf_data)
|
|
f.close()
|
|
|
|
logging.info("Done.")
|
|
|
|
return fname
|
|
|
|
def __create_suggestion_dict(self):
|
|
fname = os.path.abspath(trashfolder + "suggdict.txt")
|
|
logging.info("Make suggestion dict ({})...".format(fname))
|
|
|
|
run_builddict_cmd = "{} {} -c {} --buildstops {} 200000 --buildfreqs".format(self.index_binary,
|
|
sphinx_index_addjobj,
|
|
self.files['addrobj.conf'], fname)
|
|
#os.system(run_builddict_cmd)
|
|
logging.info("Done.")
|
|
|
|
return fname
|
|
|
|
|
|
# TRASH
|
|
# conf_data = template('aore/templates/sphinx/sphinx.conf', sphinx_var_path=sphinx_var_dir) |