Переделан конфиг (под dev и production)

aore/config.py

@@ -1,50 +0,0 @@
-# -*- coding: utf-8 -*-
-
-from platform import system
-
-config_type = "production"
-if "Windows" in system():
-    config_type = "test"
-
-DB_INSTANCES = dict(
-    test=dict(
-        host="localhost",
-        user="postgres",
-        password="intercon",
-        database="postgres",
-        port=5432
-    ),
-    production=dict(
-        host="localhost",
-        user="***",
-        password="***",
-        database="***",
-        port=5432
-    )
-)
-
-UNRAR_PATHES = dict(
-    test="C:\Program Files (x86)\WinRAR\unrar.exe",
-    production="unrar"
-)
-
-SPHINX_VAR_DIRS = dict(
-    test="C:/Sphinx",
-    production="/var/sphinx"
-)
-
-# Uncomment if you want to specify config_type manually
-# config_type = "test"
-
-# Main section
-sphinx = dict(
-    host_name="localhost",
-    port=9312,
-    index_addjobj="idx_fias_addrobj",
-    index_sugg="idx_fias_sugg",
-    var_dir=SPHINX_VAR_DIRS[config_type]
-)
-
-db = DB_INSTANCES[config_type]
-unrar = UNRAR_PATHES[config_type]
-trashfolder = "files/"

aore/config/__init__.py

@@ -0,0 +1,4 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import absolute_import
+from .dev import *

aore/config/common.py

@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+
+class sphinx_conf:
+    host_name = "localhost"
+    port = 9312
+    index_addjobj = "idx_fias_addrobj"
+    index_sugg = "idx_fias_sugg"
+    var_dir = None
+
+
+class db_conf:
+    host = None
+    user = None
+    password = None
+    database = None
+    port = None
+
+
+class unrar_config:
+    path = None
+
+
+class folders:
+    temp = None

aore/config/dev.py

@@ -0,0 +1,16 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import absolute_import
+
+from .common import *
+
+sphinx_conf.var_dir = "C:\\Sphinx"
+
+db_conf.database = "postgres"
+db_conf.host = "localhost"
+db_conf.port = 5432
+db_conf.user = "postgres"
+db_conf.password = "intercon"
+
+unrar_config.path = "C:\Program Files (x86)\WinRAR\unrar.exe"
+folders.temp = "E:\\!TEMP"

aore/dbutils/dbimpl.py

@@ -1,13 +1,15 @@
 # -*- coding: utf-8 -*-
 
 from traceback import format_exc
+
 import psycopg2.extras
 
 
 class DBImpl:
-    def __init__(self, engine, params):
+    def __init__(self, engine, db_config):
         self.db_engine = engine
-        self.connection = engine.connect(**params)
+        self.connection = engine.connect(dbname=db_config.database, user=db_config.user, password=db_config.password,
+                                         port=db_config.port, host=db_config.host)
 
     def transaction_commit(self):
         self.connection.commit()

aore/fias/fiasfactory.py

@@ -4,19 +4,20 @@ from bottle import template
 
 from aore.dbutils.dbimpl import DBImpl
 from aore.fias.search import SphinxSearch
-from aore.config import db as dbparams
+from aore.config import db_conf
 
 
 class FiasFactory:
     def __init__(self):
-        self.db = DBImpl(psycopg2, dbparams)
+        self.db = DBImpl(psycopg2, db_conf)
         self.searcher = SphinxSearch(self.db)
         self.expand_templ = template('aore/templates/postgre/expand_query.sql', aoid="//aoid")
+        self.normalize_templ = template('aore/templates/postgre/normalize_query.sql', aoid="//aoid")
 
     # text - строка поиска
     # strong - строгий поиск (True) или "мягкий" (False) (с допущением ошибок, опечаток)
-    # out_format - "full" or "simple" - полный (подробно для каждого подпункта) или простой (только строка и AOID)
+    # Строгий используется при импорте из внешних систем (автоматически), где ошибка критична
-    def find(self, text, strong=False, out_format="simple"):
+    def find(self, text, strong=False):
         try:
             results = self.searcher.find(text, strong)
         except Exception, err:
@@ -26,12 +27,26 @@ class FiasFactory:
 
     # Нормализует подаваемый AOID или AOGUID в актуальный AOID
     def normalize(self, aoid_guid):
-        pass
+        try:
+            sql_query = self.normalize_templ.replace("//aoid", aoid_guid)
+            rows = self.db.get_rows(sql_query, True)
+        except Exception, err:
+            return dict(error=err.args[0])
+
+        if len(rows) == 0:
+            return []
+        else:
+            return rows[0]
 
     # Разворачивает AOID в представление (перед этим нормализует)
     def expand(self, aoid_guid):
         try:
-            sql_query = self.expand_templ.replace("//aoid", aoid_guid)
+            normalized_id = self.normalize(aoid_guid)
+            if 'aoid' not in normalized_id:
+                raise BaseException("Invalid AOID or AOGUID")
+            else:
+                normalized_id = normalized_id['aoid']
+            sql_query = self.expand_templ.replace("//aoid", normalized_id)
             rows = self.db.get_rows(sql_query, True)
         except Exception, err:
             return dict(error=err.args[0])

aore/fias/search.py

@@ -5,7 +5,7 @@ import re
 import Levenshtein
 import sphinxapi
 
-from aore.config import sphinx
+from aore.config import sphinx_conf
 from aore.fias.wordentry import WordEntry
 from aore.miscutils.trigram import trigram
 
@@ -26,17 +26,17 @@ class SphinxSearch:
 
         self.db = db
         self.client_sugg = sphinxapi.SphinxClient()
-        self.client_sugg.SetServer(sphinx.host, sphinx.port)
+        self.client_sugg.SetServer(sphinx_conf.host_name, sphinx_conf.port)
         self.client_sugg.SetLimits(0, 10)
         self.client_sugg.SetConnectTimeout(3.0)
 
         self.client_show = sphinxapi.SphinxClient()
-        self.client_show.SetServer(sphinx.host, sphinx.port)
+        self.client_show.SetServer(sphinx_conf.host_name, sphinx_conf.port)
         self.client_show.SetLimits(0, 10)
         self.client_show.SetConnectTimeout(3.0)
 
     def __configure(self, index_name, wlen=None):
-        if index_name == sphinx.index_sugg:
+        if index_name == sphinx_conf.index_sugg:
             if wlen:
                 self.client_sugg.SetMatchMode(sphinxapi.SPH_MATCH_EXTENDED2)
                 self.client_sugg.SetRankingMode(sphinxapi.SPH_RANK_WORDCOUNT)
@@ -52,8 +52,8 @@ class SphinxSearch:
         word_len = str(len(word) / 2)
         trigrammed_word = '"{}"/1'.format(trigram(word))
 
-        self.__configure(sphinx.index_sugg, word_len)
+        self.__configure(sphinx_conf.index_sugg, word_len)
-        result = self.client_sugg.Query(trigrammed_word, sphinx.index_sugg)
+        result = self.client_sugg.Query(trigrammed_word, sphinx_conf.index_sugg)
 
         # Если по данному слову не найдено подсказок (а такое бывает?)
         # возвращаем []
@@ -67,7 +67,7 @@ class SphinxSearch:
         outlist = list()
         for match in result['matches']:
             if len(outlist) >= count:
-                break;
+                break
 
             if maxrank - match['attrs']['krank'] < self.default_rating_delta:
                 jaro_rating = Levenshtein.jaro(word, match['attrs']['word'])
@@ -117,9 +117,9 @@ class SphinxSearch:
         word_entries = self.__get_word_entries(words, strong)
         sentence = "{}".format(" MAYBE ".join(x.get_variations() for x in word_entries))
 
-        self.__configure(sphinx.index_addjobj)
+        self.__configure(sphinx_conf.index_addjobj)
         logging.info("QUERY " + sentence)
-        rs = self.client_show.Query(sentence, sphinx.index_addjobj)
+        rs = self.client_show.Query(sentence, sphinx_conf.index_addjobj)
         logging.info("OK")
 
         results = []
@@ -128,4 +128,5 @@ class SphinxSearch:
 
         if strong:
             results.sort(key=lambda x: Levenshtein.ratio(text, x['text']), reverse=True)
+
         return results

aore/fias/wordentry.py

@@ -1,6 +1,5 @@
 # -*- coding: utf-8 -*-
 import re
-import logging
 
 
 class WordEntry:

aore/miscutils/sphinx.py

@@ -5,9 +5,9 @@ import os
 
 from bottle import template
 
+from aore.config import folders, db_conf, sphinx_conf
 from aore.updater.aoxmltableentry import AoXmlTableEntry
 from aore.updater.dbhandler import DbHandler
-from aore.config import db as dbconfig, sphinx, trashfolder
 from trigram import trigram
 
 
@@ -51,15 +51,15 @@ class SphinxHelper:
         logging.info("Successfully configured. Please restart searchd.")
 
     def __create_sugg_index_config(self):
-        fname = os.path.abspath(trashfolder + "suggest.conf")
+        fname = os.path.abspath(folders.temp + "/suggest.conf")
         logging.info("Creating config {}".format(fname))
 
-        conf_data = template('aore/templates/sphinx/idx_suggest.conf', db_host=dbconfig['host'],
+        conf_data = template('aore/templates/sphinx/idx_suggest.conf', db_host=db_conf.host,
-                             db_user=dbconfig['user'],
+                             db_user=db_conf.user,
-                             db_password=dbconfig['password'],
+                             db_password=db_conf.password,
-                             db_name=dbconfig['database'], db_port=dbconfig['port'],
+                             db_name=db_conf.database, db_port=db_conf.port,
-                             index_name=sphinx.index_sugg,
+                             index_name=sphinx_conf.index_sugg,
-                             sphinx_var_path=sphinx.var_dir)
+                             sphinx_var_path=sphinx_conf.var_dir)
 
         f = open(fname, "w")
         f.write(conf_data)
@@ -71,7 +71,7 @@ class SphinxHelper:
 
     def __dbexport_sugg_dict(self):
         logging.info("Place suggestion dict to DB {}...".format(self.files['dict.txt']))
-        dict_dat_fname = os.path.abspath(trashfolder + "suggdict.csv")
+        dict_dat_fname = os.path.abspath(folders.temp + "/suggdict.csv")
 
         csv_counter = 0
         with open(self.files['dict.txt'], "r") as dict_file, open(dict_dat_fname, "w") as exit_file:
@@ -104,16 +104,16 @@ class SphinxHelper:
         logging.info("Done.")
 
     def __create_ao_index_config(self):
-        fname = os.path.abspath(trashfolder + "addrobj.conf")
+        fname = os.path.abspath(folders.temp + "/addrobj.conf")
         logging.info("Creating config {}".format(fname))
 
-        conf_data = template('aore/templates/sphinx/idx_addrobj.conf', db_host=dbconfig['host'],
+        conf_data = template('aore/templates/sphinx/idx_addrobj.conf', db_host=db_conf.host,
-                             db_user=dbconfig['user'],
+                             db_user=db_conf.user,
-                             db_password=dbconfig['password'],
+                             db_password=db_conf.password,
-                             db_name=dbconfig['database'], db_port=dbconfig['port'],
+                             db_name=db_conf.database, db_port=db_conf.port,
                              sql_query=template('aore/templates/postgre/sphinx_query.sql').replace("\n", " \\\n"),
-                             index_name=sphinx.index_addjobj,
+                             index_name=sphinx_conf.index_addjobj,
-                             sphinx_var_path=sphinx.var_dir)
+                             sphinx_var_path=sphinx_conf.var_dir)
 
         f = open(fname, "w")
         f.write(conf_data)
@@ -124,11 +124,11 @@ class SphinxHelper:
         return fname
 
     def __create_suggestion_dict(self):
-        fname = os.path.abspath(trashfolder + "suggdict.txt")
+        fname = os.path.abspath(folders.temp + "/suggdict.txt")
         logging.info("Make suggestion dict ({})...".format(fname))
 
         run_builddict_cmd = "{} {} -c {} --buildstops {} 200000 --buildfreqs".format(self.index_binary,
-                                                                                     sphinx.index_addjobj,
+                                                                                     sphinx_conf.index_addjobj,
                                                                                      self.files['addrobj.conf'], fname)
         os.system(run_builddict_cmd)
         logging.info("Done.")
@@ -139,7 +139,7 @@ class SphinxHelper:
         out_filename = os.path.abspath(config_fname)
         logging.info("Creating main config {}...".format(out_filename))
 
-        conf_data = template('aore/templates/sphinx/sphinx.conf', sphinx_var_path=sphinx.var_dir)
+        conf_data = template('aore/templates/sphinx/sphinx.conf', sphinx_var_path=sphinx_conf.var_dir)
 
         f = open(out_filename, "w")
         for fname, fpath in self.files.iteritems():

aore/templates/postgre/normalize_query.sql

@@ -0,0 +1 @@
+SELECT AOID FROM "ADDROBJ" WHERE (AOID='{{ aoid }}' OR AOGUID='{{ aoid }}') AND ACTSTATUS=True AND LIVESTATUS=True AND NEXTID IS NULL LIMIT 1;

aore/updater/aodataparser.py

@@ -1,8 +1,7 @@
 # -*- coding: utf-8 -*-
 import os
 
-from aore.updater.aoxmltableentry import AoXmlTableEntry
+from aore.config import folders
-from aore.config import trashfolder
 from aore.dbutils.dbschemas import db_shemas
 from xmlparser import XMLParser
 
@@ -52,7 +51,7 @@ class AoDataParser:
         self.data_bereit_callback = data_callback
         self.currentpage = 0
         self.base_filename = \
-            trashfolder + "fd_" + \
+            folders.temp + "/fd_" + \
             str(self.datasource.operation_type) + "_" + \
             self.datasource.table_name + ".csv.part{}"
         self.counter = self.pagesize + 1

aore/updater/aorar.py

@@ -7,21 +7,22 @@ from traceback import format_exc
 import rarfile
 import requests
 
-from aore.config import unrar, trashfolder
+from aore.config import folders, unrar_config
 from aoxmltableentry import AoXmlTableEntry
 
 
 class AoRar:
     def __init__(self):
-        rarfile.UNRAR_TOOL = unrar
+        rarfile.UNRAR_TOOL = unrar_config.path
 
     def download(self, url):
         logging.info("Downloading {}".format(url))
         try:
-            local_filename = os.path.abspath(trashfolder + url.split('/')[-1])
+            local_filename = os.path.abspath(folders.temp + "/" + url.split('/')[-1])
             if os.path.isfile(local_filename):
+                # TODO: UNCOMMENT os.remove(local_filename)
                 return local_filename
-                os.remove(local_filename)
+
 
             request = requests.get(url, stream=True)
             with open(local_filename, 'wb') as f:

aore/updater/dbhandler.py

@@ -5,15 +5,15 @@ import logging
 import psycopg2
 from bottle import template
 
-from aore.updater.aoxmltableentry import AoXmlTableEntry
+from aore.config import db_conf
-from aore.config import db as dbparams
 from aore.dbutils.dbimpl import DBImpl
 from aore.dbutils.dbschemas import db_shemas
+from aore.updater.aoxmltableentry import AoXmlTableEntry
 
 
 class DbHandler:
     def __init__(self):
-        self.db = DBImpl(psycopg2, dbparams)
+        self.db = DBImpl(psycopg2, db_conf)
 
     def bulk_csv(self, operation_type, table_name, processed_count, csv_file_name):
         sql_query = None

aore/updater/updater.py

@@ -3,12 +3,11 @@
 import logging
 from os import walk, path
 
+from aore.dbutils.dbschemas import allowed_tables
 from aore.updater.aodataparser import AoDataParser
 from aore.updater.aorar import AoRar
 from aore.updater.aoxmltableentry import AoXmlTableEntry
 from aore.updater.dbhandler import DbHandler
-from aore.updater.soapreceiver import SoapReceiver
-from aore.dbutils.dbschemas import allowed_tables
 
 
 class Updater:

manage.py

@@ -4,8 +4,8 @@ import optparse
 
 from aore.fias.fiasfactory import FiasFactory
 from aore.miscutils.sphinx import SphinxHelper
-from aore.updater.updater import Updater
 from aore.updater.soapreceiver import SoapReceiver
+from aore.updater.updater import Updater
 
 
 def print_fias_versions():
@@ -56,7 +56,7 @@ def get_allowed_updates(updates_str, mode = "create"):
         if uv_ver > current_version and (not user_defined_list or uv_ver in user_defined_list):
             out_list.append(uv)
 
-    out_list.sort(key=lambda x: x['intver'])
+    out_list.sort(key=lambda item: item['intver'])
     for ol_entry in out_list:
         yield ol_entry
 
@@ -117,9 +117,9 @@ def main():
     # 4 Debug purposes..
     if options.test:
         sph = FiasFactory()
+        print json.dumps(sph.normalize("463ce8e4-928b-45cc-be76-46c2494632b6"))
         print json.dumps(sph.expand("453091f5-2336-4aea-9b90-c4060dca0b33"))
-        print json.dumps(sph.find('с паспаул ул кедровая', True))
+        print json.dumps(sph.find('ул кемровая пасраул алтай майминский р-н'))
-        print json.dumps(sph.find('с паспаул ул кедровая'))
 
 
 if __name__ == '__main__':