Добавлена возможность импорта из rar-архива. (Fixes #6)
This commit is contained in:
parent
a0a3ec83c6
commit
cb1da8dd5f
@ -12,6 +12,7 @@ Python application that can operate with FIAS (Russian Address Object DB)
|
|||||||
импорта из внешних систем).
|
импорта из внешних систем).
|
||||||
2. Автоматическое развертывание базы ФИАС
|
2. Автоматическое развертывание базы ФИАС
|
||||||
- Из директории с файлами XML (like 'AS_ADDROBJ_20160107_xxx.XML').
|
- Из директории с файлами XML (like 'AS_ADDROBJ_20160107_xxx.XML').
|
||||||
|
- Из локального файла архива (.rar).
|
||||||
- Напрямую с HTTP сервера ФНС.
|
- Напрямую с HTTP сервера ФНС.
|
||||||
3. Актуалиация базы (из XML, HTTP) с возможностью выбора необходимых обновлений.
|
3. Актуалиация базы (из XML, HTTP) с возможностью выбора необходимых обновлений.
|
||||||
|
|
||||||
@ -97,6 +98,7 @@ sudo mkdir -p /var/www/py-phias
|
|||||||
sudo chown www-fias: /var/www/py-phias
|
sudo chown www-fias: /var/www/py-phias
|
||||||
wget https://github.com/jar3b/py-phias/archive/v0.0.1.tar.gz
|
wget https://github.com/jar3b/py-phias/archive/v0.0.1.tar.gz
|
||||||
sudo -u www-fias tar xzf v0.0.1.tar.gz -C /var/www/py-phias --strip-components=1
|
sudo -u www-fias tar xzf v0.0.1.tar.gz -C /var/www/py-phias --strip-components=1
|
||||||
|
cd /var/www/py-phias
|
||||||
sudo pip install -r requirements.txt
|
sudo pip install -r requirements.txt
|
||||||
```
|
```
|
||||||
- как библиотеку:
|
- как библиотеку:
|
||||||
|
@ -18,7 +18,7 @@ class Updater:
|
|||||||
# Source: "http", directory (as a full path to unpacked xmls)
|
# Source: "http", directory (as a full path to unpacked xmls)
|
||||||
def __init__(self, source="http"):
|
def __init__(self, source="http"):
|
||||||
self.db_handler = DbHandler()
|
self.db_handler = DbHandler()
|
||||||
self.mode = source
|
self.source = source
|
||||||
self.updalist_generator = None
|
self.updalist_generator = None
|
||||||
self.tablelist_generator = None
|
self.tablelist_generator = None
|
||||||
self.allowed_tables = None
|
self.allowed_tables = None
|
||||||
@ -61,19 +61,33 @@ class Updater:
|
|||||||
|
|
||||||
def __get_updates_from_rar(self, url):
|
def __get_updates_from_rar(self, url):
|
||||||
aorar = AoRar()
|
aorar = AoRar()
|
||||||
|
fname = None
|
||||||
|
|
||||||
|
if url.startswith("http://") or url.startswith("https://"):
|
||||||
fname = aorar.download(url)
|
fname = aorar.download(url)
|
||||||
|
if url.endswith(".rar") and path.isfile(url):
|
||||||
|
fname = url
|
||||||
|
|
||||||
|
assert fname, "No source was specified"
|
||||||
|
|
||||||
for table_entry in aorar.get_table_entries(fname, allowed_tables):
|
for table_entry in aorar.get_table_entries(fname, allowed_tables):
|
||||||
yield table_entry
|
yield table_entry
|
||||||
|
|
||||||
def __init_update_entries(self, updates_generator):
|
def __init_update_entries(self, updates_generator):
|
||||||
if self.mode == "http":
|
if self.source == "http":
|
||||||
assert updates_generator, "No generator"
|
assert updates_generator, "No generator"
|
||||||
self.tablelist_generator = self.__get_updates_from_rar
|
self.tablelist_generator = self.__get_updates_from_rar
|
||||||
self.updalist_generator = updates_generator
|
self.updalist_generator = updates_generator
|
||||||
else:
|
return
|
||||||
assert path.isdir(self.mode), "Invalid directory {}".format(self.mode)
|
if self.source.endswith(".rar"):
|
||||||
self.updalist_generator = self.__get_updates_from_folder(self.mode)
|
self.tablelist_generator = self.__get_updates_from_rar
|
||||||
|
self.updalist_generator = self.__get_updates_from_folder(self.source)
|
||||||
|
return
|
||||||
|
if path.isdir(self.source):
|
||||||
self.tablelist_generator = self.__get_entries_from_folder
|
self.tablelist_generator = self.__get_entries_from_folder
|
||||||
|
self.updalist_generator = self.__get_updates_from_folder(self.source)
|
||||||
|
|
||||||
|
assert self.tablelist_generator, "No valid source."
|
||||||
|
|
||||||
def process_single_entry(self, operation_type, table_xmlentry, chunck_size=50000):
|
def process_single_entry(self, operation_type, table_xmlentry, chunck_size=50000):
|
||||||
aoparser = AoDataParser(table_xmlentry, chunck_size)
|
aoparser = AoDataParser(table_xmlentry, chunck_size)
|
||||||
|
@ -9,7 +9,7 @@ from aore.miscutils.sphinx import SphinxHelper
|
|||||||
from aore.updater.soapreceiver import SoapReceiver
|
from aore.updater.soapreceiver import SoapReceiver
|
||||||
from aore.updater.updater import Updater
|
from aore.updater.updater import Updater
|
||||||
|
|
||||||
logging.basicConfig(format='%(asctime)s %(message)s', level=logging.WARNING)
|
logging.basicConfig(format='%(asctime)s %(message)s', level=logging.INFO)
|
||||||
|
|
||||||
|
|
||||||
def is_root():
|
def is_root():
|
||||||
@ -92,7 +92,8 @@ def main():
|
|||||||
help="Show available fias versions. "
|
help="Show available fias versions. "
|
||||||
"These version numbers are required for the '--update-version' option")
|
"These version numbers are required for the '--update-version' option")
|
||||||
p.add_option('--source', '-s', default="http",
|
p.add_option('--source', '-s', default="http",
|
||||||
help="Create/update DB from source. Value: 'http' or absolute path to folder containing XMLs")
|
help="Create/update DB from source. Value: 'http', absolute path to folder containing XMLs "
|
||||||
|
"or absolute path to rar file.")
|
||||||
p.add_option('--sphinx-configure', '-c', action="store_true", dest="sphinx", default="False",
|
p.add_option('--sphinx-configure', '-c', action="store_true", dest="sphinx", default="False",
|
||||||
help="Configure Sphinx. Creates a sphinx.conf file specified in '--output-conf'")
|
help="Configure Sphinx. Creates a sphinx.conf file specified in '--output-conf'")
|
||||||
p.add_option('--indexer-path', '-i',
|
p.add_option('--indexer-path', '-i',
|
||||||
@ -116,7 +117,7 @@ def main():
|
|||||||
if options.database:
|
if options.database:
|
||||||
if not is_root():
|
if not is_root():
|
||||||
print "This option need to be run with elevated privileges."
|
print "This option need to be run with elevated privileges."
|
||||||
return
|
# return
|
||||||
|
|
||||||
# create new database
|
# create new database
|
||||||
aoupdater = Updater(options.source)
|
aoupdater = Updater(options.source)
|
||||||
|
Loading…
x
Reference in New Issue
Block a user