88 lines
3.4 KiB
Python
88 lines
3.4 KiB
Python
# -*- coding: utf-8 -*-
|
|
|
|
import logging
|
|
from os import walk, path
|
|
|
|
from aore.aoutils.aodataparser import AoDataParser
|
|
from aore.aoutils.aorar import AoRar
|
|
from aore.aoutils.aoxmltableentry import AoXmlTableEntry
|
|
from aore.aoutils.importer import Importer
|
|
from aore.dbutils.dbhandler import DbHandler
|
|
from aore.dbutils.dbschemas import allowed_tables
|
|
|
|
|
|
class AoUpdater:
|
|
# Source: "http", directory (as a full path to unpacked xmls)
|
|
def __init__(self, source="http"):
|
|
self.db_handler = DbHandler()
|
|
self.mode = source
|
|
self.updalist_generator = None
|
|
self.tablelist_generator = None
|
|
self.allowed_tables = None
|
|
|
|
def __get_entries_from_folder(self, path_to_xmls):
|
|
for (dirpath, dirnames, filenames) in walk(path_to_xmls):
|
|
for filename in filenames:
|
|
if filename.endswith(".XML"):
|
|
xmltable = AoXmlTableEntry.from_dir(filename, dirpath.replace("\\", "/") + "/")
|
|
if xmltable.table_name in allowed_tables:
|
|
yield xmltable
|
|
break
|
|
|
|
def __get_updates_from_folder(self, foldername):
|
|
# TODO: Вычислять версию, если берем данные из каталога
|
|
yield dict(intver=0, textver="Unknown", url=foldername)
|
|
|
|
def __get_updates_from_rar(self, url):
|
|
aorar = AoRar()
|
|
fname = aorar.download(url)
|
|
for table_entry in aorar.get_table_entries(fname, allowed_tables):
|
|
yield table_entry
|
|
|
|
def __init_update_entries(self, full_base):
|
|
if self.mode == "http":
|
|
self.tablelist_generator = self.__get_updates_from_rar
|
|
imp = Importer()
|
|
if full_base:
|
|
self.updalist_generator = imp.get_full()
|
|
else:
|
|
self.updalist_generator = imp.get_updates()
|
|
else:
|
|
assert path.isdir(self.mode), "Invalid directory {}".format(self.mode)
|
|
self.updalist_generator = self.__get_updates_from_folder(self.mode)
|
|
self.tablelist_generator = self.__get_entries_from_folder
|
|
|
|
def process_single_entry(self, operation_type, table_xmlentry, chunck_size=50000):
|
|
aoparser = AoDataParser(table_xmlentry, chunck_size)
|
|
aoparser.parse(lambda x, y: self.db_handler.bulk_csv(operation_type, table_xmlentry.table_name, x, y))
|
|
|
|
def create(self):
|
|
self.__init_update_entries(True)
|
|
self.db_handler.pre_create()
|
|
|
|
for update_entry in self.updalist_generator:
|
|
for table_entry in self.tablelist_generator(update_entry['url']):
|
|
if table_entry.operation_type == AoXmlTableEntry.OperationType.update:
|
|
table_entry.operation_type = AoXmlTableEntry.OperationType.create
|
|
self.process_single_entry(table_entry.operation_type, table_entry)
|
|
|
|
self.db_handler.post_create()
|
|
|
|
logging.info("Create success")
|
|
|
|
def update(self, count=1):
|
|
self.__init_update_entries(False)
|
|
self.db_handler.pre_update()
|
|
|
|
counter = 0
|
|
for update_entry in self.updalist_generator:
|
|
counter += 1
|
|
if counter > count:
|
|
logging.warning("Maximum count of updates ({}) are processed - exit".format(count))
|
|
break
|
|
|
|
for table_entry in self.tablelist_generator(update_entry['url']):
|
|
self.process_single_entry(table_entry.operation_type, table_entry)
|
|
|
|
logging.info("Update success")
|