Reformatted
This commit is contained in:
parent
9803b1d25a
commit
acfee0cd8c
@ -1,9 +1,10 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
import os
|
||||
|
||||
from aore.aoutils.aoxmltableentry import AoXmlTableEntry
|
||||
from xmlparser import XMLParser
|
||||
from aore.config import trashfolder
|
||||
from aore.dbutils.dbschemas import db_shemas
|
||||
import os
|
||||
from xmlparser import XMLParser
|
||||
|
||||
|
||||
class AoDataParser:
|
||||
|
@ -1,10 +1,12 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import requests
|
||||
import os.path
|
||||
import rarfile
|
||||
from aore.config import unrar, trashfolder
|
||||
from traceback import format_exc
|
||||
|
||||
import rarfile
|
||||
import requests
|
||||
|
||||
from aore.config import unrar, trashfolder
|
||||
from aoxmltableentry import AoXmlTableEntry
|
||||
|
||||
|
||||
|
@ -1,13 +1,15 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import logging
|
||||
from os import walk, path
|
||||
|
||||
from aore.aoutils.aodataparser import AoDataParser
|
||||
from aore.aoutils.aorar import AoRar
|
||||
from aore.aoutils.aoxmltableentry import AoXmlTableEntry
|
||||
from aore.aoutils.importer import Importer
|
||||
from aore.dbutils.dbhandler import DbHandler
|
||||
from aore.dbutils.dbschemas import allowed_tables
|
||||
from aore.aoutils.importer import Importer
|
||||
from os import walk, path
|
||||
import logging
|
||||
|
||||
|
||||
class AoUpdater:
|
||||
# Source: "http", directory (as a full path to unpacked xmls)
|
||||
|
@ -1,6 +1,7 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import re
|
||||
|
||||
from enum import Enum
|
||||
|
||||
|
||||
|
@ -23,7 +23,7 @@ class Importer:
|
||||
assert downloadfileinfo.VersionId < self.get_current_fias_version(), "DB is already up-to-date"
|
||||
|
||||
yield dict(intver=int(downloadfileinfo.VersionId), strver=str(downloadfileinfo.TextVersion),
|
||||
url=str(downloadfileinfo.FiasCompleteXmlUrl))
|
||||
url=str(downloadfileinfo.FiasCompleteXmlUrl))
|
||||
|
||||
# return (intver, strver, url)
|
||||
def get_updates(self):
|
||||
@ -36,4 +36,4 @@ class Importer:
|
||||
for DownloadFileInfo in response.GetAllDownloadFileInfoResponse.GetAllDownloadFileInfoResult.DownloadFileInfo:
|
||||
if int(DownloadFileInfo.VersionId) > current_fias_version:
|
||||
yield dict(intver=int(DownloadFileInfo.VersionId), strver=str(DownloadFileInfo.TextVersion),
|
||||
url=str(DownloadFileInfo.FiasDeltaXmlUrl))
|
||||
url=str(DownloadFileInfo.FiasDeltaXmlUrl))
|
||||
|
@ -21,4 +21,4 @@ class XMLParser:
|
||||
|
||||
def parse_buffer(self, data_buffer, tag_name):
|
||||
context = etree.iterparse(data_buffer, events=('end',), tag=tag_name)
|
||||
self.fast_iter(context, lambda x: self.parse_function(x.attrib))
|
||||
self.fast_iter(context, lambda x: self.parse_function(x.attrib))
|
||||
|
@ -1,11 +1,13 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
from aore.dbutils.dbimpl import DBImpl
|
||||
from aore.config import db as dbparams
|
||||
from aore.dbutils.dbschemas import db_shemas, allowed_tables
|
||||
from traceback import format_exc
|
||||
import psycopg2
|
||||
import logging
|
||||
from traceback import format_exc
|
||||
|
||||
import psycopg2
|
||||
|
||||
from aore.config import db as dbparams
|
||||
from aore.dbutils.dbimpl import DBImpl
|
||||
from aore.dbutils.dbschemas import db_shemas
|
||||
|
||||
|
||||
class DbHandler:
|
||||
@ -13,12 +15,12 @@ class DbHandler:
|
||||
logging.basicConfig(format='%(asctime)s %(message)s')
|
||||
self.db = DBImpl(psycopg2, dbparams)
|
||||
|
||||
def bulk_csv(self, chunck_size, table_name, csv_file_name):
|
||||
def bulk_csv(self, chunk_size, table_name, csv_file_name):
|
||||
sql_query = "COPY \"{}\" ({}) FROM '{}' DELIMITER '\t' NULL 'NULL'". \
|
||||
format(table_name,
|
||||
", ".join(
|
||||
db_shemas[table_name].fields),
|
||||
csv_file_name)
|
||||
db_shemas[table_name].fields),
|
||||
csv_file_name)
|
||||
try:
|
||||
cur = self.db.get_cursor()
|
||||
cur.execute(sql_query)
|
||||
@ -27,7 +29,7 @@ class DbHandler:
|
||||
self.db.transaction_rollback()
|
||||
logging.error("Error updating sql. Reason : {}".format(format_exc()))
|
||||
|
||||
logging.warning("Inserted {} queries FROM {}".format(chunck_size, csv_file_name))
|
||||
logging.warning("Inserted {} queries FROM {}".format(chunk_size, csv_file_name))
|
||||
|
||||
def pre_create(self):
|
||||
f = open("aore/templates/postgre/pre_create.sql")
|
||||
@ -44,4 +46,4 @@ class DbHandler:
|
||||
|
||||
def pre_update(self):
|
||||
# TODO: update actions
|
||||
pass
|
||||
pass
|
||||
|
@ -30,4 +30,4 @@ class DBImpl:
|
||||
if cur:
|
||||
cur.close()
|
||||
|
||||
return rows
|
||||
return rows
|
||||
|
@ -1,8 +1,9 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
from aore.aoutils.aoupdater import AoUpdater
|
||||
import optparse
|
||||
|
||||
from aore.aoutils.aoupdater import AoUpdater
|
||||
|
||||
|
||||
def update_base(updates_count):
|
||||
aoupdater = AoUpdater()
|
||||
|
@ -11,4 +11,5 @@ sys.setdefaultencoding("utf-8")
|
||||
sys.path.append('/home/i/interc7j/.local/lib/python2.7/site-packages')
|
||||
|
||||
from fias import fias
|
||||
|
||||
application = fias.app
|
||||
|
Loading…
x
Reference in New Issue
Block a user