Reformatted
This commit is contained in:
parent
9803b1d25a
commit
acfee0cd8c
@ -1,9 +1,10 @@
|
|||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
|
import os
|
||||||
|
|
||||||
from aore.aoutils.aoxmltableentry import AoXmlTableEntry
|
from aore.aoutils.aoxmltableentry import AoXmlTableEntry
|
||||||
from xmlparser import XMLParser
|
|
||||||
from aore.config import trashfolder
|
from aore.config import trashfolder
|
||||||
from aore.dbutils.dbschemas import db_shemas
|
from aore.dbutils.dbschemas import db_shemas
|
||||||
import os
|
from xmlparser import XMLParser
|
||||||
|
|
||||||
|
|
||||||
class AoDataParser:
|
class AoDataParser:
|
||||||
|
@ -1,10 +1,12 @@
|
|||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
import requests
|
|
||||||
import os.path
|
import os.path
|
||||||
import rarfile
|
|
||||||
from aore.config import unrar, trashfolder
|
|
||||||
from traceback import format_exc
|
from traceback import format_exc
|
||||||
|
|
||||||
|
import rarfile
|
||||||
|
import requests
|
||||||
|
|
||||||
|
from aore.config import unrar, trashfolder
|
||||||
from aoxmltableentry import AoXmlTableEntry
|
from aoxmltableentry import AoXmlTableEntry
|
||||||
|
|
||||||
|
|
||||||
|
@ -1,13 +1,15 @@
|
|||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
|
import logging
|
||||||
|
from os import walk, path
|
||||||
|
|
||||||
from aore.aoutils.aodataparser import AoDataParser
|
from aore.aoutils.aodataparser import AoDataParser
|
||||||
from aore.aoutils.aorar import AoRar
|
from aore.aoutils.aorar import AoRar
|
||||||
from aore.aoutils.aoxmltableentry import AoXmlTableEntry
|
from aore.aoutils.aoxmltableentry import AoXmlTableEntry
|
||||||
|
from aore.aoutils.importer import Importer
|
||||||
from aore.dbutils.dbhandler import DbHandler
|
from aore.dbutils.dbhandler import DbHandler
|
||||||
from aore.dbutils.dbschemas import allowed_tables
|
from aore.dbutils.dbschemas import allowed_tables
|
||||||
from aore.aoutils.importer import Importer
|
|
||||||
from os import walk, path
|
|
||||||
import logging
|
|
||||||
|
|
||||||
class AoUpdater:
|
class AoUpdater:
|
||||||
# Source: "http", directory (as a full path to unpacked xmls)
|
# Source: "http", directory (as a full path to unpacked xmls)
|
||||||
|
@ -1,6 +1,7 @@
|
|||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
import re
|
import re
|
||||||
|
|
||||||
from enum import Enum
|
from enum import Enum
|
||||||
|
|
||||||
|
|
||||||
|
@ -23,7 +23,7 @@ class Importer:
|
|||||||
assert downloadfileinfo.VersionId < self.get_current_fias_version(), "DB is already up-to-date"
|
assert downloadfileinfo.VersionId < self.get_current_fias_version(), "DB is already up-to-date"
|
||||||
|
|
||||||
yield dict(intver=int(downloadfileinfo.VersionId), strver=str(downloadfileinfo.TextVersion),
|
yield dict(intver=int(downloadfileinfo.VersionId), strver=str(downloadfileinfo.TextVersion),
|
||||||
url=str(downloadfileinfo.FiasCompleteXmlUrl))
|
url=str(downloadfileinfo.FiasCompleteXmlUrl))
|
||||||
|
|
||||||
# return (intver, strver, url)
|
# return (intver, strver, url)
|
||||||
def get_updates(self):
|
def get_updates(self):
|
||||||
@ -36,4 +36,4 @@ class Importer:
|
|||||||
for DownloadFileInfo in response.GetAllDownloadFileInfoResponse.GetAllDownloadFileInfoResult.DownloadFileInfo:
|
for DownloadFileInfo in response.GetAllDownloadFileInfoResponse.GetAllDownloadFileInfoResult.DownloadFileInfo:
|
||||||
if int(DownloadFileInfo.VersionId) > current_fias_version:
|
if int(DownloadFileInfo.VersionId) > current_fias_version:
|
||||||
yield dict(intver=int(DownloadFileInfo.VersionId), strver=str(DownloadFileInfo.TextVersion),
|
yield dict(intver=int(DownloadFileInfo.VersionId), strver=str(DownloadFileInfo.TextVersion),
|
||||||
url=str(DownloadFileInfo.FiasDeltaXmlUrl))
|
url=str(DownloadFileInfo.FiasDeltaXmlUrl))
|
||||||
|
@ -21,4 +21,4 @@ class XMLParser:
|
|||||||
|
|
||||||
def parse_buffer(self, data_buffer, tag_name):
|
def parse_buffer(self, data_buffer, tag_name):
|
||||||
context = etree.iterparse(data_buffer, events=('end',), tag=tag_name)
|
context = etree.iterparse(data_buffer, events=('end',), tag=tag_name)
|
||||||
self.fast_iter(context, lambda x: self.parse_function(x.attrib))
|
self.fast_iter(context, lambda x: self.parse_function(x.attrib))
|
||||||
|
@ -1,11 +1,13 @@
|
|||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
from aore.dbutils.dbimpl import DBImpl
|
|
||||||
from aore.config import db as dbparams
|
|
||||||
from aore.dbutils.dbschemas import db_shemas, allowed_tables
|
|
||||||
from traceback import format_exc
|
|
||||||
import psycopg2
|
|
||||||
import logging
|
import logging
|
||||||
|
from traceback import format_exc
|
||||||
|
|
||||||
|
import psycopg2
|
||||||
|
|
||||||
|
from aore.config import db as dbparams
|
||||||
|
from aore.dbutils.dbimpl import DBImpl
|
||||||
|
from aore.dbutils.dbschemas import db_shemas
|
||||||
|
|
||||||
|
|
||||||
class DbHandler:
|
class DbHandler:
|
||||||
@ -13,12 +15,12 @@ class DbHandler:
|
|||||||
logging.basicConfig(format='%(asctime)s %(message)s')
|
logging.basicConfig(format='%(asctime)s %(message)s')
|
||||||
self.db = DBImpl(psycopg2, dbparams)
|
self.db = DBImpl(psycopg2, dbparams)
|
||||||
|
|
||||||
def bulk_csv(self, chunck_size, table_name, csv_file_name):
|
def bulk_csv(self, chunk_size, table_name, csv_file_name):
|
||||||
sql_query = "COPY \"{}\" ({}) FROM '{}' DELIMITER '\t' NULL 'NULL'". \
|
sql_query = "COPY \"{}\" ({}) FROM '{}' DELIMITER '\t' NULL 'NULL'". \
|
||||||
format(table_name,
|
format(table_name,
|
||||||
", ".join(
|
", ".join(
|
||||||
db_shemas[table_name].fields),
|
db_shemas[table_name].fields),
|
||||||
csv_file_name)
|
csv_file_name)
|
||||||
try:
|
try:
|
||||||
cur = self.db.get_cursor()
|
cur = self.db.get_cursor()
|
||||||
cur.execute(sql_query)
|
cur.execute(sql_query)
|
||||||
@ -27,7 +29,7 @@ class DbHandler:
|
|||||||
self.db.transaction_rollback()
|
self.db.transaction_rollback()
|
||||||
logging.error("Error updating sql. Reason : {}".format(format_exc()))
|
logging.error("Error updating sql. Reason : {}".format(format_exc()))
|
||||||
|
|
||||||
logging.warning("Inserted {} queries FROM {}".format(chunck_size, csv_file_name))
|
logging.warning("Inserted {} queries FROM {}".format(chunk_size, csv_file_name))
|
||||||
|
|
||||||
def pre_create(self):
|
def pre_create(self):
|
||||||
f = open("aore/templates/postgre/pre_create.sql")
|
f = open("aore/templates/postgre/pre_create.sql")
|
||||||
@ -44,4 +46,4 @@ class DbHandler:
|
|||||||
|
|
||||||
def pre_update(self):
|
def pre_update(self):
|
||||||
# TODO: update actions
|
# TODO: update actions
|
||||||
pass
|
pass
|
||||||
|
@ -30,4 +30,4 @@ class DBImpl:
|
|||||||
if cur:
|
if cur:
|
||||||
cur.close()
|
cur.close()
|
||||||
|
|
||||||
return rows
|
return rows
|
||||||
|
@ -1,8 +1,9 @@
|
|||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
from aore.aoutils.aoupdater import AoUpdater
|
|
||||||
import optparse
|
import optparse
|
||||||
|
|
||||||
|
from aore.aoutils.aoupdater import AoUpdater
|
||||||
|
|
||||||
|
|
||||||
def update_base(updates_count):
|
def update_base(updates_count):
|
||||||
aoupdater = AoUpdater()
|
aoupdater = AoUpdater()
|
||||||
|
@ -11,4 +11,5 @@ sys.setdefaultencoding("utf-8")
|
|||||||
sys.path.append('/home/i/interc7j/.local/lib/python2.7/site-packages')
|
sys.path.append('/home/i/interc7j/.local/lib/python2.7/site-packages')
|
||||||
|
|
||||||
from fias import fias
|
from fias import fias
|
||||||
|
|
||||||
application = fias.app
|
application = fias.app
|
||||||
|
Loading…
x
Reference in New Issue
Block a user