Reformatted

This commit is contained in:
Jack Stdin 2016-01-12 23:21:06 +03:00
parent 9803b1d25a
commit acfee0cd8c
10 changed files with 33 additions and 23 deletions

View File

@ -1,9 +1,10 @@
# -*- coding: utf-8 -*-
import os
from aore.aoutils.aoxmltableentry import AoXmlTableEntry
from xmlparser import XMLParser
from aore.config import trashfolder
from aore.dbutils.dbschemas import db_shemas
import os
from xmlparser import XMLParser
class AoDataParser:

View File

@ -1,10 +1,12 @@
# -*- coding: utf-8 -*-
import requests
import os.path
import rarfile
from aore.config import unrar, trashfolder
from traceback import format_exc
import rarfile
import requests
from aore.config import unrar, trashfolder
from aoxmltableentry import AoXmlTableEntry

View File

@ -1,13 +1,15 @@
# -*- coding: utf-8 -*-
import logging
from os import walk, path
from aore.aoutils.aodataparser import AoDataParser
from aore.aoutils.aorar import AoRar
from aore.aoutils.aoxmltableentry import AoXmlTableEntry
from aore.aoutils.importer import Importer
from aore.dbutils.dbhandler import DbHandler
from aore.dbutils.dbschemas import allowed_tables
from aore.aoutils.importer import Importer
from os import walk, path
import logging
class AoUpdater:
# Source: "http", directory (as a full path to unpacked xmls)

View File

@ -1,6 +1,7 @@
# -*- coding: utf-8 -*-
import re
from enum import Enum

View File

@ -23,7 +23,7 @@ class Importer:
assert downloadfileinfo.VersionId < self.get_current_fias_version(), "DB is already up-to-date"
yield dict(intver=int(downloadfileinfo.VersionId), strver=str(downloadfileinfo.TextVersion),
url=str(downloadfileinfo.FiasCompleteXmlUrl))
url=str(downloadfileinfo.FiasCompleteXmlUrl))
# return (intver, strver, url)
def get_updates(self):
@ -36,4 +36,4 @@ class Importer:
for DownloadFileInfo in response.GetAllDownloadFileInfoResponse.GetAllDownloadFileInfoResult.DownloadFileInfo:
if int(DownloadFileInfo.VersionId) > current_fias_version:
yield dict(intver=int(DownloadFileInfo.VersionId), strver=str(DownloadFileInfo.TextVersion),
url=str(DownloadFileInfo.FiasDeltaXmlUrl))
url=str(DownloadFileInfo.FiasDeltaXmlUrl))

View File

@ -21,4 +21,4 @@ class XMLParser:
def parse_buffer(self, data_buffer, tag_name):
context = etree.iterparse(data_buffer, events=('end',), tag=tag_name)
self.fast_iter(context, lambda x: self.parse_function(x.attrib))
self.fast_iter(context, lambda x: self.parse_function(x.attrib))

View File

@ -1,11 +1,13 @@
# -*- coding: utf-8 -*-
from aore.dbutils.dbimpl import DBImpl
from aore.config import db as dbparams
from aore.dbutils.dbschemas import db_shemas, allowed_tables
from traceback import format_exc
import psycopg2
import logging
from traceback import format_exc
import psycopg2
from aore.config import db as dbparams
from aore.dbutils.dbimpl import DBImpl
from aore.dbutils.dbschemas import db_shemas
class DbHandler:
@ -13,12 +15,12 @@ class DbHandler:
logging.basicConfig(format='%(asctime)s %(message)s')
self.db = DBImpl(psycopg2, dbparams)
def bulk_csv(self, chunck_size, table_name, csv_file_name):
def bulk_csv(self, chunk_size, table_name, csv_file_name):
sql_query = "COPY \"{}\" ({}) FROM '{}' DELIMITER '\t' NULL 'NULL'". \
format(table_name,
", ".join(
db_shemas[table_name].fields),
csv_file_name)
db_shemas[table_name].fields),
csv_file_name)
try:
cur = self.db.get_cursor()
cur.execute(sql_query)
@ -27,7 +29,7 @@ class DbHandler:
self.db.transaction_rollback()
logging.error("Error updating sql. Reason : {}".format(format_exc()))
logging.warning("Inserted {} queries FROM {}".format(chunck_size, csv_file_name))
logging.warning("Inserted {} queries FROM {}".format(chunk_size, csv_file_name))
def pre_create(self):
f = open("aore/templates/postgre/pre_create.sql")
@ -44,4 +46,4 @@ class DbHandler:
def pre_update(self):
# TODO: update actions
pass
pass

View File

@ -30,4 +30,4 @@ class DBImpl:
if cur:
cur.close()
return rows
return rows

View File

@ -1,8 +1,9 @@
# -*- coding: utf-8 -*-
from aore.aoutils.aoupdater import AoUpdater
import optparse
from aore.aoutils.aoupdater import AoUpdater
def update_base(updates_count):
aoupdater = AoUpdater()

View File

@ -11,4 +11,5 @@ sys.setdefaultencoding("utf-8")
sys.path.append('/home/i/interc7j/.local/lib/python2.7/site-packages')
from fias import fias
application = fias.app