stale/py-phias
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Add indexes and sphinx.conf (conf and query stolen from https://github.com/Yuego/django-fias)

Browse Source
This commit is contained in:
Jack Stdin
2016-01-14 01:36:34 +03:00
parent 67f6943dce
commit 759efc43ee
14 changed files with 216 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
aore/templates/postgre/bulk_create.sql
View File

@@ -1 +1 @@
COPY "%tablename%" (%fieldslist%) FROM '%csvname%' DELIMITER '%tab%' NULL 'NULL'
COPY "{{tablename}}" ({{fieldslist}}) FROM '{{csvname}}' DELIMITER '{{delim}}' NULL 'NULL'

10
aore/templates/postgre/bulk_delete.sql
View File

@@ -1,5 +1,5 @@
DROP TABLE IF EXISTS "%tablename%_TEMP";
CREATE TEMP TABLE "%tablename%_TEMP" ON COMMIT DROP AS SELECT *
FROM "%tablename%" WITH NO DATA;
COPY "%tablename%_TEMP" (%fieldslist%) FROM '%csvname%' DELIMITER '%tab%' NULL 'NULL';
DELETE FROM "%tablename%" WHERE %uniquekey% IN (SELECT %uniquekey% FROM "%tablename%_TEMP");
DROP TABLE IF EXISTS "{{tablename}}_TEMP";
CREATE TEMP TABLE "{{tablename}}_TEMP" ON COMMIT DROP AS SELECT *
FROM "{{tablename}}" WITH NO DATA;
COPY "{{tablename}}_TEMP" ({{fieldslist}}) FROM '{{csvname}}' DELIMITER '{{delim}}' NULL 'NULL';
DELETE FROM "{{tablename}}" WHERE {{uniquekey}} IN (SELECT {{uniquekey}} FROM "{{tablename}}_TEMP");

15
aore/templates/postgre/bulk_update.sql
View File

@@ -1,7 +1,10 @@
DROP TABLE IF EXISTS "%tablename%_TEMP";
CREATE TEMP TABLE "%tablename%_TEMP" ON COMMIT DROP AS SELECT *
FROM "%tablename%" WITH NO DATA;
COPY "%tablename%_TEMP" (%fieldslist%) FROM '%csvname%' DELIMITER '%tab%' NULL 'NULL';
INSERT INTO "%tablename%" (%fieldslist%) SELECT %fieldslist%
DROP TABLE IF EXISTS "{{tablename}}_TEMP";
CREATE TEMP TABLE "{{tablename}}_TEMP" ON COMMIT DROP AS SELECT *
FROM "{{tablename}}" WITH NO DATA;
COPY "{{tablename}}_TEMP" ({{fieldslist}}) FROM '{{csvname}}' DELIMITER '{{delim}}' NULL 'NULL';
INSERT INTO "{{tablename}}" ({{fieldslist}}) SELECT {{fieldslist}}
FROM
"%tablename%_TEMP" ON CONFLICT (%uniquekey%) DO UPDATE SET %updaterule%;
"{{tablename}}_TEMP" ON CONFLICT ({{uniquekey}}) DO UPDATE SET {{updaterule}};
% if tablename=="ADDROBJ":
DELETE FROM "{{tablename}}" WHERE ACTSTATUS = FALSE OR NEXTID IS NOT NULL;
% end

4
aore/templates/postgre/post_create.sql Normal file
View File

@@ -0,0 +1,4 @@
CREATE INDEX "sphinx_ind_aolevel" ON "ADDROBJ" USING btree ("aolevel");
CREATE INDEX "sphinx_ind_parentguid" ON "ADDROBJ" USING btree ("parentguid");
CREATE INDEX "sphinx_ind_livestatus" ON "ADDROBJ" USING btree ("livestatus");
CREATE INDEX "sphinx_ind_aoguid" ON "ADDROBJ" USING btree ("aoguid");

8
aore/templates/postgre/pre_create.sql
View File

@@ -1,5 +1,5 @@
DROP TABLE IF EXISTS "public"."ADDROBJ";
CREATE TABLE "public"."ADDROBJ" (
DROP TABLE IF EXISTS "ADDROBJ";
CREATE TABLE "ADDROBJ" (
"id" SERIAL4 NOT NULL,
"aoid" UUID NOT NULL,
"aoguid" UUID,
@@ -15,8 +15,8 @@ CREATE TABLE "public"."ADDROBJ" (
)
WITH (OIDS =FALSE
);
DROP TABLE IF EXISTS "public"."SOCRBASE";
CREATE TABLE "public"."SOCRBASE" (
DROP TABLE IF EXISTS "SOCRBASE";
CREATE TABLE "SOCRBASE" (
"id" SERIAL4 NOT NULL,
"level" INT2,
"scname" VARCHAR(10),

14
aore/templates/postgre/sphinx_query.sql Normal file
View File

@@ -0,0 +1,14 @@
WITH RECURSIVE PATH (cnt, aoid, aoguid, aolevel, fullname) AS (
SELECT ao.id as cnt, ao.aoid, ao.aoguid, ao.aolevel,
ao.shortname || ' ' || ao.formalname AS fullname
FROM "ADDROBJ" AS ao
WHERE aolevel = 1 AND livestatus = TRUE
UNION
SELECT child.id as cnt, child.aoid, child.aoguid, child.aolevel,
PATH.fullname || ', ' || child.shortname || ' ' || child.formalname AS fullname
FROM "ADDROBJ" AS child
, PATH
WHERE child.parentguid = PATH.aoguid AND livestatus = TRUE
)
SELECT * FROM PATH WHERE AOLEVEL NOT IN (1,3)

41
aore/templates/sphinx/data.conf Normal file
View File

@@ -0,0 +1,41 @@
source src_{{index_name}}
{
type = pgsql
sql_host = {{db_host}}
sql_user = {{db_user}}
sql_pass = {{db_password}}
sql_db = {{db_name}}
sql_port = {{db_port}}
sql_query = {{!sql_query}}
sql_field_string = fullname
sql_attr_string = aoid
sql_attr_string = aoguid
sql_attr_uint = aolevel
}
index index_{{ index_name }}
{
docinfo = extern
morphology = stem_ru
min_stemming_len = 2
stopwords =
min_word_len = 2
charset_type = utf-8
min_prefix_len = 1
min_infix_len = 0
enable_star = 1
# strip html by default
html_strip = 1
ignore_chars = @, -
charset_table = 0..9, A..Z->a..z, _, a..z, \
U+0401->U+0435, U+0451->U+0435, \
U+410..U+42F->U+430..U+44F, U+430..U+44F
source = src_{{index_name}}
path = {{sphinx_var_path}}/data/index_{{index_name}}
}

63
aore/templates/sphinx/sphinx.conf Normal file
View File

@@ -0,0 +1,63 @@
indexer
{
# memory limit, in bytes, kiloytes (16384K) or megabytes (256M)
# optional, default is 32M, max is 2047M, recommended is 256M to 1024M
mem_limit = 256M
# maximum IO calls per second (for I/O throttling)
# optional, default is 0 (unlimited)
#
# max_iops = 40
# maximum IO call size, bytes (for I/O throttling)
# optional, default is 0 (unlimited)
#
max_iosize = 524288
}
searchd
{
listen = 127.0.0.1:9312
# required by RT-indexes
workers = threads
# log file, searchd run info is logged here
# optional, default is 'searchd.log'
log = {{sphinx_var_path}}/log/searchd.log
# query log file, all search queries are logged here
# optional, default is empty (do not log queries)
query_log = {{sphinx_var_path}}/log/query.log
# client read timeout, seconds
# optional, default is 5
read_timeout = 5
# maximum amount of children to fork (concurrent searches to run)
# optional, default is 0 (unlimited)
max_children = 30
# PID file, searchd process ID file name
# mandatory
pid_file = {{sphinx_var_path}}/run/searchd.pid
# max amount of matches the daemon ever keeps in RAM, per-index
# WARNING, THERE'S ALSO PER-QUERY LIMIT, SEE SetLimits() API CALL
# default is 1000 (just like Google)
max_matches = 1000
# seamless rotate, prevents rotate stalls if precaching huge datasets
# optional, default is 1
seamless_rotate = 1
# whether to forcibly preopen all indexes on startup
# optional, default is 0 (do not preopen)
preopen_indexes = 0
# whether to unlink .old index copies on succesful rotation.
# optional, default is 1 (do unlink)
unlink_old = 1
}