From 957d83fc1324c529863e20b56311f78de88b9120 Mon Sep 17 00:00:00 2001 From: Alexey Kondratov Date: Fri, 25 Oct 2019 17:03:23 +0300 Subject: [PATCH 01/91] Transfer tests to python3 --- Dockerfile.tmpl | 3 +- Makefile | 2 +- mk_dockerfile.sh | 2 + run_tests.sh | 4 +- tests/pg_qs_test_runner.py | 58 ++++++++----- tests/prepare_stress.sh | 25 ++++++ tests/requirements.txt | 3 + tests/test_cases.py | 172 ++++++++++++++++++++++++++----------- 8 files changed, 194 insertions(+), 75 deletions(-) create mode 100644 tests/prepare_stress.sh create mode 100644 tests/requirements.txt diff --git a/Dockerfile.tmpl b/Dockerfile.tmpl index 1725de3..c27e380 100644 --- a/Dockerfile.tmpl +++ b/Dockerfile.tmpl @@ -7,7 +7,7 @@ RUN apk add --no-cache \ make musl-dev gcc bison flex coreutils \ zlib-dev libedit-dev \ clang clang-analyzer linux-headers \ - python2 python2-dev py2-virtualenv; + python3 python3-dev py3-virtualenv; # Install fresh valgrind @@ -29,6 +29,7 @@ RUN chown postgres:postgres ${PGDATA} && \ chmod a+rwx /usr/local/share/postgresql/extension COPY run_tests.sh /run.sh +COPY tests/requirements.txt /requirements.txt RUN chmod 755 /run.sh ADD . /pg/testdir diff --git a/Makefile b/Makefile index c142ed5..aecfb45 100644 --- a/Makefile +++ b/Makefile @@ -9,7 +9,7 @@ DATA_built = $(EXTENSION)--$(EXTVERSION).sql PGFILEDESC = "pg_query_state - facility to track progress of plan execution" EXTRA_CLEAN = ./isolation_output $(EXTENSION)--$(EXTVERSION).sql \ - Dockerfile ./tests/*.pyc + Dockerfile ./tests/*.pyc ./tmp_stress ifdef USE_PGXS PG_CONFIG ?= pg_config diff --git a/mk_dockerfile.sh b/mk_dockerfile.sh index f15433c..76a7bf3 100755 --- a/mk_dockerfile.sh +++ b/mk_dockerfile.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env sh + if [ -z ${PG_VERSION+x} ]; then echo PG_VERSION is not set! exit 1 diff --git a/run_tests.sh b/run_tests.sh index bb7b75c..29c98b9 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -143,8 +143,8 @@ if [ -f regression.diffs ]; then cat regression.diffs; fi # run python tests set +x -e -virtualenv /tmp/env && source /tmp/env/bin/activate && -pip install PyYAML && pip install psycopg2 +python3 -m venv /tmp/env && source /tmp/env/bin/activate && +pip install -r /requirements.txt set -e #exit virtualenv with error code python tests/pg_qs_test_runner.py --port $PGPORT deactivate diff --git a/tests/pg_qs_test_runner.py b/tests/pg_qs_test_runner.py index 716719e..6eaf4fe 100644 --- a/tests/pg_qs_test_runner.py +++ b/tests/pg_qs_test_runner.py @@ -3,10 +3,15 @@ Tests extract query state from running backend (including concurrent extracts) Copyright (c) 2016-2016, Postgres Professional ''' +from __future__ import print_function, division, absolute_import +import os +import sys import argparse +import getpass import psycopg2 -import sys + +sys.path.append(os.path.dirname(os.path.abspath(__file__))) from test_cases import * class PasswordPromptAction(argparse.Action): @@ -34,50 +39,51 @@ class TeardownException(Exception): pass 'drop table foo cascade', 'drop table bar cascade', 'drop extension pg_query_state cascade', - ] +] tests = [ - test_deadlock, - test_simple_query, - test_concurrent_access, - test_nested_call, - test_trigger, - test_costs, - test_buffers, - test_timing, - test_formats, - test_timing_buffers_conflicts, - test_insert_on_conflict, - ] + test_deadlock, + test_simple_query, + test_concurrent_access, + test_nested_call, + test_trigger, + test_costs, + test_buffers, + test_timing, + test_formats, + test_timing_buffers_conflicts, + test_insert_on_conflict, +] def setup(con): ''' Creates pg_query_state extension, creates tables for tests, fills it with data ''' - print 'setting up...' + print('setting up...') try: cur = con.cursor() for cmd in setup_cmd: cur.execute(cmd) con.commit() cur.close() - except Exception, e: + except Exception as e: raise SetupException('Setup failed: %s' % e) - print 'done!' + print('done!') def teardown(con): ''' Drops table and extension ''' - print 'tearing down...' + print('tearing down...') try: cur = con.cursor() for cmd in teardown_cmd: cur.execute(cmd) con.commit() cur.close() - except Exception, e: + except Exception as e: raise TeardownException('Teardown failed: %s' % e) - print 'done!' + print('done!') def main(config): ''' Main test function ''' + con = psycopg2.connect(**config) setup(con) @@ -86,19 +92,27 @@ def main(config): descr = test.__doc__ else: descr = 'test case %d' % (i+1) - print ("%s..." % descr),; sys.stdout.flush() + print(("%s..." % descr)) + sys.stdout.flush() test(config) - print 'ok!' + print('ok!') + + if os.environ['LEVEL'] == 'stress': + print('Starting stress test') + stress_test(config) + print('Stress finished successfully') teardown(con) con.close() if __name__ == '__main__': parser = argparse.ArgumentParser(description='Query state of running backends tests') + parser.add_argument('--host', default='localhost', help='postgres server host') parser.add_argument('--port', type=int, default=5432, help='postgres server port') parser.add_argument('--user', dest='user', default='postgres', help='user name') parser.add_argument('--database', dest='database', default='postgres', help='database name') parser.add_argument('--password', dest='password', nargs=0, action=PasswordPromptAction, default='') + args = parser.parse_args() main(args.__dict__) diff --git a/tests/prepare_stress.sh b/tests/prepare_stress.sh new file mode 100644 index 0000000..7501704 --- /dev/null +++ b/tests/prepare_stress.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env sh +mkdir -p tmp_stress +cd tmp_stress +rm -rf ./* + +git clone https://github.com/gregrahn/tpcds-kit.git +cd tpcds-kit/tools +make -s + +#Generate data +./dsdgen -FORCE -VERBOSE + +#Prepare data +mkdir tables -p +for i in `ls *.dat`; do + echo "Preparing file " $i + sed 's/|$//' $i > tables/$i +done + +#Generate queries +./dsqgen -DIRECTORY ../query_templates \ + -INPUT ../query_templates/templates.lst \ + -VERBOSE Y \ + -QUALIFY Y \ + -DIALECT netezza diff --git a/tests/requirements.txt b/tests/requirements.txt new file mode 100644 index 0000000..ff6b4f4 --- /dev/null +++ b/tests/requirements.txt @@ -0,0 +1,3 @@ +PyYAML +psycopg2 +progressbar2 diff --git a/tests/test_cases.py b/tests/test_cases.py index 175dbd1..e0abda7 100644 --- a/tests/test_cases.py +++ b/tests/test_cases.py @@ -1,3 +1,6 @@ +from __future__ import print_function, division, absolute_import + +import os import json import psycopg2 import psycopg2.extensions @@ -6,6 +9,9 @@ import time import xml.etree.ElementTree as ET import yaml +import subprocess +import progressbar + from time import sleep def wait(conn): @@ -28,7 +34,7 @@ def n_async_connect(config, n=1): aconfig['async'] = True result = [] - for _ in xrange(n): + for _ in range(n): conn = psycopg2.connect(**aconfig) wait(conn) result.append(conn) @@ -45,35 +51,35 @@ def n_close(conns): def debug_output(qs, qs_len, pid, query, expected): something_happened = False if (qs_len and len(qs) != qs_len ): - print "len(qs): ", len(qs), ", expected: ", qs_len + print( "len(qs): ", len(qs), ", expected: ", qs_len) something_happened = True if (pid and qs[0][0] != pid): - print "qs[0][0]: ", qs[0][0], " = ", pid + print( "qs[0][0]: ", qs[0][0], " = ", pid) something_happened = True if (qs[0][1] != 0): - print "qs[0][1]: ", qs[0][1], ", expected: 0" + print( "qs[0][1]: ", qs[0][1], ", expected: 0") something_happened = True if (qs[0][2] != query): - print "qs[0][2]:\n", qs[0][2] - print "Expected:\n", query + print( "qs[0][2]:\n", qs[0][2]) + print( "Expected:\n", query) something_happened = True if (not (re.match(expected, qs[0][3]))): - print "qs[0][3]:\n", qs[0][3] - print "Expected:\n", expected + print( "qs[0][3]:\n", qs[0][3]) + print( "Expected:\n", expected) something_happened = True if (qs[0][4] != None): - print "qs[0][4]: ", qs[0][4], "Expected: None" + print( "qs[0][4]: ", qs[0][4], "Expected: None") something_happened = True if (qs_len and len(qs) > qs_len): for i in range(qs_len, len(qs)): - print "qs[",i,"][0]: ", qs[i][0] - print "qs[",i,"][1]: ", qs[i][1] - print "qs[",i,"][2]: ", qs[i][2] - print "qs[",i,"][3]: ", qs[i][3] - print "qs[",i,"][4]: ", qs[i][4] + print( "qs[",i,"][0]: ", qs[i][0]) + print( "qs[",i,"][1]: ", qs[i][1]) + print( "qs[",i,"][2]: ", qs[i][2]) + print( "qs[",i,"][3]: ", qs[i][3]) + print( "qs[",i,"][4]: ", qs[i][4]) something_happened = True if (something_happened): - print "If test have not crashed, then it's OK" + print( "If test have not crashed, then it's OK") def notices_warning(): if (len(notices) > 0): @@ -144,7 +150,7 @@ def query_state(config, async_conn, query, args={}, num_workers=0): 'config': config, 'pid': async_conn.get_backend_pid() } - for k, v in args.iteritems(): + for k, v in args.items(): pg_qs_args[k] = v result = pg_query_state(**pg_qs_args) wait(async_conn) @@ -171,8 +177,8 @@ def test_simple_query(config): qs = query_state(config, acon, query) debug_output(qs, 1, acon.get_backend_pid(), query, expected) notices_warning() - #assert len(qs) == 1 #Skip this check while output of test can be different - assert qs[0][0] == acon.get_backend_pid() and qs[0][1] == 0 \ + #assert len(qs) == 1 #Skip this check while output of test can be different + assert qs[0][0] == acon.get_backend_pid() and qs[0][1] == 0 \ and qs[0][2] == query and re.match(expected, qs[0][3]) and qs[0][4] == None n_close((acon,)) @@ -194,13 +200,13 @@ def test_concurrent_access(config): wait(acon3) qs1, qs2 = acurs1.fetchall(), acurs2.fetchall() - assert len(qs1) == len(qs2) == 1 \ + assert len(qs1) == len(qs2) == 1 \ and qs1[0][0] == qs2[0][0] == acon3.get_backend_pid() \ and qs1[0][1] == qs2[0][1] == 0 \ and qs1[0][2] == qs2[0][2] == query \ and len(qs1[0][3]) > 0 and len(qs2[0][3]) > 0 \ and qs1[0][4] == qs2[0][4] == None - #assert len(notices) == 0 + #assert len(notices) == 0 notices_warning() n_close((acon1, acon2, acon3)) @@ -235,13 +241,13 @@ def test_nested_call(config): util_conn.commit() qs = query_state(config, acon, call_function) - assert len(qs) == 2 \ + assert len(qs) == 2 \ and qs[0][0] == qs[1][0] == acon.get_backend_pid() \ and qs[0][1] == 0 and qs[1][1] == 1 \ and qs[0][2] == call_function and qs[0][3] == expected \ and qs[1][2] == nested_query and re.match(expected_nested, qs[1][3]) \ and qs[0][4] == qs[1][4] == None - assert len(notices) == 0 + assert len(notices) == 0 util_curs.execute(drop_function) @@ -270,11 +276,11 @@ def test_insert_on_conflict(config): debug_output(qs, 1, acon.get_backend_pid(), query, expected) notices_warning() - #assert len(qs) == 1 \ - assert qs[0][0] == acon.get_backend_pid() and qs[0][1] == 0 \ + #assert len(qs) == 1 \ + assert qs[0][0] == acon.get_backend_pid() and qs[0][1] == 0 \ and qs[0][2] == query and re.match(expected, qs[0][3]) \ and qs[0][4] == None - assert len(notices) == 0 + assert len(notices) == 0 util_curs.execute(drop_field_uniqueness) @@ -322,7 +328,7 @@ def test_trigger(config): assert qs[0][0] == acon.get_backend_pid() and qs[0][1] == 0 \ and qs[0][2] == query and re.match(expected_upper, qs[0][3]) \ and qs[0][4] == None - assert len(notices) == 0 + assert len(notices) == 0 qs = query_state(config, acon, query, {'triggers': False}) debug_output(qs, None, acon.get_backend_pid(), query, expected_upper) @@ -330,7 +336,7 @@ def test_trigger(config): assert qs[0][0] == acon.get_backend_pid() and qs[0][1] == 0 \ and qs[0][2] == query and re.match(expected_upper, qs[0][3]) \ and qs[0][4] == None - assert len(notices) == 0 + assert len(notices) == 0 util_curs.execute(drop_temps) @@ -353,8 +359,8 @@ def test_costs(config): qs = query_state(config, acon, query, {'costs': True}) debug_output(qs, 1, None, query, expected) notices_warning() - assert len(qs) == 1 and re.match(expected, qs[0][3]) - assert len(notices) == 0 + assert len(qs) == 1 and re.match(expected, qs[0][3]) + assert len(notices) == 0 n_close((acon,)) @@ -378,8 +384,8 @@ def test_buffers(config): qs = query_state(config, acon, query, {'buffers': True}) debug_output(qs, 1, None, query, expected) notices_warning() - assert len(qs) == 1 and re.match(expected, qs[0][3]) - assert len(notices) == 0 + assert len(qs) == 1 and re.match(expected, qs[0][3]) + assert len(notices) == 0 n_close((acon,)) @@ -401,18 +407,18 @@ def test_timing(config): qs = query_state(config, acon, query, {'timing': True}) debug_output(qs, 1, None, query, expected) notices_warning() - assert len(qs) == 1 and re.match(expected, qs[0][3]) - assert len(notices) == 0 + assert len(qs) == 1 and re.match(expected, qs[0][3]) + assert len(notices) == 0 n_close((acon,)) def check_plan(plan): - assert plan.has_key('Current loop') + assert 'Current loop' in plan cur_loop = plan['Current loop'] - assert cur_loop.has_key('Actual Loop Number') \ - and cur_loop.has_key('Actual Rows') + assert 'Actual Loop Number' in cur_loop\ + and 'Actual Rows' in cur_loop - if not plan.has_key('Plans'): + if not 'Plans' in plan: return for subplan in plan['Plans']: @@ -422,7 +428,7 @@ def check_xml(root): prefix = '{http://www.postgresql.org/2009/explain}' for plan in root.iter(prefix + 'Plan'): cur_loop = plan.find(prefix + 'Current-loop') - assert cur_loop != None \ + assert cur_loop != None \ and cur_loop.find(prefix + 'Actual-Loop-Number') != None \ and cur_loop.find(prefix + 'Actual-Rows') != None @@ -442,21 +448,21 @@ def test_formats(config): qs = query_state(config, acon, query, {'format': 'text'}) debug_output(qs, 1, None, query, expected) notices_warning() - assert len(qs) == 1 and re.match(expected, qs[0][3]) - assert len(notices) == 0 + assert len(qs) == 1 and re.match(expected, qs[0][3]) + assert len(notices) == 0 qs = query_state(config, acon, query, {'format': 'json'}) try: js_obj = json.loads(qs[0][3]) except ValueError: assert False, 'Invalid json format' - assert len(qs) == 1 - assert len(notices) == 0 + assert len(qs) == 1 + assert len(notices) == 0 check_plan(js_obj['Plan']) qs = query_state(config, acon, query, {'format': 'xml'}) - assert len(qs) == 1 - assert len(notices) == 0 + assert len(qs) == 1 + assert len(notices) == 0 try: xml_root = ET.fromstring(qs[0][3]) except: @@ -468,8 +474,8 @@ def test_formats(config): yaml_doc = yaml.load(qs[0][3]) except: assert False, 'Invalid yaml format' - assert len(qs) == 1 - assert len(notices) == 0 + assert len(qs) == 1 + assert len(notices) == 0 check_plan(yaml_doc['Plan']) n_close((acon,)) @@ -483,17 +489,85 @@ def test_timing_buffers_conflicts(config): buffers_pattern = 'Buffers:' qs = query_state(config, acon, query, {'timing': True, 'buffers': False}) - assert len(qs) == 1 and not re.search(timing_pattern, qs[0][3]) + assert len(qs) == 1 and not re.search(timing_pattern, qs[0][3]) assert notices == ['WARNING: timing statistics disabled\n'] qs = query_state(config, acon, query, {'timing': False, 'buffers': True}) - assert len(qs) == 1 and not re.search(buffers_pattern, qs[0][3]) + assert len(qs) == 1 and not re.search(buffers_pattern, qs[0][3]) assert notices == ['WARNING: buffers statistics disabled\n'] qs = query_state(config, acon, query, {'timing': True, 'buffers': True}) - assert len(qs) == 1 and not re.search(timing_pattern, qs[0][3]) \ + assert len(qs) == 1 and not re.search(timing_pattern, qs[0][3]) \ and not re.search(buffers_pattern, qs[0][3]) assert len(notices) == 2 and 'WARNING: timing statistics disabled\n' in notices \ and 'WARNING: buffers statistics disabled\n' in notices n_close((acon,)) + +class DataLoadException(Exception): pass +class StressTestException(Exception): pass + +def load_tpcds_data(config): + print('Preparing TPC-DS...') + subprocess.call(['./tests/prepare_stress.sh']) + + try: + conn = psycopg2.connect(**config) + cur = conn.cursor() + + # Create tables + cur.execute(open('tmp_stress/tpcds-kit/tools/tpcds.sql', 'r').read()) + + # Copy table data from files + for table_datafile in os.listdir('tmp_stress/tpcds-kit/tools/'): + if table_datafile.endswith(".dat"): + table_name = os.path.splitext(os.path.basename(table_datafile))[0] + copy_cmd = "COPY %s FROM 'tmp_stress/tpcds-kit/tools/tables/%s' CSV DELIMITER '|'" % (table_name, table_datafile) + + print("Loading table ", table_name) + cur.execute("TRUNCATE %s" % table_name) + cur.execute(copy_cmd) + + except Exception as e: + cur.close() + conn.close() + raise DataLoadException('Load failed: %s' % e) + + print('done!') + +def stress_test(config): + """TPC-DS stress test""" + load_tpcds_data(config) + print('Starting test...') + + # Execute query in separate thread + with open("tests/query_tpcds.sql",'r') as f: + sql = f.read() + + queries = sql.split(';') + for i, query in enumerate(queries): + queries[i] = query.replace('%','%%') + if (len(query.strip()) == 0): + del queries[i] + + acon, = n_async_connect(config) + + timeout_list = [] + bar = progressbar.ProgressBar(max_value=len(queries)) + for i, query in enumerate(queries): + bar.update(i + 1) + try: + # Set query timeout to 10 sec + set_guc(acon, 'statement_timeout', 10000) + qs = query_state(config, acon, query) + + #TODO: Put here testgres exception when supported + except psycopg2.extensions.QueryCanceledError: + timeout_list.append(i) + finally: + pass + + n_close((acon,)) + + if len(timeout_list) > 0: + print('There were pg_query_state timeouts (10s) on queries: ', timeout_list) From bbc89f5fd676f7dfd4783e38222af4c8e69c073c Mon Sep 17 00:00:00 2001 From: Alexey Kondratov Date: Fri, 25 Oct 2019 20:34:48 +0300 Subject: [PATCH 02/91] Try TPC-DS test --- .gitignore | 2 + .travis.yml | 1 + Dockerfile.tmpl | 3 +- run_tests.sh | 2 +- tests/prepare_stress.sh | 6 +- tests/query_tpcds.sql | 4871 +++++++++++++++++++++++++++++++++++++++ tests/test_cases.py | 29 +- 7 files changed, 4897 insertions(+), 17 deletions(-) mode change 100644 => 100755 tests/prepare_stress.sh create mode 100644 tests/query_tpcds.sql diff --git a/.gitignore b/.gitignore index ad06216..587ab37 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,5 @@ pg_query_state--*.sql cscope.out tags Dockerfile +tmp_stress + diff --git a/.travis.yml b/.travis.yml index 05f37c3..96a4651 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,6 +26,7 @@ env: - PG_VERSION=10 - PG_VERSION=9.6 LEVEL=hardcore - PG_VERSION=9.6 + - PG_VERSION=12 LEVEL=stress matrix: allow_failures: diff --git a/Dockerfile.tmpl b/Dockerfile.tmpl index c27e380..73ce735 100644 --- a/Dockerfile.tmpl +++ b/Dockerfile.tmpl @@ -2,7 +2,7 @@ FROM postgres:${PG_VERSION}-alpine # Install dependencies RUN apk add --no-cache \ - openssl curl \ + openssl curl git \ perl perl-ipc-run \ make musl-dev gcc bison flex coreutils \ zlib-dev libedit-dev \ @@ -29,7 +29,6 @@ RUN chown postgres:postgres ${PGDATA} && \ chmod a+rwx /usr/local/share/postgresql/extension COPY run_tests.sh /run.sh -COPY tests/requirements.txt /requirements.txt RUN chmod 755 /run.sh ADD . /pg/testdir diff --git a/run_tests.sh b/run_tests.sh index 29c98b9..31ac49a 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -144,7 +144,7 @@ if [ -f regression.diffs ]; then cat regression.diffs; fi # run python tests set +x -e python3 -m venv /tmp/env && source /tmp/env/bin/activate && -pip install -r /requirements.txt +pip install -r tests/requirements.txt set -e #exit virtualenv with error code python tests/pg_qs_test_runner.py --port $PGPORT deactivate diff --git a/tests/prepare_stress.sh b/tests/prepare_stress.sh old mode 100644 new mode 100755 index 7501704..827e310 --- a/tests/prepare_stress.sh +++ b/tests/prepare_stress.sh @@ -1,4 +1,5 @@ #!/usr/bin/env sh + mkdir -p tmp_stress cd tmp_stress rm -rf ./* @@ -8,10 +9,10 @@ cd tpcds-kit/tools make -s #Generate data -./dsdgen -FORCE -VERBOSE +./dsdgen -FORCE -VERBOSE -SCALE 1 #Prepare data -mkdir tables -p +mkdir -p tables for i in `ls *.dat`; do echo "Preparing file " $i sed 's/|$//' $i > tables/$i @@ -22,4 +23,5 @@ done -INPUT ../query_templates/templates.lst \ -VERBOSE Y \ -QUALIFY Y \ + -SCALE 1 \ -DIALECT netezza diff --git a/tests/query_tpcds.sql b/tests/query_tpcds.sql new file mode 100644 index 0000000..e82fca1 --- /dev/null +++ b/tests/query_tpcds.sql @@ -0,0 +1,4871 @@ +-- start query 1 in stream 0 using template query1.tpl +with customer_total_return as +(select sr_customer_sk as ctr_customer_sk +,sr_store_sk as ctr_store_sk +,sum(SR_FEE) as ctr_total_return +from store_returns +,date_dim +where sr_returned_date_sk = d_date_sk +and d_year =2000 +group by sr_customer_sk +,sr_store_sk) + select c_customer_id +from customer_total_return ctr1 +,store +,customer +where ctr1.ctr_total_return > (select avg(ctr_total_return)*1.2 +from customer_total_return ctr2 +where ctr1.ctr_store_sk = ctr2.ctr_store_sk) +and s_store_sk = ctr1.ctr_store_sk +and s_state = 'TN' +and ctr1.ctr_customer_sk = c_customer_sk +order by c_customer_id +limit 100; + +-- end query 1 in stream 0 using template query1.tpl +-- start query 2 in stream 0 using template query2.tpl +with wscs as + (select sold_date_sk + ,sales_price + from (select ws_sold_date_sk sold_date_sk + ,ws_ext_sales_price sales_price + from web_sales + union all + select cs_sold_date_sk sold_date_sk + ,cs_ext_sales_price sales_price + from catalog_sales) as sdsp), + wswscs as + (select d_week_seq, + sum(case when (d_day_name='Sunday') then sales_price else null end) sun_sales, + sum(case when (d_day_name='Monday') then sales_price else null end) mon_sales, + sum(case when (d_day_name='Tuesday') then sales_price else null end) tue_sales, + sum(case when (d_day_name='Wednesday') then sales_price else null end) wed_sales, + sum(case when (d_day_name='Thursday') then sales_price else null end) thu_sales, + sum(case when (d_day_name='Friday') then sales_price else null end) fri_sales, + sum(case when (d_day_name='Saturday') then sales_price else null end) sat_sales + from wscs + ,date_dim + where d_date_sk = sold_date_sk + group by d_week_seq) + select d_week_seq1 + ,round(sun_sales1/sun_sales2,2) + ,round(mon_sales1/mon_sales2,2) + ,round(tue_sales1/tue_sales2,2) + ,round(wed_sales1/wed_sales2,2) + ,round(thu_sales1/thu_sales2,2) + ,round(fri_sales1/fri_sales2,2) + ,round(sat_sales1/sat_sales2,2) + from + (select wswscs.d_week_seq d_week_seq1 + ,sun_sales sun_sales1 + ,mon_sales mon_sales1 + ,tue_sales tue_sales1 + ,wed_sales wed_sales1 + ,thu_sales thu_sales1 + ,fri_sales fri_sales1 + ,sat_sales sat_sales1 + from wswscs,date_dim + where date_dim.d_week_seq = wswscs.d_week_seq and + d_year = 1998) y, + (select wswscs.d_week_seq d_week_seq2 + ,sun_sales sun_sales2 + ,mon_sales mon_sales2 + ,tue_sales tue_sales2 + ,wed_sales wed_sales2 + ,thu_sales thu_sales2 + ,fri_sales fri_sales2 + ,sat_sales sat_sales2 + from wswscs + ,date_dim + where date_dim.d_week_seq = wswscs.d_week_seq and + d_year = 1998+1) z + where d_week_seq1=d_week_seq2-53 + order by d_week_seq1; + +-- end query 2 in stream 0 using template query2.tpl +-- start query 3 in stream 0 using template query3.tpl +select dt.d_year + ,item.i_brand_id brand_id + ,item.i_brand brand + ,sum(ss_sales_price) sum_agg + from date_dim dt + ,store_sales + ,item + where dt.d_date_sk = store_sales.ss_sold_date_sk + and store_sales.ss_item_sk = item.i_item_sk + and item.i_manufact_id = 816 + and dt.d_moy=11 + group by dt.d_year + ,item.i_brand + ,item.i_brand_id + order by dt.d_year + ,sum_agg desc + ,brand_id + limit 100; + +-- end query 3 in stream 0 using template query3.tpl +-- start query 4 in stream 0 using template query4.tpl +with year_total as ( + select c_customer_id customer_id + ,c_first_name customer_first_name + ,c_last_name customer_last_name + ,c_preferred_cust_flag customer_preferred_cust_flag + ,c_birth_country customer_birth_country + ,c_login customer_login + ,c_email_address customer_email_address + ,d_year dyear + ,sum(((ss_ext_list_price-ss_ext_wholesale_cost-ss_ext_discount_amt)+ss_ext_sales_price)/2) year_total + ,'s' sale_type + from customer + ,store_sales + ,date_dim + where c_customer_sk = ss_customer_sk + and ss_sold_date_sk = d_date_sk + group by c_customer_id + ,c_first_name + ,c_last_name + ,c_preferred_cust_flag + ,c_birth_country + ,c_login + ,c_email_address + ,d_year + union all + select c_customer_id customer_id + ,c_first_name customer_first_name + ,c_last_name customer_last_name + ,c_preferred_cust_flag customer_preferred_cust_flag + ,c_birth_country customer_birth_country + ,c_login customer_login + ,c_email_address customer_email_address + ,d_year dyear + ,sum((((cs_ext_list_price-cs_ext_wholesale_cost-cs_ext_discount_amt)+cs_ext_sales_price)/2) ) year_total + ,'c' sale_type + from customer + ,catalog_sales + ,date_dim + where c_customer_sk = cs_bill_customer_sk + and cs_sold_date_sk = d_date_sk + group by c_customer_id + ,c_first_name + ,c_last_name + ,c_preferred_cust_flag + ,c_birth_country + ,c_login + ,c_email_address + ,d_year +union all + select c_customer_id customer_id + ,c_first_name customer_first_name + ,c_last_name customer_last_name + ,c_preferred_cust_flag customer_preferred_cust_flag + ,c_birth_country customer_birth_country + ,c_login customer_login + ,c_email_address customer_email_address + ,d_year dyear + ,sum((((ws_ext_list_price-ws_ext_wholesale_cost-ws_ext_discount_amt)+ws_ext_sales_price)/2) ) year_total + ,'w' sale_type + from customer + ,web_sales + ,date_dim + where c_customer_sk = ws_bill_customer_sk + and ws_sold_date_sk = d_date_sk + group by c_customer_id + ,c_first_name + ,c_last_name + ,c_preferred_cust_flag + ,c_birth_country + ,c_login + ,c_email_address + ,d_year + ) + select + t_s_secyear.customer_id + ,t_s_secyear.customer_first_name + ,t_s_secyear.customer_last_name + ,t_s_secyear.customer_birth_country + from year_total t_s_firstyear + ,year_total t_s_secyear + ,year_total t_c_firstyear + ,year_total t_c_secyear + ,year_total t_w_firstyear + ,year_total t_w_secyear + where t_s_secyear.customer_id = t_s_firstyear.customer_id + and t_s_firstyear.customer_id = t_c_secyear.customer_id + and t_s_firstyear.customer_id = t_c_firstyear.customer_id + and t_s_firstyear.customer_id = t_w_firstyear.customer_id + and t_s_firstyear.customer_id = t_w_secyear.customer_id + and t_s_firstyear.sale_type = 's' + and t_c_firstyear.sale_type = 'c' + and t_w_firstyear.sale_type = 'w' + and t_s_secyear.sale_type = 's' + and t_c_secyear.sale_type = 'c' + and t_w_secyear.sale_type = 'w' + and t_s_firstyear.dyear = 1999 + and t_s_secyear.dyear = 1999+1 + and t_c_firstyear.dyear = 1999 + and t_c_secyear.dyear = 1999+1 + and t_w_firstyear.dyear = 1999 + and t_w_secyear.dyear = 1999+1 + and t_s_firstyear.year_total > 0 + and t_c_firstyear.year_total > 0 + and t_w_firstyear.year_total > 0 + and case when t_c_firstyear.year_total > 0 then t_c_secyear.year_total / t_c_firstyear.year_total else null end + > case when t_s_firstyear.year_total > 0 then t_s_secyear.year_total / t_s_firstyear.year_total else null end + and case when t_c_firstyear.year_total > 0 then t_c_secyear.year_total / t_c_firstyear.year_total else null end + > case when t_w_firstyear.year_total > 0 then t_w_secyear.year_total / t_w_firstyear.year_total else null end + order by t_s_secyear.customer_id + ,t_s_secyear.customer_first_name + ,t_s_secyear.customer_last_name + ,t_s_secyear.customer_birth_country +limit 100; + +-- end query 4 in stream 0 using template query4.tpl +-- start query 5 in stream 0 using template query5.tpl +with ssr as + (select s_store_id, + sum(sales_price) as sales, + sum(profit) as profit, + sum(return_amt) as returns, + sum(net_loss) as profit_loss + from + ( select ss_store_sk as store_sk, + ss_sold_date_sk as date_sk, + ss_ext_sales_price as sales_price, + ss_net_profit as profit, + cast(0 as decimal(7,2)) as return_amt, + cast(0 as decimal(7,2)) as net_loss + from store_sales + union all + select sr_store_sk as store_sk, + sr_returned_date_sk as date_sk, + cast(0 as decimal(7,2)) as sales_price, + cast(0 as decimal(7,2)) as profit, + sr_return_amt as return_amt, + sr_net_loss as net_loss + from store_returns + ) salesreturns, + date_dim, + store + where date_sk = d_date_sk + and d_date between cast('2000-08-19' as date) + and (cast('2000-08-19' as date) + interval '14 days') + and store_sk = s_store_sk + group by s_store_id) + , + csr as + (select cp_catalog_page_id, + sum(sales_price) as sales, + sum(profit) as profit, + sum(return_amt) as returns, + sum(net_loss) as profit_loss + from + ( select cs_catalog_page_sk as page_sk, + cs_sold_date_sk as date_sk, + cs_ext_sales_price as sales_price, + cs_net_profit as profit, + cast(0 as decimal(7,2)) as return_amt, + cast(0 as decimal(7,2)) as net_loss + from catalog_sales + union all + select cr_catalog_page_sk as page_sk, + cr_returned_date_sk as date_sk, + cast(0 as decimal(7,2)) as sales_price, + cast(0 as decimal(7,2)) as profit, + cr_return_amount as return_amt, + cr_net_loss as net_loss + from catalog_returns + ) salesreturns, + date_dim, + catalog_page + where date_sk = d_date_sk + and d_date between cast('2000-08-19' as date) + and (cast('2000-08-19' as date) + interval '14 days') + and page_sk = cp_catalog_page_sk + group by cp_catalog_page_id) + , + wsr as + (select web_site_id, + sum(sales_price) as sales, + sum(profit) as profit, + sum(return_amt) as returns, + sum(net_loss) as profit_loss + from + ( select ws_web_site_sk as wsr_web_site_sk, + ws_sold_date_sk as date_sk, + ws_ext_sales_price as sales_price, + ws_net_profit as profit, + cast(0 as decimal(7,2)) as return_amt, + cast(0 as decimal(7,2)) as net_loss + from web_sales + union all + select ws_web_site_sk as wsr_web_site_sk, + wr_returned_date_sk as date_sk, + cast(0 as decimal(7,2)) as sales_price, + cast(0 as decimal(7,2)) as profit, + wr_return_amt as return_amt, + wr_net_loss as net_loss + from web_returns left outer join web_sales on + ( wr_item_sk = ws_item_sk + and wr_order_number = ws_order_number) + ) salesreturns, + date_dim, + web_site + where date_sk = d_date_sk + and d_date between cast('2000-08-19' as date) + and (cast('2000-08-19' as date) + interval '14 days') + and wsr_web_site_sk = web_site_sk + group by web_site_id) + select channel + , id + , sum(sales) as sales + , sum(returns) as returns + , sum(profit) as profit + from + (select 'store channel' as channel + , 'store' || s_store_id as id + , sales + , returns + , (profit - profit_loss) as profit + from ssr + union all + select 'catalog channel' as channel + , 'catalog_page' || cp_catalog_page_id as id + , sales + , returns + , (profit - profit_loss) as profit + from csr + union all + select 'web channel' as channel + , 'web_site' || web_site_id as id + , sales + , returns + , (profit - profit_loss) as profit + from wsr + ) x + group by rollup (channel, id) + order by channel + ,id + limit 100; + +-- end query 5 in stream 0 using template query5.tpl +-- start query 6 in stream 0 using template query6.tpl +select a.ca_state state, count(*) cnt + from customer_address a + ,customer c + ,store_sales s + ,date_dim d + ,item i + where a.ca_address_sk = c.c_current_addr_sk + and c.c_customer_sk = s.ss_customer_sk + and s.ss_sold_date_sk = d.d_date_sk + and s.ss_item_sk = i.i_item_sk + and d.d_month_seq = + (select distinct (d_month_seq) + from date_dim + where d_year = 2002 + and d_moy = 3 ) + and i.i_current_price > 1.2 * + (select avg(j.i_current_price) + from item j + where j.i_category = i.i_category) + group by a.ca_state + having count(*) >= 10 + order by cnt, a.ca_state + limit 100; + +-- end query 6 in stream 0 using template query6.tpl +-- start query 7 in stream 0 using template query7.tpl +select i_item_id, + avg(ss_quantity) agg1, + avg(ss_list_price) agg2, + avg(ss_coupon_amt) agg3, + avg(ss_sales_price) agg4 + from store_sales, customer_demographics, date_dim, item, promotion + where ss_sold_date_sk = d_date_sk and + ss_item_sk = i_item_sk and + ss_cdemo_sk = cd_demo_sk and + ss_promo_sk = p_promo_sk and + cd_gender = 'F' and + cd_marital_status = 'W' and + cd_education_status = 'College' and + (p_channel_email = 'N' or p_channel_event = 'N') and + d_year = 2001 + group by i_item_id + order by i_item_id + limit 100; + +-- end query 7 in stream 0 using template query7.tpl +-- start query 8 in stream 0 using template query8.tpl +select s_store_name + ,sum(ss_net_profit) + from store_sales + ,date_dim + ,store, + (select ca_zip + from ( + SELECT substr(ca_zip,1,5) ca_zip + FROM customer_address + WHERE substr(ca_zip,1,5) IN ( + '47602','16704','35863','28577','83910','36201', + '58412','48162','28055','41419','80332', + '38607','77817','24891','16226','18410', + '21231','59345','13918','51089','20317', + '17167','54585','67881','78366','47770', + '18360','51717','73108','14440','21800', + '89338','45859','65501','34948','25973', + '73219','25333','17291','10374','18829', + '60736','82620','41351','52094','19326', + '25214','54207','40936','21814','79077', + '25178','75742','77454','30621','89193', + '27369','41232','48567','83041','71948', + '37119','68341','14073','16891','62878', + '49130','19833','24286','27700','40979', + '50412','81504','94835','84844','71954', + '39503','57649','18434','24987','12350', + '86379','27413','44529','98569','16515', + '27287','24255','21094','16005','56436', + '91110','68293','56455','54558','10298', + '83647','32754','27052','51766','19444', + '13869','45645','94791','57631','20712', + '37788','41807','46507','21727','71836', + '81070','50632','88086','63991','20244', + '31655','51782','29818','63792','68605', + '94898','36430','57025','20601','82080', + '33869','22728','35834','29086','92645', + '98584','98072','11652','78093','57553', + '43830','71144','53565','18700','90209', + '71256','38353','54364','28571','96560', + '57839','56355','50679','45266','84680', + '34306','34972','48530','30106','15371', + '92380','84247','92292','68852','13338', + '34594','82602','70073','98069','85066', + '47289','11686','98862','26217','47529', + '63294','51793','35926','24227','14196', + '24594','32489','99060','49472','43432', + '49211','14312','88137','47369','56877', + '20534','81755','15794','12318','21060', + '73134','41255','63073','81003','73873', + '66057','51184','51195','45676','92696', + '70450','90669','98338','25264','38919', + '59226','58581','60298','17895','19489', + '52301','80846','95464','68770','51634', + '19988','18367','18421','11618','67975', + '25494','41352','95430','15734','62585', + '97173','33773','10425','75675','53535', + '17879','41967','12197','67998','79658', + '59130','72592','14851','43933','68101', + '50636','25717','71286','24660','58058', + '72991','95042','15543','33122','69280', + '11912','59386','27642','65177','17672', + '33467','64592','36335','54010','18767', + '63193','42361','49254','33113','33159', + '36479','59080','11855','81963','31016', + '49140','29392','41836','32958','53163', + '13844','73146','23952','65148','93498', + '14530','46131','58454','13376','13378', + '83986','12320','17193','59852','46081', + '98533','52389','13086','68843','31013', + '13261','60560','13443','45533','83583', + '11489','58218','19753','22911','25115', + '86709','27156','32669','13123','51933', + '39214','41331','66943','14155','69998', + '49101','70070','35076','14242','73021', + '59494','15782','29752','37914','74686', + '83086','34473','15751','81084','49230', + '91894','60624','17819','28810','63180', + '56224','39459','55233','75752','43639', + '55349','86057','62361','50788','31830', + '58062','18218','85761','60083','45484', + '21204','90229','70041','41162','35390', + '16364','39500','68908','26689','52868', + '81335','40146','11340','61527','61794', + '71997','30415','59004','29450','58117', + '69952','33562','83833','27385','61860', + '96435','48333','23065','32961','84919', + '61997','99132','22815','56600','68730', + '48017','95694','32919','88217','27116', + '28239','58032','18884','16791','21343', + '97462','18569','75660','15475') + intersect + select ca_zip + from (SELECT substr(ca_zip,1,5) ca_zip,count(*) cnt + FROM customer_address, customer + WHERE ca_address_sk = c_current_addr_sk and + c_preferred_cust_flag='Y' + group by ca_zip + having count(*) > 10)A1)A2) V1 + where ss_store_sk = s_store_sk + and ss_sold_date_sk = d_date_sk + and d_qoy = 2 and d_year = 1998 + and (substr(s_zip,1,2) = substr(V1.ca_zip,1,2)) + group by s_store_name + order by s_store_name + limit 100; + +-- end query 8 in stream 0 using template query8.tpl +-- start query 9 in stream 0 using template query9.tpl +select case when (select count(*) + from store_sales + where ss_quantity between 1 and 20) > 1071 + then (select avg(ss_ext_tax) + from store_sales + where ss_quantity between 1 and 20) + else (select avg(ss_net_paid_inc_tax) + from store_sales + where ss_quantity between 1 and 20) end bucket1 , + case when (select count(*) + from store_sales + where ss_quantity between 21 and 40) > 39161 + then (select avg(ss_ext_tax) + from store_sales + where ss_quantity between 21 and 40) + else (select avg(ss_net_paid_inc_tax) + from store_sales + where ss_quantity between 21 and 40) end bucket2, + case when (select count(*) + from store_sales + where ss_quantity between 41 and 60) > 29434 + then (select avg(ss_ext_tax) + from store_sales + where ss_quantity between 41 and 60) + else (select avg(ss_net_paid_inc_tax) + from store_sales + where ss_quantity between 41 and 60) end bucket3, + case when (select count(*) + from store_sales + where ss_quantity between 61 and 80) > 6568 + then (select avg(ss_ext_tax) + from store_sales + where ss_quantity between 61 and 80) + else (select avg(ss_net_paid_inc_tax) + from store_sales + where ss_quantity between 61 and 80) end bucket4, + case when (select count(*) + from store_sales + where ss_quantity between 81 and 100) > 21216 + then (select avg(ss_ext_tax) + from store_sales + where ss_quantity between 81 and 100) + else (select avg(ss_net_paid_inc_tax) + from store_sales + where ss_quantity between 81 and 100) end bucket5 +from reason +where r_reason_sk = 1 +; + +-- end query 9 in stream 0 using template query9.tpl +-- start query 10 in stream 0 using template query10.tpl +select + cd_gender, + cd_marital_status, + cd_education_status, + count(*) cnt1, + cd_purchase_estimate, + count(*) cnt2, + cd_credit_rating, + count(*) cnt3, + cd_dep_count, + count(*) cnt4, + cd_dep_employed_count, + count(*) cnt5, + cd_dep_college_count, + count(*) cnt6 + from + customer c,customer_address ca,customer_demographics + where + c.c_current_addr_sk = ca.ca_address_sk and + ca_county in ('Fairfield County','Campbell County','Washtenaw County','Escambia County','Cleburne County') and + cd_demo_sk = c.c_current_cdemo_sk and + exists (select * + from store_sales,date_dim + where c.c_customer_sk = ss_customer_sk and + ss_sold_date_sk = d_date_sk and + d_year = 2001 and + d_moy between 3 and 3+3) and + (exists (select * + from web_sales,date_dim + where c.c_customer_sk = ws_bill_customer_sk and + ws_sold_date_sk = d_date_sk and + d_year = 2001 and + d_moy between 3 ANd 3+3) or + exists (select * + from catalog_sales,date_dim + where c.c_customer_sk = cs_ship_customer_sk and + cs_sold_date_sk = d_date_sk and + d_year = 2001 and + d_moy between 3 and 3+3)) + group by cd_gender, + cd_marital_status, + cd_education_status, + cd_purchase_estimate, + cd_credit_rating, + cd_dep_count, + cd_dep_employed_count, + cd_dep_college_count + order by cd_gender, + cd_marital_status, + cd_education_status, + cd_purchase_estimate, + cd_credit_rating, + cd_dep_count, + cd_dep_employed_count, + cd_dep_college_count +limit 100; + +-- end query 10 in stream 0 using template query10.tpl +-- start query 11 in stream 0 using template query11.tpl +with year_total as ( + select c_customer_id customer_id + ,c_first_name customer_first_name + ,c_last_name customer_last_name + ,c_preferred_cust_flag customer_preferred_cust_flag + ,c_birth_country customer_birth_country + ,c_login customer_login + ,c_email_address customer_email_address + ,d_year dyear + ,sum(ss_ext_list_price-ss_ext_discount_amt) year_total + ,'s' sale_type + from customer + ,store_sales + ,date_dim + where c_customer_sk = ss_customer_sk + and ss_sold_date_sk = d_date_sk + group by c_customer_id + ,c_first_name + ,c_last_name + ,c_preferred_cust_flag + ,c_birth_country + ,c_login + ,c_email_address + ,d_year + union all + select c_customer_id customer_id + ,c_first_name customer_first_name + ,c_last_name customer_last_name + ,c_preferred_cust_flag customer_preferred_cust_flag + ,c_birth_country customer_birth_country + ,c_login customer_login + ,c_email_address customer_email_address + ,d_year dyear + ,sum(ws_ext_list_price-ws_ext_discount_amt) year_total + ,'w' sale_type + from customer + ,web_sales + ,date_dim + where c_customer_sk = ws_bill_customer_sk + and ws_sold_date_sk = d_date_sk + group by c_customer_id + ,c_first_name + ,c_last_name + ,c_preferred_cust_flag + ,c_birth_country + ,c_login + ,c_email_address + ,d_year + ) + select + t_s_secyear.customer_id + ,t_s_secyear.customer_first_name + ,t_s_secyear.customer_last_name + ,t_s_secyear.customer_email_address + from year_total t_s_firstyear + ,year_total t_s_secyear + ,year_total t_w_firstyear + ,year_total t_w_secyear + where t_s_secyear.customer_id = t_s_firstyear.customer_id + and t_s_firstyear.customer_id = t_w_secyear.customer_id + and t_s_firstyear.customer_id = t_w_firstyear.customer_id + and t_s_firstyear.sale_type = 's' + and t_w_firstyear.sale_type = 'w' + and t_s_secyear.sale_type = 's' + and t_w_secyear.sale_type = 'w' + and t_s_firstyear.dyear = 1998 + and t_s_secyear.dyear = 1998+1 + and t_w_firstyear.dyear = 1998 + and t_w_secyear.dyear = 1998+1 + and t_s_firstyear.year_total > 0 + and t_w_firstyear.year_total > 0 + and case when t_w_firstyear.year_total > 0 then t_w_secyear.year_total / t_w_firstyear.year_total else 0.0 end + > case when t_s_firstyear.year_total > 0 then t_s_secyear.year_total / t_s_firstyear.year_total else 0.0 end + order by t_s_secyear.customer_id + ,t_s_secyear.customer_first_name + ,t_s_secyear.customer_last_name + ,t_s_secyear.customer_email_address +limit 100; + +-- end query 11 in stream 0 using template query11.tpl +-- start query 12 in stream 0 using template query12.tpl +select i_item_id + ,i_item_desc + ,i_category + ,i_class + ,i_current_price + ,sum(ws_ext_sales_price) as itemrevenue + ,sum(ws_ext_sales_price)*100/sum(sum(ws_ext_sales_price)) over + (partition by i_class) as revenueratio +from + web_sales + ,item + ,date_dim +where + ws_item_sk = i_item_sk + and i_category in ('Men', 'Books', 'Electronics') + and ws_sold_date_sk = d_date_sk + and d_date between cast('2001-06-15' as date) + and (cast('2001-06-15' as date) + interval '30 days') +group by + i_item_id + ,i_item_desc + ,i_category + ,i_class + ,i_current_price +order by + i_category + ,i_class + ,i_item_id + ,i_item_desc + ,revenueratio +limit 100; + +-- end query 12 in stream 0 using template query12.tpl +-- start query 13 in stream 0 using template query13.tpl +select avg(ss_quantity) + ,avg(ss_ext_sales_price) + ,avg(ss_ext_wholesale_cost) + ,sum(ss_ext_wholesale_cost) + from store_sales + ,store + ,customer_demographics + ,household_demographics + ,customer_address + ,date_dim + where s_store_sk = ss_store_sk + and ss_sold_date_sk = d_date_sk and d_year = 2001 + and((ss_hdemo_sk=hd_demo_sk + and cd_demo_sk = ss_cdemo_sk + and cd_marital_status = 'M' + and cd_education_status = 'College' + and ss_sales_price between 100.00 and 150.00 + and hd_dep_count = 3 + )or + (ss_hdemo_sk=hd_demo_sk + and cd_demo_sk = ss_cdemo_sk + and cd_marital_status = 'D' + and cd_education_status = 'Primary' + and ss_sales_price between 50.00 and 100.00 + and hd_dep_count = 1 + ) or + (ss_hdemo_sk=hd_demo_sk + and cd_demo_sk = ss_cdemo_sk + and cd_marital_status = 'W' + and cd_education_status = '2 yr Degree' + and ss_sales_price between 150.00 and 200.00 + and hd_dep_count = 1 + )) + and((ss_addr_sk = ca_address_sk + and ca_country = 'United States' + and ca_state in ('IL', 'TN', 'TX') + and ss_net_profit between 100 and 200 + ) or + (ss_addr_sk = ca_address_sk + and ca_country = 'United States' + and ca_state in ('WY', 'OH', 'ID') + and ss_net_profit between 150 and 300 + ) or + (ss_addr_sk = ca_address_sk + and ca_country = 'United States' + and ca_state in ('MS', 'SC', 'IA') + and ss_net_profit between 50 and 250 + )) +; + +-- end query 13 in stream 0 using template query13.tpl +-- start query 14 in stream 0 using template query14.tpl +with cross_items as + (select i_item_sk ss_item_sk + from item, + (select iss.i_brand_id brand_id + ,iss.i_class_id class_id + ,iss.i_category_id category_id + from store_sales + ,item iss + ,date_dim d1 + where ss_item_sk = iss.i_item_sk + and ss_sold_date_sk = d1.d_date_sk + and d1.d_year between 1999 AND 1999 + 2 + intersect + select ics.i_brand_id + ,ics.i_class_id + ,ics.i_category_id + from catalog_sales + ,item ics + ,date_dim d2 + where cs_item_sk = ics.i_item_sk + and cs_sold_date_sk = d2.d_date_sk + and d2.d_year between 1999 AND 1999 + 2 + intersect + select iws.i_brand_id + ,iws.i_class_id + ,iws.i_category_id + from web_sales + ,item iws + ,date_dim d3 + where ws_item_sk = iws.i_item_sk + and ws_sold_date_sk = d3.d_date_sk + and d3.d_year between 1999 AND 1999 + 2) as sub + where i_brand_id = brand_id + and i_class_id = class_id + and i_category_id = category_id +), + avg_sales as + (select avg(quantity*list_price) average_sales + from (select ss_quantity quantity + ,ss_list_price list_price + from store_sales + ,date_dim + where ss_sold_date_sk = d_date_sk + and d_year between 1999 and 1999 + 2 + union all + select cs_quantity quantity + ,cs_list_price list_price + from catalog_sales + ,date_dim + where cs_sold_date_sk = d_date_sk + and d_year between 1999 and 1999 + 2 + union all + select ws_quantity quantity + ,ws_list_price list_price + from web_sales + ,date_dim + where ws_sold_date_sk = d_date_sk + and d_year between 1999 and 1999 + 2) as x) + select channel, i_brand_id,i_class_id,i_category_id,sum(sales), sum(number_sales) + from( + select 'store' channel, i_brand_id,i_class_id + ,i_category_id,sum(ss_quantity*ss_list_price) sales + , count(*) number_sales + from store_sales + ,item + ,date_dim + where ss_item_sk in (select ss_item_sk from cross_items) + and ss_item_sk = i_item_sk + and ss_sold_date_sk = d_date_sk + and d_year = 1999+2 + and d_moy = 11 + group by i_brand_id,i_class_id,i_category_id + having sum(ss_quantity*ss_list_price) > (select average_sales from avg_sales) + union all + select 'catalog' channel, i_brand_id,i_class_id,i_category_id, sum(cs_quantity*cs_list_price) sales, count(*) number_sales + from catalog_sales + ,item + ,date_dim + where cs_item_sk in (select ss_item_sk from cross_items) + and cs_item_sk = i_item_sk + and cs_sold_date_sk = d_date_sk + and d_year = 1999+2 + and d_moy = 11 + group by i_brand_id,i_class_id,i_category_id + having sum(cs_quantity*cs_list_price) > (select average_sales from avg_sales) + union all + select 'web' channel, i_brand_id,i_class_id,i_category_id, sum(ws_quantity*ws_list_price) sales , count(*) number_sales + from web_sales + ,item + ,date_dim + where ws_item_sk in (select ss_item_sk from cross_items) + and ws_item_sk = i_item_sk + and ws_sold_date_sk = d_date_sk + and d_year = 1999+2 + and d_moy = 11 + group by i_brand_id,i_class_id,i_category_id + having sum(ws_quantity*ws_list_price) > (select average_sales from avg_sales) + ) y + group by rollup (channel, i_brand_id,i_class_id,i_category_id) + order by channel,i_brand_id,i_class_id,i_category_id + limit 100; + +with cross_items as + (select i_item_sk ss_item_sk + from item, + (select iss.i_brand_id brand_id + ,iss.i_class_id class_id + ,iss.i_category_id category_id + from store_sales + ,item iss + ,date_dim d1 + where ss_item_sk = iss.i_item_sk + and ss_sold_date_sk = d1.d_date_sk + and d1.d_year between 1999 AND 1999 + 2 + intersect + select ics.i_brand_id + ,ics.i_class_id + ,ics.i_category_id + from catalog_sales + ,item ics + ,date_dim d2 + where cs_item_sk = ics.i_item_sk + and cs_sold_date_sk = d2.d_date_sk + and d2.d_year between 1999 AND 1999 + 2 + intersect + select iws.i_brand_id + ,iws.i_class_id + ,iws.i_category_id + from web_sales + ,item iws + ,date_dim d3 + where ws_item_sk = iws.i_item_sk + and ws_sold_date_sk = d3.d_date_sk + and d3.d_year between 1999 AND 1999 + 2) as x + where i_brand_id = brand_id + and i_class_id = class_id + and i_category_id = category_id +), + avg_sales as +(select avg(quantity*list_price) average_sales + from (select ss_quantity quantity + ,ss_list_price list_price + from store_sales + ,date_dim + where ss_sold_date_sk = d_date_sk + and d_year between 1999 and 1999 + 2 + union all + select cs_quantity quantity + ,cs_list_price list_price + from catalog_sales + ,date_dim + where cs_sold_date_sk = d_date_sk + and d_year between 1999 and 1999 + 2 + union all + select ws_quantity quantity + ,ws_list_price list_price + from web_sales + ,date_dim + where ws_sold_date_sk = d_date_sk + and d_year between 1999 and 1999 + 2) as x) + select this_year.channel ty_channel + ,this_year.i_brand_id ty_brand + ,this_year.i_class_id ty_class + ,this_year.i_category_id ty_category + ,this_year.sales ty_sales + ,this_year.number_sales ty_number_sales + ,last_year.channel ly_channel + ,last_year.i_brand_id ly_brand + ,last_year.i_class_id ly_class + ,last_year.i_category_id ly_category + ,last_year.sales ly_sales + ,last_year.number_sales ly_number_sales + from + (select 'store' channel, i_brand_id,i_class_id,i_category_id + ,sum(ss_quantity*ss_list_price) sales, count(*) number_sales + from store_sales + ,item + ,date_dim + where ss_item_sk in (select ss_item_sk from cross_items) + and ss_item_sk = i_item_sk + and ss_sold_date_sk = d_date_sk + and d_week_seq = (select d_week_seq + from date_dim + where d_year = 1999 + 1 + and d_moy = 12 + and d_dom = 3) + group by i_brand_id,i_class_id,i_category_id + having sum(ss_quantity*ss_list_price) > (select average_sales from avg_sales)) this_year, + (select 'store' channel, i_brand_id,i_class_id + ,i_category_id, sum(ss_quantity*ss_list_price) sales, count(*) number_sales + from store_sales + ,item + ,date_dim + where ss_item_sk in (select ss_item_sk from cross_items) + and ss_item_sk = i_item_sk + and ss_sold_date_sk = d_date_sk + and d_week_seq = (select d_week_seq + from date_dim + where d_year = 1999 + and d_moy = 12 + and d_dom = 3) + group by i_brand_id,i_class_id,i_category_id + having sum(ss_quantity*ss_list_price) > (select average_sales from avg_sales)) last_year + where this_year.i_brand_id= last_year.i_brand_id + and this_year.i_class_id = last_year.i_class_id + and this_year.i_category_id = last_year.i_category_id + order by this_year.channel, this_year.i_brand_id, this_year.i_class_id, this_year.i_category_id + limit 100; + +-- end query 14 in stream 0 using template query14.tpl +-- start query 15 in stream 0 using template query15.tpl +select ca_zip + ,sum(cs_sales_price) + from catalog_sales + ,customer + ,customer_address + ,date_dim + where cs_bill_customer_sk = c_customer_sk + and c_current_addr_sk = ca_address_sk + and ( substr(ca_zip,1,5) in ('85669', '86197','88274','83405','86475', + '85392', '85460', '80348', '81792') + or ca_state in ('CA','WA','GA') + or cs_sales_price > 500) + and cs_sold_date_sk = d_date_sk + and d_qoy = 2 and d_year = 2001 + group by ca_zip + order by ca_zip + limit 100; + +-- end query 15 in stream 0 using template query15.tpl +-- start query 16 in stream 0 using template query16.tpl +select + count(distinct cs_order_number) as "order count" + ,sum(cs_ext_ship_cost) as "total shipping cost" + ,sum(cs_net_profit) as "total net profit" +from + catalog_sales cs1 + ,date_dim + ,customer_address + ,call_center +where + d_date between '2002-4-01' and + (cast('2002-4-01' as date) + interval '60 days') +and cs1.cs_ship_date_sk = d_date_sk +and cs1.cs_ship_addr_sk = ca_address_sk +and ca_state = 'PA' +and cs1.cs_call_center_sk = cc_call_center_sk +and cc_county in ('Williamson County','Williamson County','Williamson County','Williamson County', + 'Williamson County' +) +and exists (select * + from catalog_sales cs2 + where cs1.cs_order_number = cs2.cs_order_number + and cs1.cs_warehouse_sk <> cs2.cs_warehouse_sk) +and not exists(select * + from catalog_returns cr1 + where cs1.cs_order_number = cr1.cr_order_number) +order by count(distinct cs_order_number) +limit 100; + +-- end query 16 in stream 0 using template query16.tpl +-- start query 17 in stream 0 using template query17.tpl +select i_item_id + ,i_item_desc + ,s_state + ,count(ss_quantity) as store_sales_quantitycount + ,avg(ss_quantity) as store_sales_quantityave + ,stddev_samp(ss_quantity) as store_sales_quantitystdev + ,stddev_samp(ss_quantity)/avg(ss_quantity) as store_sales_quantitycov + ,count(sr_return_quantity) as store_returns_quantitycount + ,avg(sr_return_quantity) as store_returns_quantityave + ,stddev_samp(sr_return_quantity) as store_returns_quantitystdev + ,stddev_samp(sr_return_quantity)/avg(sr_return_quantity) as store_returns_quantitycov + ,count(cs_quantity) as catalog_sales_quantitycount ,avg(cs_quantity) as catalog_sales_quantityave + ,stddev_samp(cs_quantity) as catalog_sales_quantitystdev + ,stddev_samp(cs_quantity)/avg(cs_quantity) as catalog_sales_quantitycov + from store_sales + ,store_returns + ,catalog_sales + ,date_dim d1 + ,date_dim d2 + ,date_dim d3 + ,store + ,item + where d1.d_quarter_name = '2001Q1' + and d1.d_date_sk = ss_sold_date_sk + and i_item_sk = ss_item_sk + and s_store_sk = ss_store_sk + and ss_customer_sk = sr_customer_sk + and ss_item_sk = sr_item_sk + and ss_ticket_number = sr_ticket_number + and sr_returned_date_sk = d2.d_date_sk + and d2.d_quarter_name in ('2001Q1','2001Q2','2001Q3') + and sr_customer_sk = cs_bill_customer_sk + and sr_item_sk = cs_item_sk + and cs_sold_date_sk = d3.d_date_sk + and d3.d_quarter_name in ('2001Q1','2001Q2','2001Q3') + group by i_item_id + ,i_item_desc + ,s_state + order by i_item_id + ,i_item_desc + ,s_state +limit 100; + +-- end query 17 in stream 0 using template query17.tpl +-- start query 18 in stream 0 using template query18.tpl +select i_item_id, + ca_country, + ca_state, + ca_county, + avg( cast(cs_quantity as decimal(12,2))) agg1, + avg( cast(cs_list_price as decimal(12,2))) agg2, + avg( cast(cs_coupon_amt as decimal(12,2))) agg3, + avg( cast(cs_sales_price as decimal(12,2))) agg4, + avg( cast(cs_net_profit as decimal(12,2))) agg5, + avg( cast(c_birth_year as decimal(12,2))) agg6, + avg( cast(cd1.cd_dep_count as decimal(12,2))) agg7 + from catalog_sales, customer_demographics cd1, + customer_demographics cd2, customer, customer_address, date_dim, item + where cs_sold_date_sk = d_date_sk and + cs_item_sk = i_item_sk and + cs_bill_cdemo_sk = cd1.cd_demo_sk and + cs_bill_customer_sk = c_customer_sk and + cd1.cd_gender = 'F' and + cd1.cd_education_status = 'Primary' and + c_current_cdemo_sk = cd2.cd_demo_sk and + c_current_addr_sk = ca_address_sk and + c_birth_month in (1,3,7,11,10,4) and + d_year = 2001 and + ca_state in ('AL','MO','TN' + ,'GA','MT','IN','CA') + group by rollup (i_item_id, ca_country, ca_state, ca_county) + order by ca_country, + ca_state, + ca_county, + i_item_id + limit 100; + +-- end query 18 in stream 0 using template query18.tpl +-- start query 19 in stream 0 using template query19.tpl +select i_brand_id brand_id, i_brand brand, i_manufact_id, i_manufact, + sum(ss_ext_sales_price) ext_price + from date_dim, store_sales, item,customer,customer_address,store + where d_date_sk = ss_sold_date_sk + and ss_item_sk = i_item_sk + and i_manager_id=14 + and d_moy=11 + and d_year=2002 + and ss_customer_sk = c_customer_sk + and c_current_addr_sk = ca_address_sk + and substr(ca_zip,1,5) <> substr(s_zip,1,5) + and ss_store_sk = s_store_sk + group by i_brand + ,i_brand_id + ,i_manufact_id + ,i_manufact + order by ext_price desc + ,i_brand + ,i_brand_id + ,i_manufact_id + ,i_manufact +limit 100 ; + +-- end query 19 in stream 0 using template query19.tpl +-- start query 20 in stream 0 using template query20.tpl +select i_item_id + ,i_item_desc + ,i_category + ,i_class + ,i_current_price + ,sum(cs_ext_sales_price) as itemrevenue + ,sum(cs_ext_sales_price)*100/sum(sum(cs_ext_sales_price)) over + (partition by i_class) as revenueratio + from catalog_sales + ,item + ,date_dim + where cs_item_sk = i_item_sk + and i_category in ('Books', 'Music', 'Sports') + and cs_sold_date_sk = d_date_sk + and d_date between cast('2002-06-18' as date) + and (cast('2002-06-18' as date) + interval '30 days') + group by i_item_id + ,i_item_desc + ,i_category + ,i_class + ,i_current_price + order by i_category + ,i_class + ,i_item_id + ,i_item_desc + ,revenueratio +limit 100; + +-- end query 20 in stream 0 using template query20.tpl +-- start query 21 in stream 0 using template query21.tpl +select * + from(select w_warehouse_name + ,i_item_id + ,sum(case when (cast(d_date as date) < cast ('1999-06-22' as date)) + then inv_quantity_on_hand + else 0 end) as inv_before + ,sum(case when (cast(d_date as date) >= cast ('1999-06-22' as date)) + then inv_quantity_on_hand + else 0 end) as inv_after + from inventory + ,warehouse + ,item + ,date_dim + where i_current_price between 0.99 and 1.49 + and i_item_sk = inv_item_sk + and inv_warehouse_sk = w_warehouse_sk + and inv_date_sk = d_date_sk + and d_date between (cast ('1999-06-22' as date) - interval '30 days') + and (cast ('1999-06-22' as date) + interval '30 days') + group by w_warehouse_name, i_item_id) x + where (case when inv_before > 0 + then inv_after / inv_before + else null + end) between 2.0/3.0 and 3.0/2.0 + order by w_warehouse_name + ,i_item_id + limit 100; + +-- end query 21 in stream 0 using template query21.tpl +-- start query 22 in stream 0 using template query22.tpl +select i_product_name + ,i_brand + ,i_class + ,i_category + ,avg(inv_quantity_on_hand) qoh + from inventory + ,date_dim + ,item + where inv_date_sk=d_date_sk + and inv_item_sk=i_item_sk + and d_month_seq between 1200 and 1200 + 11 + group by rollup(i_product_name + ,i_brand + ,i_class + ,i_category) +order by qoh, i_product_name, i_brand, i_class, i_category +limit 100; + +-- end query 22 in stream 0 using template query22.tpl +-- start query 23 in stream 0 using template query23.tpl +with frequent_ss_items as + (select substr(i_item_desc,1,30) itemdesc,i_item_sk item_sk,d_date solddate,count(*) cnt + from store_sales + ,date_dim + ,item + where ss_sold_date_sk = d_date_sk + and ss_item_sk = i_item_sk + and d_year in (2000,2000+1,2000+2,2000+3) + group by substr(i_item_desc,1,30),i_item_sk,d_date + having count(*) >4), + max_store_sales as + (select max(csales) tpcds_cmax + from (select c_customer_sk,sum(ss_quantity*ss_sales_price) as csales + from store_sales + ,customer + ,date_dim + where ss_customer_sk = c_customer_sk + and ss_sold_date_sk = d_date_sk + and d_year in (2000,2000+1,2000+2,2000+3) + group by c_customer_sk) as scsales + ), + best_ss_customer as + (select c_customer_sk,sum(ss_quantity*ss_sales_price) ssales + from store_sales + ,customer + where ss_customer_sk = c_customer_sk + group by c_customer_sk + having sum(ss_quantity*ss_sales_price) > (95/100.0) * (select + * +from + max_store_sales)) + select sum(sales) + from (select cs_quantity*cs_list_price sales + from catalog_sales + ,date_dim + where d_year = 2000 + and d_moy = 7 + and cs_sold_date_sk = d_date_sk + and cs_item_sk in (select item_sk from frequent_ss_items) + and cs_bill_customer_sk in (select c_customer_sk from best_ss_customer) + union all + select ws_quantity*ws_list_price sales + from web_sales + ,date_dim + where d_year = 2000 + and d_moy = 7 + and ws_sold_date_sk = d_date_sk + and ws_item_sk in (select item_sk from frequent_ss_items) + and ws_bill_customer_sk in (select c_customer_sk from best_ss_customer)) as foo + limit 100; + +with frequent_ss_items as + (select substr(i_item_desc,1,30) itemdesc,i_item_sk item_sk,d_date solddate,count(*) cnt + from store_sales + ,date_dim + ,item + where ss_sold_date_sk = d_date_sk + and ss_item_sk = i_item_sk + and d_year in (2000,2000 + 1,2000 + 2,2000 + 3) + group by substr(i_item_desc,1,30),i_item_sk,d_date + having count(*) >4), + max_store_sales as + (select max(csales) tpcds_cmax + from (select c_customer_sk,sum(ss_quantity*ss_sales_price) csales + from store_sales + ,customer + ,date_dim + where ss_customer_sk = c_customer_sk + and ss_sold_date_sk = d_date_sk + and d_year in (2000,2000+1,2000+2,2000+3) + group by c_customer_sk) as scsales), + best_ss_customer as + (select c_customer_sk,sum(ss_quantity*ss_sales_price) ssales + from store_sales + ,customer + where ss_customer_sk = c_customer_sk + group by c_customer_sk + having sum(ss_quantity*ss_sales_price) > (95/100.0) * (select + * + from max_store_sales)) + select c_last_name,c_first_name,sales + from (select c_last_name,c_first_name,sum(cs_quantity*cs_list_price) sales + from catalog_sales + ,customer + ,date_dim + where d_year = 2000 + and d_moy = 7 + and cs_sold_date_sk = d_date_sk + and cs_item_sk in (select item_sk from frequent_ss_items) + and cs_bill_customer_sk in (select c_customer_sk from best_ss_customer) + and cs_bill_customer_sk = c_customer_sk + group by c_last_name,c_first_name + union all + select c_last_name,c_first_name,sum(ws_quantity*ws_list_price) sales + from web_sales + ,customer + ,date_dim + where d_year = 2000 + and d_moy = 7 + and ws_sold_date_sk = d_date_sk + and ws_item_sk in (select item_sk from frequent_ss_items) + and ws_bill_customer_sk in (select c_customer_sk from best_ss_customer) + and ws_bill_customer_sk = c_customer_sk + group by c_last_name,c_first_name) as sub + order by c_last_name,c_first_name,sales + limit 100; + +-- end query 23 in stream 0 using template query23.tpl +-- start query 24 in stream 0 using template query24.tpl +with ssales as +(select c_last_name + ,c_first_name + ,s_store_name + ,ca_state + ,s_state + ,i_color + ,i_current_price + ,i_manager_id + ,i_units + ,i_size + ,sum(ss_net_paid) netpaid +from store_sales + ,store_returns + ,store + ,item + ,customer + ,customer_address +where ss_ticket_number = sr_ticket_number + and ss_item_sk = sr_item_sk + and ss_customer_sk = c_customer_sk + and ss_item_sk = i_item_sk + and ss_store_sk = s_store_sk + and c_current_addr_sk = ca_address_sk + and c_birth_country <> upper(ca_country) + and s_zip = ca_zip +and s_market_id=5 +group by c_last_name + ,c_first_name + ,s_store_name + ,ca_state + ,s_state + ,i_color + ,i_current_price + ,i_manager_id + ,i_units + ,i_size) +select c_last_name + ,c_first_name + ,s_store_name + ,sum(netpaid) paid +from ssales +where i_color = 'aquamarine' +group by c_last_name + ,c_first_name + ,s_store_name +having sum(netpaid) > (select 0.05*avg(netpaid) + from ssales) +order by c_last_name + ,c_first_name + ,s_store_name +; +with ssales as +(select c_last_name + ,c_first_name + ,s_store_name + ,ca_state + ,s_state + ,i_color + ,i_current_price + ,i_manager_id + ,i_units + ,i_size + ,sum(ss_net_paid) netpaid +from store_sales + ,store_returns + ,store + ,item + ,customer + ,customer_address +where ss_ticket_number = sr_ticket_number + and ss_item_sk = sr_item_sk + and ss_customer_sk = c_customer_sk + and ss_item_sk = i_item_sk + and ss_store_sk = s_store_sk + and c_current_addr_sk = ca_address_sk + and c_birth_country <> upper(ca_country) + and s_zip = ca_zip + and s_market_id = 5 +group by c_last_name + ,c_first_name + ,s_store_name + ,ca_state + ,s_state + ,i_color + ,i_current_price + ,i_manager_id + ,i_units + ,i_size) +select c_last_name + ,c_first_name + ,s_store_name + ,sum(netpaid) paid +from ssales +where i_color = 'seashell' +group by c_last_name + ,c_first_name + ,s_store_name +having sum(netpaid) > (select 0.05*avg(netpaid) + from ssales) +order by c_last_name + ,c_first_name + ,s_store_name +; + +-- end query 24 in stream 0 using template query24.tpl +-- start query 25 in stream 0 using template query25.tpl +select + i_item_id + ,i_item_desc + ,s_store_id + ,s_store_name + ,max(ss_net_profit) as store_sales_profit + ,max(sr_net_loss) as store_returns_loss + ,max(cs_net_profit) as catalog_sales_profit + from + store_sales + ,store_returns + ,catalog_sales + ,date_dim d1 + ,date_dim d2 + ,date_dim d3 + ,store + ,item + where + d1.d_moy = 4 + and d1.d_year = 1999 + and d1.d_date_sk = ss_sold_date_sk + and i_item_sk = ss_item_sk + and s_store_sk = ss_store_sk + and ss_customer_sk = sr_customer_sk + and ss_item_sk = sr_item_sk + and ss_ticket_number = sr_ticket_number + and sr_returned_date_sk = d2.d_date_sk + and d2.d_moy between 4 and 10 + and d2.d_year = 1999 + and sr_customer_sk = cs_bill_customer_sk + and sr_item_sk = cs_item_sk + and cs_sold_date_sk = d3.d_date_sk + and d3.d_moy between 4 and 10 + and d3.d_year = 1999 + group by + i_item_id + ,i_item_desc + ,s_store_id + ,s_store_name + order by + i_item_id + ,i_item_desc + ,s_store_id + ,s_store_name + limit 100; + +-- end query 25 in stream 0 using template query25.tpl +-- start query 26 in stream 0 using template query26.tpl +select i_item_id, + avg(cs_quantity) agg1, + avg(cs_list_price) agg2, + avg(cs_coupon_amt) agg3, + avg(cs_sales_price) agg4 + from catalog_sales, customer_demographics, date_dim, item, promotion + where cs_sold_date_sk = d_date_sk and + cs_item_sk = i_item_sk and + cs_bill_cdemo_sk = cd_demo_sk and + cs_promo_sk = p_promo_sk and + cd_gender = 'M' and + cd_marital_status = 'W' and + cd_education_status = 'Unknown' and + (p_channel_email = 'N' or p_channel_event = 'N') and + d_year = 2002 + group by i_item_id + order by i_item_id + limit 100; + +-- end query 26 in stream 0 using template query26.tpl +-- start query 27 in stream 0 using template query27.tpl +select i_item_id, + s_state, grouping(s_state) g_state, + avg(ss_quantity) agg1, + avg(ss_list_price) agg2, + avg(ss_coupon_amt) agg3, + avg(ss_sales_price) agg4 + from store_sales, customer_demographics, date_dim, store, item + where ss_sold_date_sk = d_date_sk and + ss_item_sk = i_item_sk and + ss_store_sk = s_store_sk and + ss_cdemo_sk = cd_demo_sk and + cd_gender = 'M' and + cd_marital_status = 'W' and + cd_education_status = 'Secondary' and + d_year = 1999 and + s_state in ('TN','TN', 'TN', 'TN', 'TN', 'TN') + group by rollup (i_item_id, s_state) + order by i_item_id + ,s_state + limit 100; + +-- end query 27 in stream 0 using template query27.tpl +-- start query 28 in stream 0 using template query28.tpl +select * +from (select avg(ss_list_price) B1_LP + ,count(ss_list_price) B1_CNT + ,count(distinct ss_list_price) B1_CNTD + from store_sales + where ss_quantity between 0 and 5 + and (ss_list_price between 107 and 107+10 + or ss_coupon_amt between 1319 and 1319+1000 + or ss_wholesale_cost between 60 and 60+20)) B1, + (select avg(ss_list_price) B2_LP + ,count(ss_list_price) B2_CNT + ,count(distinct ss_list_price) B2_CNTD + from store_sales + where ss_quantity between 6 and 10 + and (ss_list_price between 23 and 23+10 + or ss_coupon_amt between 825 and 825+1000 + or ss_wholesale_cost between 43 and 43+20)) B2, + (select avg(ss_list_price) B3_LP + ,count(ss_list_price) B3_CNT + ,count(distinct ss_list_price) B3_CNTD + from store_sales + where ss_quantity between 11 and 15 + and (ss_list_price between 74 and 74+10 + or ss_coupon_amt between 4381 and 4381+1000 + or ss_wholesale_cost between 57 and 57+20)) B3, + (select avg(ss_list_price) B4_LP + ,count(ss_list_price) B4_CNT + ,count(distinct ss_list_price) B4_CNTD + from store_sales + where ss_quantity between 16 and 20 + and (ss_list_price between 89 and 89+10 + or ss_coupon_amt between 3117 and 3117+1000 + or ss_wholesale_cost between 68 and 68+20)) B4, + (select avg(ss_list_price) B5_LP + ,count(ss_list_price) B5_CNT + ,count(distinct ss_list_price) B5_CNTD + from store_sales + where ss_quantity between 21 and 25 + and (ss_list_price between 58 and 58+10 + or ss_coupon_amt between 9402 and 9402+1000 + or ss_wholesale_cost between 38 and 38+20)) B5, + (select avg(ss_list_price) B6_LP + ,count(ss_list_price) B6_CNT + ,count(distinct ss_list_price) B6_CNTD + from store_sales + where ss_quantity between 26 and 30 + and (ss_list_price between 64 and 64+10 + or ss_coupon_amt between 5792 and 5792+1000 + or ss_wholesale_cost between 73 and 73+20)) B6 +limit 100; + +-- end query 28 in stream 0 using template query28.tpl +-- start query 29 in stream 0 using template query29.tpl +select + i_item_id + ,i_item_desc + ,s_store_id + ,s_store_name + ,max(ss_quantity) as store_sales_quantity + ,max(sr_return_quantity) as store_returns_quantity + ,max(cs_quantity) as catalog_sales_quantity + from + store_sales + ,store_returns + ,catalog_sales + ,date_dim d1 + ,date_dim d2 + ,date_dim d3 + ,store + ,item + where + d1.d_moy = 4 + and d1.d_year = 1998 + and d1.d_date_sk = ss_sold_date_sk + and i_item_sk = ss_item_sk + and s_store_sk = ss_store_sk + and ss_customer_sk = sr_customer_sk + and ss_item_sk = sr_item_sk + and ss_ticket_number = sr_ticket_number + and sr_returned_date_sk = d2.d_date_sk + and d2.d_moy between 4 and 4 + 3 + and d2.d_year = 1998 + and sr_customer_sk = cs_bill_customer_sk + and sr_item_sk = cs_item_sk + and cs_sold_date_sk = d3.d_date_sk + and d3.d_year in (1998,1998+1,1998+2) + group by + i_item_id + ,i_item_desc + ,s_store_id + ,s_store_name + order by + i_item_id + ,i_item_desc + ,s_store_id + ,s_store_name + limit 100; + +-- end query 29 in stream 0 using template query29.tpl +-- start query 30 in stream 0 using template query30.tpl +with customer_total_return as + (select wr_returning_customer_sk as ctr_customer_sk + ,ca_state as ctr_state, + sum(wr_return_amt) as ctr_total_return + from web_returns + ,date_dim + ,customer_address + where wr_returned_date_sk = d_date_sk + and d_year =2000 + and wr_returning_addr_sk = ca_address_sk + group by wr_returning_customer_sk + ,ca_state) + select c_customer_id,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag + ,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address + ,c_last_review_date_sk,ctr_total_return + from customer_total_return ctr1 + ,customer_address + ,customer + where ctr1.ctr_total_return > (select avg(ctr_total_return)*1.2 + from customer_total_return ctr2 + where ctr1.ctr_state = ctr2.ctr_state) + and ca_address_sk = c_current_addr_sk + and ca_state = 'AR' + and ctr1.ctr_customer_sk = c_customer_sk + order by c_customer_id,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag + ,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address + ,c_last_review_date_sk,ctr_total_return +limit 100; + +-- end query 30 in stream 0 using template query30.tpl +-- start query 31 in stream 0 using template query31.tpl +with ss as + (select ca_county,d_qoy, d_year,sum(ss_ext_sales_price) as store_sales + from store_sales,date_dim,customer_address + where ss_sold_date_sk = d_date_sk + and ss_addr_sk=ca_address_sk + group by ca_county,d_qoy, d_year), + ws as + (select ca_county,d_qoy, d_year,sum(ws_ext_sales_price) as web_sales + from web_sales,date_dim,customer_address + where ws_sold_date_sk = d_date_sk + and ws_bill_addr_sk=ca_address_sk + group by ca_county,d_qoy, d_year) + select + ss1.ca_county + ,ss1.d_year + ,ws2.web_sales/ws1.web_sales web_q1_q2_increase + ,ss2.store_sales/ss1.store_sales store_q1_q2_increase + ,ws3.web_sales/ws2.web_sales web_q2_q3_increase + ,ss3.store_sales/ss2.store_sales store_q2_q3_increase + from + ss ss1 + ,ss ss2 + ,ss ss3 + ,ws ws1 + ,ws ws2 + ,ws ws3 + where + ss1.d_qoy = 1 + and ss1.d_year = 1999 + and ss1.ca_county = ss2.ca_county + and ss2.d_qoy = 2 + and ss2.d_year = 1999 + and ss2.ca_county = ss3.ca_county + and ss3.d_qoy = 3 + and ss3.d_year = 1999 + and ss1.ca_county = ws1.ca_county + and ws1.d_qoy = 1 + and ws1.d_year = 1999 + and ws1.ca_county = ws2.ca_county + and ws2.d_qoy = 2 + and ws2.d_year = 1999 + and ws1.ca_county = ws3.ca_county + and ws3.d_qoy = 3 + and ws3.d_year =1999 + and case when ws1.web_sales > 0 then ws2.web_sales/ws1.web_sales else null end + > case when ss1.store_sales > 0 then ss2.store_sales/ss1.store_sales else null end + and case when ws2.web_sales > 0 then ws3.web_sales/ws2.web_sales else null end + > case when ss2.store_sales > 0 then ss3.store_sales/ss2.store_sales else null end + order by store_q2_q3_increase; + +-- end query 31 in stream 0 using template query31.tpl +-- start query 32 in stream 0 using template query32.tpl +select sum(cs_ext_discount_amt) as "excess discount amount" +from + catalog_sales + ,item + ,date_dim +where +i_manufact_id = 722 +and i_item_sk = cs_item_sk +and d_date between '2001-03-09' and + (cast('2001-03-09' as date) + interval '90 days') +and d_date_sk = cs_sold_date_sk +and cs_ext_discount_amt + > ( + select + 1.3 * avg(cs_ext_discount_amt) + from + catalog_sales + ,date_dim + where + cs_item_sk = i_item_sk + and d_date between '2001-03-09' and + (cast('2001-03-09' as date) + interval '90 days') + and d_date_sk = cs_sold_date_sk + ) +limit 100; + +-- end query 32 in stream 0 using template query32.tpl +-- start query 33 in stream 0 using template query33.tpl +with ss as ( + select + i_manufact_id,sum(ss_ext_sales_price) total_sales + from + store_sales, + date_dim, + customer_address, + item + where + i_manufact_id in (select + i_manufact_id +from + item +where i_category in ('Books')) + and ss_item_sk = i_item_sk + and ss_sold_date_sk = d_date_sk + and d_year = 2001 + and d_moy = 3 + and ss_addr_sk = ca_address_sk + and ca_gmt_offset = -5 + group by i_manufact_id), + cs as ( + select + i_manufact_id,sum(cs_ext_sales_price) total_sales + from + catalog_sales, + date_dim, + customer_address, + item + where + i_manufact_id in (select + i_manufact_id +from + item +where i_category in ('Books')) + and cs_item_sk = i_item_sk + and cs_sold_date_sk = d_date_sk + and d_year = 2001 + and d_moy = 3 + and cs_bill_addr_sk = ca_address_sk + and ca_gmt_offset = -5 + group by i_manufact_id), + ws as ( + select + i_manufact_id,sum(ws_ext_sales_price) total_sales + from + web_sales, + date_dim, + customer_address, + item + where + i_manufact_id in (select + i_manufact_id +from + item +where i_category in ('Books')) + and ws_item_sk = i_item_sk + and ws_sold_date_sk = d_date_sk + and d_year = 2001 + and d_moy = 3 + and ws_bill_addr_sk = ca_address_sk + and ca_gmt_offset = -5 + group by i_manufact_id) + select i_manufact_id ,sum(total_sales) total_sales + from (select * from ss + union all + select * from cs + union all + select * from ws) tmp1 + group by i_manufact_id + order by total_sales +limit 100; + +-- end query 33 in stream 0 using template query33.tpl +-- start query 34 in stream 0 using template query34.tpl +select c_last_name + ,c_first_name + ,c_salutation + ,c_preferred_cust_flag + ,ss_ticket_number + ,cnt from + (select ss_ticket_number + ,ss_customer_sk + ,count(*) cnt + from store_sales,date_dim,store,household_demographics + where store_sales.ss_sold_date_sk = date_dim.d_date_sk + and store_sales.ss_store_sk = store.s_store_sk + and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk + and (date_dim.d_dom between 1 and 3 or date_dim.d_dom between 25 and 28) + and (household_demographics.hd_buy_potential = '1001-5000' or + household_demographics.hd_buy_potential = '0-500') + and household_demographics.hd_vehicle_count > 0 + and (case when household_demographics.hd_vehicle_count > 0 + then household_demographics.hd_dep_count/ household_demographics.hd_vehicle_count + else null + end) > 1.2 + and date_dim.d_year in (2000,2000+1,2000+2) + and store.s_county in ('Williamson County','Williamson County','Williamson County','Williamson County', + 'Williamson County','Williamson County','Williamson County','Williamson County') + group by ss_ticket_number,ss_customer_sk) dn,customer + where ss_customer_sk = c_customer_sk + and cnt between 15 and 20 + order by c_last_name,c_first_name,c_salutation,c_preferred_cust_flag desc, ss_ticket_number; + +-- end query 34 in stream 0 using template query34.tpl +-- start query 35 in stream 0 using template query35.tpl +select + ca_state, + cd_gender, + cd_marital_status, + cd_dep_count, + count(*) cnt1, + avg(cd_dep_count), + stddev_samp(cd_dep_count), + sum(cd_dep_count), + cd_dep_employed_count, + count(*) cnt2, + avg(cd_dep_employed_count), + stddev_samp(cd_dep_employed_count), + sum(cd_dep_employed_count), + cd_dep_college_count, + count(*) cnt3, + avg(cd_dep_college_count), + stddev_samp(cd_dep_college_count), + sum(cd_dep_college_count) + from + customer c,customer_address ca,customer_demographics + where + c.c_current_addr_sk = ca.ca_address_sk and + cd_demo_sk = c.c_current_cdemo_sk and + exists (select * + from store_sales,date_dim + where c.c_customer_sk = ss_customer_sk and + ss_sold_date_sk = d_date_sk and + d_year = 1999 and + d_qoy < 4) and + (exists (select * + from web_sales,date_dim + where c.c_customer_sk = ws_bill_customer_sk and + ws_sold_date_sk = d_date_sk and + d_year = 1999 and + d_qoy < 4) or + exists (select * + from catalog_sales,date_dim + where c.c_customer_sk = cs_ship_customer_sk and + cs_sold_date_sk = d_date_sk and + d_year = 1999 and + d_qoy < 4)) + group by ca_state, + cd_gender, + cd_marital_status, + cd_dep_count, + cd_dep_employed_count, + cd_dep_college_count + order by ca_state, + cd_gender, + cd_marital_status, + cd_dep_count, + cd_dep_employed_count, + cd_dep_college_count + limit 100; + +-- end query 35 in stream 0 using template query35.tpl +-- start query 36 in stream 0 using template query36.tpl +select * +from +(select + sum(ss_net_profit)/sum(ss_ext_sales_price) as gross_margin + ,i_category + ,i_class + ,grouping(i_category)+grouping(i_class) as lochierarchy + ,rank() over ( + partition by grouping(i_category)+grouping(i_class), + case when grouping(i_class) = 0 then i_category end + order by sum(ss_net_profit)/sum(ss_ext_sales_price) asc) as rank_within_parent + from + store_sales + ,date_dim d1 + ,item + ,store + where + d1.d_year = 2000 + and d1.d_date_sk = ss_sold_date_sk + and i_item_sk = ss_item_sk + and s_store_sk = ss_store_sk + and s_state in ('TN','TN','TN','TN', + 'TN','TN','TN','TN') + group by rollup(i_category,i_class)) as sub + order by + lochierarchy desc + ,case when lochierarchy = 0 then i_category end + ,rank_within_parent + limit 100; + +-- end query 36 in stream 0 using template query36.tpl +-- start query 37 in stream 0 using template query37.tpl +select i_item_id + ,i_item_desc + ,i_current_price + from item, inventory, date_dim, catalog_sales + where i_current_price between 29 and 29 + 30 + and inv_item_sk = i_item_sk + and d_date_sk=inv_date_sk + and d_date between cast('2002-03-29' as date) and (cast('2002-03-29' as date) + interval '60 days') + and i_manufact_id in (705,742,777,944) + and inv_quantity_on_hand between 100 and 500 + and cs_item_sk = i_item_sk + group by i_item_id,i_item_desc,i_current_price + order by i_item_id + limit 100; + +-- end query 37 in stream 0 using template query37.tpl +-- start query 38 in stream 0 using template query38.tpl +select count(*) from ( + select distinct c_last_name, c_first_name, d_date + from store_sales, date_dim, customer + where store_sales.ss_sold_date_sk = date_dim.d_date_sk + and store_sales.ss_customer_sk = customer.c_customer_sk + and d_month_seq between 1189 and 1189 + 11 + intersect + select distinct c_last_name, c_first_name, d_date + from catalog_sales, date_dim, customer + where catalog_sales.cs_sold_date_sk = date_dim.d_date_sk + and catalog_sales.cs_bill_customer_sk = customer.c_customer_sk + and d_month_seq between 1189 and 1189 + 11 + intersect + select distinct c_last_name, c_first_name, d_date + from web_sales, date_dim, customer + where web_sales.ws_sold_date_sk = date_dim.d_date_sk + and web_sales.ws_bill_customer_sk = customer.c_customer_sk + and d_month_seq between 1189 and 1189 + 11 +) hot_cust +limit 100; + +-- end query 38 in stream 0 using template query38.tpl +-- start query 39 in stream 0 using template query39.tpl +with inv as +(select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy + ,stdev,mean, case mean when 0 then null else stdev/mean end cov + from(select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy + ,stddev_samp(inv_quantity_on_hand) stdev,avg(inv_quantity_on_hand) mean + from inventory + ,item + ,warehouse + ,date_dim + where inv_item_sk = i_item_sk + and inv_warehouse_sk = w_warehouse_sk + and inv_date_sk = d_date_sk + and d_year =2000 + group by w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy) foo + where case mean when 0 then 0 else stdev/mean end > 1) +select inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean, inv1.cov + ,inv2.w_warehouse_sk,inv2.i_item_sk,inv2.d_moy,inv2.mean, inv2.cov +from inv inv1,inv inv2 +where inv1.i_item_sk = inv2.i_item_sk + and inv1.w_warehouse_sk = inv2.w_warehouse_sk + and inv1.d_moy=1 + and inv2.d_moy=1+1 +order by inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean,inv1.cov + ,inv2.d_moy,inv2.mean, inv2.cov +; +with inv as +(select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy + ,stdev,mean, case mean when 0 then null else stdev/mean end cov + from(select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy + ,stddev_samp(inv_quantity_on_hand) stdev,avg(inv_quantity_on_hand) mean + from inventory + ,item + ,warehouse + ,date_dim + where inv_item_sk = i_item_sk + and inv_warehouse_sk = w_warehouse_sk + and inv_date_sk = d_date_sk + and d_year =2000 + group by w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy) foo + where case mean when 0 then 0 else stdev/mean end > 1) +select inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean, inv1.cov + ,inv2.w_warehouse_sk,inv2.i_item_sk,inv2.d_moy,inv2.mean, inv2.cov +from inv inv1,inv inv2 +where inv1.i_item_sk = inv2.i_item_sk + and inv1.w_warehouse_sk = inv2.w_warehouse_sk + and inv1.d_moy=1 + and inv2.d_moy=1+1 + and inv1.cov > 1.5 +order by inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean,inv1.cov + ,inv2.d_moy,inv2.mean, inv2.cov +; + +-- end query 39 in stream 0 using template query39.tpl +-- start query 40 in stream 0 using template query40.tpl +select + w_state + ,i_item_id + ,sum(case when (cast(d_date as date) < cast ('2001-05-02' as date)) + then cs_sales_price - coalesce(cr_refunded_cash,0) else 0 end) as sales_before + ,sum(case when (cast(d_date as date) >= cast ('2001-05-02' as date)) + then cs_sales_price - coalesce(cr_refunded_cash,0) else 0 end) as sales_after + from + catalog_sales left outer join catalog_returns on + (cs_order_number = cr_order_number + and cs_item_sk = cr_item_sk) + ,warehouse + ,item + ,date_dim + where + i_current_price between 0.99 and 1.49 + and i_item_sk = cs_item_sk + and cs_warehouse_sk = w_warehouse_sk + and cs_sold_date_sk = d_date_sk + and d_date between (cast ('2001-05-02' as date) - interval '30 days') + and (cast ('2001-05-02' as date) + interval '30 days') + group by + w_state,i_item_id + order by w_state,i_item_id +limit 100; + +-- end query 40 in stream 0 using template query40.tpl +-- start query 41 in stream 0 using template query41.tpl +select distinct(i_product_name) + from item i1 + where i_manufact_id between 704 and 704+40 + and (select count(*) as item_cnt + from item + where (i_manufact = i1.i_manufact and + ((i_category = 'Women' and + (i_color = 'forest' or i_color = 'lime') and + (i_units = 'Pallet' or i_units = 'Pound') and + (i_size = 'economy' or i_size = 'small') + ) or + (i_category = 'Women' and + (i_color = 'navy' or i_color = 'slate') and + (i_units = 'Gross' or i_units = 'Bunch') and + (i_size = 'extra large' or i_size = 'petite') + ) or + (i_category = 'Men' and + (i_color = 'powder' or i_color = 'sky') and + (i_units = 'Dozen' or i_units = 'Lb') and + (i_size = 'N/A' or i_size = 'large') + ) or + (i_category = 'Men' and + (i_color = 'maroon' or i_color = 'smoke') and + (i_units = 'Ounce' or i_units = 'Case') and + (i_size = 'economy' or i_size = 'small') + ))) or + (i_manufact = i1.i_manufact and + ((i_category = 'Women' and + (i_color = 'dark' or i_color = 'aquamarine') and + (i_units = 'Ton' or i_units = 'Tbl') and + (i_size = 'economy' or i_size = 'small') + ) or + (i_category = 'Women' and + (i_color = 'frosted' or i_color = 'plum') and + (i_units = 'Dram' or i_units = 'Box') and + (i_size = 'extra large' or i_size = 'petite') + ) or + (i_category = 'Men' and + (i_color = 'papaya' or i_color = 'peach') and + (i_units = 'Bundle' or i_units = 'Carton') and + (i_size = 'N/A' or i_size = 'large') + ) or + (i_category = 'Men' and + (i_color = 'firebrick' or i_color = 'sienna') and + (i_units = 'Cup' or i_units = 'Each') and + (i_size = 'economy' or i_size = 'small') + )))) > 0 + order by i_product_name + limit 100; + +-- end query 41 in stream 0 using template query41.tpl +-- start query 42 in stream 0 using template query42.tpl +select dt.d_year + ,item.i_category_id + ,item.i_category + ,sum(ss_ext_sales_price) + from date_dim dt + ,store_sales + ,item + where dt.d_date_sk = store_sales.ss_sold_date_sk + and store_sales.ss_item_sk = item.i_item_sk + and item.i_manager_id = 1 + and dt.d_moy=11 + and dt.d_year=1998 + group by dt.d_year + ,item.i_category_id + ,item.i_category + order by sum(ss_ext_sales_price) desc,dt.d_year + ,item.i_category_id + ,item.i_category +limit 100 ; + +-- end query 42 in stream 0 using template query42.tpl +-- start query 43 in stream 0 using template query43.tpl +select s_store_name, s_store_id, + sum(case when (d_day_name='Sunday') then ss_sales_price else null end) sun_sales, + sum(case when (d_day_name='Monday') then ss_sales_price else null end) mon_sales, + sum(case when (d_day_name='Tuesday') then ss_sales_price else null end) tue_sales, + sum(case when (d_day_name='Wednesday') then ss_sales_price else null end) wed_sales, + sum(case when (d_day_name='Thursday') then ss_sales_price else null end) thu_sales, + sum(case when (d_day_name='Friday') then ss_sales_price else null end) fri_sales, + sum(case when (d_day_name='Saturday') then ss_sales_price else null end) sat_sales + from date_dim, store_sales, store + where d_date_sk = ss_sold_date_sk and + s_store_sk = ss_store_sk and + s_gmt_offset = -5 and + d_year = 2000 + group by s_store_name, s_store_id + order by s_store_name, s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales + limit 100; + +-- end query 43 in stream 0 using template query43.tpl +-- start query 44 in stream 0 using template query44.tpl +select asceding.rnk, i1.i_product_name best_performing, i2.i_product_name worst_performing +from(select * + from (select item_sk,rank() over (order by rank_col asc) rnk + from (select ss_item_sk item_sk,avg(ss_net_profit) rank_col + from store_sales ss1 + where ss_store_sk = 4 + group by ss_item_sk + having avg(ss_net_profit) > 0.9*(select avg(ss_net_profit) rank_col + from store_sales + where ss_store_sk = 4 + and ss_hdemo_sk is null + group by ss_store_sk))V1)V11 + where rnk < 11) asceding, + (select * + from (select item_sk,rank() over (order by rank_col desc) rnk + from (select ss_item_sk item_sk,avg(ss_net_profit) rank_col + from store_sales ss1 + where ss_store_sk = 4 + group by ss_item_sk + having avg(ss_net_profit) > 0.9*(select avg(ss_net_profit) rank_col + from store_sales + where ss_store_sk = 4 + and ss_hdemo_sk is null + group by ss_store_sk))V2)V21 + where rnk < 11) descending, +item i1, +item i2 +where asceding.rnk = descending.rnk + and i1.i_item_sk=asceding.item_sk + and i2.i_item_sk=descending.item_sk +order by asceding.rnk +limit 100; + +-- end query 44 in stream 0 using template query44.tpl +-- start query 45 in stream 0 using template query45.tpl +select ca_zip, ca_city, sum(ws_sales_price) + from web_sales, customer, customer_address, date_dim, item + where ws_bill_customer_sk = c_customer_sk + and c_current_addr_sk = ca_address_sk + and ws_item_sk = i_item_sk + and ( substr(ca_zip,1,5) in ('85669', '86197','88274','83405','86475', '85392', '85460', '80348', '81792') + or + i_item_id in (select i_item_id + from item + where i_item_sk in (2, 3, 5, 7, 11, 13, 17, 19, 23, 29) + ) + ) + and ws_sold_date_sk = d_date_sk + and d_qoy = 1 and d_year = 2000 + group by ca_zip, ca_city + order by ca_zip, ca_city + limit 100; + +-- end query 45 in stream 0 using template query45.tpl +-- start query 46 in stream 0 using template query46.tpl +select c_last_name + ,c_first_name + ,ca_city + ,bought_city + ,ss_ticket_number + ,amt,profit + from + (select ss_ticket_number + ,ss_customer_sk + ,ca_city bought_city + ,sum(ss_coupon_amt) amt + ,sum(ss_net_profit) profit + from store_sales,date_dim,store,household_demographics,customer_address + where store_sales.ss_sold_date_sk = date_dim.d_date_sk + and store_sales.ss_store_sk = store.s_store_sk + and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk + and store_sales.ss_addr_sk = customer_address.ca_address_sk + and (household_demographics.hd_dep_count = 8 or + household_demographics.hd_vehicle_count= 0) + and date_dim.d_dow in (6,0) + and date_dim.d_year in (2000,2000+1,2000+2) + and store.s_city in ('Midway','Fairview','Fairview','Midway','Fairview') + group by ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city) dn,customer,customer_address current_addr + where ss_customer_sk = c_customer_sk + and customer.c_current_addr_sk = current_addr.ca_address_sk + and current_addr.ca_city <> bought_city + order by c_last_name + ,c_first_name + ,ca_city + ,bought_city + ,ss_ticket_number + limit 100; + +-- end query 46 in stream 0 using template query46.tpl +-- start query 47 in stream 0 using template query47.tpl +with v1 as( + select i_category, i_brand, + s_store_name, s_company_name, + d_year, d_moy, + sum(ss_sales_price) sum_sales, + avg(sum(ss_sales_price)) over + (partition by i_category, i_brand, + s_store_name, s_company_name, d_year) + avg_monthly_sales, + rank() over + (partition by i_category, i_brand, + s_store_name, s_company_name + order by d_year, d_moy) rn + from item, store_sales, date_dim, store + where ss_item_sk = i_item_sk and + ss_sold_date_sk = d_date_sk and + ss_store_sk = s_store_sk and + ( + d_year = 2000 or + ( d_year = 2000-1 and d_moy =12) or + ( d_year = 2000+1 and d_moy =1) + ) + group by i_category, i_brand, + s_store_name, s_company_name, + d_year, d_moy), + v2 as( + select v1.s_store_name, v1.s_company_name + ,v1.d_year + ,v1.avg_monthly_sales + ,v1.sum_sales, v1_lag.sum_sales psum, v1_lead.sum_sales nsum + from v1, v1 v1_lag, v1 v1_lead + where v1.i_category = v1_lag.i_category and + v1.i_category = v1_lead.i_category and + v1.i_brand = v1_lag.i_brand and + v1.i_brand = v1_lead.i_brand and + v1.s_store_name = v1_lag.s_store_name and + v1.s_store_name = v1_lead.s_store_name and + v1.s_company_name = v1_lag.s_company_name and + v1.s_company_name = v1_lead.s_company_name and + v1.rn = v1_lag.rn + 1 and + v1.rn = v1_lead.rn - 1) + select * + from v2 + where d_year = 2000 and + avg_monthly_sales > 0 and + case when avg_monthly_sales > 0 then abs(sum_sales - avg_monthly_sales) / avg_monthly_sales else null end > 0.1 + order by sum_sales - avg_monthly_sales, nsum + limit 100; + +-- end query 47 in stream 0 using template query47.tpl +-- start query 48 in stream 0 using template query48.tpl +select sum (ss_quantity) + from store_sales, store, customer_demographics, customer_address, date_dim + where s_store_sk = ss_store_sk + and ss_sold_date_sk = d_date_sk and d_year = 2001 + and + ( + ( + cd_demo_sk = ss_cdemo_sk + and + cd_marital_status = 'S' + and + cd_education_status = 'Secondary' + and + ss_sales_price between 100.00 and 150.00 + ) + or + ( + cd_demo_sk = ss_cdemo_sk + and + cd_marital_status = 'M' + and + cd_education_status = '2 yr Degree' + and + ss_sales_price between 50.00 and 100.00 + ) + or + ( + cd_demo_sk = ss_cdemo_sk + and + cd_marital_status = 'D' + and + cd_education_status = 'Advanced Degree' + and + ss_sales_price between 150.00 and 200.00 + ) + ) + and + ( + ( + ss_addr_sk = ca_address_sk + and + ca_country = 'United States' + and + ca_state in ('ND', 'NY', 'SD') + and ss_net_profit between 0 and 2000 + ) + or + (ss_addr_sk = ca_address_sk + and + ca_country = 'United States' + and + ca_state in ('MD', 'GA', 'KS') + and ss_net_profit between 150 and 3000 + ) + or + (ss_addr_sk = ca_address_sk + and + ca_country = 'United States' + and + ca_state in ('CO', 'MN', 'NC') + and ss_net_profit between 50 and 25000 + ) + ) +; + +-- end query 48 in stream 0 using template query48.tpl +-- start query 49 in stream 0 using template query49.tpl +select channel, item, return_ratio, return_rank, currency_rank from + (select + 'web' as channel + ,web.item + ,web.return_ratio + ,web.return_rank + ,web.currency_rank + from ( + select + item + ,return_ratio + ,currency_ratio + ,rank() over (order by return_ratio) as return_rank + ,rank() over (order by currency_ratio) as currency_rank + from + ( select ws.ws_item_sk as item + ,(cast(sum(coalesce(wr.wr_return_quantity,0)) as decimal(15,4))/ + cast(sum(coalesce(ws.ws_quantity,0)) as decimal(15,4) )) as return_ratio + ,(cast(sum(coalesce(wr.wr_return_amt,0)) as decimal(15,4))/ + cast(sum(coalesce(ws.ws_net_paid,0)) as decimal(15,4) )) as currency_ratio + from + web_sales ws left outer join web_returns wr + on (ws.ws_order_number = wr.wr_order_number and + ws.ws_item_sk = wr.wr_item_sk) + ,date_dim + where + wr.wr_return_amt > 10000 + and ws.ws_net_profit > 1 + and ws.ws_net_paid > 0 + and ws.ws_quantity > 0 + and ws_sold_date_sk = d_date_sk + and d_year = 1998 + and d_moy = 11 + group by ws.ws_item_sk + ) in_web + ) as web + where + ( + web.return_rank <= 10 + or + web.currency_rank <= 10 + ) + union + select + 'catalog' as channel + ,catalog.item + ,catalog.return_ratio + ,catalog.return_rank + ,catalog.currency_rank + from ( + select + item + ,return_ratio + ,currency_ratio + ,rank() over (order by return_ratio) as return_rank + ,rank() over (order by currency_ratio) as currency_rank + from + ( select + cs.cs_item_sk as item + ,(cast(sum(coalesce(cr.cr_return_quantity,0)) as decimal(15,4))/ + cast(sum(coalesce(cs.cs_quantity,0)) as decimal(15,4) )) as return_ratio + ,(cast(sum(coalesce(cr.cr_return_amount,0)) as decimal(15,4))/ + cast(sum(coalesce(cs.cs_net_paid,0)) as decimal(15,4) )) as currency_ratio + from + catalog_sales cs left outer join catalog_returns cr + on (cs.cs_order_number = cr.cr_order_number and + cs.cs_item_sk = cr.cr_item_sk) + ,date_dim + where + cr.cr_return_amount > 10000 + and cs.cs_net_profit > 1 + and cs.cs_net_paid > 0 + and cs.cs_quantity > 0 + and cs_sold_date_sk = d_date_sk + and d_year = 1998 + and d_moy = 11 + group by cs.cs_item_sk + ) in_cat + ) as catalog + where + ( + catalog.return_rank <= 10 + or + catalog.currency_rank <=10 + ) + union + select + 'store' as channel + ,store.item + ,store.return_ratio + ,store.return_rank + ,store.currency_rank + from ( + select + item + ,return_ratio + ,currency_ratio + ,rank() over (order by return_ratio) as return_rank + ,rank() over (order by currency_ratio) as currency_rank + from + ( select sts.ss_item_sk as item + ,(cast(sum(coalesce(sr.sr_return_quantity,0)) as decimal(15,4))/cast(sum(coalesce(sts.ss_quantity,0)) as decimal(15,4) )) as return_ratio + ,(cast(sum(coalesce(sr.sr_return_amt,0)) as decimal(15,4))/cast(sum(coalesce(sts.ss_net_paid,0)) as decimal(15,4) )) as currency_ratio + from + store_sales sts left outer join store_returns sr + on (sts.ss_ticket_number = sr.sr_ticket_number and sts.ss_item_sk = sr.sr_item_sk) + ,date_dim + where + sr.sr_return_amt > 10000 + and sts.ss_net_profit > 1 + and sts.ss_net_paid > 0 + and sts.ss_quantity > 0 + and ss_sold_date_sk = d_date_sk + and d_year = 1998 + and d_moy = 11 + group by sts.ss_item_sk + ) in_store + ) as store + where ( + store.return_rank <= 10 + or + store.currency_rank <= 10 + ) + )as tab + order by 1,4,5,2 + limit 100; + +-- end query 49 in stream 0 using template query49.tpl +-- start query 50 in stream 0 using template query50.tpl +select + s_store_name + ,s_company_id + ,s_street_number + ,s_street_name + ,s_street_type + ,s_suite_number + ,s_city + ,s_county + ,s_state + ,s_zip + ,sum(case when (sr_returned_date_sk - ss_sold_date_sk <= 30 ) then 1 else 0 end) as "30 days" + ,sum(case when (sr_returned_date_sk - ss_sold_date_sk > 30) and + (sr_returned_date_sk - ss_sold_date_sk <= 60) then 1 else 0 end ) as "31-60 days" + ,sum(case when (sr_returned_date_sk - ss_sold_date_sk > 60) and + (sr_returned_date_sk - ss_sold_date_sk <= 90) then 1 else 0 end) as "61-90 days" + ,sum(case when (sr_returned_date_sk - ss_sold_date_sk > 90) and + (sr_returned_date_sk - ss_sold_date_sk <= 120) then 1 else 0 end) as "91-120 days" + ,sum(case when (sr_returned_date_sk - ss_sold_date_sk > 120) then 1 else 0 end) as ">120 days" +from + store_sales + ,store_returns + ,store + ,date_dim d1 + ,date_dim d2 +where + d2.d_year = 2001 +and d2.d_moy = 8 +and ss_ticket_number = sr_ticket_number +and ss_item_sk = sr_item_sk +and ss_sold_date_sk = d1.d_date_sk +and sr_returned_date_sk = d2.d_date_sk +and ss_customer_sk = sr_customer_sk +and ss_store_sk = s_store_sk +group by + s_store_name + ,s_company_id + ,s_street_number + ,s_street_name + ,s_street_type + ,s_suite_number + ,s_city + ,s_county + ,s_state + ,s_zip +order by s_store_name + ,s_company_id + ,s_street_number + ,s_street_name + ,s_street_type + ,s_suite_number + ,s_city + ,s_county + ,s_state + ,s_zip +limit 100; + +-- end query 50 in stream 0 using template query50.tpl +-- start query 51 in stream 0 using template query51.tpl +WITH web_v1 as ( +select + ws_item_sk item_sk, d_date, + sum(sum(ws_sales_price)) + over (partition by ws_item_sk order by d_date rows between unbounded preceding and current row) cume_sales +from web_sales + ,date_dim +where ws_sold_date_sk=d_date_sk + and d_month_seq between 1212 and 1212+11 + and ws_item_sk is not NULL +group by ws_item_sk, d_date), +store_v1 as ( +select + ss_item_sk item_sk, d_date, + sum(sum(ss_sales_price)) + over (partition by ss_item_sk order by d_date rows between unbounded preceding and current row) cume_sales +from store_sales + ,date_dim +where ss_sold_date_sk=d_date_sk + and d_month_seq between 1212 and 1212+11 + and ss_item_sk is not NULL +group by ss_item_sk, d_date) + select * +from (select item_sk + ,d_date + ,web_sales + ,store_sales + ,max(web_sales) + over (partition by item_sk order by d_date rows between unbounded preceding and current row) web_cumulative + ,max(store_sales) + over (partition by item_sk order by d_date rows between unbounded preceding and current row) store_cumulative + from (select case when web.item_sk is not null then web.item_sk else store.item_sk end item_sk + ,case when web.d_date is not null then web.d_date else store.d_date end d_date + ,web.cume_sales web_sales + ,store.cume_sales store_sales + from web_v1 web full outer join store_v1 store on (web.item_sk = store.item_sk + and web.d_date = store.d_date) + )x )y +where web_cumulative > store_cumulative +order by item_sk + ,d_date +limit 100; + +-- end query 51 in stream 0 using template query51.tpl +-- start query 52 in stream 0 using template query52.tpl +select dt.d_year + ,item.i_brand_id brand_id + ,item.i_brand brand + ,sum(ss_ext_sales_price) ext_price + from date_dim dt + ,store_sales + ,item + where dt.d_date_sk = store_sales.ss_sold_date_sk + and store_sales.ss_item_sk = item.i_item_sk + and item.i_manager_id = 1 + and dt.d_moy=12 + and dt.d_year=2000 + group by dt.d_year + ,item.i_brand + ,item.i_brand_id + order by dt.d_year + ,ext_price desc + ,brand_id +limit 100 ; + +-- end query 52 in stream 0 using template query52.tpl +-- start query 53 in stream 0 using template query53.tpl +select * from +(select i_manufact_id, +sum(ss_sales_price) sum_sales, +avg(sum(ss_sales_price)) over (partition by i_manufact_id) avg_quarterly_sales +from item, store_sales, date_dim, store +where ss_item_sk = i_item_sk and +ss_sold_date_sk = d_date_sk and +ss_store_sk = s_store_sk and +d_month_seq in (1186,1186+1,1186+2,1186+3,1186+4,1186+5,1186+6,1186+7,1186+8,1186+9,1186+10,1186+11) and +((i_category in ('Books','Children','Electronics') and +i_class in ('personal','portable','reference','self-help') and +i_brand in ('scholaramalgamalg #14','scholaramalgamalg #7', + 'exportiunivamalg #9','scholaramalgamalg #9')) +or(i_category in ('Women','Music','Men') and +i_class in ('accessories','classical','fragrances','pants') and +i_brand in ('amalgimporto #1','edu packscholar #1','exportiimporto #1', + 'importoamalg #1'))) +group by i_manufact_id, d_qoy ) tmp1 +where case when avg_quarterly_sales > 0 + then abs (sum_sales - avg_quarterly_sales)/ avg_quarterly_sales + else null end > 0.1 +order by avg_quarterly_sales, + sum_sales, + i_manufact_id +limit 100; + +-- end query 53 in stream 0 using template query53.tpl +-- start query 54 in stream 0 using template query54.tpl +with my_customers as ( + select distinct c_customer_sk + , c_current_addr_sk + from + ( select cs_sold_date_sk sold_date_sk, + cs_bill_customer_sk customer_sk, + cs_item_sk item_sk + from catalog_sales + union all + select ws_sold_date_sk sold_date_sk, + ws_bill_customer_sk customer_sk, + ws_item_sk item_sk + from web_sales + ) cs_or_ws_sales, + item, + date_dim, + customer + where sold_date_sk = d_date_sk + and item_sk = i_item_sk + and i_category = 'Music' + and i_class = 'country' + and c_customer_sk = cs_or_ws_sales.customer_sk + and d_moy = 1 + and d_year = 1999 + ) + , my_revenue as ( + select c_customer_sk, + sum(ss_ext_sales_price) as revenue + from my_customers, + store_sales, + customer_address, + store, + date_dim + where c_current_addr_sk = ca_address_sk + and ca_county = s_county + and ca_state = s_state + and ss_sold_date_sk = d_date_sk + and c_customer_sk = ss_customer_sk + and d_month_seq between (select distinct d_month_seq+1 + from date_dim where d_year = 1999 and d_moy = 1) + and (select distinct d_month_seq+3 + from date_dim where d_year = 1999 and d_moy = 1) + group by c_customer_sk + ) + , segments as + (select cast((revenue/50) as int) as segment + from my_revenue + ) + select segment, count(*) as num_customers, segment*50 as segment_base + from segments + group by segment + order by segment, num_customers + limit 100; + +-- end query 54 in stream 0 using template query54.tpl +-- start query 55 in stream 0 using template query55.tpl +select i_brand_id brand_id, i_brand brand, + sum(ss_ext_sales_price) ext_price + from date_dim, store_sales, item + where d_date_sk = ss_sold_date_sk + and ss_item_sk = i_item_sk + and i_manager_id=52 + and d_moy=11 + and d_year=2000 + group by i_brand, i_brand_id + order by ext_price desc, i_brand_id +limit 100 ; + +-- end query 55 in stream 0 using template query55.tpl +-- start query 56 in stream 0 using template query56.tpl +with ss as ( + select i_item_id,sum(ss_ext_sales_price) total_sales + from + store_sales, + date_dim, + customer_address, + item + where i_item_id in (select + i_item_id +from item +where i_color in ('powder','orchid','pink')) + and ss_item_sk = i_item_sk + and ss_sold_date_sk = d_date_sk + and d_year = 2000 + and d_moy = 3 + and ss_addr_sk = ca_address_sk + and ca_gmt_offset = -6 + group by i_item_id), + cs as ( + select i_item_id,sum(cs_ext_sales_price) total_sales + from + catalog_sales, + date_dim, + customer_address, + item + where + i_item_id in (select + i_item_id +from item +where i_color in ('powder','orchid','pink')) + and cs_item_sk = i_item_sk + and cs_sold_date_sk = d_date_sk + and d_year = 2000 + and d_moy = 3 + and cs_bill_addr_sk = ca_address_sk + and ca_gmt_offset = -6 + group by i_item_id), + ws as ( + select i_item_id,sum(ws_ext_sales_price) total_sales + from + web_sales, + date_dim, + customer_address, + item + where + i_item_id in (select + i_item_id +from item +where i_color in ('powder','orchid','pink')) + and ws_item_sk = i_item_sk + and ws_sold_date_sk = d_date_sk + and d_year = 2000 + and d_moy = 3 + and ws_bill_addr_sk = ca_address_sk + and ca_gmt_offset = -6 + group by i_item_id) + select i_item_id ,sum(total_sales) total_sales + from (select * from ss + union all + select * from cs + union all + select * from ws) tmp1 + group by i_item_id + order by total_sales, + i_item_id + limit 100; + +-- end query 56 in stream 0 using template query56.tpl +-- start query 57 in stream 0 using template query57.tpl +with v1 as( + select i_category, i_brand, + cc_name, + d_year, d_moy, + sum(cs_sales_price) sum_sales, + avg(sum(cs_sales_price)) over + (partition by i_category, i_brand, + cc_name, d_year) + avg_monthly_sales, + rank() over + (partition by i_category, i_brand, + cc_name + order by d_year, d_moy) rn + from item, catalog_sales, date_dim, call_center + where cs_item_sk = i_item_sk and + cs_sold_date_sk = d_date_sk and + cc_call_center_sk= cs_call_center_sk and + ( + d_year = 2001 or + ( d_year = 2001-1 and d_moy =12) or + ( d_year = 2001+1 and d_moy =1) + ) + group by i_category, i_brand, + cc_name , d_year, d_moy), + v2 as( + select v1.i_category, v1.i_brand, v1.cc_name + ,v1.d_year + ,v1.avg_monthly_sales + ,v1.sum_sales, v1_lag.sum_sales psum, v1_lead.sum_sales nsum + from v1, v1 v1_lag, v1 v1_lead + where v1.i_category = v1_lag.i_category and + v1.i_category = v1_lead.i_category and + v1.i_brand = v1_lag.i_brand and + v1.i_brand = v1_lead.i_brand and + v1. cc_name = v1_lag. cc_name and + v1. cc_name = v1_lead. cc_name and + v1.rn = v1_lag.rn + 1 and + v1.rn = v1_lead.rn - 1) + select * + from v2 + where d_year = 2001 and + avg_monthly_sales > 0 and + case when avg_monthly_sales > 0 then abs(sum_sales - avg_monthly_sales) / avg_monthly_sales else null end > 0.1 + order by sum_sales - avg_monthly_sales, avg_monthly_sales + limit 100; + +-- end query 57 in stream 0 using template query57.tpl +-- start query 58 in stream 0 using template query58.tpl +with ss_items as + (select i_item_id item_id + ,sum(ss_ext_sales_price) ss_item_rev + from store_sales + ,item + ,date_dim + where ss_item_sk = i_item_sk + and d_date in (select d_date + from date_dim + where d_week_seq = (select d_week_seq + from date_dim + where d_date = '2001-06-16')) + and ss_sold_date_sk = d_date_sk + group by i_item_id), + cs_items as + (select i_item_id item_id + ,sum(cs_ext_sales_price) cs_item_rev + from catalog_sales + ,item + ,date_dim + where cs_item_sk = i_item_sk + and d_date in (select d_date + from date_dim + where d_week_seq = (select d_week_seq + from date_dim + where d_date = '2001-06-16')) + and cs_sold_date_sk = d_date_sk + group by i_item_id), + ws_items as + (select i_item_id item_id + ,sum(ws_ext_sales_price) ws_item_rev + from web_sales + ,item + ,date_dim + where ws_item_sk = i_item_sk + and d_date in (select d_date + from date_dim + where d_week_seq =(select d_week_seq + from date_dim + where d_date = '2001-06-16')) + and ws_sold_date_sk = d_date_sk + group by i_item_id) + select ss_items.item_id + ,ss_item_rev + ,ss_item_rev/((ss_item_rev+cs_item_rev+ws_item_rev)/3) * 100 ss_dev + ,cs_item_rev + ,cs_item_rev/((ss_item_rev+cs_item_rev+ws_item_rev)/3) * 100 cs_dev + ,ws_item_rev + ,ws_item_rev/((ss_item_rev+cs_item_rev+ws_item_rev)/3) * 100 ws_dev + ,(ss_item_rev+cs_item_rev+ws_item_rev)/3 average + from ss_items,cs_items,ws_items + where ss_items.item_id=cs_items.item_id + and ss_items.item_id=ws_items.item_id + and ss_item_rev between 0.9 * cs_item_rev and 1.1 * cs_item_rev + and ss_item_rev between 0.9 * ws_item_rev and 1.1 * ws_item_rev + and cs_item_rev between 0.9 * ss_item_rev and 1.1 * ss_item_rev + and cs_item_rev between 0.9 * ws_item_rev and 1.1 * ws_item_rev + and ws_item_rev between 0.9 * ss_item_rev and 1.1 * ss_item_rev + and ws_item_rev between 0.9 * cs_item_rev and 1.1 * cs_item_rev + order by item_id + ,ss_item_rev + limit 100; + +-- end query 58 in stream 0 using template query58.tpl +-- start query 59 in stream 0 using template query59.tpl +with wss as + (select d_week_seq, + ss_store_sk, + sum(case when (d_day_name='Sunday') then ss_sales_price else null end) sun_sales, + sum(case when (d_day_name='Monday') then ss_sales_price else null end) mon_sales, + sum(case when (d_day_name='Tuesday') then ss_sales_price else null end) tue_sales, + sum(case when (d_day_name='Wednesday') then ss_sales_price else null end) wed_sales, + sum(case when (d_day_name='Thursday') then ss_sales_price else null end) thu_sales, + sum(case when (d_day_name='Friday') then ss_sales_price else null end) fri_sales, + sum(case when (d_day_name='Saturday') then ss_sales_price else null end) sat_sales + from store_sales,date_dim + where d_date_sk = ss_sold_date_sk + group by d_week_seq,ss_store_sk + ) + select s_store_name1,s_store_id1,d_week_seq1 + ,sun_sales1/sun_sales2,mon_sales1/mon_sales2 + ,tue_sales1/tue_sales2,wed_sales1/wed_sales2,thu_sales1/thu_sales2 + ,fri_sales1/fri_sales2,sat_sales1/sat_sales2 + from + (select s_store_name s_store_name1,wss.d_week_seq d_week_seq1 + ,s_store_id s_store_id1,sun_sales sun_sales1 + ,mon_sales mon_sales1,tue_sales tue_sales1 + ,wed_sales wed_sales1,thu_sales thu_sales1 + ,fri_sales fri_sales1,sat_sales sat_sales1 + from wss,store,date_dim d + where d.d_week_seq = wss.d_week_seq and + ss_store_sk = s_store_sk and + d_month_seq between 1195 and 1195 + 11) y, + (select s_store_name s_store_name2,wss.d_week_seq d_week_seq2 + ,s_store_id s_store_id2,sun_sales sun_sales2 + ,mon_sales mon_sales2,tue_sales tue_sales2 + ,wed_sales wed_sales2,thu_sales thu_sales2 + ,fri_sales fri_sales2,sat_sales sat_sales2 + from wss,store,date_dim d + where d.d_week_seq = wss.d_week_seq and + ss_store_sk = s_store_sk and + d_month_seq between 1195+ 12 and 1195 + 23) x + where s_store_id1=s_store_id2 + and d_week_seq1=d_week_seq2-52 + order by s_store_name1,s_store_id1,d_week_seq1 +limit 100; + +-- end query 59 in stream 0 using template query59.tpl +-- start query 60 in stream 0 using template query60.tpl +with ss as ( + select + i_item_id,sum(ss_ext_sales_price) total_sales + from + store_sales, + date_dim, + customer_address, + item + where + i_item_id in (select + i_item_id +from + item +where i_category in ('Jewelry')) + and ss_item_sk = i_item_sk + and ss_sold_date_sk = d_date_sk + and d_year = 2000 + and d_moy = 10 + and ss_addr_sk = ca_address_sk + and ca_gmt_offset = -5 + group by i_item_id), + cs as ( + select + i_item_id,sum(cs_ext_sales_price) total_sales + from + catalog_sales, + date_dim, + customer_address, + item + where + i_item_id in (select + i_item_id +from + item +where i_category in ('Jewelry')) + and cs_item_sk = i_item_sk + and cs_sold_date_sk = d_date_sk + and d_year = 2000 + and d_moy = 10 + and cs_bill_addr_sk = ca_address_sk + and ca_gmt_offset = -5 + group by i_item_id), + ws as ( + select + i_item_id,sum(ws_ext_sales_price) total_sales + from + web_sales, + date_dim, + customer_address, + item + where + i_item_id in (select + i_item_id +from + item +where i_category in ('Jewelry')) + and ws_item_sk = i_item_sk + and ws_sold_date_sk = d_date_sk + and d_year = 2000 + and d_moy = 10 + and ws_bill_addr_sk = ca_address_sk + and ca_gmt_offset = -5 + group by i_item_id) + select + i_item_id +,sum(total_sales) total_sales + from (select * from ss + union all + select * from cs + union all + select * from ws) tmp1 + group by i_item_id + order by i_item_id + ,total_sales + limit 100; + +-- end query 60 in stream 0 using template query60.tpl +-- start query 61 in stream 0 using template query61.tpl +select promotions,total,cast(promotions as decimal(15,4))/cast(total as decimal(15,4))*100 +from + (select sum(ss_ext_sales_price) promotions + from store_sales + ,store + ,promotion + ,date_dim + ,customer + ,customer_address + ,item + where ss_sold_date_sk = d_date_sk + and ss_store_sk = s_store_sk + and ss_promo_sk = p_promo_sk + and ss_customer_sk= c_customer_sk + and ca_address_sk = c_current_addr_sk + and ss_item_sk = i_item_sk + and ca_gmt_offset = -7 + and i_category = 'Home' + and (p_channel_dmail = 'Y' or p_channel_email = 'Y' or p_channel_tv = 'Y') + and s_gmt_offset = -7 + and d_year = 2000 + and d_moy = 12) promotional_sales, + (select sum(ss_ext_sales_price) total + from store_sales + ,store + ,date_dim + ,customer + ,customer_address + ,item + where ss_sold_date_sk = d_date_sk + and ss_store_sk = s_store_sk + and ss_customer_sk= c_customer_sk + and ca_address_sk = c_current_addr_sk + and ss_item_sk = i_item_sk + and ca_gmt_offset = -7 + and i_category = 'Home' + and s_gmt_offset = -7 + and d_year = 2000 + and d_moy = 12) all_sales +order by promotions, total +limit 100; + +-- end query 61 in stream 0 using template query61.tpl +-- start query 62 in stream 0 using template query62.tpl +select + substr(w_warehouse_name,1,20) + ,sm_type + ,web_name + ,sum(case when (ws_ship_date_sk - ws_sold_date_sk <= 30 ) then 1 else 0 end) as "30 days" + ,sum(case when (ws_ship_date_sk - ws_sold_date_sk > 30) and + (ws_ship_date_sk - ws_sold_date_sk <= 60) then 1 else 0 end ) as "31-60 days" + ,sum(case when (ws_ship_date_sk - ws_sold_date_sk > 60) and + (ws_ship_date_sk - ws_sold_date_sk <= 90) then 1 else 0 end) as "61-90 days" + ,sum(case when (ws_ship_date_sk - ws_sold_date_sk > 90) and + (ws_ship_date_sk - ws_sold_date_sk <= 120) then 1 else 0 end) as "91-120 days" + ,sum(case when (ws_ship_date_sk - ws_sold_date_sk > 120) then 1 else 0 end) as ">120 days" +from + web_sales + ,warehouse + ,ship_mode + ,web_site + ,date_dim +where + d_month_seq between 1223 and 1223 + 11 +and ws_ship_date_sk = d_date_sk +and ws_warehouse_sk = w_warehouse_sk +and ws_ship_mode_sk = sm_ship_mode_sk +and ws_web_site_sk = web_site_sk +group by + substr(w_warehouse_name,1,20) + ,sm_type + ,web_name +order by substr(w_warehouse_name,1,20) + ,sm_type + ,web_name +limit 100; + +-- end query 62 in stream 0 using template query62.tpl +-- start query 63 in stream 0 using template query63.tpl +select * +from (select i_manager_id + ,sum(ss_sales_price) sum_sales + ,avg(sum(ss_sales_price)) over (partition by i_manager_id) avg_monthly_sales + from item + ,store_sales + ,date_dim + ,store + where ss_item_sk = i_item_sk + and ss_sold_date_sk = d_date_sk + and ss_store_sk = s_store_sk + and d_month_seq in (1222,1222+1,1222+2,1222+3,1222+4,1222+5,1222+6,1222+7,1222+8,1222+9,1222+10,1222+11) + and (( i_category in ('Books','Children','Electronics') + and i_class in ('personal','portable','reference','self-help') + and i_brand in ('scholaramalgamalg #14','scholaramalgamalg #7', + 'exportiunivamalg #9','scholaramalgamalg #9')) + or( i_category in ('Women','Music','Men') + and i_class in ('accessories','classical','fragrances','pants') + and i_brand in ('amalgimporto #1','edu packscholar #1','exportiimporto #1', + 'importoamalg #1'))) +group by i_manager_id, d_moy) tmp1 +where case when avg_monthly_sales > 0 then abs (sum_sales - avg_monthly_sales) / avg_monthly_sales else null end > 0.1 +order by i_manager_id + ,avg_monthly_sales + ,sum_sales +limit 100; + +-- end query 63 in stream 0 using template query63.tpl +-- start query 64 in stream 0 using template query64.tpl +with cs_ui as + (select cs_item_sk + ,sum(cs_ext_list_price) as sale,sum(cr_refunded_cash+cr_reversed_charge+cr_store_credit) as refund + from catalog_sales + ,catalog_returns + where cs_item_sk = cr_item_sk + and cs_order_number = cr_order_number + group by cs_item_sk + having sum(cs_ext_list_price)>2*sum(cr_refunded_cash+cr_reversed_charge+cr_store_credit)), +cross_sales as + (select i_product_name product_name + ,i_item_sk item_sk + ,s_store_name store_name + ,s_zip store_zip + ,ad1.ca_street_number b_street_number + ,ad1.ca_street_name b_street_name + ,ad1.ca_city b_city + ,ad1.ca_zip b_zip + ,ad2.ca_street_number c_street_number + ,ad2.ca_street_name c_street_name + ,ad2.ca_city c_city + ,ad2.ca_zip c_zip + ,d1.d_year as syear + ,d2.d_year as fsyear + ,d3.d_year s2year + ,count(*) cnt + ,sum(ss_wholesale_cost) s1 + ,sum(ss_list_price) s2 + ,sum(ss_coupon_amt) s3 + FROM store_sales + ,store_returns + ,cs_ui + ,date_dim d1 + ,date_dim d2 + ,date_dim d3 + ,store + ,customer + ,customer_demographics cd1 + ,customer_demographics cd2 + ,promotion + ,household_demographics hd1 + ,household_demographics hd2 + ,customer_address ad1 + ,customer_address ad2 + ,income_band ib1 + ,income_band ib2 + ,item + WHERE ss_store_sk = s_store_sk AND + ss_sold_date_sk = d1.d_date_sk AND + ss_customer_sk = c_customer_sk AND + ss_cdemo_sk= cd1.cd_demo_sk AND + ss_hdemo_sk = hd1.hd_demo_sk AND + ss_addr_sk = ad1.ca_address_sk and + ss_item_sk = i_item_sk and + ss_item_sk = sr_item_sk and + ss_ticket_number = sr_ticket_number and + ss_item_sk = cs_ui.cs_item_sk and + c_current_cdemo_sk = cd2.cd_demo_sk AND + c_current_hdemo_sk = hd2.hd_demo_sk AND + c_current_addr_sk = ad2.ca_address_sk and + c_first_sales_date_sk = d2.d_date_sk and + c_first_shipto_date_sk = d3.d_date_sk and + ss_promo_sk = p_promo_sk and + hd1.hd_income_band_sk = ib1.ib_income_band_sk and + hd2.hd_income_band_sk = ib2.ib_income_band_sk and + cd1.cd_marital_status <> cd2.cd_marital_status and + i_color in ('orange','lace','lawn','misty','blush','pink') and + i_current_price between 48 and 48 + 10 and + i_current_price between 48 + 1 and 48 + 15 +group by i_product_name + ,i_item_sk + ,s_store_name + ,s_zip + ,ad1.ca_street_number + ,ad1.ca_street_name + ,ad1.ca_city + ,ad1.ca_zip + ,ad2.ca_street_number + ,ad2.ca_street_name + ,ad2.ca_city + ,ad2.ca_zip + ,d1.d_year + ,d2.d_year + ,d3.d_year +) +select cs1.product_name + ,cs1.store_name + ,cs1.store_zip + ,cs1.b_street_number + ,cs1.b_street_name + ,cs1.b_city + ,cs1.b_zip + ,cs1.c_street_number + ,cs1.c_street_name + ,cs1.c_city + ,cs1.c_zip + ,cs1.syear + ,cs1.cnt + ,cs1.s1 as s11 + ,cs1.s2 as s21 + ,cs1.s3 as s31 + ,cs2.s1 as s12 + ,cs2.s2 as s22 + ,cs2.s3 as s32 + ,cs2.syear + ,cs2.cnt +from cross_sales cs1,cross_sales cs2 +where cs1.item_sk=cs2.item_sk and + cs1.syear = 1999 and + cs2.syear = 1999 + 1 and + cs2.cnt <= cs1.cnt and + cs1.store_name = cs2.store_name and + cs1.store_zip = cs2.store_zip +order by cs1.product_name + ,cs1.store_name + ,cs2.cnt + ,cs1.s1 + ,cs2.s1; + +-- end query 64 in stream 0 using template query64.tpl +-- start query 65 in stream 0 using template query65.tpl +select + s_store_name, + i_item_desc, + sc.revenue, + i_current_price, + i_wholesale_cost, + i_brand + from store, item, + (select ss_store_sk, avg(revenue) as ave + from + (select ss_store_sk, ss_item_sk, + sum(ss_sales_price) as revenue + from store_sales, date_dim + where ss_sold_date_sk = d_date_sk and d_month_seq between 1176 and 1176+11 + group by ss_store_sk, ss_item_sk) sa + group by ss_store_sk) sb, + (select ss_store_sk, ss_item_sk, sum(ss_sales_price) as revenue + from store_sales, date_dim + where ss_sold_date_sk = d_date_sk and d_month_seq between 1176 and 1176+11 + group by ss_store_sk, ss_item_sk) sc + where sb.ss_store_sk = sc.ss_store_sk and + sc.revenue <= 0.1 * sb.ave and + s_store_sk = sc.ss_store_sk and + i_item_sk = sc.ss_item_sk + order by s_store_name, i_item_desc +limit 100; + +-- end query 65 in stream 0 using template query65.tpl +-- start query 66 in stream 0 using template query66.tpl +select + w_warehouse_name + ,w_warehouse_sq_ft + ,w_city + ,w_county + ,w_state + ,w_country + ,ship_carriers + ,year + ,sum(jan_sales) as jan_sales + ,sum(feb_sales) as feb_sales + ,sum(mar_sales) as mar_sales + ,sum(apr_sales) as apr_sales + ,sum(may_sales) as may_sales + ,sum(jun_sales) as jun_sales + ,sum(jul_sales) as jul_sales + ,sum(aug_sales) as aug_sales + ,sum(sep_sales) as sep_sales + ,sum(oct_sales) as oct_sales + ,sum(nov_sales) as nov_sales + ,sum(dec_sales) as dec_sales + ,sum(jan_sales/w_warehouse_sq_ft) as jan_sales_per_sq_foot + ,sum(feb_sales/w_warehouse_sq_ft) as feb_sales_per_sq_foot + ,sum(mar_sales/w_warehouse_sq_ft) as mar_sales_per_sq_foot + ,sum(apr_sales/w_warehouse_sq_ft) as apr_sales_per_sq_foot + ,sum(may_sales/w_warehouse_sq_ft) as may_sales_per_sq_foot + ,sum(jun_sales/w_warehouse_sq_ft) as jun_sales_per_sq_foot + ,sum(jul_sales/w_warehouse_sq_ft) as jul_sales_per_sq_foot + ,sum(aug_sales/w_warehouse_sq_ft) as aug_sales_per_sq_foot + ,sum(sep_sales/w_warehouse_sq_ft) as sep_sales_per_sq_foot + ,sum(oct_sales/w_warehouse_sq_ft) as oct_sales_per_sq_foot + ,sum(nov_sales/w_warehouse_sq_ft) as nov_sales_per_sq_foot + ,sum(dec_sales/w_warehouse_sq_ft) as dec_sales_per_sq_foot + ,sum(jan_net) as jan_net + ,sum(feb_net) as feb_net + ,sum(mar_net) as mar_net + ,sum(apr_net) as apr_net + ,sum(may_net) as may_net + ,sum(jun_net) as jun_net + ,sum(jul_net) as jul_net + ,sum(aug_net) as aug_net + ,sum(sep_net) as sep_net + ,sum(oct_net) as oct_net + ,sum(nov_net) as nov_net + ,sum(dec_net) as dec_net + from ( + select + w_warehouse_name + ,w_warehouse_sq_ft + ,w_city + ,w_county + ,w_state + ,w_country + ,'ORIENTAL' || ',' || 'BOXBUNDLES' as ship_carriers + ,d_year as year + ,sum(case when d_moy = 1 + then ws_ext_sales_price* ws_quantity else 0 end) as jan_sales + ,sum(case when d_moy = 2 + then ws_ext_sales_price* ws_quantity else 0 end) as feb_sales + ,sum(case when d_moy = 3 + then ws_ext_sales_price* ws_quantity else 0 end) as mar_sales + ,sum(case when d_moy = 4 + then ws_ext_sales_price* ws_quantity else 0 end) as apr_sales + ,sum(case when d_moy = 5 + then ws_ext_sales_price* ws_quantity else 0 end) as may_sales + ,sum(case when d_moy = 6 + then ws_ext_sales_price* ws_quantity else 0 end) as jun_sales + ,sum(case when d_moy = 7 + then ws_ext_sales_price* ws_quantity else 0 end) as jul_sales + ,sum(case when d_moy = 8 + then ws_ext_sales_price* ws_quantity else 0 end) as aug_sales + ,sum(case when d_moy = 9 + then ws_ext_sales_price* ws_quantity else 0 end) as sep_sales + ,sum(case when d_moy = 10 + then ws_ext_sales_price* ws_quantity else 0 end) as oct_sales + ,sum(case when d_moy = 11 + then ws_ext_sales_price* ws_quantity else 0 end) as nov_sales + ,sum(case when d_moy = 12 + then ws_ext_sales_price* ws_quantity else 0 end) as dec_sales + ,sum(case when d_moy = 1 + then ws_net_paid_inc_ship * ws_quantity else 0 end) as jan_net + ,sum(case when d_moy = 2 + then ws_net_paid_inc_ship * ws_quantity else 0 end) as feb_net + ,sum(case when d_moy = 3 + then ws_net_paid_inc_ship * ws_quantity else 0 end) as mar_net + ,sum(case when d_moy = 4 + then ws_net_paid_inc_ship * ws_quantity else 0 end) as apr_net + ,sum(case when d_moy = 5 + then ws_net_paid_inc_ship * ws_quantity else 0 end) as may_net + ,sum(case when d_moy = 6 + then ws_net_paid_inc_ship * ws_quantity else 0 end) as jun_net + ,sum(case when d_moy = 7 + then ws_net_paid_inc_ship * ws_quantity else 0 end) as jul_net + ,sum(case when d_moy = 8 + then ws_net_paid_inc_ship * ws_quantity else 0 end) as aug_net + ,sum(case when d_moy = 9 + then ws_net_paid_inc_ship * ws_quantity else 0 end) as sep_net + ,sum(case when d_moy = 10 + then ws_net_paid_inc_ship * ws_quantity else 0 end) as oct_net + ,sum(case when d_moy = 11 + then ws_net_paid_inc_ship * ws_quantity else 0 end) as nov_net + ,sum(case when d_moy = 12 + then ws_net_paid_inc_ship * ws_quantity else 0 end) as dec_net + from + web_sales + ,warehouse + ,date_dim + ,time_dim + ,ship_mode + where + ws_warehouse_sk = w_warehouse_sk + and ws_sold_date_sk = d_date_sk + and ws_sold_time_sk = t_time_sk + and ws_ship_mode_sk = sm_ship_mode_sk + and d_year = 2001 + and t_time between 42970 and 42970+28800 + and sm_carrier in ('ORIENTAL','BOXBUNDLES') + group by + w_warehouse_name + ,w_warehouse_sq_ft + ,w_city + ,w_county + ,w_state + ,w_country + ,d_year + union all + select + w_warehouse_name + ,w_warehouse_sq_ft + ,w_city + ,w_county + ,w_state + ,w_country + ,'ORIENTAL' || ',' || 'BOXBUNDLES' as ship_carriers + ,d_year as year + ,sum(case when d_moy = 1 + then cs_ext_list_price* cs_quantity else 0 end) as jan_sales + ,sum(case when d_moy = 2 + then cs_ext_list_price* cs_quantity else 0 end) as feb_sales + ,sum(case when d_moy = 3 + then cs_ext_list_price* cs_quantity else 0 end) as mar_sales + ,sum(case when d_moy = 4 + then cs_ext_list_price* cs_quantity else 0 end) as apr_sales + ,sum(case when d_moy = 5 + then cs_ext_list_price* cs_quantity else 0 end) as may_sales + ,sum(case when d_moy = 6 + then cs_ext_list_price* cs_quantity else 0 end) as jun_sales + ,sum(case when d_moy = 7 + then cs_ext_list_price* cs_quantity else 0 end) as jul_sales + ,sum(case when d_moy = 8 + then cs_ext_list_price* cs_quantity else 0 end) as aug_sales + ,sum(case when d_moy = 9 + then cs_ext_list_price* cs_quantity else 0 end) as sep_sales + ,sum(case when d_moy = 10 + then cs_ext_list_price* cs_quantity else 0 end) as oct_sales + ,sum(case when d_moy = 11 + then cs_ext_list_price* cs_quantity else 0 end) as nov_sales + ,sum(case when d_moy = 12 + then cs_ext_list_price* cs_quantity else 0 end) as dec_sales + ,sum(case when d_moy = 1 + then cs_net_paid * cs_quantity else 0 end) as jan_net + ,sum(case when d_moy = 2 + then cs_net_paid * cs_quantity else 0 end) as feb_net + ,sum(case when d_moy = 3 + then cs_net_paid * cs_quantity else 0 end) as mar_net + ,sum(case when d_moy = 4 + then cs_net_paid * cs_quantity else 0 end) as apr_net + ,sum(case when d_moy = 5 + then cs_net_paid * cs_quantity else 0 end) as may_net + ,sum(case when d_moy = 6 + then cs_net_paid * cs_quantity else 0 end) as jun_net + ,sum(case when d_moy = 7 + then cs_net_paid * cs_quantity else 0 end) as jul_net + ,sum(case when d_moy = 8 + then cs_net_paid * cs_quantity else 0 end) as aug_net + ,sum(case when d_moy = 9 + then cs_net_paid * cs_quantity else 0 end) as sep_net + ,sum(case when d_moy = 10 + then cs_net_paid * cs_quantity else 0 end) as oct_net + ,sum(case when d_moy = 11 + then cs_net_paid * cs_quantity else 0 end) as nov_net + ,sum(case when d_moy = 12 + then cs_net_paid * cs_quantity else 0 end) as dec_net + from + catalog_sales + ,warehouse + ,date_dim + ,time_dim + ,ship_mode + where + cs_warehouse_sk = w_warehouse_sk + and cs_sold_date_sk = d_date_sk + and cs_sold_time_sk = t_time_sk + and cs_ship_mode_sk = sm_ship_mode_sk + and d_year = 2001 + and t_time between 42970 AND 42970+28800 + and sm_carrier in ('ORIENTAL','BOXBUNDLES') + group by + w_warehouse_name + ,w_warehouse_sq_ft + ,w_city + ,w_county + ,w_state + ,w_country + ,d_year + ) x + group by + w_warehouse_name + ,w_warehouse_sq_ft + ,w_city + ,w_county + ,w_state + ,w_country + ,ship_carriers + ,year + order by w_warehouse_name + limit 100; + +-- end query 66 in stream 0 using template query66.tpl +-- start query 67 in stream 0 using template query67.tpl +select * +from (select i_category + ,i_class + ,i_brand + ,i_product_name + ,d_year + ,d_qoy + ,d_moy + ,s_store_id + ,sumsales + ,rank() over (partition by i_category order by sumsales desc) rk + from (select i_category + ,i_class + ,i_brand + ,i_product_name + ,d_year + ,d_qoy + ,d_moy + ,s_store_id + ,sum(coalesce(ss_sales_price*ss_quantity,0)) sumsales + from store_sales + ,date_dim + ,store + ,item + where ss_sold_date_sk=d_date_sk + and ss_item_sk=i_item_sk + and ss_store_sk = s_store_sk + and d_month_seq between 1217 and 1217+11 + group by rollup(i_category, i_class, i_brand, i_product_name, d_year, d_qoy, d_moy,s_store_id))dw1) dw2 +where rk <= 100 +order by i_category + ,i_class + ,i_brand + ,i_product_name + ,d_year + ,d_qoy + ,d_moy + ,s_store_id + ,sumsales + ,rk +limit 100; + +-- end query 67 in stream 0 using template query67.tpl +-- start query 68 in stream 0 using template query68.tpl +select c_last_name + ,c_first_name + ,ca_city + ,bought_city + ,ss_ticket_number + ,extended_price + ,extended_tax + ,list_price + from (select ss_ticket_number + ,ss_customer_sk + ,ca_city bought_city + ,sum(ss_ext_sales_price) extended_price + ,sum(ss_ext_list_price) list_price + ,sum(ss_ext_tax) extended_tax + from store_sales + ,date_dim + ,store + ,household_demographics + ,customer_address + where store_sales.ss_sold_date_sk = date_dim.d_date_sk + and store_sales.ss_store_sk = store.s_store_sk + and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk + and store_sales.ss_addr_sk = customer_address.ca_address_sk + and date_dim.d_dom between 1 and 2 + and (household_demographics.hd_dep_count = 3 or + household_demographics.hd_vehicle_count= 4) + and date_dim.d_year in (1998,1998+1,1998+2) + and store.s_city in ('Fairview','Midway') + group by ss_ticket_number + ,ss_customer_sk + ,ss_addr_sk,ca_city) dn + ,customer + ,customer_address current_addr + where ss_customer_sk = c_customer_sk + and customer.c_current_addr_sk = current_addr.ca_address_sk + and current_addr.ca_city <> bought_city + order by c_last_name + ,ss_ticket_number + limit 100; + +-- end query 68 in stream 0 using template query68.tpl +-- start query 69 in stream 0 using template query69.tpl +select + cd_gender, + cd_marital_status, + cd_education_status, + count(*) cnt1, + cd_purchase_estimate, + count(*) cnt2, + cd_credit_rating, + count(*) cnt3 + from + customer c,customer_address ca,customer_demographics + where + c.c_current_addr_sk = ca.ca_address_sk and + ca_state in ('IL','TX','ME') and + cd_demo_sk = c.c_current_cdemo_sk and + exists (select * + from store_sales,date_dim + where c.c_customer_sk = ss_customer_sk and + ss_sold_date_sk = d_date_sk and + d_year = 2002 and + d_moy between 1 and 1+2) and + (not exists (select * + from web_sales,date_dim + where c.c_customer_sk = ws_bill_customer_sk and + ws_sold_date_sk = d_date_sk and + d_year = 2002 and + d_moy between 1 and 1+2) and + not exists (select * + from catalog_sales,date_dim + where c.c_customer_sk = cs_ship_customer_sk and + cs_sold_date_sk = d_date_sk and + d_year = 2002 and + d_moy between 1 and 1+2)) + group by cd_gender, + cd_marital_status, + cd_education_status, + cd_purchase_estimate, + cd_credit_rating + order by cd_gender, + cd_marital_status, + cd_education_status, + cd_purchase_estimate, + cd_credit_rating + limit 100; + +-- end query 69 in stream 0 using template query69.tpl +-- start query 70 in stream 0 using template query70.tpl +select * +from (select + sum(ss_net_profit) as total_sum + ,s_state + ,s_county + ,grouping(s_state)+grouping(s_county) as lochierarchy + ,rank() over ( + partition by grouping(s_state)+grouping(s_county), + case when grouping(s_county) = 0 then s_state end + order by sum(ss_net_profit) desc) as rank_within_parent + from + store_sales + ,date_dim d1 + ,store + where + d1.d_month_seq between 1220 and 1220+11 + and d1.d_date_sk = ss_sold_date_sk + and s_store_sk = ss_store_sk + and s_state in + ( select s_state + from (select s_state as s_state, + rank() over ( partition by s_state order by sum(ss_net_profit) desc) as ranking + from store_sales, store, date_dim + where d_month_seq between 1220 and 1220+11 + and d_date_sk = ss_sold_date_sk + and s_store_sk = ss_store_sk + group by s_state + ) tmp1 + where ranking <= 5 + ) + group by rollup(s_state,s_county)) as sub + order by + lochierarchy desc + ,case when lochierarchy = 0 then s_state end + ,rank_within_parent + limit 100; + +-- end query 70 in stream 0 using template query70.tpl +-- start query 71 in stream 0 using template query71.tpl +select i_brand_id brand_id, i_brand brand,t_hour,t_minute, + sum(ext_price) ext_price + from item, (select ws_ext_sales_price as ext_price, + ws_sold_date_sk as sold_date_sk, + ws_item_sk as sold_item_sk, + ws_sold_time_sk as time_sk + from web_sales,date_dim + where d_date_sk = ws_sold_date_sk + and d_moy=12 + and d_year=2002 + union all + select cs_ext_sales_price as ext_price, + cs_sold_date_sk as sold_date_sk, + cs_item_sk as sold_item_sk, + cs_sold_time_sk as time_sk + from catalog_sales,date_dim + where d_date_sk = cs_sold_date_sk + and d_moy=12 + and d_year=2002 + union all + select ss_ext_sales_price as ext_price, + ss_sold_date_sk as sold_date_sk, + ss_item_sk as sold_item_sk, + ss_sold_time_sk as time_sk + from store_sales,date_dim + where d_date_sk = ss_sold_date_sk + and d_moy=12 + and d_year=2002 + ) tmp,time_dim + where + sold_item_sk = i_item_sk + and i_manager_id=1 + and time_sk = t_time_sk + and (t_meal_time = 'breakfast' or t_meal_time = 'dinner') + group by i_brand, i_brand_id,t_hour,t_minute + order by ext_price desc, i_brand_id + ; + +-- end query 71 in stream 0 using template query71.tpl +-- start query 72 in stream 0 using template query72.tpl +select i_item_desc + ,w_warehouse_name + ,d1.d_week_seq + ,sum(case when p_promo_sk is null then 1 else 0 end) no_promo + ,sum(case when p_promo_sk is not null then 1 else 0 end) promo + ,count(*) total_cnt +from catalog_sales +join inventory on (cs_item_sk = inv_item_sk) +join warehouse on (w_warehouse_sk=inv_warehouse_sk) +join item on (i_item_sk = cs_item_sk) +join customer_demographics on (cs_bill_cdemo_sk = cd_demo_sk) +join household_demographics on (cs_bill_hdemo_sk = hd_demo_sk) +join date_dim d1 on (cs_sold_date_sk = d1.d_date_sk) +join date_dim d2 on (inv_date_sk = d2.d_date_sk) +join date_dim d3 on (cs_ship_date_sk = d3.d_date_sk) +left outer join promotion on (cs_promo_sk=p_promo_sk) +left outer join catalog_returns on (cr_item_sk = cs_item_sk and cr_order_number = cs_order_number) +where d1.d_week_seq = d2.d_week_seq + and inv_quantity_on_hand < cs_quantity + and d3.d_date > d1.d_date + 5 + and hd_buy_potential = '1001-5000' + and d1.d_year = 1998 + and cd_marital_status = 'S' +group by i_item_desc,w_warehouse_name,d1.d_week_seq +order by total_cnt desc, i_item_desc, w_warehouse_name, d_week_seq +limit 100; + +-- end query 72 in stream 0 using template query72.tpl +-- start query 73 in stream 0 using template query73.tpl +select c_last_name + ,c_first_name + ,c_salutation + ,c_preferred_cust_flag + ,ss_ticket_number + ,cnt from + (select ss_ticket_number + ,ss_customer_sk + ,count(*) cnt + from store_sales,date_dim,store,household_demographics + where store_sales.ss_sold_date_sk = date_dim.d_date_sk + and store_sales.ss_store_sk = store.s_store_sk + and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk + and date_dim.d_dom between 1 and 2 + and (household_demographics.hd_buy_potential = '1001-5000' or + household_demographics.hd_buy_potential = '5001-10000') + and household_demographics.hd_vehicle_count > 0 + and case when household_demographics.hd_vehicle_count > 0 then + household_demographics.hd_dep_count/ household_demographics.hd_vehicle_count else null end > 1 + and date_dim.d_year in (2000,2000+1,2000+2) + and store.s_county in ('Williamson County','Williamson County','Williamson County','Williamson County') + group by ss_ticket_number,ss_customer_sk) dj,customer + where ss_customer_sk = c_customer_sk + and cnt between 1 and 5 + order by cnt desc, c_last_name asc; + +-- end query 73 in stream 0 using template query73.tpl +-- start query 74 in stream 0 using template query74.tpl +with year_total as ( + select c_customer_id customer_id + ,c_first_name customer_first_name + ,c_last_name customer_last_name + ,d_year as year + ,max(ss_net_paid) year_total + ,'s' sale_type + from customer + ,store_sales + ,date_dim + where c_customer_sk = ss_customer_sk + and ss_sold_date_sk = d_date_sk + and d_year in (1999,1999+1) + group by c_customer_id + ,c_first_name + ,c_last_name + ,d_year + union all + select c_customer_id customer_id + ,c_first_name customer_first_name + ,c_last_name customer_last_name + ,d_year as year + ,max(ws_net_paid) year_total + ,'w' sale_type + from customer + ,web_sales + ,date_dim + where c_customer_sk = ws_bill_customer_sk + and ws_sold_date_sk = d_date_sk + and d_year in (1999,1999+1) + group by c_customer_id + ,c_first_name + ,c_last_name + ,d_year + ) + select + t_s_secyear.customer_id, t_s_secyear.customer_first_name, t_s_secyear.customer_last_name + from year_total t_s_firstyear + ,year_total t_s_secyear + ,year_total t_w_firstyear + ,year_total t_w_secyear + where t_s_secyear.customer_id = t_s_firstyear.customer_id + and t_s_firstyear.customer_id = t_w_secyear.customer_id + and t_s_firstyear.customer_id = t_w_firstyear.customer_id + and t_s_firstyear.sale_type = 's' + and t_w_firstyear.sale_type = 'w' + and t_s_secyear.sale_type = 's' + and t_w_secyear.sale_type = 'w' + and t_s_firstyear.year = 1999 + and t_s_secyear.year = 1999+1 + and t_w_firstyear.year = 1999 + and t_w_secyear.year = 1999+1 + and t_s_firstyear.year_total > 0 + and t_w_firstyear.year_total > 0 + and case when t_w_firstyear.year_total > 0 then t_w_secyear.year_total / t_w_firstyear.year_total else null end + > case when t_s_firstyear.year_total > 0 then t_s_secyear.year_total / t_s_firstyear.year_total else null end + order by 1,3,2 +limit 100; + +-- end query 74 in stream 0 using template query74.tpl +-- start query 75 in stream 0 using template query75.tpl +WITH all_sales AS ( + SELECT d_year + ,i_brand_id + ,i_class_id + ,i_category_id + ,i_manufact_id + ,SUM(sales_cnt) AS sales_cnt + ,SUM(sales_amt) AS sales_amt + FROM (SELECT d_year + ,i_brand_id + ,i_class_id + ,i_category_id + ,i_manufact_id + ,cs_quantity - COALESCE(cr_return_quantity,0) AS sales_cnt + ,cs_ext_sales_price - COALESCE(cr_return_amount,0.0) AS sales_amt + FROM catalog_sales JOIN item ON i_item_sk=cs_item_sk + JOIN date_dim ON d_date_sk=cs_sold_date_sk + LEFT JOIN catalog_returns ON (cs_order_number=cr_order_number + AND cs_item_sk=cr_item_sk) + WHERE i_category='Sports' + UNION + SELECT d_year + ,i_brand_id + ,i_class_id + ,i_category_id + ,i_manufact_id + ,ss_quantity - COALESCE(sr_return_quantity,0) AS sales_cnt + ,ss_ext_sales_price - COALESCE(sr_return_amt,0.0) AS sales_amt + FROM store_sales JOIN item ON i_item_sk=ss_item_sk + JOIN date_dim ON d_date_sk=ss_sold_date_sk + LEFT JOIN store_returns ON (ss_ticket_number=sr_ticket_number + AND ss_item_sk=sr_item_sk) + WHERE i_category='Sports' + UNION + SELECT d_year + ,i_brand_id + ,i_class_id + ,i_category_id + ,i_manufact_id + ,ws_quantity - COALESCE(wr_return_quantity,0) AS sales_cnt + ,ws_ext_sales_price - COALESCE(wr_return_amt,0.0) AS sales_amt + FROM web_sales JOIN item ON i_item_sk=ws_item_sk + JOIN date_dim ON d_date_sk=ws_sold_date_sk + LEFT JOIN web_returns ON (ws_order_number=wr_order_number + AND ws_item_sk=wr_item_sk) + WHERE i_category='Sports') sales_detail + GROUP BY d_year, i_brand_id, i_class_id, i_category_id, i_manufact_id) + SELECT prev_yr.d_year AS prev_year + ,curr_yr.d_year AS year + ,curr_yr.i_brand_id + ,curr_yr.i_class_id + ,curr_yr.i_category_id + ,curr_yr.i_manufact_id + ,prev_yr.sales_cnt AS prev_yr_cnt + ,curr_yr.sales_cnt AS curr_yr_cnt + ,curr_yr.sales_cnt-prev_yr.sales_cnt AS sales_cnt_diff + ,curr_yr.sales_amt-prev_yr.sales_amt AS sales_amt_diff + FROM all_sales curr_yr, all_sales prev_yr + WHERE curr_yr.i_brand_id=prev_yr.i_brand_id + AND curr_yr.i_class_id=prev_yr.i_class_id + AND curr_yr.i_category_id=prev_yr.i_category_id + AND curr_yr.i_manufact_id=prev_yr.i_manufact_id + AND curr_yr.d_year=2002 + AND prev_yr.d_year=2002-1 + AND CAST(curr_yr.sales_cnt AS DECIMAL(17,2))/CAST(prev_yr.sales_cnt AS DECIMAL(17,2))<0.9 + ORDER BY sales_cnt_diff,sales_amt_diff + limit 100; + +-- end query 75 in stream 0 using template query75.tpl +-- start query 76 in stream 0 using template query76.tpl +select channel, col_name, d_year, d_qoy, i_category, COUNT(*) sales_cnt, SUM(ext_sales_price) sales_amt FROM ( + SELECT 'store' as channel, 'ss_customer_sk' col_name, d_year, d_qoy, i_category, ss_ext_sales_price ext_sales_price + FROM store_sales, item, date_dim + WHERE ss_customer_sk IS NULL + AND ss_sold_date_sk=d_date_sk + AND ss_item_sk=i_item_sk + UNION ALL + SELECT 'web' as channel, 'ws_promo_sk' col_name, d_year, d_qoy, i_category, ws_ext_sales_price ext_sales_price + FROM web_sales, item, date_dim + WHERE ws_promo_sk IS NULL + AND ws_sold_date_sk=d_date_sk + AND ws_item_sk=i_item_sk + UNION ALL + SELECT 'catalog' as channel, 'cs_bill_customer_sk' col_name, d_year, d_qoy, i_category, cs_ext_sales_price ext_sales_price + FROM catalog_sales, item, date_dim + WHERE cs_bill_customer_sk IS NULL + AND cs_sold_date_sk=d_date_sk + AND cs_item_sk=i_item_sk) foo +GROUP BY channel, col_name, d_year, d_qoy, i_category +ORDER BY channel, col_name, d_year, d_qoy, i_category +limit 100; + +-- end query 76 in stream 0 using template query76.tpl +-- start query 77 in stream 0 using template query77.tpl +with ss as + (select s_store_sk, + sum(ss_ext_sales_price) as sales, + sum(ss_net_profit) as profit + from store_sales, + date_dim, + store + where ss_sold_date_sk = d_date_sk + and d_date between cast('2000-08-10' as date) + and (cast('2000-08-10' as date) + interval '30 days') + and ss_store_sk = s_store_sk + group by s_store_sk) + , + sr as + (select s_store_sk, + sum(sr_return_amt) as returns, + sum(sr_net_loss) as profit_loss + from store_returns, + date_dim, + store + where sr_returned_date_sk = d_date_sk + and d_date between cast('2000-08-10' as date) + and (cast('2000-08-10' as date) + interval '30 days') + and sr_store_sk = s_store_sk + group by s_store_sk), + cs as + (select cs_call_center_sk, + sum(cs_ext_sales_price) as sales, + sum(cs_net_profit) as profit + from catalog_sales, + date_dim + where cs_sold_date_sk = d_date_sk + and d_date between cast('2000-08-10' as date) + and (cast('2000-08-10' as date) + interval '30 days') + group by cs_call_center_sk + ), + cr as + (select cr_call_center_sk, + sum(cr_return_amount) as returns, + sum(cr_net_loss) as profit_loss + from catalog_returns, + date_dim + where cr_returned_date_sk = d_date_sk + and d_date between cast('2000-08-10' as date) + and (cast('2000-08-10' as date) + interval '30 days') + group by cr_call_center_sk + ), + ws as + ( select wp_web_page_sk, + sum(ws_ext_sales_price) as sales, + sum(ws_net_profit) as profit + from web_sales, + date_dim, + web_page + where ws_sold_date_sk = d_date_sk + and d_date between cast('2000-08-10' as date) + and (cast('2000-08-10' as date) + interval '30 days') + and ws_web_page_sk = wp_web_page_sk + group by wp_web_page_sk), + wr as + (select wp_web_page_sk, + sum(wr_return_amt) as returns, + sum(wr_net_loss) as profit_loss + from web_returns, + date_dim, + web_page + where wr_returned_date_sk = d_date_sk + and d_date between cast('2000-08-10' as date) + and (cast('2000-08-10' as date) + interval '30 days') + and wr_web_page_sk = wp_web_page_sk + group by wp_web_page_sk) + select channel + , id + , sum(sales) as sales + , sum(returns) as returns + , sum(profit) as profit + from + (select 'store channel' as channel + , ss.s_store_sk as id + , sales + , coalesce(returns, 0) as returns + , (profit - coalesce(profit_loss,0)) as profit + from ss left join sr + on ss.s_store_sk = sr.s_store_sk + union all + select 'catalog channel' as channel + , cs_call_center_sk as id + , sales + , returns + , (profit - profit_loss) as profit + from cs + , cr + union all + select 'web channel' as channel + , ws.wp_web_page_sk as id + , sales + , coalesce(returns, 0) as returns + , (profit - coalesce(profit_loss,0)) as profit + from ws left join wr + on ws.wp_web_page_sk = wr.wp_web_page_sk + ) x + group by rollup (channel, id) + order by channel + ,id + limit 100; + +-- end query 77 in stream 0 using template query77.tpl +-- start query 78 in stream 0 using template query78.tpl +with ws as + (select d_year AS ws_sold_year, ws_item_sk, + ws_bill_customer_sk ws_customer_sk, + sum(ws_quantity) ws_qty, + sum(ws_wholesale_cost) ws_wc, + sum(ws_sales_price) ws_sp + from web_sales + left join web_returns on wr_order_number=ws_order_number and ws_item_sk=wr_item_sk + join date_dim on ws_sold_date_sk = d_date_sk + where wr_order_number is null + group by d_year, ws_item_sk, ws_bill_customer_sk + ), +cs as + (select d_year AS cs_sold_year, cs_item_sk, + cs_bill_customer_sk cs_customer_sk, + sum(cs_quantity) cs_qty, + sum(cs_wholesale_cost) cs_wc, + sum(cs_sales_price) cs_sp + from catalog_sales + left join catalog_returns on cr_order_number=cs_order_number and cs_item_sk=cr_item_sk + join date_dim on cs_sold_date_sk = d_date_sk + where cr_order_number is null + group by d_year, cs_item_sk, cs_bill_customer_sk + ), +ss as + (select d_year AS ss_sold_year, ss_item_sk, + ss_customer_sk, + sum(ss_quantity) ss_qty, + sum(ss_wholesale_cost) ss_wc, + sum(ss_sales_price) ss_sp + from store_sales + left join store_returns on sr_ticket_number=ss_ticket_number and ss_item_sk=sr_item_sk + join date_dim on ss_sold_date_sk = d_date_sk + where sr_ticket_number is null + group by d_year, ss_item_sk, ss_customer_sk + ) + select +ss_customer_sk, +round(ss_qty/(coalesce(ws_qty,0)+coalesce(cs_qty,0)),2) ratio, +ss_qty store_qty, ss_wc store_wholesale_cost, ss_sp store_sales_price, +coalesce(ws_qty,0)+coalesce(cs_qty,0) other_chan_qty, +coalesce(ws_wc,0)+coalesce(cs_wc,0) other_chan_wholesale_cost, +coalesce(ws_sp,0)+coalesce(cs_sp,0) other_chan_sales_price +from ss +left join ws on (ws_sold_year=ss_sold_year and ws_item_sk=ss_item_sk and ws_customer_sk=ss_customer_sk) +left join cs on (cs_sold_year=ss_sold_year and cs_item_sk=ss_item_sk and cs_customer_sk=ss_customer_sk) +where (coalesce(ws_qty,0)>0 or coalesce(cs_qty, 0)>0) and ss_sold_year=1998 +order by + ss_customer_sk, + ss_qty desc, ss_wc desc, ss_sp desc, + other_chan_qty, + other_chan_wholesale_cost, + other_chan_sales_price, + ratio +limit 100; + +-- end query 78 in stream 0 using template query78.tpl +-- start query 79 in stream 0 using template query79.tpl +select + c_last_name,c_first_name,substr(s_city,1,30),ss_ticket_number,amt,profit + from + (select ss_ticket_number + ,ss_customer_sk + ,store.s_city + ,sum(ss_coupon_amt) amt + ,sum(ss_net_profit) profit + from store_sales,date_dim,store,household_demographics + where store_sales.ss_sold_date_sk = date_dim.d_date_sk + and store_sales.ss_store_sk = store.s_store_sk + and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk + and (household_demographics.hd_dep_count = 7 or household_demographics.hd_vehicle_count > -1) + and date_dim.d_dow = 1 + and date_dim.d_year in (2000,2000+1,2000+2) + and store.s_number_employees between 200 and 295 + group by ss_ticket_number,ss_customer_sk,ss_addr_sk,store.s_city) ms,customer + where ss_customer_sk = c_customer_sk + order by c_last_name,c_first_name,substr(s_city,1,30), profit +limit 100; + +-- end query 79 in stream 0 using template query79.tpl +-- start query 80 in stream 0 using template query80.tpl +with ssr as + (select s_store_id as store_id, + sum(ss_ext_sales_price) as sales, + sum(coalesce(sr_return_amt, 0)) as returns, + sum(ss_net_profit - coalesce(sr_net_loss, 0)) as profit + from store_sales left outer join store_returns on + (ss_item_sk = sr_item_sk and ss_ticket_number = sr_ticket_number), + date_dim, + store, + item, + promotion + where ss_sold_date_sk = d_date_sk + and d_date between cast('2002-08-14' as date) + and (cast('2002-08-14' as date) + interval '30 days') + and ss_store_sk = s_store_sk + and ss_item_sk = i_item_sk + and i_current_price > 50 + and ss_promo_sk = p_promo_sk + and p_channel_tv = 'N' + group by s_store_id) + , + csr as + (select cp_catalog_page_id as catalog_page_id, + sum(cs_ext_sales_price) as sales, + sum(coalesce(cr_return_amount, 0)) as returns, + sum(cs_net_profit - coalesce(cr_net_loss, 0)) as profit + from catalog_sales left outer join catalog_returns on + (cs_item_sk = cr_item_sk and cs_order_number = cr_order_number), + date_dim, + catalog_page, + item, + promotion + where cs_sold_date_sk = d_date_sk + and d_date between cast('2002-08-14' as date) + and (cast('2002-08-14' as date) + interval '30 days') + and cs_catalog_page_sk = cp_catalog_page_sk + and cs_item_sk = i_item_sk + and i_current_price > 50 + and cs_promo_sk = p_promo_sk + and p_channel_tv = 'N' +group by cp_catalog_page_id) + , + wsr as + (select web_site_id, + sum(ws_ext_sales_price) as sales, + sum(coalesce(wr_return_amt, 0)) as returns, + sum(ws_net_profit - coalesce(wr_net_loss, 0)) as profit + from web_sales left outer join web_returns on + (ws_item_sk = wr_item_sk and ws_order_number = wr_order_number), + date_dim, + web_site, + item, + promotion + where ws_sold_date_sk = d_date_sk + and d_date between cast('2002-08-14' as date) + and (cast('2002-08-14' as date) + interval '30 days') + and ws_web_site_sk = web_site_sk + and ws_item_sk = i_item_sk + and i_current_price > 50 + and ws_promo_sk = p_promo_sk + and p_channel_tv = 'N' +group by web_site_id) + select channel + , id + , sum(sales) as sales + , sum(returns) as returns + , sum(profit) as profit + from + (select 'store channel' as channel + , 'store' || store_id as id + , sales + , returns + , profit + from ssr + union all + select 'catalog channel' as channel + , 'catalog_page' || catalog_page_id as id + , sales + , returns + , profit + from csr + union all + select 'web channel' as channel + , 'web_site' || web_site_id as id + , sales + , returns + , profit + from wsr + ) x + group by rollup (channel, id) + order by channel + ,id + limit 100; + +-- end query 80 in stream 0 using template query80.tpl +-- start query 81 in stream 0 using template query81.tpl +with customer_total_return as + (select cr_returning_customer_sk as ctr_customer_sk + ,ca_state as ctr_state, + sum(cr_return_amt_inc_tax) as ctr_total_return + from catalog_returns + ,date_dim + ,customer_address + where cr_returned_date_sk = d_date_sk + and d_year =2001 + and cr_returning_addr_sk = ca_address_sk + group by cr_returning_customer_sk + ,ca_state ) + select c_customer_id,c_salutation,c_first_name,c_last_name,ca_street_number,ca_street_name + ,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset + ,ca_location_type,ctr_total_return + from customer_total_return ctr1 + ,customer_address + ,customer + where ctr1.ctr_total_return > (select avg(ctr_total_return)*1.2 + from customer_total_return ctr2 + where ctr1.ctr_state = ctr2.ctr_state) + and ca_address_sk = c_current_addr_sk + and ca_state = 'TN' + and ctr1.ctr_customer_sk = c_customer_sk + order by c_customer_id,c_salutation,c_first_name,c_last_name,ca_street_number,ca_street_name + ,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset + ,ca_location_type,ctr_total_return + limit 100; + +-- end query 81 in stream 0 using template query81.tpl +-- start query 82 in stream 0 using template query82.tpl +select i_item_id + ,i_item_desc + ,i_current_price + from item, inventory, date_dim, store_sales + where i_current_price between 58 and 58+30 + and inv_item_sk = i_item_sk + and d_date_sk=inv_date_sk + and d_date between cast('2001-01-13' as date) and (cast('2001-01-13' as date) + interval '60 days') + and i_manufact_id in (259,559,580,485) + and inv_quantity_on_hand between 100 and 500 + and ss_item_sk = i_item_sk + group by i_item_id,i_item_desc,i_current_price + order by i_item_id + limit 100; + +-- end query 82 in stream 0 using template query82.tpl +-- start query 83 in stream 0 using template query83.tpl +with sr_items as + (select i_item_id item_id, + sum(sr_return_quantity) sr_item_qty + from store_returns, + item, + date_dim + where sr_item_sk = i_item_sk + and d_date in + (select d_date + from date_dim + where d_week_seq in + (select d_week_seq + from date_dim + where d_date in ('2001-07-13','2001-09-10','2001-11-16'))) + and sr_returned_date_sk = d_date_sk + group by i_item_id), + cr_items as + (select i_item_id item_id, + sum(cr_return_quantity) cr_item_qty + from catalog_returns, + item, + date_dim + where cr_item_sk = i_item_sk + and d_date in + (select d_date + from date_dim + where d_week_seq in + (select d_week_seq + from date_dim + where d_date in ('2001-07-13','2001-09-10','2001-11-16'))) + and cr_returned_date_sk = d_date_sk + group by i_item_id), + wr_items as + (select i_item_id item_id, + sum(wr_return_quantity) wr_item_qty + from web_returns, + item, + date_dim + where wr_item_sk = i_item_sk + and d_date in + (select d_date + from date_dim + where d_week_seq in + (select d_week_seq + from date_dim + where d_date in ('2001-07-13','2001-09-10','2001-11-16'))) + and wr_returned_date_sk = d_date_sk + group by i_item_id) + select sr_items.item_id + ,sr_item_qty + ,sr_item_qty/(sr_item_qty+cr_item_qty+wr_item_qty)/3.0 * 100 sr_dev + ,cr_item_qty + ,cr_item_qty/(sr_item_qty+cr_item_qty+wr_item_qty)/3.0 * 100 cr_dev + ,wr_item_qty + ,wr_item_qty/(sr_item_qty+cr_item_qty+wr_item_qty)/3.0 * 100 wr_dev + ,(sr_item_qty+cr_item_qty+wr_item_qty)/3.0 average + from sr_items + ,cr_items + ,wr_items + where sr_items.item_id=cr_items.item_id + and sr_items.item_id=wr_items.item_id + order by sr_items.item_id + ,sr_item_qty + limit 100; + +-- end query 83 in stream 0 using template query83.tpl +-- start query 84 in stream 0 using template query84.tpl +select c_customer_id as customer_id + , coalesce(c_last_name,'') || ', ' || coalesce(c_first_name,'') as customername + from customer + ,customer_address + ,customer_demographics + ,household_demographics + ,income_band + ,store_returns + where ca_city = 'Woodland' + and c_current_addr_sk = ca_address_sk + and ib_lower_bound >= 60306 + and ib_upper_bound <= 60306 + 50000 + and ib_income_band_sk = hd_income_band_sk + and cd_demo_sk = c_current_cdemo_sk + and hd_demo_sk = c_current_hdemo_sk + and sr_cdemo_sk = cd_demo_sk + order by c_customer_id + limit 100; + +-- end query 84 in stream 0 using template query84.tpl +-- start query 85 in stream 0 using template query85.tpl +select substr(r_reason_desc,1,20) + ,avg(ws_quantity) + ,avg(wr_refunded_cash) + ,avg(wr_fee) + from web_sales, web_returns, web_page, customer_demographics cd1, + customer_demographics cd2, customer_address, date_dim, reason + where ws_web_page_sk = wp_web_page_sk + and ws_item_sk = wr_item_sk + and ws_order_number = wr_order_number + and ws_sold_date_sk = d_date_sk and d_year = 1998 + and cd1.cd_demo_sk = wr_refunded_cdemo_sk + and cd2.cd_demo_sk = wr_returning_cdemo_sk + and ca_address_sk = wr_refunded_addr_sk + and r_reason_sk = wr_reason_sk + and + ( + ( + cd1.cd_marital_status = 'D' + and + cd1.cd_marital_status = cd2.cd_marital_status + and + cd1.cd_education_status = 'Primary' + and + cd1.cd_education_status = cd2.cd_education_status + and + ws_sales_price between 100.00 and 150.00 + ) + or + ( + cd1.cd_marital_status = 'S' + and + cd1.cd_marital_status = cd2.cd_marital_status + and + cd1.cd_education_status = 'College' + and + cd1.cd_education_status = cd2.cd_education_status + and + ws_sales_price between 50.00 and 100.00 + ) + or + ( + cd1.cd_marital_status = 'U' + and + cd1.cd_marital_status = cd2.cd_marital_status + and + cd1.cd_education_status = 'Advanced Degree' + and + cd1.cd_education_status = cd2.cd_education_status + and + ws_sales_price between 150.00 and 200.00 + ) + ) + and + ( + ( + ca_country = 'United States' + and + ca_state in ('NC', 'TX', 'IA') + and ws_net_profit between 100 and 200 + ) + or + ( + ca_country = 'United States' + and + ca_state in ('WI', 'WV', 'GA') + and ws_net_profit between 150 and 300 + ) + or + ( + ca_country = 'United States' + and + ca_state in ('OK', 'VA', 'KY') + and ws_net_profit between 50 and 250 + ) + ) +group by r_reason_desc +order by substr(r_reason_desc,1,20) + ,avg(ws_quantity) + ,avg(wr_refunded_cash) + ,avg(wr_fee) +limit 100; + +-- end query 85 in stream 0 using template query85.tpl +-- start query 86 in stream 0 using template query86.tpl +select * +from (select + sum(ws_net_paid) as total_sum + ,i_category + ,i_class + ,grouping(i_category)+grouping(i_class) as lochierarchy + ,rank() over ( + partition by grouping(i_category)+grouping(i_class), + case when grouping(i_class) = 0 then i_category end + order by sum(ws_net_paid) desc) as rank_within_parent + from + web_sales + ,date_dim d1 + ,item + where + d1.d_month_seq between 1186 and 1186+11 + and d1.d_date_sk = ws_sold_date_sk + and i_item_sk = ws_item_sk + group by rollup(i_category,i_class)) as sub + order by + lochierarchy desc, + case when lochierarchy = 0 then i_category end, + rank_within_parent + limit 100; + +-- end query 86 in stream 0 using template query86.tpl +-- start query 87 in stream 0 using template query87.tpl +select count(*) +from ((select distinct c_last_name, c_first_name, d_date + from store_sales, date_dim, customer + where store_sales.ss_sold_date_sk = date_dim.d_date_sk + and store_sales.ss_customer_sk = customer.c_customer_sk + and d_month_seq between 1202 and 1202+11) + except + (select distinct c_last_name, c_first_name, d_date + from catalog_sales, date_dim, customer + where catalog_sales.cs_sold_date_sk = date_dim.d_date_sk + and catalog_sales.cs_bill_customer_sk = customer.c_customer_sk + and d_month_seq between 1202 and 1202+11) + except + (select distinct c_last_name, c_first_name, d_date + from web_sales, date_dim, customer + where web_sales.ws_sold_date_sk = date_dim.d_date_sk + and web_sales.ws_bill_customer_sk = customer.c_customer_sk + and d_month_seq between 1202 and 1202+11) +) cool_cust +; + +-- end query 87 in stream 0 using template query87.tpl +-- start query 88 in stream 0 using template query88.tpl +select * +from + (select count(*) h8_30_to_9 + from store_sales, household_demographics , time_dim, store + where ss_sold_time_sk = time_dim.t_time_sk + and ss_hdemo_sk = household_demographics.hd_demo_sk + and ss_store_sk = s_store_sk + and time_dim.t_hour = 8 + and time_dim.t_minute >= 30 + and ((household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2) or + (household_demographics.hd_dep_count = -1 and household_demographics.hd_vehicle_count<=-1+2) or + (household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2)) + and store.s_store_name = 'ese') s1, + (select count(*) h9_to_9_30 + from store_sales, household_demographics , time_dim, store + where ss_sold_time_sk = time_dim.t_time_sk + and ss_hdemo_sk = household_demographics.hd_demo_sk + and ss_store_sk = s_store_sk + and time_dim.t_hour = 9 + and time_dim.t_minute < 30 + and ((household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2) or + (household_demographics.hd_dep_count = -1 and household_demographics.hd_vehicle_count<=-1+2) or + (household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2)) + and store.s_store_name = 'ese') s2, + (select count(*) h9_30_to_10 + from store_sales, household_demographics , time_dim, store + where ss_sold_time_sk = time_dim.t_time_sk + and ss_hdemo_sk = household_demographics.hd_demo_sk + and ss_store_sk = s_store_sk + and time_dim.t_hour = 9 + and time_dim.t_minute >= 30 + and ((household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2) or + (household_demographics.hd_dep_count = -1 and household_demographics.hd_vehicle_count<=-1+2) or + (household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2)) + and store.s_store_name = 'ese') s3, + (select count(*) h10_to_10_30 + from store_sales, household_demographics , time_dim, store + where ss_sold_time_sk = time_dim.t_time_sk + and ss_hdemo_sk = household_demographics.hd_demo_sk + and ss_store_sk = s_store_sk + and time_dim.t_hour = 10 + and time_dim.t_minute < 30 + and ((household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2) or + (household_demographics.hd_dep_count = -1 and household_demographics.hd_vehicle_count<=-1+2) or + (household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2)) + and store.s_store_name = 'ese') s4, + (select count(*) h10_30_to_11 + from store_sales, household_demographics , time_dim, store + where ss_sold_time_sk = time_dim.t_time_sk + and ss_hdemo_sk = household_demographics.hd_demo_sk + and ss_store_sk = s_store_sk + and time_dim.t_hour = 10 + and time_dim.t_minute >= 30 + and ((household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2) or + (household_demographics.hd_dep_count = -1 and household_demographics.hd_vehicle_count<=-1+2) or + (household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2)) + and store.s_store_name = 'ese') s5, + (select count(*) h11_to_11_30 + from store_sales, household_demographics , time_dim, store + where ss_sold_time_sk = time_dim.t_time_sk + and ss_hdemo_sk = household_demographics.hd_demo_sk + and ss_store_sk = s_store_sk + and time_dim.t_hour = 11 + and time_dim.t_minute < 30 + and ((household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2) or + (household_demographics.hd_dep_count = -1 and household_demographics.hd_vehicle_count<=-1+2) or + (household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2)) + and store.s_store_name = 'ese') s6, + (select count(*) h11_30_to_12 + from store_sales, household_demographics , time_dim, store + where ss_sold_time_sk = time_dim.t_time_sk + and ss_hdemo_sk = household_demographics.hd_demo_sk + and ss_store_sk = s_store_sk + and time_dim.t_hour = 11 + and time_dim.t_minute >= 30 + and ((household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2) or + (household_demographics.hd_dep_count = -1 and household_demographics.hd_vehicle_count<=-1+2) or + (household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2)) + and store.s_store_name = 'ese') s7, + (select count(*) h12_to_12_30 + from store_sales, household_demographics , time_dim, store + where ss_sold_time_sk = time_dim.t_time_sk + and ss_hdemo_sk = household_demographics.hd_demo_sk + and ss_store_sk = s_store_sk + and time_dim.t_hour = 12 + and time_dim.t_minute < 30 + and ((household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2) or + (household_demographics.hd_dep_count = -1 and household_demographics.hd_vehicle_count<=-1+2) or + (household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2)) + and store.s_store_name = 'ese') s8 +; + +-- end query 88 in stream 0 using template query88.tpl +-- start query 89 in stream 0 using template query89.tpl +select * +from( +select i_category, i_class, i_brand, + s_store_name, s_company_name, + d_moy, + sum(ss_sales_price) sum_sales, + avg(sum(ss_sales_price)) over + (partition by i_category, i_brand, s_store_name, s_company_name) + avg_monthly_sales +from item, store_sales, date_dim, store +where ss_item_sk = i_item_sk and + ss_sold_date_sk = d_date_sk and + ss_store_sk = s_store_sk and + d_year in (2001) and + ((i_category in ('Books','Children','Electronics') and + i_class in ('history','school-uniforms','audio') + ) + or (i_category in ('Men','Sports','Shoes') and + i_class in ('pants','tennis','womens') + )) +group by i_category, i_class, i_brand, + s_store_name, s_company_name, d_moy) tmp1 +where case when (avg_monthly_sales <> 0) then (abs(sum_sales - avg_monthly_sales) / avg_monthly_sales) else null end > 0.1 +order by sum_sales - avg_monthly_sales, s_store_name +limit 100; + +-- end query 89 in stream 0 using template query89.tpl +-- start query 90 in stream 0 using template query90.tpl +select cast(amc as decimal(15,4))/cast(pmc as decimal(15,4)) am_pm_ratio + from ( select count(*) amc + from web_sales, household_demographics , time_dim, web_page + where ws_sold_time_sk = time_dim.t_time_sk + and ws_ship_hdemo_sk = household_demographics.hd_demo_sk + and ws_web_page_sk = web_page.wp_web_page_sk + and time_dim.t_hour between 12 and 12+1 + and household_demographics.hd_dep_count = 6 + and web_page.wp_char_count between 5000 and 5200) at, + ( select count(*) pmc + from web_sales, household_demographics , time_dim, web_page + where ws_sold_time_sk = time_dim.t_time_sk + and ws_ship_hdemo_sk = household_demographics.hd_demo_sk + and ws_web_page_sk = web_page.wp_web_page_sk + and time_dim.t_hour between 14 and 14+1 + and household_demographics.hd_dep_count = 6 + and web_page.wp_char_count between 5000 and 5200) pt + order by am_pm_ratio + limit 100; + +-- end query 90 in stream 0 using template query90.tpl +-- start query 91 in stream 0 using template query91.tpl +select + cc_call_center_id Call_Center, + cc_name Call_Center_Name, + cc_manager Manager, + sum(cr_net_loss) Returns_Loss +from + call_center, + catalog_returns, + date_dim, + customer, + customer_address, + customer_demographics, + household_demographics +where + cr_call_center_sk = cc_call_center_sk +and cr_returned_date_sk = d_date_sk +and cr_returning_customer_sk= c_customer_sk +and cd_demo_sk = c_current_cdemo_sk +and hd_demo_sk = c_current_hdemo_sk +and ca_address_sk = c_current_addr_sk +and d_year = 2000 +and d_moy = 12 +and ( (cd_marital_status = 'M' and cd_education_status = 'Unknown') + or(cd_marital_status = 'W' and cd_education_status = 'Advanced Degree')) +and hd_buy_potential like 'Unknown%' +and ca_gmt_offset = -7 +group by cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status +order by sum(cr_net_loss) desc; + +-- end query 91 in stream 0 using template query91.tpl +-- start query 92 in stream 0 using template query92.tpl +select + sum(ws_ext_discount_amt) as "Excess Discount Amount" +from + web_sales + ,item + ,date_dim +where +i_manufact_id = 714 +and i_item_sk = ws_item_sk +and d_date between '2000-02-01' and + (cast('2000-02-01' as date) + interval '90 days') +and d_date_sk = ws_sold_date_sk +and ws_ext_discount_amt + > ( + SELECT + 1.3 * avg(ws_ext_discount_amt) + FROM + web_sales + ,date_dim + WHERE + ws_item_sk = i_item_sk + and d_date between '2000-02-01' and + (cast('2000-02-01' as date) + interval '90 days') + and d_date_sk = ws_sold_date_sk + ) +order by sum(ws_ext_discount_amt) +limit 100; + +-- end query 92 in stream 0 using template query92.tpl +-- start query 93 in stream 0 using template query93.tpl +select ss_customer_sk + ,sum(act_sales) sumsales + from (select ss_item_sk + ,ss_ticket_number + ,ss_customer_sk + ,case when sr_return_quantity is not null then (ss_quantity-sr_return_quantity)*ss_sales_price + else (ss_quantity*ss_sales_price) end act_sales + from store_sales left outer join store_returns on (sr_item_sk = ss_item_sk + and sr_ticket_number = ss_ticket_number) + ,reason + where sr_reason_sk = r_reason_sk + and r_reason_desc = 'reason 58') t + group by ss_customer_sk + order by sumsales, ss_customer_sk +limit 100; + +-- end query 93 in stream 0 using template query93.tpl +-- start query 94 in stream 0 using template query94.tpl +select + count(distinct ws_order_number) as "order count" + ,sum(ws_ext_ship_cost) as "total shipping cost" + ,sum(ws_net_profit) as "total net profit" +from + web_sales ws1 + ,date_dim + ,customer_address + ,web_site +where + d_date between '2002-5-01' and + (cast('2002-5-01' as date) + interval '60 days') +and ws1.ws_ship_date_sk = d_date_sk +and ws1.ws_ship_addr_sk = ca_address_sk +and ca_state = 'OK' +and ws1.ws_web_site_sk = web_site_sk +and web_company_name = 'pri' +and exists (select * + from web_sales ws2 + where ws1.ws_order_number = ws2.ws_order_number + and ws1.ws_warehouse_sk <> ws2.ws_warehouse_sk) +and not exists(select * + from web_returns wr1 + where ws1.ws_order_number = wr1.wr_order_number) +order by count(distinct ws_order_number) +limit 100; + +-- end query 94 in stream 0 using template query94.tpl +-- start query 95 in stream 0 using template query95.tpl +with ws_wh as +(select ws1.ws_order_number,ws1.ws_warehouse_sk wh1,ws2.ws_warehouse_sk wh2 + from web_sales ws1,web_sales ws2 + where ws1.ws_order_number = ws2.ws_order_number + and ws1.ws_warehouse_sk <> ws2.ws_warehouse_sk) + select + count(distinct ws_order_number) as "order count" + ,sum(ws_ext_ship_cost) as "total shipping cost" + ,sum(ws_net_profit) as "total net profit" +from + web_sales ws1 + ,date_dim + ,customer_address + ,web_site +where + d_date between '2001-4-01' and + (cast('2001-4-01' as date) + interval '60 days') +and ws1.ws_ship_date_sk = d_date_sk +and ws1.ws_ship_addr_sk = ca_address_sk +and ca_state = 'VA' +and ws1.ws_web_site_sk = web_site_sk +and web_company_name = 'pri' +and ws1.ws_order_number in (select ws_order_number + from ws_wh) +and ws1.ws_order_number in (select wr_order_number + from web_returns,ws_wh + where wr_order_number = ws_wh.ws_order_number) +order by count(distinct ws_order_number) +limit 100; + +-- end query 95 in stream 0 using template query95.tpl +-- start query 96 in stream 0 using template query96.tpl +select count(*) +from store_sales + ,household_demographics + ,time_dim, store +where ss_sold_time_sk = time_dim.t_time_sk + and ss_hdemo_sk = household_demographics.hd_demo_sk + and ss_store_sk = s_store_sk + and time_dim.t_hour = 8 + and time_dim.t_minute >= 30 + and household_demographics.hd_dep_count = 0 + and store.s_store_name = 'ese' +order by count(*) +limit 100; + +-- end query 96 in stream 0 using template query96.tpl +-- start query 97 in stream 0 using template query97.tpl +with ssci as ( +select ss_customer_sk customer_sk + ,ss_item_sk item_sk +from store_sales,date_dim +where ss_sold_date_sk = d_date_sk + and d_month_seq between 1199 and 1199 + 11 +group by ss_customer_sk + ,ss_item_sk), +csci as( + select cs_bill_customer_sk customer_sk + ,cs_item_sk item_sk +from catalog_sales,date_dim +where cs_sold_date_sk = d_date_sk + and d_month_seq between 1199 and 1199 + 11 +group by cs_bill_customer_sk + ,cs_item_sk) + select sum(case when ssci.customer_sk is not null and csci.customer_sk is null then 1 else 0 end) store_only + ,sum(case when ssci.customer_sk is null and csci.customer_sk is not null then 1 else 0 end) catalog_only + ,sum(case when ssci.customer_sk is not null and csci.customer_sk is not null then 1 else 0 end) store_and_catalog +from ssci full outer join csci on (ssci.customer_sk=csci.customer_sk + and ssci.item_sk = csci.item_sk) +limit 100; + +-- end query 97 in stream 0 using template query97.tpl +-- start query 98 in stream 0 using template query98.tpl +select i_item_id + ,i_item_desc + ,i_category + ,i_class + ,i_current_price + ,sum(ss_ext_sales_price) as itemrevenue + ,sum(ss_ext_sales_price)*100/sum(sum(ss_ext_sales_price)) over + (partition by i_class) as revenueratio +from + store_sales + ,item + ,date_dim +where + ss_item_sk = i_item_sk + and i_category in ('Men', 'Sports', 'Jewelry') + and ss_sold_date_sk = d_date_sk + and d_date between cast('1999-02-05' as date) + and (cast('1999-02-05' as date) + interval '60 days') +group by + i_item_id + ,i_item_desc + ,i_category + ,i_class + ,i_current_price +order by + i_category + ,i_class + ,i_item_id + ,i_item_desc + ,revenueratio; + +-- end query 98 in stream 0 using template query98.tpl +-- start query 99 in stream 0 using template query99.tpl +select + substr(w_warehouse_name,1,20) + ,sm_type + ,cc_name + ,sum(case when (cs_ship_date_sk - cs_sold_date_sk <= 30 ) then 1 else 0 end) as "30 days" + ,sum(case when (cs_ship_date_sk - cs_sold_date_sk > 30) and + (cs_ship_date_sk - cs_sold_date_sk <= 60) then 1 else 0 end ) as "31-60 days" + ,sum(case when (cs_ship_date_sk - cs_sold_date_sk > 60) and + (cs_ship_date_sk - cs_sold_date_sk <= 90) then 1 else 0 end) as "61-90 days" + ,sum(case when (cs_ship_date_sk - cs_sold_date_sk > 90) and + (cs_ship_date_sk - cs_sold_date_sk <= 120) then 1 else 0 end) as "91-120 days" + ,sum(case when (cs_ship_date_sk - cs_sold_date_sk > 120) then 1 else 0 end) as ">120 days" +from + catalog_sales + ,warehouse + ,ship_mode + ,call_center + ,date_dim +where + d_month_seq between 1194 and 1194 + 11 +and cs_ship_date_sk = d_date_sk +and cs_warehouse_sk = w_warehouse_sk +and cs_ship_mode_sk = sm_ship_mode_sk +and cs_call_center_sk = cc_call_center_sk +group by + substr(w_warehouse_name,1,20) + ,sm_type + ,cc_name +order by substr(w_warehouse_name,1,20) + ,sm_type + ,cc_name +limit 100; diff --git a/tests/test_cases.py b/tests/test_cases.py index e0abda7..6797170 100644 --- a/tests/test_cases.py +++ b/tests/test_cases.py @@ -471,7 +471,7 @@ def test_formats(config): qs = query_state(config, acon, query, {'format': 'yaml'}) try: - yaml_doc = yaml.load(qs[0][3]) + yaml_doc = yaml.load(qs[0][3], Loader=yaml.FullLoader) except: assert False, 'Invalid yaml format' assert len(qs) == 1 @@ -508,26 +508,29 @@ class DataLoadException(Exception): pass class StressTestException(Exception): pass def load_tpcds_data(config): - print('Preparing TPC-DS...') - subprocess.call(['./tests/prepare_stress.sh']) + print('Loading TPC-DS data...') + # subprocess.call(['./tests/prepare_stress.sh']) try: conn = psycopg2.connect(**config) cur = conn.cursor() # Create tables - cur.execute(open('tmp_stress/tpcds-kit/tools/tpcds.sql', 'r').read()) + with open('tmp_stress/tpcds-kit/tools/tpcds.sql', 'r') as f: + cur.execute(f.read()) # Copy table data from files for table_datafile in os.listdir('tmp_stress/tpcds-kit/tools/'): - if table_datafile.endswith(".dat"): + if table_datafile.endswith('.dat'): table_name = os.path.splitext(os.path.basename(table_datafile))[0] - copy_cmd = "COPY %s FROM 'tmp_stress/tpcds-kit/tools/tables/%s' CSV DELIMITER '|'" % (table_name, table_datafile) + copy_cmd = "COPY %s FROM '/pg/testdir/tmp_stress/tpcds-kit/tools/tables/%s' CSV DELIMITER '|'" % (table_name, table_datafile) - print("Loading table ", table_name) - cur.execute("TRUNCATE %s" % table_name) + print("Loading table", table_name) + # cur.execute("TRUNCATE %s" % table_name) cur.execute(copy_cmd) + conn.commit() + except Exception as e: cur.close() conn.close() @@ -538,10 +541,10 @@ def load_tpcds_data(config): def stress_test(config): """TPC-DS stress test""" load_tpcds_data(config) - print('Starting test...') + print('Preparing TPC-DS queries...') # Execute query in separate thread - with open("tests/query_tpcds.sql",'r') as f: + with open('tmp_stress/tpcds-kit/tools/query_0.sql', 'r') as f: sql = f.read() queries = sql.split(';') @@ -552,10 +555,14 @@ def stress_test(config): acon, = n_async_connect(config) + print('Starting test...') timeout_list = [] + exclude_list = [2] bar = progressbar.ProgressBar(max_value=len(queries)) for i, query in enumerate(queries): bar.update(i + 1) + if i + 1 in exclude_list: + continue try: # Set query timeout to 10 sec set_guc(acon, 'statement_timeout', 10000) @@ -564,8 +571,6 @@ def stress_test(config): #TODO: Put here testgres exception when supported except psycopg2.extensions.QueryCanceledError: timeout_list.append(i) - finally: - pass n_close((acon,)) From 459a060d3cb3e0176c54a2ac05b5096c70cbfc02 Mon Sep 17 00:00:00 2001 From: Alexey Kondratov Date: Fri, 25 Oct 2019 21:16:29 +0300 Subject: [PATCH 03/91] Use committed TPC-DS queries for now --- .travis.yml | 2 ++ tests/test_cases.py | 14 ++++++++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 96a4651..66bf741 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,6 +27,8 @@ env: - PG_VERSION=9.6 LEVEL=hardcore - PG_VERSION=9.6 - PG_VERSION=12 LEVEL=stress + - PG_VERSION=11 LEVEL=stress + - PG_VERSION=10 LEVEL=stress matrix: allow_failures: diff --git a/tests/test_cases.py b/tests/test_cases.py index 6797170..59b1e17 100644 --- a/tests/test_cases.py +++ b/tests/test_cases.py @@ -98,10 +98,19 @@ def pg_query_state(config, pid, verbose=False, costs=False, timing=False, \ conn = psycopg2.connect(**config) curs = conn.cursor() + set_guc(conn, 'statement_timeout', 10000) + result = [] + n_retries = 0 while not result: curs.callproc('pg_query_state', (pid, verbose, costs, timing, buffers, triggers, format)) result = curs.fetchall() + n_retries += 1 + + if n_retries == 25: + print('pg_query_state tried 25 times with no effect') + break + notices = conn.notices[:] conn.close() return result @@ -544,7 +553,8 @@ def stress_test(config): print('Preparing TPC-DS queries...') # Execute query in separate thread - with open('tmp_stress/tpcds-kit/tools/query_0.sql', 'r') as f: + # with open('tmp_stress/tpcds-kit/tools/query_0.sql', 'r') as f: + with open('tests/query_tpcds.sql', 'r') as f: sql = f.read() queries = sql.split(';') @@ -557,7 +567,7 @@ def stress_test(config): print('Starting test...') timeout_list = [] - exclude_list = [2] + exclude_list = [] bar = progressbar.ProgressBar(max_value=len(queries)) for i, query in enumerate(queries): bar.update(i + 1) From 37bedbe10526511d38b0c08d0621c7f485dd2aee Mon Sep 17 00:00:00 2001 From: Alexey Kondratov Date: Fri, 25 Oct 2019 21:19:02 +0300 Subject: [PATCH 04/91] Prepare TPC-DS data --- tests/test_cases.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_cases.py b/tests/test_cases.py index 59b1e17..633a577 100644 --- a/tests/test_cases.py +++ b/tests/test_cases.py @@ -518,7 +518,7 @@ class StressTestException(Exception): pass def load_tpcds_data(config): print('Loading TPC-DS data...') - # subprocess.call(['./tests/prepare_stress.sh']) + subprocess.call(['./tests/prepare_stress.sh']) try: conn = psycopg2.connect(**config) From 166ead3144761f88965f63ca07239e26101a4b25 Mon Sep 17 00:00:00 2001 From: Alexey Kondratov Date: Sat, 26 Oct 2019 14:40:44 +0300 Subject: [PATCH 05/91] New tests refactoring --- tests/prepare_stress.sh | 2 +- tests/test_cases.py | 37 +++++++++++++++++++++++-------------- 2 files changed, 24 insertions(+), 15 deletions(-) diff --git a/tests/prepare_stress.sh b/tests/prepare_stress.sh index 827e310..aaa0261 100755 --- a/tests/prepare_stress.sh +++ b/tests/prepare_stress.sh @@ -4,7 +4,7 @@ mkdir -p tmp_stress cd tmp_stress rm -rf ./* -git clone https://github.com/gregrahn/tpcds-kit.git +git clone --depth 1 --single-branch --branch master https://github.com/gregrahn/tpcds-kit.git cd tpcds-kit/tools make -s diff --git a/tests/test_cases.py b/tests/test_cases.py index 633a577..4d51e1f 100644 --- a/tests/test_cases.py +++ b/tests/test_cases.py @@ -14,6 +14,11 @@ from time import sleep +MAX_PG_QS_RETRIES = 50 +TPC_DS_EXCLUDE_LIST = [] # actual numbers of TPC-DS tests to exclude +TPC_DS_STATEMENT_TIMEOUT = 20000 # statement_timeout in ms +stress_in_progress = False + def wait(conn): """wait for some event on connection to postgres""" while 1: @@ -98,18 +103,21 @@ def pg_query_state(config, pid, verbose=False, costs=False, timing=False, \ conn = psycopg2.connect(**config) curs = conn.cursor() - set_guc(conn, 'statement_timeout', 10000) + + if stress_in_progress: + set_guc(conn, 'statement_timeout', TPC_DS_STATEMENT_TIMEOUT) + n_retries = 0 result = [] - n_retries = 0 while not result: curs.callproc('pg_query_state', (pid, verbose, costs, timing, buffers, triggers, format)) result = curs.fetchall() - n_retries += 1 - if n_retries == 25: - print('pg_query_state tried 25 times with no effect') - break + if stress_in_progress: + n_retries += 1 + if n_retries >= MAX_PG_QS_RETRIES: + print('\npg_query_state tried %s times with no effect, giving up' % MAX_PG_QS_RETRIES) + break notices = conn.notices[:] conn.close() @@ -549,6 +557,7 @@ def load_tpcds_data(config): def stress_test(config): """TPC-DS stress test""" + stress_in_progress = True load_tpcds_data(config) print('Preparing TPC-DS queries...') @@ -565,24 +574,24 @@ def stress_test(config): acon, = n_async_connect(config) - print('Starting test...') + print('Starting TPC-DS queries...') timeout_list = [] - exclude_list = [] bar = progressbar.ProgressBar(max_value=len(queries)) for i, query in enumerate(queries): bar.update(i + 1) - if i + 1 in exclude_list: + if i + 1 in TPC_DS_EXCLUDE_LIST: continue try: - # Set query timeout to 10 sec - set_guc(acon, 'statement_timeout', 10000) + # Set query timeout to TPC_DS_STATEMENT_TIMEOUT / 1000 seconds + set_guc(acon, 'statement_timeout', TPC_DS_STATEMENT_TIMEOUT) qs = query_state(config, acon, query) - #TODO: Put here testgres exception when supported except psycopg2.extensions.QueryCanceledError: - timeout_list.append(i) + timeout_list.append(i + 1) n_close((acon,)) if len(timeout_list) > 0: - print('There were pg_query_state timeouts (10s) on queries: ', timeout_list) + print('There were pg_query_state timeouts (%s s) on queries:' % TPC_DS_STATEMENT_TIMEOUT, timeout_list) + + stress_in_progress = False From 91d77543d9ea08b8fc5cc179dde56f64f02a2e23 Mon Sep 17 00:00:00 2001 From: Alexey Kondratov Date: Sat, 26 Oct 2019 22:05:16 +0300 Subject: [PATCH 06/91] Add comment --- tests/test_cases.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_cases.py b/tests/test_cases.py index 4d51e1f..3bd95b9 100644 --- a/tests/test_cases.py +++ b/tests/test_cases.py @@ -14,6 +14,10 @@ from time import sleep +# Some queries from TPC-DS may freeze or be even broken, +# so we allow some sort of failure, since we do not test +# Postgres, but rather that pg_query_state do not crash +# anything under stress load. MAX_PG_QS_RETRIES = 50 TPC_DS_EXCLUDE_LIST = [] # actual numbers of TPC-DS tests to exclude TPC_DS_STATEMENT_TIMEOUT = 20000 # statement_timeout in ms From 5bc77690957450435691d37697eea6ef805df8e0 Mon Sep 17 00:00:00 2001 From: Alexey Kondratov Date: Sat, 26 Oct 2019 22:30:02 +0300 Subject: [PATCH 07/91] Set stress_in_progress as global --- tests/test_cases.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_cases.py b/tests/test_cases.py index 3bd95b9..ca38248 100644 --- a/tests/test_cases.py +++ b/tests/test_cases.py @@ -103,7 +103,7 @@ def pg_query_state(config, pid, verbose=False, costs=False, timing=False, \ Save any warning, info, notice and log data in global variable 'notices' """ - global notices + global notices, stress_in_progress conn = psycopg2.connect(**config) curs = conn.cursor() @@ -561,6 +561,8 @@ def load_tpcds_data(config): def stress_test(config): """TPC-DS stress test""" + global stress_in_progress + stress_in_progress = True load_tpcds_data(config) From 8c1a8d75d1abc0545251c99e1e26fc0c0381d757 Mon Sep 17 00:00:00 2001 From: Alexey Kondratov Date: Sun, 27 Oct 2019 00:38:05 +0300 Subject: [PATCH 08/91] Some formatting and new queries --- tests/pg_qs_test_runner.py | 2 +- tests/prepare_stress.sh | 3 +- tests/test_cases.py | 58 ++++++++++++++++++++------------------ 3 files changed, 34 insertions(+), 29 deletions(-) diff --git a/tests/pg_qs_test_runner.py b/tests/pg_qs_test_runner.py index 6eaf4fe..bc9e638 100644 --- a/tests/pg_qs_test_runner.py +++ b/tests/pg_qs_test_runner.py @@ -99,7 +99,7 @@ def main(config): if os.environ['LEVEL'] == 'stress': print('Starting stress test') - stress_test(config) + test_tpc_ds(config) print('Stress finished successfully') teardown(con) diff --git a/tests/prepare_stress.sh b/tests/prepare_stress.sh index aaa0261..1ae3c75 100755 --- a/tests/prepare_stress.sh +++ b/tests/prepare_stress.sh @@ -5,6 +5,7 @@ cd tmp_stress rm -rf ./* git clone --depth 1 --single-branch --branch master https://github.com/gregrahn/tpcds-kit.git +git clone --depth 1 --single-branch --branch master https://github.com/cwida/tpcds-result-reproduction.git cd tpcds-kit/tools make -s @@ -14,7 +15,7 @@ make -s #Prepare data mkdir -p tables for i in `ls *.dat`; do - echo "Preparing file " $i + echo "Preparing file" $i sed 's/|$//' $i > tables/$i done diff --git a/tests/test_cases.py b/tests/test_cases.py index ca38248..1b13b63 100644 --- a/tests/test_cases.py +++ b/tests/test_cases.py @@ -60,35 +60,35 @@ def n_close(conns): def debug_output(qs, qs_len, pid, query, expected): something_happened = False if (qs_len and len(qs) != qs_len ): - print( "len(qs): ", len(qs), ", expected: ", qs_len) + print("len(qs): ", len(qs), ", expected: ", qs_len) something_happened = True if (pid and qs[0][0] != pid): - print( "qs[0][0]: ", qs[0][0], " = ", pid) + print("qs[0][0]: ", qs[0][0], " = ", pid) something_happened = True if (qs[0][1] != 0): - print( "qs[0][1]: ", qs[0][1], ", expected: 0") + print("qs[0][1]: ", qs[0][1], ", expected: 0") something_happened = True if (qs[0][2] != query): - print( "qs[0][2]:\n", qs[0][2]) - print( "Expected:\n", query) + print("qs[0][2]:\n", qs[0][2]) + print("Expected:\n", query) something_happened = True if (not (re.match(expected, qs[0][3]))): - print( "qs[0][3]:\n", qs[0][3]) - print( "Expected:\n", expected) + print("qs[0][3]:\n", qs[0][3]) + print("Expected:\n", expected) something_happened = True if (qs[0][4] != None): - print( "qs[0][4]: ", qs[0][4], "Expected: None") + print("qs[0][4]: ", qs[0][4], "Expected: None") something_happened = True if (qs_len and len(qs) > qs_len): for i in range(qs_len, len(qs)): - print( "qs[",i,"][0]: ", qs[i][0]) - print( "qs[",i,"][1]: ", qs[i][1]) - print( "qs[",i,"][2]: ", qs[i][2]) - print( "qs[",i,"][3]: ", qs[i][3]) - print( "qs[",i,"][4]: ", qs[i][4]) + print("qs[",i,"][0]: ", qs[i][0]) + print("qs[",i,"][1]: ", qs[i][1]) + print("qs[",i,"][2]: ", qs[i][2]) + print("qs[",i,"][3]: ", qs[i][3]) + print("qs[",i,"][4]: ", qs[i][4]) something_happened = True if (something_happened): - print( "If test have not crashed, then it's OK") + print("If test have not crashed, then it's OK") def notices_warning(): if (len(notices) > 0): @@ -546,7 +546,7 @@ def load_tpcds_data(config): table_name = os.path.splitext(os.path.basename(table_datafile))[0] copy_cmd = "COPY %s FROM '/pg/testdir/tmp_stress/tpcds-kit/tools/tables/%s' CSV DELIMITER '|'" % (table_name, table_datafile) - print("Loading table", table_name) + print('Loading table', table_name) # cur.execute("TRUNCATE %s" % table_name) cur.execute(copy_cmd) @@ -559,7 +559,7 @@ def load_tpcds_data(config): print('done!') -def stress_test(config): +def test_tpc_ds(config): """TPC-DS stress test""" global stress_in_progress @@ -567,16 +567,20 @@ def stress_test(config): load_tpcds_data(config) print('Preparing TPC-DS queries...') - # Execute query in separate thread - # with open('tmp_stress/tpcds-kit/tools/query_0.sql', 'r') as f: - with open('tests/query_tpcds.sql', 'r') as f: - sql = f.read() - - queries = sql.split(';') - for i, query in enumerate(queries): - queries[i] = query.replace('%','%%') - if (len(query.strip()) == 0): - del queries[i] + # # Execute query in separate thread + # # with open('tmp_stress/tpcds-kit/tools/query_0.sql', 'r') as f: + # with open('tests/query_tpcds.sql', 'r') as f: + # sql = f.read() + + # queries = sql.split(';') + # for i, query in enumerate(queries): + # queries[i] = query.replace('%','%%') + # if (len(query.strip()) == 0): + # del queries[i] + queries = [] + for query_file in sorted(os.listdir('tmp_stress/tpcds-result-reproduction/query_qualification/')): + with open('tmp_stress/tpcds-result-reproduction/query_qualification/%s' % query_file, 'r') as f: + queries.append(f.read()) acon, = n_async_connect(config) @@ -598,6 +602,6 @@ def stress_test(config): n_close((acon,)) if len(timeout_list) > 0: - print('There were pg_query_state timeouts (%s s) on queries:' % TPC_DS_STATEMENT_TIMEOUT, timeout_list) + print('\nThere were pg_query_state timeouts (%s s) on queries:' % TPC_DS_STATEMENT_TIMEOUT, timeout_list) stress_in_progress = False From 7118ea27b8b4bb436a9df88e3121297667be522b Mon Sep 17 00:00:00 2001 From: Alexey Kondratov Date: Sun, 27 Oct 2019 01:01:19 +0300 Subject: [PATCH 09/91] Remove committed TCP-DS queries --- tests/query_tpcds.sql | 4871 ----------------------------------------- 1 file changed, 4871 deletions(-) delete mode 100644 tests/query_tpcds.sql diff --git a/tests/query_tpcds.sql b/tests/query_tpcds.sql deleted file mode 100644 index e82fca1..0000000 --- a/tests/query_tpcds.sql +++ /dev/null @@ -1,4871 +0,0 @@ --- start query 1 in stream 0 using template query1.tpl -with customer_total_return as -(select sr_customer_sk as ctr_customer_sk -,sr_store_sk as ctr_store_sk -,sum(SR_FEE) as ctr_total_return -from store_returns -,date_dim -where sr_returned_date_sk = d_date_sk -and d_year =2000 -group by sr_customer_sk -,sr_store_sk) - select c_customer_id -from customer_total_return ctr1 -,store -,customer -where ctr1.ctr_total_return > (select avg(ctr_total_return)*1.2 -from customer_total_return ctr2 -where ctr1.ctr_store_sk = ctr2.ctr_store_sk) -and s_store_sk = ctr1.ctr_store_sk -and s_state = 'TN' -and ctr1.ctr_customer_sk = c_customer_sk -order by c_customer_id -limit 100; - --- end query 1 in stream 0 using template query1.tpl --- start query 2 in stream 0 using template query2.tpl -with wscs as - (select sold_date_sk - ,sales_price - from (select ws_sold_date_sk sold_date_sk - ,ws_ext_sales_price sales_price - from web_sales - union all - select cs_sold_date_sk sold_date_sk - ,cs_ext_sales_price sales_price - from catalog_sales) as sdsp), - wswscs as - (select d_week_seq, - sum(case when (d_day_name='Sunday') then sales_price else null end) sun_sales, - sum(case when (d_day_name='Monday') then sales_price else null end) mon_sales, - sum(case when (d_day_name='Tuesday') then sales_price else null end) tue_sales, - sum(case when (d_day_name='Wednesday') then sales_price else null end) wed_sales, - sum(case when (d_day_name='Thursday') then sales_price else null end) thu_sales, - sum(case when (d_day_name='Friday') then sales_price else null end) fri_sales, - sum(case when (d_day_name='Saturday') then sales_price else null end) sat_sales - from wscs - ,date_dim - where d_date_sk = sold_date_sk - group by d_week_seq) - select d_week_seq1 - ,round(sun_sales1/sun_sales2,2) - ,round(mon_sales1/mon_sales2,2) - ,round(tue_sales1/tue_sales2,2) - ,round(wed_sales1/wed_sales2,2) - ,round(thu_sales1/thu_sales2,2) - ,round(fri_sales1/fri_sales2,2) - ,round(sat_sales1/sat_sales2,2) - from - (select wswscs.d_week_seq d_week_seq1 - ,sun_sales sun_sales1 - ,mon_sales mon_sales1 - ,tue_sales tue_sales1 - ,wed_sales wed_sales1 - ,thu_sales thu_sales1 - ,fri_sales fri_sales1 - ,sat_sales sat_sales1 - from wswscs,date_dim - where date_dim.d_week_seq = wswscs.d_week_seq and - d_year = 1998) y, - (select wswscs.d_week_seq d_week_seq2 - ,sun_sales sun_sales2 - ,mon_sales mon_sales2 - ,tue_sales tue_sales2 - ,wed_sales wed_sales2 - ,thu_sales thu_sales2 - ,fri_sales fri_sales2 - ,sat_sales sat_sales2 - from wswscs - ,date_dim - where date_dim.d_week_seq = wswscs.d_week_seq and - d_year = 1998+1) z - where d_week_seq1=d_week_seq2-53 - order by d_week_seq1; - --- end query 2 in stream 0 using template query2.tpl --- start query 3 in stream 0 using template query3.tpl -select dt.d_year - ,item.i_brand_id brand_id - ,item.i_brand brand - ,sum(ss_sales_price) sum_agg - from date_dim dt - ,store_sales - ,item - where dt.d_date_sk = store_sales.ss_sold_date_sk - and store_sales.ss_item_sk = item.i_item_sk - and item.i_manufact_id = 816 - and dt.d_moy=11 - group by dt.d_year - ,item.i_brand - ,item.i_brand_id - order by dt.d_year - ,sum_agg desc - ,brand_id - limit 100; - --- end query 3 in stream 0 using template query3.tpl --- start query 4 in stream 0 using template query4.tpl -with year_total as ( - select c_customer_id customer_id - ,c_first_name customer_first_name - ,c_last_name customer_last_name - ,c_preferred_cust_flag customer_preferred_cust_flag - ,c_birth_country customer_birth_country - ,c_login customer_login - ,c_email_address customer_email_address - ,d_year dyear - ,sum(((ss_ext_list_price-ss_ext_wholesale_cost-ss_ext_discount_amt)+ss_ext_sales_price)/2) year_total - ,'s' sale_type - from customer - ,store_sales - ,date_dim - where c_customer_sk = ss_customer_sk - and ss_sold_date_sk = d_date_sk - group by c_customer_id - ,c_first_name - ,c_last_name - ,c_preferred_cust_flag - ,c_birth_country - ,c_login - ,c_email_address - ,d_year - union all - select c_customer_id customer_id - ,c_first_name customer_first_name - ,c_last_name customer_last_name - ,c_preferred_cust_flag customer_preferred_cust_flag - ,c_birth_country customer_birth_country - ,c_login customer_login - ,c_email_address customer_email_address - ,d_year dyear - ,sum((((cs_ext_list_price-cs_ext_wholesale_cost-cs_ext_discount_amt)+cs_ext_sales_price)/2) ) year_total - ,'c' sale_type - from customer - ,catalog_sales - ,date_dim - where c_customer_sk = cs_bill_customer_sk - and cs_sold_date_sk = d_date_sk - group by c_customer_id - ,c_first_name - ,c_last_name - ,c_preferred_cust_flag - ,c_birth_country - ,c_login - ,c_email_address - ,d_year -union all - select c_customer_id customer_id - ,c_first_name customer_first_name - ,c_last_name customer_last_name - ,c_preferred_cust_flag customer_preferred_cust_flag - ,c_birth_country customer_birth_country - ,c_login customer_login - ,c_email_address customer_email_address - ,d_year dyear - ,sum((((ws_ext_list_price-ws_ext_wholesale_cost-ws_ext_discount_amt)+ws_ext_sales_price)/2) ) year_total - ,'w' sale_type - from customer - ,web_sales - ,date_dim - where c_customer_sk = ws_bill_customer_sk - and ws_sold_date_sk = d_date_sk - group by c_customer_id - ,c_first_name - ,c_last_name - ,c_preferred_cust_flag - ,c_birth_country - ,c_login - ,c_email_address - ,d_year - ) - select - t_s_secyear.customer_id - ,t_s_secyear.customer_first_name - ,t_s_secyear.customer_last_name - ,t_s_secyear.customer_birth_country - from year_total t_s_firstyear - ,year_total t_s_secyear - ,year_total t_c_firstyear - ,year_total t_c_secyear - ,year_total t_w_firstyear - ,year_total t_w_secyear - where t_s_secyear.customer_id = t_s_firstyear.customer_id - and t_s_firstyear.customer_id = t_c_secyear.customer_id - and t_s_firstyear.customer_id = t_c_firstyear.customer_id - and t_s_firstyear.customer_id = t_w_firstyear.customer_id - and t_s_firstyear.customer_id = t_w_secyear.customer_id - and t_s_firstyear.sale_type = 's' - and t_c_firstyear.sale_type = 'c' - and t_w_firstyear.sale_type = 'w' - and t_s_secyear.sale_type = 's' - and t_c_secyear.sale_type = 'c' - and t_w_secyear.sale_type = 'w' - and t_s_firstyear.dyear = 1999 - and t_s_secyear.dyear = 1999+1 - and t_c_firstyear.dyear = 1999 - and t_c_secyear.dyear = 1999+1 - and t_w_firstyear.dyear = 1999 - and t_w_secyear.dyear = 1999+1 - and t_s_firstyear.year_total > 0 - and t_c_firstyear.year_total > 0 - and t_w_firstyear.year_total > 0 - and case when t_c_firstyear.year_total > 0 then t_c_secyear.year_total / t_c_firstyear.year_total else null end - > case when t_s_firstyear.year_total > 0 then t_s_secyear.year_total / t_s_firstyear.year_total else null end - and case when t_c_firstyear.year_total > 0 then t_c_secyear.year_total / t_c_firstyear.year_total else null end - > case when t_w_firstyear.year_total > 0 then t_w_secyear.year_total / t_w_firstyear.year_total else null end - order by t_s_secyear.customer_id - ,t_s_secyear.customer_first_name - ,t_s_secyear.customer_last_name - ,t_s_secyear.customer_birth_country -limit 100; - --- end query 4 in stream 0 using template query4.tpl --- start query 5 in stream 0 using template query5.tpl -with ssr as - (select s_store_id, - sum(sales_price) as sales, - sum(profit) as profit, - sum(return_amt) as returns, - sum(net_loss) as profit_loss - from - ( select ss_store_sk as store_sk, - ss_sold_date_sk as date_sk, - ss_ext_sales_price as sales_price, - ss_net_profit as profit, - cast(0 as decimal(7,2)) as return_amt, - cast(0 as decimal(7,2)) as net_loss - from store_sales - union all - select sr_store_sk as store_sk, - sr_returned_date_sk as date_sk, - cast(0 as decimal(7,2)) as sales_price, - cast(0 as decimal(7,2)) as profit, - sr_return_amt as return_amt, - sr_net_loss as net_loss - from store_returns - ) salesreturns, - date_dim, - store - where date_sk = d_date_sk - and d_date between cast('2000-08-19' as date) - and (cast('2000-08-19' as date) + interval '14 days') - and store_sk = s_store_sk - group by s_store_id) - , - csr as - (select cp_catalog_page_id, - sum(sales_price) as sales, - sum(profit) as profit, - sum(return_amt) as returns, - sum(net_loss) as profit_loss - from - ( select cs_catalog_page_sk as page_sk, - cs_sold_date_sk as date_sk, - cs_ext_sales_price as sales_price, - cs_net_profit as profit, - cast(0 as decimal(7,2)) as return_amt, - cast(0 as decimal(7,2)) as net_loss - from catalog_sales - union all - select cr_catalog_page_sk as page_sk, - cr_returned_date_sk as date_sk, - cast(0 as decimal(7,2)) as sales_price, - cast(0 as decimal(7,2)) as profit, - cr_return_amount as return_amt, - cr_net_loss as net_loss - from catalog_returns - ) salesreturns, - date_dim, - catalog_page - where date_sk = d_date_sk - and d_date between cast('2000-08-19' as date) - and (cast('2000-08-19' as date) + interval '14 days') - and page_sk = cp_catalog_page_sk - group by cp_catalog_page_id) - , - wsr as - (select web_site_id, - sum(sales_price) as sales, - sum(profit) as profit, - sum(return_amt) as returns, - sum(net_loss) as profit_loss - from - ( select ws_web_site_sk as wsr_web_site_sk, - ws_sold_date_sk as date_sk, - ws_ext_sales_price as sales_price, - ws_net_profit as profit, - cast(0 as decimal(7,2)) as return_amt, - cast(0 as decimal(7,2)) as net_loss - from web_sales - union all - select ws_web_site_sk as wsr_web_site_sk, - wr_returned_date_sk as date_sk, - cast(0 as decimal(7,2)) as sales_price, - cast(0 as decimal(7,2)) as profit, - wr_return_amt as return_amt, - wr_net_loss as net_loss - from web_returns left outer join web_sales on - ( wr_item_sk = ws_item_sk - and wr_order_number = ws_order_number) - ) salesreturns, - date_dim, - web_site - where date_sk = d_date_sk - and d_date between cast('2000-08-19' as date) - and (cast('2000-08-19' as date) + interval '14 days') - and wsr_web_site_sk = web_site_sk - group by web_site_id) - select channel - , id - , sum(sales) as sales - , sum(returns) as returns - , sum(profit) as profit - from - (select 'store channel' as channel - , 'store' || s_store_id as id - , sales - , returns - , (profit - profit_loss) as profit - from ssr - union all - select 'catalog channel' as channel - , 'catalog_page' || cp_catalog_page_id as id - , sales - , returns - , (profit - profit_loss) as profit - from csr - union all - select 'web channel' as channel - , 'web_site' || web_site_id as id - , sales - , returns - , (profit - profit_loss) as profit - from wsr - ) x - group by rollup (channel, id) - order by channel - ,id - limit 100; - --- end query 5 in stream 0 using template query5.tpl --- start query 6 in stream 0 using template query6.tpl -select a.ca_state state, count(*) cnt - from customer_address a - ,customer c - ,store_sales s - ,date_dim d - ,item i - where a.ca_address_sk = c.c_current_addr_sk - and c.c_customer_sk = s.ss_customer_sk - and s.ss_sold_date_sk = d.d_date_sk - and s.ss_item_sk = i.i_item_sk - and d.d_month_seq = - (select distinct (d_month_seq) - from date_dim - where d_year = 2002 - and d_moy = 3 ) - and i.i_current_price > 1.2 * - (select avg(j.i_current_price) - from item j - where j.i_category = i.i_category) - group by a.ca_state - having count(*) >= 10 - order by cnt, a.ca_state - limit 100; - --- end query 6 in stream 0 using template query6.tpl --- start query 7 in stream 0 using template query7.tpl -select i_item_id, - avg(ss_quantity) agg1, - avg(ss_list_price) agg2, - avg(ss_coupon_amt) agg3, - avg(ss_sales_price) agg4 - from store_sales, customer_demographics, date_dim, item, promotion - where ss_sold_date_sk = d_date_sk and - ss_item_sk = i_item_sk and - ss_cdemo_sk = cd_demo_sk and - ss_promo_sk = p_promo_sk and - cd_gender = 'F' and - cd_marital_status = 'W' and - cd_education_status = 'College' and - (p_channel_email = 'N' or p_channel_event = 'N') and - d_year = 2001 - group by i_item_id - order by i_item_id - limit 100; - --- end query 7 in stream 0 using template query7.tpl --- start query 8 in stream 0 using template query8.tpl -select s_store_name - ,sum(ss_net_profit) - from store_sales - ,date_dim - ,store, - (select ca_zip - from ( - SELECT substr(ca_zip,1,5) ca_zip - FROM customer_address - WHERE substr(ca_zip,1,5) IN ( - '47602','16704','35863','28577','83910','36201', - '58412','48162','28055','41419','80332', - '38607','77817','24891','16226','18410', - '21231','59345','13918','51089','20317', - '17167','54585','67881','78366','47770', - '18360','51717','73108','14440','21800', - '89338','45859','65501','34948','25973', - '73219','25333','17291','10374','18829', - '60736','82620','41351','52094','19326', - '25214','54207','40936','21814','79077', - '25178','75742','77454','30621','89193', - '27369','41232','48567','83041','71948', - '37119','68341','14073','16891','62878', - '49130','19833','24286','27700','40979', - '50412','81504','94835','84844','71954', - '39503','57649','18434','24987','12350', - '86379','27413','44529','98569','16515', - '27287','24255','21094','16005','56436', - '91110','68293','56455','54558','10298', - '83647','32754','27052','51766','19444', - '13869','45645','94791','57631','20712', - '37788','41807','46507','21727','71836', - '81070','50632','88086','63991','20244', - '31655','51782','29818','63792','68605', - '94898','36430','57025','20601','82080', - '33869','22728','35834','29086','92645', - '98584','98072','11652','78093','57553', - '43830','71144','53565','18700','90209', - '71256','38353','54364','28571','96560', - '57839','56355','50679','45266','84680', - '34306','34972','48530','30106','15371', - '92380','84247','92292','68852','13338', - '34594','82602','70073','98069','85066', - '47289','11686','98862','26217','47529', - '63294','51793','35926','24227','14196', - '24594','32489','99060','49472','43432', - '49211','14312','88137','47369','56877', - '20534','81755','15794','12318','21060', - '73134','41255','63073','81003','73873', - '66057','51184','51195','45676','92696', - '70450','90669','98338','25264','38919', - '59226','58581','60298','17895','19489', - '52301','80846','95464','68770','51634', - '19988','18367','18421','11618','67975', - '25494','41352','95430','15734','62585', - '97173','33773','10425','75675','53535', - '17879','41967','12197','67998','79658', - '59130','72592','14851','43933','68101', - '50636','25717','71286','24660','58058', - '72991','95042','15543','33122','69280', - '11912','59386','27642','65177','17672', - '33467','64592','36335','54010','18767', - '63193','42361','49254','33113','33159', - '36479','59080','11855','81963','31016', - '49140','29392','41836','32958','53163', - '13844','73146','23952','65148','93498', - '14530','46131','58454','13376','13378', - '83986','12320','17193','59852','46081', - '98533','52389','13086','68843','31013', - '13261','60560','13443','45533','83583', - '11489','58218','19753','22911','25115', - '86709','27156','32669','13123','51933', - '39214','41331','66943','14155','69998', - '49101','70070','35076','14242','73021', - '59494','15782','29752','37914','74686', - '83086','34473','15751','81084','49230', - '91894','60624','17819','28810','63180', - '56224','39459','55233','75752','43639', - '55349','86057','62361','50788','31830', - '58062','18218','85761','60083','45484', - '21204','90229','70041','41162','35390', - '16364','39500','68908','26689','52868', - '81335','40146','11340','61527','61794', - '71997','30415','59004','29450','58117', - '69952','33562','83833','27385','61860', - '96435','48333','23065','32961','84919', - '61997','99132','22815','56600','68730', - '48017','95694','32919','88217','27116', - '28239','58032','18884','16791','21343', - '97462','18569','75660','15475') - intersect - select ca_zip - from (SELECT substr(ca_zip,1,5) ca_zip,count(*) cnt - FROM customer_address, customer - WHERE ca_address_sk = c_current_addr_sk and - c_preferred_cust_flag='Y' - group by ca_zip - having count(*) > 10)A1)A2) V1 - where ss_store_sk = s_store_sk - and ss_sold_date_sk = d_date_sk - and d_qoy = 2 and d_year = 1998 - and (substr(s_zip,1,2) = substr(V1.ca_zip,1,2)) - group by s_store_name - order by s_store_name - limit 100; - --- end query 8 in stream 0 using template query8.tpl --- start query 9 in stream 0 using template query9.tpl -select case when (select count(*) - from store_sales - where ss_quantity between 1 and 20) > 1071 - then (select avg(ss_ext_tax) - from store_sales - where ss_quantity between 1 and 20) - else (select avg(ss_net_paid_inc_tax) - from store_sales - where ss_quantity between 1 and 20) end bucket1 , - case when (select count(*) - from store_sales - where ss_quantity between 21 and 40) > 39161 - then (select avg(ss_ext_tax) - from store_sales - where ss_quantity between 21 and 40) - else (select avg(ss_net_paid_inc_tax) - from store_sales - where ss_quantity between 21 and 40) end bucket2, - case when (select count(*) - from store_sales - where ss_quantity between 41 and 60) > 29434 - then (select avg(ss_ext_tax) - from store_sales - where ss_quantity between 41 and 60) - else (select avg(ss_net_paid_inc_tax) - from store_sales - where ss_quantity between 41 and 60) end bucket3, - case when (select count(*) - from store_sales - where ss_quantity between 61 and 80) > 6568 - then (select avg(ss_ext_tax) - from store_sales - where ss_quantity between 61 and 80) - else (select avg(ss_net_paid_inc_tax) - from store_sales - where ss_quantity between 61 and 80) end bucket4, - case when (select count(*) - from store_sales - where ss_quantity between 81 and 100) > 21216 - then (select avg(ss_ext_tax) - from store_sales - where ss_quantity between 81 and 100) - else (select avg(ss_net_paid_inc_tax) - from store_sales - where ss_quantity between 81 and 100) end bucket5 -from reason -where r_reason_sk = 1 -; - --- end query 9 in stream 0 using template query9.tpl --- start query 10 in stream 0 using template query10.tpl -select - cd_gender, - cd_marital_status, - cd_education_status, - count(*) cnt1, - cd_purchase_estimate, - count(*) cnt2, - cd_credit_rating, - count(*) cnt3, - cd_dep_count, - count(*) cnt4, - cd_dep_employed_count, - count(*) cnt5, - cd_dep_college_count, - count(*) cnt6 - from - customer c,customer_address ca,customer_demographics - where - c.c_current_addr_sk = ca.ca_address_sk and - ca_county in ('Fairfield County','Campbell County','Washtenaw County','Escambia County','Cleburne County') and - cd_demo_sk = c.c_current_cdemo_sk and - exists (select * - from store_sales,date_dim - where c.c_customer_sk = ss_customer_sk and - ss_sold_date_sk = d_date_sk and - d_year = 2001 and - d_moy between 3 and 3+3) and - (exists (select * - from web_sales,date_dim - where c.c_customer_sk = ws_bill_customer_sk and - ws_sold_date_sk = d_date_sk and - d_year = 2001 and - d_moy between 3 ANd 3+3) or - exists (select * - from catalog_sales,date_dim - where c.c_customer_sk = cs_ship_customer_sk and - cs_sold_date_sk = d_date_sk and - d_year = 2001 and - d_moy between 3 and 3+3)) - group by cd_gender, - cd_marital_status, - cd_education_status, - cd_purchase_estimate, - cd_credit_rating, - cd_dep_count, - cd_dep_employed_count, - cd_dep_college_count - order by cd_gender, - cd_marital_status, - cd_education_status, - cd_purchase_estimate, - cd_credit_rating, - cd_dep_count, - cd_dep_employed_count, - cd_dep_college_count -limit 100; - --- end query 10 in stream 0 using template query10.tpl --- start query 11 in stream 0 using template query11.tpl -with year_total as ( - select c_customer_id customer_id - ,c_first_name customer_first_name - ,c_last_name customer_last_name - ,c_preferred_cust_flag customer_preferred_cust_flag - ,c_birth_country customer_birth_country - ,c_login customer_login - ,c_email_address customer_email_address - ,d_year dyear - ,sum(ss_ext_list_price-ss_ext_discount_amt) year_total - ,'s' sale_type - from customer - ,store_sales - ,date_dim - where c_customer_sk = ss_customer_sk - and ss_sold_date_sk = d_date_sk - group by c_customer_id - ,c_first_name - ,c_last_name - ,c_preferred_cust_flag - ,c_birth_country - ,c_login - ,c_email_address - ,d_year - union all - select c_customer_id customer_id - ,c_first_name customer_first_name - ,c_last_name customer_last_name - ,c_preferred_cust_flag customer_preferred_cust_flag - ,c_birth_country customer_birth_country - ,c_login customer_login - ,c_email_address customer_email_address - ,d_year dyear - ,sum(ws_ext_list_price-ws_ext_discount_amt) year_total - ,'w' sale_type - from customer - ,web_sales - ,date_dim - where c_customer_sk = ws_bill_customer_sk - and ws_sold_date_sk = d_date_sk - group by c_customer_id - ,c_first_name - ,c_last_name - ,c_preferred_cust_flag - ,c_birth_country - ,c_login - ,c_email_address - ,d_year - ) - select - t_s_secyear.customer_id - ,t_s_secyear.customer_first_name - ,t_s_secyear.customer_last_name - ,t_s_secyear.customer_email_address - from year_total t_s_firstyear - ,year_total t_s_secyear - ,year_total t_w_firstyear - ,year_total t_w_secyear - where t_s_secyear.customer_id = t_s_firstyear.customer_id - and t_s_firstyear.customer_id = t_w_secyear.customer_id - and t_s_firstyear.customer_id = t_w_firstyear.customer_id - and t_s_firstyear.sale_type = 's' - and t_w_firstyear.sale_type = 'w' - and t_s_secyear.sale_type = 's' - and t_w_secyear.sale_type = 'w' - and t_s_firstyear.dyear = 1998 - and t_s_secyear.dyear = 1998+1 - and t_w_firstyear.dyear = 1998 - and t_w_secyear.dyear = 1998+1 - and t_s_firstyear.year_total > 0 - and t_w_firstyear.year_total > 0 - and case when t_w_firstyear.year_total > 0 then t_w_secyear.year_total / t_w_firstyear.year_total else 0.0 end - > case when t_s_firstyear.year_total > 0 then t_s_secyear.year_total / t_s_firstyear.year_total else 0.0 end - order by t_s_secyear.customer_id - ,t_s_secyear.customer_first_name - ,t_s_secyear.customer_last_name - ,t_s_secyear.customer_email_address -limit 100; - --- end query 11 in stream 0 using template query11.tpl --- start query 12 in stream 0 using template query12.tpl -select i_item_id - ,i_item_desc - ,i_category - ,i_class - ,i_current_price - ,sum(ws_ext_sales_price) as itemrevenue - ,sum(ws_ext_sales_price)*100/sum(sum(ws_ext_sales_price)) over - (partition by i_class) as revenueratio -from - web_sales - ,item - ,date_dim -where - ws_item_sk = i_item_sk - and i_category in ('Men', 'Books', 'Electronics') - and ws_sold_date_sk = d_date_sk - and d_date between cast('2001-06-15' as date) - and (cast('2001-06-15' as date) + interval '30 days') -group by - i_item_id - ,i_item_desc - ,i_category - ,i_class - ,i_current_price -order by - i_category - ,i_class - ,i_item_id - ,i_item_desc - ,revenueratio -limit 100; - --- end query 12 in stream 0 using template query12.tpl --- start query 13 in stream 0 using template query13.tpl -select avg(ss_quantity) - ,avg(ss_ext_sales_price) - ,avg(ss_ext_wholesale_cost) - ,sum(ss_ext_wholesale_cost) - from store_sales - ,store - ,customer_demographics - ,household_demographics - ,customer_address - ,date_dim - where s_store_sk = ss_store_sk - and ss_sold_date_sk = d_date_sk and d_year = 2001 - and((ss_hdemo_sk=hd_demo_sk - and cd_demo_sk = ss_cdemo_sk - and cd_marital_status = 'M' - and cd_education_status = 'College' - and ss_sales_price between 100.00 and 150.00 - and hd_dep_count = 3 - )or - (ss_hdemo_sk=hd_demo_sk - and cd_demo_sk = ss_cdemo_sk - and cd_marital_status = 'D' - and cd_education_status = 'Primary' - and ss_sales_price between 50.00 and 100.00 - and hd_dep_count = 1 - ) or - (ss_hdemo_sk=hd_demo_sk - and cd_demo_sk = ss_cdemo_sk - and cd_marital_status = 'W' - and cd_education_status = '2 yr Degree' - and ss_sales_price between 150.00 and 200.00 - and hd_dep_count = 1 - )) - and((ss_addr_sk = ca_address_sk - and ca_country = 'United States' - and ca_state in ('IL', 'TN', 'TX') - and ss_net_profit between 100 and 200 - ) or - (ss_addr_sk = ca_address_sk - and ca_country = 'United States' - and ca_state in ('WY', 'OH', 'ID') - and ss_net_profit between 150 and 300 - ) or - (ss_addr_sk = ca_address_sk - and ca_country = 'United States' - and ca_state in ('MS', 'SC', 'IA') - and ss_net_profit between 50 and 250 - )) -; - --- end query 13 in stream 0 using template query13.tpl --- start query 14 in stream 0 using template query14.tpl -with cross_items as - (select i_item_sk ss_item_sk - from item, - (select iss.i_brand_id brand_id - ,iss.i_class_id class_id - ,iss.i_category_id category_id - from store_sales - ,item iss - ,date_dim d1 - where ss_item_sk = iss.i_item_sk - and ss_sold_date_sk = d1.d_date_sk - and d1.d_year between 1999 AND 1999 + 2 - intersect - select ics.i_brand_id - ,ics.i_class_id - ,ics.i_category_id - from catalog_sales - ,item ics - ,date_dim d2 - where cs_item_sk = ics.i_item_sk - and cs_sold_date_sk = d2.d_date_sk - and d2.d_year between 1999 AND 1999 + 2 - intersect - select iws.i_brand_id - ,iws.i_class_id - ,iws.i_category_id - from web_sales - ,item iws - ,date_dim d3 - where ws_item_sk = iws.i_item_sk - and ws_sold_date_sk = d3.d_date_sk - and d3.d_year between 1999 AND 1999 + 2) as sub - where i_brand_id = brand_id - and i_class_id = class_id - and i_category_id = category_id -), - avg_sales as - (select avg(quantity*list_price) average_sales - from (select ss_quantity quantity - ,ss_list_price list_price - from store_sales - ,date_dim - where ss_sold_date_sk = d_date_sk - and d_year between 1999 and 1999 + 2 - union all - select cs_quantity quantity - ,cs_list_price list_price - from catalog_sales - ,date_dim - where cs_sold_date_sk = d_date_sk - and d_year between 1999 and 1999 + 2 - union all - select ws_quantity quantity - ,ws_list_price list_price - from web_sales - ,date_dim - where ws_sold_date_sk = d_date_sk - and d_year between 1999 and 1999 + 2) as x) - select channel, i_brand_id,i_class_id,i_category_id,sum(sales), sum(number_sales) - from( - select 'store' channel, i_brand_id,i_class_id - ,i_category_id,sum(ss_quantity*ss_list_price) sales - , count(*) number_sales - from store_sales - ,item - ,date_dim - where ss_item_sk in (select ss_item_sk from cross_items) - and ss_item_sk = i_item_sk - and ss_sold_date_sk = d_date_sk - and d_year = 1999+2 - and d_moy = 11 - group by i_brand_id,i_class_id,i_category_id - having sum(ss_quantity*ss_list_price) > (select average_sales from avg_sales) - union all - select 'catalog' channel, i_brand_id,i_class_id,i_category_id, sum(cs_quantity*cs_list_price) sales, count(*) number_sales - from catalog_sales - ,item - ,date_dim - where cs_item_sk in (select ss_item_sk from cross_items) - and cs_item_sk = i_item_sk - and cs_sold_date_sk = d_date_sk - and d_year = 1999+2 - and d_moy = 11 - group by i_brand_id,i_class_id,i_category_id - having sum(cs_quantity*cs_list_price) > (select average_sales from avg_sales) - union all - select 'web' channel, i_brand_id,i_class_id,i_category_id, sum(ws_quantity*ws_list_price) sales , count(*) number_sales - from web_sales - ,item - ,date_dim - where ws_item_sk in (select ss_item_sk from cross_items) - and ws_item_sk = i_item_sk - and ws_sold_date_sk = d_date_sk - and d_year = 1999+2 - and d_moy = 11 - group by i_brand_id,i_class_id,i_category_id - having sum(ws_quantity*ws_list_price) > (select average_sales from avg_sales) - ) y - group by rollup (channel, i_brand_id,i_class_id,i_category_id) - order by channel,i_brand_id,i_class_id,i_category_id - limit 100; - -with cross_items as - (select i_item_sk ss_item_sk - from item, - (select iss.i_brand_id brand_id - ,iss.i_class_id class_id - ,iss.i_category_id category_id - from store_sales - ,item iss - ,date_dim d1 - where ss_item_sk = iss.i_item_sk - and ss_sold_date_sk = d1.d_date_sk - and d1.d_year between 1999 AND 1999 + 2 - intersect - select ics.i_brand_id - ,ics.i_class_id - ,ics.i_category_id - from catalog_sales - ,item ics - ,date_dim d2 - where cs_item_sk = ics.i_item_sk - and cs_sold_date_sk = d2.d_date_sk - and d2.d_year between 1999 AND 1999 + 2 - intersect - select iws.i_brand_id - ,iws.i_class_id - ,iws.i_category_id - from web_sales - ,item iws - ,date_dim d3 - where ws_item_sk = iws.i_item_sk - and ws_sold_date_sk = d3.d_date_sk - and d3.d_year between 1999 AND 1999 + 2) as x - where i_brand_id = brand_id - and i_class_id = class_id - and i_category_id = category_id -), - avg_sales as -(select avg(quantity*list_price) average_sales - from (select ss_quantity quantity - ,ss_list_price list_price - from store_sales - ,date_dim - where ss_sold_date_sk = d_date_sk - and d_year between 1999 and 1999 + 2 - union all - select cs_quantity quantity - ,cs_list_price list_price - from catalog_sales - ,date_dim - where cs_sold_date_sk = d_date_sk - and d_year between 1999 and 1999 + 2 - union all - select ws_quantity quantity - ,ws_list_price list_price - from web_sales - ,date_dim - where ws_sold_date_sk = d_date_sk - and d_year between 1999 and 1999 + 2) as x) - select this_year.channel ty_channel - ,this_year.i_brand_id ty_brand - ,this_year.i_class_id ty_class - ,this_year.i_category_id ty_category - ,this_year.sales ty_sales - ,this_year.number_sales ty_number_sales - ,last_year.channel ly_channel - ,last_year.i_brand_id ly_brand - ,last_year.i_class_id ly_class - ,last_year.i_category_id ly_category - ,last_year.sales ly_sales - ,last_year.number_sales ly_number_sales - from - (select 'store' channel, i_brand_id,i_class_id,i_category_id - ,sum(ss_quantity*ss_list_price) sales, count(*) number_sales - from store_sales - ,item - ,date_dim - where ss_item_sk in (select ss_item_sk from cross_items) - and ss_item_sk = i_item_sk - and ss_sold_date_sk = d_date_sk - and d_week_seq = (select d_week_seq - from date_dim - where d_year = 1999 + 1 - and d_moy = 12 - and d_dom = 3) - group by i_brand_id,i_class_id,i_category_id - having sum(ss_quantity*ss_list_price) > (select average_sales from avg_sales)) this_year, - (select 'store' channel, i_brand_id,i_class_id - ,i_category_id, sum(ss_quantity*ss_list_price) sales, count(*) number_sales - from store_sales - ,item - ,date_dim - where ss_item_sk in (select ss_item_sk from cross_items) - and ss_item_sk = i_item_sk - and ss_sold_date_sk = d_date_sk - and d_week_seq = (select d_week_seq - from date_dim - where d_year = 1999 - and d_moy = 12 - and d_dom = 3) - group by i_brand_id,i_class_id,i_category_id - having sum(ss_quantity*ss_list_price) > (select average_sales from avg_sales)) last_year - where this_year.i_brand_id= last_year.i_brand_id - and this_year.i_class_id = last_year.i_class_id - and this_year.i_category_id = last_year.i_category_id - order by this_year.channel, this_year.i_brand_id, this_year.i_class_id, this_year.i_category_id - limit 100; - --- end query 14 in stream 0 using template query14.tpl --- start query 15 in stream 0 using template query15.tpl -select ca_zip - ,sum(cs_sales_price) - from catalog_sales - ,customer - ,customer_address - ,date_dim - where cs_bill_customer_sk = c_customer_sk - and c_current_addr_sk = ca_address_sk - and ( substr(ca_zip,1,5) in ('85669', '86197','88274','83405','86475', - '85392', '85460', '80348', '81792') - or ca_state in ('CA','WA','GA') - or cs_sales_price > 500) - and cs_sold_date_sk = d_date_sk - and d_qoy = 2 and d_year = 2001 - group by ca_zip - order by ca_zip - limit 100; - --- end query 15 in stream 0 using template query15.tpl --- start query 16 in stream 0 using template query16.tpl -select - count(distinct cs_order_number) as "order count" - ,sum(cs_ext_ship_cost) as "total shipping cost" - ,sum(cs_net_profit) as "total net profit" -from - catalog_sales cs1 - ,date_dim - ,customer_address - ,call_center -where - d_date between '2002-4-01' and - (cast('2002-4-01' as date) + interval '60 days') -and cs1.cs_ship_date_sk = d_date_sk -and cs1.cs_ship_addr_sk = ca_address_sk -and ca_state = 'PA' -and cs1.cs_call_center_sk = cc_call_center_sk -and cc_county in ('Williamson County','Williamson County','Williamson County','Williamson County', - 'Williamson County' -) -and exists (select * - from catalog_sales cs2 - where cs1.cs_order_number = cs2.cs_order_number - and cs1.cs_warehouse_sk <> cs2.cs_warehouse_sk) -and not exists(select * - from catalog_returns cr1 - where cs1.cs_order_number = cr1.cr_order_number) -order by count(distinct cs_order_number) -limit 100; - --- end query 16 in stream 0 using template query16.tpl --- start query 17 in stream 0 using template query17.tpl -select i_item_id - ,i_item_desc - ,s_state - ,count(ss_quantity) as store_sales_quantitycount - ,avg(ss_quantity) as store_sales_quantityave - ,stddev_samp(ss_quantity) as store_sales_quantitystdev - ,stddev_samp(ss_quantity)/avg(ss_quantity) as store_sales_quantitycov - ,count(sr_return_quantity) as store_returns_quantitycount - ,avg(sr_return_quantity) as store_returns_quantityave - ,stddev_samp(sr_return_quantity) as store_returns_quantitystdev - ,stddev_samp(sr_return_quantity)/avg(sr_return_quantity) as store_returns_quantitycov - ,count(cs_quantity) as catalog_sales_quantitycount ,avg(cs_quantity) as catalog_sales_quantityave - ,stddev_samp(cs_quantity) as catalog_sales_quantitystdev - ,stddev_samp(cs_quantity)/avg(cs_quantity) as catalog_sales_quantitycov - from store_sales - ,store_returns - ,catalog_sales - ,date_dim d1 - ,date_dim d2 - ,date_dim d3 - ,store - ,item - where d1.d_quarter_name = '2001Q1' - and d1.d_date_sk = ss_sold_date_sk - and i_item_sk = ss_item_sk - and s_store_sk = ss_store_sk - and ss_customer_sk = sr_customer_sk - and ss_item_sk = sr_item_sk - and ss_ticket_number = sr_ticket_number - and sr_returned_date_sk = d2.d_date_sk - and d2.d_quarter_name in ('2001Q1','2001Q2','2001Q3') - and sr_customer_sk = cs_bill_customer_sk - and sr_item_sk = cs_item_sk - and cs_sold_date_sk = d3.d_date_sk - and d3.d_quarter_name in ('2001Q1','2001Q2','2001Q3') - group by i_item_id - ,i_item_desc - ,s_state - order by i_item_id - ,i_item_desc - ,s_state -limit 100; - --- end query 17 in stream 0 using template query17.tpl --- start query 18 in stream 0 using template query18.tpl -select i_item_id, - ca_country, - ca_state, - ca_county, - avg( cast(cs_quantity as decimal(12,2))) agg1, - avg( cast(cs_list_price as decimal(12,2))) agg2, - avg( cast(cs_coupon_amt as decimal(12,2))) agg3, - avg( cast(cs_sales_price as decimal(12,2))) agg4, - avg( cast(cs_net_profit as decimal(12,2))) agg5, - avg( cast(c_birth_year as decimal(12,2))) agg6, - avg( cast(cd1.cd_dep_count as decimal(12,2))) agg7 - from catalog_sales, customer_demographics cd1, - customer_demographics cd2, customer, customer_address, date_dim, item - where cs_sold_date_sk = d_date_sk and - cs_item_sk = i_item_sk and - cs_bill_cdemo_sk = cd1.cd_demo_sk and - cs_bill_customer_sk = c_customer_sk and - cd1.cd_gender = 'F' and - cd1.cd_education_status = 'Primary' and - c_current_cdemo_sk = cd2.cd_demo_sk and - c_current_addr_sk = ca_address_sk and - c_birth_month in (1,3,7,11,10,4) and - d_year = 2001 and - ca_state in ('AL','MO','TN' - ,'GA','MT','IN','CA') - group by rollup (i_item_id, ca_country, ca_state, ca_county) - order by ca_country, - ca_state, - ca_county, - i_item_id - limit 100; - --- end query 18 in stream 0 using template query18.tpl --- start query 19 in stream 0 using template query19.tpl -select i_brand_id brand_id, i_brand brand, i_manufact_id, i_manufact, - sum(ss_ext_sales_price) ext_price - from date_dim, store_sales, item,customer,customer_address,store - where d_date_sk = ss_sold_date_sk - and ss_item_sk = i_item_sk - and i_manager_id=14 - and d_moy=11 - and d_year=2002 - and ss_customer_sk = c_customer_sk - and c_current_addr_sk = ca_address_sk - and substr(ca_zip,1,5) <> substr(s_zip,1,5) - and ss_store_sk = s_store_sk - group by i_brand - ,i_brand_id - ,i_manufact_id - ,i_manufact - order by ext_price desc - ,i_brand - ,i_brand_id - ,i_manufact_id - ,i_manufact -limit 100 ; - --- end query 19 in stream 0 using template query19.tpl --- start query 20 in stream 0 using template query20.tpl -select i_item_id - ,i_item_desc - ,i_category - ,i_class - ,i_current_price - ,sum(cs_ext_sales_price) as itemrevenue - ,sum(cs_ext_sales_price)*100/sum(sum(cs_ext_sales_price)) over - (partition by i_class) as revenueratio - from catalog_sales - ,item - ,date_dim - where cs_item_sk = i_item_sk - and i_category in ('Books', 'Music', 'Sports') - and cs_sold_date_sk = d_date_sk - and d_date between cast('2002-06-18' as date) - and (cast('2002-06-18' as date) + interval '30 days') - group by i_item_id - ,i_item_desc - ,i_category - ,i_class - ,i_current_price - order by i_category - ,i_class - ,i_item_id - ,i_item_desc - ,revenueratio -limit 100; - --- end query 20 in stream 0 using template query20.tpl --- start query 21 in stream 0 using template query21.tpl -select * - from(select w_warehouse_name - ,i_item_id - ,sum(case when (cast(d_date as date) < cast ('1999-06-22' as date)) - then inv_quantity_on_hand - else 0 end) as inv_before - ,sum(case when (cast(d_date as date) >= cast ('1999-06-22' as date)) - then inv_quantity_on_hand - else 0 end) as inv_after - from inventory - ,warehouse - ,item - ,date_dim - where i_current_price between 0.99 and 1.49 - and i_item_sk = inv_item_sk - and inv_warehouse_sk = w_warehouse_sk - and inv_date_sk = d_date_sk - and d_date between (cast ('1999-06-22' as date) - interval '30 days') - and (cast ('1999-06-22' as date) + interval '30 days') - group by w_warehouse_name, i_item_id) x - where (case when inv_before > 0 - then inv_after / inv_before - else null - end) between 2.0/3.0 and 3.0/2.0 - order by w_warehouse_name - ,i_item_id - limit 100; - --- end query 21 in stream 0 using template query21.tpl --- start query 22 in stream 0 using template query22.tpl -select i_product_name - ,i_brand - ,i_class - ,i_category - ,avg(inv_quantity_on_hand) qoh - from inventory - ,date_dim - ,item - where inv_date_sk=d_date_sk - and inv_item_sk=i_item_sk - and d_month_seq between 1200 and 1200 + 11 - group by rollup(i_product_name - ,i_brand - ,i_class - ,i_category) -order by qoh, i_product_name, i_brand, i_class, i_category -limit 100; - --- end query 22 in stream 0 using template query22.tpl --- start query 23 in stream 0 using template query23.tpl -with frequent_ss_items as - (select substr(i_item_desc,1,30) itemdesc,i_item_sk item_sk,d_date solddate,count(*) cnt - from store_sales - ,date_dim - ,item - where ss_sold_date_sk = d_date_sk - and ss_item_sk = i_item_sk - and d_year in (2000,2000+1,2000+2,2000+3) - group by substr(i_item_desc,1,30),i_item_sk,d_date - having count(*) >4), - max_store_sales as - (select max(csales) tpcds_cmax - from (select c_customer_sk,sum(ss_quantity*ss_sales_price) as csales - from store_sales - ,customer - ,date_dim - where ss_customer_sk = c_customer_sk - and ss_sold_date_sk = d_date_sk - and d_year in (2000,2000+1,2000+2,2000+3) - group by c_customer_sk) as scsales - ), - best_ss_customer as - (select c_customer_sk,sum(ss_quantity*ss_sales_price) ssales - from store_sales - ,customer - where ss_customer_sk = c_customer_sk - group by c_customer_sk - having sum(ss_quantity*ss_sales_price) > (95/100.0) * (select - * -from - max_store_sales)) - select sum(sales) - from (select cs_quantity*cs_list_price sales - from catalog_sales - ,date_dim - where d_year = 2000 - and d_moy = 7 - and cs_sold_date_sk = d_date_sk - and cs_item_sk in (select item_sk from frequent_ss_items) - and cs_bill_customer_sk in (select c_customer_sk from best_ss_customer) - union all - select ws_quantity*ws_list_price sales - from web_sales - ,date_dim - where d_year = 2000 - and d_moy = 7 - and ws_sold_date_sk = d_date_sk - and ws_item_sk in (select item_sk from frequent_ss_items) - and ws_bill_customer_sk in (select c_customer_sk from best_ss_customer)) as foo - limit 100; - -with frequent_ss_items as - (select substr(i_item_desc,1,30) itemdesc,i_item_sk item_sk,d_date solddate,count(*) cnt - from store_sales - ,date_dim - ,item - where ss_sold_date_sk = d_date_sk - and ss_item_sk = i_item_sk - and d_year in (2000,2000 + 1,2000 + 2,2000 + 3) - group by substr(i_item_desc,1,30),i_item_sk,d_date - having count(*) >4), - max_store_sales as - (select max(csales) tpcds_cmax - from (select c_customer_sk,sum(ss_quantity*ss_sales_price) csales - from store_sales - ,customer - ,date_dim - where ss_customer_sk = c_customer_sk - and ss_sold_date_sk = d_date_sk - and d_year in (2000,2000+1,2000+2,2000+3) - group by c_customer_sk) as scsales), - best_ss_customer as - (select c_customer_sk,sum(ss_quantity*ss_sales_price) ssales - from store_sales - ,customer - where ss_customer_sk = c_customer_sk - group by c_customer_sk - having sum(ss_quantity*ss_sales_price) > (95/100.0) * (select - * - from max_store_sales)) - select c_last_name,c_first_name,sales - from (select c_last_name,c_first_name,sum(cs_quantity*cs_list_price) sales - from catalog_sales - ,customer - ,date_dim - where d_year = 2000 - and d_moy = 7 - and cs_sold_date_sk = d_date_sk - and cs_item_sk in (select item_sk from frequent_ss_items) - and cs_bill_customer_sk in (select c_customer_sk from best_ss_customer) - and cs_bill_customer_sk = c_customer_sk - group by c_last_name,c_first_name - union all - select c_last_name,c_first_name,sum(ws_quantity*ws_list_price) sales - from web_sales - ,customer - ,date_dim - where d_year = 2000 - and d_moy = 7 - and ws_sold_date_sk = d_date_sk - and ws_item_sk in (select item_sk from frequent_ss_items) - and ws_bill_customer_sk in (select c_customer_sk from best_ss_customer) - and ws_bill_customer_sk = c_customer_sk - group by c_last_name,c_first_name) as sub - order by c_last_name,c_first_name,sales - limit 100; - --- end query 23 in stream 0 using template query23.tpl --- start query 24 in stream 0 using template query24.tpl -with ssales as -(select c_last_name - ,c_first_name - ,s_store_name - ,ca_state - ,s_state - ,i_color - ,i_current_price - ,i_manager_id - ,i_units - ,i_size - ,sum(ss_net_paid) netpaid -from store_sales - ,store_returns - ,store - ,item - ,customer - ,customer_address -where ss_ticket_number = sr_ticket_number - and ss_item_sk = sr_item_sk - and ss_customer_sk = c_customer_sk - and ss_item_sk = i_item_sk - and ss_store_sk = s_store_sk - and c_current_addr_sk = ca_address_sk - and c_birth_country <> upper(ca_country) - and s_zip = ca_zip -and s_market_id=5 -group by c_last_name - ,c_first_name - ,s_store_name - ,ca_state - ,s_state - ,i_color - ,i_current_price - ,i_manager_id - ,i_units - ,i_size) -select c_last_name - ,c_first_name - ,s_store_name - ,sum(netpaid) paid -from ssales -where i_color = 'aquamarine' -group by c_last_name - ,c_first_name - ,s_store_name -having sum(netpaid) > (select 0.05*avg(netpaid) - from ssales) -order by c_last_name - ,c_first_name - ,s_store_name -; -with ssales as -(select c_last_name - ,c_first_name - ,s_store_name - ,ca_state - ,s_state - ,i_color - ,i_current_price - ,i_manager_id - ,i_units - ,i_size - ,sum(ss_net_paid) netpaid -from store_sales - ,store_returns - ,store - ,item - ,customer - ,customer_address -where ss_ticket_number = sr_ticket_number - and ss_item_sk = sr_item_sk - and ss_customer_sk = c_customer_sk - and ss_item_sk = i_item_sk - and ss_store_sk = s_store_sk - and c_current_addr_sk = ca_address_sk - and c_birth_country <> upper(ca_country) - and s_zip = ca_zip - and s_market_id = 5 -group by c_last_name - ,c_first_name - ,s_store_name - ,ca_state - ,s_state - ,i_color - ,i_current_price - ,i_manager_id - ,i_units - ,i_size) -select c_last_name - ,c_first_name - ,s_store_name - ,sum(netpaid) paid -from ssales -where i_color = 'seashell' -group by c_last_name - ,c_first_name - ,s_store_name -having sum(netpaid) > (select 0.05*avg(netpaid) - from ssales) -order by c_last_name - ,c_first_name - ,s_store_name -; - --- end query 24 in stream 0 using template query24.tpl --- start query 25 in stream 0 using template query25.tpl -select - i_item_id - ,i_item_desc - ,s_store_id - ,s_store_name - ,max(ss_net_profit) as store_sales_profit - ,max(sr_net_loss) as store_returns_loss - ,max(cs_net_profit) as catalog_sales_profit - from - store_sales - ,store_returns - ,catalog_sales - ,date_dim d1 - ,date_dim d2 - ,date_dim d3 - ,store - ,item - where - d1.d_moy = 4 - and d1.d_year = 1999 - and d1.d_date_sk = ss_sold_date_sk - and i_item_sk = ss_item_sk - and s_store_sk = ss_store_sk - and ss_customer_sk = sr_customer_sk - and ss_item_sk = sr_item_sk - and ss_ticket_number = sr_ticket_number - and sr_returned_date_sk = d2.d_date_sk - and d2.d_moy between 4 and 10 - and d2.d_year = 1999 - and sr_customer_sk = cs_bill_customer_sk - and sr_item_sk = cs_item_sk - and cs_sold_date_sk = d3.d_date_sk - and d3.d_moy between 4 and 10 - and d3.d_year = 1999 - group by - i_item_id - ,i_item_desc - ,s_store_id - ,s_store_name - order by - i_item_id - ,i_item_desc - ,s_store_id - ,s_store_name - limit 100; - --- end query 25 in stream 0 using template query25.tpl --- start query 26 in stream 0 using template query26.tpl -select i_item_id, - avg(cs_quantity) agg1, - avg(cs_list_price) agg2, - avg(cs_coupon_amt) agg3, - avg(cs_sales_price) agg4 - from catalog_sales, customer_demographics, date_dim, item, promotion - where cs_sold_date_sk = d_date_sk and - cs_item_sk = i_item_sk and - cs_bill_cdemo_sk = cd_demo_sk and - cs_promo_sk = p_promo_sk and - cd_gender = 'M' and - cd_marital_status = 'W' and - cd_education_status = 'Unknown' and - (p_channel_email = 'N' or p_channel_event = 'N') and - d_year = 2002 - group by i_item_id - order by i_item_id - limit 100; - --- end query 26 in stream 0 using template query26.tpl --- start query 27 in stream 0 using template query27.tpl -select i_item_id, - s_state, grouping(s_state) g_state, - avg(ss_quantity) agg1, - avg(ss_list_price) agg2, - avg(ss_coupon_amt) agg3, - avg(ss_sales_price) agg4 - from store_sales, customer_demographics, date_dim, store, item - where ss_sold_date_sk = d_date_sk and - ss_item_sk = i_item_sk and - ss_store_sk = s_store_sk and - ss_cdemo_sk = cd_demo_sk and - cd_gender = 'M' and - cd_marital_status = 'W' and - cd_education_status = 'Secondary' and - d_year = 1999 and - s_state in ('TN','TN', 'TN', 'TN', 'TN', 'TN') - group by rollup (i_item_id, s_state) - order by i_item_id - ,s_state - limit 100; - --- end query 27 in stream 0 using template query27.tpl --- start query 28 in stream 0 using template query28.tpl -select * -from (select avg(ss_list_price) B1_LP - ,count(ss_list_price) B1_CNT - ,count(distinct ss_list_price) B1_CNTD - from store_sales - where ss_quantity between 0 and 5 - and (ss_list_price between 107 and 107+10 - or ss_coupon_amt between 1319 and 1319+1000 - or ss_wholesale_cost between 60 and 60+20)) B1, - (select avg(ss_list_price) B2_LP - ,count(ss_list_price) B2_CNT - ,count(distinct ss_list_price) B2_CNTD - from store_sales - where ss_quantity between 6 and 10 - and (ss_list_price between 23 and 23+10 - or ss_coupon_amt between 825 and 825+1000 - or ss_wholesale_cost between 43 and 43+20)) B2, - (select avg(ss_list_price) B3_LP - ,count(ss_list_price) B3_CNT - ,count(distinct ss_list_price) B3_CNTD - from store_sales - where ss_quantity between 11 and 15 - and (ss_list_price between 74 and 74+10 - or ss_coupon_amt between 4381 and 4381+1000 - or ss_wholesale_cost between 57 and 57+20)) B3, - (select avg(ss_list_price) B4_LP - ,count(ss_list_price) B4_CNT - ,count(distinct ss_list_price) B4_CNTD - from store_sales - where ss_quantity between 16 and 20 - and (ss_list_price between 89 and 89+10 - or ss_coupon_amt between 3117 and 3117+1000 - or ss_wholesale_cost between 68 and 68+20)) B4, - (select avg(ss_list_price) B5_LP - ,count(ss_list_price) B5_CNT - ,count(distinct ss_list_price) B5_CNTD - from store_sales - where ss_quantity between 21 and 25 - and (ss_list_price between 58 and 58+10 - or ss_coupon_amt between 9402 and 9402+1000 - or ss_wholesale_cost between 38 and 38+20)) B5, - (select avg(ss_list_price) B6_LP - ,count(ss_list_price) B6_CNT - ,count(distinct ss_list_price) B6_CNTD - from store_sales - where ss_quantity between 26 and 30 - and (ss_list_price between 64 and 64+10 - or ss_coupon_amt between 5792 and 5792+1000 - or ss_wholesale_cost between 73 and 73+20)) B6 -limit 100; - --- end query 28 in stream 0 using template query28.tpl --- start query 29 in stream 0 using template query29.tpl -select - i_item_id - ,i_item_desc - ,s_store_id - ,s_store_name - ,max(ss_quantity) as store_sales_quantity - ,max(sr_return_quantity) as store_returns_quantity - ,max(cs_quantity) as catalog_sales_quantity - from - store_sales - ,store_returns - ,catalog_sales - ,date_dim d1 - ,date_dim d2 - ,date_dim d3 - ,store - ,item - where - d1.d_moy = 4 - and d1.d_year = 1998 - and d1.d_date_sk = ss_sold_date_sk - and i_item_sk = ss_item_sk - and s_store_sk = ss_store_sk - and ss_customer_sk = sr_customer_sk - and ss_item_sk = sr_item_sk - and ss_ticket_number = sr_ticket_number - and sr_returned_date_sk = d2.d_date_sk - and d2.d_moy between 4 and 4 + 3 - and d2.d_year = 1998 - and sr_customer_sk = cs_bill_customer_sk - and sr_item_sk = cs_item_sk - and cs_sold_date_sk = d3.d_date_sk - and d3.d_year in (1998,1998+1,1998+2) - group by - i_item_id - ,i_item_desc - ,s_store_id - ,s_store_name - order by - i_item_id - ,i_item_desc - ,s_store_id - ,s_store_name - limit 100; - --- end query 29 in stream 0 using template query29.tpl --- start query 30 in stream 0 using template query30.tpl -with customer_total_return as - (select wr_returning_customer_sk as ctr_customer_sk - ,ca_state as ctr_state, - sum(wr_return_amt) as ctr_total_return - from web_returns - ,date_dim - ,customer_address - where wr_returned_date_sk = d_date_sk - and d_year =2000 - and wr_returning_addr_sk = ca_address_sk - group by wr_returning_customer_sk - ,ca_state) - select c_customer_id,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag - ,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address - ,c_last_review_date_sk,ctr_total_return - from customer_total_return ctr1 - ,customer_address - ,customer - where ctr1.ctr_total_return > (select avg(ctr_total_return)*1.2 - from customer_total_return ctr2 - where ctr1.ctr_state = ctr2.ctr_state) - and ca_address_sk = c_current_addr_sk - and ca_state = 'AR' - and ctr1.ctr_customer_sk = c_customer_sk - order by c_customer_id,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag - ,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address - ,c_last_review_date_sk,ctr_total_return -limit 100; - --- end query 30 in stream 0 using template query30.tpl --- start query 31 in stream 0 using template query31.tpl -with ss as - (select ca_county,d_qoy, d_year,sum(ss_ext_sales_price) as store_sales - from store_sales,date_dim,customer_address - where ss_sold_date_sk = d_date_sk - and ss_addr_sk=ca_address_sk - group by ca_county,d_qoy, d_year), - ws as - (select ca_county,d_qoy, d_year,sum(ws_ext_sales_price) as web_sales - from web_sales,date_dim,customer_address - where ws_sold_date_sk = d_date_sk - and ws_bill_addr_sk=ca_address_sk - group by ca_county,d_qoy, d_year) - select - ss1.ca_county - ,ss1.d_year - ,ws2.web_sales/ws1.web_sales web_q1_q2_increase - ,ss2.store_sales/ss1.store_sales store_q1_q2_increase - ,ws3.web_sales/ws2.web_sales web_q2_q3_increase - ,ss3.store_sales/ss2.store_sales store_q2_q3_increase - from - ss ss1 - ,ss ss2 - ,ss ss3 - ,ws ws1 - ,ws ws2 - ,ws ws3 - where - ss1.d_qoy = 1 - and ss1.d_year = 1999 - and ss1.ca_county = ss2.ca_county - and ss2.d_qoy = 2 - and ss2.d_year = 1999 - and ss2.ca_county = ss3.ca_county - and ss3.d_qoy = 3 - and ss3.d_year = 1999 - and ss1.ca_county = ws1.ca_county - and ws1.d_qoy = 1 - and ws1.d_year = 1999 - and ws1.ca_county = ws2.ca_county - and ws2.d_qoy = 2 - and ws2.d_year = 1999 - and ws1.ca_county = ws3.ca_county - and ws3.d_qoy = 3 - and ws3.d_year =1999 - and case when ws1.web_sales > 0 then ws2.web_sales/ws1.web_sales else null end - > case when ss1.store_sales > 0 then ss2.store_sales/ss1.store_sales else null end - and case when ws2.web_sales > 0 then ws3.web_sales/ws2.web_sales else null end - > case when ss2.store_sales > 0 then ss3.store_sales/ss2.store_sales else null end - order by store_q2_q3_increase; - --- end query 31 in stream 0 using template query31.tpl --- start query 32 in stream 0 using template query32.tpl -select sum(cs_ext_discount_amt) as "excess discount amount" -from - catalog_sales - ,item - ,date_dim -where -i_manufact_id = 722 -and i_item_sk = cs_item_sk -and d_date between '2001-03-09' and - (cast('2001-03-09' as date) + interval '90 days') -and d_date_sk = cs_sold_date_sk -and cs_ext_discount_amt - > ( - select - 1.3 * avg(cs_ext_discount_amt) - from - catalog_sales - ,date_dim - where - cs_item_sk = i_item_sk - and d_date between '2001-03-09' and - (cast('2001-03-09' as date) + interval '90 days') - and d_date_sk = cs_sold_date_sk - ) -limit 100; - --- end query 32 in stream 0 using template query32.tpl --- start query 33 in stream 0 using template query33.tpl -with ss as ( - select - i_manufact_id,sum(ss_ext_sales_price) total_sales - from - store_sales, - date_dim, - customer_address, - item - where - i_manufact_id in (select - i_manufact_id -from - item -where i_category in ('Books')) - and ss_item_sk = i_item_sk - and ss_sold_date_sk = d_date_sk - and d_year = 2001 - and d_moy = 3 - and ss_addr_sk = ca_address_sk - and ca_gmt_offset = -5 - group by i_manufact_id), - cs as ( - select - i_manufact_id,sum(cs_ext_sales_price) total_sales - from - catalog_sales, - date_dim, - customer_address, - item - where - i_manufact_id in (select - i_manufact_id -from - item -where i_category in ('Books')) - and cs_item_sk = i_item_sk - and cs_sold_date_sk = d_date_sk - and d_year = 2001 - and d_moy = 3 - and cs_bill_addr_sk = ca_address_sk - and ca_gmt_offset = -5 - group by i_manufact_id), - ws as ( - select - i_manufact_id,sum(ws_ext_sales_price) total_sales - from - web_sales, - date_dim, - customer_address, - item - where - i_manufact_id in (select - i_manufact_id -from - item -where i_category in ('Books')) - and ws_item_sk = i_item_sk - and ws_sold_date_sk = d_date_sk - and d_year = 2001 - and d_moy = 3 - and ws_bill_addr_sk = ca_address_sk - and ca_gmt_offset = -5 - group by i_manufact_id) - select i_manufact_id ,sum(total_sales) total_sales - from (select * from ss - union all - select * from cs - union all - select * from ws) tmp1 - group by i_manufact_id - order by total_sales -limit 100; - --- end query 33 in stream 0 using template query33.tpl --- start query 34 in stream 0 using template query34.tpl -select c_last_name - ,c_first_name - ,c_salutation - ,c_preferred_cust_flag - ,ss_ticket_number - ,cnt from - (select ss_ticket_number - ,ss_customer_sk - ,count(*) cnt - from store_sales,date_dim,store,household_demographics - where store_sales.ss_sold_date_sk = date_dim.d_date_sk - and store_sales.ss_store_sk = store.s_store_sk - and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk - and (date_dim.d_dom between 1 and 3 or date_dim.d_dom between 25 and 28) - and (household_demographics.hd_buy_potential = '1001-5000' or - household_demographics.hd_buy_potential = '0-500') - and household_demographics.hd_vehicle_count > 0 - and (case when household_demographics.hd_vehicle_count > 0 - then household_demographics.hd_dep_count/ household_demographics.hd_vehicle_count - else null - end) > 1.2 - and date_dim.d_year in (2000,2000+1,2000+2) - and store.s_county in ('Williamson County','Williamson County','Williamson County','Williamson County', - 'Williamson County','Williamson County','Williamson County','Williamson County') - group by ss_ticket_number,ss_customer_sk) dn,customer - where ss_customer_sk = c_customer_sk - and cnt between 15 and 20 - order by c_last_name,c_first_name,c_salutation,c_preferred_cust_flag desc, ss_ticket_number; - --- end query 34 in stream 0 using template query34.tpl --- start query 35 in stream 0 using template query35.tpl -select - ca_state, - cd_gender, - cd_marital_status, - cd_dep_count, - count(*) cnt1, - avg(cd_dep_count), - stddev_samp(cd_dep_count), - sum(cd_dep_count), - cd_dep_employed_count, - count(*) cnt2, - avg(cd_dep_employed_count), - stddev_samp(cd_dep_employed_count), - sum(cd_dep_employed_count), - cd_dep_college_count, - count(*) cnt3, - avg(cd_dep_college_count), - stddev_samp(cd_dep_college_count), - sum(cd_dep_college_count) - from - customer c,customer_address ca,customer_demographics - where - c.c_current_addr_sk = ca.ca_address_sk and - cd_demo_sk = c.c_current_cdemo_sk and - exists (select * - from store_sales,date_dim - where c.c_customer_sk = ss_customer_sk and - ss_sold_date_sk = d_date_sk and - d_year = 1999 and - d_qoy < 4) and - (exists (select * - from web_sales,date_dim - where c.c_customer_sk = ws_bill_customer_sk and - ws_sold_date_sk = d_date_sk and - d_year = 1999 and - d_qoy < 4) or - exists (select * - from catalog_sales,date_dim - where c.c_customer_sk = cs_ship_customer_sk and - cs_sold_date_sk = d_date_sk and - d_year = 1999 and - d_qoy < 4)) - group by ca_state, - cd_gender, - cd_marital_status, - cd_dep_count, - cd_dep_employed_count, - cd_dep_college_count - order by ca_state, - cd_gender, - cd_marital_status, - cd_dep_count, - cd_dep_employed_count, - cd_dep_college_count - limit 100; - --- end query 35 in stream 0 using template query35.tpl --- start query 36 in stream 0 using template query36.tpl -select * -from -(select - sum(ss_net_profit)/sum(ss_ext_sales_price) as gross_margin - ,i_category - ,i_class - ,grouping(i_category)+grouping(i_class) as lochierarchy - ,rank() over ( - partition by grouping(i_category)+grouping(i_class), - case when grouping(i_class) = 0 then i_category end - order by sum(ss_net_profit)/sum(ss_ext_sales_price) asc) as rank_within_parent - from - store_sales - ,date_dim d1 - ,item - ,store - where - d1.d_year = 2000 - and d1.d_date_sk = ss_sold_date_sk - and i_item_sk = ss_item_sk - and s_store_sk = ss_store_sk - and s_state in ('TN','TN','TN','TN', - 'TN','TN','TN','TN') - group by rollup(i_category,i_class)) as sub - order by - lochierarchy desc - ,case when lochierarchy = 0 then i_category end - ,rank_within_parent - limit 100; - --- end query 36 in stream 0 using template query36.tpl --- start query 37 in stream 0 using template query37.tpl -select i_item_id - ,i_item_desc - ,i_current_price - from item, inventory, date_dim, catalog_sales - where i_current_price between 29 and 29 + 30 - and inv_item_sk = i_item_sk - and d_date_sk=inv_date_sk - and d_date between cast('2002-03-29' as date) and (cast('2002-03-29' as date) + interval '60 days') - and i_manufact_id in (705,742,777,944) - and inv_quantity_on_hand between 100 and 500 - and cs_item_sk = i_item_sk - group by i_item_id,i_item_desc,i_current_price - order by i_item_id - limit 100; - --- end query 37 in stream 0 using template query37.tpl --- start query 38 in stream 0 using template query38.tpl -select count(*) from ( - select distinct c_last_name, c_first_name, d_date - from store_sales, date_dim, customer - where store_sales.ss_sold_date_sk = date_dim.d_date_sk - and store_sales.ss_customer_sk = customer.c_customer_sk - and d_month_seq between 1189 and 1189 + 11 - intersect - select distinct c_last_name, c_first_name, d_date - from catalog_sales, date_dim, customer - where catalog_sales.cs_sold_date_sk = date_dim.d_date_sk - and catalog_sales.cs_bill_customer_sk = customer.c_customer_sk - and d_month_seq between 1189 and 1189 + 11 - intersect - select distinct c_last_name, c_first_name, d_date - from web_sales, date_dim, customer - where web_sales.ws_sold_date_sk = date_dim.d_date_sk - and web_sales.ws_bill_customer_sk = customer.c_customer_sk - and d_month_seq between 1189 and 1189 + 11 -) hot_cust -limit 100; - --- end query 38 in stream 0 using template query38.tpl --- start query 39 in stream 0 using template query39.tpl -with inv as -(select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy - ,stdev,mean, case mean when 0 then null else stdev/mean end cov - from(select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy - ,stddev_samp(inv_quantity_on_hand) stdev,avg(inv_quantity_on_hand) mean - from inventory - ,item - ,warehouse - ,date_dim - where inv_item_sk = i_item_sk - and inv_warehouse_sk = w_warehouse_sk - and inv_date_sk = d_date_sk - and d_year =2000 - group by w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy) foo - where case mean when 0 then 0 else stdev/mean end > 1) -select inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean, inv1.cov - ,inv2.w_warehouse_sk,inv2.i_item_sk,inv2.d_moy,inv2.mean, inv2.cov -from inv inv1,inv inv2 -where inv1.i_item_sk = inv2.i_item_sk - and inv1.w_warehouse_sk = inv2.w_warehouse_sk - and inv1.d_moy=1 - and inv2.d_moy=1+1 -order by inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean,inv1.cov - ,inv2.d_moy,inv2.mean, inv2.cov -; -with inv as -(select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy - ,stdev,mean, case mean when 0 then null else stdev/mean end cov - from(select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy - ,stddev_samp(inv_quantity_on_hand) stdev,avg(inv_quantity_on_hand) mean - from inventory - ,item - ,warehouse - ,date_dim - where inv_item_sk = i_item_sk - and inv_warehouse_sk = w_warehouse_sk - and inv_date_sk = d_date_sk - and d_year =2000 - group by w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy) foo - where case mean when 0 then 0 else stdev/mean end > 1) -select inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean, inv1.cov - ,inv2.w_warehouse_sk,inv2.i_item_sk,inv2.d_moy,inv2.mean, inv2.cov -from inv inv1,inv inv2 -where inv1.i_item_sk = inv2.i_item_sk - and inv1.w_warehouse_sk = inv2.w_warehouse_sk - and inv1.d_moy=1 - and inv2.d_moy=1+1 - and inv1.cov > 1.5 -order by inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean,inv1.cov - ,inv2.d_moy,inv2.mean, inv2.cov -; - --- end query 39 in stream 0 using template query39.tpl --- start query 40 in stream 0 using template query40.tpl -select - w_state - ,i_item_id - ,sum(case when (cast(d_date as date) < cast ('2001-05-02' as date)) - then cs_sales_price - coalesce(cr_refunded_cash,0) else 0 end) as sales_before - ,sum(case when (cast(d_date as date) >= cast ('2001-05-02' as date)) - then cs_sales_price - coalesce(cr_refunded_cash,0) else 0 end) as sales_after - from - catalog_sales left outer join catalog_returns on - (cs_order_number = cr_order_number - and cs_item_sk = cr_item_sk) - ,warehouse - ,item - ,date_dim - where - i_current_price between 0.99 and 1.49 - and i_item_sk = cs_item_sk - and cs_warehouse_sk = w_warehouse_sk - and cs_sold_date_sk = d_date_sk - and d_date between (cast ('2001-05-02' as date) - interval '30 days') - and (cast ('2001-05-02' as date) + interval '30 days') - group by - w_state,i_item_id - order by w_state,i_item_id -limit 100; - --- end query 40 in stream 0 using template query40.tpl --- start query 41 in stream 0 using template query41.tpl -select distinct(i_product_name) - from item i1 - where i_manufact_id between 704 and 704+40 - and (select count(*) as item_cnt - from item - where (i_manufact = i1.i_manufact and - ((i_category = 'Women' and - (i_color = 'forest' or i_color = 'lime') and - (i_units = 'Pallet' or i_units = 'Pound') and - (i_size = 'economy' or i_size = 'small') - ) or - (i_category = 'Women' and - (i_color = 'navy' or i_color = 'slate') and - (i_units = 'Gross' or i_units = 'Bunch') and - (i_size = 'extra large' or i_size = 'petite') - ) or - (i_category = 'Men' and - (i_color = 'powder' or i_color = 'sky') and - (i_units = 'Dozen' or i_units = 'Lb') and - (i_size = 'N/A' or i_size = 'large') - ) or - (i_category = 'Men' and - (i_color = 'maroon' or i_color = 'smoke') and - (i_units = 'Ounce' or i_units = 'Case') and - (i_size = 'economy' or i_size = 'small') - ))) or - (i_manufact = i1.i_manufact and - ((i_category = 'Women' and - (i_color = 'dark' or i_color = 'aquamarine') and - (i_units = 'Ton' or i_units = 'Tbl') and - (i_size = 'economy' or i_size = 'small') - ) or - (i_category = 'Women' and - (i_color = 'frosted' or i_color = 'plum') and - (i_units = 'Dram' or i_units = 'Box') and - (i_size = 'extra large' or i_size = 'petite') - ) or - (i_category = 'Men' and - (i_color = 'papaya' or i_color = 'peach') and - (i_units = 'Bundle' or i_units = 'Carton') and - (i_size = 'N/A' or i_size = 'large') - ) or - (i_category = 'Men' and - (i_color = 'firebrick' or i_color = 'sienna') and - (i_units = 'Cup' or i_units = 'Each') and - (i_size = 'economy' or i_size = 'small') - )))) > 0 - order by i_product_name - limit 100; - --- end query 41 in stream 0 using template query41.tpl --- start query 42 in stream 0 using template query42.tpl -select dt.d_year - ,item.i_category_id - ,item.i_category - ,sum(ss_ext_sales_price) - from date_dim dt - ,store_sales - ,item - where dt.d_date_sk = store_sales.ss_sold_date_sk - and store_sales.ss_item_sk = item.i_item_sk - and item.i_manager_id = 1 - and dt.d_moy=11 - and dt.d_year=1998 - group by dt.d_year - ,item.i_category_id - ,item.i_category - order by sum(ss_ext_sales_price) desc,dt.d_year - ,item.i_category_id - ,item.i_category -limit 100 ; - --- end query 42 in stream 0 using template query42.tpl --- start query 43 in stream 0 using template query43.tpl -select s_store_name, s_store_id, - sum(case when (d_day_name='Sunday') then ss_sales_price else null end) sun_sales, - sum(case when (d_day_name='Monday') then ss_sales_price else null end) mon_sales, - sum(case when (d_day_name='Tuesday') then ss_sales_price else null end) tue_sales, - sum(case when (d_day_name='Wednesday') then ss_sales_price else null end) wed_sales, - sum(case when (d_day_name='Thursday') then ss_sales_price else null end) thu_sales, - sum(case when (d_day_name='Friday') then ss_sales_price else null end) fri_sales, - sum(case when (d_day_name='Saturday') then ss_sales_price else null end) sat_sales - from date_dim, store_sales, store - where d_date_sk = ss_sold_date_sk and - s_store_sk = ss_store_sk and - s_gmt_offset = -5 and - d_year = 2000 - group by s_store_name, s_store_id - order by s_store_name, s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales - limit 100; - --- end query 43 in stream 0 using template query43.tpl --- start query 44 in stream 0 using template query44.tpl -select asceding.rnk, i1.i_product_name best_performing, i2.i_product_name worst_performing -from(select * - from (select item_sk,rank() over (order by rank_col asc) rnk - from (select ss_item_sk item_sk,avg(ss_net_profit) rank_col - from store_sales ss1 - where ss_store_sk = 4 - group by ss_item_sk - having avg(ss_net_profit) > 0.9*(select avg(ss_net_profit) rank_col - from store_sales - where ss_store_sk = 4 - and ss_hdemo_sk is null - group by ss_store_sk))V1)V11 - where rnk < 11) asceding, - (select * - from (select item_sk,rank() over (order by rank_col desc) rnk - from (select ss_item_sk item_sk,avg(ss_net_profit) rank_col - from store_sales ss1 - where ss_store_sk = 4 - group by ss_item_sk - having avg(ss_net_profit) > 0.9*(select avg(ss_net_profit) rank_col - from store_sales - where ss_store_sk = 4 - and ss_hdemo_sk is null - group by ss_store_sk))V2)V21 - where rnk < 11) descending, -item i1, -item i2 -where asceding.rnk = descending.rnk - and i1.i_item_sk=asceding.item_sk - and i2.i_item_sk=descending.item_sk -order by asceding.rnk -limit 100; - --- end query 44 in stream 0 using template query44.tpl --- start query 45 in stream 0 using template query45.tpl -select ca_zip, ca_city, sum(ws_sales_price) - from web_sales, customer, customer_address, date_dim, item - where ws_bill_customer_sk = c_customer_sk - and c_current_addr_sk = ca_address_sk - and ws_item_sk = i_item_sk - and ( substr(ca_zip,1,5) in ('85669', '86197','88274','83405','86475', '85392', '85460', '80348', '81792') - or - i_item_id in (select i_item_id - from item - where i_item_sk in (2, 3, 5, 7, 11, 13, 17, 19, 23, 29) - ) - ) - and ws_sold_date_sk = d_date_sk - and d_qoy = 1 and d_year = 2000 - group by ca_zip, ca_city - order by ca_zip, ca_city - limit 100; - --- end query 45 in stream 0 using template query45.tpl --- start query 46 in stream 0 using template query46.tpl -select c_last_name - ,c_first_name - ,ca_city - ,bought_city - ,ss_ticket_number - ,amt,profit - from - (select ss_ticket_number - ,ss_customer_sk - ,ca_city bought_city - ,sum(ss_coupon_amt) amt - ,sum(ss_net_profit) profit - from store_sales,date_dim,store,household_demographics,customer_address - where store_sales.ss_sold_date_sk = date_dim.d_date_sk - and store_sales.ss_store_sk = store.s_store_sk - and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk - and store_sales.ss_addr_sk = customer_address.ca_address_sk - and (household_demographics.hd_dep_count = 8 or - household_demographics.hd_vehicle_count= 0) - and date_dim.d_dow in (6,0) - and date_dim.d_year in (2000,2000+1,2000+2) - and store.s_city in ('Midway','Fairview','Fairview','Midway','Fairview') - group by ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city) dn,customer,customer_address current_addr - where ss_customer_sk = c_customer_sk - and customer.c_current_addr_sk = current_addr.ca_address_sk - and current_addr.ca_city <> bought_city - order by c_last_name - ,c_first_name - ,ca_city - ,bought_city - ,ss_ticket_number - limit 100; - --- end query 46 in stream 0 using template query46.tpl --- start query 47 in stream 0 using template query47.tpl -with v1 as( - select i_category, i_brand, - s_store_name, s_company_name, - d_year, d_moy, - sum(ss_sales_price) sum_sales, - avg(sum(ss_sales_price)) over - (partition by i_category, i_brand, - s_store_name, s_company_name, d_year) - avg_monthly_sales, - rank() over - (partition by i_category, i_brand, - s_store_name, s_company_name - order by d_year, d_moy) rn - from item, store_sales, date_dim, store - where ss_item_sk = i_item_sk and - ss_sold_date_sk = d_date_sk and - ss_store_sk = s_store_sk and - ( - d_year = 2000 or - ( d_year = 2000-1 and d_moy =12) or - ( d_year = 2000+1 and d_moy =1) - ) - group by i_category, i_brand, - s_store_name, s_company_name, - d_year, d_moy), - v2 as( - select v1.s_store_name, v1.s_company_name - ,v1.d_year - ,v1.avg_monthly_sales - ,v1.sum_sales, v1_lag.sum_sales psum, v1_lead.sum_sales nsum - from v1, v1 v1_lag, v1 v1_lead - where v1.i_category = v1_lag.i_category and - v1.i_category = v1_lead.i_category and - v1.i_brand = v1_lag.i_brand and - v1.i_brand = v1_lead.i_brand and - v1.s_store_name = v1_lag.s_store_name and - v1.s_store_name = v1_lead.s_store_name and - v1.s_company_name = v1_lag.s_company_name and - v1.s_company_name = v1_lead.s_company_name and - v1.rn = v1_lag.rn + 1 and - v1.rn = v1_lead.rn - 1) - select * - from v2 - where d_year = 2000 and - avg_monthly_sales > 0 and - case when avg_monthly_sales > 0 then abs(sum_sales - avg_monthly_sales) / avg_monthly_sales else null end > 0.1 - order by sum_sales - avg_monthly_sales, nsum - limit 100; - --- end query 47 in stream 0 using template query47.tpl --- start query 48 in stream 0 using template query48.tpl -select sum (ss_quantity) - from store_sales, store, customer_demographics, customer_address, date_dim - where s_store_sk = ss_store_sk - and ss_sold_date_sk = d_date_sk and d_year = 2001 - and - ( - ( - cd_demo_sk = ss_cdemo_sk - and - cd_marital_status = 'S' - and - cd_education_status = 'Secondary' - and - ss_sales_price between 100.00 and 150.00 - ) - or - ( - cd_demo_sk = ss_cdemo_sk - and - cd_marital_status = 'M' - and - cd_education_status = '2 yr Degree' - and - ss_sales_price between 50.00 and 100.00 - ) - or - ( - cd_demo_sk = ss_cdemo_sk - and - cd_marital_status = 'D' - and - cd_education_status = 'Advanced Degree' - and - ss_sales_price between 150.00 and 200.00 - ) - ) - and - ( - ( - ss_addr_sk = ca_address_sk - and - ca_country = 'United States' - and - ca_state in ('ND', 'NY', 'SD') - and ss_net_profit between 0 and 2000 - ) - or - (ss_addr_sk = ca_address_sk - and - ca_country = 'United States' - and - ca_state in ('MD', 'GA', 'KS') - and ss_net_profit between 150 and 3000 - ) - or - (ss_addr_sk = ca_address_sk - and - ca_country = 'United States' - and - ca_state in ('CO', 'MN', 'NC') - and ss_net_profit between 50 and 25000 - ) - ) -; - --- end query 48 in stream 0 using template query48.tpl --- start query 49 in stream 0 using template query49.tpl -select channel, item, return_ratio, return_rank, currency_rank from - (select - 'web' as channel - ,web.item - ,web.return_ratio - ,web.return_rank - ,web.currency_rank - from ( - select - item - ,return_ratio - ,currency_ratio - ,rank() over (order by return_ratio) as return_rank - ,rank() over (order by currency_ratio) as currency_rank - from - ( select ws.ws_item_sk as item - ,(cast(sum(coalesce(wr.wr_return_quantity,0)) as decimal(15,4))/ - cast(sum(coalesce(ws.ws_quantity,0)) as decimal(15,4) )) as return_ratio - ,(cast(sum(coalesce(wr.wr_return_amt,0)) as decimal(15,4))/ - cast(sum(coalesce(ws.ws_net_paid,0)) as decimal(15,4) )) as currency_ratio - from - web_sales ws left outer join web_returns wr - on (ws.ws_order_number = wr.wr_order_number and - ws.ws_item_sk = wr.wr_item_sk) - ,date_dim - where - wr.wr_return_amt > 10000 - and ws.ws_net_profit > 1 - and ws.ws_net_paid > 0 - and ws.ws_quantity > 0 - and ws_sold_date_sk = d_date_sk - and d_year = 1998 - and d_moy = 11 - group by ws.ws_item_sk - ) in_web - ) as web - where - ( - web.return_rank <= 10 - or - web.currency_rank <= 10 - ) - union - select - 'catalog' as channel - ,catalog.item - ,catalog.return_ratio - ,catalog.return_rank - ,catalog.currency_rank - from ( - select - item - ,return_ratio - ,currency_ratio - ,rank() over (order by return_ratio) as return_rank - ,rank() over (order by currency_ratio) as currency_rank - from - ( select - cs.cs_item_sk as item - ,(cast(sum(coalesce(cr.cr_return_quantity,0)) as decimal(15,4))/ - cast(sum(coalesce(cs.cs_quantity,0)) as decimal(15,4) )) as return_ratio - ,(cast(sum(coalesce(cr.cr_return_amount,0)) as decimal(15,4))/ - cast(sum(coalesce(cs.cs_net_paid,0)) as decimal(15,4) )) as currency_ratio - from - catalog_sales cs left outer join catalog_returns cr - on (cs.cs_order_number = cr.cr_order_number and - cs.cs_item_sk = cr.cr_item_sk) - ,date_dim - where - cr.cr_return_amount > 10000 - and cs.cs_net_profit > 1 - and cs.cs_net_paid > 0 - and cs.cs_quantity > 0 - and cs_sold_date_sk = d_date_sk - and d_year = 1998 - and d_moy = 11 - group by cs.cs_item_sk - ) in_cat - ) as catalog - where - ( - catalog.return_rank <= 10 - or - catalog.currency_rank <=10 - ) - union - select - 'store' as channel - ,store.item - ,store.return_ratio - ,store.return_rank - ,store.currency_rank - from ( - select - item - ,return_ratio - ,currency_ratio - ,rank() over (order by return_ratio) as return_rank - ,rank() over (order by currency_ratio) as currency_rank - from - ( select sts.ss_item_sk as item - ,(cast(sum(coalesce(sr.sr_return_quantity,0)) as decimal(15,4))/cast(sum(coalesce(sts.ss_quantity,0)) as decimal(15,4) )) as return_ratio - ,(cast(sum(coalesce(sr.sr_return_amt,0)) as decimal(15,4))/cast(sum(coalesce(sts.ss_net_paid,0)) as decimal(15,4) )) as currency_ratio - from - store_sales sts left outer join store_returns sr - on (sts.ss_ticket_number = sr.sr_ticket_number and sts.ss_item_sk = sr.sr_item_sk) - ,date_dim - where - sr.sr_return_amt > 10000 - and sts.ss_net_profit > 1 - and sts.ss_net_paid > 0 - and sts.ss_quantity > 0 - and ss_sold_date_sk = d_date_sk - and d_year = 1998 - and d_moy = 11 - group by sts.ss_item_sk - ) in_store - ) as store - where ( - store.return_rank <= 10 - or - store.currency_rank <= 10 - ) - )as tab - order by 1,4,5,2 - limit 100; - --- end query 49 in stream 0 using template query49.tpl --- start query 50 in stream 0 using template query50.tpl -select - s_store_name - ,s_company_id - ,s_street_number - ,s_street_name - ,s_street_type - ,s_suite_number - ,s_city - ,s_county - ,s_state - ,s_zip - ,sum(case when (sr_returned_date_sk - ss_sold_date_sk <= 30 ) then 1 else 0 end) as "30 days" - ,sum(case when (sr_returned_date_sk - ss_sold_date_sk > 30) and - (sr_returned_date_sk - ss_sold_date_sk <= 60) then 1 else 0 end ) as "31-60 days" - ,sum(case when (sr_returned_date_sk - ss_sold_date_sk > 60) and - (sr_returned_date_sk - ss_sold_date_sk <= 90) then 1 else 0 end) as "61-90 days" - ,sum(case when (sr_returned_date_sk - ss_sold_date_sk > 90) and - (sr_returned_date_sk - ss_sold_date_sk <= 120) then 1 else 0 end) as "91-120 days" - ,sum(case when (sr_returned_date_sk - ss_sold_date_sk > 120) then 1 else 0 end) as ">120 days" -from - store_sales - ,store_returns - ,store - ,date_dim d1 - ,date_dim d2 -where - d2.d_year = 2001 -and d2.d_moy = 8 -and ss_ticket_number = sr_ticket_number -and ss_item_sk = sr_item_sk -and ss_sold_date_sk = d1.d_date_sk -and sr_returned_date_sk = d2.d_date_sk -and ss_customer_sk = sr_customer_sk -and ss_store_sk = s_store_sk -group by - s_store_name - ,s_company_id - ,s_street_number - ,s_street_name - ,s_street_type - ,s_suite_number - ,s_city - ,s_county - ,s_state - ,s_zip -order by s_store_name - ,s_company_id - ,s_street_number - ,s_street_name - ,s_street_type - ,s_suite_number - ,s_city - ,s_county - ,s_state - ,s_zip -limit 100; - --- end query 50 in stream 0 using template query50.tpl --- start query 51 in stream 0 using template query51.tpl -WITH web_v1 as ( -select - ws_item_sk item_sk, d_date, - sum(sum(ws_sales_price)) - over (partition by ws_item_sk order by d_date rows between unbounded preceding and current row) cume_sales -from web_sales - ,date_dim -where ws_sold_date_sk=d_date_sk - and d_month_seq between 1212 and 1212+11 - and ws_item_sk is not NULL -group by ws_item_sk, d_date), -store_v1 as ( -select - ss_item_sk item_sk, d_date, - sum(sum(ss_sales_price)) - over (partition by ss_item_sk order by d_date rows between unbounded preceding and current row) cume_sales -from store_sales - ,date_dim -where ss_sold_date_sk=d_date_sk - and d_month_seq between 1212 and 1212+11 - and ss_item_sk is not NULL -group by ss_item_sk, d_date) - select * -from (select item_sk - ,d_date - ,web_sales - ,store_sales - ,max(web_sales) - over (partition by item_sk order by d_date rows between unbounded preceding and current row) web_cumulative - ,max(store_sales) - over (partition by item_sk order by d_date rows between unbounded preceding and current row) store_cumulative - from (select case when web.item_sk is not null then web.item_sk else store.item_sk end item_sk - ,case when web.d_date is not null then web.d_date else store.d_date end d_date - ,web.cume_sales web_sales - ,store.cume_sales store_sales - from web_v1 web full outer join store_v1 store on (web.item_sk = store.item_sk - and web.d_date = store.d_date) - )x )y -where web_cumulative > store_cumulative -order by item_sk - ,d_date -limit 100; - --- end query 51 in stream 0 using template query51.tpl --- start query 52 in stream 0 using template query52.tpl -select dt.d_year - ,item.i_brand_id brand_id - ,item.i_brand brand - ,sum(ss_ext_sales_price) ext_price - from date_dim dt - ,store_sales - ,item - where dt.d_date_sk = store_sales.ss_sold_date_sk - and store_sales.ss_item_sk = item.i_item_sk - and item.i_manager_id = 1 - and dt.d_moy=12 - and dt.d_year=2000 - group by dt.d_year - ,item.i_brand - ,item.i_brand_id - order by dt.d_year - ,ext_price desc - ,brand_id -limit 100 ; - --- end query 52 in stream 0 using template query52.tpl --- start query 53 in stream 0 using template query53.tpl -select * from -(select i_manufact_id, -sum(ss_sales_price) sum_sales, -avg(sum(ss_sales_price)) over (partition by i_manufact_id) avg_quarterly_sales -from item, store_sales, date_dim, store -where ss_item_sk = i_item_sk and -ss_sold_date_sk = d_date_sk and -ss_store_sk = s_store_sk and -d_month_seq in (1186,1186+1,1186+2,1186+3,1186+4,1186+5,1186+6,1186+7,1186+8,1186+9,1186+10,1186+11) and -((i_category in ('Books','Children','Electronics') and -i_class in ('personal','portable','reference','self-help') and -i_brand in ('scholaramalgamalg #14','scholaramalgamalg #7', - 'exportiunivamalg #9','scholaramalgamalg #9')) -or(i_category in ('Women','Music','Men') and -i_class in ('accessories','classical','fragrances','pants') and -i_brand in ('amalgimporto #1','edu packscholar #1','exportiimporto #1', - 'importoamalg #1'))) -group by i_manufact_id, d_qoy ) tmp1 -where case when avg_quarterly_sales > 0 - then abs (sum_sales - avg_quarterly_sales)/ avg_quarterly_sales - else null end > 0.1 -order by avg_quarterly_sales, - sum_sales, - i_manufact_id -limit 100; - --- end query 53 in stream 0 using template query53.tpl --- start query 54 in stream 0 using template query54.tpl -with my_customers as ( - select distinct c_customer_sk - , c_current_addr_sk - from - ( select cs_sold_date_sk sold_date_sk, - cs_bill_customer_sk customer_sk, - cs_item_sk item_sk - from catalog_sales - union all - select ws_sold_date_sk sold_date_sk, - ws_bill_customer_sk customer_sk, - ws_item_sk item_sk - from web_sales - ) cs_or_ws_sales, - item, - date_dim, - customer - where sold_date_sk = d_date_sk - and item_sk = i_item_sk - and i_category = 'Music' - and i_class = 'country' - and c_customer_sk = cs_or_ws_sales.customer_sk - and d_moy = 1 - and d_year = 1999 - ) - , my_revenue as ( - select c_customer_sk, - sum(ss_ext_sales_price) as revenue - from my_customers, - store_sales, - customer_address, - store, - date_dim - where c_current_addr_sk = ca_address_sk - and ca_county = s_county - and ca_state = s_state - and ss_sold_date_sk = d_date_sk - and c_customer_sk = ss_customer_sk - and d_month_seq between (select distinct d_month_seq+1 - from date_dim where d_year = 1999 and d_moy = 1) - and (select distinct d_month_seq+3 - from date_dim where d_year = 1999 and d_moy = 1) - group by c_customer_sk - ) - , segments as - (select cast((revenue/50) as int) as segment - from my_revenue - ) - select segment, count(*) as num_customers, segment*50 as segment_base - from segments - group by segment - order by segment, num_customers - limit 100; - --- end query 54 in stream 0 using template query54.tpl --- start query 55 in stream 0 using template query55.tpl -select i_brand_id brand_id, i_brand brand, - sum(ss_ext_sales_price) ext_price - from date_dim, store_sales, item - where d_date_sk = ss_sold_date_sk - and ss_item_sk = i_item_sk - and i_manager_id=52 - and d_moy=11 - and d_year=2000 - group by i_brand, i_brand_id - order by ext_price desc, i_brand_id -limit 100 ; - --- end query 55 in stream 0 using template query55.tpl --- start query 56 in stream 0 using template query56.tpl -with ss as ( - select i_item_id,sum(ss_ext_sales_price) total_sales - from - store_sales, - date_dim, - customer_address, - item - where i_item_id in (select - i_item_id -from item -where i_color in ('powder','orchid','pink')) - and ss_item_sk = i_item_sk - and ss_sold_date_sk = d_date_sk - and d_year = 2000 - and d_moy = 3 - and ss_addr_sk = ca_address_sk - and ca_gmt_offset = -6 - group by i_item_id), - cs as ( - select i_item_id,sum(cs_ext_sales_price) total_sales - from - catalog_sales, - date_dim, - customer_address, - item - where - i_item_id in (select - i_item_id -from item -where i_color in ('powder','orchid','pink')) - and cs_item_sk = i_item_sk - and cs_sold_date_sk = d_date_sk - and d_year = 2000 - and d_moy = 3 - and cs_bill_addr_sk = ca_address_sk - and ca_gmt_offset = -6 - group by i_item_id), - ws as ( - select i_item_id,sum(ws_ext_sales_price) total_sales - from - web_sales, - date_dim, - customer_address, - item - where - i_item_id in (select - i_item_id -from item -where i_color in ('powder','orchid','pink')) - and ws_item_sk = i_item_sk - and ws_sold_date_sk = d_date_sk - and d_year = 2000 - and d_moy = 3 - and ws_bill_addr_sk = ca_address_sk - and ca_gmt_offset = -6 - group by i_item_id) - select i_item_id ,sum(total_sales) total_sales - from (select * from ss - union all - select * from cs - union all - select * from ws) tmp1 - group by i_item_id - order by total_sales, - i_item_id - limit 100; - --- end query 56 in stream 0 using template query56.tpl --- start query 57 in stream 0 using template query57.tpl -with v1 as( - select i_category, i_brand, - cc_name, - d_year, d_moy, - sum(cs_sales_price) sum_sales, - avg(sum(cs_sales_price)) over - (partition by i_category, i_brand, - cc_name, d_year) - avg_monthly_sales, - rank() over - (partition by i_category, i_brand, - cc_name - order by d_year, d_moy) rn - from item, catalog_sales, date_dim, call_center - where cs_item_sk = i_item_sk and - cs_sold_date_sk = d_date_sk and - cc_call_center_sk= cs_call_center_sk and - ( - d_year = 2001 or - ( d_year = 2001-1 and d_moy =12) or - ( d_year = 2001+1 and d_moy =1) - ) - group by i_category, i_brand, - cc_name , d_year, d_moy), - v2 as( - select v1.i_category, v1.i_brand, v1.cc_name - ,v1.d_year - ,v1.avg_monthly_sales - ,v1.sum_sales, v1_lag.sum_sales psum, v1_lead.sum_sales nsum - from v1, v1 v1_lag, v1 v1_lead - where v1.i_category = v1_lag.i_category and - v1.i_category = v1_lead.i_category and - v1.i_brand = v1_lag.i_brand and - v1.i_brand = v1_lead.i_brand and - v1. cc_name = v1_lag. cc_name and - v1. cc_name = v1_lead. cc_name and - v1.rn = v1_lag.rn + 1 and - v1.rn = v1_lead.rn - 1) - select * - from v2 - where d_year = 2001 and - avg_monthly_sales > 0 and - case when avg_monthly_sales > 0 then abs(sum_sales - avg_monthly_sales) / avg_monthly_sales else null end > 0.1 - order by sum_sales - avg_monthly_sales, avg_monthly_sales - limit 100; - --- end query 57 in stream 0 using template query57.tpl --- start query 58 in stream 0 using template query58.tpl -with ss_items as - (select i_item_id item_id - ,sum(ss_ext_sales_price) ss_item_rev - from store_sales - ,item - ,date_dim - where ss_item_sk = i_item_sk - and d_date in (select d_date - from date_dim - where d_week_seq = (select d_week_seq - from date_dim - where d_date = '2001-06-16')) - and ss_sold_date_sk = d_date_sk - group by i_item_id), - cs_items as - (select i_item_id item_id - ,sum(cs_ext_sales_price) cs_item_rev - from catalog_sales - ,item - ,date_dim - where cs_item_sk = i_item_sk - and d_date in (select d_date - from date_dim - where d_week_seq = (select d_week_seq - from date_dim - where d_date = '2001-06-16')) - and cs_sold_date_sk = d_date_sk - group by i_item_id), - ws_items as - (select i_item_id item_id - ,sum(ws_ext_sales_price) ws_item_rev - from web_sales - ,item - ,date_dim - where ws_item_sk = i_item_sk - and d_date in (select d_date - from date_dim - where d_week_seq =(select d_week_seq - from date_dim - where d_date = '2001-06-16')) - and ws_sold_date_sk = d_date_sk - group by i_item_id) - select ss_items.item_id - ,ss_item_rev - ,ss_item_rev/((ss_item_rev+cs_item_rev+ws_item_rev)/3) * 100 ss_dev - ,cs_item_rev - ,cs_item_rev/((ss_item_rev+cs_item_rev+ws_item_rev)/3) * 100 cs_dev - ,ws_item_rev - ,ws_item_rev/((ss_item_rev+cs_item_rev+ws_item_rev)/3) * 100 ws_dev - ,(ss_item_rev+cs_item_rev+ws_item_rev)/3 average - from ss_items,cs_items,ws_items - where ss_items.item_id=cs_items.item_id - and ss_items.item_id=ws_items.item_id - and ss_item_rev between 0.9 * cs_item_rev and 1.1 * cs_item_rev - and ss_item_rev between 0.9 * ws_item_rev and 1.1 * ws_item_rev - and cs_item_rev between 0.9 * ss_item_rev and 1.1 * ss_item_rev - and cs_item_rev between 0.9 * ws_item_rev and 1.1 * ws_item_rev - and ws_item_rev between 0.9 * ss_item_rev and 1.1 * ss_item_rev - and ws_item_rev between 0.9 * cs_item_rev and 1.1 * cs_item_rev - order by item_id - ,ss_item_rev - limit 100; - --- end query 58 in stream 0 using template query58.tpl --- start query 59 in stream 0 using template query59.tpl -with wss as - (select d_week_seq, - ss_store_sk, - sum(case when (d_day_name='Sunday') then ss_sales_price else null end) sun_sales, - sum(case when (d_day_name='Monday') then ss_sales_price else null end) mon_sales, - sum(case when (d_day_name='Tuesday') then ss_sales_price else null end) tue_sales, - sum(case when (d_day_name='Wednesday') then ss_sales_price else null end) wed_sales, - sum(case when (d_day_name='Thursday') then ss_sales_price else null end) thu_sales, - sum(case when (d_day_name='Friday') then ss_sales_price else null end) fri_sales, - sum(case when (d_day_name='Saturday') then ss_sales_price else null end) sat_sales - from store_sales,date_dim - where d_date_sk = ss_sold_date_sk - group by d_week_seq,ss_store_sk - ) - select s_store_name1,s_store_id1,d_week_seq1 - ,sun_sales1/sun_sales2,mon_sales1/mon_sales2 - ,tue_sales1/tue_sales2,wed_sales1/wed_sales2,thu_sales1/thu_sales2 - ,fri_sales1/fri_sales2,sat_sales1/sat_sales2 - from - (select s_store_name s_store_name1,wss.d_week_seq d_week_seq1 - ,s_store_id s_store_id1,sun_sales sun_sales1 - ,mon_sales mon_sales1,tue_sales tue_sales1 - ,wed_sales wed_sales1,thu_sales thu_sales1 - ,fri_sales fri_sales1,sat_sales sat_sales1 - from wss,store,date_dim d - where d.d_week_seq = wss.d_week_seq and - ss_store_sk = s_store_sk and - d_month_seq between 1195 and 1195 + 11) y, - (select s_store_name s_store_name2,wss.d_week_seq d_week_seq2 - ,s_store_id s_store_id2,sun_sales sun_sales2 - ,mon_sales mon_sales2,tue_sales tue_sales2 - ,wed_sales wed_sales2,thu_sales thu_sales2 - ,fri_sales fri_sales2,sat_sales sat_sales2 - from wss,store,date_dim d - where d.d_week_seq = wss.d_week_seq and - ss_store_sk = s_store_sk and - d_month_seq between 1195+ 12 and 1195 + 23) x - where s_store_id1=s_store_id2 - and d_week_seq1=d_week_seq2-52 - order by s_store_name1,s_store_id1,d_week_seq1 -limit 100; - --- end query 59 in stream 0 using template query59.tpl --- start query 60 in stream 0 using template query60.tpl -with ss as ( - select - i_item_id,sum(ss_ext_sales_price) total_sales - from - store_sales, - date_dim, - customer_address, - item - where - i_item_id in (select - i_item_id -from - item -where i_category in ('Jewelry')) - and ss_item_sk = i_item_sk - and ss_sold_date_sk = d_date_sk - and d_year = 2000 - and d_moy = 10 - and ss_addr_sk = ca_address_sk - and ca_gmt_offset = -5 - group by i_item_id), - cs as ( - select - i_item_id,sum(cs_ext_sales_price) total_sales - from - catalog_sales, - date_dim, - customer_address, - item - where - i_item_id in (select - i_item_id -from - item -where i_category in ('Jewelry')) - and cs_item_sk = i_item_sk - and cs_sold_date_sk = d_date_sk - and d_year = 2000 - and d_moy = 10 - and cs_bill_addr_sk = ca_address_sk - and ca_gmt_offset = -5 - group by i_item_id), - ws as ( - select - i_item_id,sum(ws_ext_sales_price) total_sales - from - web_sales, - date_dim, - customer_address, - item - where - i_item_id in (select - i_item_id -from - item -where i_category in ('Jewelry')) - and ws_item_sk = i_item_sk - and ws_sold_date_sk = d_date_sk - and d_year = 2000 - and d_moy = 10 - and ws_bill_addr_sk = ca_address_sk - and ca_gmt_offset = -5 - group by i_item_id) - select - i_item_id -,sum(total_sales) total_sales - from (select * from ss - union all - select * from cs - union all - select * from ws) tmp1 - group by i_item_id - order by i_item_id - ,total_sales - limit 100; - --- end query 60 in stream 0 using template query60.tpl --- start query 61 in stream 0 using template query61.tpl -select promotions,total,cast(promotions as decimal(15,4))/cast(total as decimal(15,4))*100 -from - (select sum(ss_ext_sales_price) promotions - from store_sales - ,store - ,promotion - ,date_dim - ,customer - ,customer_address - ,item - where ss_sold_date_sk = d_date_sk - and ss_store_sk = s_store_sk - and ss_promo_sk = p_promo_sk - and ss_customer_sk= c_customer_sk - and ca_address_sk = c_current_addr_sk - and ss_item_sk = i_item_sk - and ca_gmt_offset = -7 - and i_category = 'Home' - and (p_channel_dmail = 'Y' or p_channel_email = 'Y' or p_channel_tv = 'Y') - and s_gmt_offset = -7 - and d_year = 2000 - and d_moy = 12) promotional_sales, - (select sum(ss_ext_sales_price) total - from store_sales - ,store - ,date_dim - ,customer - ,customer_address - ,item - where ss_sold_date_sk = d_date_sk - and ss_store_sk = s_store_sk - and ss_customer_sk= c_customer_sk - and ca_address_sk = c_current_addr_sk - and ss_item_sk = i_item_sk - and ca_gmt_offset = -7 - and i_category = 'Home' - and s_gmt_offset = -7 - and d_year = 2000 - and d_moy = 12) all_sales -order by promotions, total -limit 100; - --- end query 61 in stream 0 using template query61.tpl --- start query 62 in stream 0 using template query62.tpl -select - substr(w_warehouse_name,1,20) - ,sm_type - ,web_name - ,sum(case when (ws_ship_date_sk - ws_sold_date_sk <= 30 ) then 1 else 0 end) as "30 days" - ,sum(case when (ws_ship_date_sk - ws_sold_date_sk > 30) and - (ws_ship_date_sk - ws_sold_date_sk <= 60) then 1 else 0 end ) as "31-60 days" - ,sum(case when (ws_ship_date_sk - ws_sold_date_sk > 60) and - (ws_ship_date_sk - ws_sold_date_sk <= 90) then 1 else 0 end) as "61-90 days" - ,sum(case when (ws_ship_date_sk - ws_sold_date_sk > 90) and - (ws_ship_date_sk - ws_sold_date_sk <= 120) then 1 else 0 end) as "91-120 days" - ,sum(case when (ws_ship_date_sk - ws_sold_date_sk > 120) then 1 else 0 end) as ">120 days" -from - web_sales - ,warehouse - ,ship_mode - ,web_site - ,date_dim -where - d_month_seq between 1223 and 1223 + 11 -and ws_ship_date_sk = d_date_sk -and ws_warehouse_sk = w_warehouse_sk -and ws_ship_mode_sk = sm_ship_mode_sk -and ws_web_site_sk = web_site_sk -group by - substr(w_warehouse_name,1,20) - ,sm_type - ,web_name -order by substr(w_warehouse_name,1,20) - ,sm_type - ,web_name -limit 100; - --- end query 62 in stream 0 using template query62.tpl --- start query 63 in stream 0 using template query63.tpl -select * -from (select i_manager_id - ,sum(ss_sales_price) sum_sales - ,avg(sum(ss_sales_price)) over (partition by i_manager_id) avg_monthly_sales - from item - ,store_sales - ,date_dim - ,store - where ss_item_sk = i_item_sk - and ss_sold_date_sk = d_date_sk - and ss_store_sk = s_store_sk - and d_month_seq in (1222,1222+1,1222+2,1222+3,1222+4,1222+5,1222+6,1222+7,1222+8,1222+9,1222+10,1222+11) - and (( i_category in ('Books','Children','Electronics') - and i_class in ('personal','portable','reference','self-help') - and i_brand in ('scholaramalgamalg #14','scholaramalgamalg #7', - 'exportiunivamalg #9','scholaramalgamalg #9')) - or( i_category in ('Women','Music','Men') - and i_class in ('accessories','classical','fragrances','pants') - and i_brand in ('amalgimporto #1','edu packscholar #1','exportiimporto #1', - 'importoamalg #1'))) -group by i_manager_id, d_moy) tmp1 -where case when avg_monthly_sales > 0 then abs (sum_sales - avg_monthly_sales) / avg_monthly_sales else null end > 0.1 -order by i_manager_id - ,avg_monthly_sales - ,sum_sales -limit 100; - --- end query 63 in stream 0 using template query63.tpl --- start query 64 in stream 0 using template query64.tpl -with cs_ui as - (select cs_item_sk - ,sum(cs_ext_list_price) as sale,sum(cr_refunded_cash+cr_reversed_charge+cr_store_credit) as refund - from catalog_sales - ,catalog_returns - where cs_item_sk = cr_item_sk - and cs_order_number = cr_order_number - group by cs_item_sk - having sum(cs_ext_list_price)>2*sum(cr_refunded_cash+cr_reversed_charge+cr_store_credit)), -cross_sales as - (select i_product_name product_name - ,i_item_sk item_sk - ,s_store_name store_name - ,s_zip store_zip - ,ad1.ca_street_number b_street_number - ,ad1.ca_street_name b_street_name - ,ad1.ca_city b_city - ,ad1.ca_zip b_zip - ,ad2.ca_street_number c_street_number - ,ad2.ca_street_name c_street_name - ,ad2.ca_city c_city - ,ad2.ca_zip c_zip - ,d1.d_year as syear - ,d2.d_year as fsyear - ,d3.d_year s2year - ,count(*) cnt - ,sum(ss_wholesale_cost) s1 - ,sum(ss_list_price) s2 - ,sum(ss_coupon_amt) s3 - FROM store_sales - ,store_returns - ,cs_ui - ,date_dim d1 - ,date_dim d2 - ,date_dim d3 - ,store - ,customer - ,customer_demographics cd1 - ,customer_demographics cd2 - ,promotion - ,household_demographics hd1 - ,household_demographics hd2 - ,customer_address ad1 - ,customer_address ad2 - ,income_band ib1 - ,income_band ib2 - ,item - WHERE ss_store_sk = s_store_sk AND - ss_sold_date_sk = d1.d_date_sk AND - ss_customer_sk = c_customer_sk AND - ss_cdemo_sk= cd1.cd_demo_sk AND - ss_hdemo_sk = hd1.hd_demo_sk AND - ss_addr_sk = ad1.ca_address_sk and - ss_item_sk = i_item_sk and - ss_item_sk = sr_item_sk and - ss_ticket_number = sr_ticket_number and - ss_item_sk = cs_ui.cs_item_sk and - c_current_cdemo_sk = cd2.cd_demo_sk AND - c_current_hdemo_sk = hd2.hd_demo_sk AND - c_current_addr_sk = ad2.ca_address_sk and - c_first_sales_date_sk = d2.d_date_sk and - c_first_shipto_date_sk = d3.d_date_sk and - ss_promo_sk = p_promo_sk and - hd1.hd_income_band_sk = ib1.ib_income_band_sk and - hd2.hd_income_band_sk = ib2.ib_income_band_sk and - cd1.cd_marital_status <> cd2.cd_marital_status and - i_color in ('orange','lace','lawn','misty','blush','pink') and - i_current_price between 48 and 48 + 10 and - i_current_price between 48 + 1 and 48 + 15 -group by i_product_name - ,i_item_sk - ,s_store_name - ,s_zip - ,ad1.ca_street_number - ,ad1.ca_street_name - ,ad1.ca_city - ,ad1.ca_zip - ,ad2.ca_street_number - ,ad2.ca_street_name - ,ad2.ca_city - ,ad2.ca_zip - ,d1.d_year - ,d2.d_year - ,d3.d_year -) -select cs1.product_name - ,cs1.store_name - ,cs1.store_zip - ,cs1.b_street_number - ,cs1.b_street_name - ,cs1.b_city - ,cs1.b_zip - ,cs1.c_street_number - ,cs1.c_street_name - ,cs1.c_city - ,cs1.c_zip - ,cs1.syear - ,cs1.cnt - ,cs1.s1 as s11 - ,cs1.s2 as s21 - ,cs1.s3 as s31 - ,cs2.s1 as s12 - ,cs2.s2 as s22 - ,cs2.s3 as s32 - ,cs2.syear - ,cs2.cnt -from cross_sales cs1,cross_sales cs2 -where cs1.item_sk=cs2.item_sk and - cs1.syear = 1999 and - cs2.syear = 1999 + 1 and - cs2.cnt <= cs1.cnt and - cs1.store_name = cs2.store_name and - cs1.store_zip = cs2.store_zip -order by cs1.product_name - ,cs1.store_name - ,cs2.cnt - ,cs1.s1 - ,cs2.s1; - --- end query 64 in stream 0 using template query64.tpl --- start query 65 in stream 0 using template query65.tpl -select - s_store_name, - i_item_desc, - sc.revenue, - i_current_price, - i_wholesale_cost, - i_brand - from store, item, - (select ss_store_sk, avg(revenue) as ave - from - (select ss_store_sk, ss_item_sk, - sum(ss_sales_price) as revenue - from store_sales, date_dim - where ss_sold_date_sk = d_date_sk and d_month_seq between 1176 and 1176+11 - group by ss_store_sk, ss_item_sk) sa - group by ss_store_sk) sb, - (select ss_store_sk, ss_item_sk, sum(ss_sales_price) as revenue - from store_sales, date_dim - where ss_sold_date_sk = d_date_sk and d_month_seq between 1176 and 1176+11 - group by ss_store_sk, ss_item_sk) sc - where sb.ss_store_sk = sc.ss_store_sk and - sc.revenue <= 0.1 * sb.ave and - s_store_sk = sc.ss_store_sk and - i_item_sk = sc.ss_item_sk - order by s_store_name, i_item_desc -limit 100; - --- end query 65 in stream 0 using template query65.tpl --- start query 66 in stream 0 using template query66.tpl -select - w_warehouse_name - ,w_warehouse_sq_ft - ,w_city - ,w_county - ,w_state - ,w_country - ,ship_carriers - ,year - ,sum(jan_sales) as jan_sales - ,sum(feb_sales) as feb_sales - ,sum(mar_sales) as mar_sales - ,sum(apr_sales) as apr_sales - ,sum(may_sales) as may_sales - ,sum(jun_sales) as jun_sales - ,sum(jul_sales) as jul_sales - ,sum(aug_sales) as aug_sales - ,sum(sep_sales) as sep_sales - ,sum(oct_sales) as oct_sales - ,sum(nov_sales) as nov_sales - ,sum(dec_sales) as dec_sales - ,sum(jan_sales/w_warehouse_sq_ft) as jan_sales_per_sq_foot - ,sum(feb_sales/w_warehouse_sq_ft) as feb_sales_per_sq_foot - ,sum(mar_sales/w_warehouse_sq_ft) as mar_sales_per_sq_foot - ,sum(apr_sales/w_warehouse_sq_ft) as apr_sales_per_sq_foot - ,sum(may_sales/w_warehouse_sq_ft) as may_sales_per_sq_foot - ,sum(jun_sales/w_warehouse_sq_ft) as jun_sales_per_sq_foot - ,sum(jul_sales/w_warehouse_sq_ft) as jul_sales_per_sq_foot - ,sum(aug_sales/w_warehouse_sq_ft) as aug_sales_per_sq_foot - ,sum(sep_sales/w_warehouse_sq_ft) as sep_sales_per_sq_foot - ,sum(oct_sales/w_warehouse_sq_ft) as oct_sales_per_sq_foot - ,sum(nov_sales/w_warehouse_sq_ft) as nov_sales_per_sq_foot - ,sum(dec_sales/w_warehouse_sq_ft) as dec_sales_per_sq_foot - ,sum(jan_net) as jan_net - ,sum(feb_net) as feb_net - ,sum(mar_net) as mar_net - ,sum(apr_net) as apr_net - ,sum(may_net) as may_net - ,sum(jun_net) as jun_net - ,sum(jul_net) as jul_net - ,sum(aug_net) as aug_net - ,sum(sep_net) as sep_net - ,sum(oct_net) as oct_net - ,sum(nov_net) as nov_net - ,sum(dec_net) as dec_net - from ( - select - w_warehouse_name - ,w_warehouse_sq_ft - ,w_city - ,w_county - ,w_state - ,w_country - ,'ORIENTAL' || ',' || 'BOXBUNDLES' as ship_carriers - ,d_year as year - ,sum(case when d_moy = 1 - then ws_ext_sales_price* ws_quantity else 0 end) as jan_sales - ,sum(case when d_moy = 2 - then ws_ext_sales_price* ws_quantity else 0 end) as feb_sales - ,sum(case when d_moy = 3 - then ws_ext_sales_price* ws_quantity else 0 end) as mar_sales - ,sum(case when d_moy = 4 - then ws_ext_sales_price* ws_quantity else 0 end) as apr_sales - ,sum(case when d_moy = 5 - then ws_ext_sales_price* ws_quantity else 0 end) as may_sales - ,sum(case when d_moy = 6 - then ws_ext_sales_price* ws_quantity else 0 end) as jun_sales - ,sum(case when d_moy = 7 - then ws_ext_sales_price* ws_quantity else 0 end) as jul_sales - ,sum(case when d_moy = 8 - then ws_ext_sales_price* ws_quantity else 0 end) as aug_sales - ,sum(case when d_moy = 9 - then ws_ext_sales_price* ws_quantity else 0 end) as sep_sales - ,sum(case when d_moy = 10 - then ws_ext_sales_price* ws_quantity else 0 end) as oct_sales - ,sum(case when d_moy = 11 - then ws_ext_sales_price* ws_quantity else 0 end) as nov_sales - ,sum(case when d_moy = 12 - then ws_ext_sales_price* ws_quantity else 0 end) as dec_sales - ,sum(case when d_moy = 1 - then ws_net_paid_inc_ship * ws_quantity else 0 end) as jan_net - ,sum(case when d_moy = 2 - then ws_net_paid_inc_ship * ws_quantity else 0 end) as feb_net - ,sum(case when d_moy = 3 - then ws_net_paid_inc_ship * ws_quantity else 0 end) as mar_net - ,sum(case when d_moy = 4 - then ws_net_paid_inc_ship * ws_quantity else 0 end) as apr_net - ,sum(case when d_moy = 5 - then ws_net_paid_inc_ship * ws_quantity else 0 end) as may_net - ,sum(case when d_moy = 6 - then ws_net_paid_inc_ship * ws_quantity else 0 end) as jun_net - ,sum(case when d_moy = 7 - then ws_net_paid_inc_ship * ws_quantity else 0 end) as jul_net - ,sum(case when d_moy = 8 - then ws_net_paid_inc_ship * ws_quantity else 0 end) as aug_net - ,sum(case when d_moy = 9 - then ws_net_paid_inc_ship * ws_quantity else 0 end) as sep_net - ,sum(case when d_moy = 10 - then ws_net_paid_inc_ship * ws_quantity else 0 end) as oct_net - ,sum(case when d_moy = 11 - then ws_net_paid_inc_ship * ws_quantity else 0 end) as nov_net - ,sum(case when d_moy = 12 - then ws_net_paid_inc_ship * ws_quantity else 0 end) as dec_net - from - web_sales - ,warehouse - ,date_dim - ,time_dim - ,ship_mode - where - ws_warehouse_sk = w_warehouse_sk - and ws_sold_date_sk = d_date_sk - and ws_sold_time_sk = t_time_sk - and ws_ship_mode_sk = sm_ship_mode_sk - and d_year = 2001 - and t_time between 42970 and 42970+28800 - and sm_carrier in ('ORIENTAL','BOXBUNDLES') - group by - w_warehouse_name - ,w_warehouse_sq_ft - ,w_city - ,w_county - ,w_state - ,w_country - ,d_year - union all - select - w_warehouse_name - ,w_warehouse_sq_ft - ,w_city - ,w_county - ,w_state - ,w_country - ,'ORIENTAL' || ',' || 'BOXBUNDLES' as ship_carriers - ,d_year as year - ,sum(case when d_moy = 1 - then cs_ext_list_price* cs_quantity else 0 end) as jan_sales - ,sum(case when d_moy = 2 - then cs_ext_list_price* cs_quantity else 0 end) as feb_sales - ,sum(case when d_moy = 3 - then cs_ext_list_price* cs_quantity else 0 end) as mar_sales - ,sum(case when d_moy = 4 - then cs_ext_list_price* cs_quantity else 0 end) as apr_sales - ,sum(case when d_moy = 5 - then cs_ext_list_price* cs_quantity else 0 end) as may_sales - ,sum(case when d_moy = 6 - then cs_ext_list_price* cs_quantity else 0 end) as jun_sales - ,sum(case when d_moy = 7 - then cs_ext_list_price* cs_quantity else 0 end) as jul_sales - ,sum(case when d_moy = 8 - then cs_ext_list_price* cs_quantity else 0 end) as aug_sales - ,sum(case when d_moy = 9 - then cs_ext_list_price* cs_quantity else 0 end) as sep_sales - ,sum(case when d_moy = 10 - then cs_ext_list_price* cs_quantity else 0 end) as oct_sales - ,sum(case when d_moy = 11 - then cs_ext_list_price* cs_quantity else 0 end) as nov_sales - ,sum(case when d_moy = 12 - then cs_ext_list_price* cs_quantity else 0 end) as dec_sales - ,sum(case when d_moy = 1 - then cs_net_paid * cs_quantity else 0 end) as jan_net - ,sum(case when d_moy = 2 - then cs_net_paid * cs_quantity else 0 end) as feb_net - ,sum(case when d_moy = 3 - then cs_net_paid * cs_quantity else 0 end) as mar_net - ,sum(case when d_moy = 4 - then cs_net_paid * cs_quantity else 0 end) as apr_net - ,sum(case when d_moy = 5 - then cs_net_paid * cs_quantity else 0 end) as may_net - ,sum(case when d_moy = 6 - then cs_net_paid * cs_quantity else 0 end) as jun_net - ,sum(case when d_moy = 7 - then cs_net_paid * cs_quantity else 0 end) as jul_net - ,sum(case when d_moy = 8 - then cs_net_paid * cs_quantity else 0 end) as aug_net - ,sum(case when d_moy = 9 - then cs_net_paid * cs_quantity else 0 end) as sep_net - ,sum(case when d_moy = 10 - then cs_net_paid * cs_quantity else 0 end) as oct_net - ,sum(case when d_moy = 11 - then cs_net_paid * cs_quantity else 0 end) as nov_net - ,sum(case when d_moy = 12 - then cs_net_paid * cs_quantity else 0 end) as dec_net - from - catalog_sales - ,warehouse - ,date_dim - ,time_dim - ,ship_mode - where - cs_warehouse_sk = w_warehouse_sk - and cs_sold_date_sk = d_date_sk - and cs_sold_time_sk = t_time_sk - and cs_ship_mode_sk = sm_ship_mode_sk - and d_year = 2001 - and t_time between 42970 AND 42970+28800 - and sm_carrier in ('ORIENTAL','BOXBUNDLES') - group by - w_warehouse_name - ,w_warehouse_sq_ft - ,w_city - ,w_county - ,w_state - ,w_country - ,d_year - ) x - group by - w_warehouse_name - ,w_warehouse_sq_ft - ,w_city - ,w_county - ,w_state - ,w_country - ,ship_carriers - ,year - order by w_warehouse_name - limit 100; - --- end query 66 in stream 0 using template query66.tpl --- start query 67 in stream 0 using template query67.tpl -select * -from (select i_category - ,i_class - ,i_brand - ,i_product_name - ,d_year - ,d_qoy - ,d_moy - ,s_store_id - ,sumsales - ,rank() over (partition by i_category order by sumsales desc) rk - from (select i_category - ,i_class - ,i_brand - ,i_product_name - ,d_year - ,d_qoy - ,d_moy - ,s_store_id - ,sum(coalesce(ss_sales_price*ss_quantity,0)) sumsales - from store_sales - ,date_dim - ,store - ,item - where ss_sold_date_sk=d_date_sk - and ss_item_sk=i_item_sk - and ss_store_sk = s_store_sk - and d_month_seq between 1217 and 1217+11 - group by rollup(i_category, i_class, i_brand, i_product_name, d_year, d_qoy, d_moy,s_store_id))dw1) dw2 -where rk <= 100 -order by i_category - ,i_class - ,i_brand - ,i_product_name - ,d_year - ,d_qoy - ,d_moy - ,s_store_id - ,sumsales - ,rk -limit 100; - --- end query 67 in stream 0 using template query67.tpl --- start query 68 in stream 0 using template query68.tpl -select c_last_name - ,c_first_name - ,ca_city - ,bought_city - ,ss_ticket_number - ,extended_price - ,extended_tax - ,list_price - from (select ss_ticket_number - ,ss_customer_sk - ,ca_city bought_city - ,sum(ss_ext_sales_price) extended_price - ,sum(ss_ext_list_price) list_price - ,sum(ss_ext_tax) extended_tax - from store_sales - ,date_dim - ,store - ,household_demographics - ,customer_address - where store_sales.ss_sold_date_sk = date_dim.d_date_sk - and store_sales.ss_store_sk = store.s_store_sk - and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk - and store_sales.ss_addr_sk = customer_address.ca_address_sk - and date_dim.d_dom between 1 and 2 - and (household_demographics.hd_dep_count = 3 or - household_demographics.hd_vehicle_count= 4) - and date_dim.d_year in (1998,1998+1,1998+2) - and store.s_city in ('Fairview','Midway') - group by ss_ticket_number - ,ss_customer_sk - ,ss_addr_sk,ca_city) dn - ,customer - ,customer_address current_addr - where ss_customer_sk = c_customer_sk - and customer.c_current_addr_sk = current_addr.ca_address_sk - and current_addr.ca_city <> bought_city - order by c_last_name - ,ss_ticket_number - limit 100; - --- end query 68 in stream 0 using template query68.tpl --- start query 69 in stream 0 using template query69.tpl -select - cd_gender, - cd_marital_status, - cd_education_status, - count(*) cnt1, - cd_purchase_estimate, - count(*) cnt2, - cd_credit_rating, - count(*) cnt3 - from - customer c,customer_address ca,customer_demographics - where - c.c_current_addr_sk = ca.ca_address_sk and - ca_state in ('IL','TX','ME') and - cd_demo_sk = c.c_current_cdemo_sk and - exists (select * - from store_sales,date_dim - where c.c_customer_sk = ss_customer_sk and - ss_sold_date_sk = d_date_sk and - d_year = 2002 and - d_moy between 1 and 1+2) and - (not exists (select * - from web_sales,date_dim - where c.c_customer_sk = ws_bill_customer_sk and - ws_sold_date_sk = d_date_sk and - d_year = 2002 and - d_moy between 1 and 1+2) and - not exists (select * - from catalog_sales,date_dim - where c.c_customer_sk = cs_ship_customer_sk and - cs_sold_date_sk = d_date_sk and - d_year = 2002 and - d_moy between 1 and 1+2)) - group by cd_gender, - cd_marital_status, - cd_education_status, - cd_purchase_estimate, - cd_credit_rating - order by cd_gender, - cd_marital_status, - cd_education_status, - cd_purchase_estimate, - cd_credit_rating - limit 100; - --- end query 69 in stream 0 using template query69.tpl --- start query 70 in stream 0 using template query70.tpl -select * -from (select - sum(ss_net_profit) as total_sum - ,s_state - ,s_county - ,grouping(s_state)+grouping(s_county) as lochierarchy - ,rank() over ( - partition by grouping(s_state)+grouping(s_county), - case when grouping(s_county) = 0 then s_state end - order by sum(ss_net_profit) desc) as rank_within_parent - from - store_sales - ,date_dim d1 - ,store - where - d1.d_month_seq between 1220 and 1220+11 - and d1.d_date_sk = ss_sold_date_sk - and s_store_sk = ss_store_sk - and s_state in - ( select s_state - from (select s_state as s_state, - rank() over ( partition by s_state order by sum(ss_net_profit) desc) as ranking - from store_sales, store, date_dim - where d_month_seq between 1220 and 1220+11 - and d_date_sk = ss_sold_date_sk - and s_store_sk = ss_store_sk - group by s_state - ) tmp1 - where ranking <= 5 - ) - group by rollup(s_state,s_county)) as sub - order by - lochierarchy desc - ,case when lochierarchy = 0 then s_state end - ,rank_within_parent - limit 100; - --- end query 70 in stream 0 using template query70.tpl --- start query 71 in stream 0 using template query71.tpl -select i_brand_id brand_id, i_brand brand,t_hour,t_minute, - sum(ext_price) ext_price - from item, (select ws_ext_sales_price as ext_price, - ws_sold_date_sk as sold_date_sk, - ws_item_sk as sold_item_sk, - ws_sold_time_sk as time_sk - from web_sales,date_dim - where d_date_sk = ws_sold_date_sk - and d_moy=12 - and d_year=2002 - union all - select cs_ext_sales_price as ext_price, - cs_sold_date_sk as sold_date_sk, - cs_item_sk as sold_item_sk, - cs_sold_time_sk as time_sk - from catalog_sales,date_dim - where d_date_sk = cs_sold_date_sk - and d_moy=12 - and d_year=2002 - union all - select ss_ext_sales_price as ext_price, - ss_sold_date_sk as sold_date_sk, - ss_item_sk as sold_item_sk, - ss_sold_time_sk as time_sk - from store_sales,date_dim - where d_date_sk = ss_sold_date_sk - and d_moy=12 - and d_year=2002 - ) tmp,time_dim - where - sold_item_sk = i_item_sk - and i_manager_id=1 - and time_sk = t_time_sk - and (t_meal_time = 'breakfast' or t_meal_time = 'dinner') - group by i_brand, i_brand_id,t_hour,t_minute - order by ext_price desc, i_brand_id - ; - --- end query 71 in stream 0 using template query71.tpl --- start query 72 in stream 0 using template query72.tpl -select i_item_desc - ,w_warehouse_name - ,d1.d_week_seq - ,sum(case when p_promo_sk is null then 1 else 0 end) no_promo - ,sum(case when p_promo_sk is not null then 1 else 0 end) promo - ,count(*) total_cnt -from catalog_sales -join inventory on (cs_item_sk = inv_item_sk) -join warehouse on (w_warehouse_sk=inv_warehouse_sk) -join item on (i_item_sk = cs_item_sk) -join customer_demographics on (cs_bill_cdemo_sk = cd_demo_sk) -join household_demographics on (cs_bill_hdemo_sk = hd_demo_sk) -join date_dim d1 on (cs_sold_date_sk = d1.d_date_sk) -join date_dim d2 on (inv_date_sk = d2.d_date_sk) -join date_dim d3 on (cs_ship_date_sk = d3.d_date_sk) -left outer join promotion on (cs_promo_sk=p_promo_sk) -left outer join catalog_returns on (cr_item_sk = cs_item_sk and cr_order_number = cs_order_number) -where d1.d_week_seq = d2.d_week_seq - and inv_quantity_on_hand < cs_quantity - and d3.d_date > d1.d_date + 5 - and hd_buy_potential = '1001-5000' - and d1.d_year = 1998 - and cd_marital_status = 'S' -group by i_item_desc,w_warehouse_name,d1.d_week_seq -order by total_cnt desc, i_item_desc, w_warehouse_name, d_week_seq -limit 100; - --- end query 72 in stream 0 using template query72.tpl --- start query 73 in stream 0 using template query73.tpl -select c_last_name - ,c_first_name - ,c_salutation - ,c_preferred_cust_flag - ,ss_ticket_number - ,cnt from - (select ss_ticket_number - ,ss_customer_sk - ,count(*) cnt - from store_sales,date_dim,store,household_demographics - where store_sales.ss_sold_date_sk = date_dim.d_date_sk - and store_sales.ss_store_sk = store.s_store_sk - and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk - and date_dim.d_dom between 1 and 2 - and (household_demographics.hd_buy_potential = '1001-5000' or - household_demographics.hd_buy_potential = '5001-10000') - and household_demographics.hd_vehicle_count > 0 - and case when household_demographics.hd_vehicle_count > 0 then - household_demographics.hd_dep_count/ household_demographics.hd_vehicle_count else null end > 1 - and date_dim.d_year in (2000,2000+1,2000+2) - and store.s_county in ('Williamson County','Williamson County','Williamson County','Williamson County') - group by ss_ticket_number,ss_customer_sk) dj,customer - where ss_customer_sk = c_customer_sk - and cnt between 1 and 5 - order by cnt desc, c_last_name asc; - --- end query 73 in stream 0 using template query73.tpl --- start query 74 in stream 0 using template query74.tpl -with year_total as ( - select c_customer_id customer_id - ,c_first_name customer_first_name - ,c_last_name customer_last_name - ,d_year as year - ,max(ss_net_paid) year_total - ,'s' sale_type - from customer - ,store_sales - ,date_dim - where c_customer_sk = ss_customer_sk - and ss_sold_date_sk = d_date_sk - and d_year in (1999,1999+1) - group by c_customer_id - ,c_first_name - ,c_last_name - ,d_year - union all - select c_customer_id customer_id - ,c_first_name customer_first_name - ,c_last_name customer_last_name - ,d_year as year - ,max(ws_net_paid) year_total - ,'w' sale_type - from customer - ,web_sales - ,date_dim - where c_customer_sk = ws_bill_customer_sk - and ws_sold_date_sk = d_date_sk - and d_year in (1999,1999+1) - group by c_customer_id - ,c_first_name - ,c_last_name - ,d_year - ) - select - t_s_secyear.customer_id, t_s_secyear.customer_first_name, t_s_secyear.customer_last_name - from year_total t_s_firstyear - ,year_total t_s_secyear - ,year_total t_w_firstyear - ,year_total t_w_secyear - where t_s_secyear.customer_id = t_s_firstyear.customer_id - and t_s_firstyear.customer_id = t_w_secyear.customer_id - and t_s_firstyear.customer_id = t_w_firstyear.customer_id - and t_s_firstyear.sale_type = 's' - and t_w_firstyear.sale_type = 'w' - and t_s_secyear.sale_type = 's' - and t_w_secyear.sale_type = 'w' - and t_s_firstyear.year = 1999 - and t_s_secyear.year = 1999+1 - and t_w_firstyear.year = 1999 - and t_w_secyear.year = 1999+1 - and t_s_firstyear.year_total > 0 - and t_w_firstyear.year_total > 0 - and case when t_w_firstyear.year_total > 0 then t_w_secyear.year_total / t_w_firstyear.year_total else null end - > case when t_s_firstyear.year_total > 0 then t_s_secyear.year_total / t_s_firstyear.year_total else null end - order by 1,3,2 -limit 100; - --- end query 74 in stream 0 using template query74.tpl --- start query 75 in stream 0 using template query75.tpl -WITH all_sales AS ( - SELECT d_year - ,i_brand_id - ,i_class_id - ,i_category_id - ,i_manufact_id - ,SUM(sales_cnt) AS sales_cnt - ,SUM(sales_amt) AS sales_amt - FROM (SELECT d_year - ,i_brand_id - ,i_class_id - ,i_category_id - ,i_manufact_id - ,cs_quantity - COALESCE(cr_return_quantity,0) AS sales_cnt - ,cs_ext_sales_price - COALESCE(cr_return_amount,0.0) AS sales_amt - FROM catalog_sales JOIN item ON i_item_sk=cs_item_sk - JOIN date_dim ON d_date_sk=cs_sold_date_sk - LEFT JOIN catalog_returns ON (cs_order_number=cr_order_number - AND cs_item_sk=cr_item_sk) - WHERE i_category='Sports' - UNION - SELECT d_year - ,i_brand_id - ,i_class_id - ,i_category_id - ,i_manufact_id - ,ss_quantity - COALESCE(sr_return_quantity,0) AS sales_cnt - ,ss_ext_sales_price - COALESCE(sr_return_amt,0.0) AS sales_amt - FROM store_sales JOIN item ON i_item_sk=ss_item_sk - JOIN date_dim ON d_date_sk=ss_sold_date_sk - LEFT JOIN store_returns ON (ss_ticket_number=sr_ticket_number - AND ss_item_sk=sr_item_sk) - WHERE i_category='Sports' - UNION - SELECT d_year - ,i_brand_id - ,i_class_id - ,i_category_id - ,i_manufact_id - ,ws_quantity - COALESCE(wr_return_quantity,0) AS sales_cnt - ,ws_ext_sales_price - COALESCE(wr_return_amt,0.0) AS sales_amt - FROM web_sales JOIN item ON i_item_sk=ws_item_sk - JOIN date_dim ON d_date_sk=ws_sold_date_sk - LEFT JOIN web_returns ON (ws_order_number=wr_order_number - AND ws_item_sk=wr_item_sk) - WHERE i_category='Sports') sales_detail - GROUP BY d_year, i_brand_id, i_class_id, i_category_id, i_manufact_id) - SELECT prev_yr.d_year AS prev_year - ,curr_yr.d_year AS year - ,curr_yr.i_brand_id - ,curr_yr.i_class_id - ,curr_yr.i_category_id - ,curr_yr.i_manufact_id - ,prev_yr.sales_cnt AS prev_yr_cnt - ,curr_yr.sales_cnt AS curr_yr_cnt - ,curr_yr.sales_cnt-prev_yr.sales_cnt AS sales_cnt_diff - ,curr_yr.sales_amt-prev_yr.sales_amt AS sales_amt_diff - FROM all_sales curr_yr, all_sales prev_yr - WHERE curr_yr.i_brand_id=prev_yr.i_brand_id - AND curr_yr.i_class_id=prev_yr.i_class_id - AND curr_yr.i_category_id=prev_yr.i_category_id - AND curr_yr.i_manufact_id=prev_yr.i_manufact_id - AND curr_yr.d_year=2002 - AND prev_yr.d_year=2002-1 - AND CAST(curr_yr.sales_cnt AS DECIMAL(17,2))/CAST(prev_yr.sales_cnt AS DECIMAL(17,2))<0.9 - ORDER BY sales_cnt_diff,sales_amt_diff - limit 100; - --- end query 75 in stream 0 using template query75.tpl --- start query 76 in stream 0 using template query76.tpl -select channel, col_name, d_year, d_qoy, i_category, COUNT(*) sales_cnt, SUM(ext_sales_price) sales_amt FROM ( - SELECT 'store' as channel, 'ss_customer_sk' col_name, d_year, d_qoy, i_category, ss_ext_sales_price ext_sales_price - FROM store_sales, item, date_dim - WHERE ss_customer_sk IS NULL - AND ss_sold_date_sk=d_date_sk - AND ss_item_sk=i_item_sk - UNION ALL - SELECT 'web' as channel, 'ws_promo_sk' col_name, d_year, d_qoy, i_category, ws_ext_sales_price ext_sales_price - FROM web_sales, item, date_dim - WHERE ws_promo_sk IS NULL - AND ws_sold_date_sk=d_date_sk - AND ws_item_sk=i_item_sk - UNION ALL - SELECT 'catalog' as channel, 'cs_bill_customer_sk' col_name, d_year, d_qoy, i_category, cs_ext_sales_price ext_sales_price - FROM catalog_sales, item, date_dim - WHERE cs_bill_customer_sk IS NULL - AND cs_sold_date_sk=d_date_sk - AND cs_item_sk=i_item_sk) foo -GROUP BY channel, col_name, d_year, d_qoy, i_category -ORDER BY channel, col_name, d_year, d_qoy, i_category -limit 100; - --- end query 76 in stream 0 using template query76.tpl --- start query 77 in stream 0 using template query77.tpl -with ss as - (select s_store_sk, - sum(ss_ext_sales_price) as sales, - sum(ss_net_profit) as profit - from store_sales, - date_dim, - store - where ss_sold_date_sk = d_date_sk - and d_date between cast('2000-08-10' as date) - and (cast('2000-08-10' as date) + interval '30 days') - and ss_store_sk = s_store_sk - group by s_store_sk) - , - sr as - (select s_store_sk, - sum(sr_return_amt) as returns, - sum(sr_net_loss) as profit_loss - from store_returns, - date_dim, - store - where sr_returned_date_sk = d_date_sk - and d_date between cast('2000-08-10' as date) - and (cast('2000-08-10' as date) + interval '30 days') - and sr_store_sk = s_store_sk - group by s_store_sk), - cs as - (select cs_call_center_sk, - sum(cs_ext_sales_price) as sales, - sum(cs_net_profit) as profit - from catalog_sales, - date_dim - where cs_sold_date_sk = d_date_sk - and d_date between cast('2000-08-10' as date) - and (cast('2000-08-10' as date) + interval '30 days') - group by cs_call_center_sk - ), - cr as - (select cr_call_center_sk, - sum(cr_return_amount) as returns, - sum(cr_net_loss) as profit_loss - from catalog_returns, - date_dim - where cr_returned_date_sk = d_date_sk - and d_date between cast('2000-08-10' as date) - and (cast('2000-08-10' as date) + interval '30 days') - group by cr_call_center_sk - ), - ws as - ( select wp_web_page_sk, - sum(ws_ext_sales_price) as sales, - sum(ws_net_profit) as profit - from web_sales, - date_dim, - web_page - where ws_sold_date_sk = d_date_sk - and d_date between cast('2000-08-10' as date) - and (cast('2000-08-10' as date) + interval '30 days') - and ws_web_page_sk = wp_web_page_sk - group by wp_web_page_sk), - wr as - (select wp_web_page_sk, - sum(wr_return_amt) as returns, - sum(wr_net_loss) as profit_loss - from web_returns, - date_dim, - web_page - where wr_returned_date_sk = d_date_sk - and d_date between cast('2000-08-10' as date) - and (cast('2000-08-10' as date) + interval '30 days') - and wr_web_page_sk = wp_web_page_sk - group by wp_web_page_sk) - select channel - , id - , sum(sales) as sales - , sum(returns) as returns - , sum(profit) as profit - from - (select 'store channel' as channel - , ss.s_store_sk as id - , sales - , coalesce(returns, 0) as returns - , (profit - coalesce(profit_loss,0)) as profit - from ss left join sr - on ss.s_store_sk = sr.s_store_sk - union all - select 'catalog channel' as channel - , cs_call_center_sk as id - , sales - , returns - , (profit - profit_loss) as profit - from cs - , cr - union all - select 'web channel' as channel - , ws.wp_web_page_sk as id - , sales - , coalesce(returns, 0) as returns - , (profit - coalesce(profit_loss,0)) as profit - from ws left join wr - on ws.wp_web_page_sk = wr.wp_web_page_sk - ) x - group by rollup (channel, id) - order by channel - ,id - limit 100; - --- end query 77 in stream 0 using template query77.tpl --- start query 78 in stream 0 using template query78.tpl -with ws as - (select d_year AS ws_sold_year, ws_item_sk, - ws_bill_customer_sk ws_customer_sk, - sum(ws_quantity) ws_qty, - sum(ws_wholesale_cost) ws_wc, - sum(ws_sales_price) ws_sp - from web_sales - left join web_returns on wr_order_number=ws_order_number and ws_item_sk=wr_item_sk - join date_dim on ws_sold_date_sk = d_date_sk - where wr_order_number is null - group by d_year, ws_item_sk, ws_bill_customer_sk - ), -cs as - (select d_year AS cs_sold_year, cs_item_sk, - cs_bill_customer_sk cs_customer_sk, - sum(cs_quantity) cs_qty, - sum(cs_wholesale_cost) cs_wc, - sum(cs_sales_price) cs_sp - from catalog_sales - left join catalog_returns on cr_order_number=cs_order_number and cs_item_sk=cr_item_sk - join date_dim on cs_sold_date_sk = d_date_sk - where cr_order_number is null - group by d_year, cs_item_sk, cs_bill_customer_sk - ), -ss as - (select d_year AS ss_sold_year, ss_item_sk, - ss_customer_sk, - sum(ss_quantity) ss_qty, - sum(ss_wholesale_cost) ss_wc, - sum(ss_sales_price) ss_sp - from store_sales - left join store_returns on sr_ticket_number=ss_ticket_number and ss_item_sk=sr_item_sk - join date_dim on ss_sold_date_sk = d_date_sk - where sr_ticket_number is null - group by d_year, ss_item_sk, ss_customer_sk - ) - select -ss_customer_sk, -round(ss_qty/(coalesce(ws_qty,0)+coalesce(cs_qty,0)),2) ratio, -ss_qty store_qty, ss_wc store_wholesale_cost, ss_sp store_sales_price, -coalesce(ws_qty,0)+coalesce(cs_qty,0) other_chan_qty, -coalesce(ws_wc,0)+coalesce(cs_wc,0) other_chan_wholesale_cost, -coalesce(ws_sp,0)+coalesce(cs_sp,0) other_chan_sales_price -from ss -left join ws on (ws_sold_year=ss_sold_year and ws_item_sk=ss_item_sk and ws_customer_sk=ss_customer_sk) -left join cs on (cs_sold_year=ss_sold_year and cs_item_sk=ss_item_sk and cs_customer_sk=ss_customer_sk) -where (coalesce(ws_qty,0)>0 or coalesce(cs_qty, 0)>0) and ss_sold_year=1998 -order by - ss_customer_sk, - ss_qty desc, ss_wc desc, ss_sp desc, - other_chan_qty, - other_chan_wholesale_cost, - other_chan_sales_price, - ratio -limit 100; - --- end query 78 in stream 0 using template query78.tpl --- start query 79 in stream 0 using template query79.tpl -select - c_last_name,c_first_name,substr(s_city,1,30),ss_ticket_number,amt,profit - from - (select ss_ticket_number - ,ss_customer_sk - ,store.s_city - ,sum(ss_coupon_amt) amt - ,sum(ss_net_profit) profit - from store_sales,date_dim,store,household_demographics - where store_sales.ss_sold_date_sk = date_dim.d_date_sk - and store_sales.ss_store_sk = store.s_store_sk - and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk - and (household_demographics.hd_dep_count = 7 or household_demographics.hd_vehicle_count > -1) - and date_dim.d_dow = 1 - and date_dim.d_year in (2000,2000+1,2000+2) - and store.s_number_employees between 200 and 295 - group by ss_ticket_number,ss_customer_sk,ss_addr_sk,store.s_city) ms,customer - where ss_customer_sk = c_customer_sk - order by c_last_name,c_first_name,substr(s_city,1,30), profit -limit 100; - --- end query 79 in stream 0 using template query79.tpl --- start query 80 in stream 0 using template query80.tpl -with ssr as - (select s_store_id as store_id, - sum(ss_ext_sales_price) as sales, - sum(coalesce(sr_return_amt, 0)) as returns, - sum(ss_net_profit - coalesce(sr_net_loss, 0)) as profit - from store_sales left outer join store_returns on - (ss_item_sk = sr_item_sk and ss_ticket_number = sr_ticket_number), - date_dim, - store, - item, - promotion - where ss_sold_date_sk = d_date_sk - and d_date between cast('2002-08-14' as date) - and (cast('2002-08-14' as date) + interval '30 days') - and ss_store_sk = s_store_sk - and ss_item_sk = i_item_sk - and i_current_price > 50 - and ss_promo_sk = p_promo_sk - and p_channel_tv = 'N' - group by s_store_id) - , - csr as - (select cp_catalog_page_id as catalog_page_id, - sum(cs_ext_sales_price) as sales, - sum(coalesce(cr_return_amount, 0)) as returns, - sum(cs_net_profit - coalesce(cr_net_loss, 0)) as profit - from catalog_sales left outer join catalog_returns on - (cs_item_sk = cr_item_sk and cs_order_number = cr_order_number), - date_dim, - catalog_page, - item, - promotion - where cs_sold_date_sk = d_date_sk - and d_date between cast('2002-08-14' as date) - and (cast('2002-08-14' as date) + interval '30 days') - and cs_catalog_page_sk = cp_catalog_page_sk - and cs_item_sk = i_item_sk - and i_current_price > 50 - and cs_promo_sk = p_promo_sk - and p_channel_tv = 'N' -group by cp_catalog_page_id) - , - wsr as - (select web_site_id, - sum(ws_ext_sales_price) as sales, - sum(coalesce(wr_return_amt, 0)) as returns, - sum(ws_net_profit - coalesce(wr_net_loss, 0)) as profit - from web_sales left outer join web_returns on - (ws_item_sk = wr_item_sk and ws_order_number = wr_order_number), - date_dim, - web_site, - item, - promotion - where ws_sold_date_sk = d_date_sk - and d_date between cast('2002-08-14' as date) - and (cast('2002-08-14' as date) + interval '30 days') - and ws_web_site_sk = web_site_sk - and ws_item_sk = i_item_sk - and i_current_price > 50 - and ws_promo_sk = p_promo_sk - and p_channel_tv = 'N' -group by web_site_id) - select channel - , id - , sum(sales) as sales - , sum(returns) as returns - , sum(profit) as profit - from - (select 'store channel' as channel - , 'store' || store_id as id - , sales - , returns - , profit - from ssr - union all - select 'catalog channel' as channel - , 'catalog_page' || catalog_page_id as id - , sales - , returns - , profit - from csr - union all - select 'web channel' as channel - , 'web_site' || web_site_id as id - , sales - , returns - , profit - from wsr - ) x - group by rollup (channel, id) - order by channel - ,id - limit 100; - --- end query 80 in stream 0 using template query80.tpl --- start query 81 in stream 0 using template query81.tpl -with customer_total_return as - (select cr_returning_customer_sk as ctr_customer_sk - ,ca_state as ctr_state, - sum(cr_return_amt_inc_tax) as ctr_total_return - from catalog_returns - ,date_dim - ,customer_address - where cr_returned_date_sk = d_date_sk - and d_year =2001 - and cr_returning_addr_sk = ca_address_sk - group by cr_returning_customer_sk - ,ca_state ) - select c_customer_id,c_salutation,c_first_name,c_last_name,ca_street_number,ca_street_name - ,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset - ,ca_location_type,ctr_total_return - from customer_total_return ctr1 - ,customer_address - ,customer - where ctr1.ctr_total_return > (select avg(ctr_total_return)*1.2 - from customer_total_return ctr2 - where ctr1.ctr_state = ctr2.ctr_state) - and ca_address_sk = c_current_addr_sk - and ca_state = 'TN' - and ctr1.ctr_customer_sk = c_customer_sk - order by c_customer_id,c_salutation,c_first_name,c_last_name,ca_street_number,ca_street_name - ,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset - ,ca_location_type,ctr_total_return - limit 100; - --- end query 81 in stream 0 using template query81.tpl --- start query 82 in stream 0 using template query82.tpl -select i_item_id - ,i_item_desc - ,i_current_price - from item, inventory, date_dim, store_sales - where i_current_price between 58 and 58+30 - and inv_item_sk = i_item_sk - and d_date_sk=inv_date_sk - and d_date between cast('2001-01-13' as date) and (cast('2001-01-13' as date) + interval '60 days') - and i_manufact_id in (259,559,580,485) - and inv_quantity_on_hand between 100 and 500 - and ss_item_sk = i_item_sk - group by i_item_id,i_item_desc,i_current_price - order by i_item_id - limit 100; - --- end query 82 in stream 0 using template query82.tpl --- start query 83 in stream 0 using template query83.tpl -with sr_items as - (select i_item_id item_id, - sum(sr_return_quantity) sr_item_qty - from store_returns, - item, - date_dim - where sr_item_sk = i_item_sk - and d_date in - (select d_date - from date_dim - where d_week_seq in - (select d_week_seq - from date_dim - where d_date in ('2001-07-13','2001-09-10','2001-11-16'))) - and sr_returned_date_sk = d_date_sk - group by i_item_id), - cr_items as - (select i_item_id item_id, - sum(cr_return_quantity) cr_item_qty - from catalog_returns, - item, - date_dim - where cr_item_sk = i_item_sk - and d_date in - (select d_date - from date_dim - where d_week_seq in - (select d_week_seq - from date_dim - where d_date in ('2001-07-13','2001-09-10','2001-11-16'))) - and cr_returned_date_sk = d_date_sk - group by i_item_id), - wr_items as - (select i_item_id item_id, - sum(wr_return_quantity) wr_item_qty - from web_returns, - item, - date_dim - where wr_item_sk = i_item_sk - and d_date in - (select d_date - from date_dim - where d_week_seq in - (select d_week_seq - from date_dim - where d_date in ('2001-07-13','2001-09-10','2001-11-16'))) - and wr_returned_date_sk = d_date_sk - group by i_item_id) - select sr_items.item_id - ,sr_item_qty - ,sr_item_qty/(sr_item_qty+cr_item_qty+wr_item_qty)/3.0 * 100 sr_dev - ,cr_item_qty - ,cr_item_qty/(sr_item_qty+cr_item_qty+wr_item_qty)/3.0 * 100 cr_dev - ,wr_item_qty - ,wr_item_qty/(sr_item_qty+cr_item_qty+wr_item_qty)/3.0 * 100 wr_dev - ,(sr_item_qty+cr_item_qty+wr_item_qty)/3.0 average - from sr_items - ,cr_items - ,wr_items - where sr_items.item_id=cr_items.item_id - and sr_items.item_id=wr_items.item_id - order by sr_items.item_id - ,sr_item_qty - limit 100; - --- end query 83 in stream 0 using template query83.tpl --- start query 84 in stream 0 using template query84.tpl -select c_customer_id as customer_id - , coalesce(c_last_name,'') || ', ' || coalesce(c_first_name,'') as customername - from customer - ,customer_address - ,customer_demographics - ,household_demographics - ,income_band - ,store_returns - where ca_city = 'Woodland' - and c_current_addr_sk = ca_address_sk - and ib_lower_bound >= 60306 - and ib_upper_bound <= 60306 + 50000 - and ib_income_band_sk = hd_income_band_sk - and cd_demo_sk = c_current_cdemo_sk - and hd_demo_sk = c_current_hdemo_sk - and sr_cdemo_sk = cd_demo_sk - order by c_customer_id - limit 100; - --- end query 84 in stream 0 using template query84.tpl --- start query 85 in stream 0 using template query85.tpl -select substr(r_reason_desc,1,20) - ,avg(ws_quantity) - ,avg(wr_refunded_cash) - ,avg(wr_fee) - from web_sales, web_returns, web_page, customer_demographics cd1, - customer_demographics cd2, customer_address, date_dim, reason - where ws_web_page_sk = wp_web_page_sk - and ws_item_sk = wr_item_sk - and ws_order_number = wr_order_number - and ws_sold_date_sk = d_date_sk and d_year = 1998 - and cd1.cd_demo_sk = wr_refunded_cdemo_sk - and cd2.cd_demo_sk = wr_returning_cdemo_sk - and ca_address_sk = wr_refunded_addr_sk - and r_reason_sk = wr_reason_sk - and - ( - ( - cd1.cd_marital_status = 'D' - and - cd1.cd_marital_status = cd2.cd_marital_status - and - cd1.cd_education_status = 'Primary' - and - cd1.cd_education_status = cd2.cd_education_status - and - ws_sales_price between 100.00 and 150.00 - ) - or - ( - cd1.cd_marital_status = 'S' - and - cd1.cd_marital_status = cd2.cd_marital_status - and - cd1.cd_education_status = 'College' - and - cd1.cd_education_status = cd2.cd_education_status - and - ws_sales_price between 50.00 and 100.00 - ) - or - ( - cd1.cd_marital_status = 'U' - and - cd1.cd_marital_status = cd2.cd_marital_status - and - cd1.cd_education_status = 'Advanced Degree' - and - cd1.cd_education_status = cd2.cd_education_status - and - ws_sales_price between 150.00 and 200.00 - ) - ) - and - ( - ( - ca_country = 'United States' - and - ca_state in ('NC', 'TX', 'IA') - and ws_net_profit between 100 and 200 - ) - or - ( - ca_country = 'United States' - and - ca_state in ('WI', 'WV', 'GA') - and ws_net_profit between 150 and 300 - ) - or - ( - ca_country = 'United States' - and - ca_state in ('OK', 'VA', 'KY') - and ws_net_profit between 50 and 250 - ) - ) -group by r_reason_desc -order by substr(r_reason_desc,1,20) - ,avg(ws_quantity) - ,avg(wr_refunded_cash) - ,avg(wr_fee) -limit 100; - --- end query 85 in stream 0 using template query85.tpl --- start query 86 in stream 0 using template query86.tpl -select * -from (select - sum(ws_net_paid) as total_sum - ,i_category - ,i_class - ,grouping(i_category)+grouping(i_class) as lochierarchy - ,rank() over ( - partition by grouping(i_category)+grouping(i_class), - case when grouping(i_class) = 0 then i_category end - order by sum(ws_net_paid) desc) as rank_within_parent - from - web_sales - ,date_dim d1 - ,item - where - d1.d_month_seq between 1186 and 1186+11 - and d1.d_date_sk = ws_sold_date_sk - and i_item_sk = ws_item_sk - group by rollup(i_category,i_class)) as sub - order by - lochierarchy desc, - case when lochierarchy = 0 then i_category end, - rank_within_parent - limit 100; - --- end query 86 in stream 0 using template query86.tpl --- start query 87 in stream 0 using template query87.tpl -select count(*) -from ((select distinct c_last_name, c_first_name, d_date - from store_sales, date_dim, customer - where store_sales.ss_sold_date_sk = date_dim.d_date_sk - and store_sales.ss_customer_sk = customer.c_customer_sk - and d_month_seq between 1202 and 1202+11) - except - (select distinct c_last_name, c_first_name, d_date - from catalog_sales, date_dim, customer - where catalog_sales.cs_sold_date_sk = date_dim.d_date_sk - and catalog_sales.cs_bill_customer_sk = customer.c_customer_sk - and d_month_seq between 1202 and 1202+11) - except - (select distinct c_last_name, c_first_name, d_date - from web_sales, date_dim, customer - where web_sales.ws_sold_date_sk = date_dim.d_date_sk - and web_sales.ws_bill_customer_sk = customer.c_customer_sk - and d_month_seq between 1202 and 1202+11) -) cool_cust -; - --- end query 87 in stream 0 using template query87.tpl --- start query 88 in stream 0 using template query88.tpl -select * -from - (select count(*) h8_30_to_9 - from store_sales, household_demographics , time_dim, store - where ss_sold_time_sk = time_dim.t_time_sk - and ss_hdemo_sk = household_demographics.hd_demo_sk - and ss_store_sk = s_store_sk - and time_dim.t_hour = 8 - and time_dim.t_minute >= 30 - and ((household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2) or - (household_demographics.hd_dep_count = -1 and household_demographics.hd_vehicle_count<=-1+2) or - (household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2)) - and store.s_store_name = 'ese') s1, - (select count(*) h9_to_9_30 - from store_sales, household_demographics , time_dim, store - where ss_sold_time_sk = time_dim.t_time_sk - and ss_hdemo_sk = household_demographics.hd_demo_sk - and ss_store_sk = s_store_sk - and time_dim.t_hour = 9 - and time_dim.t_minute < 30 - and ((household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2) or - (household_demographics.hd_dep_count = -1 and household_demographics.hd_vehicle_count<=-1+2) or - (household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2)) - and store.s_store_name = 'ese') s2, - (select count(*) h9_30_to_10 - from store_sales, household_demographics , time_dim, store - where ss_sold_time_sk = time_dim.t_time_sk - and ss_hdemo_sk = household_demographics.hd_demo_sk - and ss_store_sk = s_store_sk - and time_dim.t_hour = 9 - and time_dim.t_minute >= 30 - and ((household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2) or - (household_demographics.hd_dep_count = -1 and household_demographics.hd_vehicle_count<=-1+2) or - (household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2)) - and store.s_store_name = 'ese') s3, - (select count(*) h10_to_10_30 - from store_sales, household_demographics , time_dim, store - where ss_sold_time_sk = time_dim.t_time_sk - and ss_hdemo_sk = household_demographics.hd_demo_sk - and ss_store_sk = s_store_sk - and time_dim.t_hour = 10 - and time_dim.t_minute < 30 - and ((household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2) or - (household_demographics.hd_dep_count = -1 and household_demographics.hd_vehicle_count<=-1+2) or - (household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2)) - and store.s_store_name = 'ese') s4, - (select count(*) h10_30_to_11 - from store_sales, household_demographics , time_dim, store - where ss_sold_time_sk = time_dim.t_time_sk - and ss_hdemo_sk = household_demographics.hd_demo_sk - and ss_store_sk = s_store_sk - and time_dim.t_hour = 10 - and time_dim.t_minute >= 30 - and ((household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2) or - (household_demographics.hd_dep_count = -1 and household_demographics.hd_vehicle_count<=-1+2) or - (household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2)) - and store.s_store_name = 'ese') s5, - (select count(*) h11_to_11_30 - from store_sales, household_demographics , time_dim, store - where ss_sold_time_sk = time_dim.t_time_sk - and ss_hdemo_sk = household_demographics.hd_demo_sk - and ss_store_sk = s_store_sk - and time_dim.t_hour = 11 - and time_dim.t_minute < 30 - and ((household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2) or - (household_demographics.hd_dep_count = -1 and household_demographics.hd_vehicle_count<=-1+2) or - (household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2)) - and store.s_store_name = 'ese') s6, - (select count(*) h11_30_to_12 - from store_sales, household_demographics , time_dim, store - where ss_sold_time_sk = time_dim.t_time_sk - and ss_hdemo_sk = household_demographics.hd_demo_sk - and ss_store_sk = s_store_sk - and time_dim.t_hour = 11 - and time_dim.t_minute >= 30 - and ((household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2) or - (household_demographics.hd_dep_count = -1 and household_demographics.hd_vehicle_count<=-1+2) or - (household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2)) - and store.s_store_name = 'ese') s7, - (select count(*) h12_to_12_30 - from store_sales, household_demographics , time_dim, store - where ss_sold_time_sk = time_dim.t_time_sk - and ss_hdemo_sk = household_demographics.hd_demo_sk - and ss_store_sk = s_store_sk - and time_dim.t_hour = 12 - and time_dim.t_minute < 30 - and ((household_demographics.hd_dep_count = 0 and household_demographics.hd_vehicle_count<=0+2) or - (household_demographics.hd_dep_count = -1 and household_demographics.hd_vehicle_count<=-1+2) or - (household_demographics.hd_dep_count = 3 and household_demographics.hd_vehicle_count<=3+2)) - and store.s_store_name = 'ese') s8 -; - --- end query 88 in stream 0 using template query88.tpl --- start query 89 in stream 0 using template query89.tpl -select * -from( -select i_category, i_class, i_brand, - s_store_name, s_company_name, - d_moy, - sum(ss_sales_price) sum_sales, - avg(sum(ss_sales_price)) over - (partition by i_category, i_brand, s_store_name, s_company_name) - avg_monthly_sales -from item, store_sales, date_dim, store -where ss_item_sk = i_item_sk and - ss_sold_date_sk = d_date_sk and - ss_store_sk = s_store_sk and - d_year in (2001) and - ((i_category in ('Books','Children','Electronics') and - i_class in ('history','school-uniforms','audio') - ) - or (i_category in ('Men','Sports','Shoes') and - i_class in ('pants','tennis','womens') - )) -group by i_category, i_class, i_brand, - s_store_name, s_company_name, d_moy) tmp1 -where case when (avg_monthly_sales <> 0) then (abs(sum_sales - avg_monthly_sales) / avg_monthly_sales) else null end > 0.1 -order by sum_sales - avg_monthly_sales, s_store_name -limit 100; - --- end query 89 in stream 0 using template query89.tpl --- start query 90 in stream 0 using template query90.tpl -select cast(amc as decimal(15,4))/cast(pmc as decimal(15,4)) am_pm_ratio - from ( select count(*) amc - from web_sales, household_demographics , time_dim, web_page - where ws_sold_time_sk = time_dim.t_time_sk - and ws_ship_hdemo_sk = household_demographics.hd_demo_sk - and ws_web_page_sk = web_page.wp_web_page_sk - and time_dim.t_hour between 12 and 12+1 - and household_demographics.hd_dep_count = 6 - and web_page.wp_char_count between 5000 and 5200) at, - ( select count(*) pmc - from web_sales, household_demographics , time_dim, web_page - where ws_sold_time_sk = time_dim.t_time_sk - and ws_ship_hdemo_sk = household_demographics.hd_demo_sk - and ws_web_page_sk = web_page.wp_web_page_sk - and time_dim.t_hour between 14 and 14+1 - and household_demographics.hd_dep_count = 6 - and web_page.wp_char_count between 5000 and 5200) pt - order by am_pm_ratio - limit 100; - --- end query 90 in stream 0 using template query90.tpl --- start query 91 in stream 0 using template query91.tpl -select - cc_call_center_id Call_Center, - cc_name Call_Center_Name, - cc_manager Manager, - sum(cr_net_loss) Returns_Loss -from - call_center, - catalog_returns, - date_dim, - customer, - customer_address, - customer_demographics, - household_demographics -where - cr_call_center_sk = cc_call_center_sk -and cr_returned_date_sk = d_date_sk -and cr_returning_customer_sk= c_customer_sk -and cd_demo_sk = c_current_cdemo_sk -and hd_demo_sk = c_current_hdemo_sk -and ca_address_sk = c_current_addr_sk -and d_year = 2000 -and d_moy = 12 -and ( (cd_marital_status = 'M' and cd_education_status = 'Unknown') - or(cd_marital_status = 'W' and cd_education_status = 'Advanced Degree')) -and hd_buy_potential like 'Unknown%' -and ca_gmt_offset = -7 -group by cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status -order by sum(cr_net_loss) desc; - --- end query 91 in stream 0 using template query91.tpl --- start query 92 in stream 0 using template query92.tpl -select - sum(ws_ext_discount_amt) as "Excess Discount Amount" -from - web_sales - ,item - ,date_dim -where -i_manufact_id = 714 -and i_item_sk = ws_item_sk -and d_date between '2000-02-01' and - (cast('2000-02-01' as date) + interval '90 days') -and d_date_sk = ws_sold_date_sk -and ws_ext_discount_amt - > ( - SELECT - 1.3 * avg(ws_ext_discount_amt) - FROM - web_sales - ,date_dim - WHERE - ws_item_sk = i_item_sk - and d_date between '2000-02-01' and - (cast('2000-02-01' as date) + interval '90 days') - and d_date_sk = ws_sold_date_sk - ) -order by sum(ws_ext_discount_amt) -limit 100; - --- end query 92 in stream 0 using template query92.tpl --- start query 93 in stream 0 using template query93.tpl -select ss_customer_sk - ,sum(act_sales) sumsales - from (select ss_item_sk - ,ss_ticket_number - ,ss_customer_sk - ,case when sr_return_quantity is not null then (ss_quantity-sr_return_quantity)*ss_sales_price - else (ss_quantity*ss_sales_price) end act_sales - from store_sales left outer join store_returns on (sr_item_sk = ss_item_sk - and sr_ticket_number = ss_ticket_number) - ,reason - where sr_reason_sk = r_reason_sk - and r_reason_desc = 'reason 58') t - group by ss_customer_sk - order by sumsales, ss_customer_sk -limit 100; - --- end query 93 in stream 0 using template query93.tpl --- start query 94 in stream 0 using template query94.tpl -select - count(distinct ws_order_number) as "order count" - ,sum(ws_ext_ship_cost) as "total shipping cost" - ,sum(ws_net_profit) as "total net profit" -from - web_sales ws1 - ,date_dim - ,customer_address - ,web_site -where - d_date between '2002-5-01' and - (cast('2002-5-01' as date) + interval '60 days') -and ws1.ws_ship_date_sk = d_date_sk -and ws1.ws_ship_addr_sk = ca_address_sk -and ca_state = 'OK' -and ws1.ws_web_site_sk = web_site_sk -and web_company_name = 'pri' -and exists (select * - from web_sales ws2 - where ws1.ws_order_number = ws2.ws_order_number - and ws1.ws_warehouse_sk <> ws2.ws_warehouse_sk) -and not exists(select * - from web_returns wr1 - where ws1.ws_order_number = wr1.wr_order_number) -order by count(distinct ws_order_number) -limit 100; - --- end query 94 in stream 0 using template query94.tpl --- start query 95 in stream 0 using template query95.tpl -with ws_wh as -(select ws1.ws_order_number,ws1.ws_warehouse_sk wh1,ws2.ws_warehouse_sk wh2 - from web_sales ws1,web_sales ws2 - where ws1.ws_order_number = ws2.ws_order_number - and ws1.ws_warehouse_sk <> ws2.ws_warehouse_sk) - select - count(distinct ws_order_number) as "order count" - ,sum(ws_ext_ship_cost) as "total shipping cost" - ,sum(ws_net_profit) as "total net profit" -from - web_sales ws1 - ,date_dim - ,customer_address - ,web_site -where - d_date between '2001-4-01' and - (cast('2001-4-01' as date) + interval '60 days') -and ws1.ws_ship_date_sk = d_date_sk -and ws1.ws_ship_addr_sk = ca_address_sk -and ca_state = 'VA' -and ws1.ws_web_site_sk = web_site_sk -and web_company_name = 'pri' -and ws1.ws_order_number in (select ws_order_number - from ws_wh) -and ws1.ws_order_number in (select wr_order_number - from web_returns,ws_wh - where wr_order_number = ws_wh.ws_order_number) -order by count(distinct ws_order_number) -limit 100; - --- end query 95 in stream 0 using template query95.tpl --- start query 96 in stream 0 using template query96.tpl -select count(*) -from store_sales - ,household_demographics - ,time_dim, store -where ss_sold_time_sk = time_dim.t_time_sk - and ss_hdemo_sk = household_demographics.hd_demo_sk - and ss_store_sk = s_store_sk - and time_dim.t_hour = 8 - and time_dim.t_minute >= 30 - and household_demographics.hd_dep_count = 0 - and store.s_store_name = 'ese' -order by count(*) -limit 100; - --- end query 96 in stream 0 using template query96.tpl --- start query 97 in stream 0 using template query97.tpl -with ssci as ( -select ss_customer_sk customer_sk - ,ss_item_sk item_sk -from store_sales,date_dim -where ss_sold_date_sk = d_date_sk - and d_month_seq between 1199 and 1199 + 11 -group by ss_customer_sk - ,ss_item_sk), -csci as( - select cs_bill_customer_sk customer_sk - ,cs_item_sk item_sk -from catalog_sales,date_dim -where cs_sold_date_sk = d_date_sk - and d_month_seq between 1199 and 1199 + 11 -group by cs_bill_customer_sk - ,cs_item_sk) - select sum(case when ssci.customer_sk is not null and csci.customer_sk is null then 1 else 0 end) store_only - ,sum(case when ssci.customer_sk is null and csci.customer_sk is not null then 1 else 0 end) catalog_only - ,sum(case when ssci.customer_sk is not null and csci.customer_sk is not null then 1 else 0 end) store_and_catalog -from ssci full outer join csci on (ssci.customer_sk=csci.customer_sk - and ssci.item_sk = csci.item_sk) -limit 100; - --- end query 97 in stream 0 using template query97.tpl --- start query 98 in stream 0 using template query98.tpl -select i_item_id - ,i_item_desc - ,i_category - ,i_class - ,i_current_price - ,sum(ss_ext_sales_price) as itemrevenue - ,sum(ss_ext_sales_price)*100/sum(sum(ss_ext_sales_price)) over - (partition by i_class) as revenueratio -from - store_sales - ,item - ,date_dim -where - ss_item_sk = i_item_sk - and i_category in ('Men', 'Sports', 'Jewelry') - and ss_sold_date_sk = d_date_sk - and d_date between cast('1999-02-05' as date) - and (cast('1999-02-05' as date) + interval '60 days') -group by - i_item_id - ,i_item_desc - ,i_category - ,i_class - ,i_current_price -order by - i_category - ,i_class - ,i_item_id - ,i_item_desc - ,revenueratio; - --- end query 98 in stream 0 using template query98.tpl --- start query 99 in stream 0 using template query99.tpl -select - substr(w_warehouse_name,1,20) - ,sm_type - ,cc_name - ,sum(case when (cs_ship_date_sk - cs_sold_date_sk <= 30 ) then 1 else 0 end) as "30 days" - ,sum(case when (cs_ship_date_sk - cs_sold_date_sk > 30) and - (cs_ship_date_sk - cs_sold_date_sk <= 60) then 1 else 0 end ) as "31-60 days" - ,sum(case when (cs_ship_date_sk - cs_sold_date_sk > 60) and - (cs_ship_date_sk - cs_sold_date_sk <= 90) then 1 else 0 end) as "61-90 days" - ,sum(case when (cs_ship_date_sk - cs_sold_date_sk > 90) and - (cs_ship_date_sk - cs_sold_date_sk <= 120) then 1 else 0 end) as "91-120 days" - ,sum(case when (cs_ship_date_sk - cs_sold_date_sk > 120) then 1 else 0 end) as ">120 days" -from - catalog_sales - ,warehouse - ,ship_mode - ,call_center - ,date_dim -where - d_month_seq between 1194 and 1194 + 11 -and cs_ship_date_sk = d_date_sk -and cs_warehouse_sk = w_warehouse_sk -and cs_ship_mode_sk = sm_ship_mode_sk -and cs_call_center_sk = cc_call_center_sk -group by - substr(w_warehouse_name,1,20) - ,sm_type - ,cc_name -order by substr(w_warehouse_name,1,20) - ,sm_type - ,cc_name -limit 100; From 7bb3c33122f759c43b5c5cb4d9ddfa28b084b9bd Mon Sep 17 00:00:00 2001 From: Alexey Kondratov Date: Mon, 28 Oct 2019 17:00:30 +0300 Subject: [PATCH 10/91] README.md updated to reflect recent changes --- README.md | 37 +++++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 34ecdb0..803c731 100644 --- a/README.md +++ b/README.md @@ -26,20 +26,21 @@ Add module name to the `shared_preload_libraries` parameter in `postgresql.conf` shared_preload_libraries = 'pg_query_state' ``` It is essential to restart the PostgreSQL instance. After that, execute the following query in psql: -``` +```sql CREATE EXTENSION pg_query_state; ``` Done! ## Tests -Tests using parallel sessions using python 2.7 script: - ``` - python tests/pg_qs_test_runner.py [OPTION]... - ``` +Test using parallel sessions with Python 2.7/3+ compatible script: +```shell +python tests/pg_qs_test_runner.py [OPTION]... +``` *prerequisite packages*: * `psycopg2` version 2.6 or later * `PyYAML` version 3.11 or later - +* `progressbar2` for stress test progress reporting + *options*: * *- -host* --- postgres server host, default value is *localhost* * *- -port* --- postgres server port, default value is *5432* @@ -47,6 +48,18 @@ Tests using parallel sessions using python 2.7 script: * *- -user* --- user name, default value is *postgres* * *- -password* --- user's password, default value is empty +Or run all tests in `Docker` using: + +```shell +export LEVEL=stress +export PG_VERSION=12 + +docker-compose build +docker-compose run tests +``` + +There are different test levels: `hardcore`, `nightmare` (runs tests under `valgrind`) and `stress` (runs tests under `TPC-DS` load). + ## Function pg\_query\_state ```plpgsql pg_query_state(integer pid, @@ -92,11 +105,11 @@ This parameters is set on called side before running any queries whose states ar ## Examples Set maximum number of parallel workers on `gather` node equals `2`: -``` +```sql postgres=# set max_parallel_workers_per_gather = 2; ``` Assume one backend with pid = 49265 performs a simple query: -``` +```sql postgres=# select pg_backend_pid(); pg_backend_pid ---------------- @@ -105,7 +118,7 @@ postgres=# select pg_backend_pid(); postgres=# select count(*) from foo join bar on foo.c1=bar.c1; ``` Other backend can extract intermediate state of execution that query: -``` +```sql postgres=# \x postgres=# select * from pg_query_state(49265); -[ RECORD 1 ]+------------------------------------------------------------------------------------------------------------------------- @@ -150,11 +163,11 @@ In example above working backend spawns two parallel workers with pids `49324` a `Seq Scan` node has statistics on passed loops (average number of rows delivered to `Nested Loop` and number of passed loops are shown) and statistics on current loop. Other nodes has statistics only for current loop as this loop is first (`loop number` = 1). Assume first backend executes some function: -``` +```sql postgres=# select n_join_foo_bar(); ``` Other backend can get the follow output: -``` +```sql postgres=# select * from pg_query_state(49265); -[ RECORD 1 ]+------------------------------------------------------------------------------------------------------------------ pid | 49265 @@ -180,7 +193,7 @@ leader_pid | (null) First row corresponds to function call, second - to query which is in the body of that function. We can get result plans in different format (e.g. `json`): -``` +```sql postgres=# select * from pg_query_state(pid := 49265, format := 'json'); -[ RECORD 1 ]+------------------------------------------------------------ pid | 49265 From 087b0c84fbf3bd07dd0a896374405417160e1e90 Mon Sep 17 00:00:00 2001 From: Alexey Kondratov Date: Mon, 28 Oct 2019 17:44:49 +0300 Subject: [PATCH 11/91] Support only Python 3+ for running tests --- README.md | 2 +- tests/pg_qs_test_runner.py | 1 - tests/test_cases.py | 2 -- 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/README.md b/README.md index 803c731..4ec2bb6 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ CREATE EXTENSION pg_query_state; Done! ## Tests -Test using parallel sessions with Python 2.7/3+ compatible script: +Test using parallel sessions with Python 3+ compatible script: ```shell python tests/pg_qs_test_runner.py [OPTION]... ``` diff --git a/tests/pg_qs_test_runner.py b/tests/pg_qs_test_runner.py index bc9e638..2098809 100644 --- a/tests/pg_qs_test_runner.py +++ b/tests/pg_qs_test_runner.py @@ -3,7 +3,6 @@ Tests extract query state from running backend (including concurrent extracts) Copyright (c) 2016-2016, Postgres Professional ''' -from __future__ import print_function, division, absolute_import import os import sys diff --git a/tests/test_cases.py b/tests/test_cases.py index 1b13b63..9f7e3b2 100644 --- a/tests/test_cases.py +++ b/tests/test_cases.py @@ -1,5 +1,3 @@ -from __future__ import print_function, division, absolute_import - import os import json import psycopg2 From 4d4734a7b5c20c9406bab6c684f995a1f64c2f40 Mon Sep 17 00:00:00 2001 From: Alexey Kondratov Date: Mon, 28 Oct 2019 19:06:07 +0300 Subject: [PATCH 12/91] Use copy_from from psycopg2 for TPC-DS data load --- tests/prepare_stress.sh | 1 + tests/test_cases.py | 5 ++--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/prepare_stress.sh b/tests/prepare_stress.sh index 1ae3c75..da5ae48 100755 --- a/tests/prepare_stress.sh +++ b/tests/prepare_stress.sh @@ -6,6 +6,7 @@ rm -rf ./* git clone --depth 1 --single-branch --branch master https://github.com/gregrahn/tpcds-kit.git git clone --depth 1 --single-branch --branch master https://github.com/cwida/tpcds-result-reproduction.git + cd tpcds-kit/tools make -s diff --git a/tests/test_cases.py b/tests/test_cases.py index 9f7e3b2..a7af8cf 100644 --- a/tests/test_cases.py +++ b/tests/test_cases.py @@ -542,11 +542,10 @@ def load_tpcds_data(config): for table_datafile in os.listdir('tmp_stress/tpcds-kit/tools/'): if table_datafile.endswith('.dat'): table_name = os.path.splitext(os.path.basename(table_datafile))[0] - copy_cmd = "COPY %s FROM '/pg/testdir/tmp_stress/tpcds-kit/tools/tables/%s' CSV DELIMITER '|'" % (table_name, table_datafile) print('Loading table', table_name) - # cur.execute("TRUNCATE %s" % table_name) - cur.execute(copy_cmd) + with open('tmp_stress/tpcds-kit/tools/tables/%s' % table_datafile) as f: + cur.copy_from(f, table_name, sep='|', null='') conn.commit() From 80a5e1af648e46fd71ac46999f28a10dc3ddece4 Mon Sep 17 00:00:00 2001 From: Milyutin Maksim Date: Tue, 29 Oct 2019 17:41:24 +0300 Subject: [PATCH 13/91] Segregate stress test based on TPC-DS from common ones --- .travis.yml | 11 ++++------- README.md | 4 +++- run_tests.sh | 5 ++++- tests/pg_qs_test_runner.py | 32 ++++++++++++++++++-------------- 4 files changed, 29 insertions(+), 23 deletions(-) diff --git a/.travis.yml b/.travis.yml index 66bf741..58aadf3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,17 +18,14 @@ notifications: on_failure: always env: - - PG_VERSION=12 LEVEL=hardcore + - PG_VERSION=12 LEVEL=hardcore USE_TPCDS=1 - PG_VERSION=12 - - PG_VERSION=11 LEVEL=hardcore + - PG_VERSION=11 LEVEL=hardcore USE_TPCDS=1 - PG_VERSION=11 - - PG_VERSION=10 LEVEL=hardcore + - PG_VERSION=10 LEVEL=hardcore USE_TPCDS=1 - PG_VERSION=10 - - PG_VERSION=9.6 LEVEL=hardcore + - PG_VERSION=9.6 LEVEL=hardcore USE_TPCDS=1 - PG_VERSION=9.6 - - PG_VERSION=12 LEVEL=stress - - PG_VERSION=11 LEVEL=stress - - PG_VERSION=10 LEVEL=stress matrix: allow_failures: diff --git a/README.md b/README.md index 4ec2bb6..795f571 100644 --- a/README.md +++ b/README.md @@ -47,11 +47,13 @@ python tests/pg_qs_test_runner.py [OPTION]... * *- -database* --- database name, default value is *postgres* * *- -user* --- user name, default value is *postgres* * *- -password* --- user's password, default value is empty +* *- -tpc-ds* --- runs only stress tests on TPC-DS benchmark Or run all tests in `Docker` using: ```shell -export LEVEL=stress +export LEVEL=hardcore +export USE_TPCDS=1 export PG_VERSION=12 docker-compose build diff --git a/run_tests.sh b/run_tests.sh index 31ac49a..0f1f059 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -147,6 +147,9 @@ python3 -m venv /tmp/env && source /tmp/env/bin/activate && pip install -r tests/requirements.txt set -e #exit virtualenv with error code python tests/pg_qs_test_runner.py --port $PGPORT +if [ -n $USE_TPCDS ]; then + python tests/pg_qs_test_runner.py --port $PGPORT --tpc-ds +fi deactivate set -x @@ -170,4 +173,4 @@ gcov *.c *.h set +ux # send coverage stats to Codecov -bash <(curl -s https://codecov.io/bash) \ No newline at end of file +bash <(curl -s https://codecov.io/bash) diff --git a/tests/pg_qs_test_runner.py b/tests/pg_qs_test_runner.py index 2098809..a3fec0b 100644 --- a/tests/pg_qs_test_runner.py +++ b/tests/pg_qs_test_runner.py @@ -32,7 +32,7 @@ class TeardownException(Exception): pass 'insert into bar select i, i%2=1 from generate_series(1, 500000) as i', 'analyze foo', 'analyze bar', - ] +] teardown_cmd = [ 'drop table foo cascade', @@ -82,10 +82,19 @@ def teardown(con): def main(config): ''' Main test function ''' + conn_params = { + key:config.__dict__[key] for key in ('host', 'port', 'user', 'database', 'password') + } - con = psycopg2.connect(**config) - setup(con) + if config.use_tpcds: + print('Starting stress test') + test_tpc_ds(conn_params) + print('Stress finished successfully') + return + # run default tests + init_conn = psycopg2.connect(**conn_params) + setup(init_conn) for i, test in enumerate(tests): if test.__doc__: descr = test.__doc__ @@ -93,16 +102,10 @@ def main(config): descr = 'test case %d' % (i+1) print(("%s..." % descr)) sys.stdout.flush() - test(config) + test(conn_params) print('ok!') - - if os.environ['LEVEL'] == 'stress': - print('Starting stress test') - test_tpc_ds(config) - print('Stress finished successfully') - - teardown(con) - con.close() + teardown(init_conn) + init_conn.close() if __name__ == '__main__': parser = argparse.ArgumentParser(description='Query state of running backends tests') @@ -111,7 +114,8 @@ def main(config): parser.add_argument('--port', type=int, default=5432, help='postgres server port') parser.add_argument('--user', dest='user', default='postgres', help='user name') parser.add_argument('--database', dest='database', default='postgres', help='database name') - parser.add_argument('--password', dest='password', nargs=0, action=PasswordPromptAction, default='') + parser.add_argument('--password', dest='password', nargs=0, action=PasswordPromptAction, default='', help='password') + parser.add_argument('--tpc-ds', dest='use_tpcds', action='store_true', help='run only stress test based on TPC-DS benchmark') args = parser.parse_args() - main(args.__dict__) + main(args) From 302350ce79b2ad7e2abf06d580704495ccf6f726 Mon Sep 17 00:00:00 2001 From: Maksim Milyutin Date: Tue, 29 Oct 2019 19:44:50 +0300 Subject: [PATCH 14/91] Fix tests runner script --- run_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run_tests.sh b/run_tests.sh index 0f1f059..5f871f0 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -147,7 +147,7 @@ python3 -m venv /tmp/env && source /tmp/env/bin/activate && pip install -r tests/requirements.txt set -e #exit virtualenv with error code python tests/pg_qs_test_runner.py --port $PGPORT -if [ -n $USE_TPCDS ]; then +if [[ -n $USE_TPCDS ]]; then python tests/pg_qs_test_runner.py --port $PGPORT --tpc-ds fi deactivate From 18f5a380e741cca6a0cd792f804b48b7e076340e Mon Sep 17 00:00:00 2001 From: Alexey Kondratov Date: Wed, 30 Oct 2019 12:27:17 +0300 Subject: [PATCH 15/91] Try to fix usage of USE_TPCDS variable in test script --- run_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run_tests.sh b/run_tests.sh index 5f871f0..4280da6 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -147,7 +147,7 @@ python3 -m venv /tmp/env && source /tmp/env/bin/activate && pip install -r tests/requirements.txt set -e #exit virtualenv with error code python tests/pg_qs_test_runner.py --port $PGPORT -if [[ -n $USE_TPCDS ]]; then +if [[ "$USE_TPCDS" == "1" ]]; then python tests/pg_qs_test_runner.py --port $PGPORT --tpc-ds fi deactivate From 02049e10afa38b1807ad830fb887a5f5199c4391 Mon Sep 17 00:00:00 2001 From: Alexey Kondratov Date: Wed, 30 Oct 2019 12:46:38 +0300 Subject: [PATCH 16/91] Pass USE_TPCDS env variable to Docker container --- Dockerfile.tmpl | 2 +- mk_dockerfile.sh | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/Dockerfile.tmpl b/Dockerfile.tmpl index 73ce735..a9fdac2 100644 --- a/Dockerfile.tmpl +++ b/Dockerfile.tmpl @@ -35,4 +35,4 @@ ADD . /pg/testdir WORKDIR /pg/testdir USER postgres -ENTRYPOINT LEVEL=${LEVEL} /run.sh +ENTRYPOINT LEVEL=${LEVEL} USE_TPCDS=${USE_TPCDS} /run.sh diff --git a/mk_dockerfile.sh b/mk_dockerfile.sh index 76a7bf3..86f72a4 100755 --- a/mk_dockerfile.sh +++ b/mk_dockerfile.sh @@ -9,10 +9,16 @@ if [ -z ${LEVEL+x} ]; then LEVEL=scan-build fi +if [ -z ${USE_TPCDS+x} ]; then + USE_TPCDS=0 +fi + echo PG_VERSION=${PG_VERSION} echo LEVEL=${LEVEL} +echo USE_TPCDS=${USE_TPCDS} sed \ -e 's/${PG_VERSION}/'${PG_VERSION}/g \ -e 's/${LEVEL}/'${LEVEL}/g \ + -e 's/${USE_TPCDS}/'${USE_TPCDS}/g \ Dockerfile.tmpl > Dockerfile From a4a2ec365396bd5745184ad2d4fe6af65d905d75 Mon Sep 17 00:00:00 2001 From: Alexey Kondratov Date: Wed, 30 Oct 2019 14:19:17 +0300 Subject: [PATCH 17/91] Create pg_query_state extension in the case of TPC-DS test --- tests/pg_qs_test_runner.py | 2 +- tests/test_cases.py | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/pg_qs_test_runner.py b/tests/pg_qs_test_runner.py index a3fec0b..123d91e 100644 --- a/tests/pg_qs_test_runner.py +++ b/tests/pg_qs_test_runner.py @@ -83,7 +83,7 @@ def teardown(con): def main(config): ''' Main test function ''' conn_params = { - key:config.__dict__[key] for key in ('host', 'port', 'user', 'database', 'password') + key:config.__dict__[key] for key in ('host', 'port', 'user', 'database', 'password') } if config.use_tpcds: diff --git a/tests/test_cases.py b/tests/test_cases.py index a7af8cf..a1bb34a 100644 --- a/tests/test_cases.py +++ b/tests/test_cases.py @@ -526,14 +526,17 @@ def test_timing_buffers_conflicts(config): class DataLoadException(Exception): pass class StressTestException(Exception): pass -def load_tpcds_data(config): - print('Loading TPC-DS data...') +def setup_tpcds(config): + print('Setting up TPC-DS test...') subprocess.call(['./tests/prepare_stress.sh']) try: conn = psycopg2.connect(**config) cur = conn.cursor() + # Create pg_query_state extension + cur.execute('CREATE EXTENSION IF NOT EXISTS pg_query_state') + # Create tables with open('tmp_stress/tpcds-kit/tools/tpcds.sql', 'r') as f: cur.execute(f.read()) @@ -561,7 +564,7 @@ def test_tpc_ds(config): global stress_in_progress stress_in_progress = True - load_tpcds_data(config) + setup_tpcds(config) print('Preparing TPC-DS queries...') # # Execute query in separate thread From ba34af1a51abc27c1e99685ddb6a598a26ac5d86 Mon Sep 17 00:00:00 2001 From: Alexey Kondratov Date: Wed, 30 Oct 2019 16:34:11 +0300 Subject: [PATCH 18/91] Do not run TPC-DS on 9.6 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 58aadf3..5d26b7a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,7 +24,7 @@ env: - PG_VERSION=11 - PG_VERSION=10 LEVEL=hardcore USE_TPCDS=1 - PG_VERSION=10 - - PG_VERSION=9.6 LEVEL=hardcore USE_TPCDS=1 + - PG_VERSION=9.6 LEVEL=hardcore - PG_VERSION=9.6 matrix: From b2f8e82687545fa0fe9e38d82f2b1998da863a47 Mon Sep 17 00:00:00 2001 From: Maksim Milyutin Date: Sat, 21 Dec 2019 19:50:56 +0300 Subject: [PATCH 19/91] Split common tests and tpc-ds stress test on different modules And add option to run TPC-DS stress test separately. --- tests/common.py | 112 ++++++++++++ tests/pg_qs_test_runner.py | 12 +- tests/test_cases.py | 355 ++++++------------------------------- tests/tpcds.py | 92 ++++++++++ 4 files changed, 262 insertions(+), 309 deletions(-) create mode 100644 tests/common.py create mode 100644 tests/tpcds.py diff --git a/tests/common.py b/tests/common.py new file mode 100644 index 0000000..a392019 --- /dev/null +++ b/tests/common.py @@ -0,0 +1,112 @@ +''' +common.py +Copyright (c) 2016-2019, Postgres Professional +''' + +import psycopg2 +import psycopg2.extensions +import select + +# Some queries from TPC-DS may freeze or be even broken, +# so we allow some sort of failure, since we do not test +# Postgres, but rather that pg_query_state do not crash +# anything under stress load. +MAX_PG_QS_RETRIES = 50 + + +def wait(conn): + """wait for some event on connection to postgres""" + while 1: + state = conn.poll() + if state == psycopg2.extensions.POLL_OK: + break + elif state == psycopg2.extensions.POLL_WRITE: + select.select([], [conn.fileno()], []) + elif state == psycopg2.extensions.POLL_READ: + select.select([conn.fileno()], [], []) + else: + raise psycopg2.OperationalError("poll() returned %s" % state) + +def n_async_connect(config, n=1): + """establish n asynchronious connections to the postgres with specified config""" + + aconfig = config.copy() + aconfig['async'] = True + + result = [] + for _ in range(n): + conn = psycopg2.connect(**aconfig) + wait(conn) + result.append(conn) + return result + +def n_close(conns): + """close connections to postgres""" + + for conn in conns: + conn.close() + +def pg_query_state(config, pid, verbose=False, costs=False, timing=False, \ + buffers=False, triggers=False, format='text', \ + stress_in_progress=False): + """ + Get query state from backend with specified pid and optional parameters. + Save any warning, info, notice and log data in global variable 'notices' + """ + + conn = psycopg2.connect(**config) + curs = conn.cursor() + + if stress_in_progress: + set_guc(conn, 'statement_timeout', TPC_DS_STATEMENT_TIMEOUT) + n_retries = 0 + + result = [] + while not result: + curs.callproc('pg_query_state', (pid, verbose, costs, timing, buffers, triggers, format)) + result = curs.fetchall() + + if stress_in_progress: + n_retries += 1 + if n_retries >= MAX_PG_QS_RETRIES: + print('\npg_query_state tried %s times with no effect, giving up' % MAX_PG_QS_RETRIES) + break + + notices = conn.notices[:] + conn.close() + return result, notices + +def query_state(config, async_conn, query, args={}, num_workers=0, stress_in_progress=False): + """ + Get intermediate state of 'query' on connection 'async_conn' after number of 'steps' + of node executions from start of query + """ + + acurs = async_conn.cursor() + conn = psycopg2.connect(**config) + curs = conn.cursor() + + set_guc(async_conn, 'enable_mergejoin', 'off') + set_guc(async_conn, 'max_parallel_workers_per_gather', num_workers) + acurs.execute(query) + + # extract current state of query progress + pg_qs_args = { + 'config': config, + 'pid': async_conn.get_backend_pid() + } + for k, v in args.items(): + pg_qs_args[k] = v + result, notices = pg_query_state(**pg_qs_args) + wait(async_conn) + + set_guc(async_conn, 'pg_query_state.executor_trace', 'off') + set_guc(async_conn, 'enable_mergejoin', 'on') + + conn.close() + return result, notices + +def set_guc(async_conn, param, value): + acurs = async_conn.cursor() + acurs.execute('set %s to %s' % (param, value)) + wait(async_conn) diff --git a/tests/pg_qs_test_runner.py b/tests/pg_qs_test_runner.py index 123d91e..b83a5c3 100644 --- a/tests/pg_qs_test_runner.py +++ b/tests/pg_qs_test_runner.py @@ -1,17 +1,17 @@ ''' -pg_qs_test_cases.py - Tests extract query state from running backend (including concurrent extracts) -Copyright (c) 2016-2016, Postgres Professional +pg_qs_test_runner.py +Copyright (c) 2016-2019, Postgres Professional ''' -import os -import sys import argparse import getpass +import os import psycopg2 +import sys sys.path.append(os.path.dirname(os.path.abspath(__file__))) from test_cases import * +import tpcds class PasswordPromptAction(argparse.Action): def __call__(self, parser, args, values, option_string=None): @@ -88,7 +88,7 @@ def main(config): if config.use_tpcds: print('Starting stress test') - test_tpc_ds(conn_params) + tpcds.test_tpc_ds(conn_params) print('Stress finished successfully') return diff --git a/tests/test_cases.py b/tests/test_cases.py index a1bb34a..78aa17a 100644 --- a/tests/test_cases.py +++ b/tests/test_cases.py @@ -1,134 +1,21 @@ -import os +''' +test_cases.py +Copyright (c) 2016-2019, Postgres Professional +''' + +import common import json import psycopg2 -import psycopg2.extensions import re import select import time import xml.etree.ElementTree as ET import yaml -import subprocess -import progressbar - -from time import sleep - -# Some queries from TPC-DS may freeze or be even broken, -# so we allow some sort of failure, since we do not test -# Postgres, but rather that pg_query_state do not crash -# anything under stress load. -MAX_PG_QS_RETRIES = 50 -TPC_DS_EXCLUDE_LIST = [] # actual numbers of TPC-DS tests to exclude -TPC_DS_STATEMENT_TIMEOUT = 20000 # statement_timeout in ms -stress_in_progress = False - -def wait(conn): - """wait for some event on connection to postgres""" - while 1: - state = conn.poll() - if state == psycopg2.extensions.POLL_OK: - break - elif state == psycopg2.extensions.POLL_WRITE: - select.select([], [conn.fileno()], []) - elif state == psycopg2.extensions.POLL_READ: - select.select([conn.fileno()], [], []) - else: - raise psycopg2.OperationalError("poll() returned %s" % state) - -def n_async_connect(config, n=1): - """establish n asynchronious connections to the postgres with specified config""" - - aconfig = config.copy() - aconfig['async'] = True - - result = [] - for _ in range(n): - conn = psycopg2.connect(**aconfig) - wait(conn) - result.append(conn) - return result - -def n_close(conns): - """close connections to postgres""" - - for conn in conns: - conn.close() - -notices = [] - -def debug_output(qs, qs_len, pid, query, expected): - something_happened = False - if (qs_len and len(qs) != qs_len ): - print("len(qs): ", len(qs), ", expected: ", qs_len) - something_happened = True - if (pid and qs[0][0] != pid): - print("qs[0][0]: ", qs[0][0], " = ", pid) - something_happened = True - if (qs[0][1] != 0): - print("qs[0][1]: ", qs[0][1], ", expected: 0") - something_happened = True - if (qs[0][2] != query): - print("qs[0][2]:\n", qs[0][2]) - print("Expected:\n", query) - something_happened = True - if (not (re.match(expected, qs[0][3]))): - print("qs[0][3]:\n", qs[0][3]) - print("Expected:\n", expected) - something_happened = True - if (qs[0][4] != None): - print("qs[0][4]: ", qs[0][4], "Expected: None") - something_happened = True - if (qs_len and len(qs) > qs_len): - for i in range(qs_len, len(qs)): - print("qs[",i,"][0]: ", qs[i][0]) - print("qs[",i,"][1]: ", qs[i][1]) - print("qs[",i,"][2]: ", qs[i][2]) - print("qs[",i,"][3]: ", qs[i][3]) - print("qs[",i,"][4]: ", qs[i][4]) - something_happened = True - if (something_happened): - print("If test have not crashed, then it's OK") - -def notices_warning(): - if (len(notices) > 0): - print("") - print("WARNING:") - print(notices) - -def pg_query_state(config, pid, verbose=False, costs=False, timing=False, \ - buffers=False, triggers=False, format='text'): - """ - Get query state from backend with specified pid and optional parameters. - Save any warning, info, notice and log data in global variable 'notices' - """ - - global notices, stress_in_progress - - conn = psycopg2.connect(**config) - curs = conn.cursor() - - if stress_in_progress: - set_guc(conn, 'statement_timeout', TPC_DS_STATEMENT_TIMEOUT) - n_retries = 0 - - result = [] - while not result: - curs.callproc('pg_query_state', (pid, verbose, costs, timing, buffers, triggers, format)) - result = curs.fetchall() - - if stress_in_progress: - n_retries += 1 - if n_retries >= MAX_PG_QS_RETRIES: - print('\npg_query_state tried %s times with no effect, giving up' % MAX_PG_QS_RETRIES) - break - - notices = conn.notices[:] - conn.close() - return result def test_deadlock(config): """test when two backends try to extract state of each other""" - acon1, acon2 = n_async_connect(config, 2) + acon1, acon2 = common.n_async_connect(config, 2) acurs1 = acon1.cursor() acurs2 = acon2.cursor() @@ -140,50 +27,20 @@ def test_deadlock(config): r, w, x = select.select([acon1.fileno(), acon2.fileno()], [], [], 10) assert (r or w or x), "Deadlock is happened under cross reading of query states" - wait(acon1) - wait(acon2) + common.wait(acon1) + common.wait(acon2) # exit from loop if one backend could read state of execution 'pg_query_state' # from other backend if acurs1.fetchone() or acurs2.fetchone(): break - n_close((acon1, acon2)) - -def query_state(config, async_conn, query, args={}, num_workers=0): - """ - Get intermediate state of 'query' on connection 'async_conn' after number of 'steps' - of node executions from start of query - """ - - acurs = async_conn.cursor() - conn = psycopg2.connect(**config) - curs = conn.cursor() - - set_guc(async_conn, 'enable_mergejoin', 'off') - set_guc(async_conn, 'max_parallel_workers_per_gather', num_workers) - acurs.execute(query) - - # extract current state of query progress - pg_qs_args = { - 'config': config, - 'pid': async_conn.get_backend_pid() - } - for k, v in args.items(): - pg_qs_args[k] = v - result = pg_query_state(**pg_qs_args) - wait(async_conn) - - set_guc(async_conn, 'pg_query_state.executor_trace', 'off') - set_guc(async_conn, 'enable_mergejoin', 'on') - - conn.close() - return result + common.n_close((acon1, acon2)) def test_simple_query(config): """test statistics of simple query""" - acon, = n_async_connect(config) + acon, = common.n_async_connect(config) query = 'select count(*) from foo join bar on foo.c1=bar.c1' expected = r"""Aggregate \(Current loop: actual rows=\d+, loop number=1\) -> Hash Join \(Current loop: actual rows=\d+, loop number=1\) @@ -193,30 +50,27 @@ def test_simple_query(config): Buckets: \d+ Batches: \d+ Memory Usage: \d+kB -> Seq Scan on bar \(Current loop: actual rows=\d+, loop number=1\)""" - qs = query_state(config, acon, query) - debug_output(qs, 1, acon.get_backend_pid(), query, expected) - notices_warning() - #assert len(qs) == 1 #Skip this check while output of test can be different + qs, _ = common.query_state(config, acon, query) assert qs[0][0] == acon.get_backend_pid() and qs[0][1] == 0 \ and qs[0][2] == query and re.match(expected, qs[0][3]) and qs[0][4] == None - n_close((acon,)) + common.n_close((acon,)) def test_concurrent_access(config): """test when two backends compete with each other to extract state from third running backend""" - acon1, acon2, acon3 = n_async_connect(config, 3) + acon1, acon2, acon3 = common.n_async_connect(config, 3) acurs1, acurs2, acurs3 = acon1.cursor(), acon2.cursor(), acon3.cursor() query = 'select count(*) from foo join bar on foo.c1=bar.c1' - set_guc(acon3, 'max_parallel_workers_per_gather', 0) + common.set_guc(acon3, 'max_parallel_workers_per_gather', 0) acurs3.execute(query) time.sleep(0.1) acurs1.callproc('pg_query_state', (acon3.get_backend_pid(),)) acurs2.callproc('pg_query_state', (acon3.get_backend_pid(),)) - wait(acon1) - wait(acon2) - wait(acon3) + common.wait(acon1) + common.wait(acon2) + common.wait(acon3) qs1, qs2 = acurs1.fetchall(), acurs2.fetchall() assert len(qs1) == len(qs2) == 1 \ @@ -225,15 +79,13 @@ def test_concurrent_access(config): and qs1[0][2] == qs2[0][2] == query \ and len(qs1[0][3]) > 0 and len(qs2[0][3]) > 0 \ and qs1[0][4] == qs2[0][4] == None - #assert len(notices) == 0 - notices_warning() - n_close((acon1, acon2, acon3)) + common.n_close((acon1, acon2, acon3)) def test_nested_call(config): """test statistics under calling function""" - acon, = n_async_connect(config) + acon, = common.n_async_connect(config) util_conn = psycopg2.connect(**config) util_curs = util_conn.cursor() create_function = """ @@ -259,7 +111,7 @@ def test_nested_call(config): util_curs.execute(create_function) util_conn.commit() - qs = query_state(config, acon, call_function) + qs, notices = common.query_state(config, acon, call_function) assert len(qs) == 2 \ and qs[0][0] == qs[1][0] == acon.get_backend_pid() \ and qs[0][1] == 0 and qs[1][1] == 1 \ @@ -271,12 +123,12 @@ def test_nested_call(config): util_curs.execute(drop_function) util_conn.close() - n_close((acon,)) + common.n_close((acon,)) def test_insert_on_conflict(config): """test statistics on conflicting tuples under INSERT ON CONFLICT query""" - acon, = n_async_connect(config) + acon, = common.n_async_connect(config) util_conn = psycopg2.connect(**config) util_curs = util_conn.cursor() add_field_uniqueness = 'alter table foo add constraint unique_c1 unique(c1)' @@ -291,11 +143,8 @@ def test_insert_on_conflict(config): util_curs.execute(add_field_uniqueness) util_conn.commit() - qs = query_state(config, acon, query) + qs, notices = common.query_state(config, acon, query) - debug_output(qs, 1, acon.get_backend_pid(), query, expected) - notices_warning() - #assert len(qs) == 1 \ assert qs[0][0] == acon.get_backend_pid() and qs[0][1] == 0 \ and qs[0][2] == query and re.match(expected, qs[0][3]) \ and qs[0][4] == None @@ -304,17 +153,12 @@ def test_insert_on_conflict(config): util_curs.execute(drop_field_uniqueness) util_conn.close() - n_close((acon,)) - -def set_guc(async_conn, param, value): - acurs = async_conn.cursor() - acurs.execute('set %s to %s' % (param, value)) - wait(async_conn) + common.n_close((acon,)) def test_trigger(config): """test trigger statistics""" - acon, = n_async_connect(config) + acon, = common.n_async_connect(config) acurs = acon.cursor() util_conn = psycopg2.connect(**config) util_curs = util_conn.cursor() @@ -330,7 +174,7 @@ def test_trigger(config): create_trigger = """ create trigger unique_foo_c1 before insert or update of c1 on foo for row - execute procedure unique_c1_in_foo()""" + execute procedure unique_c1_in_foo()""" drop_temps = 'drop function unique_c1_in_foo() cascade' query = 'insert into foo select i, md5(random()::text) from generate_series(1, 10000) as i' expected_upper = r"""Insert on foo \(Current loop: actual rows=\d+, loop number=1\) @@ -341,17 +185,13 @@ def test_trigger(config): util_curs.execute(create_trigger) util_conn.commit() - qs = query_state(config, acon, query, {'triggers': True}) - debug_output(qs, None, acon.get_backend_pid(), query, expected_upper) - notices_warning() + qs, notices = common.query_state(config, acon, query, {'triggers': True}) assert qs[0][0] == acon.get_backend_pid() and qs[0][1] == 0 \ and qs[0][2] == query and re.match(expected_upper, qs[0][3]) \ and qs[0][4] == None assert len(notices) == 0 - qs = query_state(config, acon, query, {'triggers': False}) - debug_output(qs, None, acon.get_backend_pid(), query, expected_upper) - notices_warning() + qs, notices = common.query_state(config, acon, query, {'triggers': False}) assert qs[0][0] == acon.get_backend_pid() and qs[0][1] == 0 \ and qs[0][2] == query and re.match(expected_upper, qs[0][3]) \ and qs[0][4] == None @@ -360,12 +200,12 @@ def test_trigger(config): util_curs.execute(drop_temps) util_conn.close() - n_close((acon,)) + common.n_close((acon,)) def test_costs(config): """test plan costs""" - acon, = n_async_connect(config) + acon, = common.n_async_connect(config) query = 'select count(*) from foo join bar on foo.c1=bar.c1' expected = r"""Aggregate \(cost=\d+.\d+..\d+.\d+ rows=\d+ width=8\) \(Current loop: actual rows=0, loop number=1\) -> Hash Join \(cost=\d+.\d+..\d+.\d+ rows=\d+ width=0\) \(Current loop: actual rows=0, loop number=1\) @@ -375,18 +215,16 @@ def test_costs(config): Buckets: \d+ Batches: \d+ Memory Usage: \d+kB -> Seq Scan on bar \(cost=0.00..\d+.\d+ rows=\d+ width=4\) \(Current loop: actual rows=\d+, loop number=1\)""" - qs = query_state(config, acon, query, {'costs': True}) - debug_output(qs, 1, None, query, expected) - notices_warning() + qs, notices = common.query_state(config, acon, query, {'costs': True}) assert len(qs) == 1 and re.match(expected, qs[0][3]) assert len(notices) == 0 - n_close((acon,)) + common.n_close((acon,)) def test_buffers(config): """test buffer statistics""" - acon, = n_async_connect(config) + acon, = common.n_async_connect(config) query = 'select count(*) from foo join bar on foo.c1=bar.c1' expected = r"""Aggregate \(Current loop: actual rows=0, loop number=1\) -> Hash Join \(Current loop: actual rows=0, loop number=1\) @@ -398,20 +236,18 @@ def test_buffers(config): -> Seq Scan on bar \(Current loop: actual rows=\d+, loop number=1\) Buffers: .*""" - set_guc(acon, 'pg_query_state.enable_buffers', 'on') + common.set_guc(acon, 'pg_query_state.enable_buffers', 'on') - qs = query_state(config, acon, query, {'buffers': True}) - debug_output(qs, 1, None, query, expected) - notices_warning() + qs, notices = common.query_state(config, acon, query, {'buffers': True}) assert len(qs) == 1 and re.match(expected, qs[0][3]) assert len(notices) == 0 - n_close((acon,)) + common.n_close((acon,)) def test_timing(config): """test timing statistics""" - acon, = n_async_connect(config) + acon, = common.n_async_connect(config) query = 'select count(*) from foo join bar on foo.c1=bar.c1' expected = r"""Aggregate \(Current loop: running time=\d+.\d+ actual rows=0, loop number=1\) -> Hash Join \(Current loop: running time=\d+.\d+ actual rows=0, loop number=1\) @@ -421,15 +257,13 @@ def test_timing(config): Buckets: \d+ Batches: \d+ Memory Usage: \d+kB -> Seq Scan on bar \(Current loop: actual time=\d+.\d+..\d+.\d+ rows=\d+, loop number=1\)""" - set_guc(acon, 'pg_query_state.enable_timing', 'on') + common.set_guc(acon, 'pg_query_state.enable_timing', 'on') - qs = query_state(config, acon, query, {'timing': True}) - debug_output(qs, 1, None, query, expected) - notices_warning() + qs, notices = common.query_state(config, acon, query, {'timing': True}) assert len(qs) == 1 and re.match(expected, qs[0][3]) assert len(notices) == 0 - n_close((acon,)) + common.n_close((acon,)) def check_plan(plan): assert 'Current loop' in plan @@ -454,7 +288,7 @@ def check_xml(root): def test_formats(config): """test all formats of pg_query_state output""" - acon, = n_async_connect(config) + acon, = common.n_async_connect(config) query = 'select count(*) from foo join bar on foo.c1=bar.c1' expected = r"""Aggregate \(Current loop: actual rows=0, loop number=1\) -> Hash Join \(Current loop: actual rows=0, loop number=1\) @@ -464,13 +298,11 @@ def test_formats(config): Buckets: \d+ Batches: \d+ Memory Usage: \d+kB -> Seq Scan on bar \(Current loop: actual rows=\d+, loop number=1\)""" - qs = query_state(config, acon, query, {'format': 'text'}) - debug_output(qs, 1, None, query, expected) - notices_warning() + qs, notices = common.query_state(config, acon, query, {'format': 'text'}) assert len(qs) == 1 and re.match(expected, qs[0][3]) assert len(notices) == 0 - qs = query_state(config, acon, query, {'format': 'json'}) + qs, notices = common.query_state(config, acon, query, {'format': 'json'}) try: js_obj = json.loads(qs[0][3]) except ValueError: @@ -479,7 +311,7 @@ def test_formats(config): assert len(notices) == 0 check_plan(js_obj['Plan']) - qs = query_state(config, acon, query, {'format': 'xml'}) + qs, notices = common.query_state(config, acon, query, {'format': 'xml'}) assert len(qs) == 1 assert len(notices) == 0 try: @@ -488,7 +320,7 @@ def test_formats(config): assert False, 'Invalid xml format' check_xml(xml_root) - qs = query_state(config, acon, query, {'format': 'yaml'}) + qs, _ = common.query_state(config, acon, query, {'format': 'yaml'}) try: yaml_doc = yaml.load(qs[0][3], Loader=yaml.FullLoader) except: @@ -497,111 +329,28 @@ def test_formats(config): assert len(notices) == 0 check_plan(yaml_doc['Plan']) - n_close((acon,)) + common.n_close((acon,)) def test_timing_buffers_conflicts(config): """test when caller requests timing and buffers but counterpart turned off its""" - acon, = n_async_connect(config) + acon, = common.n_async_connect(config) query = 'select count(*) from foo join bar on foo.c1=bar.c1' timing_pattern = '(?:running time=\d+.\d+)|(?:actual time=\d+.\d+..\d+.\d+)' buffers_pattern = 'Buffers:' - qs = query_state(config, acon, query, {'timing': True, 'buffers': False}) + qs, notices = common.query_state(config, acon, query, {'timing': True, 'buffers': False}) assert len(qs) == 1 and not re.search(timing_pattern, qs[0][3]) assert notices == ['WARNING: timing statistics disabled\n'] - qs = query_state(config, acon, query, {'timing': False, 'buffers': True}) + qs, notices = common.query_state(config, acon, query, {'timing': False, 'buffers': True}) assert len(qs) == 1 and not re.search(buffers_pattern, qs[0][3]) assert notices == ['WARNING: buffers statistics disabled\n'] - qs = query_state(config, acon, query, {'timing': True, 'buffers': True}) + qs, notices = common.query_state(config, acon, query, {'timing': True, 'buffers': True}) assert len(qs) == 1 and not re.search(timing_pattern, qs[0][3]) \ and not re.search(buffers_pattern, qs[0][3]) assert len(notices) == 2 and 'WARNING: timing statistics disabled\n' in notices \ and 'WARNING: buffers statistics disabled\n' in notices - n_close((acon,)) - -class DataLoadException(Exception): pass -class StressTestException(Exception): pass - -def setup_tpcds(config): - print('Setting up TPC-DS test...') - subprocess.call(['./tests/prepare_stress.sh']) - - try: - conn = psycopg2.connect(**config) - cur = conn.cursor() - - # Create pg_query_state extension - cur.execute('CREATE EXTENSION IF NOT EXISTS pg_query_state') - - # Create tables - with open('tmp_stress/tpcds-kit/tools/tpcds.sql', 'r') as f: - cur.execute(f.read()) - - # Copy table data from files - for table_datafile in os.listdir('tmp_stress/tpcds-kit/tools/'): - if table_datafile.endswith('.dat'): - table_name = os.path.splitext(os.path.basename(table_datafile))[0] - - print('Loading table', table_name) - with open('tmp_stress/tpcds-kit/tools/tables/%s' % table_datafile) as f: - cur.copy_from(f, table_name, sep='|', null='') - - conn.commit() - - except Exception as e: - cur.close() - conn.close() - raise DataLoadException('Load failed: %s' % e) - - print('done!') - -def test_tpc_ds(config): - """TPC-DS stress test""" - global stress_in_progress - - stress_in_progress = True - setup_tpcds(config) - - print('Preparing TPC-DS queries...') - # # Execute query in separate thread - # # with open('tmp_stress/tpcds-kit/tools/query_0.sql', 'r') as f: - # with open('tests/query_tpcds.sql', 'r') as f: - # sql = f.read() - - # queries = sql.split(';') - # for i, query in enumerate(queries): - # queries[i] = query.replace('%','%%') - # if (len(query.strip()) == 0): - # del queries[i] - queries = [] - for query_file in sorted(os.listdir('tmp_stress/tpcds-result-reproduction/query_qualification/')): - with open('tmp_stress/tpcds-result-reproduction/query_qualification/%s' % query_file, 'r') as f: - queries.append(f.read()) - - acon, = n_async_connect(config) - - print('Starting TPC-DS queries...') - timeout_list = [] - bar = progressbar.ProgressBar(max_value=len(queries)) - for i, query in enumerate(queries): - bar.update(i + 1) - if i + 1 in TPC_DS_EXCLUDE_LIST: - continue - try: - # Set query timeout to TPC_DS_STATEMENT_TIMEOUT / 1000 seconds - set_guc(acon, 'statement_timeout', TPC_DS_STATEMENT_TIMEOUT) - qs = query_state(config, acon, query) - - except psycopg2.extensions.QueryCanceledError: - timeout_list.append(i + 1) - - n_close((acon,)) - - if len(timeout_list) > 0: - print('\nThere were pg_query_state timeouts (%s s) on queries:' % TPC_DS_STATEMENT_TIMEOUT, timeout_list) - - stress_in_progress = False + common.n_close((acon,)) diff --git a/tests/tpcds.py b/tests/tpcds.py new file mode 100644 index 0000000..60589b0 --- /dev/null +++ b/tests/tpcds.py @@ -0,0 +1,92 @@ +''' +test_cases.py +Copyright (c) 2016-2019, Postgres Professional +''' + +import common +import os +import progressbar +import psycopg2.extensions +import subprocess + +class DataLoadException(Exception): pass +class StressTestException(Exception): pass + +TPC_DS_EXCLUDE_LIST = [] # actual numbers of TPC-DS tests to exclude +TPC_DS_STATEMENT_TIMEOUT = 20000 # statement_timeout in ms + +def setup_tpcds(config): + print('Setting up TPC-DS test...') + subprocess.call(['./tests/prepare_stress.sh']) + + try: + conn = psycopg2.connect(**config) + cur = conn.cursor() + + # Create pg_query_state extension + cur.execute('CREATE EXTENSION IF NOT EXISTS pg_query_state') + + # Create tables + with open('tmp_stress/tpcds-kit/tools/tpcds.sql', 'r') as f: + cur.execute(f.read()) + + # Copy table data from files + for table_datafile in os.listdir('tmp_stress/tpcds-kit/tools/'): + if table_datafile.endswith('.dat'): + table_name = os.path.splitext(os.path.basename(table_datafile))[0] + + print('Loading table', table_name) + with open('tmp_stress/tpcds-kit/tools/tables/%s' % table_datafile) as f: + cur.copy_from(f, table_name, sep='|', null='') + + conn.commit() + + except Exception as e: + cur.close() + conn.close() + raise DataLoadException('Load failed: %s' % e) + + print('done!') + +def test_tpc_ds(config): + """TPC-DS stress test""" + + setup_tpcds(config) + + print('Preparing TPC-DS queries...') + # # Execute query in separate thread + # # with open('tmp_stress/tpcds-kit/tools/query_0.sql', 'r') as f: + # with open('tests/query_tpcds.sql', 'r') as f: + # sql = f.read() + + # queries = sql.split(';') + # for i, query in enumerate(queries): + # queries[i] = query.replace('%','%%') + # if (len(query.strip()) == 0): + # del queries[i] + queries = [] + for query_file in sorted(os.listdir('tmp_stress/tpcds-result-reproduction/query_qualification/')): + with open('tmp_stress/tpcds-result-reproduction/query_qualification/%s' % query_file, 'r') as f: + queries.append(f.read()) + + acon, = common.n_async_connect(config) + + print('Starting TPC-DS queries...') + timeout_list = [] + bar = progressbar.ProgressBar(max_value=len(queries)) + for i, query in enumerate(queries): + bar.update(i + 1) + if i + 1 in TPC_DS_EXCLUDE_LIST: + continue + try: + # Set query timeout to TPC_DS_STATEMENT_TIMEOUT / 1000 seconds + common.set_guc(acon, 'statement_timeout', TPC_DS_STATEMENT_TIMEOUT) + qs = common.query_state(config, acon, query, stress_in_progress=True) + + except psycopg2.extensions.QueryCanceledError: + timeout_list.append(i + 1) + + common.n_close((acon,)) + + if len(timeout_list) > 0: + print('\nThere were pg_query_state timeouts (%s s) on queries:' % TPC_DS_STATEMENT_TIMEOUT, timeout_list) From 772062a87ee85349e768f1b8652ac862255b3cd2 Mon Sep 17 00:00:00 2001 From: Maksim Date: Sun, 22 Dec 2019 00:29:28 +0300 Subject: [PATCH 20/91] Segregate setup and running of stress test --- run_tests.sh | 7 ++++--- tests/pg_qs_test_runner.py | 13 ++++++++++--- tests/tpcds.py | 14 +------------- 3 files changed, 15 insertions(+), 19 deletions(-) diff --git a/run_tests.sh b/run_tests.sh index 4280da6..651e9a1 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # -# Copyright (c) 2018, Postgres Professional +# Copyright (c) 2019, Postgres Professional # # supported levels: # * standard @@ -55,7 +55,7 @@ fi # build and install PostgreSQL if [ "$LEVEL" = "hardcore" ] || \ - [ "$LEVEL" = "nightmare" ]; then + [ "$LEVEL" = "nightmare" ]; then # enable Valgrind support sed -i.bak "s/\/* #define USE_VALGRIND *\//#define USE_VALGRIND/g" src/include/pg_config_manual.h @@ -148,7 +148,8 @@ pip install -r tests/requirements.txt set -e #exit virtualenv with error code python tests/pg_qs_test_runner.py --port $PGPORT if [[ "$USE_TPCDS" == "1" ]]; then - python tests/pg_qs_test_runner.py --port $PGPORT --tpc-ds + python tests/pg_qs_test_runner.py --port $PGPORT --tpc-ds-setup + python tests/pg_qs_test_runner.py --port $PGPORT --tpc-ds-run fi deactivate set -x diff --git a/tests/pg_qs_test_runner.py b/tests/pg_qs_test_runner.py index b83a5c3..a186589 100644 --- a/tests/pg_qs_test_runner.py +++ b/tests/pg_qs_test_runner.py @@ -86,9 +86,15 @@ def main(config): key:config.__dict__[key] for key in ('host', 'port', 'user', 'database', 'password') } - if config.use_tpcds: + if config.tpcds_setup: + print('Setup database for TPC-DS bench') + tpcds.setup_tpcds(conn_params) + print('Database is setup successfully') + return + + if config.tpcds_run: print('Starting stress test') - tpcds.test_tpc_ds(conn_params) + tpcds.run_tpcds(conn_params) print('Stress finished successfully') return @@ -115,7 +121,8 @@ def main(config): parser.add_argument('--user', dest='user', default='postgres', help='user name') parser.add_argument('--database', dest='database', default='postgres', help='database name') parser.add_argument('--password', dest='password', nargs=0, action=PasswordPromptAction, default='', help='password') - parser.add_argument('--tpc-ds', dest='use_tpcds', action='store_true', help='run only stress test based on TPC-DS benchmark') + parser.add_argument('--tpc-ds-setup', dest='tpcds_setup', action='store_true', help='setup database to run TPC-DS benchmark') + parser.add_argument('--tpc-ds-run', dest='tpcds_run', action='store_true', help='run only stress test based on TPC-DS benchmark') args = parser.parse_args() main(args) diff --git a/tests/tpcds.py b/tests/tpcds.py index 60589b0..338d8c8 100644 --- a/tests/tpcds.py +++ b/tests/tpcds.py @@ -48,22 +48,10 @@ def setup_tpcds(config): print('done!') -def test_tpc_ds(config): +def run_tpcds(config): """TPC-DS stress test""" - setup_tpcds(config) - print('Preparing TPC-DS queries...') - # # Execute query in separate thread - # # with open('tmp_stress/tpcds-kit/tools/query_0.sql', 'r') as f: - # with open('tests/query_tpcds.sql', 'r') as f: - # sql = f.read() - - # queries = sql.split(';') - # for i, query in enumerate(queries): - # queries[i] = query.replace('%','%%') - # if (len(query.strip()) == 0): - # del queries[i] queries = [] for query_file in sorted(os.listdir('tmp_stress/tpcds-result-reproduction/query_qualification/')): with open('tmp_stress/tpcds-result-reproduction/query_qualification/%s' % query_file, 'r') as f: From 2ac1f8a4d0794021877e06be5ad3942e0f9df1e2 Mon Sep 17 00:00:00 2001 From: Maksim Date: Sun, 22 Dec 2019 02:12:50 +0300 Subject: [PATCH 21/91] Make periodic pg_query_state calls to backend running TPC-DS bench Also refactor common functions for python tests --- tests/common.py | 50 +++++++++++++++++---------------------------- tests/test_cases.py | 30 +++++++++++++-------------- tests/tpcds.py | 33 +++++++++++++++++++++++++++++- 3 files changed, 66 insertions(+), 47 deletions(-) diff --git a/tests/common.py b/tests/common.py index a392019..c5cf4cd 100644 --- a/tests/common.py +++ b/tests/common.py @@ -6,13 +6,7 @@ import psycopg2 import psycopg2.extensions import select - -# Some queries from TPC-DS may freeze or be even broken, -# so we allow some sort of failure, since we do not test -# Postgres, but rather that pg_query_state do not crash -# anything under stress load. -MAX_PG_QS_RETRIES = 50 - +import time def wait(conn): """wait for some event on connection to postgres""" @@ -47,8 +41,7 @@ def n_close(conns): conn.close() def pg_query_state(config, pid, verbose=False, costs=False, timing=False, \ - buffers=False, triggers=False, format='text', \ - stress_in_progress=False): + buffers=False, triggers=False, format='text'): """ Get query state from backend with specified pid and optional parameters. Save any warning, info, notice and log data in global variable 'notices' @@ -57,53 +50,48 @@ def pg_query_state(config, pid, verbose=False, costs=False, timing=False, \ conn = psycopg2.connect(**config) curs = conn.cursor() - if stress_in_progress: - set_guc(conn, 'statement_timeout', TPC_DS_STATEMENT_TIMEOUT) - n_retries = 0 - - result = [] - while not result: - curs.callproc('pg_query_state', (pid, verbose, costs, timing, buffers, triggers, format)) - result = curs.fetchall() - - if stress_in_progress: - n_retries += 1 - if n_retries >= MAX_PG_QS_RETRIES: - print('\npg_query_state tried %s times with no effect, giving up' % MAX_PG_QS_RETRIES) - break - + curs.callproc('pg_query_state', (pid, verbose, costs, timing, buffers, triggers, format)) + result = curs.fetchall() notices = conn.notices[:] conn.close() + return result, notices -def query_state(config, async_conn, query, args={}, num_workers=0, stress_in_progress=False): +def onetime_query_state(config, async_conn, query, args={}, num_workers=0): """ Get intermediate state of 'query' on connection 'async_conn' after number of 'steps' of node executions from start of query """ acurs = async_conn.cursor() - conn = psycopg2.connect(**config) - curs = conn.cursor() set_guc(async_conn, 'enable_mergejoin', 'off') set_guc(async_conn, 'max_parallel_workers_per_gather', num_workers) acurs.execute(query) # extract current state of query progress + MAX_PG_QS_RETRIES = 10 + DELAY_BETWEEN_RETRIES = 0.1 pg_qs_args = { 'config': config, 'pid': async_conn.get_backend_pid() } for k, v in args.items(): pg_qs_args[k] = v - result, notices = pg_query_state(**pg_qs_args) + n_retries = 0 + while True: + result, notices = pg_query_state(**pg_qs_args) + n_retries += 1 + if len(result) > 0: + break + if n_retries >= MAX_PG_QS_RETRIES: + # pg_query_state callings don't return any result, more likely run + # query has completed + break + time.sleep(DELAY_BETWEEN_RETRIES) wait(async_conn) - set_guc(async_conn, 'pg_query_state.executor_trace', 'off') set_guc(async_conn, 'enable_mergejoin', 'on') - - conn.close() return result, notices def set_guc(async_conn, param, value): diff --git a/tests/test_cases.py b/tests/test_cases.py index 78aa17a..7a41892 100644 --- a/tests/test_cases.py +++ b/tests/test_cases.py @@ -50,7 +50,7 @@ def test_simple_query(config): Buckets: \d+ Batches: \d+ Memory Usage: \d+kB -> Seq Scan on bar \(Current loop: actual rows=\d+, loop number=1\)""" - qs, _ = common.query_state(config, acon, query) + qs, _ = common.onetime_query_state(config, acon, query) assert qs[0][0] == acon.get_backend_pid() and qs[0][1] == 0 \ and qs[0][2] == query and re.match(expected, qs[0][3]) and qs[0][4] == None @@ -111,7 +111,7 @@ def test_nested_call(config): util_curs.execute(create_function) util_conn.commit() - qs, notices = common.query_state(config, acon, call_function) + qs, notices = common.onetime_query_state(config, acon, call_function) assert len(qs) == 2 \ and qs[0][0] == qs[1][0] == acon.get_backend_pid() \ and qs[0][1] == 0 and qs[1][1] == 1 \ @@ -143,7 +143,7 @@ def test_insert_on_conflict(config): util_curs.execute(add_field_uniqueness) util_conn.commit() - qs, notices = common.query_state(config, acon, query) + qs, notices = common.onetime_query_state(config, acon, query) assert qs[0][0] == acon.get_backend_pid() and qs[0][1] == 0 \ and qs[0][2] == query and re.match(expected, qs[0][3]) \ @@ -185,13 +185,13 @@ def test_trigger(config): util_curs.execute(create_trigger) util_conn.commit() - qs, notices = common.query_state(config, acon, query, {'triggers': True}) + qs, notices = common.onetime_query_state(config, acon, query, {'triggers': True}) assert qs[0][0] == acon.get_backend_pid() and qs[0][1] == 0 \ and qs[0][2] == query and re.match(expected_upper, qs[0][3]) \ and qs[0][4] == None assert len(notices) == 0 - qs, notices = common.query_state(config, acon, query, {'triggers': False}) + qs, notices = common.onetime_query_state(config, acon, query, {'triggers': False}) assert qs[0][0] == acon.get_backend_pid() and qs[0][1] == 0 \ and qs[0][2] == query and re.match(expected_upper, qs[0][3]) \ and qs[0][4] == None @@ -215,7 +215,7 @@ def test_costs(config): Buckets: \d+ Batches: \d+ Memory Usage: \d+kB -> Seq Scan on bar \(cost=0.00..\d+.\d+ rows=\d+ width=4\) \(Current loop: actual rows=\d+, loop number=1\)""" - qs, notices = common.query_state(config, acon, query, {'costs': True}) + qs, notices = common.onetime_query_state(config, acon, query, {'costs': True}) assert len(qs) == 1 and re.match(expected, qs[0][3]) assert len(notices) == 0 @@ -238,7 +238,7 @@ def test_buffers(config): common.set_guc(acon, 'pg_query_state.enable_buffers', 'on') - qs, notices = common.query_state(config, acon, query, {'buffers': True}) + qs, notices = common.onetime_query_state(config, acon, query, {'buffers': True}) assert len(qs) == 1 and re.match(expected, qs[0][3]) assert len(notices) == 0 @@ -259,7 +259,7 @@ def test_timing(config): common.set_guc(acon, 'pg_query_state.enable_timing', 'on') - qs, notices = common.query_state(config, acon, query, {'timing': True}) + qs, notices = common.onetime_query_state(config, acon, query, {'timing': True}) assert len(qs) == 1 and re.match(expected, qs[0][3]) assert len(notices) == 0 @@ -298,11 +298,11 @@ def test_formats(config): Buckets: \d+ Batches: \d+ Memory Usage: \d+kB -> Seq Scan on bar \(Current loop: actual rows=\d+, loop number=1\)""" - qs, notices = common.query_state(config, acon, query, {'format': 'text'}) + qs, notices = common.onetime_query_state(config, acon, query, {'format': 'text'}) assert len(qs) == 1 and re.match(expected, qs[0][3]) assert len(notices) == 0 - qs, notices = common.query_state(config, acon, query, {'format': 'json'}) + qs, notices = common.onetime_query_state(config, acon, query, {'format': 'json'}) try: js_obj = json.loads(qs[0][3]) except ValueError: @@ -311,7 +311,7 @@ def test_formats(config): assert len(notices) == 0 check_plan(js_obj['Plan']) - qs, notices = common.query_state(config, acon, query, {'format': 'xml'}) + qs, notices = common.onetime_query_state(config, acon, query, {'format': 'xml'}) assert len(qs) == 1 assert len(notices) == 0 try: @@ -320,7 +320,7 @@ def test_formats(config): assert False, 'Invalid xml format' check_xml(xml_root) - qs, _ = common.query_state(config, acon, query, {'format': 'yaml'}) + qs, _ = common.onetime_query_state(config, acon, query, {'format': 'yaml'}) try: yaml_doc = yaml.load(qs[0][3], Loader=yaml.FullLoader) except: @@ -339,15 +339,15 @@ def test_timing_buffers_conflicts(config): timing_pattern = '(?:running time=\d+.\d+)|(?:actual time=\d+.\d+..\d+.\d+)' buffers_pattern = 'Buffers:' - qs, notices = common.query_state(config, acon, query, {'timing': True, 'buffers': False}) + qs, notices = common.onetime_query_state(config, acon, query, {'timing': True, 'buffers': False}) assert len(qs) == 1 and not re.search(timing_pattern, qs[0][3]) assert notices == ['WARNING: timing statistics disabled\n'] - qs, notices = common.query_state(config, acon, query, {'timing': False, 'buffers': True}) + qs, notices = common.onetime_query_state(config, acon, query, {'timing': False, 'buffers': True}) assert len(qs) == 1 and not re.search(buffers_pattern, qs[0][3]) assert notices == ['WARNING: buffers statistics disabled\n'] - qs, notices = common.query_state(config, acon, query, {'timing': True, 'buffers': True}) + qs, notices = common.onetime_query_state(config, acon, query, {'timing': True, 'buffers': True}) assert len(qs) == 1 and not re.search(timing_pattern, qs[0][3]) \ and not re.search(buffers_pattern, qs[0][3]) assert len(notices) == 2 and 'WARNING: timing statistics disabled\n' in notices \ diff --git a/tests/tpcds.py b/tests/tpcds.py index 338d8c8..dfba0bf 100644 --- a/tests/tpcds.py +++ b/tests/tpcds.py @@ -8,6 +8,7 @@ import progressbar import psycopg2.extensions import subprocess +import time class DataLoadException(Exception): pass class StressTestException(Exception): pass @@ -58,6 +59,7 @@ def run_tpcds(config): queries.append(f.read()) acon, = common.n_async_connect(config) + pid = acon.get_backend_pid() print('Starting TPC-DS queries...') timeout_list = [] @@ -69,7 +71,36 @@ def run_tpcds(config): try: # Set query timeout to TPC_DS_STATEMENT_TIMEOUT / 1000 seconds common.set_guc(acon, 'statement_timeout', TPC_DS_STATEMENT_TIMEOUT) - qs = common.query_state(config, acon, query, stress_in_progress=True) + + # run query + acurs = acon.cursor() + acurs.execute(query) + + # periodically run pg_query_state on running backend trying to get + # crash of PostgreSQL + MAX_PG_QS_RETRIES = 10 + PG_QS_DELAY, BEFORE_GOT_QS_DELAY = 0.1, 0.1 + BEFORE_GOT_QS, GOT_QS = range(2) + state, n_retries = BEFORE_GOT_QS, 0 + while True: + result, _ = common.pg_query_state(config, pid) + if state == BEFORE_GOT_QS: + if len(result) > 0: + state = GOT_QS + continue + n_retries += 1 + if n_retries >= MAX_PG_QS_RETRIES: + # pg_query_state callings don't return any result, more likely run + # query has completed + break + time.sleep(BEFORE_GOT_QS_DELAY) + if state == GOT_QS: + if len(result) == 0: + break + time.sleep(PG_QS_DELAY) + + # wait for real query completion + common.wait(acon) except psycopg2.extensions.QueryCanceledError: timeout_list.append(i + 1) From 61cf837f0d72ff8ffe897d0b41bcd51f535452c6 Mon Sep 17 00:00:00 2001 From: Maksim Date: Sat, 28 Dec 2019 21:35:34 +0300 Subject: [PATCH 22/91] Refactor the main cycle of tpc-ds stress test --- tests/common.py | 3 +++ tests/tpcds.py | 30 ++++++++++++++++-------------- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/tests/common.py b/tests/common.py index c5cf4cd..68434b1 100644 --- a/tests/common.py +++ b/tests/common.py @@ -8,6 +8,9 @@ import select import time +BACKEND_IS_IDLE_INFO = 'INFO: state of backend is idle\n' +BACKEND_IS_ACTIVE_INFO = 'INFO: state of backend is active\n' + def wait(conn): """wait for some event on connection to postgres""" while 1: diff --git a/tests/tpcds.py b/tests/tpcds.py index dfba0bf..8df96d7 100644 --- a/tests/tpcds.py +++ b/tests/tpcds.py @@ -78,26 +78,28 @@ def run_tpcds(config): # periodically run pg_query_state on running backend trying to get # crash of PostgreSQL - MAX_PG_QS_RETRIES = 10 - PG_QS_DELAY, BEFORE_GOT_QS_DELAY = 0.1, 0.1 - BEFORE_GOT_QS, GOT_QS = range(2) - state, n_retries = BEFORE_GOT_QS, 0 + MAX_FIRST_GETTING_QS_RETRIES = 10 + PG_QS_DELAY, BEFORE_GETTING_QS_DELAY = 0.1, 0.1 + BEFORE_GETTING_QS, GETTING_QS = range(2) + state, n_first_getting_qs_retries = BEFORE_GETTING_QS, 0 while True: - result, _ = common.pg_query_state(config, pid) - if state == BEFORE_GOT_QS: - if len(result) > 0: - state = GOT_QS + result, notices = common.pg_query_state(config, pid) + # run state machine to determine the first getting query state + # and query finishing + if state == BEFORE_GETTING_QS: + if len(result) > 0 or common.BACKEND_IS_ACTIVE_INFO in notices: + state = GETTING_QS continue - n_retries += 1 - if n_retries >= MAX_PG_QS_RETRIES: + n_first_getting_qs_retries += 1 + if n_first_getting_qs_retries >= MAX_FIRST_GETTING_QS_RETRIES: # pg_query_state callings don't return any result, more likely run # query has completed break - time.sleep(BEFORE_GOT_QS_DELAY) - if state == GOT_QS: - if len(result) == 0: + time.sleep(BEFORE_GETTING_QS_DELAY) + elif state == GETTING_QS: + if common.BACKEND_IS_IDLE_INFO in notices: break - time.sleep(PG_QS_DELAY) + time.sleep(PG_QS_DELAY) # wait for real query completion common.wait(acon) From c1776bc5c028d72259098327323338afd243ab24 Mon Sep 17 00:00:00 2001 From: Maksim Date: Thu, 9 Jan 2020 20:13:34 +0300 Subject: [PATCH 23/91] Make light refactoring after review from ololobus --- tests/common.py | 2 +- tests/pg_qs_test_runner.py | 5 +++-- tests/test_cases.py | 8 +++++--- tests/tpcds.py | 17 +++++++++-------- 4 files changed, 18 insertions(+), 14 deletions(-) diff --git a/tests/common.py b/tests/common.py index 68434b1..3f4f9c2 100644 --- a/tests/common.py +++ b/tests/common.py @@ -1,6 +1,6 @@ ''' common.py -Copyright (c) 2016-2019, Postgres Professional +Copyright (c) 2016-2020, Postgres Professional ''' import psycopg2 diff --git a/tests/pg_qs_test_runner.py b/tests/pg_qs_test_runner.py index a186589..28db807 100644 --- a/tests/pg_qs_test_runner.py +++ b/tests/pg_qs_test_runner.py @@ -1,14 +1,15 @@ ''' pg_qs_test_runner.py -Copyright (c) 2016-2019, Postgres Professional +Copyright (c) 2016-2020, Postgres Professional ''' import argparse import getpass import os -import psycopg2 import sys +import psycopg2 + sys.path.append(os.path.dirname(os.path.abspath(__file__))) from test_cases import * import tpcds diff --git a/tests/test_cases.py b/tests/test_cases.py index 7a41892..8d7fc28 100644 --- a/tests/test_cases.py +++ b/tests/test_cases.py @@ -1,17 +1,19 @@ ''' test_cases.py -Copyright (c) 2016-2019, Postgres Professional +Copyright (c) 2016-2020, Postgres Professional ''' -import common import json -import psycopg2 import re import select import time import xml.etree.ElementTree as ET + +import psycopg2 import yaml +import common + def test_deadlock(config): """test when two backends try to extract state of each other""" diff --git a/tests/tpcds.py b/tests/tpcds.py index 8df96d7..892540c 100644 --- a/tests/tpcds.py +++ b/tests/tpcds.py @@ -1,21 +1,20 @@ ''' test_cases.py -Copyright (c) 2016-2019, Postgres Professional +Copyright (c) 2016-2020, Postgres Professional ''' -import common import os -import progressbar -import psycopg2.extensions import subprocess import time +import progressbar +import psycopg2.extensions + +import common + class DataLoadException(Exception): pass class StressTestException(Exception): pass -TPC_DS_EXCLUDE_LIST = [] # actual numbers of TPC-DS tests to exclude -TPC_DS_STATEMENT_TIMEOUT = 20000 # statement_timeout in ms - def setup_tpcds(config): print('Setting up TPC-DS test...') subprocess.call(['./tests/prepare_stress.sh']) @@ -62,6 +61,7 @@ def run_tpcds(config): pid = acon.get_backend_pid() print('Starting TPC-DS queries...') + TPC_DS_EXCLUDE_LIST = [] # actual numbers of TPC-DS tests to exclude timeout_list = [] bar = progressbar.ProgressBar(max_value=len(queries)) for i, query in enumerate(queries): @@ -70,6 +70,7 @@ def run_tpcds(config): continue try: # Set query timeout to TPC_DS_STATEMENT_TIMEOUT / 1000 seconds + TPC_DS_STATEMENT_TIMEOUT = 20000 common.set_guc(acon, 'statement_timeout', TPC_DS_STATEMENT_TIMEOUT) # run query @@ -84,7 +85,7 @@ def run_tpcds(config): state, n_first_getting_qs_retries = BEFORE_GETTING_QS, 0 while True: result, notices = common.pg_query_state(config, pid) - # run state machine to determine the first getting query state + # run state machine to determine the first getting of query state # and query finishing if state == BEFORE_GETTING_QS: if len(result) > 0 or common.BACKEND_IS_ACTIVE_INFO in notices: From 8545705524632fdc8c8f953476c9dcd0bc26e7f9 Mon Sep 17 00:00:00 2001 From: Maksim Date: Fri, 10 Jan 2020 12:14:22 +0300 Subject: [PATCH 24/91] Light refactoring regarding constants in run_tpcds function --- tests/tpcds.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/tpcds.py b/tests/tpcds.py index 892540c..8ac7183 100644 --- a/tests/tpcds.py +++ b/tests/tpcds.py @@ -51,6 +51,9 @@ def setup_tpcds(config): def run_tpcds(config): """TPC-DS stress test""" + TPC_DS_EXCLUDE_LIST = [] # actual numbers of TPC-DS tests to exclude + TPC_DS_STATEMENT_TIMEOUT = 20000 # statement_timeout in ms + print('Preparing TPC-DS queries...') queries = [] for query_file in sorted(os.listdir('tmp_stress/tpcds-result-reproduction/query_qualification/')): @@ -61,7 +64,6 @@ def run_tpcds(config): pid = acon.get_backend_pid() print('Starting TPC-DS queries...') - TPC_DS_EXCLUDE_LIST = [] # actual numbers of TPC-DS tests to exclude timeout_list = [] bar = progressbar.ProgressBar(max_value=len(queries)) for i, query in enumerate(queries): @@ -70,7 +72,6 @@ def run_tpcds(config): continue try: # Set query timeout to TPC_DS_STATEMENT_TIMEOUT / 1000 seconds - TPC_DS_STATEMENT_TIMEOUT = 20000 common.set_guc(acon, 'statement_timeout', TPC_DS_STATEMENT_TIMEOUT) # run query From 1b095a10e6c29ac2ccf657238ab724fb7457b67a Mon Sep 17 00:00:00 2001 From: Maksim Milyutin Date: Mon, 13 Jan 2020 14:32:34 +0300 Subject: [PATCH 25/91] Fix README --- README.md | 50 +++++++++++++++++++++++++++----------------------- 1 file changed, 27 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 93e60a7..e151bbb 100644 --- a/README.md +++ b/README.md @@ -2,12 +2,12 @@ [![codecov](https://codecov.io/gh/postgrespro/pg_query_state/branch/master/graph/badge.svg)](https://codecov.io/gh/postgrespro/pg_query_state) # pg\_query\_state -The `pg_query_state` module provides facility to know the current state of query execution on working backend. To enable this extension you have to patch the latest stable version of PostgreSQL. Different branches are intended for different version numbers of PostgreSQL, e.g., branch _PG9_5_ corresponds to PostgreSQL 9.5. +The `pg_query_state` module provides facility to know the current state of query execution on working backend. To enable this extension you have to patch the stable version of PostgreSQL, recompile it and deploy new binaries. All patch files are located in `patches/` directory and tagged with suffix of PostgreSQL version number. ## Overview -Each nonutility query statement (SELECT/INSERT/UPDATE/DELETE) after optimization/planning stage is translated into plan tree which is kind of imperative representation of declarative SQL query. EXPLAIN ANALYZE request allows to demonstrate execution statistics gathered from each node of plan tree (full time of execution, number rows emitted to upper nodes, etc). But this statistics is collected after execution of query. This module allows to show actual statistics of query running on external backend. At that, format of resulting output is almost identical to ordinal EXPLAIN ANALYZE. Thus users are able to track of query execution in progress. +Each nonutility query statement (SELECT/INSERT/UPDATE/DELETE) after optimization/planning stage is translated into plan tree which is kind of imperative representation of SQL query execution algorithm. EXPLAIN ANALYZE request allows to demonstrate execution statistics gathered from each node of plan tree (full time of execution, number rows emitted to upper nodes, etc). But this statistics is collected after execution of query. This module allows to show actual statistics of query running gathered from external backend. At that, format of resulting output is almost identical to ordinal EXPLAIN ANALYZE. Thus users are able to track of query execution in progress. -In fact, this module is able to explore external backend and determine its actual state. Particularly it's helpful when backend executes a heavy query or gets stuck. +In fact, this module is able to explore external backend and determine its actual state. Particularly it's helpful when backend executes a heavy query and gets stuck. ## Use cases Using this module there can help in the following things: @@ -15,8 +15,7 @@ Using this module there can help in the following things: - overwatch the query execution ## Installation -To install `pg_query_state`, please apply corresponding patches `custom_signal_(PG_VERSION).patch` and `runtime_explain.patch` (or `runtime_explain_11.0.patch` for PG11) to reqired stable version of PostgreSQL and rebuild PostgreSQL. - +To install `pg_query_state`, please apply corresponding patches `custom_signal_(PG_VERSION).patch` and `runtime_explain_(PG_VERSION).patch` (or `runtime_explain.patch` for PG version <= 10.0) in `patches/` directory to reqired stable version of PostgreSQL and rebuild PostgreSQL. Then execute this in the module's directory: ``` @@ -35,7 +34,7 @@ Done! ## Tests Test using parallel sessions with Python 3+ compatible script: ```shell -python tests/pg_qs_test_runner.py [OPTION]... +python3 tests/pg_qs_test_runner.py [OPTION]... ``` *prerequisite packages*: * `psycopg2` version 2.6 or later @@ -48,7 +47,8 @@ python tests/pg_qs_test_runner.py [OPTION]... * *- -database* --- database name, default value is *postgres* * *- -user* --- user name, default value is *postgres* * *- -password* --- user's password, default value is empty -* *- -tpc-ds* --- runs only stress tests on TPC-DS benchmark +* *- -tpc-ds-setup* --- setup database to run TPC-DS benchmark +* *- -tpc-ds-run* --- runs only stress tests on TPC-DS benchmark Or run all tests in `Docker` using: @@ -61,24 +61,27 @@ docker-compose build docker-compose run tests ``` -There are different test levels: `hardcore`, `nightmare` (runs tests under `valgrind`) and `stress` (runs tests under `TPC-DS` load). +There are different test levels: `hardcore`, `nightmare` (runs tests under `valgrind`) and `stress` (runs tests under `TPC-DS` load). ## Function pg\_query\_state ```plpgsql -pg_query_state(integer pid, - verbose boolean DEFAULT FALSE, - costs boolean DEFAULT FALSE, - timing boolean DEFAULT FALSE, - buffers boolean DEFAULT FALSE, - triggers boolean DEFAULT FALSE, - format text DEFAULT 'text') - returns TABLE ( pid integer, - frame_number integer, - query_text text, - plan text, - leader_pid integer) +pg_query_state( + integer pid, + verbose boolean DEFAULT FALSE, + costs boolean DEFAULT FALSE, + timing boolean DEFAULT FALSE, + buffers boolean DEFAULT FALSE, + triggers boolean DEFAULT FALSE, + format text DEFAULT 'text' +) returns TABLE ( + pid integer, + frame_number integer, + query_text text, + plan text, + leader_pid integer +) ``` -Extract current query state from backend with specified `pid`. Since parallel query can spawn multiple workers and function call causes nested subqueries so that state of execution may be viewed as stack of running queries, return value of `pg_query_state` has type `TABLE (pid integer, frame_number integer, query_text text, plan text, leader_pid integer)`. It represents tree structure consisting of leader process and its spawned workers identified by `pid`. Each worker refers to leader through `leader_pid` column. For leader process the value of this column is` null`. The state of each process is represented as stack of function calls. Each frame of that stack is specified as correspondence between `frame_number` starting from zero, `query_text` and `plan` with online statistics columns. +extracts the current query state from backend with specified `pid`. Since parallel query can spawn multiple workers and function call causes nested subqueries so that state of execution may be viewed as stack of running queries, return value of `pg_query_state` has type `TABLE (pid integer, frame_number integer, query_text text, plan text, leader_pid integer)`. It represents tree structure consisting of leader process and its spawned workers identified by `pid`. Each worker refers to leader through `leader_pid` column. For leader process the value of this column is` null`. The state of each process is represented as stack of function calls. Each frame of that stack is specified as correspondence between `frame_number` starting from zero, `query_text` and `plan` with online statistics columns. Thus, user can see the states of main query and queries generated from function calls for leader process and all workers spawned from it. @@ -95,7 +98,7 @@ Optional arguments: If callable backend is not executing any query the function prints INFO message about backend's state taken from `pg_stat_activity` view if it exists there. -Calling role have to be superuser or member of the role whose backend is being called. Otherwise function prints ERROR message `permission denied`. +**_Warning_**: Calling role have to be superuser or member of the role whose backend is being called. Otherwise function prints ERROR message `permission denied`. ## Configuration settings There are several user-accessible [GUC](https://www.postgresql.org/docs/9.5/static/config-setting.html) variables designed to toggle the whole module and the collecting of specific statistic parameters while query is running: @@ -301,4 +304,5 @@ leader_pid | (null) Do not hesitate to post your issues, questions and new ideas at the [issues](https://github.com/postgrespro/pg_query_state/issues) page. ## Authors -Maksim Milyutin Postgres Professional Ltd., Russia +[Maksim Milyutin](https://github.com/maksm90) +Alexey Kondratov Postgres Professional Ltd., Russia From 267c6cedb2d9cdaf03e0138524aa197e0f92e9ae Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Fri, 20 Nov 2020 18:28:24 +0300 Subject: [PATCH 26/91] [refer #PGPRO-4197] Fix shm_mq access sycnhronization --- pg_query_state.c | 21 ++++++++++++++++++++- pg_query_state.h | 1 + signal_handler.c | 1 + 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/pg_query_state.c b/pg_query_state.c index 55aec13..6b5a01e 100644 --- a/pg_query_state.c +++ b/pg_query_state.c @@ -86,6 +86,7 @@ typedef struct slock_t mutex; /* protect concurrent access to `userid` */ Oid userid; Latch *caller; + pg_atomic_uint32 n_peers; } RemoteUserIdResult; static void SendCurrentUserId(void); @@ -150,6 +151,7 @@ pg_qs_shmem_startup(void) counterpart_userid = shm_toc_allocate(toc, sizeof(RemoteUserIdResult)); shm_toc_insert(toc, num_toc++, counterpart_userid); SpinLockInit(&counterpart_userid->mutex); + pg_atomic_init_u32(&counterpart_userid->n_peers, 0); params = shm_toc_allocate(toc, sizeof(pg_qs_params)); shm_toc_insert(toc, num_toc++, params); @@ -481,6 +483,7 @@ pg_query_state(PG_FUNCTION_ARGS) shm_mq_msg *msg; List *bg_worker_procs = NIL; List *msgs; + int i; if (!module_initialized) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -513,6 +516,13 @@ pg_query_state(PG_FUNCTION_ARGS) init_lock_tag(&tag, PG_QUERY_STATE_KEY); LockAcquire(&tag, ExclusiveLock, false, false); + for (i = 0; pg_atomic_read_u32(&counterpart_userid->n_peers) != 0 && i < MIN_TIMEOUT/1000; i++) + { + pg_usleep(1000000); /* wait one second */ + CHECK_FOR_INTERRUPTS(); + } + pg_atomic_write_u32(&counterpart_userid->n_peers, 1); + counterpart_user_id = GetRemoteBackendUserId(proc); if (!(superuser() || GetUserId() == counterpart_user_id)) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), @@ -970,6 +980,7 @@ GetRemoteBackendQueryStates(PGPROC *leader, continue; } + pg_atomic_add_fetch_u32(&counterpart_userid->n_peers, 1); alive_procs = lappend(alive_procs, proc); } @@ -1023,7 +1034,6 @@ GetRemoteBackendQueryStates(PGPROC *leader, shm_mq_detach(mqh); #endif } - return result; signal_error: @@ -1033,3 +1043,12 @@ GetRemoteBackendQueryStates(PGPROC *leader, ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("error in message queue data transmitting"))); } + +void +DetachPeer(void) +{ + int n_peers = pg_atomic_fetch_sub_u32(&counterpart_userid->n_peers, 1); + if (n_peers <= 0) + ereport(LOG, (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("pg_query_state peer is not responding"))); +} diff --git a/pg_query_state.h b/pg_query_state.h index a8c13d9..3963ba0 100644 --- a/pg_query_state.h +++ b/pg_query_state.h @@ -69,5 +69,6 @@ extern shm_mq *mq; /* signal_handler.c */ extern void SendQueryState(void); +extern void DetachPeer(void); #endif diff --git a/signal_handler.c b/signal_handler.c index f46d264..bfe069f 100644 --- a/signal_handler.c +++ b/signal_handler.c @@ -214,4 +214,5 @@ SendQueryState(void) serialize_stack(msg->stack, qs_stack); shm_mq_send(mqh, msglen, msg, false); } + DetachPeer(); } From 1a8558c8756a94a453858cb8acb21985ff23326b Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Mon, 23 Nov 2020 09:49:01 +0300 Subject: [PATCH 27/91] [refer #PGPRO-4197] Allow pg_query_state only for nomal backends --- pg_query_state.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pg_query_state.c b/pg_query_state.c index 6b5a01e..535b124 100644 --- a/pg_query_state.c +++ b/pg_query_state.c @@ -494,7 +494,7 @@ pg_query_state(PG_FUNCTION_ARGS) errmsg("attempt to extract state of current process"))); proc = BackendPidGetProc(pid); - if (!proc || proc->backendId == InvalidBackendId) + if (!proc || proc->backendId == InvalidBackendId || proc->databaseId == InvalidOid || proc->roleId == InvalidOid) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("backend with pid=%d not found", pid))); @@ -885,7 +885,7 @@ GetRemoteBackendWorkers(PGPROC *proc) mqh = shm_mq_attach(mq, NULL, NULL); mq_receive_result = shm_mq_receive(mqh, &msg_len, (void **) &msg, false); - if (mq_receive_result != SHM_MQ_SUCCESS) + if (mq_receive_result != SHM_MQ_SUCCESS || msg == NULL || msg_len != sizeof(int) + msg->number*sizeof(pid_t)) goto mq_error; for (i = 0; i < msg->number; i++) From fbdc41acc96e9c43ae114ba79ba8075921d753a5 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Mon, 23 Nov 2020 17:02:49 +0300 Subject: [PATCH 28/91] [refer #PGPRO-4197] Correctly handle timeouts --- pg_query_state.c | 23 ++++++++++++++--------- pg_query_state.h | 3 +++ signal_handler.c | 21 ++++++++++++++++++--- 3 files changed, 35 insertions(+), 12 deletions(-) diff --git a/pg_query_state.c b/pg_query_state.c index 535b124..fe654f5 100644 --- a/pg_query_state.c +++ b/pg_query_state.c @@ -36,8 +36,6 @@ PG_MODULE_MAGIC; #define PG_QS_MODULE_KEY 0xCA94B108 #define PG_QUERY_STATE_KEY 0 -#define MIN_TIMEOUT 5000 - #define TEXT_CSTR_CMP(text, cstr) \ (memcmp(VARDATA(text), (cstr), VARSIZE(text) - VARHDRSZ)) @@ -516,11 +514,14 @@ pg_query_state(PG_FUNCTION_ARGS) init_lock_tag(&tag, PG_QUERY_STATE_KEY); LockAcquire(&tag, ExclusiveLock, false, false); - for (i = 0; pg_atomic_read_u32(&counterpart_userid->n_peers) != 0 && i < MIN_TIMEOUT/1000; i++) + for (i = 0; pg_atomic_read_u32(&counterpart_userid->n_peers) != 0 && i <= MAX_TIMEOUT/1000; i++) { pg_usleep(1000000); /* wait one second */ CHECK_FOR_INTERRUPTS(); } + if (i > MAX_TIMEOUT/1000) + elog(WARNING, "pg_query_state: last request was interrupted"); + pg_atomic_write_u32(&counterpart_userid->n_peers, 1); counterpart_user_id = GetRemoteBackendUserId(proc); @@ -741,15 +742,15 @@ shm_mq_receive_with_timeout(shm_mq_handle *mqh, { int rc = 0; long delay = timeout; + instr_time start_time; + instr_time cur_time; + + INSTR_TIME_SET_CURRENT(start_time); for (;;) { - instr_time start_time; - instr_time cur_time; shm_mq_result mq_receive_result; - INSTR_TIME_SET_CURRENT(start_time); - mq_receive_result = shm_mq_receive(mqh, nbytesp, datap, true); if (mq_receive_result != SHM_MQ_WOULD_BLOCK) return mq_receive_result; @@ -772,6 +773,8 @@ shm_mq_receive_with_timeout(shm_mq_handle *mqh, INSTR_TIME_SUBTRACT(cur_time, start_time); delay = timeout - (long) INSTR_TIME_GET_MILLISEC(cur_time); + if (delay <= 0) + return SHM_MQ_WOULD_BLOCK; CHECK_FOR_INTERRUPTS(); ResetLatch(MyLatch); @@ -970,6 +973,9 @@ GetRemoteBackendQueryStates(PGPROC *leader, PGPROC *proc = (PGPROC *) lfirst(iter); if (!proc || !proc->pid) continue; + + pg_atomic_add_fetch_u32(&counterpart_userid->n_peers, 1); + sig_result = SendProcSignal(proc->pid, QueryStatePollReason, proc->backendId); @@ -980,7 +986,6 @@ GetRemoteBackendQueryStates(PGPROC *leader, continue; } - pg_atomic_add_fetch_u32(&counterpart_userid->n_peers, 1); alive_procs = lappend(alive_procs, proc); } @@ -1018,7 +1023,7 @@ GetRemoteBackendQueryStates(PGPROC *leader, mq_receive_result = shm_mq_receive_with_timeout(mqh, &len, (void **) &msg, - MIN_TIMEOUT); + MAX_TIMEOUT); if (mq_receive_result != SHM_MQ_SUCCESS) /* counterpart is died, not consider it */ continue; diff --git a/pg_query_state.h b/pg_query_state.h index 3963ba0..f652a51 100644 --- a/pg_query_state.h +++ b/pg_query_state.h @@ -22,6 +22,9 @@ #define TIMINIG_OFF_WARNING 1 #define BUFFERS_OFF_WARNING 2 +#define MAX_TIMEOUT 5000 /* 5 seconds */ + + /* * Result status on query state request from asked backend */ diff --git a/signal_handler.c b/signal_handler.c index bfe069f..fa047e4 100644 --- a/signal_handler.c +++ b/signal_handler.c @@ -157,6 +157,11 @@ void SendQueryState(void) { shm_mq_handle *mqh; + instr_time start_time; + instr_time cur_time; + long delay = MAX_TIMEOUT; + + INSTR_TIME_SET_CURRENT(start_time); /* wait until caller sets this process as sender to message queue */ for (;;) @@ -165,12 +170,22 @@ SendQueryState(void) break; #if PG_VERSION_NUM < 100000 - WaitLatch(MyLatch, WL_LATCH_SET, 0); + WaitLatch(MyLatch, WL_LATCH_SET | WL_TIMEOUT, delay); #elif PG_VERSION_NUM < 120000 - WaitLatch(MyLatch, WL_LATCH_SET, 0, PG_WAIT_IPC); + WaitLatch(MyLatch, WL_LATCH_SET | WL_TIMEOUT, delay, PG_WAIT_IPC); #else - WaitLatch(MyLatch, WL_LATCH_SET | WL_EXIT_ON_PM_DEATH, 0, PG_WAIT_IPC); + WaitLatch(MyLatch, WL_LATCH_SET | WL_EXIT_ON_PM_DEATH | WL_TIMEOUT, delay, PG_WAIT_IPC); #endif + INSTR_TIME_SET_CURRENT(cur_time); + INSTR_TIME_SUBTRACT(cur_time, start_time); + + delay = MAX_TIMEOUT - (long) INSTR_TIME_GET_MILLISEC(cur_time); + if (delay <= 0) + { + elog(WARNING, "pg_query_state: failed to receive request from leader"); + DetachPeer(); + return; + } CHECK_FOR_INTERRUPTS(); ResetLatch(MyLatch); } From c0f0363f50d0d0695f22e20acd1a4d584a1ea720 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Tue, 24 Nov 2020 08:43:46 +0300 Subject: [PATCH 29/91] Make send timeout smaller than receive timeout --- pg_query_state.c | 21 +++++++++++++++------ pg_query_state.h | 4 +++- signal_handler.c | 4 ++-- 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/pg_query_state.c b/pg_query_state.c index fe654f5..655e68d 100644 --- a/pg_query_state.c +++ b/pg_query_state.c @@ -481,7 +481,8 @@ pg_query_state(PG_FUNCTION_ARGS) shm_mq_msg *msg; List *bg_worker_procs = NIL; List *msgs; - int i; + instr_time start_time; + instr_time cur_time; if (!module_initialized) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -514,14 +515,22 @@ pg_query_state(PG_FUNCTION_ARGS) init_lock_tag(&tag, PG_QUERY_STATE_KEY); LockAcquire(&tag, ExclusiveLock, false, false); - for (i = 0; pg_atomic_read_u32(&counterpart_userid->n_peers) != 0 && i <= MAX_TIMEOUT/1000; i++) + INSTR_TIME_SET_CURRENT(start_time); + + while (pg_atomic_read_u32(&counterpart_userid->n_peers) != 0) { pg_usleep(1000000); /* wait one second */ CHECK_FOR_INTERRUPTS(); - } - if (i > MAX_TIMEOUT/1000) - elog(WARNING, "pg_query_state: last request was interrupted"); + INSTR_TIME_SET_CURRENT(cur_time); + INSTR_TIME_SUBTRACT(cur_time, start_time); + + if (INSTR_TIME_GET_MILLISEC(cur_time) > MAX_RCV_TIMEOUT) + { + elog(WARNING, "pg_query_state: last request was interrupted"); + break; + } + } pg_atomic_write_u32(&counterpart_userid->n_peers, 1); counterpart_user_id = GetRemoteBackendUserId(proc); @@ -1023,7 +1032,7 @@ GetRemoteBackendQueryStates(PGPROC *leader, mq_receive_result = shm_mq_receive_with_timeout(mqh, &len, (void **) &msg, - MAX_TIMEOUT); + MAX_RCV_TIMEOUT); if (mq_receive_result != SHM_MQ_SUCCESS) /* counterpart is died, not consider it */ continue; diff --git a/pg_query_state.h b/pg_query_state.h index f652a51..bda5875 100644 --- a/pg_query_state.h +++ b/pg_query_state.h @@ -22,7 +22,9 @@ #define TIMINIG_OFF_WARNING 1 #define BUFFERS_OFF_WARNING 2 -#define MAX_TIMEOUT 5000 /* 5 seconds */ +/* Receive timeout should be larger than send timeout to let workers stop waiting before polling process */ +#define MAX_RCV_TIMEOUT 6000 /* 6 seconds */ +#define MAX_SND_TIMEOUT 3000 /* 3 seconds */ /* diff --git a/signal_handler.c b/signal_handler.c index fa047e4..50559a9 100644 --- a/signal_handler.c +++ b/signal_handler.c @@ -159,7 +159,7 @@ SendQueryState(void) shm_mq_handle *mqh; instr_time start_time; instr_time cur_time; - long delay = MAX_TIMEOUT; + long delay = MAX_SND_TIMEOUT; INSTR_TIME_SET_CURRENT(start_time); @@ -179,7 +179,7 @@ SendQueryState(void) INSTR_TIME_SET_CURRENT(cur_time); INSTR_TIME_SUBTRACT(cur_time, start_time); - delay = MAX_TIMEOUT - (long) INSTR_TIME_GET_MILLISEC(cur_time); + delay = MAX_SND_TIMEOUT - (long) INSTR_TIME_GET_MILLISEC(cur_time); if (delay <= 0) { elog(WARNING, "pg_query_state: failed to receive request from leader"); From 81a915aacc4ec66f911e2fc255f34bdd4306fe79 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Tue, 24 Nov 2020 14:28:11 +0300 Subject: [PATCH 30/91] Set mq sender before sending signal --- pg_query_state.c | 10 +++++----- signal_handler.c | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pg_query_state.c b/pg_query_state.c index 655e68d..2a79b5e 100644 --- a/pg_query_state.c +++ b/pg_query_state.c @@ -747,10 +747,10 @@ static shm_mq_result shm_mq_receive_with_timeout(shm_mq_handle *mqh, Size *nbytesp, void **datap, - long timeout) + int64 timeout) { int rc = 0; - long delay = timeout; + int64 delay = timeout; instr_time start_time; instr_time cur_time; @@ -781,7 +781,7 @@ shm_mq_receive_with_timeout(shm_mq_handle *mqh, INSTR_TIME_SET_CURRENT(cur_time); INSTR_TIME_SUBTRACT(cur_time, start_time); - delay = timeout - (long) INSTR_TIME_GET_MILLISEC(cur_time); + delay = timeout - (int64) INSTR_TIME_GET_MILLISEC(cur_time); if (delay <= 0) return SHM_MQ_WOULD_BLOCK; @@ -967,6 +967,8 @@ GetRemoteBackendQueryStates(PGPROC *leader, /* initialize message queue that will transfer query states */ mq = shm_mq_create(mq, QUEUE_SIZE); + shm_mq_set_sender(mq, leader); + shm_mq_set_receiver(mq, MyProc); /* * send signal `QueryStatePollReason` to all processes and define all alive @@ -999,8 +1001,6 @@ GetRemoteBackendQueryStates(PGPROC *leader, } /* extract query state from leader process */ - shm_mq_set_sender(mq, leader); - shm_mq_set_receiver(mq, MyProc); mqh = shm_mq_attach(mq, NULL, NULL); mq_receive_result = shm_mq_receive(mqh, &len, (void **) &msg, false); if (mq_receive_result != SHM_MQ_SUCCESS) diff --git a/signal_handler.c b/signal_handler.c index 50559a9..bd2d423 100644 --- a/signal_handler.c +++ b/signal_handler.c @@ -159,7 +159,7 @@ SendQueryState(void) shm_mq_handle *mqh; instr_time start_time; instr_time cur_time; - long delay = MAX_SND_TIMEOUT; + int64 delay = MAX_SND_TIMEOUT; INSTR_TIME_SET_CURRENT(start_time); @@ -179,7 +179,7 @@ SendQueryState(void) INSTR_TIME_SET_CURRENT(cur_time); INSTR_TIME_SUBTRACT(cur_time, start_time); - delay = MAX_SND_TIMEOUT - (long) INSTR_TIME_GET_MILLISEC(cur_time); + delay = MAX_SND_TIMEOUT - (int64) INSTR_TIME_GET_MILLISEC(cur_time); if (delay <= 0) { elog(WARNING, "pg_query_state: failed to receive request from leader"); From 9ad9d93516fee5aaac0ab3d66fb72ddd08a6fe45 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Tue, 1 Dec 2020 14:44:11 +0300 Subject: [PATCH 31/91] [refer #PGPRO-4197] Add request ID --- pg_query_state.c | 19 +++++++++++++++++-- pg_query_state.h | 2 ++ signal_handler.c | 9 ++++++--- 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/pg_query_state.c b/pg_query_state.c index 2a79b5e..6848c10 100644 --- a/pg_query_state.c +++ b/pg_query_state.c @@ -952,6 +952,7 @@ GetRemoteBackendQueryStates(PGPROC *leader, shm_mq_result mq_receive_result; shm_mq_msg *msg; Size len; + static int reqid = 0; Assert(QueryStatePollReason != INVALID_PROCSIGNAL); Assert(mq); @@ -963,6 +964,7 @@ GetRemoteBackendQueryStates(PGPROC *leader, params->buffers = buffers; params->triggers = triggers; params->format = format; + params->reqid = ++reqid; pg_write_barrier(); /* initialize message queue that will transfer query states */ @@ -1002,9 +1004,13 @@ GetRemoteBackendQueryStates(PGPROC *leader, /* extract query state from leader process */ mqh = shm_mq_attach(mq, NULL, NULL); + elog(DEBUG1, "Wait response from leader %d", leader->pid); mq_receive_result = shm_mq_receive(mqh, &len, (void **) &msg, false); if (mq_receive_result != SHM_MQ_SUCCESS) goto mq_error; + if (msg->reqid != reqid) + goto mq_error; + Assert(len == msg->length); result = lappend(result, copy_msg(msg)); #if PG_VERSION_NUM < 100000 @@ -1021,6 +1027,7 @@ GetRemoteBackendQueryStates(PGPROC *leader, PGPROC *proc = (PGPROC *) lfirst(iter); /* prepare message queue to transfer data */ + elog(DEBUG1, "Wait response from worker %d", proc->pid); mq = shm_mq_create(mq, QUEUE_SIZE); shm_mq_set_sender(mq, proc); shm_mq_set_receiver(mq, MyProc); /* this function notifies the @@ -1034,9 +1041,12 @@ GetRemoteBackendQueryStates(PGPROC *leader, (void **) &msg, MAX_RCV_TIMEOUT); if (mq_receive_result != SHM_MQ_SUCCESS) + { /* counterpart is died, not consider it */ - continue; - + goto mq_error; + } + if (msg->reqid != reqid) + goto mq_error; Assert(len == msg->length); /* aggregate result data */ @@ -1054,6 +1064,11 @@ GetRemoteBackendQueryStates(PGPROC *leader, ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("invalid send signal"))); mq_error: +#if PG_VERSION_NUM < 100000 + shm_mq_detach(mq); +#else + shm_mq_detach(mqh); +#endif ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("error in message queue data transmitting"))); } diff --git a/pg_query_state.h b/pg_query_state.h index bda5875..9cae5aa 100644 --- a/pg_query_state.h +++ b/pg_query_state.h @@ -42,6 +42,7 @@ typedef enum */ typedef struct { + int reqid; int length; /* size of message record, for sanity check */ PGPROC *proc; PG_QS_RequestResult result_code; @@ -56,6 +57,7 @@ typedef struct /* pg_query_state arguments */ typedef struct { + int reqid; bool verbose; bool costs; bool timing; diff --git a/signal_handler.c b/signal_handler.c index bd2d423..2780b81 100644 --- a/signal_handler.c +++ b/signal_handler.c @@ -160,6 +160,7 @@ SendQueryState(void) instr_time start_time; instr_time cur_time; int64 delay = MAX_SND_TIMEOUT; + int reqid = params->reqid; INSTR_TIME_SET_CURRENT(start_time); @@ -189,13 +190,13 @@ SendQueryState(void) CHECK_FOR_INTERRUPTS(); ResetLatch(MyLatch); } - + elog(DEBUG1, "Worker %d receives pg_query_state request from %d", shm_mq_get_sender(mq)->pid, shm_mq_get_receiver(mq)->pid); mqh = shm_mq_attach(mq, NULL, NULL); /* check if module is enabled */ if (!pg_qs_enable) { - shm_mq_msg msg = { BASE_SIZEOF_SHM_MQ_MSG, MyProc, STAT_DISABLED }; + shm_mq_msg msg = { reqid, BASE_SIZEOF_SHM_MQ_MSG, MyProc, STAT_DISABLED }; shm_mq_send(mqh, msg.length, &msg, false); } @@ -203,7 +204,7 @@ SendQueryState(void) /* check if backend doesn't execute any query */ else if (list_length(QueryDescStack) == 0) { - shm_mq_msg msg = { BASE_SIZEOF_SHM_MQ_MSG, MyProc, QUERY_NOT_RUNNING }; + shm_mq_msg msg = { reqid, BASE_SIZEOF_SHM_MQ_MSG, MyProc, QUERY_NOT_RUNNING }; shm_mq_send(mqh, msg.length, &msg, false); } @@ -215,6 +216,7 @@ SendQueryState(void) int msglen = sizeof(shm_mq_msg) + serialized_stack_length(qs_stack); shm_mq_msg *msg = palloc(msglen); + msg->reqid = reqid; msg->length = msglen; msg->proc = MyProc; msg->result_code = QS_RETURNED; @@ -229,5 +231,6 @@ SendQueryState(void) serialize_stack(msg->stack, qs_stack); shm_mq_send(mqh, msglen, msg, false); } + elog(DEBUG1, "Worker %d sends response for pg_query_state to %d", shm_mq_get_sender(mq)->pid, shm_mq_get_receiver(mq)->pid); DetachPeer(); } From 404cd45eedcedc95dbe77a8c3e887278b934c65d Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Tue, 1 Dec 2020 19:38:28 +0300 Subject: [PATCH 32/91] [refer #4197]] Protect queue access by lock --- pg_query_state.c | 54 +++++++++++++++++++++++++++++++++--------------- pg_query_state.h | 9 +++++++- signal_handler.c | 10 +++++++++ 3 files changed, 55 insertions(+), 18 deletions(-) diff --git a/pg_query_state.c b/pg_query_state.c index 6848c10..abf8ffa 100644 --- a/pg_query_state.c +++ b/pg_query_state.c @@ -33,9 +33,6 @@ PG_MODULE_MAGIC; #endif -#define PG_QS_MODULE_KEY 0xCA94B108 -#define PG_QUERY_STATE_KEY 0 - #define TEXT_CSTR_CMP(text, cstr) \ (memcmp(VARDATA(text), (cstr), VARSIZE(text) - VARHDRSZ)) @@ -78,6 +75,7 @@ static const char *be_state_str[] = { /* BackendState -> string repr */ "idle in transaction (aborted)", /* STATE_IDLEINTRANSACTION_ABORTED */ "disabled", /* STATE_DISABLED */ }; +static int reqid = 0; typedef struct { @@ -376,20 +374,29 @@ search_be_status(int pid) return NULL; } -/* - * Init userlock - */ -static void -init_lock_tag(LOCKTAG *tag, uint32 key) + +void +UnlockShmem(LOCKTAG *tag) { + LockRelease(tag, ExclusiveLock, false); +} + +void +LockShmem(LOCKTAG *tag, uint32 key) +{ + LockAcquireResult result; tag->locktag_field1 = PG_QS_MODULE_KEY; tag->locktag_field2 = key; tag->locktag_field3 = 0; tag->locktag_field4 = 0; tag->locktag_type = LOCKTAG_USERLOCK; tag->locktag_lockmethodid = USER_LOCKMETHOD; + result = LockAcquire(tag, ExclusiveLock, false, false); + Assert(result == LOCKACQUIRE_OK); } + + /* * Structure of stack frame of fucntion call which transfers through message queue */ @@ -512,8 +519,7 @@ pg_query_state(PG_FUNCTION_ARGS) * init and acquire lock so that any other concurrent calls of this fuction * can not occupy shared queue for transfering query state */ - init_lock_tag(&tag, PG_QUERY_STATE_KEY); - LockAcquire(&tag, ExclusiveLock, false, false); + LockShmem(&tag, PG_QS_RCV_KEY); INSTR_TIME_SET_CURRENT(start_time); @@ -532,6 +538,8 @@ pg_query_state(PG_FUNCTION_ARGS) } } pg_atomic_write_u32(&counterpart_userid->n_peers, 1); + params->reqid = ++reqid; + pg_write_barrier(); counterpart_user_id = GetRemoteBackendUserId(proc); if (!(superuser() || GetUserId() == counterpart_user_id)) @@ -553,7 +561,7 @@ pg_query_state(PG_FUNCTION_ARGS) if (list_length(msgs) == 0) { elog(WARNING, "backend does not reply"); - LockRelease(&tag, ExclusiveLock, false); + UnlockShmem(&tag); SRF_RETURN_DONE(funcctx); } @@ -570,12 +578,12 @@ pg_query_state(PG_FUNCTION_ARGS) else elog(INFO, "backend is not running query"); - LockRelease(&tag, ExclusiveLock, false); + UnlockShmem(&tag); SRF_RETURN_DONE(funcctx); } case STAT_DISABLED: elog(INFO, "query execution statistics disabled"); - LockRelease(&tag, ExclusiveLock, false); + UnlockShmem(&tag); SRF_RETURN_DONE(funcctx); case QS_RETURNED: { @@ -636,7 +644,7 @@ pg_query_state(PG_FUNCTION_ARGS) TupleDescInitEntry(tupdesc, (AttrNumber) 5, "leader_pid", INT4OID, -1, 0); funcctx->tuple_desc = BlessTupleDesc(tupdesc); - LockRelease(&tag, ExclusiveLock, false); + UnlockShmem(&tag); MemoryContextSwitchTo(oldcontext); } break; @@ -828,6 +836,7 @@ extract_running_bgworkers(PlanState *node, List **result) typedef struct { + int reqid; int number; pid_t pids[FLEXIBLE_ARRAY_MEMBER]; } BgWorkerPids; @@ -841,6 +850,9 @@ SendBgWorkerPids(void) int msg_len; int i; shm_mq_handle *mqh; + LOCKTAG tag; + + LockShmem(&tag, PG_QS_SND_KEY); mqh = shm_mq_attach(mq, NULL, NULL); @@ -856,6 +868,7 @@ SendBgWorkerPids(void) msg_len = offsetof(BgWorkerPids, pids) + sizeof(pid_t) * list_length(all_workers); msg = palloc(msg_len); + msg->reqid = params->reqid; msg->number = list_length(all_workers); i = 0; foreach(iter, all_workers) @@ -867,6 +880,7 @@ SendBgWorkerPids(void) } shm_mq_send(mqh, msg_len, msg, false); + UnlockShmem(&tag); } /* @@ -882,14 +896,17 @@ GetRemoteBackendWorkers(PGPROC *proc) Size msg_len; int i; List *result = NIL; + LOCKTAG tag; Assert(proc && proc->backendId != InvalidBackendId); Assert(WorkerPollReason != INVALID_PROCSIGNAL); Assert(mq); + LockShmem(&tag, PG_QS_SND_KEY); mq = shm_mq_create(mq, QUEUE_SIZE); shm_mq_set_sender(mq, proc); shm_mq_set_receiver(mq, MyProc); + UnlockShmem(&tag); sig_result = SendProcSignal(proc->pid, WorkerPollReason, proc->backendId); if (sig_result == -1) @@ -897,7 +914,7 @@ GetRemoteBackendWorkers(PGPROC *proc) mqh = shm_mq_attach(mq, NULL, NULL); mq_receive_result = shm_mq_receive(mqh, &msg_len, (void **) &msg, false); - if (mq_receive_result != SHM_MQ_SUCCESS || msg == NULL || msg_len != sizeof(int) + msg->number*sizeof(pid_t)) + if (mq_receive_result != SHM_MQ_SUCCESS || msg == NULL || msg->reqid != reqid || msg_len != offsetof(BgWorkerPids, pids) + msg->number*sizeof(pid_t)) goto mq_error; for (i = 0; i < msg->number; i++) @@ -952,7 +969,7 @@ GetRemoteBackendQueryStates(PGPROC *leader, shm_mq_result mq_receive_result; shm_mq_msg *msg; Size len; - static int reqid = 0; + LOCKTAG tag; Assert(QueryStatePollReason != INVALID_PROCSIGNAL); Assert(mq); @@ -964,13 +981,14 @@ GetRemoteBackendQueryStates(PGPROC *leader, params->buffers = buffers; params->triggers = triggers; params->format = format; - params->reqid = ++reqid; pg_write_barrier(); /* initialize message queue that will transfer query states */ + LockShmem(&tag, PG_QS_SND_KEY); mq = shm_mq_create(mq, QUEUE_SIZE); shm_mq_set_sender(mq, leader); shm_mq_set_receiver(mq, MyProc); + UnlockShmem(&tag); /* * send signal `QueryStatePollReason` to all processes and define all alive @@ -1028,11 +1046,13 @@ GetRemoteBackendQueryStates(PGPROC *leader, /* prepare message queue to transfer data */ elog(DEBUG1, "Wait response from worker %d", proc->pid); + LockShmem(&tag, PG_QS_SND_KEY); mq = shm_mq_create(mq, QUEUE_SIZE); shm_mq_set_sender(mq, proc); shm_mq_set_receiver(mq, MyProc); /* this function notifies the counterpart to come into data transfer */ + UnlockShmem(&tag); /* retrieve result data from message queue */ mqh = shm_mq_attach(mq, NULL, NULL); diff --git a/pg_query_state.h b/pg_query_state.h index 9cae5aa..5c6fbef 100644 --- a/pg_query_state.h +++ b/pg_query_state.h @@ -22,6 +22,10 @@ #define TIMINIG_OFF_WARNING 1 #define BUFFERS_OFF_WARNING 2 +#define PG_QS_MODULE_KEY 0xCA94B108 +#define PG_QS_RCV_KEY 0 +#define PG_QS_SND_KEY 1 + /* Receive timeout should be larger than send timeout to let workers stop waiting before polling process */ #define MAX_RCV_TIMEOUT 6000 /* 6 seconds */ #define MAX_SND_TIMEOUT 3000 /* 3 seconds */ @@ -34,7 +38,7 @@ typedef enum { QUERY_NOT_RUNNING, /* Backend doesn't execute any query */ STAT_DISABLED, /* Collection of execution statistics is disabled */ - QS_RETURNED /* Backend succesfully returned its query state */ + QS_RETURNED /* Backend succx[esfully returned its query state */ } PG_QS_RequestResult; /* @@ -48,6 +52,7 @@ typedef struct PG_QS_RequestResult result_code; int warnings; /* bitmap of warnings */ int stack_depth; + char filler[1024*1024]; char stack[FLEXIBLE_ARRAY_MEMBER]; /* sequencially laid out stack frames in form of text records */ } shm_mq_msg; @@ -77,5 +82,7 @@ extern shm_mq *mq; /* signal_handler.c */ extern void SendQueryState(void); extern void DetachPeer(void); +extern void UnlockShmem(LOCKTAG *tag); +extern void LockShmem(LOCKTAG *tag, uint32 key); #endif diff --git a/signal_handler.c b/signal_handler.c index 2780b81..c9de5d7 100644 --- a/signal_handler.c +++ b/signal_handler.c @@ -161,6 +161,7 @@ SendQueryState(void) instr_time cur_time; int64 delay = MAX_SND_TIMEOUT; int reqid = params->reqid; + LOCKTAG tag; INSTR_TIME_SET_CURRENT(start_time); @@ -190,9 +191,17 @@ SendQueryState(void) CHECK_FOR_INTERRUPTS(); ResetLatch(MyLatch); } + + LockShmem(&tag, PG_QS_SND_KEY); + elog(DEBUG1, "Worker %d receives pg_query_state request from %d", shm_mq_get_sender(mq)->pid, shm_mq_get_receiver(mq)->pid); mqh = shm_mq_attach(mq, NULL, NULL); + if (reqid != params->reqid || shm_mq_get_sender(mq) != MyProc) + { + UnlockShmem(&tag); + return; + } /* check if module is enabled */ if (!pg_qs_enable) { @@ -233,4 +242,5 @@ SendQueryState(void) } elog(DEBUG1, "Worker %d sends response for pg_query_state to %d", shm_mq_get_sender(mq)->pid, shm_mq_get_receiver(mq)->pid); DetachPeer(); + UnlockShmem(&tag); } From feaebcda914454f946c979f1167128d7ce0734c6 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Tue, 1 Dec 2020 19:39:28 +0300 Subject: [PATCH 33/91] [refer #4197]] Remove dummy field --- pg_query_state.h | 1 - 1 file changed, 1 deletion(-) diff --git a/pg_query_state.h b/pg_query_state.h index 5c6fbef..0cb8923 100644 --- a/pg_query_state.h +++ b/pg_query_state.h @@ -52,7 +52,6 @@ typedef struct PG_QS_RequestResult result_code; int warnings; /* bitmap of warnings */ int stack_depth; - char filler[1024*1024]; char stack[FLEXIBLE_ARRAY_MEMBER]; /* sequencially laid out stack frames in form of text records */ } shm_mq_msg; From 3402536cc1e802a3f78d768c66021e59be96408a Mon Sep 17 00:00:00 2001 From: Daria Lepikhova Date: Thu, 25 Mar 2021 14:06:26 +0500 Subject: [PATCH 34/91] Added PGRPO13 support. Modify Makefile for run installcheck. Modify script for docker running --- .travis.yml | 2 + Dockerfile.tmpl | 2 +- Makefile | 15 +- expected/corner_cases_3.out | 67 ++++++++ patches/custom_signals_13.0.patch | 206 +++++++++++++++++++++++ patches/runtime_explain_13.0.patch | 261 +++++++++++++++++++++++++++++ pg_query_state.c | 8 + run_tests.sh | 14 +- 8 files changed, 559 insertions(+), 16 deletions(-) create mode 100644 expected/corner_cases_3.out create mode 100644 patches/custom_signals_13.0.patch create mode 100644 patches/runtime_explain_13.0.patch mode change 100755 => 100644 run_tests.sh diff --git a/.travis.yml b/.travis.yml index 5d26b7a..c6c06bf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,6 +18,8 @@ notifications: on_failure: always env: + - PG_VERSION=13 LEVEL=hardcore USE_TPCDS=1 + - PG_VERSION=13 - PG_VERSION=12 LEVEL=hardcore USE_TPCDS=1 - PG_VERSION=12 - PG_VERSION=11 LEVEL=hardcore USE_TPCDS=1 diff --git a/Dockerfile.tmpl b/Dockerfile.tmpl index a9fdac2..1e512bc 100644 --- a/Dockerfile.tmpl +++ b/Dockerfile.tmpl @@ -2,7 +2,7 @@ FROM postgres:${PG_VERSION}-alpine # Install dependencies RUN apk add --no-cache \ - openssl curl git \ + openssl curl git patch \ perl perl-ipc-run \ make musl-dev gcc bison flex coreutils \ zlib-dev libedit-dev \ diff --git a/Makefile b/Makefile index aecfb45..4b2ada5 100644 --- a/Makefile +++ b/Makefile @@ -25,9 +25,12 @@ endif $(EXTENSION)--$(EXTVERSION).sql: init.sql cat $^ > $@ +ISOLATIONCHECKS=corner_cases + check: isolationcheck -ISOLATIONCHECKS=corner_cases +installcheck: + $(pg_isolation_regress_installcheck) $(ISOLATIONCHECKS) submake-isolation: $(MAKE) -C $(top_builddir)/src/test/isolation all @@ -37,14 +40,6 @@ isolationcheck: | submake-isolation temp-install $(pg_isolation_regress_check) \ --temp-config $(top_srcdir)/contrib/pg_query_state/test.conf \ --outputdir=isolation_output \ - $(ISOLATIONCHECKS) - -isolationcheck-install-force: all | submake-isolation temp-install - $(MKDIR_P) isolation_output - $(pg_isolation_regress_installcheck) \ - --outputdir=isolation_output \ - $(ISOLATIONCHECKS) - -.PHONY: isolationcheck isolationcheck-install-force check + $(ISOLATIONCHECKS) temp-install: EXTRA_INSTALL=contrib/pg_query_state diff --git a/expected/corner_cases_3.out b/expected/corner_cases_3.out new file mode 100644 index 0000000..845db75 --- /dev/null +++ b/expected/corner_cases_3.out @@ -0,0 +1,67 @@ +Parsed test spec with 2 sessions + +starting permutation: s1_pg_qs_1 +step s1_pg_qs_1: select pg_query_state(1); +ERROR: backend with pid=1 not found + +starting permutation: s1_pg_qs_2 +step s1_pg_qs_2: select pg_query_state(pg_backend_pid()); +ERROR: attempt to extract state of current process + +starting permutation: s1_save_pid s2_pg_qs_counterpart +step s1_save_pid: select save_own_pid(0); +save_own_pid + + +s2: INFO: state of backend is idle +step s2_pg_qs_counterpart: select pg_query_state(counterpart_pid(0)); +pg_query_state + + +starting permutation: s1_save_pid s1_disable_pg_qs s2_pg_qs_counterpart +step s1_save_pid: select save_own_pid(0); +save_own_pid + + +step s1_disable_pg_qs: set pg_query_state.enable to off; +s2: INFO: query execution statistics disabled +step s2_pg_qs_counterpart: select pg_query_state(counterpart_pid(0)); +pg_query_state + + +starting permutation: s1_set_bob s2_set_bob s1_save_pid s2_pg_qs_counterpart +step s1_set_bob: set role bob; +step s2_set_bob: set role bob; +step s1_save_pid: select save_own_pid(0); +save_own_pid + + +s2: INFO: state of backend is idle +step s2_pg_qs_counterpart: select pg_query_state(counterpart_pid(0)); +pg_query_state + + +starting permutation: s1_set_bob s2_set_su s1_save_pid s2_pg_qs_counterpart +step s1_set_bob: set role bob; +step s2_set_su: set role super; +step s1_save_pid: select save_own_pid(0); +save_own_pid + + +s2: INFO: state of backend is idle +step s2_pg_qs_counterpart: select pg_query_state(counterpart_pid(0)); +pg_query_state + + +starting permutation: s1_set_bob s2_set_alice s1_save_pid s2_pg_qs_counterpart +step s1_set_bob: set role bob; +step s2_set_alice: set role alice; +step s1_save_pid: select save_own_pid(0); +save_own_pid + + +step s2_pg_qs_counterpart: select pg_query_state(counterpart_pid(0)); +ERROR: permission denied +unused step name: s1_enable_pg_qs +unused step name: s1_pg_qs_counterpart +unused step name: s2_save_pid diff --git a/patches/custom_signals_13.0.patch b/patches/custom_signals_13.0.patch new file mode 100644 index 0000000..add965c --- /dev/null +++ b/patches/custom_signals_13.0.patch @@ -0,0 +1,206 @@ +diff --git a/src/backend/storage/ipc/procsignal.c b/src/backend/storage/ipc/procsignal.c +index 4fa385b0ece..fc1637a2e28 100644 +--- a/src/backend/storage/ipc/procsignal.c ++++ b/src/backend/storage/ipc/procsignal.c +@@ -88,12 +88,21 @@ typedef struct + (((flags) & (((uint32) 1) << (uint32) (type))) != 0) + + static ProcSignalHeader *ProcSignal = NULL; ++#define IsCustomProcSignalReason(reason) \ ++ ((reason) >= PROCSIG_CUSTOM_1 && (reason) <= PROCSIG_CUSTOM_N) ++ ++static bool CustomSignalPendings[NUM_CUSTOM_PROCSIGNALS]; ++static bool CustomSignalProcessing[NUM_CUSTOM_PROCSIGNALS]; ++static ProcSignalHandler_type CustomInterruptHandlers[NUM_CUSTOM_PROCSIGNALS]; ++ + static volatile ProcSignalSlot *MyProcSignalSlot = NULL; + + static bool CheckProcSignal(ProcSignalReason reason); + static void CleanupProcSignalState(int status, Datum arg); + static void ProcessBarrierPlaceholder(void); + ++static void CheckAndSetCustomSignalInterrupts(void); ++ + /* + * ProcSignalShmemSize + * Compute space needed for procsignal's shared memory +@@ -235,6 +244,36 @@ CleanupProcSignalState(int status, Datum arg) + slot->pss_pid = 0; + } + ++/* ++ * RegisterCustomProcSignalHandler ++ * Assign specific handler of custom process signal with new ++ * ProcSignalReason key. ++ * ++ * This function has to be called in _PG_init function of extensions at the ++ * stage of loading shared preloaded libraries. Otherwise it throws fatal error. ++ * ++ * Return INVALID_PROCSIGNAL if all slots for custom signals are occupied. ++ */ ++ProcSignalReason ++RegisterCustomProcSignalHandler(ProcSignalHandler_type handler) ++{ ++ ProcSignalReason reason; ++ ++ if (!process_shared_preload_libraries_in_progress) ++ ereport(FATAL, (errcode(ERRCODE_INTERNAL_ERROR), ++ errmsg("cannot register custom signal after startup"))); ++ ++ /* Iterate through custom signal slots to find a free one */ ++ for (reason = PROCSIG_CUSTOM_1; reason <= PROCSIG_CUSTOM_N; reason++) ++ if (!CustomInterruptHandlers[reason - PROCSIG_CUSTOM_1]) ++ { ++ CustomInterruptHandlers[reason - PROCSIG_CUSTOM_1] = handler; ++ return reason; ++ } ++ ++ return INVALID_PROCSIGNAL; ++} ++ + /* + * SendProcSignal + * Send a signal to a Postgres process +@@ -585,9 +624,64 @@ procsignal_sigusr1_handler(SIGNAL_ARGS) + if (CheckProcSignal(PROCSIG_RECOVERY_CONFLICT_BUFFERPIN)) + RecoveryConflictInterrupt(PROCSIG_RECOVERY_CONFLICT_BUFFERPIN); + ++ CheckAndSetCustomSignalInterrupts(); ++ + SetLatch(MyLatch); + + latch_sigusr1_handler(); + + errno = save_errno; + } ++ ++/* ++ * Handle receipt of an interrupt indicating any of custom process signals. ++ */ ++static void ++CheckAndSetCustomSignalInterrupts() ++{ ++ ProcSignalReason reason; ++ ++ for (reason = PROCSIG_CUSTOM_1; reason <= PROCSIG_CUSTOM_N; reason++) ++ { ++ if (CheckProcSignal(reason)) ++ { ++ ++ /* set interrupt flags */ ++ InterruptPending = true; ++ CustomSignalPendings[reason - PROCSIG_CUSTOM_1] = true; ++ } ++ } ++ ++ SetLatch(MyLatch); ++} ++ ++/* ++ * CheckAndHandleCustomSignals ++ * Check custom signal flags and call handler assigned to that signal ++ * if it is not NULL ++ * ++ * This function is called within CHECK_FOR_INTERRUPTS if interrupt occurred. ++ */ ++void ++CheckAndHandleCustomSignals(void) ++{ ++ int i; ++ ++ /* Check on expiring of custom signals and call its handlers if exist */ ++ for (i = 0; i < NUM_CUSTOM_PROCSIGNALS; i++) ++ { ++ if (!CustomSignalProcessing[i] && CustomSignalPendings[i]) ++ { ++ ProcSignalHandler_type handler; ++ ++ CustomSignalPendings[i] = false; ++ handler = CustomInterruptHandlers[i]; ++ if (handler != NULL) ++ { ++ CustomSignalProcessing[i] = true; ++ handler(); ++ CustomSignalProcessing[i] = false; ++ } ++ } ++ } ++} +diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c +index 174c72a14bc..0e7366bd58f 100644 +--- a/src/backend/tcop/postgres.c ++++ b/src/backend/tcop/postgres.c +@@ -3221,6 +3221,8 @@ ProcessInterrupts(void) + + if (ParallelMessagePending) + HandleParallelMessages(); ++ ++ CheckAndHandleCustomSignals(); + } + + +diff --git a/src/include/storage/procsignal.h b/src/include/storage/procsignal.h +index 5cb39697f38..c05f60fa719 100644 +--- a/src/include/storage/procsignal.h ++++ b/src/include/storage/procsignal.h +@@ -17,6 +17,8 @@ + #include "storage/backendid.h" + + ++#define NUM_CUSTOM_PROCSIGNALS 64 ++ + /* + * Reasons for signaling a Postgres child process (a backend or an auxiliary + * process, like checkpointer). We can cope with concurrent signals for different +@@ -29,6 +31,8 @@ + */ + typedef enum + { ++ INVALID_PROCSIGNAL = -1, /* Must be first */ ++ + PROCSIG_CATCHUP_INTERRUPT, /* sinval catchup interrupt */ + PROCSIG_NOTIFY_INTERRUPT, /* listen/notify interrupt */ + PROCSIG_PARALLEL_MESSAGE, /* message from cooperating parallel backend */ +@@ -43,6 +47,14 @@ typedef enum + PROCSIG_RECOVERY_CONFLICT_BUFFERPIN, + PROCSIG_RECOVERY_CONFLICT_STARTUP_DEADLOCK, + ++ PROCSIG_CUSTOM_1, ++ /* ++ * PROCSIG_CUSTOM_2, ++ * ..., ++ * PROCSIG_CUSTOM_N-1, ++ */ ++ PROCSIG_CUSTOM_N = PROCSIG_CUSTOM_1 + NUM_CUSTOM_PROCSIGNALS - 1, ++ + NUM_PROCSIGNALS /* Must be last! */ + } ProcSignalReason; + +@@ -55,6 +67,8 @@ typedef enum + */ + PROCSIGNAL_BARRIER_PLACEHOLDER = 0 + } ProcSignalBarrierType; ++/* Handler of custom process signal */ ++typedef void (*ProcSignalHandler_type) (void); + + /* + * prototypes for functions in procsignal.c +@@ -63,12 +77,15 @@ extern Size ProcSignalShmemSize(void); + extern void ProcSignalShmemInit(void); + + extern void ProcSignalInit(int pss_idx); ++extern ProcSignalReason ++ RegisterCustomProcSignalHandler(ProcSignalHandler_type handler); + extern int SendProcSignal(pid_t pid, ProcSignalReason reason, + BackendId backendId); + + extern uint64 EmitProcSignalBarrier(ProcSignalBarrierType type); + extern void WaitForProcSignalBarrier(uint64 generation); + extern void ProcessProcSignalBarrier(void); ++extern void CheckAndHandleCustomSignals(void); + + extern void procsignal_sigusr1_handler(SIGNAL_ARGS); + +-- +2.25.1 + diff --git a/patches/runtime_explain_13.0.patch b/patches/runtime_explain_13.0.patch new file mode 100644 index 0000000..973ebd5 --- /dev/null +++ b/patches/runtime_explain_13.0.patch @@ -0,0 +1,261 @@ +diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c +index 20708db9f12..866948bd0c1 100644 +--- a/src/backend/commands/explain.c ++++ b/src/backend/commands/explain.c +@@ -955,14 +955,36 @@ report_triggers(ResultRelInfo *rInfo, bool show_relname, ExplainState *es) + char *relname; + char *conname = NULL; + ++ instr_time starttimespan; ++ double total; ++ double ntuples; ++ double ncalls; ++ ++ if (!es->runtime) ++ { + /* Must clean up instrumentation state */ + InstrEndLoop(instr); ++ } ++ ++ /* Collect statistic variables */ ++ if (!INSTR_TIME_IS_ZERO(instr->starttime)) ++ { ++ INSTR_TIME_SET_CURRENT(starttimespan); ++ INSTR_TIME_SUBTRACT(starttimespan, instr->starttime); ++ } ++ else ++ INSTR_TIME_SET_ZERO(starttimespan); ++ ++ total = instr->total + INSTR_TIME_GET_DOUBLE(instr->counter) ++ + INSTR_TIME_GET_DOUBLE(starttimespan); ++ ntuples = instr->ntuples + instr->tuplecount; ++ ncalls = ntuples + !INSTR_TIME_IS_ZERO(starttimespan); + + /* + * We ignore triggers that were never invoked; they likely aren't + * relevant to the current query type. + */ +- if (instr->ntuples == 0) ++ if (ncalls == 0) + continue; + + ExplainOpenGroup("Trigger", NULL, true, es); +@@ -988,9 +1010,9 @@ report_triggers(ResultRelInfo *rInfo, bool show_relname, ExplainState *es) + appendStringInfo(es->str, " on %s", relname); + if (es->timing) + appendStringInfo(es->str, ": time=%.3f calls=%.0f\n", +- 1000.0 * instr->total, instr->ntuples); ++ 1000.0 * total, ncalls); + else +- appendStringInfo(es->str, ": calls=%.0f\n", instr->ntuples); ++ appendStringInfo(es->str, ": calls=%.0f\n", ncalls); + } + else + { +@@ -999,9 +1021,8 @@ report_triggers(ResultRelInfo *rInfo, bool show_relname, ExplainState *es) + ExplainPropertyText("Constraint Name", conname, es); + ExplainPropertyText("Relation", relname, es); + if (es->timing) +- ExplainPropertyFloat("Time", "ms", 1000.0 * instr->total, 3, +- es); +- ExplainPropertyFloat("Calls", NULL, instr->ntuples, 0, es); ++ ExplainPropertyFloat("Time", "ms", 1000.0 * total, 3, es); ++ ExplainPropertyFloat("Calls", NULL, ncalls, 0, es); + } + + if (conname) +@@ -1560,8 +1581,11 @@ ExplainNode(PlanState *planstate, List *ancestors, + * instrumentation results the user didn't ask for. But we do the + * InstrEndLoop call anyway, if possible, to reduce the number of cases + * auto_explain has to contend with. ++ * ++ * If flag es->stateinfo is set, i.e. when printing the current execution ++ * state, this step of cleaning up is missed. + */ +- if (planstate->instrument) ++ if (planstate->instrument && !es->runtime) + InstrEndLoop(planstate->instrument); + + if (es->analyze && +@@ -1596,7 +1620,7 @@ ExplainNode(PlanState *planstate, List *ancestors, + ExplainPropertyFloat("Actual Loops", NULL, nloops, 0, es); + } + } +- else if (es->analyze) ++ else if (es->analyze && !es->runtime) + { + if (es->format == EXPLAIN_FORMAT_TEXT) + appendStringInfoString(es->str, " (never executed)"); +@@ -1612,6 +1636,75 @@ ExplainNode(PlanState *planstate, List *ancestors, + } + } + ++ /* ++ * Print the progress of node execution at current loop. ++ */ ++ if (planstate->instrument && es->analyze && es->runtime) ++ { ++ instr_time starttimespan; ++ double startup_sec; ++ double total_sec; ++ double rows; ++ double loop_num; ++ bool finished; ++ ++ if (!INSTR_TIME_IS_ZERO(planstate->instrument->starttime)) ++ { ++ INSTR_TIME_SET_CURRENT(starttimespan); ++ INSTR_TIME_SUBTRACT(starttimespan, planstate->instrument->starttime); ++ } ++ else ++ INSTR_TIME_SET_ZERO(starttimespan); ++ startup_sec = 1000.0 * planstate->instrument->firsttuple; ++ total_sec = 1000.0 * (INSTR_TIME_GET_DOUBLE(planstate->instrument->counter) ++ + INSTR_TIME_GET_DOUBLE(starttimespan)); ++ rows = planstate->instrument->tuplecount; ++ loop_num = planstate->instrument->nloops + 1; ++ ++ finished = planstate->instrument->nloops > 0 ++ && !planstate->instrument->running ++ && INSTR_TIME_IS_ZERO(starttimespan); ++ ++ if (!finished) ++ { ++ ExplainOpenGroup("Current loop", "Current loop", true, es); ++ if (es->format == EXPLAIN_FORMAT_TEXT) ++ { ++ if (es->timing) ++ { ++ if (planstate->instrument->running) ++ appendStringInfo(es->str, ++ " (Current loop: actual time=%.3f..%.3f rows=%.0f, loop number=%.0f)", ++ startup_sec, total_sec, rows, loop_num); ++ else ++ appendStringInfo(es->str, ++ " (Current loop: running time=%.3f actual rows=0, loop number=%.0f)", ++ total_sec, loop_num); ++ } ++ else ++ appendStringInfo(es->str, ++ " (Current loop: actual rows=%.0f, loop number=%.0f)", ++ rows, loop_num); ++ } ++ else ++ { ++ ExplainPropertyFloat("Actual Loop Number", NULL, loop_num, 0, es); ++ if (es->timing) ++ { ++ if (planstate->instrument->running) ++ { ++ ExplainPropertyFloat("Actual Startup Time", NULL, startup_sec, 3, es); ++ ExplainPropertyFloat("Actual Total Time", NULL, total_sec, 3, es); ++ } ++ else ++ ExplainPropertyFloat("Running Time", NULL, total_sec, 3, es); ++ } ++ ExplainPropertyFloat("Actual Rows", NULL, rows, 0, es); ++ } ++ ExplainCloseGroup("Current loop", "Current loop", true, es); ++ } ++ } ++ + /* in text format, first line ends here */ + if (es->format == EXPLAIN_FORMAT_TEXT) + appendStringInfoChar(es->str, '\n'); +@@ -1990,6 +2083,9 @@ ExplainNode(PlanState *planstate, List *ancestors, + + /* Prepare per-worker buffer/WAL usage */ + if (es->workers_state && (es->buffers || es->wal) && es->verbose) ++ /* Show worker detail after query execution */ ++ if (es->analyze && es->verbose && planstate->worker_instrument ++ && !es->runtime) + { + WorkerInstrumentation *w = planstate->worker_instrument; + +@@ -2960,6 +3056,11 @@ show_hash_info(HashState *hashstate, ExplainState *es) + memcpy(&hinstrument, hashstate->hinstrument, + sizeof(HashInstrumentation)); + ++ if (hashstate->hashtable) ++ { ++ ExecHashAccumInstrumentation(&hinstrument, hashstate->hashtable); ++ } ++ + /* + * Merge results from workers. In the parallel-oblivious case, the + * results from all participants should be identical, except where +@@ -3196,20 +3297,16 @@ show_instrumentation_count(const char *qlabel, int which, + if (!es->analyze || !planstate->instrument) + return; + ++ nloops = planstate->instrument->nloops; + if (which == 2) +- nfiltered = planstate->instrument->nfiltered2; ++ nfiltered = ((nloops > 0) ? planstate->instrument->nfiltered2 / nloops : 0); + else +- nfiltered = planstate->instrument->nfiltered1; ++ nfiltered = ((nloops > 0) ? planstate->instrument->nfiltered1 / nloops : 0); + nloops = planstate->instrument->nloops; + + /* In text mode, suppress zero counts; they're not interesting enough */ + if (nfiltered > 0 || es->format != EXPLAIN_FORMAT_TEXT) +- { +- if (nloops > 0) +- ExplainPropertyFloat(qlabel, NULL, nfiltered / nloops, 0, es); +- else +- ExplainPropertyFloat(qlabel, NULL, 0.0, 0, es); +- } ++ ExplainPropertyFloat(qlabel, NULL, nfiltered, 0, es); + } + + /* +@@ -3781,15 +3878,27 @@ show_modifytable_info(ModifyTableState *mtstate, List *ancestors, + double insert_path; + double other_path; + +- InstrEndLoop(mtstate->mt_plans[0]->instrument); ++ if (!es->runtime) ++ InstrEndLoop(mtstate->mt_plans[0]->instrument); + + /* count the number of source rows */ +- total = mtstate->mt_plans[0]->instrument->ntuples; +- other_path = mtstate->ps.instrument->ntuples2; +- insert_path = total - other_path; ++ other_path = mtstate->ps.instrument->nfiltered2; ++ ++ /* ++ * Insert occurs after extracting row from subplan and in runtime mode ++ * we can appear between these two operations - situation when ++ * total > insert_path + other_path. Therefore we don't know exactly ++ * whether last row from subplan is inserted. ++ * We don't print inserted tuples in runtime mode in order to not print ++ * inconsistent data ++ */ ++ if (!es->runtime) ++ { ++ total = mtstate->mt_plans[0]->instrument->ntuples; ++ insert_path = total - other_path; ++ ExplainPropertyFloat("Tuples Inserted", NULL, insert_path, 0, es); ++ } + +- ExplainPropertyFloat("Tuples Inserted", NULL, +- insert_path, 0, es); + ExplainPropertyFloat("Conflicting Tuples", NULL, + other_path, 0, es); + } +diff --git a/src/include/commands/explain.h b/src/include/commands/explain.h +index ba661d32a63..ee49febfce5 100644 +--- a/src/include/commands/explain.h ++++ b/src/include/commands/explain.h +@@ -47,6 +47,8 @@ typedef struct ExplainState + bool summary; /* print total planning and execution timing */ + bool settings; /* print modified settings */ + ExplainFormat format; /* output format */ ++ bool runtime; /* print intermediate state of query execution, ++ not after completion */ + /* state for output formatting --- not reset for each new plan tree */ + int indent; /* current indentation level */ + List *grouping_stack; /* format-specific grouping state */ +-- +2.25.1 + diff --git a/pg_query_state.c b/pg_query_state.c index abf8ffa..246cabc 100644 --- a/pg_query_state.c +++ b/pg_query_state.c @@ -677,11 +677,19 @@ pg_query_state(PG_FUNCTION_ARGS) tuple = heap_form_tuple(funcctx->tuple_desc, values, nulls); /* increment cursor */ +#if PG_VERSION_NUM >= 130000 + p_state->frame_cursor = lnext(p_state->stack, p_state->frame_cursor); +#else p_state->frame_cursor = lnext(p_state->frame_cursor); +#endif p_state->frame_index++; if (p_state->frame_cursor == NULL) +#if PG_VERSION_NUM >= 130000 + fctx->proc_cursor = lnext(fctx->procs, fctx->proc_cursor); +#else fctx->proc_cursor = lnext(fctx->proc_cursor); +#endif SRF_RETURN_NEXT(funcctx, HeapTupleGetDatum(tuple)); } diff --git a/run_tests.sh b/run_tests.sh old mode 100755 new mode 100644 index 651e9a1..cc61b33 --- a/run_tests.sh +++ b/run_tests.sh @@ -99,12 +99,16 @@ if [ "$LEVEL" = "scan-build" ] || \ fi +# XXX: Hackish way to make possible to run all contrib tests +mkdir $CUSTOM_PG_SRC/contrib/pg_query_state +cp -r * $CUSTOM_PG_SRC/contrib/pg_query_state/ + # don't forget to "make clean" -make USE_PGXS=1 clean +make -C $CUSTOM_PG_SRC/contrib/pg_query_state clean # build and install extension (using PG_CPPFLAGS and SHLIB_LINK for gcov) -make USE_PGXS=1 PG_CPPFLAGS="-coverage" SHLIB_LINK="-coverage" -make USE_PGXS=1 install +make -C $CUSTOM_PG_SRC/contrib/pg_query_state PG_CPPFLAGS="-coverage" SHLIB_LINK="-coverage" +make -C $CUSTOM_PG_SRC/contrib/pg_query_state install # initialize database initdb -D $PGDATA @@ -136,7 +140,7 @@ if [ $status -ne 0 ]; then cat /tmp/postgres.log; exit 1; fi # run regression tests export PG_REGRESS_DIFF_OPTS="-w -U3" # for alpine's diff (BusyBox) -make USE_PGXS=1 installcheck || status=$? +make -C $CUSTOM_PG_SRC/contrib/pg_query_state installcheck || status=$? # show diff if it exists if [ -f regression.diffs ]; then cat regression.diffs; fi @@ -169,7 +173,7 @@ fi if [ $status -ne 0 ]; then exit 1; fi set +e # generate *.gcov files -gcov *.c *.h +gcov $CUSTOM_PG_SRC/contrib/pg_query_state/*.c $CUSTOM_PG_SRC/contrib/pg_query_state/*.h set +ux From 02017e3927df5d7767605371b4c6a90a9ba19aa4 Mon Sep 17 00:00:00 2001 From: Alexey Kondratov Date: Wed, 24 Feb 2021 19:30:52 +0300 Subject: [PATCH 35/91] Print some debug output before assertion in test_nested_call() --- tests/test_cases.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_cases.py b/tests/test_cases.py index 8d7fc28..440f32f 100644 --- a/tests/test_cases.py +++ b/tests/test_cases.py @@ -114,6 +114,11 @@ def test_nested_call(config): util_conn.commit() qs, notices = common.onetime_query_state(config, acon, call_function) + + # Print some debug output before assertion + if len(qs) < 2: + print(qs) + assert len(qs) == 2 \ and qs[0][0] == qs[1][0] == acon.get_backend_pid() \ and qs[0][1] == 0 and qs[1][1] == 1 \ From 18b365364b99a5e4860da5b8a2b2f73b65acf002 Mon Sep 17 00:00:00 2001 From: Alexey Kondratov Date: Thu, 15 Apr 2021 14:18:02 +0300 Subject: [PATCH 36/91] Update Tests section in README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index e151bbb..f645a7d 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,8 @@ export LEVEL=hardcore export USE_TPCDS=1 export PG_VERSION=12 +./mk_dockerfile.sh + docker-compose build docker-compose run tests ``` From b668154806d7ca6cd8d1586b226141b6573de049 Mon Sep 17 00:00:00 2001 From: Daria Lepikhova Date: Wed, 21 Apr 2021 14:28:48 +0500 Subject: [PATCH 37/91] Fixed warning in pg_query_state.c --- pg_query_state.c | 1 + 1 file changed, 1 insertion(+) diff --git a/pg_query_state.c b/pg_query_state.c index 246cabc..2aaa948 100644 --- a/pg_query_state.c +++ b/pg_query_state.c @@ -393,6 +393,7 @@ LockShmem(LOCKTAG *tag, uint32 key) tag->locktag_lockmethodid = USER_LOCKMETHOD; result = LockAcquire(tag, ExclusiveLock, false, false); Assert(result == LOCKACQUIRE_OK); + elog(DEBUG1, "LockAcquireResult is not OK %d", result); } From 93eda9871d3852f1b176a9d9fbd4d295501bfddb Mon Sep 17 00:00:00 2001 From: Daria Lepikhova Date: Thu, 29 Apr 2021 14:00:24 +0500 Subject: [PATCH 38/91] Fix paths for running docker tests --- run_tests.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/run_tests.sh b/run_tests.sh index cc61b33..7bc6d99 100644 --- a/run_tests.sh +++ b/run_tests.sh @@ -138,6 +138,8 @@ fi # something's wrong, exit now! if [ $status -ne 0 ]; then cat /tmp/postgres.log; exit 1; fi +cd $CUSTOM_PG_SRC + # run regression tests export PG_REGRESS_DIFF_OPTS="-w -U3" # for alpine's diff (BusyBox) make -C $CUSTOM_PG_SRC/contrib/pg_query_state installcheck || status=$? @@ -145,10 +147,12 @@ make -C $CUSTOM_PG_SRC/contrib/pg_query_state installcheck || status=$? # show diff if it exists if [ -f regression.diffs ]; then cat regression.diffs; fi +cd $CUSTOM_PG_SRC/contrib/pg_query_state + # run python tests set +x -e python3 -m venv /tmp/env && source /tmp/env/bin/activate && -pip install -r tests/requirements.txt +pip install -r ./tests/requirements.txt set -e #exit virtualenv with error code python tests/pg_qs_test_runner.py --port $PGPORT if [[ "$USE_TPCDS" == "1" ]]; then From 5a2486d822b34e4a36159b5132f2a12d29f072f1 Mon Sep 17 00:00:00 2001 From: Daria Lepikhova Date: Fri, 7 May 2021 13:25:18 +0500 Subject: [PATCH 39/91] Another version of installcheck to run docker tests correctly --- Makefile | 18 +++++++++++++----- run_tests.sh | 7 ++----- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/Makefile b/Makefile index 4b2ada5..288ee34 100644 --- a/Makefile +++ b/Makefile @@ -22,6 +22,8 @@ include $(top_builddir)/src/Makefile.global include $(top_srcdir)/contrib/contrib-global.mk endif +EXTRA_REGRESS_OPTS=--temp-config=$(top_srcdir)/$(subdir)/test.conf + $(EXTENSION)--$(EXTVERSION).sql: init.sql cat $^ > $@ @@ -29,11 +31,7 @@ ISOLATIONCHECKS=corner_cases check: isolationcheck -installcheck: - $(pg_isolation_regress_installcheck) $(ISOLATIONCHECKS) - -submake-isolation: - $(MAKE) -C $(top_builddir)/src/test/isolation all +installcheck: isolationcheck-install-force isolationcheck: | submake-isolation temp-install $(MKDIR_P) isolation_output @@ -42,4 +40,14 @@ isolationcheck: | submake-isolation temp-install --outputdir=isolation_output \ $(ISOLATIONCHECKS) +isolationcheck-install-force: all | submake-isolation submake-pg_query_state temp-install + $(pg_isolation_regress_installcheck) \ + $(ISOLATIONCHECKS) + +submake-isolation: + $(MAKE) -C $(top_builddir)/src/test/isolation all + +submake-pg_query_state: + $(MAKE) -C $(top_builddir)/contrib/pg_query_state + temp-install: EXTRA_INSTALL=contrib/pg_query_state diff --git a/run_tests.sh b/run_tests.sh index 7bc6d99..fbf2da1 100644 --- a/run_tests.sh +++ b/run_tests.sh @@ -138,17 +138,14 @@ fi # something's wrong, exit now! if [ $status -ne 0 ]; then cat /tmp/postgres.log; exit 1; fi -cd $CUSTOM_PG_SRC - # run regression tests export PG_REGRESS_DIFF_OPTS="-w -U3" # for alpine's diff (BusyBox) -make -C $CUSTOM_PG_SRC/contrib/pg_query_state installcheck || status=$? +cd $CUSTOM_PG_SRC/contrib/pg_query_state +make installcheck || status=$? # show diff if it exists if [ -f regression.diffs ]; then cat regression.diffs; fi -cd $CUSTOM_PG_SRC/contrib/pg_query_state - # run python tests set +x -e python3 -m venv /tmp/env && source /tmp/env/bin/activate && From b492d62045866fff5e0bd1c317f84929c8453d33 Mon Sep 17 00:00:00 2001 From: Alexey Kondratov Date: Tue, 11 May 2021 20:18:34 +0300 Subject: [PATCH 40/91] Fix TPC-DS tests, i.e. test data and schema generation --- tests/prepare_stress.sh | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/tests/prepare_stress.sh b/tests/prepare_stress.sh index da5ae48..3bdb2a5 100755 --- a/tests/prepare_stress.sh +++ b/tests/prepare_stress.sh @@ -4,23 +4,26 @@ mkdir -p tmp_stress cd tmp_stress rm -rf ./* -git clone --depth 1 --single-branch --branch master https://github.com/gregrahn/tpcds-kit.git -git clone --depth 1 --single-branch --branch master https://github.com/cwida/tpcds-result-reproduction.git +git clone --depth 1 --single-branch --branch master https://github.com/gregrahn/tpcds-kit.git # used for data and schema +git clone --depth 1 --single-branch --branch master https://github.com/cwida/tpcds-result-reproduction.git # used for queries only cd tpcds-kit/tools -make -s -#Generate data +# This is a palliative care, since tpcds-kit is old and doesn't compile with modern ld. +# Anyway, now it works and this is better than nothing. +make LDFLAGS=-zmuldefs -s + +# Generate data ./dsdgen -FORCE -VERBOSE -SCALE 1 -#Prepare data +# Prepare data mkdir -p tables for i in `ls *.dat`; do echo "Preparing file" $i sed 's/|$//' $i > tables/$i done -#Generate queries +# Generate queries ./dsqgen -DIRECTORY ../query_templates \ -INPUT ../query_templates/templates.lst \ -VERBOSE Y \ From 0b3be2b1b5f43dbb7244aced14602a27b018dc68 Mon Sep 17 00:00:00 2001 From: Anna Akenteva Date: Mon, 17 May 2021 14:10:24 +0500 Subject: [PATCH 41/91] PGPRO-4197: attempt to send stuff byte-wise in pg_query_state --- pg_query_state.c | 51 +++++++++++++++++++++++++++++++++++++++++++++++- pg_query_state.h | 1 + signal_handler.c | 37 ++++++++++++++++++++++++++++++++--- 3 files changed, 85 insertions(+), 4 deletions(-) diff --git a/pg_query_state.c b/pg_query_state.c index 2aaa948..3ec04e4 100644 --- a/pg_query_state.c +++ b/pg_query_state.c @@ -960,6 +960,55 @@ copy_msg(shm_mq_msg *msg) return result; } +// ----------------- DEBUG ----------------- +static void +print_recv_bytes(int num, char *src, int offset) +{ + elog(INFO, "======= RECV MSG SEGMENT START (%d bytes) =======", num); + for (int i = offset; i < offset + num; i++) + elog(INFO, "RECV byte #%d = %02x", i, (unsigned char) *(src + i)); +} +// ----------------- DEBUG ----------------- + +static shm_mq_result +shm_mq_receive_by_bytes(shm_mq_handle *mqh, Size *total, void **datap) +{ + shm_mq_result mq_receive_result; + shm_mq_msg *buff; + int ii; + int offset; + int *expected; + Size len; + + /* Get the expected number of bytes in message */ + mq_receive_result = shm_mq_receive(mqh, &len, (void **) &expected, false); + if (mq_receive_result != SHM_MQ_SUCCESS) + return mq_receive_result; + Assert(len == sizeof(int)); +// elog(INFO, "======= RECV MSG (expecting %d bytes) =======", *expected); + + *datap = palloc0(*expected); + + /* Get the message itself */ + for (offset = 0, ii = 0; offset < *expected; ii++) + { + // Keep receiving new messages until we assemble the full message + mq_receive_result = shm_mq_receive(mqh, &len, ((void **) &buff), false); + memcpy((char *) *datap + offset, buff, len); +// print_recv_bytes(len, (char *) *datap, offset); + offset += len; + if (mq_receive_result != SHM_MQ_SUCCESS) + return mq_receive_result; + } + +// elog(INFO, "RECV: END cycle - %d", ii); + *total = offset; +// mq_receive_result = shm_mq_receive(mqh, &len, (void **) &msg, false); +// *datap = buff; + + return mq_receive_result; +} + static List * GetRemoteBackendQueryStates(PGPROC *leader, List *pworkers, @@ -1032,7 +1081,7 @@ GetRemoteBackendQueryStates(PGPROC *leader, /* extract query state from leader process */ mqh = shm_mq_attach(mq, NULL, NULL); elog(DEBUG1, "Wait response from leader %d", leader->pid); - mq_receive_result = shm_mq_receive(mqh, &len, (void **) &msg, false); + mq_receive_result = shm_mq_receive_by_bytes(mqh, &len, ((void **) &msg)); if (mq_receive_result != SHM_MQ_SUCCESS) goto mq_error; if (msg->reqid != reqid) diff --git a/pg_query_state.h b/pg_query_state.h index 0cb8923..bd647e6 100644 --- a/pg_query_state.h +++ b/pg_query_state.h @@ -18,6 +18,7 @@ #include "storage/shm_mq.h" #define QUEUE_SIZE (16 * 1024) +#define BUF_SIZE 7 #define TIMINIG_OFF_WARNING 1 #define BUFFERS_OFF_WARNING 2 diff --git a/signal_handler.c b/signal_handler.c index c9de5d7..1d14e6e 100644 --- a/signal_handler.c +++ b/signal_handler.c @@ -27,6 +27,8 @@ typedef struct char *plan; } stack_frame; +static void send_msg_by_bits(shm_mq_handle *mqh, Size nbytes, const void *data); + /* * Get List of stack_frames as a stack of function calls starting from outermost call. * Each entry contains query text and query state in form of EXPLAIN ANALYZE output. @@ -148,6 +150,35 @@ serialize_stack(char *dest, List *qs_stack) serialize_stack_frame(&dest, qs_frame); } } +// ----------------- DEBUG ----------------- +static void +print_sent_bytes(int num, char *src, int offset) +{ + elog(INFO, "======= SEND MSG SEGMENT START (%d bytes) =======", num); + for (int i = offset; i < offset + num; i++) + elog(INFO, "SENT byte #%d = %02x", i, (unsigned char) *(src + i)); +} +// ----------------- DEBUG ----------------- + +static void +send_msg_by_bits(shm_mq_handle *mqh, Size nbytes, const void *data) +{ + int bytes_left; + int bytes_send; + + /* Send the expected message length */ + shm_mq_send(mqh, sizeof(int), &nbytes, false); + +// elog(INFO, "======= SEND MSG (%lu bytes) =======", nbytes); + for (int offset = 0; offset < nbytes; offset += bytes_send) + { + bytes_left = nbytes - offset; + bytes_send = (bytes_left < BUF_SIZE) ? bytes_left : BUF_SIZE; + shm_mq_send(mqh, bytes_send, &(((unsigned char*)data)[offset]), false); + // DEBUG: print message that we just sent +// print_sent_bytes(bytes_send, (char *) data, offset); + } +} /* * Send state of current query to shared queue. @@ -207,7 +238,7 @@ SendQueryState(void) { shm_mq_msg msg = { reqid, BASE_SIZEOF_SHM_MQ_MSG, MyProc, STAT_DISABLED }; - shm_mq_send(mqh, msg.length, &msg, false); + send_msg_by_bits(mqh, msg.length, &msg); } /* check if backend doesn't execute any query */ @@ -215,7 +246,7 @@ SendQueryState(void) { shm_mq_msg msg = { reqid, BASE_SIZEOF_SHM_MQ_MSG, MyProc, QUERY_NOT_RUNNING }; - shm_mq_send(mqh, msg.length, &msg, false); + send_msg_by_bits(mqh, msg.length, &msg); } /* happy path */ @@ -238,7 +269,7 @@ SendQueryState(void) msg->stack_depth = list_length(qs_stack); serialize_stack(msg->stack, qs_stack); - shm_mq_send(mqh, msglen, msg, false); + send_msg_by_bits(mqh, msglen, msg); } elog(DEBUG1, "Worker %d sends response for pg_query_state to %d", shm_mq_get_sender(mq)->pid, shm_mq_get_receiver(mq)->pid); DetachPeer(); From bc32577a29c878f14a7a4613fad38882ec91ee8e Mon Sep 17 00:00:00 2001 From: Anna Akenteva Date: Mon, 17 May 2021 14:21:12 +0500 Subject: [PATCH 42/91] PGPRO-4197: code cleanup + send stuff by 1024 bytes now --- pg_query_state.c | 35 ++++++++++++----------------------- pg_query_state.h | 2 +- signal_handler.c | 24 ++++++------------------ 3 files changed, 19 insertions(+), 42 deletions(-) diff --git a/pg_query_state.c b/pg_query_state.c index 3ec04e4..46556d1 100644 --- a/pg_query_state.c +++ b/pg_query_state.c @@ -60,6 +60,9 @@ static void qs_ExecutorRun(QueryDesc *queryDesc, ScanDirection direction, #endif static void qs_ExecutorFinish(QueryDesc *queryDesc); +static shm_mq_result receive_msg_by_parts(shm_mq_handle *mqh, Size *total, + void **datap, bool nowait); + /* Global variables */ List *QueryDescStack = NIL; static ProcSignalReason UserIdPollReason = INVALID_PROCSIGNAL; @@ -777,7 +780,7 @@ shm_mq_receive_with_timeout(shm_mq_handle *mqh, { shm_mq_result mq_receive_result; - mq_receive_result = shm_mq_receive(mqh, nbytesp, datap, true); + mq_receive_result = receive_msg_by_parts(mqh, nbytesp, datap, true); if (mq_receive_result != SHM_MQ_WOULD_BLOCK) return mq_receive_result; if (rc & WL_TIMEOUT || delay <= 0) @@ -960,51 +963,36 @@ copy_msg(shm_mq_msg *msg) return result; } -// ----------------- DEBUG ----------------- -static void -print_recv_bytes(int num, char *src, int offset) -{ - elog(INFO, "======= RECV MSG SEGMENT START (%d bytes) =======", num); - for (int i = offset; i < offset + num; i++) - elog(INFO, "RECV byte #%d = %02x", i, (unsigned char) *(src + i)); -} -// ----------------- DEBUG ----------------- - static shm_mq_result -shm_mq_receive_by_bytes(shm_mq_handle *mqh, Size *total, void **datap) +receive_msg_by_parts(shm_mq_handle *mqh, Size *total, void **datap, + bool nowait) { shm_mq_result mq_receive_result; shm_mq_msg *buff; - int ii; int offset; int *expected; Size len; /* Get the expected number of bytes in message */ - mq_receive_result = shm_mq_receive(mqh, &len, (void **) &expected, false); + mq_receive_result = shm_mq_receive(mqh, &len, (void **) &expected, nowait); if (mq_receive_result != SHM_MQ_SUCCESS) return mq_receive_result; Assert(len == sizeof(int)); -// elog(INFO, "======= RECV MSG (expecting %d bytes) =======", *expected); *datap = palloc0(*expected); /* Get the message itself */ - for (offset = 0, ii = 0; offset < *expected; ii++) + for (offset = 0; offset < *expected; ) { - // Keep receiving new messages until we assemble the full message - mq_receive_result = shm_mq_receive(mqh, &len, ((void **) &buff), false); + /* Keep receiving new messages until we assemble the full message */ + mq_receive_result = shm_mq_receive(mqh, &len, ((void **) &buff), nowait); memcpy((char *) *datap + offset, buff, len); -// print_recv_bytes(len, (char *) *datap, offset); offset += len; if (mq_receive_result != SHM_MQ_SUCCESS) return mq_receive_result; } -// elog(INFO, "RECV: END cycle - %d", ii); *total = offset; -// mq_receive_result = shm_mq_receive(mqh, &len, (void **) &msg, false); -// *datap = buff; return mq_receive_result; } @@ -1081,7 +1069,8 @@ GetRemoteBackendQueryStates(PGPROC *leader, /* extract query state from leader process */ mqh = shm_mq_attach(mq, NULL, NULL); elog(DEBUG1, "Wait response from leader %d", leader->pid); - mq_receive_result = shm_mq_receive_by_bytes(mqh, &len, ((void **) &msg)); + mq_receive_result = receive_msg_by_parts(mqh, &len, (void **) &msg, + false); if (mq_receive_result != SHM_MQ_SUCCESS) goto mq_error; if (msg->reqid != reqid) diff --git a/pg_query_state.h b/pg_query_state.h index bd647e6..8192ae2 100644 --- a/pg_query_state.h +++ b/pg_query_state.h @@ -18,7 +18,7 @@ #include "storage/shm_mq.h" #define QUEUE_SIZE (16 * 1024) -#define BUF_SIZE 7 +#define MSG_MAX_SIZE 1024 #define TIMINIG_OFF_WARNING 1 #define BUFFERS_OFF_WARNING 2 diff --git a/signal_handler.c b/signal_handler.c index 1d14e6e..d8120b8 100644 --- a/signal_handler.c +++ b/signal_handler.c @@ -27,7 +27,7 @@ typedef struct char *plan; } stack_frame; -static void send_msg_by_bits(shm_mq_handle *mqh, Size nbytes, const void *data); +static void send_msg_by_parts(shm_mq_handle *mqh, Size nbytes, const void *data); /* * Get List of stack_frames as a stack of function calls starting from outermost call. @@ -150,18 +150,9 @@ serialize_stack(char *dest, List *qs_stack) serialize_stack_frame(&dest, qs_frame); } } -// ----------------- DEBUG ----------------- -static void -print_sent_bytes(int num, char *src, int offset) -{ - elog(INFO, "======= SEND MSG SEGMENT START (%d bytes) =======", num); - for (int i = offset; i < offset + num; i++) - elog(INFO, "SENT byte #%d = %02x", i, (unsigned char) *(src + i)); -} -// ----------------- DEBUG ----------------- static void -send_msg_by_bits(shm_mq_handle *mqh, Size nbytes, const void *data) +send_msg_by_parts(shm_mq_handle *mqh, Size nbytes, const void *data) { int bytes_left; int bytes_send; @@ -169,14 +160,11 @@ send_msg_by_bits(shm_mq_handle *mqh, Size nbytes, const void *data) /* Send the expected message length */ shm_mq_send(mqh, sizeof(int), &nbytes, false); -// elog(INFO, "======= SEND MSG (%lu bytes) =======", nbytes); for (int offset = 0; offset < nbytes; offset += bytes_send) { bytes_left = nbytes - offset; - bytes_send = (bytes_left < BUF_SIZE) ? bytes_left : BUF_SIZE; + bytes_send = (bytes_left < MSG_MAX_SIZE) ? bytes_left : MSG_MAX_SIZE; shm_mq_send(mqh, bytes_send, &(((unsigned char*)data)[offset]), false); - // DEBUG: print message that we just sent -// print_sent_bytes(bytes_send, (char *) data, offset); } } @@ -238,7 +226,7 @@ SendQueryState(void) { shm_mq_msg msg = { reqid, BASE_SIZEOF_SHM_MQ_MSG, MyProc, STAT_DISABLED }; - send_msg_by_bits(mqh, msg.length, &msg); + send_msg_by_parts(mqh, msg.length, &msg); } /* check if backend doesn't execute any query */ @@ -246,7 +234,7 @@ SendQueryState(void) { shm_mq_msg msg = { reqid, BASE_SIZEOF_SHM_MQ_MSG, MyProc, QUERY_NOT_RUNNING }; - send_msg_by_bits(mqh, msg.length, &msg); + send_msg_by_parts(mqh, msg.length, &msg); } /* happy path */ @@ -269,7 +257,7 @@ SendQueryState(void) msg->stack_depth = list_length(qs_stack); serialize_stack(msg->stack, qs_stack); - send_msg_by_bits(mqh, msglen, msg); + send_msg_by_parts(mqh, msglen, msg); } elog(DEBUG1, "Worker %d sends response for pg_query_state to %d", shm_mq_get_sender(mq)->pid, shm_mq_get_receiver(mq)->pid); DetachPeer(); From 6e9b1b519d70da2305d32dba36474250a47dbf85 Mon Sep 17 00:00:00 2001 From: Daria Lepikhova Date: Thu, 20 May 2021 17:15:37 +0500 Subject: [PATCH 43/91] Add isolation_output to .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 587ab37..c1b6c2d 100644 --- a/.gitignore +++ b/.gitignore @@ -10,4 +10,5 @@ cscope.out tags Dockerfile tmp_stress +isolation_output From 284e3a53bd01336879e76efc48ef56fa655a3f10 Mon Sep 17 00:00:00 2001 From: Victor Wagner Date: Fri, 21 May 2021 16:53:20 +0300 Subject: [PATCH 44/91] Fix too modern syntax --- signal_handler.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/signal_handler.c b/signal_handler.c index d8120b8..74c2c6d 100644 --- a/signal_handler.c +++ b/signal_handler.c @@ -156,11 +156,12 @@ send_msg_by_parts(shm_mq_handle *mqh, Size nbytes, const void *data) { int bytes_left; int bytes_send; + int offset; /* Send the expected message length */ shm_mq_send(mqh, sizeof(int), &nbytes, false); - for (int offset = 0; offset < nbytes; offset += bytes_send) + for (offset = 0; offset < nbytes; offset += bytes_send) { bytes_left = nbytes - offset; bytes_send = (bytes_left < MSG_MAX_SIZE) ? bytes_left : MSG_MAX_SIZE; From c431f8f4fb322a9ab683c31731fdc33e681b583a Mon Sep 17 00:00:00 2001 From: Daria Lepikhova Date: Mon, 24 May 2021 12:28:34 +0500 Subject: [PATCH 45/91] Fixed installcheck in Makefile --- Makefile | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index 288ee34..dfed34c 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,4 @@ + # contrib/pg_query_state/Makefile MODULE_big = pg_query_state @@ -22,16 +23,16 @@ include $(top_builddir)/src/Makefile.global include $(top_srcdir)/contrib/contrib-global.mk endif -EXTRA_REGRESS_OPTS=--temp-config=$(top_srcdir)/$(subdir)/test.conf - $(EXTENSION)--$(EXTVERSION).sql: init.sql cat $^ > $@ -ISOLATIONCHECKS=corner_cases +ISOLATIONCHECKS = corner_cases check: isolationcheck -installcheck: isolationcheck-install-force +installcheck: submake-isolation + $(pg_isolation_regress_installcheck) \ + $(ISOLATIONCHECKS) isolationcheck: | submake-isolation temp-install $(MKDIR_P) isolation_output @@ -40,14 +41,7 @@ isolationcheck: | submake-isolation temp-install --outputdir=isolation_output \ $(ISOLATIONCHECKS) -isolationcheck-install-force: all | submake-isolation submake-pg_query_state temp-install - $(pg_isolation_regress_installcheck) \ - $(ISOLATIONCHECKS) - submake-isolation: $(MAKE) -C $(top_builddir)/src/test/isolation all -submake-pg_query_state: - $(MAKE) -C $(top_builddir)/contrib/pg_query_state - temp-install: EXTRA_INSTALL=contrib/pg_query_state From 3910194261749f456070df0c253a2e1dc94e4d62 Mon Sep 17 00:00:00 2001 From: Victor Wagner Date: Mon, 24 May 2021 12:56:49 +0300 Subject: [PATCH 46/91] Fix passing of text config to make sure buildfarm quld find it --- Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile b/Makefile index dfed34c..f94dca4 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ EXTVERSION = 1.1 DATA = pg_query_state--1.0--1.1.sql DATA_built = $(EXTENSION)--$(EXTVERSION).sql PGFILEDESC = "pg_query_state - facility to track progress of plan execution" - +EXTRA_REGRESS_OPTS=--temp-config=$(top_srcdir)/$(subdir)/test.conf EXTRA_CLEAN = ./isolation_output $(EXTENSION)--$(EXTVERSION).sql \ Dockerfile ./tests/*.pyc ./tmp_stress @@ -37,7 +37,6 @@ installcheck: submake-isolation isolationcheck: | submake-isolation temp-install $(MKDIR_P) isolation_output $(pg_isolation_regress_check) \ - --temp-config $(top_srcdir)/contrib/pg_query_state/test.conf \ --outputdir=isolation_output \ $(ISOLATIONCHECKS) From 1d0d7979bf5d7ddc77404e9f11ce26d98eca33f0 Mon Sep 17 00:00:00 2001 From: Marina Polyakova Date: Fri, 4 Jun 2021 17:46:01 +0300 Subject: [PATCH 47/91] PGPRO-4978: add outputdir=isolation_output to installcheck This way it will be processed by .gitignore as opposed to outputdir=output_iso which is the default in pg_isolation_regress_installcheck since PostgreSQL 10. Also: replace spaces with a tab in the same place of isolationcheck. --- Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index f94dca4..ca9faab 100644 --- a/Makefile +++ b/Makefile @@ -31,13 +31,15 @@ ISOLATIONCHECKS = corner_cases check: isolationcheck installcheck: submake-isolation + $(MKDIR_P) isolation_output $(pg_isolation_regress_installcheck) \ + --outputdir=isolation_output \ $(ISOLATIONCHECKS) isolationcheck: | submake-isolation temp-install $(MKDIR_P) isolation_output $(pg_isolation_regress_check) \ - --outputdir=isolation_output \ + --outputdir=isolation_output \ $(ISOLATIONCHECKS) submake-isolation: From 5a9f8b2f75f1a70e0808905f998e018211d63937 Mon Sep 17 00:00:00 2001 From: Vyacheslav Makarov Date: Fri, 11 Jun 2021 18:20:29 +0300 Subject: [PATCH 48/91] [PGPRO-5172] Fixed infinite looping in the receive_msg_by_parts function. --- pg_query_state.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pg_query_state.c b/pg_query_state.c index 46556d1..d461e20 100644 --- a/pg_query_state.c +++ b/pg_query_state.c @@ -970,19 +970,21 @@ receive_msg_by_parts(shm_mq_handle *mqh, Size *total, void **datap, shm_mq_result mq_receive_result; shm_mq_msg *buff; int offset; - int *expected; + int *expected; + int expected_data; Size len; /* Get the expected number of bytes in message */ mq_receive_result = shm_mq_receive(mqh, &len, (void **) &expected, nowait); + expected_data = *expected; if (mq_receive_result != SHM_MQ_SUCCESS) return mq_receive_result; Assert(len == sizeof(int)); - *datap = palloc0(*expected); + *datap = palloc0(expected_data); /* Get the message itself */ - for (offset = 0; offset < *expected; ) + for (offset = 0; offset < expected_data; ) { /* Keep receiving new messages until we assemble the full message */ mq_receive_result = shm_mq_receive(mqh, &len, ((void **) &buff), nowait); From 3d6ca52bcf9a82794440197fcfb472a233a92683 Mon Sep 17 00:00:00 2001 From: Daria Lepikhova Date: Wed, 30 Jun 2021 18:19:49 +0500 Subject: [PATCH 49/91] Fix isolation test output format --- expected/corner_cases.out | 57 ++++++++++++++++++++++++++------------- 1 file changed, 39 insertions(+), 18 deletions(-) diff --git a/expected/corner_cases.out b/expected/corner_cases.out index 8320e1f..b475e33 100644 --- a/expected/corner_cases.out +++ b/expected/corner_cases.out @@ -1,3 +1,6 @@ +unused step name: s1_enable_pg_qs +unused step name: s1_pg_qs_counterpart +unused step name: s2_save_pid Parsed test spec with 2 sessions starting permutation: s1_pg_qs_1 @@ -10,55 +13,73 @@ ERROR: attempt to extract state of current process starting permutation: s1_save_pid s2_pg_qs_counterpart step s1_save_pid: select save_own_pid(0); -save_own_pid +save_own_pid +------------ + +(1 row) - -INFO: state of backend is idle +s2: INFO: state of backend is idle step s2_pg_qs_counterpart: select pg_query_state(counterpart_pid(0)); -pg_query_state +pg_query_state +-------------- +(0 rows) starting permutation: s1_save_pid s1_disable_pg_qs s2_pg_qs_counterpart step s1_save_pid: select save_own_pid(0); -save_own_pid +save_own_pid +------------ + +(1 row) - step s1_disable_pg_qs: set pg_query_state.enable to off; -INFO: query execution statistics disabled +s2: INFO: query execution statistics disabled step s2_pg_qs_counterpart: select pg_query_state(counterpart_pid(0)); -pg_query_state +pg_query_state +-------------- +(0 rows) starting permutation: s1_set_bob s2_set_bob s1_save_pid s2_pg_qs_counterpart step s1_set_bob: set role bob; step s2_set_bob: set role bob; step s1_save_pid: select save_own_pid(0); -save_own_pid +save_own_pid +------------ + +(1 row) - -INFO: state of backend is idle +s2: INFO: state of backend is idle step s2_pg_qs_counterpart: select pg_query_state(counterpart_pid(0)); -pg_query_state +pg_query_state +-------------- +(0 rows) starting permutation: s1_set_bob s2_set_su s1_save_pid s2_pg_qs_counterpart step s1_set_bob: set role bob; step s2_set_su: set role super; step s1_save_pid: select save_own_pid(0); -save_own_pid +save_own_pid +------------ + +(1 row) - -INFO: state of backend is idle +s2: INFO: state of backend is idle step s2_pg_qs_counterpart: select pg_query_state(counterpart_pid(0)); -pg_query_state +pg_query_state +-------------- +(0 rows) starting permutation: s1_set_bob s2_set_alice s1_save_pid s2_pg_qs_counterpart step s1_set_bob: set role bob; step s2_set_alice: set role alice; step s1_save_pid: select save_own_pid(0); -save_own_pid +save_own_pid +------------ + +(1 row) - step s2_pg_qs_counterpart: select pg_query_state(counterpart_pid(0)); ERROR: permission denied From 1ca39b92b14d8364e918654164594b3f9f40bb81 Mon Sep 17 00:00:00 2001 From: Daria Lepikhova Date: Fri, 30 Jul 2021 10:07:46 +0500 Subject: [PATCH 50/91] Add return Nil; after error messages for fix warnings --- pg_query_state.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pg_query_state.c b/pg_query_state.c index d461e20..9927be2 100644 --- a/pg_query_state.c +++ b/pg_query_state.c @@ -952,6 +952,8 @@ GetRemoteBackendWorkers(PGPROC *proc) mq_error: ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("error in message queue data transmitting"))); + + return NIL; } static shm_mq_msg * @@ -1140,6 +1142,8 @@ GetRemoteBackendQueryStates(PGPROC *leader, #endif ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("error in message queue data transmitting"))); + + return NIL; } void From beb8587a3b60ec6e1a378b1a2c60a6368aeafb43 Mon Sep 17 00:00:00 2001 From: Kovalenko Anastasia Date: Fri, 30 Jul 2021 18:53:26 +0300 Subject: [PATCH 51/91] Python tests fixed --- tests/common.py | 66 +++++++++++++++++++- tests/pg_qs_test_runner.py | 17 +++++- tests/test_cases.py | 121 ++++++++++++++++++++++--------------- 3 files changed, 154 insertions(+), 50 deletions(-) diff --git a/tests/common.py b/tests/common.py index 3f4f9c2..ac24e76 100644 --- a/tests/common.py +++ b/tests/common.py @@ -43,6 +43,21 @@ def n_close(conns): for conn in conns: conn.close() +def pg_query_state_locks(config, pid, conn, verbose=False, costs=False, timing=False, \ + buffers=False, triggers=False, format='text'): + """ + Get query state from backend with specified pid and optional parameters. + Save any warning, info, notice and log data in global variable 'notices' + """ + + curs = conn.cursor() + curs.callproc('pg_query_state', (pid, verbose, costs, timing, buffers, triggers, format)) + wait(conn) + result = curs.fetchall() + notices = conn.notices[:] + + return result, notices + def pg_query_state(config, pid, verbose=False, costs=False, timing=False, \ buffers=False, triggers=False, format='text'): """ @@ -52,7 +67,6 @@ def pg_query_state(config, pid, verbose=False, costs=False, timing=False, \ conn = psycopg2.connect(**config) curs = conn.cursor() - curs.callproc('pg_query_state', (pid, verbose, costs, timing, buffers, triggers, format)) result = curs.fetchall() notices = conn.notices[:] @@ -60,6 +74,56 @@ def pg_query_state(config, pid, verbose=False, costs=False, timing=False, \ return result, notices +def onetime_query_state_locks(config, acon_query, acon_pg, query, args={}, num_workers=0): + """ + Get intermediate state of 'query' on connection 'acon_query' after number of 'steps' + of node executions from start of query + """ + + curs_query = acon_query.cursor() + curs_pg = acon_pg.cursor() + curs_query.execute("select pg_advisory_lock(1);") + curs_pg.execute("select pg_advisory_lock(2);") + wait(acon_query) + wait(acon_pg) + curs_pg.execute("select pg_advisory_lock(1);") + set_guc(acon_query, 'enable_mergejoin', 'off') + set_guc(acon_query, 'max_parallel_workers_per_gather', num_workers) + curs_query.execute(query) + # extract current state of query progress + MAX_PG_QS_RETRIES = 10 + DELAY_BETWEEN_RETRIES = 0.1 + pg_qs_args = { + 'config': config, + 'pid': acon_query.get_backend_pid(), + 'conn': acon_pg + } + for k, v in args.items(): + pg_qs_args[k] = v + n_retries = 0 + + wait(acon_pg) + + while True: + result, notices = pg_query_state_locks(**pg_qs_args) + n_retries += 1 + if len(result) > 0: + break + if n_retries >= MAX_PG_QS_RETRIES: + # pg_query_state callings don't return any result, more likely run + # query has completed + break + time.sleep(DELAY_BETWEEN_RETRIES) + + curs_pg.execute("select pg_advisory_unlock(2);") + wait(acon_pg) + wait(acon_query) + + set_guc(acon_query, 'enable_mergejoin', 'on') + curs_query.execute("select pg_advisory_unlock(2);") + curs_pg.execute("select pg_advisory_unlock(1);") + return result, notices + def onetime_query_state(config, async_conn, query, args={}, num_workers=0): """ Get intermediate state of 'query' on connection 'async_conn' after number of 'steps' diff --git a/tests/pg_qs_test_runner.py b/tests/pg_qs_test_runner.py index 28db807..a6e02e9 100644 --- a/tests/pg_qs_test_runner.py +++ b/tests/pg_qs_test_runner.py @@ -1,6 +1,6 @@ ''' pg_qs_test_runner.py -Copyright (c) 2016-2020, Postgres Professional +Copyright (c) 2016-2021, Postgres Professional ''' import argparse @@ -22,6 +22,20 @@ def __call__(self, parser, args, values, option_string=None): class SetupException(Exception): pass class TeardownException(Exception): pass +unlock_if_eq_1 = """ + CREATE OR REPLACE FUNCTION unlock_if_eq_1(x integer) RETURNS integer AS $$ + BEGIN + IF x = 1 THEN + perform pg_advisory_unlock(1); + perform pg_advisory_lock(2); + return 1; + ELSE + return x; + END IF; + END; + $$ LANGUAGE plpgsql + """ + setup_cmd = [ 'drop extension if exists pg_query_state cascade', 'drop table if exists foo cascade', @@ -33,6 +47,7 @@ class TeardownException(Exception): pass 'insert into bar select i, i%2=1 from generate_series(1, 500000) as i', 'analyze foo', 'analyze bar', + unlock_if_eq_1, ] teardown_cmd = [ diff --git a/tests/test_cases.py b/tests/test_cases.py index 440f32f..1750bb1 100644 --- a/tests/test_cases.py +++ b/tests/test_cases.py @@ -1,6 +1,6 @@ ''' test_cases.py -Copyright (c) 2016-2020, Postgres Professional +Copyright (c) 2016-2021, Postgres Professional ''' import json @@ -42,21 +42,28 @@ def test_deadlock(config): def test_simple_query(config): """test statistics of simple query""" - acon, = common.n_async_connect(config) - query = 'select count(*) from foo join bar on foo.c1=bar.c1' + acon1, acon2 = common.n_async_connect(config, 2) + query = 'select count(*) from foo join bar on foo.c1=bar.c1 and unlock_if_eq_1(foo.c1)=bar.c1' expected = r"""Aggregate \(Current loop: actual rows=\d+, loop number=1\) - -> Hash Join \(Current loop: actual rows=\d+, loop number=1\) + -> Hash Join \(Current loop: actual rows=62473, loop number=1\) Hash Cond: \(foo.c1 = bar.c1\) + Join Filter: \(unlock_if_eq_1\(foo.c1\) = bar.c1\) -> Seq Scan on foo \(Current loop: actual rows=\d+, loop number=1\) -> Hash \(Current loop: actual rows=\d+, loop number=1\) Buckets: \d+ Batches: \d+ Memory Usage: \d+kB -> Seq Scan on bar \(Current loop: actual rows=\d+, loop number=1\)""" - qs, _ = common.onetime_query_state(config, acon, query) - assert qs[0][0] == acon.get_backend_pid() and qs[0][1] == 0 \ - and qs[0][2] == query and re.match(expected, qs[0][3]) and qs[0][4] == None + qs, _ = common.onetime_query_state_locks(config, acon1, acon2, query) - common.n_close((acon,)) + assert qs[0][0] == acon1.get_backend_pid() + assert qs[0][1] == 0 + assert qs[0][2] == query + assert re.match(expected, qs[0][3]) + assert qs[0][4] == None + # assert qs[0][0] == acon.get_backend_pid() and qs[0][1] == 0 \ + # and qs[0][2] == query and re.match(expected, qs[0][3]) and qs[0][4] == None + + common.n_close((acon1, acon2)) def test_concurrent_access(config): """test when two backends compete with each other to extract state from third running backend""" @@ -87,50 +94,56 @@ def test_concurrent_access(config): def test_nested_call(config): """test statistics under calling function""" - acon, = common.n_async_connect(config) + acon1, acon2 = common.n_async_connect(config, 2) util_conn = psycopg2.connect(**config) util_curs = util_conn.cursor() create_function = """ create or replace function n_join_foo_bar() returns integer as $$ begin - return (select count(*) from foo join bar on foo.c1=bar.c1); + return (select count(*) from foo join bar on foo.c1=bar.c1 and unlock_if_eq_1(foo.c1)=bar.c1); end; $$ language plpgsql""" drop_function = 'drop function n_join_foo_bar()' call_function = 'select * from n_join_foo_bar()' - nested_query = 'SELECT (select count(*) from foo join bar on foo.c1=bar.c1)' + nested_query1 = '(select count(*) from foo join bar on foo.c1=bar.c1 and unlock_if_eq_1(foo.c1)=bar.c1)' + nested_query2 = 'SELECT (select count(*) from foo join bar on foo.c1=bar.c1 and unlock_if_eq_1(foo.c1)=bar.c1)' expected = 'Function Scan on n_join_foo_bar (Current loop: actual rows=0, loop number=1)' expected_nested = r"""Result \(Current loop: actual rows=0, loop number=1\) InitPlan 1 \(returns \$0\) -> Aggregate \(Current loop: actual rows=0, loop number=1\) - -> Hash Join \(Current loop: actual rows=0, loop number=1\) + -> Hash Join \(Current loop: actual rows=62473, loop number=1\) Hash Cond: \(foo.c1 = bar.c1\) - -> Seq Scan on foo \(Current loop: actual rows=1, loop number=1\) - -> Hash \(Current loop: actual rows=0, loop number=1\) + Join Filter: \(unlock_if_eq_1\(foo.c1\) = bar.c1\) + -> Seq Scan on foo \(Current loop: actual rows=1000000, loop number=1\) + -> Hash \(Current loop: actual rows=500000, loop number=1\) Buckets: \d+ Batches: \d+ Memory Usage: \d+kB -> Seq Scan on bar \(Current loop: actual rows=\d+, loop number=1\)""" + util_curs.execute(create_function) util_conn.commit() - qs, notices = common.onetime_query_state(config, acon, call_function) + qs, notices = common.onetime_query_state_locks(config, acon1, acon2, call_function) # Print some debug output before assertion if len(qs) < 2: print(qs) - assert len(qs) == 2 \ - and qs[0][0] == qs[1][0] == acon.get_backend_pid() \ - and qs[0][1] == 0 and qs[1][1] == 1 \ - and qs[0][2] == call_function and qs[0][3] == expected \ - and qs[1][2] == nested_query and re.match(expected_nested, qs[1][3]) \ - and qs[0][4] == qs[1][4] == None + assert len(qs) == 3 + assert qs[0][0] == qs[1][0] == acon1.get_backend_pid() + assert qs[0][1] == 0 + assert qs[1][1] == 1 + assert qs[0][2] == call_function + assert qs[0][3] == expected + assert qs[1][2] == nested_query1 or qs[1][2] == nested_query2 + assert re.match(expected_nested, qs[1][3]) + assert qs[0][4] == qs[1][4] == None assert len(notices) == 0 util_curs.execute(drop_function) util_conn.close() - common.n_close((acon,)) + common.n_close((acon1, acon2)) def test_insert_on_conflict(config): """test statistics on conflicting tuples under INSERT ON CONFLICT query""" @@ -212,65 +225,77 @@ def test_trigger(config): def test_costs(config): """test plan costs""" - acon, = common.n_async_connect(config) - query = 'select count(*) from foo join bar on foo.c1=bar.c1' + acon1, acon2 = common.n_async_connect(config, 2) + query = 'select count(*) from foo join bar on foo.c1=bar.c1 and unlock_if_eq_1(foo.c1)=bar.c1;' + expected = r"""Aggregate \(cost=\d+.\d+..\d+.\d+ rows=\d+ width=8\) \(Current loop: actual rows=0, loop number=1\) - -> Hash Join \(cost=\d+.\d+..\d+.\d+ rows=\d+ width=0\) \(Current loop: actual rows=0, loop number=1\) + -> Hash Join \(cost=\d+.\d+..\d+.\d+ rows=\d+ width=0\) \(Current loop: actual rows=\d+, loop number=1\) Hash Cond: \(foo.c1 = bar.c1\) - -> Seq Scan on foo \(cost=0.00..\d+.\d+ rows=\d+ width=4\) \(Current loop: actual rows=1, loop number=1\) - -> Hash \(cost=\d+.\d+..\d+.\d+ rows=\d+ width=4\) \(Current loop: actual rows=0, loop number=1\) + Join Filter: \(unlock_if_eq_1\(foo.c1\) = bar.c1\) + -> Seq Scan on foo \(cost=0.00..\d+.\d+ rows=\d+ width=4\) \(Current loop: actual rows=1000000, loop number=1\) + -> Hash \(cost=\d+.\d+..\d+.\d+ rows=\d+ width=4\) \(Current loop: actual rows=500000, loop number=1\) Buckets: \d+ Batches: \d+ Memory Usage: \d+kB -> Seq Scan on bar \(cost=0.00..\d+.\d+ rows=\d+ width=4\) \(Current loop: actual rows=\d+, loop number=1\)""" - qs, notices = common.onetime_query_state(config, acon, query, {'costs': True}) - assert len(qs) == 1 and re.match(expected, qs[0][3]) + qs, notices = common.onetime_query_state_locks(config, acon1, acon2, query, {'costs': True}) + + assert len(qs) == 2 and re.match(expected, qs[0][3]) assert len(notices) == 0 - common.n_close((acon,)) + common.n_close((acon1, acon2)) def test_buffers(config): """test buffer statistics""" - acon, = common.n_async_connect(config) - query = 'select count(*) from foo join bar on foo.c1=bar.c1' + acon1, acon2 = common.n_async_connect(config, 2) + query = 'select count(*) from foo join bar on foo.c1=bar.c1 and unlock_if_eq_1(foo.c1)=bar.c1' expected = r"""Aggregate \(Current loop: actual rows=0, loop number=1\) - -> Hash Join \(Current loop: actual rows=0, loop number=1\) + -> Hash Join \(Current loop: actual rows=\d+, loop number=1\) Hash Cond: \(foo.c1 = bar.c1\) - -> Seq Scan on foo \(Current loop: actual rows=1, loop number=1\) + Join Filter: \(unlock_if_eq_1\(foo.c1\) = bar.c1\) + Buffers: shared hit=\d+, temp read=\d+ written=\d+ + -> Seq Scan on foo \(Current loop: actual rows=1000000, loop number=1\) Buffers: [^\n]* - -> Hash \(Current loop: actual rows=0, loop number=1\) + -> Hash \(Current loop: actual rows=500000, loop number=1\) Buckets: \d+ Batches: \d+ Memory Usage: \d+kB + Buffers: shared hit=\d+, temp written=\d+ -> Seq Scan on bar \(Current loop: actual rows=\d+, loop number=1\) Buffers: .*""" - common.set_guc(acon, 'pg_query_state.enable_buffers', 'on') + common.set_guc(acon1, 'pg_query_state.enable_buffers', 'on') - qs, notices = common.onetime_query_state(config, acon, query, {'buffers': True}) - assert len(qs) == 1 and re.match(expected, qs[0][3]) + qs, notices = common.onetime_query_state_locks(config, acon1, acon2, query, {'buffers': True}) + + assert len(qs) == 2 + assert re.match(expected, qs[0][3]) assert len(notices) == 0 - common.n_close((acon,)) + common.n_close((acon1, acon2)) def test_timing(config): """test timing statistics""" - acon, = common.n_async_connect(config) - query = 'select count(*) from foo join bar on foo.c1=bar.c1' + acon1, acon2 = common.n_async_connect(config, 2) + query = 'select count(*) from foo join bar on foo.c1=bar.c1 and unlock_if_eq_1(foo.c1)=bar.c1' + expected = r"""Aggregate \(Current loop: running time=\d+.\d+ actual rows=0, loop number=1\) - -> Hash Join \(Current loop: running time=\d+.\d+ actual rows=0, loop number=1\) + -> Hash Join \(Current loop: actual time=\d+.\d+..\d+.\d+ rows=\d+, loop number=1\) Hash Cond: \(foo.c1 = bar.c1\) - -> Seq Scan on foo \(Current loop: actual time=\d+.\d+..\d+.\d+ rows=1, loop number=1\) - -> Hash \(Current loop: running time=\d+.\d+ actual rows=0, loop number=1\) + Join Filter: \(unlock_if_eq_1\(foo.c1\) = bar.c1\) + -> Seq Scan on foo \(Current loop: actual time=\d+.\d+..\d+.\d+ rows=1000000, loop number=1\) + -> Hash \(Current loop: actual time=\d+.\d+..\d+.\d+ rows=500000, loop number=1\) Buckets: \d+ Batches: \d+ Memory Usage: \d+kB -> Seq Scan on bar \(Current loop: actual time=\d+.\d+..\d+.\d+ rows=\d+, loop number=1\)""" - common.set_guc(acon, 'pg_query_state.enable_timing', 'on') + common.set_guc(acon1, 'pg_query_state.enable_timing', 'on') - qs, notices = common.onetime_query_state(config, acon, query, {'timing': True}) - assert len(qs) == 1 and re.match(expected, qs[0][3]) + qs, notices = common.onetime_query_state_locks(config, acon1, acon2, query, {'timing': True}) + + assert len(qs) == 2 + assert re.match(expected, qs[0][3]) assert len(notices) == 0 - common.n_close((acon,)) + common.n_close((acon1, acon2)) def check_plan(plan): assert 'Current loop' in plan From bd92be082309d1ad0ab012fc3fe180a0f0be0b9c Mon Sep 17 00:00:00 2001 From: Daniel Shelepanov Date: Mon, 30 Aug 2021 19:15:57 +0300 Subject: [PATCH 52/91] [PGPRO-5146] Byte manipulations with Size fixed in send/receive_msg_by_parts On 64-bit architectures sizeof(Size) (which is in fact size_t) not always equals to sizeof(int) so the Size type manipulations in send/receive_msg_by_parts should be implemented using the Size type explicitly. tags: pg_query_state --- pg_query_state.c | 6 +++--- signal_handler.c | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pg_query_state.c b/pg_query_state.c index 9927be2..d4a58fb 100644 --- a/pg_query_state.c +++ b/pg_query_state.c @@ -972,8 +972,8 @@ receive_msg_by_parts(shm_mq_handle *mqh, Size *total, void **datap, shm_mq_result mq_receive_result; shm_mq_msg *buff; int offset; - int *expected; - int expected_data; + Size *expected; + Size expected_data; Size len; /* Get the expected number of bytes in message */ @@ -981,7 +981,7 @@ receive_msg_by_parts(shm_mq_handle *mqh, Size *total, void **datap, expected_data = *expected; if (mq_receive_result != SHM_MQ_SUCCESS) return mq_receive_result; - Assert(len == sizeof(int)); + Assert(len == sizeof(Size)); *datap = palloc0(expected_data); diff --git a/signal_handler.c b/signal_handler.c index 74c2c6d..8b61d36 100644 --- a/signal_handler.c +++ b/signal_handler.c @@ -159,7 +159,7 @@ send_msg_by_parts(shm_mq_handle *mqh, Size nbytes, const void *data) int offset; /* Send the expected message length */ - shm_mq_send(mqh, sizeof(int), &nbytes, false); + shm_mq_send(mqh, sizeof(Size), &nbytes, false); for (offset = 0; offset < nbytes; offset += bytes_send) { From 49bd4e6afcb0374d1b6fc8cc853f6afd98aa549b Mon Sep 17 00:00:00 2001 From: Daniel Shelepanov Date: Wed, 1 Sep 2021 21:01:05 +0300 Subject: [PATCH 53/91] pipeline fix related to 'pg:config: Operation not permitted' --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index c6c06bf..ecf1670 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,10 @@ sudo: required language: c +os: linux + +dist: focal + services: - docker From 70003a5f205a634d9c877d40b62967e7624a54ea Mon Sep 17 00:00:00 2001 From: Daniel Shelepanov Date: Mon, 6 Sep 2021 17:57:31 +0300 Subject: [PATCH 54/91] 5146: minor readme fix --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f645a7d..3ba5eb2 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -[![Build Status](https://travis-ci.org/postgrespro/pg_query_state.svg?branch=master)](https://travis-ci.org/postgrespro/pg_query_state) +[![Build Status](https://travis-ci.com/postgrespro/pg_query_state.svg?branch=master)](https://travis-ci.com/postgrespro/pg_query_state) [![codecov](https://codecov.io/gh/postgrespro/pg_query_state/branch/master/graph/badge.svg)](https://codecov.io/gh/postgrespro/pg_query_state) # pg\_query\_state From 799b23c787c9ca35989e596ce6a5f8f22a418225 Mon Sep 17 00:00:00 2001 From: Koval Dmitry Date: Thu, 2 Sep 2021 14:52:10 +0300 Subject: [PATCH 55/91] [PGPRO-5531] Fixed crashes in the receive_msg_by_parts function --- pg_query_state.c | 48 ++++++++++++++++++++++++++++++++++++++---------- pg_query_state.h | 5 +++++ 2 files changed, 43 insertions(+), 10 deletions(-) diff --git a/pg_query_state.c b/pg_query_state.c index d4a58fb..fc3f547 100644 --- a/pg_query_state.c +++ b/pg_query_state.c @@ -61,7 +61,7 @@ static void qs_ExecutorRun(QueryDesc *queryDesc, ScanDirection direction, static void qs_ExecutorFinish(QueryDesc *queryDesc); static shm_mq_result receive_msg_by_parts(shm_mq_handle *mqh, Size *total, - void **datap, bool nowait); + void **datap, int64 timeout, int *rc, bool nowait); /* Global variables */ List *QueryDescStack = NIL; @@ -780,7 +780,7 @@ shm_mq_receive_with_timeout(shm_mq_handle *mqh, { shm_mq_result mq_receive_result; - mq_receive_result = receive_msg_by_parts(mqh, nbytesp, datap, true); + mq_receive_result = receive_msg_by_parts(mqh, nbytesp, datap, timeout, &rc, true); if (mq_receive_result != SHM_MQ_WOULD_BLOCK) return mq_receive_result; if (rc & WL_TIMEOUT || delay <= 0) @@ -967,33 +967,61 @@ copy_msg(shm_mq_msg *msg) static shm_mq_result receive_msg_by_parts(shm_mq_handle *mqh, Size *total, void **datap, - bool nowait) + int64 timeout, int *rc, bool nowait) { shm_mq_result mq_receive_result; shm_mq_msg *buff; int offset; - Size *expected; - Size expected_data; + Size *expected; + Size expected_data; Size len; /* Get the expected number of bytes in message */ mq_receive_result = shm_mq_receive(mqh, &len, (void **) &expected, nowait); - expected_data = *expected; if (mq_receive_result != SHM_MQ_SUCCESS) return mq_receive_result; Assert(len == sizeof(Size)); + expected_data = *expected; *datap = palloc0(expected_data); /* Get the message itself */ for (offset = 0; offset < expected_data; ) { + int64 delay = timeout; /* Keep receiving new messages until we assemble the full message */ - mq_receive_result = shm_mq_receive(mqh, &len, ((void **) &buff), nowait); + for (;;) + { + mq_receive_result = shm_mq_receive(mqh, &len, ((void **) &buff), nowait); + if (mq_receive_result != SHM_MQ_SUCCESS) + { + if (nowait && mq_receive_result == SHM_MQ_WOULD_BLOCK) + { + /* + * We can't leave this function during reading parts with + * error code SHM_MQ_WOULD_BLOCK because can be be error + * at next call receive_msg_by_parts() with continuing + * reading non-readed parts. + * So we should wait whole MAX_RCV_TIMEOUT timeout and + * return error after that only. + */ + if (delay > 0) + { + pg_usleep(PART_RCV_DELAY * 1000); + delay -= PART_RCV_DELAY; + continue; + } + if (rc) + { /* Mark that the timeout has expired: */ + *rc |= WL_TIMEOUT; + } + } + return mq_receive_result; + } + break; + } memcpy((char *) *datap + offset, buff, len); offset += len; - if (mq_receive_result != SHM_MQ_SUCCESS) - return mq_receive_result; } *total = offset; @@ -1074,7 +1102,7 @@ GetRemoteBackendQueryStates(PGPROC *leader, mqh = shm_mq_attach(mq, NULL, NULL); elog(DEBUG1, "Wait response from leader %d", leader->pid); mq_receive_result = receive_msg_by_parts(mqh, &len, (void **) &msg, - false); + 0, NULL, false); if (mq_receive_result != SHM_MQ_SUCCESS) goto mq_error; if (msg->reqid != reqid) diff --git a/pg_query_state.h b/pg_query_state.h index 8192ae2..c812cd4 100644 --- a/pg_query_state.h +++ b/pg_query_state.h @@ -31,6 +31,11 @@ #define MAX_RCV_TIMEOUT 6000 /* 6 seconds */ #define MAX_SND_TIMEOUT 3000 /* 3 seconds */ +/* + * Delay for receiving parts of full message (in case SHM_MQ_WOULD_BLOCK code), + * should be tess than MAX_RCV_TIMEOUT + */ +#define PART_RCV_DELAY 1000 /* 1 second */ /* * Result status on query state request from asked backend From c769e11bf205687d3e0feab3d0c423aef6988e8d Mon Sep 17 00:00:00 2001 From: Vyacheslav Makarov Date: Tue, 14 Sep 2021 10:48:22 +0300 Subject: [PATCH 56/91] [PGPRO-5575]: Fixed installcheck in Makefile. When running "make installcheck" again, the tests fell. This change fixed this. --- Makefile | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/Makefile b/Makefile index ca9faab..e017e60 100644 --- a/Makefile +++ b/Makefile @@ -30,11 +30,7 @@ ISOLATIONCHECKS = corner_cases check: isolationcheck -installcheck: submake-isolation - $(MKDIR_P) isolation_output - $(pg_isolation_regress_installcheck) \ - --outputdir=isolation_output \ - $(ISOLATIONCHECKS) +installcheck: isolationcheck isolationcheck: | submake-isolation temp-install $(MKDIR_P) isolation_output From f79ebd98da2aeccd983f5bf85ccc0511a24f10d7 Mon Sep 17 00:00:00 2001 From: Daniel Shelepanov Date: Wed, 22 Sep 2021 17:33:25 +0300 Subject: [PATCH 57/91] [PGPRO-4561] Non-blocking writing to the queue To avoid deadlocking while canceling the pg_query_state() call writing to the queue is implemented non-blocking way. --- pg_query_state.c | 2 +- pg_query_state.h | 2 ++ signal_handler.c | 76 +++++++++++++++++++++++++++++++++++++++++++----- 3 files changed, 72 insertions(+), 8 deletions(-) diff --git a/pg_query_state.c b/pg_query_state.c index fc3f547..a4a7d1b 100644 --- a/pg_query_state.c +++ b/pg_query_state.c @@ -1141,7 +1141,7 @@ GetRemoteBackendQueryStates(PGPROC *leader, MAX_RCV_TIMEOUT); if (mq_receive_result != SHM_MQ_SUCCESS) { - /* counterpart is died, not consider it */ + /* counterpart is dead, not considering it */ goto mq_error; } if (msg->reqid != reqid) diff --git a/pg_query_state.h b/pg_query_state.h index c812cd4..382f910 100644 --- a/pg_query_state.h +++ b/pg_query_state.h @@ -19,6 +19,8 @@ #define QUEUE_SIZE (16 * 1024) #define MSG_MAX_SIZE 1024 +#define WRITING_DELAY (100 * 1000) // 100ms +#define NUM_OF_ATTEMPTS 6 #define TIMINIG_OFF_WARNING 1 #define BUFFERS_OFF_WARNING 2 diff --git a/signal_handler.c b/signal_handler.c index 8b61d36..2369fec 100644 --- a/signal_handler.c +++ b/signal_handler.c @@ -27,7 +27,16 @@ typedef struct char *plan; } stack_frame; -static void send_msg_by_parts(shm_mq_handle *mqh, Size nbytes, const void *data); +/* + * An self-explanarory enum describing the send_msg_by_parts results + */ +typedef enum +{ + MSG_BY_PARTS_SUCCEEDED, + MSG_BY_PARTS_FAILED +} msg_by_parts_result; + +static msg_by_parts_result send_msg_by_parts(shm_mq_handle *mqh, Size nbytes, const void *data); /* * Get List of stack_frames as a stack of function calls starting from outermost call. @@ -151,7 +160,36 @@ serialize_stack(char *dest, List *qs_stack) } } -static void +static msg_by_parts_result +shm_mq_send_nonblocking(shm_mq_handle *mqh, Size nbytes, const void *data, Size attempts) +{ + int i; + shm_mq_result res; + + for(i = 0; i < attempts; i++) + { + res = shm_mq_send(mqh, nbytes, data, true); + + if(res == SHM_MQ_SUCCESS) + break; + else if (res == SHM_MQ_DETACHED) + return MSG_BY_PARTS_FAILED; + + /* SHM_MQ_WOULD_BLOCK - sleeping for some delay */ + pg_usleep(WRITING_DELAY); + } + + if(i == attempts) + return MSG_BY_PARTS_FAILED; + + return MSG_BY_PARTS_SUCCEEDED; +} + +/* + * send_msg_by_parts sends data throurh the queue as a bunch of messages + * of smaller size + */ +static msg_by_parts_result send_msg_by_parts(shm_mq_handle *mqh, Size nbytes, const void *data) { int bytes_left; @@ -159,14 +197,20 @@ send_msg_by_parts(shm_mq_handle *mqh, Size nbytes, const void *data) int offset; /* Send the expected message length */ - shm_mq_send(mqh, sizeof(Size), &nbytes, false); + if(shm_mq_send_nonblocking(mqh, sizeof(Size), &nbytes, NUM_OF_ATTEMPTS) == MSG_BY_PARTS_FAILED) + return MSG_BY_PARTS_FAILED; + /* Send the message itself */ for (offset = 0; offset < nbytes; offset += bytes_send) { bytes_left = nbytes - offset; bytes_send = (bytes_left < MSG_MAX_SIZE) ? bytes_left : MSG_MAX_SIZE; - shm_mq_send(mqh, bytes_send, &(((unsigned char*)data)[offset]), false); + if(shm_mq_send_nonblocking(mqh, bytes_send, &(((unsigned char*)data)[offset]), NUM_OF_ATTEMPTS) + == MSG_BY_PARTS_FAILED) + return MSG_BY_PARTS_FAILED; } + + return MSG_BY_PARTS_SUCCEEDED; } /* @@ -227,7 +271,8 @@ SendQueryState(void) { shm_mq_msg msg = { reqid, BASE_SIZEOF_SHM_MQ_MSG, MyProc, STAT_DISABLED }; - send_msg_by_parts(mqh, msg.length, &msg); + if(send_msg_by_parts(mqh, msg.length, &msg) != MSG_BY_PARTS_SUCCEEDED) + goto connection_cleanup; } /* check if backend doesn't execute any query */ @@ -235,7 +280,8 @@ SendQueryState(void) { shm_mq_msg msg = { reqid, BASE_SIZEOF_SHM_MQ_MSG, MyProc, QUERY_NOT_RUNNING }; - send_msg_by_parts(mqh, msg.length, &msg); + if(send_msg_by_parts(mqh, msg.length, &msg) != MSG_BY_PARTS_SUCCEEDED) + goto connection_cleanup; } /* happy path */ @@ -258,9 +304,25 @@ SendQueryState(void) msg->stack_depth = list_length(qs_stack); serialize_stack(msg->stack, qs_stack); - send_msg_by_parts(mqh, msglen, msg); + + if(send_msg_by_parts(mqh, msglen, msg) != MSG_BY_PARTS_SUCCEEDED) + { + elog(WARNING, "pg_query_state: peer seems to have detached"); + goto connection_cleanup; + } } elog(DEBUG1, "Worker %d sends response for pg_query_state to %d", shm_mq_get_sender(mq)->pid, shm_mq_get_receiver(mq)->pid); DetachPeer(); UnlockShmem(&tag); + + return; + +connection_cleanup: +#if PG_VERSION_NUM < 100000 + shm_mq_detach(mq); +#else + shm_mq_detach(mqh); +#endif + DetachPeer(); + UnlockShmem(&tag); } From 25384f2513ef5c7a97194bdba0d71b7a17310a84 Mon Sep 17 00:00:00 2001 From: Kovalenko Anastasia Date: Fri, 23 Jul 2021 12:55:50 +0300 Subject: [PATCH 58/91] Patches for version 14 Docker scripts improved for version 14 --- .travis.yml | 2 + patches/custom_signals_14.0.patch | 217 +++++++++++++++++++++++ patches/runtime_explain_14.0.patch | 271 +++++++++++++++++++++++++++++ 3 files changed, 490 insertions(+) create mode 100644 patches/custom_signals_14.0.patch create mode 100644 patches/runtime_explain_14.0.patch diff --git a/.travis.yml b/.travis.yml index c6c06bf..8b3a5a7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,6 +18,8 @@ notifications: on_failure: always env: + - PG_VERSION=14 LEVEL=hardcore USE_TPCDS=1 + - PG_VERSION=14 - PG_VERSION=13 LEVEL=hardcore USE_TPCDS=1 - PG_VERSION=13 - PG_VERSION=12 LEVEL=hardcore USE_TPCDS=1 diff --git a/patches/custom_signals_14.0.patch b/patches/custom_signals_14.0.patch new file mode 100644 index 0000000..fbb83de --- /dev/null +++ b/patches/custom_signals_14.0.patch @@ -0,0 +1,217 @@ +From f2632ea7cd03119c55b8aa0ef60f529380ca2536 Mon Sep 17 00:00:00 2001 +From: Kovalenko Anastasia +Date: Tue, 24 Aug 2021 16:22:28 +0300 +Subject: [PATCH] custom-signals + +--- + src/backend/storage/ipc/procsignal.c | 94 ++++++++++++++++++++++++++++ + src/backend/tcop/postgres.c | 2 + + src/include/storage/procsignal.h | 17 +++++ + 3 files changed, 113 insertions(+) + +diff --git a/src/backend/storage/ipc/procsignal.c b/src/backend/storage/ipc/procsignal.c +index defb75a..4245d28 100644 +--- a/src/backend/storage/ipc/procsignal.c ++++ b/src/backend/storage/ipc/procsignal.c +@@ -95,6 +95,13 @@ typedef struct + #define BARRIER_CLEAR_BIT(flags, type) \ + ((flags) &= ~(((uint32) 1) << (uint32) (type))) + ++#define IsCustomProcSignalReason(reason) \ ++ ((reason) >= PROCSIG_CUSTOM_1 && (reason) <= PROCSIG_CUSTOM_N) ++ ++static bool CustomSignalPendings[NUM_CUSTOM_PROCSIGNALS]; ++static bool CustomSignalProcessing[NUM_CUSTOM_PROCSIGNALS]; ++static ProcSignalHandler_type CustomInterruptHandlers[NUM_CUSTOM_PROCSIGNALS]; ++ + static ProcSignalHeader *ProcSignal = NULL; + static ProcSignalSlot *MyProcSignalSlot = NULL; + +@@ -103,6 +110,8 @@ static void CleanupProcSignalState(int status, Datum arg); + static void ResetProcSignalBarrierBits(uint32 flags); + static bool ProcessBarrierPlaceholder(void); + ++static void CheckAndSetCustomSignalInterrupts(void); ++ + /* + * ProcSignalShmemSize + * Compute space needed for procsignal's shared memory +@@ -246,6 +255,36 @@ CleanupProcSignalState(int status, Datum arg) + slot->pss_pid = 0; + } + ++/* ++ * RegisterCustomProcSignalHandler ++ * Assign specific handler of custom process signal with new ++ * ProcSignalReason key. ++ * ++ * This function has to be called in _PG_init function of extensions at the ++ * stage of loading shared preloaded libraries. Otherwise it throws fatal error. ++ * ++ * Return INVALID_PROCSIGNAL if all slots for custom signals are occupied. ++ */ ++ProcSignalReason ++RegisterCustomProcSignalHandler(ProcSignalHandler_type handler) ++{ ++ ProcSignalReason reason; ++ ++ if (!process_shared_preload_libraries_in_progress) ++ ereport(FATAL, (errcode(ERRCODE_INTERNAL_ERROR), ++ errmsg("cannot register custom signal after startup"))); ++ ++ /* Iterate through custom signal slots to find a free one */ ++ for (reason = PROCSIG_CUSTOM_1; reason <= PROCSIG_CUSTOM_N; reason++) ++ if (!CustomInterruptHandlers[reason - PROCSIG_CUSTOM_1]) ++ { ++ CustomInterruptHandlers[reason - PROCSIG_CUSTOM_1] = handler; ++ return reason; ++ } ++ ++ return INVALID_PROCSIGNAL; ++} ++ + /* + * SendProcSignal + * Send a signal to a Postgres process +@@ -679,7 +718,62 @@ procsignal_sigusr1_handler(SIGNAL_ARGS) + if (CheckProcSignal(PROCSIG_RECOVERY_CONFLICT_BUFFERPIN)) + RecoveryConflictInterrupt(PROCSIG_RECOVERY_CONFLICT_BUFFERPIN); + ++ CheckAndSetCustomSignalInterrupts(); ++ + SetLatch(MyLatch); + + errno = save_errno; + } ++ ++/* ++ * Handle receipt of an interrupt indicating any of custom process signals. ++ */ ++static void ++CheckAndSetCustomSignalInterrupts() ++{ ++ ProcSignalReason reason; ++ ++ for (reason = PROCSIG_CUSTOM_1; reason <= PROCSIG_CUSTOM_N; reason++) ++ { ++ if (CheckProcSignal(reason)) ++ { ++ ++ /* set interrupt flags */ ++ InterruptPending = true; ++ CustomSignalPendings[reason - PROCSIG_CUSTOM_1] = true; ++ } ++ } ++ ++ SetLatch(MyLatch); ++} ++ ++/* ++ * CheckAndHandleCustomSignals ++ * Check custom signal flags and call handler assigned to that signal ++ * if it is not NULL ++ * ++ * This function is called within CHECK_FOR_INTERRUPTS if interrupt occurred. ++ */ ++void ++CheckAndHandleCustomSignals(void) ++{ ++ int i; ++ ++ /* Check on expiring of custom signals and call its handlers if exist */ ++ for (i = 0; i < NUM_CUSTOM_PROCSIGNALS; i++) ++ { ++ if (!CustomSignalProcessing[i] && CustomSignalPendings[i]) ++ { ++ ProcSignalHandler_type handler; ++ ++ CustomSignalPendings[i] = false; ++ handler = CustomInterruptHandlers[i]; ++ if (handler != NULL) ++ { ++ CustomSignalProcessing[i] = true; ++ handler(); ++ CustomSignalProcessing[i] = false; ++ } ++ } ++ } ++} +\ No newline at end of file +diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c +index 8cea10c..dd77c98 100644 +--- a/src/backend/tcop/postgres.c ++++ b/src/backend/tcop/postgres.c +@@ -3364,6 +3364,8 @@ ProcessInterrupts(void) + if (ParallelMessagePending) + HandleParallelMessages(); + ++ CheckAndHandleCustomSignals(); ++ + if (LogMemoryContextPending) + ProcessLogMemoryContextInterrupt(); + } +diff --git a/src/include/storage/procsignal.h b/src/include/storage/procsignal.h +index eec186b..74af186 100644 +--- a/src/include/storage/procsignal.h ++++ b/src/include/storage/procsignal.h +@@ -17,6 +17,8 @@ + #include "storage/backendid.h" + + ++#define NUM_CUSTOM_PROCSIGNALS 64 ++ + /* + * Reasons for signaling a Postgres child process (a backend or an auxiliary + * process, like checkpointer). We can cope with concurrent signals for different +@@ -29,6 +31,8 @@ + */ + typedef enum + { ++ INVALID_PROCSIGNAL = -1, /* Must be first */ ++ + PROCSIG_CATCHUP_INTERRUPT, /* sinval catchup interrupt */ + PROCSIG_NOTIFY_INTERRUPT, /* listen/notify interrupt */ + PROCSIG_PARALLEL_MESSAGE, /* message from cooperating parallel backend */ +@@ -44,6 +48,14 @@ typedef enum + PROCSIG_RECOVERY_CONFLICT_BUFFERPIN, + PROCSIG_RECOVERY_CONFLICT_STARTUP_DEADLOCK, + ++ PROCSIG_CUSTOM_1, ++ /* ++ * PROCSIG_CUSTOM_2, ++ * ..., ++ * PROCSIG_CUSTOM_N-1, ++ */ ++ PROCSIG_CUSTOM_N = PROCSIG_CUSTOM_1 + NUM_CUSTOM_PROCSIGNALS - 1, ++ + NUM_PROCSIGNALS /* Must be last! */ + } ProcSignalReason; + +@@ -56,6 +68,8 @@ typedef enum + */ + PROCSIGNAL_BARRIER_PLACEHOLDER = 0 + } ProcSignalBarrierType; ++/* Handler of custom process signal */ ++typedef void (*ProcSignalHandler_type) (void); + + /* + * prototypes for functions in procsignal.c +@@ -64,12 +78,15 @@ extern Size ProcSignalShmemSize(void); + extern void ProcSignalShmemInit(void); + + extern void ProcSignalInit(int pss_idx); ++extern ProcSignalReason ++ RegisterCustomProcSignalHandler(ProcSignalHandler_type handler); + extern int SendProcSignal(pid_t pid, ProcSignalReason reason, + BackendId backendId); + + extern uint64 EmitProcSignalBarrier(ProcSignalBarrierType type); + extern void WaitForProcSignalBarrier(uint64 generation); + extern void ProcessProcSignalBarrier(void); ++extern void CheckAndHandleCustomSignals(void); + + extern void procsignal_sigusr1_handler(SIGNAL_ARGS); + +-- +2.25.1 + diff --git a/patches/runtime_explain_14.0.patch b/patches/runtime_explain_14.0.patch new file mode 100644 index 0000000..3dbba9a --- /dev/null +++ b/patches/runtime_explain_14.0.patch @@ -0,0 +1,271 @@ +From 71a353d4cac663db43c57452f925082a233c0e49 Mon Sep 17 00:00:00 2001 +From: Kovalenko Anastasia +Date: Mon, 23 Aug 2021 15:29:59 +0300 +Subject: [PATCH] runtime-explain + +--- + src/backend/commands/explain.c | 153 ++++++++++++++++++++++++++++----- + src/include/commands/explain.h | 2 + + 2 files changed, 133 insertions(+), 22 deletions(-) + +diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c +index 10644dfac4..7106ed4257 100644 +--- a/src/backend/commands/explain.c ++++ b/src/backend/commands/explain.c +@@ -984,14 +984,36 @@ report_triggers(ResultRelInfo *rInfo, bool show_relname, ExplainState *es) + char *relname; + char *conname = NULL; + ++ instr_time starttimespan; ++ double total; ++ double ntuples; ++ double ncalls; ++ ++ if (!es->runtime) ++ { + /* Must clean up instrumentation state */ + InstrEndLoop(instr); ++ } ++ ++ /* Collect statistic variables */ ++ if (!INSTR_TIME_IS_ZERO(instr->starttime)) ++ { ++ INSTR_TIME_SET_CURRENT(starttimespan); ++ INSTR_TIME_SUBTRACT(starttimespan, instr->starttime); ++ } ++ else ++ INSTR_TIME_SET_ZERO(starttimespan); ++ ++ total = instr->total + INSTR_TIME_GET_DOUBLE(instr->counter) ++ + INSTR_TIME_GET_DOUBLE(starttimespan); ++ ntuples = instr->ntuples + instr->tuplecount; ++ ncalls = ntuples + !INSTR_TIME_IS_ZERO(starttimespan); + + /* + * We ignore triggers that were never invoked; they likely aren't + * relevant to the current query type. + */ +- if (instr->ntuples == 0) ++ if (ncalls == 0) + continue; + + ExplainOpenGroup("Trigger", NULL, true, es); +@@ -1017,9 +1039,9 @@ report_triggers(ResultRelInfo *rInfo, bool show_relname, ExplainState *es) + appendStringInfo(es->str, " on %s", relname); + if (es->timing) + appendStringInfo(es->str, ": time=%.3f calls=%.0f\n", +- 1000.0 * instr->total, instr->ntuples); ++ 1000.0 * total, ncalls); + else +- appendStringInfo(es->str, ": calls=%.0f\n", instr->ntuples); ++ appendStringInfo(es->str, ": calls=%.0f\n", ncalls); + } + else + { +@@ -1028,9 +1050,8 @@ report_triggers(ResultRelInfo *rInfo, bool show_relname, ExplainState *es) + ExplainPropertyText("Constraint Name", conname, es); + ExplainPropertyText("Relation", relname, es); + if (es->timing) +- ExplainPropertyFloat("Time", "ms", 1000.0 * instr->total, 3, +- es); +- ExplainPropertyFloat("Calls", NULL, instr->ntuples, 0, es); ++ ExplainPropertyFloat("Time", "ms", 1000.0 * total, 3, es); ++ ExplainPropertyFloat("Calls", NULL, ncalls, 0, es); + } + + if (conname) +@@ -1600,8 +1621,11 @@ ExplainNode(PlanState *planstate, List *ancestors, + * instrumentation results the user didn't ask for. But we do the + * InstrEndLoop call anyway, if possible, to reduce the number of cases + * auto_explain has to contend with. ++ * ++ * If flag es->stateinfo is set, i.e. when printing the current execution ++ * state, this step of cleaning up is missed. + */ +- if (planstate->instrument) ++ if (planstate->instrument && !es->runtime) + InstrEndLoop(planstate->instrument); + + if (es->analyze && +@@ -1636,7 +1660,7 @@ ExplainNode(PlanState *planstate, List *ancestors, + ExplainPropertyFloat("Actual Loops", NULL, nloops, 0, es); + } + } +- else if (es->analyze) ++ else if (es->analyze && !es->runtime) + { + if (es->format == EXPLAIN_FORMAT_TEXT) + appendStringInfoString(es->str, " (never executed)"); +@@ -1652,6 +1676,75 @@ ExplainNode(PlanState *planstate, List *ancestors, + } + } + ++ /* ++ * Print the progress of node execution at current loop. ++ */ ++ if (planstate->instrument && es->analyze && es->runtime) ++ { ++ instr_time starttimespan; ++ double startup_sec; ++ double total_sec; ++ double rows; ++ double loop_num; ++ bool finished; ++ ++ if (!INSTR_TIME_IS_ZERO(planstate->instrument->starttime)) ++ { ++ INSTR_TIME_SET_CURRENT(starttimespan); ++ INSTR_TIME_SUBTRACT(starttimespan, planstate->instrument->starttime); ++ } ++ else ++ INSTR_TIME_SET_ZERO(starttimespan); ++ startup_sec = 1000.0 * planstate->instrument->firsttuple; ++ total_sec = 1000.0 * (INSTR_TIME_GET_DOUBLE(planstate->instrument->counter) ++ + INSTR_TIME_GET_DOUBLE(starttimespan)); ++ rows = planstate->instrument->tuplecount; ++ loop_num = planstate->instrument->nloops + 1; ++ ++ finished = planstate->instrument->nloops > 0 ++ && !planstate->instrument->running ++ && INSTR_TIME_IS_ZERO(starttimespan); ++ ++ if (!finished) ++ { ++ ExplainOpenGroup("Current loop", "Current loop", true, es); ++ if (es->format == EXPLAIN_FORMAT_TEXT) ++ { ++ if (es->timing) ++ { ++ if (planstate->instrument->running) ++ appendStringInfo(es->str, ++ " (Current loop: actual time=%.3f..%.3f rows=%.0f, loop number=%.0f)", ++ startup_sec, total_sec, rows, loop_num); ++ else ++ appendStringInfo(es->str, ++ " (Current loop: running time=%.3f actual rows=0, loop number=%.0f)", ++ total_sec, loop_num); ++ } ++ else ++ appendStringInfo(es->str, ++ " (Current loop: actual rows=%.0f, loop number=%.0f)", ++ rows, loop_num); ++ } ++ else ++ { ++ ExplainPropertyFloat("Actual Loop Number", NULL, loop_num, 0, es); ++ if (es->timing) ++ { ++ if (planstate->instrument->running) ++ { ++ ExplainPropertyFloat("Actual Startup Time", NULL, startup_sec, 3, es); ++ ExplainPropertyFloat("Actual Total Time", NULL, total_sec, 3, es); ++ } ++ else ++ ExplainPropertyFloat("Running Time", NULL, total_sec, 3, es); ++ } ++ ExplainPropertyFloat("Actual Rows", NULL, rows, 0, es); ++ } ++ ExplainCloseGroup("Current loop", "Current loop", true, es); ++ } ++ } ++ + /* in text format, first line ends here */ + if (es->format == EXPLAIN_FORMAT_TEXT) + appendStringInfoChar(es->str, '\n'); +@@ -2051,6 +2144,9 @@ ExplainNode(PlanState *planstate, List *ancestors, + + /* Prepare per-worker buffer/WAL usage */ + if (es->workers_state && (es->buffers || es->wal) && es->verbose) ++ /* Show worker detail after query execution */ ++ if (es->analyze && es->verbose && planstate->worker_instrument ++ && !es->runtime) + { + WorkerInstrumentation *w = planstate->worker_instrument; + +@@ -3015,6 +3111,11 @@ show_hash_info(HashState *hashstate, ExplainState *es) + memcpy(&hinstrument, hashstate->hinstrument, + sizeof(HashInstrumentation)); + ++ if (hashstate->hashtable) ++ { ++ ExecHashAccumInstrumentation(&hinstrument, hashstate->hashtable); ++ } ++ + /* + * Merge results from workers. In the parallel-oblivious case, the + * results from all participants should be identical, except where +@@ -3392,20 +3493,16 @@ show_instrumentation_count(const char *qlabel, int which, + if (!es->analyze || !planstate->instrument) + return; + ++ nloops = planstate->instrument->nloops; + if (which == 2) +- nfiltered = planstate->instrument->nfiltered2; ++ nfiltered = ((nloops > 0) ? planstate->instrument->nfiltered2 / nloops : 0); + else +- nfiltered = planstate->instrument->nfiltered1; ++ nfiltered = ((nloops > 0) ? planstate->instrument->nfiltered1 / nloops : 0); + nloops = planstate->instrument->nloops; + + /* In text mode, suppress zero counts; they're not interesting enough */ + if (nfiltered > 0 || es->format != EXPLAIN_FORMAT_TEXT) +- { +- if (nloops > 0) +- ExplainPropertyFloat(qlabel, NULL, nfiltered / nloops, 0, es); +- else +- ExplainPropertyFloat(qlabel, NULL, 0.0, 0, es); +- } ++ ExplainPropertyFloat(qlabel, NULL, nfiltered, 0, es); + } + + /* +@@ -3977,15 +4074,27 @@ show_modifytable_info(ModifyTableState *mtstate, List *ancestors, + double insert_path; + double other_path; + +- InstrEndLoop(outerPlanState(mtstate)->instrument); ++ if (!es->runtime) ++ InstrEndLoop(outerPlanState(mtstate)->instrument); + + /* count the number of source rows */ +- total = outerPlanState(mtstate)->instrument->ntuples; +- other_path = mtstate->ps.instrument->ntuples2; +- insert_path = total - other_path; ++ other_path = mtstate->ps.instrument->nfiltered2; ++ ++ /* ++ * Insert occurs after extracting row from subplan and in runtime mode ++ * we can appear between these two operations - situation when ++ * total > insert_path + other_path. Therefore we don't know exactly ++ * whether last row from subplan is inserted. ++ * We don't print inserted tuples in runtime mode in order to not print ++ * inconsistent data ++ */ ++ if (!es->runtime) ++ { ++ total = outerPlanState(mtstate)->instrument->ntuples; ++ insert_path = total - other_path; ++ ExplainPropertyFloat("Tuples Inserted", NULL, insert_path, 0, es); ++ } + +- ExplainPropertyFloat("Tuples Inserted", NULL, +- insert_path, 0, es); + ExplainPropertyFloat("Conflicting Tuples", NULL, + other_path, 0, es); + } +diff --git a/src/include/commands/explain.h b/src/include/commands/explain.h +index e94d9e49cf..6a157b8bc0 100644 +--- a/src/include/commands/explain.h ++++ b/src/include/commands/explain.h +@@ -47,6 +47,8 @@ typedef struct ExplainState + bool summary; /* print total planning and execution timing */ + bool settings; /* print modified settings */ + ExplainFormat format; /* output format */ ++ bool runtime; /* print intermediate state of query execution, ++ not after completion */ + /* state for output formatting --- not reset for each new plan tree */ + int indent; /* current indentation level */ + List *grouping_stack; /* format-specific grouping state */ +-- +2.25.1 + From a4f4cd7e383f03c4a0bb6e96d7ef5526a1273ac2 Mon Sep 17 00:00:00 2001 From: Daniel Shelepanov Date: Wed, 13 Oct 2021 17:04:39 +0300 Subject: [PATCH 59/91] [PGPRO-4561] indentation and typo fixed after review --- pg_query_state.h | 2 +- signal_handler.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pg_query_state.h b/pg_query_state.h index 382f910..2b13234 100644 --- a/pg_query_state.h +++ b/pg_query_state.h @@ -19,7 +19,7 @@ #define QUEUE_SIZE (16 * 1024) #define MSG_MAX_SIZE 1024 -#define WRITING_DELAY (100 * 1000) // 100ms +#define WRITING_DELAY (100 * 1000) /* 100ms */ #define NUM_OF_ATTEMPTS 6 #define TIMINIG_OFF_WARNING 1 diff --git a/signal_handler.c b/signal_handler.c index 2369fec..2af69fd 100644 --- a/signal_handler.c +++ b/signal_handler.c @@ -163,7 +163,7 @@ serialize_stack(char *dest, List *qs_stack) static msg_by_parts_result shm_mq_send_nonblocking(shm_mq_handle *mqh, Size nbytes, const void *data, Size attempts) { - int i; + int i; shm_mq_result res; for(i = 0; i < attempts; i++) @@ -186,7 +186,7 @@ shm_mq_send_nonblocking(shm_mq_handle *mqh, Size nbytes, const void *data, Size } /* - * send_msg_by_parts sends data throurh the queue as a bunch of messages + * send_msg_by_parts sends data through the queue as a bunch of messages * of smaller size */ static msg_by_parts_result From 4b5b13feded6c554cc2bf332db2fadbcbcfa4c87 Mon Sep 17 00:00:00 2001 From: Vyacheslav Makarov Date: Thu, 14 Oct 2021 14:24:45 +0300 Subject: [PATCH 60/91] Temporarily turning off TPCDS tests. --- .travis.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8b3a5a7..300ddca 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,15 +18,15 @@ notifications: on_failure: always env: - - PG_VERSION=14 LEVEL=hardcore USE_TPCDS=1 + - PG_VERSION=14 LEVEL=hardcore USE_TPCDS=0 - PG_VERSION=14 - - PG_VERSION=13 LEVEL=hardcore USE_TPCDS=1 + - PG_VERSION=13 LEVEL=hardcore USE_TPCDS=0 - PG_VERSION=13 - - PG_VERSION=12 LEVEL=hardcore USE_TPCDS=1 + - PG_VERSION=12 LEVEL=hardcore USE_TPCDS=0 - PG_VERSION=12 - - PG_VERSION=11 LEVEL=hardcore USE_TPCDS=1 + - PG_VERSION=11 LEVEL=hardcore USE_TPCDS=0 - PG_VERSION=11 - - PG_VERSION=10 LEVEL=hardcore USE_TPCDS=1 + - PG_VERSION=10 LEVEL=hardcore USE_TPCDS=0 - PG_VERSION=10 - PG_VERSION=9.6 LEVEL=hardcore - PG_VERSION=9.6 From b166a3778d7ae41f4c53c0d0e25497c13089a294 Mon Sep 17 00:00:00 2001 From: "Anton A. Melnikov" Date: Sun, 24 Oct 2021 18:53:01 +0300 Subject: [PATCH 61/91] [PGPRO-5703] Run installcheck against an existing installation. Tags: pg_query_state --- Makefile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index e017e60..ca9faab 100644 --- a/Makefile +++ b/Makefile @@ -30,7 +30,11 @@ ISOLATIONCHECKS = corner_cases check: isolationcheck -installcheck: isolationcheck +installcheck: submake-isolation + $(MKDIR_P) isolation_output + $(pg_isolation_regress_installcheck) \ + --outputdir=isolation_output \ + $(ISOLATIONCHECKS) isolationcheck: | submake-isolation temp-install $(MKDIR_P) isolation_output From a9168b94ba446f46471143922bc4b1414f74be86 Mon Sep 17 00:00:00 2001 From: "Anton A. Melnikov" Date: Wed, 27 Oct 2021 10:36:41 +0300 Subject: [PATCH 62/91] [PGPRO-5703] Place setting of n_peers after the exit on permission denied. Previously setting of n_peers before error exit leds to a warning: pg_query_state: last request was interrupted and MAX_RCV_TIMEOUT pause at second function call. Tags: pg_query_state --- pg_query_state.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pg_query_state.c b/pg_query_state.c index a4a7d1b..935fbf8 100644 --- a/pg_query_state.c +++ b/pg_query_state.c @@ -541,14 +541,18 @@ pg_query_state(PG_FUNCTION_ARGS) break; } } - pg_atomic_write_u32(&counterpart_userid->n_peers, 1); - params->reqid = ++reqid; - pg_write_barrier(); counterpart_user_id = GetRemoteBackendUserId(proc); if (!(superuser() || GetUserId() == counterpart_user_id)) + { + UnlockShmem(&tag); ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("permission denied"))); + } + + pg_atomic_write_u32(&counterpart_userid->n_peers, 1); + params->reqid = ++reqid; + pg_write_barrier(); bg_worker_procs = GetRemoteBackendWorkers(proc); From 612f955ac9619b4d10ed2834582628c54f304c74 Mon Sep 17 00:00:00 2001 From: Vyacheslav Makarov Date: Fri, 29 Oct 2021 07:20:47 +0300 Subject: [PATCH 63/91] porting to version 15 and minor improvements to version 14 Fixed minor bugs in patches for version 14. Added patches for version 15. shm_mq_send started accepting one more argument. Solved this problem with #ifdef PG_VERSION_NUM. --- patches/custom_signals_14.0.patch | 11 -- patches/custom_signals_15.0.patch | 206 +++++++++++++++++++++++ patches/runtime_explain_14.0.patch | 10 -- patches/runtime_explain_15.0.patch | 261 +++++++++++++++++++++++++++++ pg_query_state.c | 4 + signal_handler.c | 4 + 6 files changed, 475 insertions(+), 21 deletions(-) create mode 100644 patches/custom_signals_15.0.patch create mode 100644 patches/runtime_explain_15.0.patch diff --git a/patches/custom_signals_14.0.patch b/patches/custom_signals_14.0.patch index fbb83de..9d640cb 100644 --- a/patches/custom_signals_14.0.patch +++ b/patches/custom_signals_14.0.patch @@ -1,14 +1,3 @@ -From f2632ea7cd03119c55b8aa0ef60f529380ca2536 Mon Sep 17 00:00:00 2001 -From: Kovalenko Anastasia -Date: Tue, 24 Aug 2021 16:22:28 +0300 -Subject: [PATCH] custom-signals - ---- - src/backend/storage/ipc/procsignal.c | 94 ++++++++++++++++++++++++++++ - src/backend/tcop/postgres.c | 2 + - src/include/storage/procsignal.h | 17 +++++ - 3 files changed, 113 insertions(+) - diff --git a/src/backend/storage/ipc/procsignal.c b/src/backend/storage/ipc/procsignal.c index defb75a..4245d28 100644 --- a/src/backend/storage/ipc/procsignal.c diff --git a/patches/custom_signals_15.0.patch b/patches/custom_signals_15.0.patch new file mode 100644 index 0000000..9d640cb --- /dev/null +++ b/patches/custom_signals_15.0.patch @@ -0,0 +1,206 @@ +diff --git a/src/backend/storage/ipc/procsignal.c b/src/backend/storage/ipc/procsignal.c +index defb75a..4245d28 100644 +--- a/src/backend/storage/ipc/procsignal.c ++++ b/src/backend/storage/ipc/procsignal.c +@@ -95,6 +95,13 @@ typedef struct + #define BARRIER_CLEAR_BIT(flags, type) \ + ((flags) &= ~(((uint32) 1) << (uint32) (type))) + ++#define IsCustomProcSignalReason(reason) \ ++ ((reason) >= PROCSIG_CUSTOM_1 && (reason) <= PROCSIG_CUSTOM_N) ++ ++static bool CustomSignalPendings[NUM_CUSTOM_PROCSIGNALS]; ++static bool CustomSignalProcessing[NUM_CUSTOM_PROCSIGNALS]; ++static ProcSignalHandler_type CustomInterruptHandlers[NUM_CUSTOM_PROCSIGNALS]; ++ + static ProcSignalHeader *ProcSignal = NULL; + static ProcSignalSlot *MyProcSignalSlot = NULL; + +@@ -103,6 +110,8 @@ static void CleanupProcSignalState(int status, Datum arg); + static void ResetProcSignalBarrierBits(uint32 flags); + static bool ProcessBarrierPlaceholder(void); + ++static void CheckAndSetCustomSignalInterrupts(void); ++ + /* + * ProcSignalShmemSize + * Compute space needed for procsignal's shared memory +@@ -246,6 +255,36 @@ CleanupProcSignalState(int status, Datum arg) + slot->pss_pid = 0; + } + ++/* ++ * RegisterCustomProcSignalHandler ++ * Assign specific handler of custom process signal with new ++ * ProcSignalReason key. ++ * ++ * This function has to be called in _PG_init function of extensions at the ++ * stage of loading shared preloaded libraries. Otherwise it throws fatal error. ++ * ++ * Return INVALID_PROCSIGNAL if all slots for custom signals are occupied. ++ */ ++ProcSignalReason ++RegisterCustomProcSignalHandler(ProcSignalHandler_type handler) ++{ ++ ProcSignalReason reason; ++ ++ if (!process_shared_preload_libraries_in_progress) ++ ereport(FATAL, (errcode(ERRCODE_INTERNAL_ERROR), ++ errmsg("cannot register custom signal after startup"))); ++ ++ /* Iterate through custom signal slots to find a free one */ ++ for (reason = PROCSIG_CUSTOM_1; reason <= PROCSIG_CUSTOM_N; reason++) ++ if (!CustomInterruptHandlers[reason - PROCSIG_CUSTOM_1]) ++ { ++ CustomInterruptHandlers[reason - PROCSIG_CUSTOM_1] = handler; ++ return reason; ++ } ++ ++ return INVALID_PROCSIGNAL; ++} ++ + /* + * SendProcSignal + * Send a signal to a Postgres process +@@ -679,7 +718,62 @@ procsignal_sigusr1_handler(SIGNAL_ARGS) + if (CheckProcSignal(PROCSIG_RECOVERY_CONFLICT_BUFFERPIN)) + RecoveryConflictInterrupt(PROCSIG_RECOVERY_CONFLICT_BUFFERPIN); + ++ CheckAndSetCustomSignalInterrupts(); ++ + SetLatch(MyLatch); + + errno = save_errno; + } ++ ++/* ++ * Handle receipt of an interrupt indicating any of custom process signals. ++ */ ++static void ++CheckAndSetCustomSignalInterrupts() ++{ ++ ProcSignalReason reason; ++ ++ for (reason = PROCSIG_CUSTOM_1; reason <= PROCSIG_CUSTOM_N; reason++) ++ { ++ if (CheckProcSignal(reason)) ++ { ++ ++ /* set interrupt flags */ ++ InterruptPending = true; ++ CustomSignalPendings[reason - PROCSIG_CUSTOM_1] = true; ++ } ++ } ++ ++ SetLatch(MyLatch); ++} ++ ++/* ++ * CheckAndHandleCustomSignals ++ * Check custom signal flags and call handler assigned to that signal ++ * if it is not NULL ++ * ++ * This function is called within CHECK_FOR_INTERRUPTS if interrupt occurred. ++ */ ++void ++CheckAndHandleCustomSignals(void) ++{ ++ int i; ++ ++ /* Check on expiring of custom signals and call its handlers if exist */ ++ for (i = 0; i < NUM_CUSTOM_PROCSIGNALS; i++) ++ { ++ if (!CustomSignalProcessing[i] && CustomSignalPendings[i]) ++ { ++ ProcSignalHandler_type handler; ++ ++ CustomSignalPendings[i] = false; ++ handler = CustomInterruptHandlers[i]; ++ if (handler != NULL) ++ { ++ CustomSignalProcessing[i] = true; ++ handler(); ++ CustomSignalProcessing[i] = false; ++ } ++ } ++ } ++} +\ No newline at end of file +diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c +index 8cea10c..dd77c98 100644 +--- a/src/backend/tcop/postgres.c ++++ b/src/backend/tcop/postgres.c +@@ -3364,6 +3364,8 @@ ProcessInterrupts(void) + if (ParallelMessagePending) + HandleParallelMessages(); + ++ CheckAndHandleCustomSignals(); ++ + if (LogMemoryContextPending) + ProcessLogMemoryContextInterrupt(); + } +diff --git a/src/include/storage/procsignal.h b/src/include/storage/procsignal.h +index eec186b..74af186 100644 +--- a/src/include/storage/procsignal.h ++++ b/src/include/storage/procsignal.h +@@ -17,6 +17,8 @@ + #include "storage/backendid.h" + + ++#define NUM_CUSTOM_PROCSIGNALS 64 ++ + /* + * Reasons for signaling a Postgres child process (a backend or an auxiliary + * process, like checkpointer). We can cope with concurrent signals for different +@@ -29,6 +31,8 @@ + */ + typedef enum + { ++ INVALID_PROCSIGNAL = -1, /* Must be first */ ++ + PROCSIG_CATCHUP_INTERRUPT, /* sinval catchup interrupt */ + PROCSIG_NOTIFY_INTERRUPT, /* listen/notify interrupt */ + PROCSIG_PARALLEL_MESSAGE, /* message from cooperating parallel backend */ +@@ -44,6 +48,14 @@ typedef enum + PROCSIG_RECOVERY_CONFLICT_BUFFERPIN, + PROCSIG_RECOVERY_CONFLICT_STARTUP_DEADLOCK, + ++ PROCSIG_CUSTOM_1, ++ /* ++ * PROCSIG_CUSTOM_2, ++ * ..., ++ * PROCSIG_CUSTOM_N-1, ++ */ ++ PROCSIG_CUSTOM_N = PROCSIG_CUSTOM_1 + NUM_CUSTOM_PROCSIGNALS - 1, ++ + NUM_PROCSIGNALS /* Must be last! */ + } ProcSignalReason; + +@@ -56,6 +68,8 @@ typedef enum + */ + PROCSIGNAL_BARRIER_PLACEHOLDER = 0 + } ProcSignalBarrierType; ++/* Handler of custom process signal */ ++typedef void (*ProcSignalHandler_type) (void); + + /* + * prototypes for functions in procsignal.c +@@ -64,12 +78,15 @@ extern Size ProcSignalShmemSize(void); + extern void ProcSignalShmemInit(void); + + extern void ProcSignalInit(int pss_idx); ++extern ProcSignalReason ++ RegisterCustomProcSignalHandler(ProcSignalHandler_type handler); + extern int SendProcSignal(pid_t pid, ProcSignalReason reason, + BackendId backendId); + + extern uint64 EmitProcSignalBarrier(ProcSignalBarrierType type); + extern void WaitForProcSignalBarrier(uint64 generation); + extern void ProcessProcSignalBarrier(void); ++extern void CheckAndHandleCustomSignals(void); + + extern void procsignal_sigusr1_handler(SIGNAL_ARGS); + +-- +2.25.1 + diff --git a/patches/runtime_explain_14.0.patch b/patches/runtime_explain_14.0.patch index 3dbba9a..7904cc2 100644 --- a/patches/runtime_explain_14.0.patch +++ b/patches/runtime_explain_14.0.patch @@ -1,13 +1,3 @@ -From 71a353d4cac663db43c57452f925082a233c0e49 Mon Sep 17 00:00:00 2001 -From: Kovalenko Anastasia -Date: Mon, 23 Aug 2021 15:29:59 +0300 -Subject: [PATCH] runtime-explain - ---- - src/backend/commands/explain.c | 153 ++++++++++++++++++++++++++++----- - src/include/commands/explain.h | 2 + - 2 files changed, 133 insertions(+), 22 deletions(-) - diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c index 10644dfac4..7106ed4257 100644 --- a/src/backend/commands/explain.c diff --git a/patches/runtime_explain_15.0.patch b/patches/runtime_explain_15.0.patch new file mode 100644 index 0000000..7904cc2 --- /dev/null +++ b/patches/runtime_explain_15.0.patch @@ -0,0 +1,261 @@ +diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c +index 10644dfac4..7106ed4257 100644 +--- a/src/backend/commands/explain.c ++++ b/src/backend/commands/explain.c +@@ -984,14 +984,36 @@ report_triggers(ResultRelInfo *rInfo, bool show_relname, ExplainState *es) + char *relname; + char *conname = NULL; + ++ instr_time starttimespan; ++ double total; ++ double ntuples; ++ double ncalls; ++ ++ if (!es->runtime) ++ { + /* Must clean up instrumentation state */ + InstrEndLoop(instr); ++ } ++ ++ /* Collect statistic variables */ ++ if (!INSTR_TIME_IS_ZERO(instr->starttime)) ++ { ++ INSTR_TIME_SET_CURRENT(starttimespan); ++ INSTR_TIME_SUBTRACT(starttimespan, instr->starttime); ++ } ++ else ++ INSTR_TIME_SET_ZERO(starttimespan); ++ ++ total = instr->total + INSTR_TIME_GET_DOUBLE(instr->counter) ++ + INSTR_TIME_GET_DOUBLE(starttimespan); ++ ntuples = instr->ntuples + instr->tuplecount; ++ ncalls = ntuples + !INSTR_TIME_IS_ZERO(starttimespan); + + /* + * We ignore triggers that were never invoked; they likely aren't + * relevant to the current query type. + */ +- if (instr->ntuples == 0) ++ if (ncalls == 0) + continue; + + ExplainOpenGroup("Trigger", NULL, true, es); +@@ -1017,9 +1039,9 @@ report_triggers(ResultRelInfo *rInfo, bool show_relname, ExplainState *es) + appendStringInfo(es->str, " on %s", relname); + if (es->timing) + appendStringInfo(es->str, ": time=%.3f calls=%.0f\n", +- 1000.0 * instr->total, instr->ntuples); ++ 1000.0 * total, ncalls); + else +- appendStringInfo(es->str, ": calls=%.0f\n", instr->ntuples); ++ appendStringInfo(es->str, ": calls=%.0f\n", ncalls); + } + else + { +@@ -1028,9 +1050,8 @@ report_triggers(ResultRelInfo *rInfo, bool show_relname, ExplainState *es) + ExplainPropertyText("Constraint Name", conname, es); + ExplainPropertyText("Relation", relname, es); + if (es->timing) +- ExplainPropertyFloat("Time", "ms", 1000.0 * instr->total, 3, +- es); +- ExplainPropertyFloat("Calls", NULL, instr->ntuples, 0, es); ++ ExplainPropertyFloat("Time", "ms", 1000.0 * total, 3, es); ++ ExplainPropertyFloat("Calls", NULL, ncalls, 0, es); + } + + if (conname) +@@ -1600,8 +1621,11 @@ ExplainNode(PlanState *planstate, List *ancestors, + * instrumentation results the user didn't ask for. But we do the + * InstrEndLoop call anyway, if possible, to reduce the number of cases + * auto_explain has to contend with. ++ * ++ * If flag es->stateinfo is set, i.e. when printing the current execution ++ * state, this step of cleaning up is missed. + */ +- if (planstate->instrument) ++ if (planstate->instrument && !es->runtime) + InstrEndLoop(planstate->instrument); + + if (es->analyze && +@@ -1636,7 +1660,7 @@ ExplainNode(PlanState *planstate, List *ancestors, + ExplainPropertyFloat("Actual Loops", NULL, nloops, 0, es); + } + } +- else if (es->analyze) ++ else if (es->analyze && !es->runtime) + { + if (es->format == EXPLAIN_FORMAT_TEXT) + appendStringInfoString(es->str, " (never executed)"); +@@ -1652,6 +1676,75 @@ ExplainNode(PlanState *planstate, List *ancestors, + } + } + ++ /* ++ * Print the progress of node execution at current loop. ++ */ ++ if (planstate->instrument && es->analyze && es->runtime) ++ { ++ instr_time starttimespan; ++ double startup_sec; ++ double total_sec; ++ double rows; ++ double loop_num; ++ bool finished; ++ ++ if (!INSTR_TIME_IS_ZERO(planstate->instrument->starttime)) ++ { ++ INSTR_TIME_SET_CURRENT(starttimespan); ++ INSTR_TIME_SUBTRACT(starttimespan, planstate->instrument->starttime); ++ } ++ else ++ INSTR_TIME_SET_ZERO(starttimespan); ++ startup_sec = 1000.0 * planstate->instrument->firsttuple; ++ total_sec = 1000.0 * (INSTR_TIME_GET_DOUBLE(planstate->instrument->counter) ++ + INSTR_TIME_GET_DOUBLE(starttimespan)); ++ rows = planstate->instrument->tuplecount; ++ loop_num = planstate->instrument->nloops + 1; ++ ++ finished = planstate->instrument->nloops > 0 ++ && !planstate->instrument->running ++ && INSTR_TIME_IS_ZERO(starttimespan); ++ ++ if (!finished) ++ { ++ ExplainOpenGroup("Current loop", "Current loop", true, es); ++ if (es->format == EXPLAIN_FORMAT_TEXT) ++ { ++ if (es->timing) ++ { ++ if (planstate->instrument->running) ++ appendStringInfo(es->str, ++ " (Current loop: actual time=%.3f..%.3f rows=%.0f, loop number=%.0f)", ++ startup_sec, total_sec, rows, loop_num); ++ else ++ appendStringInfo(es->str, ++ " (Current loop: running time=%.3f actual rows=0, loop number=%.0f)", ++ total_sec, loop_num); ++ } ++ else ++ appendStringInfo(es->str, ++ " (Current loop: actual rows=%.0f, loop number=%.0f)", ++ rows, loop_num); ++ } ++ else ++ { ++ ExplainPropertyFloat("Actual Loop Number", NULL, loop_num, 0, es); ++ if (es->timing) ++ { ++ if (planstate->instrument->running) ++ { ++ ExplainPropertyFloat("Actual Startup Time", NULL, startup_sec, 3, es); ++ ExplainPropertyFloat("Actual Total Time", NULL, total_sec, 3, es); ++ } ++ else ++ ExplainPropertyFloat("Running Time", NULL, total_sec, 3, es); ++ } ++ ExplainPropertyFloat("Actual Rows", NULL, rows, 0, es); ++ } ++ ExplainCloseGroup("Current loop", "Current loop", true, es); ++ } ++ } ++ + /* in text format, first line ends here */ + if (es->format == EXPLAIN_FORMAT_TEXT) + appendStringInfoChar(es->str, '\n'); +@@ -2051,6 +2144,9 @@ ExplainNode(PlanState *planstate, List *ancestors, + + /* Prepare per-worker buffer/WAL usage */ + if (es->workers_state && (es->buffers || es->wal) && es->verbose) ++ /* Show worker detail after query execution */ ++ if (es->analyze && es->verbose && planstate->worker_instrument ++ && !es->runtime) + { + WorkerInstrumentation *w = planstate->worker_instrument; + +@@ -3015,6 +3111,11 @@ show_hash_info(HashState *hashstate, ExplainState *es) + memcpy(&hinstrument, hashstate->hinstrument, + sizeof(HashInstrumentation)); + ++ if (hashstate->hashtable) ++ { ++ ExecHashAccumInstrumentation(&hinstrument, hashstate->hashtable); ++ } ++ + /* + * Merge results from workers. In the parallel-oblivious case, the + * results from all participants should be identical, except where +@@ -3392,20 +3493,16 @@ show_instrumentation_count(const char *qlabel, int which, + if (!es->analyze || !planstate->instrument) + return; + ++ nloops = planstate->instrument->nloops; + if (which == 2) +- nfiltered = planstate->instrument->nfiltered2; ++ nfiltered = ((nloops > 0) ? planstate->instrument->nfiltered2 / nloops : 0); + else +- nfiltered = planstate->instrument->nfiltered1; ++ nfiltered = ((nloops > 0) ? planstate->instrument->nfiltered1 / nloops : 0); + nloops = planstate->instrument->nloops; + + /* In text mode, suppress zero counts; they're not interesting enough */ + if (nfiltered > 0 || es->format != EXPLAIN_FORMAT_TEXT) +- { +- if (nloops > 0) +- ExplainPropertyFloat(qlabel, NULL, nfiltered / nloops, 0, es); +- else +- ExplainPropertyFloat(qlabel, NULL, 0.0, 0, es); +- } ++ ExplainPropertyFloat(qlabel, NULL, nfiltered, 0, es); + } + + /* +@@ -3977,15 +4074,27 @@ show_modifytable_info(ModifyTableState *mtstate, List *ancestors, + double insert_path; + double other_path; + +- InstrEndLoop(outerPlanState(mtstate)->instrument); ++ if (!es->runtime) ++ InstrEndLoop(outerPlanState(mtstate)->instrument); + + /* count the number of source rows */ +- total = outerPlanState(mtstate)->instrument->ntuples; +- other_path = mtstate->ps.instrument->ntuples2; +- insert_path = total - other_path; ++ other_path = mtstate->ps.instrument->nfiltered2; ++ ++ /* ++ * Insert occurs after extracting row from subplan and in runtime mode ++ * we can appear between these two operations - situation when ++ * total > insert_path + other_path. Therefore we don't know exactly ++ * whether last row from subplan is inserted. ++ * We don't print inserted tuples in runtime mode in order to not print ++ * inconsistent data ++ */ ++ if (!es->runtime) ++ { ++ total = outerPlanState(mtstate)->instrument->ntuples; ++ insert_path = total - other_path; ++ ExplainPropertyFloat("Tuples Inserted", NULL, insert_path, 0, es); ++ } + +- ExplainPropertyFloat("Tuples Inserted", NULL, +- insert_path, 0, es); + ExplainPropertyFloat("Conflicting Tuples", NULL, + other_path, 0, es); + } +diff --git a/src/include/commands/explain.h b/src/include/commands/explain.h +index e94d9e49cf..6a157b8bc0 100644 +--- a/src/include/commands/explain.h ++++ b/src/include/commands/explain.h +@@ -47,6 +47,8 @@ typedef struct ExplainState + bool summary; /* print total planning and execution timing */ + bool settings; /* print modified settings */ + ExplainFormat format; /* output format */ ++ bool runtime; /* print intermediate state of query execution, ++ not after completion */ + /* state for output formatting --- not reset for each new plan tree */ + int indent; /* current indentation level */ + List *grouping_stack; /* format-specific grouping state */ +-- +2.25.1 + diff --git a/pg_query_state.c b/pg_query_state.c index a4a7d1b..21a406b 100644 --- a/pg_query_state.c +++ b/pg_query_state.c @@ -891,7 +891,11 @@ SendBgWorkerPids(void) msg->pids[i++] = current_pid; } +#if PG_VERSION_NUM <= 140000 shm_mq_send(mqh, msg_len, msg, false); +#else + shm_mq_send(mqh, msg_len, msg, false, true); +#endif UnlockShmem(&tag); } diff --git a/signal_handler.c b/signal_handler.c index 2af69fd..20325e7 100644 --- a/signal_handler.c +++ b/signal_handler.c @@ -168,7 +168,11 @@ shm_mq_send_nonblocking(shm_mq_handle *mqh, Size nbytes, const void *data, Size for(i = 0; i < attempts; i++) { +#if PG_VERSION_NUM <= 140000 res = shm_mq_send(mqh, nbytes, data, true); +#else + res = shm_mq_send(mqh, nbytes, data, true, true); +#endif if(res == SHM_MQ_SUCCESS) break; From 8ecd905fcd31a42a3e310eb591fc6fba8edd07fb Mon Sep 17 00:00:00 2001 From: Maxim Orlov Date: Mon, 8 Nov 2021 14:02:02 +0300 Subject: [PATCH 64/91] Issue #36: fix script output varying due to timing. tags: pg_query_state --- expected/corner_cases.out | 15 ++++++++++----- specs/corner_cases.spec | 10 +++++----- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/expected/corner_cases.out b/expected/corner_cases.out index b475e33..725addc 100644 --- a/expected/corner_cases.out +++ b/expected/corner_cases.out @@ -18,8 +18,9 @@ save_own_pid (1 row) +step s2_pg_qs_counterpart: select pg_query_state(counterpart_pid(0)); s2: INFO: state of backend is idle -step s2_pg_qs_counterpart: select pg_query_state(counterpart_pid(0)); +step s2_pg_qs_counterpart: <... completed> pg_query_state -------------- (0 rows) @@ -33,8 +34,9 @@ save_own_pid (1 row) step s1_disable_pg_qs: set pg_query_state.enable to off; +step s2_pg_qs_counterpart: select pg_query_state(counterpart_pid(0)); s2: INFO: query execution statistics disabled -step s2_pg_qs_counterpart: select pg_query_state(counterpart_pid(0)); +step s2_pg_qs_counterpart: <... completed> pg_query_state -------------- (0 rows) @@ -49,8 +51,9 @@ save_own_pid (1 row) +step s2_pg_qs_counterpart: select pg_query_state(counterpart_pid(0)); s2: INFO: state of backend is idle -step s2_pg_qs_counterpart: select pg_query_state(counterpart_pid(0)); +step s2_pg_qs_counterpart: <... completed> pg_query_state -------------- (0 rows) @@ -65,8 +68,9 @@ save_own_pid (1 row) +step s2_pg_qs_counterpart: select pg_query_state(counterpart_pid(0)); s2: INFO: state of backend is idle -step s2_pg_qs_counterpart: select pg_query_state(counterpart_pid(0)); +step s2_pg_qs_counterpart: <... completed> pg_query_state -------------- (0 rows) @@ -81,5 +85,6 @@ save_own_pid (1 row) -step s2_pg_qs_counterpart: select pg_query_state(counterpart_pid(0)); +step s2_pg_qs_counterpart: select pg_query_state(counterpart_pid(0)); +step s2_pg_qs_counterpart: <... completed> ERROR: permission denied diff --git a/specs/corner_cases.spec b/specs/corner_cases.spec index 292b39d..c9f3fde 100644 --- a/specs/corner_cases.spec +++ b/specs/corner_cases.spec @@ -64,12 +64,12 @@ permutation "s1_pg_qs_1" permutation "s1_pg_qs_2" # Check idle -permutation "s1_save_pid" "s2_pg_qs_counterpart" +permutation "s1_save_pid" "s2_pg_qs_counterpart"(*) # Check module disable -permutation "s1_save_pid" "s1_disable_pg_qs" "s2_pg_qs_counterpart" +permutation "s1_save_pid" "s1_disable_pg_qs" "s2_pg_qs_counterpart"(*) # Check roles correspondence -permutation "s1_set_bob" "s2_set_bob" "s1_save_pid" "s2_pg_qs_counterpart" -permutation "s1_set_bob" "s2_set_su" "s1_save_pid" "s2_pg_qs_counterpart" -permutation "s1_set_bob" "s2_set_alice" "s1_save_pid" "s2_pg_qs_counterpart" +permutation "s1_set_bob" "s2_set_bob" "s1_save_pid" "s2_pg_qs_counterpart"(*) +permutation "s1_set_bob" "s2_set_su" "s1_save_pid" "s2_pg_qs_counterpart"(*) +permutation "s1_set_bob" "s2_set_alice" "s1_save_pid" "s2_pg_qs_counterpart"(*) From 4b53e0370ffcc01ed7e5187666f3385d2de1b547 Mon Sep 17 00:00:00 2001 From: Maxim Orlov Date: Mon, 8 Nov 2021 14:35:46 +0300 Subject: [PATCH 65/91] Issue #36: fix script output varying due to timing. Also fix alternative output. tags: pg_query_state --- expected/corner_cases_2.out | 15 ++++++++++----- expected/corner_cases_3.out | 15 ++++++++++----- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/expected/corner_cases_2.out b/expected/corner_cases_2.out index da624f7..df7495f 100644 --- a/expected/corner_cases_2.out +++ b/expected/corner_cases_2.out @@ -13,8 +13,9 @@ step s1_save_pid: select save_own_pid(0); save_own_pid +step s2_pg_qs_counterpart: select pg_query_state(counterpart_pid(0)); s2: INFO: state of backend is idle -step s2_pg_qs_counterpart: select pg_query_state(counterpart_pid(0)); +step s2_pg_qs_counterpart: <... completed> pg_query_state @@ -24,8 +25,9 @@ save_own_pid step s1_disable_pg_qs: set pg_query_state.enable to off; +step s2_pg_qs_counterpart: select pg_query_state(counterpart_pid(0)); s2: INFO: query execution statistics disabled -step s2_pg_qs_counterpart: select pg_query_state(counterpart_pid(0)); +step s2_pg_qs_counterpart: <... completed> pg_query_state @@ -36,8 +38,9 @@ step s1_save_pid: select save_own_pid(0); save_own_pid +step s2_pg_qs_counterpart: select pg_query_state(counterpart_pid(0)); s2: INFO: state of backend is idle -step s2_pg_qs_counterpart: select pg_query_state(counterpart_pid(0)); +step s2_pg_qs_counterpart: <... completed> pg_query_state @@ -48,8 +51,9 @@ step s1_save_pid: select save_own_pid(0); save_own_pid +step s2_pg_qs_counterpart: select pg_query_state(counterpart_pid(0)); s2: INFO: state of backend is idle -step s2_pg_qs_counterpart: select pg_query_state(counterpart_pid(0)); +step s2_pg_qs_counterpart: <... completed> pg_query_state @@ -60,5 +64,6 @@ step s1_save_pid: select save_own_pid(0); save_own_pid -step s2_pg_qs_counterpart: select pg_query_state(counterpart_pid(0)); +step s2_pg_qs_counterpart: select pg_query_state(counterpart_pid(0)); +step s2_pg_qs_counterpart: <... completed> ERROR: permission denied diff --git a/expected/corner_cases_3.out b/expected/corner_cases_3.out index 845db75..8f6a8ef 100644 --- a/expected/corner_cases_3.out +++ b/expected/corner_cases_3.out @@ -13,8 +13,9 @@ step s1_save_pid: select save_own_pid(0); save_own_pid +step s2_pg_qs_counterpart: select pg_query_state(counterpart_pid(0)); s2: INFO: state of backend is idle -step s2_pg_qs_counterpart: select pg_query_state(counterpart_pid(0)); +step s2_pg_qs_counterpart: <... completed> pg_query_state @@ -24,8 +25,9 @@ save_own_pid step s1_disable_pg_qs: set pg_query_state.enable to off; +step s2_pg_qs_counterpart: select pg_query_state(counterpart_pid(0)); s2: INFO: query execution statistics disabled -step s2_pg_qs_counterpart: select pg_query_state(counterpart_pid(0)); +step s2_pg_qs_counterpart: <... completed> pg_query_state @@ -36,8 +38,9 @@ step s1_save_pid: select save_own_pid(0); save_own_pid +step s2_pg_qs_counterpart: select pg_query_state(counterpart_pid(0)); s2: INFO: state of backend is idle -step s2_pg_qs_counterpart: select pg_query_state(counterpart_pid(0)); +step s2_pg_qs_counterpart: <... completed> pg_query_state @@ -48,8 +51,9 @@ step s1_save_pid: select save_own_pid(0); save_own_pid +step s2_pg_qs_counterpart: select pg_query_state(counterpart_pid(0)); s2: INFO: state of backend is idle -step s2_pg_qs_counterpart: select pg_query_state(counterpart_pid(0)); +step s2_pg_qs_counterpart: <... completed> pg_query_state @@ -60,7 +64,8 @@ step s1_save_pid: select save_own_pid(0); save_own_pid -step s2_pg_qs_counterpart: select pg_query_state(counterpart_pid(0)); +step s2_pg_qs_counterpart: select pg_query_state(counterpart_pid(0)); +step s2_pg_qs_counterpart: <... completed> ERROR: permission denied unused step name: s1_enable_pg_qs unused step name: s1_pg_qs_counterpart From 2aeee15863615174a28bca525020dc48a1f36e22 Mon Sep 17 00:00:00 2001 From: Marina Polyakova Date: Wed, 10 Nov 2021 08:56:26 +0300 Subject: [PATCH 66/91] PGPRO-5826: fix build for PostgreSQL 14.1 The previous code did not work for PostgreSQL 14.1 because the signature of the function shm_mq_send was changed only in PostgreSQL 15devel which has PG_VERSION_NUM = 150000. --- pg_query_state.c | 2 +- signal_handler.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pg_query_state.c b/pg_query_state.c index f495f1d..f5a9a4d 100644 --- a/pg_query_state.c +++ b/pg_query_state.c @@ -895,7 +895,7 @@ SendBgWorkerPids(void) msg->pids[i++] = current_pid; } -#if PG_VERSION_NUM <= 140000 +#if PG_VERSION_NUM < 150000 shm_mq_send(mqh, msg_len, msg, false); #else shm_mq_send(mqh, msg_len, msg, false, true); diff --git a/signal_handler.c b/signal_handler.c index 20325e7..d4f1099 100644 --- a/signal_handler.c +++ b/signal_handler.c @@ -168,7 +168,7 @@ shm_mq_send_nonblocking(shm_mq_handle *mqh, Size nbytes, const void *data, Size for(i = 0; i < attempts; i++) { -#if PG_VERSION_NUM <= 140000 +#if PG_VERSION_NUM < 150000 res = shm_mq_send(mqh, nbytes, data, true); #else res = shm_mq_send(mqh, nbytes, data, true, true); From 3445b458376783ec5f52d3175aa657adfa8b46eb Mon Sep 17 00:00:00 2001 From: Daniel Shelepanov Date: Mon, 8 Nov 2021 23:09:22 +0300 Subject: [PATCH 67/91] [PGPRO-4561] Blocking custom signals while handling one Without HOLD_INTERRUPTS()/RESUME_INTERRUPTS() there is a risk of accepting a signal while executing another signal handler which leads to a hcs memory context corruption. tags: pg_query_state --- patches/custom_signals_13.0.patch | 134 +++++++++++++++++++----------- patches/custom_signals_14.0.patch | 86 +++++++++++++++---- patches/custom_signals_15.0.patch | 10 +++ 3 files changed, 165 insertions(+), 65 deletions(-) diff --git a/patches/custom_signals_13.0.patch b/patches/custom_signals_13.0.patch index add965c..266cba8 100644 --- a/patches/custom_signals_13.0.patch +++ b/patches/custom_signals_13.0.patch @@ -1,10 +1,10 @@ diff --git a/src/backend/storage/ipc/procsignal.c b/src/backend/storage/ipc/procsignal.c -index 4fa385b0ece..fc1637a2e28 100644 +index 4fa385b0ece..60854eee386 100644 --- a/src/backend/storage/ipc/procsignal.c +++ b/src/backend/storage/ipc/procsignal.c @@ -88,12 +88,21 @@ typedef struct - (((flags) & (((uint32) 1) << (uint32) (type))) != 0) - + (((flags) & (((uint32) 1) << (uint32) (type))) != 0) + static ProcSignalHeader *ProcSignal = NULL; +#define IsCustomProcSignalReason(reason) \ + ((reason) >= PROCSIG_CUSTOM_1 && (reason) <= PROCSIG_CUSTOM_N) @@ -14,20 +14,20 @@ index 4fa385b0ece..fc1637a2e28 100644 +static ProcSignalHandler_type CustomInterruptHandlers[NUM_CUSTOM_PROCSIGNALS]; + static volatile ProcSignalSlot *MyProcSignalSlot = NULL; - + static bool CheckProcSignal(ProcSignalReason reason); static void CleanupProcSignalState(int status, Datum arg); static void ProcessBarrierPlaceholder(void); - + +static void CheckAndSetCustomSignalInterrupts(void); + /* * ProcSignalShmemSize * Compute space needed for procsignal's shared memory @@ -235,6 +244,36 @@ CleanupProcSignalState(int status, Datum arg) - slot->pss_pid = 0; + slot->pss_pid = 0; } - + +/* + * RegisterCustomProcSignalHandler + * Assign specific handler of custom process signal with new @@ -61,17 +61,17 @@ index 4fa385b0ece..fc1637a2e28 100644 /* * SendProcSignal * Send a signal to a Postgres process -@@ -585,9 +624,64 @@ procsignal_sigusr1_handler(SIGNAL_ARGS) - if (CheckProcSignal(PROCSIG_RECOVERY_CONFLICT_BUFFERPIN)) - RecoveryConflictInterrupt(PROCSIG_RECOVERY_CONFLICT_BUFFERPIN); - +@@ -585,9 +624,71 @@ procsignal_sigusr1_handler(SIGNAL_ARGS) + if (CheckProcSignal(PROCSIG_RECOVERY_CONFLICT_BUFFERPIN)) + RecoveryConflictInterrupt(PROCSIG_RECOVERY_CONFLICT_BUFFERPIN); + + CheckAndSetCustomSignalInterrupts(); + - SetLatch(MyLatch); - - latch_sigusr1_handler(); - - errno = save_errno; + SetLatch(MyLatch); + + latch_sigusr1_handler(); + + errno = save_errno; } + +/* @@ -108,9 +108,15 @@ index 4fa385b0ece..fc1637a2e28 100644 +{ + int i; + -+ /* Check on expiring of custom signals and call its handlers if exist */ ++ /* ++ * This is invoked from ProcessInterrupts(), and since some of the ++ * functions it calls contain CHECK_FOR_INTERRUPTS(), there is a potential ++ * for recursive calls if more signals are received while this runs, so ++ * let's block interrupts until done. ++ */ ++ HOLD_INTERRUPTS(); ++ + for (i = 0; i < NUM_CUSTOM_PROCSIGNALS; i++) -+ { + if (!CustomSignalProcessing[i] && CustomSignalPendings[i]) + { + ProcSignalHandler_type handler; @@ -124,29 +130,66 @@ index 4fa385b0ece..fc1637a2e28 100644 + CustomSignalProcessing[i] = false; + } + } -+ } ++ ++ RESUME_INTERRUPTS(); +} diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c -index 174c72a14bc..0e7366bd58f 100644 +index 7bc03ae4edc..3debd63bd7d 100644 --- a/src/backend/tcop/postgres.c +++ b/src/backend/tcop/postgres.c -@@ -3221,6 +3221,8 @@ ProcessInterrupts(void) - - if (ParallelMessagePending) - HandleParallelMessages(); +@@ -5,6 +5,7 @@ + * + * Portions Copyright (c) 1996-2020, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California ++ * Portions Copyright (c) 2020-2021, Postgres Professional + * + * + * IDENTIFICATION +@@ -74,6 +75,7 @@ + #include "tcop/pquery.h" + #include "tcop/tcopprot.h" + #include "tcop/utility.h" ++#include "utils/builtins.h" + #include "utils/lsyscache.h" + #include "utils/memutils.h" + #include "utils/ps_status.h" +@@ -3231,6 +3233,8 @@ ProcessInterrupts(void) + + if (ParallelMessagePending) + HandleParallelMessages(); + + CheckAndHandleCustomSignals(); } - - + + +@@ -3576,7 +3580,7 @@ process_postgres_switches(int argc, char *argv[], GucContext ctx, + * postmaster/postmaster.c (the option sets should not conflict) and with + * the common help() function in main/main.c. + */ +- while ((flag = getopt(argc, argv, "B:bc:C:D:d:EeFf:h:ijk:lN:nOo:Pp:r:S:sTt:v:W:-:")) != -1) ++ while ((flag = getopt(argc, argv, "B:bc:C:D:d:EeFf:h:ijk:lN:nOo:Pp:r:S:sTt:v:W:Z-:")) != -1) + { + switch (flag) + { +@@ -3712,6 +3716,10 @@ process_postgres_switches(int argc, char *argv[], GucContext ctx, + SetConfigOption("post_auth_delay", optarg, ctx, gucsource); + break; + ++ case 'Z': ++ /* ignored for consistency with the postmaster */ ++ break; ++ + case 'c': + case '-': + { diff --git a/src/include/storage/procsignal.h b/src/include/storage/procsignal.h index 5cb39697f38..c05f60fa719 100644 --- a/src/include/storage/procsignal.h +++ b/src/include/storage/procsignal.h @@ -17,6 +17,8 @@ #include "storage/backendid.h" - - + + +#define NUM_CUSTOM_PROCSIGNALS 64 + /* @@ -158,13 +201,13 @@ index 5cb39697f38..c05f60fa719 100644 { + INVALID_PROCSIGNAL = -1, /* Must be first */ + - PROCSIG_CATCHUP_INTERRUPT, /* sinval catchup interrupt */ - PROCSIG_NOTIFY_INTERRUPT, /* listen/notify interrupt */ - PROCSIG_PARALLEL_MESSAGE, /* message from cooperating parallel backend */ + PROCSIG_CATCHUP_INTERRUPT, /* sinval catchup interrupt */ + PROCSIG_NOTIFY_INTERRUPT, /* listen/notify interrupt */ + PROCSIG_PARALLEL_MESSAGE, /* message from cooperating parallel backend */ @@ -43,6 +47,14 @@ typedef enum - PROCSIG_RECOVERY_CONFLICT_BUFFERPIN, - PROCSIG_RECOVERY_CONFLICT_STARTUP_DEADLOCK, - + PROCSIG_RECOVERY_CONFLICT_BUFFERPIN, + PROCSIG_RECOVERY_CONFLICT_STARTUP_DEADLOCK, + + PROCSIG_CUSTOM_1, + /* + * PROCSIG_CUSTOM_2, @@ -173,34 +216,31 @@ index 5cb39697f38..c05f60fa719 100644 + */ + PROCSIG_CUSTOM_N = PROCSIG_CUSTOM_1 + NUM_CUSTOM_PROCSIGNALS - 1, + - NUM_PROCSIGNALS /* Must be last! */ + NUM_PROCSIGNALS /* Must be last! */ } ProcSignalReason; - + @@ -55,6 +67,8 @@ typedef enum - */ - PROCSIGNAL_BARRIER_PLACEHOLDER = 0 + */ + PROCSIGNAL_BARRIER_PLACEHOLDER = 0 } ProcSignalBarrierType; +/* Handler of custom process signal */ +typedef void (*ProcSignalHandler_type) (void); - + /* * prototypes for functions in procsignal.c @@ -63,12 +77,15 @@ extern Size ProcSignalShmemSize(void); extern void ProcSignalShmemInit(void); - + extern void ProcSignalInit(int pss_idx); +extern ProcSignalReason + RegisterCustomProcSignalHandler(ProcSignalHandler_type handler); extern int SendProcSignal(pid_t pid, ProcSignalReason reason, - BackendId backendId); - + BackendId backendId); + extern uint64 EmitProcSignalBarrier(ProcSignalBarrierType type); extern void WaitForProcSignalBarrier(uint64 generation); extern void ProcessProcSignalBarrier(void); +extern void CheckAndHandleCustomSignals(void); - + extern void procsignal_sigusr1_handler(SIGNAL_ARGS); - --- -2.25.1 - + diff --git a/patches/custom_signals_14.0.patch b/patches/custom_signals_14.0.patch index 9d640cb..d02f2b5 100644 --- a/patches/custom_signals_14.0.patch +++ b/patches/custom_signals_14.0.patch @@ -1,11 +1,19 @@ diff --git a/src/backend/storage/ipc/procsignal.c b/src/backend/storage/ipc/procsignal.c -index defb75a..4245d28 100644 +index defb75aa26a..cd7d44977ca 100644 --- a/src/backend/storage/ipc/procsignal.c +++ b/src/backend/storage/ipc/procsignal.c -@@ -95,6 +95,13 @@ typedef struct - #define BARRIER_CLEAR_BIT(flags, type) \ +@@ -6,6 +6,7 @@ + * + * Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California ++ * Portions Copyright (c) 2021, Postgres Professional + * + * IDENTIFICATION + * src/backend/storage/ipc/procsignal.c +@@ -96,6 +97,13 @@ typedef struct ((flags) &= ~(((uint32) 1) << (uint32) (type))) + static ProcSignalHeader *ProcSignal = NULL; +#define IsCustomProcSignalReason(reason) \ + ((reason) >= PROCSIG_CUSTOM_1 && (reason) <= PROCSIG_CUSTOM_N) + @@ -13,10 +21,10 @@ index defb75a..4245d28 100644 +static bool CustomSignalProcessing[NUM_CUSTOM_PROCSIGNALS]; +static ProcSignalHandler_type CustomInterruptHandlers[NUM_CUSTOM_PROCSIGNALS]; + - static ProcSignalHeader *ProcSignal = NULL; static ProcSignalSlot *MyProcSignalSlot = NULL; -@@ -103,6 +110,8 @@ static void CleanupProcSignalState(int status, Datum arg); + static bool CheckProcSignal(ProcSignalReason reason); +@@ -103,6 +111,8 @@ static void CleanupProcSignalState(int status, Datum arg); static void ResetProcSignalBarrierBits(uint32 flags); static bool ProcessBarrierPlaceholder(void); @@ -25,7 +33,7 @@ index defb75a..4245d28 100644 /* * ProcSignalShmemSize * Compute space needed for procsignal's shared memory -@@ -246,6 +255,36 @@ CleanupProcSignalState(int status, Datum arg) +@@ -246,6 +256,36 @@ CleanupProcSignalState(int status, Datum arg) slot->pss_pid = 0; } @@ -62,7 +70,7 @@ index defb75a..4245d28 100644 /* * SendProcSignal * Send a signal to a Postgres process -@@ -679,7 +718,62 @@ procsignal_sigusr1_handler(SIGNAL_ARGS) +@@ -679,7 +719,72 @@ procsignal_sigusr1_handler(SIGNAL_ARGS) if (CheckProcSignal(PROCSIG_RECOVERY_CONFLICT_BUFFERPIN)) RecoveryConflictInterrupt(PROCSIG_RECOVERY_CONFLICT_BUFFERPIN); @@ -107,6 +115,14 @@ index defb75a..4245d28 100644 +{ + int i; + ++ /* ++ * This is invoked from ProcessInterrupts(), and since some of the ++ * functions it calls contain CHECK_FOR_INTERRUPTS(), there is a potential ++ * for recursive calls if more signals are received while this runs, so ++ * let's block interrupts until done. ++ */ ++ HOLD_INTERRUPTS(); ++ + /* Check on expiring of custom signals and call its handlers if exist */ + for (i = 0; i < NUM_CUSTOM_PROCSIGNALS; i++) + { @@ -124,23 +140,60 @@ index defb75a..4245d28 100644 + } + } + } ++ ++ RESUME_INTERRUPTS(); +} -\ No newline at end of file diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c -index 8cea10c..dd77c98 100644 +index 171f3a95006..e6fe26fb19a 100644 --- a/src/backend/tcop/postgres.c +++ b/src/backend/tcop/postgres.c -@@ -3364,6 +3364,8 @@ ProcessInterrupts(void) - if (ParallelMessagePending) - HandleParallelMessages(); +@@ -5,6 +5,7 @@ + * + * Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California ++ * Portions Copyright (c) 2021, Postgres Professional + * + * + * IDENTIFICATION +@@ -75,6 +76,7 @@ + #include "tcop/pquery.h" + #include "tcop/tcopprot.h" + #include "tcop/utility.h" ++#include "utils/builtins.h" + #include "utils/lsyscache.h" + #include "utils/memutils.h" + #include "utils/ps_status.h" +@@ -3366,6 +3368,8 @@ ProcessInterrupts(void) -+ CheckAndHandleCustomSignals(); -+ if (LogMemoryContextPending) ProcessLogMemoryContextInterrupt(); ++ ++ CheckAndHandleCustomSignals(); } + + +@@ -3711,7 +3715,7 @@ process_postgres_switches(int argc, char *argv[], GucContext ctx, + * postmaster/postmaster.c (the option sets should not conflict) and with + * the common help() function in main/main.c. + */ +- while ((flag = getopt(argc, argv, "B:bc:C:D:d:EeFf:h:ijk:lN:nOPp:r:S:sTt:v:W:-:")) != -1) ++ while ((flag = getopt(argc, argv, "B:bc:C:D:d:EeFf:h:ijk:lN:nOPp:r:S:sTt:v:W:Z-:")) != -1) + { + switch (flag) + { +@@ -3843,6 +3847,10 @@ process_postgres_switches(int argc, char *argv[], GucContext ctx, + SetConfigOption("post_auth_delay", optarg, ctx, gucsource); + break; + ++ case 'Z': ++ /* ignored for consistency with the postmaster */ ++ break; ++ + case 'c': + case '-': + { diff --git a/src/include/storage/procsignal.h b/src/include/storage/procsignal.h -index eec186b..74af186 100644 +index eec186be2ee..74af186bf53 100644 --- a/src/include/storage/procsignal.h +++ b/src/include/storage/procsignal.h @@ -17,6 +17,8 @@ @@ -201,6 +254,3 @@ index eec186b..74af186 100644 extern void procsignal_sigusr1_handler(SIGNAL_ARGS); --- -2.25.1 - diff --git a/patches/custom_signals_15.0.patch b/patches/custom_signals_15.0.patch index 9d640cb..7678dbe 100644 --- a/patches/custom_signals_15.0.patch +++ b/patches/custom_signals_15.0.patch @@ -107,6 +107,14 @@ index defb75a..4245d28 100644 +{ + int i; + ++ /* ++ * This is invoked from ProcessInterrupts(), and since some of the ++ * functions it calls contain CHECK_FOR_INTERRUPTS(), there is a potential ++ * for recursive calls if more signals are received while this runs, so ++ * let's block interrupts until done. ++ */ ++ HOLD_INTERRUPTS(); ++ + /* Check on expiring of custom signals and call its handlers if exist */ + for (i = 0; i < NUM_CUSTOM_PROCSIGNALS; i++) + { @@ -124,6 +132,8 @@ index defb75a..4245d28 100644 + } + } + } ++ ++ RESUME_INTERRUPTS(); +} \ No newline at end of file diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c From 45f8c63a12b41907b326592c30c583ce7ec38301 Mon Sep 17 00:00:00 2001 From: Anton Voloshin Date: Wed, 20 Jul 2022 13:25:22 +0300 Subject: [PATCH 68/91] adapt pg_query_state for upcoming PostgreSQL 15 1. Only call RequestAddinShmemSpace from within our implementation of shmem_request_hook (as required after commit 4f2400cb3 in PostgreSQL 15). 2. While we are here, remove _PG_fini, as it is now officially dead after commit ab02d702e in PostgreSQL 15. --- pg_query_state.c | 35 +++++++++++++++-------------------- 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/pg_query_state.c b/pg_query_state.c index f5a9a4d..d499902 100644 --- a/pg_query_state.c +++ b/pg_query_state.c @@ -48,7 +48,6 @@ static ExecutorFinish_hook_type prev_ExecutorFinish = NULL; static shmem_startup_hook_type prev_shmem_startup_hook = NULL; void _PG_init(void); -void _PG_fini(void); /* hooks defined in this module */ static void qs_ExecutorStart(QueryDesc *queryDesc, int eflags); @@ -179,6 +178,11 @@ pg_qs_shmem_startup(void) module_initialized = true; } +#if PG_VERSION_NUM >= 150000 +static shmem_request_hook_type prev_shmem_request_hook = NULL; +static void pg_qs_shmem_request(void); +#endif + /* * Module load callback */ @@ -188,12 +192,12 @@ _PG_init(void) if (!process_shared_preload_libraries_in_progress) return; - /* - * Request additional shared resources. (These are no-ops if we're not in - * the postmaster process.) We'll allocate or attach to the shared - * resources in qs_shmem_startup(). - */ +#if PG_VERSION_NUM >= 150000 + prev_shmem_request_hook = shmem_request_hook; + shmem_request_hook = pg_qs_shmem_request; +#else RequestAddinShmemSpace(pg_qs_shmem_size()); +#endif /* Register interrupt on custom signal of polling query state */ UserIdPollReason = RegisterCustomProcSignalHandler(SendCurrentUserId); @@ -252,22 +256,13 @@ _PG_init(void) shmem_startup_hook = pg_qs_shmem_startup; } -/* - * Module unload callback - */ -void -_PG_fini(void) +static void +pg_qs_shmem_request(void) { - module_initialized = false; - - /* clear global state */ - list_free(QueryDescStack); + if (prev_shmem_request_hook) + prev_shmem_request_hook(); - /* Uninstall hooks. */ - ExecutorStart_hook = prev_ExecutorStart; - ExecutorRun_hook = prev_ExecutorRun; - ExecutorFinish_hook = prev_ExecutorFinish; - shmem_startup_hook = prev_shmem_startup_hook; + RequestAddinShmemSpace(pg_qs_shmem_size()); } /* From 410cf8f5c872087bf96b47d2383fda2b5094e54d Mon Sep 17 00:00:00 2001 From: Yura Sokolov Date: Mon, 8 Aug 2022 12:50:55 +0300 Subject: [PATCH 69/91] fix compilation on pre-15 --- pg_query_state.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pg_query_state.c b/pg_query_state.c index d499902..dabaa71 100644 --- a/pg_query_state.c +++ b/pg_query_state.c @@ -256,6 +256,7 @@ _PG_init(void) shmem_startup_hook = pg_qs_shmem_startup; } +#if PG_VERSION_NUM >= 150000 static void pg_qs_shmem_request(void) { @@ -264,6 +265,7 @@ pg_qs_shmem_request(void) RequestAddinShmemSpace(pg_qs_shmem_size()); } +#endif /* * ExecutorStart hook: From 6fa1219fa6b7f21031a3e92dc9c565a549272336 Mon Sep 17 00:00:00 2001 From: Ekaterina Sokolova Date: Mon, 31 Oct 2022 16:05:45 +0300 Subject: [PATCH 70/91] [PGPRO-6693] Checking the result of shm_mq_send (according to Svace). Tags: pg_query_state. --- pg_query_state.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pg_query_state.c b/pg_query_state.c index dabaa71..ba79050 100644 --- a/pg_query_state.c +++ b/pg_query_state.c @@ -864,6 +864,7 @@ SendBgWorkerPids(void) int i; shm_mq_handle *mqh; LOCKTAG tag; + shm_mq_result result; LockShmem(&tag, PG_QS_SND_KEY); @@ -893,10 +894,15 @@ SendBgWorkerPids(void) } #if PG_VERSION_NUM < 150000 - shm_mq_send(mqh, msg_len, msg, false); + result = shm_mq_send(mqh, msg_len, msg, false); #else - shm_mq_send(mqh, msg_len, msg, false, true); + result = shm_mq_send(mqh, msg_len, msg, false, true); #endif + + /* Check for failure. */ + if(result == SHM_MQ_DETACHED) + elog(WARNING, "could not send message queue to shared-memory queue: receiver has been detached"); + UnlockShmem(&tag); } From 484600c9073d4fec4d24d829655d4c792be23121 Mon Sep 17 00:00:00 2001 From: Ekaterina Sokolova Date: Fri, 11 Nov 2022 09:54:51 +0300 Subject: [PATCH 71/91] [6693] Refactoring source code of pg_query_state. Tags: pg_query_state. --- pg_query_state.c | 40 ++++++++++++++++++++-------------------- signal_handler.c | 12 ++++++------ 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/pg_query_state.c b/pg_query_state.c index ba79050..2c8a917 100644 --- a/pg_query_state.c +++ b/pg_query_state.c @@ -101,10 +101,10 @@ static List *GetRemoteBackendQueryStates(PGPROC *leader, ExplainFormat format); /* Shared memory variables */ -shm_toc *toc = NULL; +shm_toc *toc = NULL; RemoteUserIdResult *counterpart_userid = NULL; -pg_qs_params *params = NULL; -shm_mq *mq = NULL; +pg_qs_params *params = NULL; +shm_mq *mq = NULL; /* * Estimate amount of shared memory needed. @@ -208,7 +208,7 @@ _PG_init(void) || UserIdPollReason == INVALID_PROCSIGNAL) { ereport(WARNING, (errcode(ERRCODE_INSUFFICIENT_RESOURCES), - errmsg("pg_query_state isn't loaded: insufficient custom ProcSignal slots"))); + errmsg("pg_query_state isn't loaded: insufficient custom ProcSignal slots"))); return; } @@ -435,7 +435,7 @@ deserialize_stack(char *src, int stack_depth) { List *result = NIL; char *curr_ptr = src; - int i; + int i; for (i = 0; i < stack_depth; i++) { @@ -599,10 +599,10 @@ pg_query_state(PG_FUNCTION_ARGS) /* print warnings if exist */ if (msg->warnings & TIMINIG_OFF_WARNING) ereport(WARNING, (errcode(ERRCODE_WARNING), - errmsg("timing statistics disabled"))); + errmsg("timing statistics disabled"))); if (msg->warnings & BUFFERS_OFF_WARNING) ereport(WARNING, (errcode(ERRCODE_WARNING), - errmsg("buffers statistics disabled"))); + errmsg("buffers statistics disabled"))); oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx); @@ -864,7 +864,7 @@ SendBgWorkerPids(void) int i; shm_mq_handle *mqh; LOCKTAG tag; - shm_mq_result result; + shm_mq_result result; LockShmem(&tag, PG_QS_SND_KEY); @@ -959,10 +959,10 @@ GetRemoteBackendWorkers(PGPROC *proc) signal_error: ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), - errmsg("invalid send signal"))); + errmsg("invalid send signal"))); mq_error: ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), - errmsg("error in message queue data transmitting"))); + errmsg("error in message queue data transmitting"))); return NIL; } @@ -980,12 +980,12 @@ static shm_mq_result receive_msg_by_parts(shm_mq_handle *mqh, Size *total, void **datap, int64 timeout, int *rc, bool nowait) { - shm_mq_result mq_receive_result; - shm_mq_msg *buff; - int offset; - Size *expected; - Size expected_data; - Size len; + shm_mq_result mq_receive_result; + shm_mq_msg *buff; + int offset; + Size *expected; + Size expected_data; + Size len; /* Get the expected number of bytes in message */ mq_receive_result = shm_mq_receive(mqh, &len, (void **) &expected, nowait); @@ -1113,7 +1113,7 @@ GetRemoteBackendQueryStates(PGPROC *leader, mqh = shm_mq_attach(mq, NULL, NULL); elog(DEBUG1, "Wait response from leader %d", leader->pid); mq_receive_result = receive_msg_by_parts(mqh, &len, (void **) &msg, - 0, NULL, false); + 0, NULL, false); if (mq_receive_result != SHM_MQ_SUCCESS) goto mq_error; if (msg->reqid != reqid) @@ -1132,7 +1132,7 @@ GetRemoteBackendQueryStates(PGPROC *leader, */ foreach(iter, alive_procs) { - PGPROC *proc = (PGPROC *) lfirst(iter); + PGPROC *proc = (PGPROC *) lfirst(iter); /* prepare message queue to transfer data */ elog(DEBUG1, "Wait response from worker %d", proc->pid); @@ -1172,7 +1172,7 @@ GetRemoteBackendQueryStates(PGPROC *leader, signal_error: ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), - errmsg("invalid send signal"))); + errmsg("invalid send signal"))); mq_error: #if PG_VERSION_NUM < 100000 shm_mq_detach(mq); @@ -1180,7 +1180,7 @@ GetRemoteBackendQueryStates(PGPROC *leader, shm_mq_detach(mqh); #endif ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), - errmsg("error in message queue data transmitting"))); + errmsg("error in message queue data transmitting"))); return NIL; } diff --git a/signal_handler.c b/signal_handler.c index d4f1099..c8f2950 100644 --- a/signal_handler.c +++ b/signal_handler.c @@ -224,12 +224,12 @@ send_msg_by_parts(shm_mq_handle *mqh, Size nbytes, const void *data) void SendQueryState(void) { - shm_mq_handle *mqh; - instr_time start_time; - instr_time cur_time; - int64 delay = MAX_SND_TIMEOUT; - int reqid = params->reqid; - LOCKTAG tag; + shm_mq_handle *mqh; + instr_time start_time; + instr_time cur_time; + int64 delay = MAX_SND_TIMEOUT; + int reqid = params->reqid; + LOCKTAG tag; INSTR_TIME_SET_CURRENT(start_time); From 5ba877945f712a6bcdec29b905effef0867ba46b Mon Sep 17 00:00:00 2001 From: Marina Polyakova Date: Mon, 21 Nov 2022 16:20:39 +0300 Subject: [PATCH 72/91] Fix compiler warnings due to new checks in PostgreSQL 16 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See the commit 0fe954c28584169938e5c0738cfaa9930ce77577 (Add -Wshadow=compatible-local to the standard compilation flags) in PostgreSQL 16. pg_query_state.c: In function ‘pg_query_state’: pg_query_state.c:615:66: warning: declaration of ‘msg’ shadows a previous local [-Wshadow=compatible-local] 615 | shm_mq_msg *msg = (shm_mq_msg *) lfirst(i); | ^~~ pg_query_state.c:489:42: note: shadowed declaration is here 489 | shm_mq_msg *msg; | ^~~ pg_query_state.c: In function ‘GetRemoteBackendWorkers’: pg_query_state.c:946:25: warning: declaration of ‘proc’ shadows a parameter [-Wshadow=compatible-local] 946 | PGPROC *proc = BackendPidGetProc(pid); | ^~~~ pg_query_state.c:913:33: note: shadowed declaration is here 913 | GetRemoteBackendWorkers(PGPROC *proc) | ~~~~~~~~^~~~ --- pg_query_state.c | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/pg_query_state.c b/pg_query_state.c index 2c8a917..3419d64 100644 --- a/pg_query_state.c +++ b/pg_query_state.c @@ -612,17 +612,18 @@ pg_query_state(PG_FUNCTION_ARGS) foreach(i, msgs) { List *qs_stack; - shm_mq_msg *msg = (shm_mq_msg *) lfirst(i); + shm_mq_msg *current_msg = (shm_mq_msg *) lfirst(i); proc_state *p_state = (proc_state *) palloc(sizeof(proc_state)); - if (msg->result_code != QS_RETURNED) + if (current_msg->result_code != QS_RETURNED) continue; - AssertState(msg->result_code == QS_RETURNED); + AssertState(current_msg->result_code == QS_RETURNED); - qs_stack = deserialize_stack(msg->stack, msg->stack_depth); + qs_stack = deserialize_stack(current_msg->stack, + current_msg->stack_depth); - p_state->proc = msg->proc; + p_state->proc = current_msg->proc; p_state->stack = qs_stack; p_state->frame_index = 0; p_state->frame_cursor = list_head(qs_stack); @@ -943,10 +944,10 @@ GetRemoteBackendWorkers(PGPROC *proc) for (i = 0; i < msg->number; i++) { pid_t pid = msg->pids[i]; - PGPROC *proc = BackendPidGetProc(pid); - if (!proc || !proc->pid) + PGPROC *current_proc = BackendPidGetProc(pid); + if (!current_proc || !current_proc->pid) continue; - result = lcons(proc, result); + result = lcons(current_proc, result); } #if PG_VERSION_NUM < 100000 From 3b2942dc4b601d1f5bee54243a12021464f3e8b2 Mon Sep 17 00:00:00 2001 From: Marina Polyakova Date: Wed, 14 Dec 2022 09:49:12 +0300 Subject: [PATCH 73/91] Remove AssertState See the commit b1099eca8f38ff5cfaf0901bb91cb6a22f909bc6 (Remove AssertArg and AssertState) in PostgreSQL 16. --- pg_query_state.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pg_query_state.c b/pg_query_state.c index 3419d64..a5a3f47 100644 --- a/pg_query_state.c +++ b/pg_query_state.c @@ -618,7 +618,7 @@ pg_query_state(PG_FUNCTION_ARGS) if (current_msg->result_code != QS_RETURNED) continue; - AssertState(current_msg->result_code == QS_RETURNED); + Assert(current_msg->result_code == QS_RETURNED); qs_stack = deserialize_stack(current_msg->stack, current_msg->stack_depth); @@ -890,7 +890,7 @@ SendBgWorkerPids(void) { pid_t current_pid = lfirst_int(iter); - AssertState(current_pid > 0); + Assert(current_pid > 0); msg->pids[i++] = current_pid; } From 93afc916dfccdb85b2d8f6604690f2348f2fd0c7 Mon Sep 17 00:00:00 2001 From: Vyacheslav Makarov Date: Thu, 26 Jan 2023 11:36:30 +0300 Subject: [PATCH 74/91] [PGPRO-7010] - Revision of docker tests for pg_query_state under version 15 tags: pg_query_state --- .travis.yml | 2 ++ patches/custom_signals_15.0.patch | 31 +++++++++++++++--------------- patches/runtime_explain_15.0.patch | 20 +++++++++---------- tests/test_cases.py | 28 +++++++++++++++++---------- 4 files changed, 46 insertions(+), 35 deletions(-) diff --git a/.travis.yml b/.travis.yml index 697aba2..e311dbc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,6 +22,8 @@ notifications: on_failure: always env: + - PG_VERSION=15 LEVEL=hardcore USE_TPCDS=0 + - PG_VERSION=15 - PG_VERSION=14 LEVEL=hardcore USE_TPCDS=0 - PG_VERSION=14 - PG_VERSION=13 LEVEL=hardcore USE_TPCDS=0 diff --git a/patches/custom_signals_15.0.patch b/patches/custom_signals_15.0.patch index 7678dbe..4e99c69 100644 --- a/patches/custom_signals_15.0.patch +++ b/patches/custom_signals_15.0.patch @@ -2,10 +2,10 @@ diff --git a/src/backend/storage/ipc/procsignal.c b/src/backend/storage/ipc/proc index defb75a..4245d28 100644 --- a/src/backend/storage/ipc/procsignal.c +++ b/src/backend/storage/ipc/procsignal.c -@@ -95,6 +95,13 @@ typedef struct +@@ -96,6 +96,13 @@ typedef struct #define BARRIER_CLEAR_BIT(flags, type) \ ((flags) &= ~(((uint32) 1) << (uint32) (type))) - + +#define IsCustomProcSignalReason(reason) \ + ((reason) >= PROCSIG_CUSTOM_1 && (reason) <= PROCSIG_CUSTOM_N) + @@ -15,20 +15,20 @@ index defb75a..4245d28 100644 + static ProcSignalHeader *ProcSignal = NULL; static ProcSignalSlot *MyProcSignalSlot = NULL; - -@@ -103,6 +110,8 @@ static void CleanupProcSignalState(int status, Datum arg); + +@@ -103,6 +110,8 @@ static bool CheckProcSignal(ProcSignalReason reason); + static void CleanupProcSignalState(int status, Datum arg); static void ResetProcSignalBarrierBits(uint32 flags); - static bool ProcessBarrierPlaceholder(void); - + +static void CheckAndSetCustomSignalInterrupts(void); + /* * ProcSignalShmemSize - * Compute space needed for procsignal's shared memory + * Compute space needed for ProcSignal's shared memory @@ -246,6 +255,36 @@ CleanupProcSignalState(int status, Datum arg) slot->pss_pid = 0; } - + +/* + * RegisterCustomProcSignalHandler + * Assign specific handler of custom process signal with new @@ -62,7 +62,7 @@ index defb75a..4245d28 100644 /* * SendProcSignal * Send a signal to a Postgres process -@@ -679,7 +718,62 @@ procsignal_sigusr1_handler(SIGNAL_ARGS) +@@ -675,7 +714,72 @@ procsignal_sigusr1_handler(SIGNAL_ARGS) if (CheckProcSignal(PROCSIG_RECOVERY_CONFLICT_BUFFERPIN)) RecoveryConflictInterrupt(PROCSIG_RECOVERY_CONFLICT_BUFFERPIN); @@ -135,12 +135,12 @@ index defb75a..4245d28 100644 + + RESUME_INTERRUPTS(); +} -\ No newline at end of file ++ diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c index 8cea10c..dd77c98 100644 --- a/src/backend/tcop/postgres.c +++ b/src/backend/tcop/postgres.c -@@ -3364,6 +3364,8 @@ ProcessInterrupts(void) +@@ -3402,6 +3402,8 @@ ProcessInterrupts(void) if (ParallelMessagePending) HandleParallelMessages(); @@ -186,16 +186,17 @@ index eec186b..74af186 100644 NUM_PROCSIGNALS /* Must be last! */ } ProcSignalReason; -@@ -56,6 +68,8 @@ typedef enum - */ - PROCSIGNAL_BARRIER_PLACEHOLDER = 0 +@@ -51,6 +63,9 @@ typedef enum + { + PROCSIGNAL_BARRIER_SMGRRELEASE /* ask smgr to close files */ } ProcSignalBarrierType; ++ +/* Handler of custom process signal */ +typedef void (*ProcSignalHandler_type) (void); /* * prototypes for functions in procsignal.c -@@ -64,12 +78,15 @@ extern Size ProcSignalShmemSize(void); +@@ -59,12 +74,15 @@ extern Size ProcSignalShmemSize(void); extern void ProcSignalShmemInit(void); extern void ProcSignalInit(int pss_idx); diff --git a/patches/runtime_explain_15.0.patch b/patches/runtime_explain_15.0.patch index 7904cc2..adab6dc 100644 --- a/patches/runtime_explain_15.0.patch +++ b/patches/runtime_explain_15.0.patch @@ -2,7 +2,7 @@ diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c index 10644dfac4..7106ed4257 100644 --- a/src/backend/commands/explain.c +++ b/src/backend/commands/explain.c -@@ -984,14 +984,36 @@ report_triggers(ResultRelInfo *rInfo, bool show_relname, ExplainState *es) +@@ -990,14 +990,36 @@ report_triggers(ResultRelInfo *rInfo, bool show_relname, ExplainState *es) char *relname; char *conname = NULL; @@ -40,7 +40,7 @@ index 10644dfac4..7106ed4257 100644 continue; ExplainOpenGroup("Trigger", NULL, true, es); -@@ -1017,9 +1039,9 @@ report_triggers(ResultRelInfo *rInfo, bool show_relname, ExplainState *es) +@@ -1023,9 +1045,9 @@ report_triggers(ResultRelInfo *rInfo, bool show_relname, ExplainState *es) appendStringInfo(es->str, " on %s", relname); if (es->timing) appendStringInfo(es->str, ": time=%.3f calls=%.0f\n", @@ -52,7 +52,7 @@ index 10644dfac4..7106ed4257 100644 } else { -@@ -1028,9 +1050,8 @@ report_triggers(ResultRelInfo *rInfo, bool show_relname, ExplainState *es) +@@ -1034,9 +1056,8 @@ report_triggers(ResultRelInfo *rInfo, bool show_relname, ExplainState *es) ExplainPropertyText("Constraint Name", conname, es); ExplainPropertyText("Relation", relname, es); if (es->timing) @@ -64,7 +64,7 @@ index 10644dfac4..7106ed4257 100644 } if (conname) -@@ -1600,8 +1621,11 @@ ExplainNode(PlanState *planstate, List *ancestors, +@@ -1609,8 +1630,11 @@ ExplainNode(PlanState *planstate, List *ancestors, * instrumentation results the user didn't ask for. But we do the * InstrEndLoop call anyway, if possible, to reduce the number of cases * auto_explain has to contend with. @@ -77,7 +77,7 @@ index 10644dfac4..7106ed4257 100644 InstrEndLoop(planstate->instrument); if (es->analyze && -@@ -1636,7 +1660,7 @@ ExplainNode(PlanState *planstate, List *ancestors, +@@ -1645,7 +1669,7 @@ ExplainNode(PlanState *planstate, List *ancestors, ExplainPropertyFloat("Actual Loops", NULL, nloops, 0, es); } } @@ -86,7 +86,7 @@ index 10644dfac4..7106ed4257 100644 { if (es->format == EXPLAIN_FORMAT_TEXT) appendStringInfoString(es->str, " (never executed)"); -@@ -1652,6 +1676,75 @@ ExplainNode(PlanState *planstate, List *ancestors, +@@ -1661,6 +1685,75 @@ ExplainNode(PlanState *planstate, List *ancestors, } } @@ -162,7 +162,7 @@ index 10644dfac4..7106ed4257 100644 /* in text format, first line ends here */ if (es->format == EXPLAIN_FORMAT_TEXT) appendStringInfoChar(es->str, '\n'); -@@ -2051,6 +2144,9 @@ ExplainNode(PlanState *planstate, List *ancestors, +@@ -2068,6 +2161,9 @@ ExplainNode(PlanState *planstate, List *ancestors, /* Prepare per-worker buffer/WAL usage */ if (es->workers_state && (es->buffers || es->wal) && es->verbose) @@ -172,7 +172,7 @@ index 10644dfac4..7106ed4257 100644 { WorkerInstrumentation *w = planstate->worker_instrument; -@@ -3015,6 +3111,11 @@ show_hash_info(HashState *hashstate, ExplainState *es) +@@ -3032,6 +3128,11 @@ show_hash_info(HashState *hashstate, ExplainState *es) memcpy(&hinstrument, hashstate->hinstrument, sizeof(HashInstrumentation)); @@ -184,7 +184,7 @@ index 10644dfac4..7106ed4257 100644 /* * Merge results from workers. In the parallel-oblivious case, the * results from all participants should be identical, except where -@@ -3392,20 +3493,16 @@ show_instrumentation_count(const char *qlabel, int which, +@@ -3412,20 +3513,16 @@ show_instrumentation_count(const char *qlabel, int which, if (!es->analyze || !planstate->instrument) return; @@ -209,7 +209,7 @@ index 10644dfac4..7106ed4257 100644 } /* -@@ -3977,15 +4074,27 @@ show_modifytable_info(ModifyTableState *mtstate, List *ancestors, +@@ -4028,15 +4125,27 @@ show_modifytable_info(ModifyTableState *mtstate, List *ancestors, double insert_path; double other_path; diff --git a/tests/test_cases.py b/tests/test_cases.py index 1750bb1..b4bbbb3 100644 --- a/tests/test_cases.py +++ b/tests/test_cases.py @@ -45,7 +45,7 @@ def test_simple_query(config): acon1, acon2 = common.n_async_connect(config, 2) query = 'select count(*) from foo join bar on foo.c1=bar.c1 and unlock_if_eq_1(foo.c1)=bar.c1' expected = r"""Aggregate \(Current loop: actual rows=\d+, loop number=1\) - -> Hash Join \(Current loop: actual rows=62473, loop number=1\) + -> Hash Join \(Current loop: actual rows=\d+, loop number=1\) Hash Cond: \(foo.c1 = bar.c1\) Join Filter: \(unlock_if_eq_1\(foo.c1\) = bar.c1\) -> Seq Scan on foo \(Current loop: actual rows=\d+, loop number=1\) @@ -111,10 +111,10 @@ def test_nested_call(config): expected_nested = r"""Result \(Current loop: actual rows=0, loop number=1\) InitPlan 1 \(returns \$0\) -> Aggregate \(Current loop: actual rows=0, loop number=1\) - -> Hash Join \(Current loop: actual rows=62473, loop number=1\) + -> Hash Join \(Current loop: actual rows=\d+, loop number=1\) Hash Cond: \(foo.c1 = bar.c1\) Join Filter: \(unlock_if_eq_1\(foo.c1\) = bar.c1\) - -> Seq Scan on foo \(Current loop: actual rows=1000000, loop number=1\) + -> Seq Scan on foo \(Current loop: actual rows=\d+, loop number=1\) -> Hash \(Current loop: actual rows=500000, loop number=1\) Buckets: \d+ Batches: \d+ Memory Usage: \d+kB -> Seq Scan on bar \(Current loop: actual rows=\d+, loop number=1\)""" @@ -232,7 +232,7 @@ def test_costs(config): -> Hash Join \(cost=\d+.\d+..\d+.\d+ rows=\d+ width=0\) \(Current loop: actual rows=\d+, loop number=1\) Hash Cond: \(foo.c1 = bar.c1\) Join Filter: \(unlock_if_eq_1\(foo.c1\) = bar.c1\) - -> Seq Scan on foo \(cost=0.00..\d+.\d+ rows=\d+ width=4\) \(Current loop: actual rows=1000000, loop number=1\) + -> Seq Scan on foo \(cost=0.00..\d+.\d+ rows=\d+ width=4\) \(Current loop: actual rows=\d+, loop number=1\) -> Hash \(cost=\d+.\d+..\d+.\d+ rows=\d+ width=4\) \(Current loop: actual rows=500000, loop number=1\) Buckets: \d+ Batches: \d+ Memory Usage: \d+kB -> Seq Scan on bar \(cost=0.00..\d+.\d+ rows=\d+ width=4\) \(Current loop: actual rows=\d+, loop number=1\)""" @@ -249,25 +249,33 @@ def test_buffers(config): acon1, acon2 = common.n_async_connect(config, 2) query = 'select count(*) from foo join bar on foo.c1=bar.c1 and unlock_if_eq_1(foo.c1)=bar.c1' - expected = r"""Aggregate \(Current loop: actual rows=0, loop number=1\) + temporary = r"""Aggregate \(Current loop: actual rows=0, loop number=1\) -> Hash Join \(Current loop: actual rows=\d+, loop number=1\) Hash Cond: \(foo.c1 = bar.c1\) - Join Filter: \(unlock_if_eq_1\(foo.c1\) = bar.c1\) - Buffers: shared hit=\d+, temp read=\d+ written=\d+ - -> Seq Scan on foo \(Current loop: actual rows=1000000, loop number=1\) + Join Filter: \(unlock_if_eq_1\(foo.c1\) = bar.c1\)""" + expected = temporary + expected_15 = temporary + expected += r""" + Buffers: shared hit=\d+, temp read=\d+ written=\d+""" + expected_15 += r""" + Buffers: shared hit=\d+, temp written=\d+""" + temporary = r""" + -> Seq Scan on foo \(Current loop: actual rows=\d+, loop number=1\) Buffers: [^\n]* -> Hash \(Current loop: actual rows=500000, loop number=1\) Buckets: \d+ Batches: \d+ Memory Usage: \d+kB Buffers: shared hit=\d+, temp written=\d+ -> Seq Scan on bar \(Current loop: actual rows=\d+, loop number=1\) Buffers: .*""" + expected += temporary + expected_15 += temporary common.set_guc(acon1, 'pg_query_state.enable_buffers', 'on') qs, notices = common.onetime_query_state_locks(config, acon1, acon2, query, {'buffers': True}) assert len(qs) == 2 - assert re.match(expected, qs[0][3]) + assert (re.match(expected, qs[0][3]) or re.match(expected_15, qs[0][3])) assert len(notices) == 0 common.n_close((acon1, acon2)) @@ -282,7 +290,7 @@ def test_timing(config): -> Hash Join \(Current loop: actual time=\d+.\d+..\d+.\d+ rows=\d+, loop number=1\) Hash Cond: \(foo.c1 = bar.c1\) Join Filter: \(unlock_if_eq_1\(foo.c1\) = bar.c1\) - -> Seq Scan on foo \(Current loop: actual time=\d+.\d+..\d+.\d+ rows=1000000, loop number=1\) + -> Seq Scan on foo \(Current loop: actual time=\d+.\d+..\d+.\d+ rows=\d+, loop number=1\) -> Hash \(Current loop: actual time=\d+.\d+..\d+.\d+ rows=500000, loop number=1\) Buckets: \d+ Batches: \d+ Memory Usage: \d+kB -> Seq Scan on bar \(Current loop: actual time=\d+.\d+..\d+.\d+ rows=\d+, loop number=1\)""" From c9f90cb47d0a5ed884cd7f1943493777524750eb Mon Sep 17 00:00:00 2001 From: Maxim Orlov Date: Mon, 15 May 2023 11:31:53 +0300 Subject: [PATCH 75/91] fixes in Makefile In order to be ready to the meson build and be able to support parallel installcheck do the following: - fix use of USE_MODULE_DB - isolation target should be included from the gloabl makefile - reorder vars to make it easy to read + fix out of source check target tags: pg_query_state --- Makefile | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index ca9faab..4468c51 100644 --- a/Makefile +++ b/Makefile @@ -8,10 +8,20 @@ EXTVERSION = 1.1 DATA = pg_query_state--1.0--1.1.sql DATA_built = $(EXTENSION)--$(EXTVERSION).sql PGFILEDESC = "pg_query_state - facility to track progress of plan execution" -EXTRA_REGRESS_OPTS=--temp-config=$(top_srcdir)/$(subdir)/test.conf + EXTRA_CLEAN = ./isolation_output $(EXTENSION)--$(EXTVERSION).sql \ Dockerfile ./tests/*.pyc ./tmp_stress +ISOLATION = corner_cases +# +# PG11 doesn't support ISOLATION_OPTS variable. We have to use +# "CREATE/DROP EXTENTION" command in spec. +# +# One day, when we'll get rid of PG11, it will be possible to uncomment this +# variable and remove "CREATE EXTENTION" from spec. +# +# ISOLATION_OPTS = --load-extension=pg_query_state + ifdef USE_PGXS PG_CONFIG ?= pg_config PGXS := $(shell $(PG_CONFIG) --pgxs) @@ -21,11 +31,17 @@ subdir = contrib/pg_query_state top_builddir = ../.. include $(top_builddir)/src/Makefile.global include $(top_srcdir)/contrib/contrib-global.mk +# need this to provide make check in case of "in source" build +EXTRA_REGRESS_OPTS=--temp-config=$(top_srcdir)/$(subdir)/test.conf endif $(EXTENSION)--$(EXTVERSION).sql: init.sql cat $^ > $@ +# +# Make conditional targets to save backward compatibility with PG11. +# +ifeq ($(MAJORVERSION),11) ISOLATIONCHECKS = corner_cases check: isolationcheck @@ -46,3 +62,4 @@ submake-isolation: $(MAKE) -C $(top_builddir)/src/test/isolation all temp-install: EXTRA_INSTALL=contrib/pg_query_state +endif From 26948428d741caf292097e236750bf5e679816de Mon Sep 17 00:00:00 2001 From: Ekaterina Sokolova Date: Tue, 20 Jun 2023 14:36:14 +0300 Subject: [PATCH 76/91] Fix Copyrights. Only for basic files, not patches. --- LICENSE | 2 +- pg_query_state.c | 2 +- pg_query_state.h | 2 +- run_tests.sh | 2 +- signal_handler.c | 2 +- tests/common.py | 2 +- tests/pg_qs_test_runner.py | 2 +- tests/test_cases.py | 2 +- tests/tpcds.py | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/LICENSE b/LICENSE index 1bafabf..5d50c25 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ pg_query_state is released under the PostgreSQL License, a liberal Open Source license, similar to the BSD or MIT licenses. -Copyright (c) 2016-2019, Postgres Professional +Copyright (c) 2016-2023, Postgres Professional Portions Copyright (c) 1996-2019, PostgreSQL Global Development Group Portions Copyright (c) 1994, The Regents of the University of California diff --git a/pg_query_state.c b/pg_query_state.c index a5a3f47..a52e051 100644 --- a/pg_query_state.c +++ b/pg_query_state.c @@ -2,7 +2,7 @@ * pg_query_state.c * Extract information about query state from other backend * - * Copyright (c) 2016-2016, Postgres Professional + * Copyright (c) 2016-2023, Postgres Professional * * contrib/pg_query_state/pg_query_state.c * IDENTIFICATION diff --git a/pg_query_state.h b/pg_query_state.h index 2b13234..9152560 100644 --- a/pg_query_state.h +++ b/pg_query_state.h @@ -2,7 +2,7 @@ * pg_query_state.h * Headers for pg_query_state extension. * - * Copyright (c) 2016-2016, Postgres Professional + * Copyright (c) 2016-2023, Postgres Professional * * IDENTIFICATION * contrib/pg_query_state/pg_query_state.h diff --git a/run_tests.sh b/run_tests.sh index fbf2da1..1c43847 100644 --- a/run_tests.sh +++ b/run_tests.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # -# Copyright (c) 2019, Postgres Professional +# Copyright (c) 2019-2023, Postgres Professional # # supported levels: # * standard diff --git a/signal_handler.c b/signal_handler.c index c8f2950..7e4b602 100644 --- a/signal_handler.c +++ b/signal_handler.c @@ -2,7 +2,7 @@ * signal_handler.c * Collect current query state and send it to requestor in custom signal handler * - * Copyright (c) 2016-2016, Postgres Professional + * Copyright (c) 2016-2023, Postgres Professional * * IDENTIFICATION * contrib/pg_query_state/signal_handler.c diff --git a/tests/common.py b/tests/common.py index ac24e76..c83abb1 100644 --- a/tests/common.py +++ b/tests/common.py @@ -1,6 +1,6 @@ ''' common.py -Copyright (c) 2016-2020, Postgres Professional +Copyright (c) 2016-2023, Postgres Professional ''' import psycopg2 diff --git a/tests/pg_qs_test_runner.py b/tests/pg_qs_test_runner.py index a6e02e9..a0df6a9 100644 --- a/tests/pg_qs_test_runner.py +++ b/tests/pg_qs_test_runner.py @@ -1,6 +1,6 @@ ''' pg_qs_test_runner.py -Copyright (c) 2016-2021, Postgres Professional +Copyright (c) 2016-2023, Postgres Professional ''' import argparse diff --git a/tests/test_cases.py b/tests/test_cases.py index b4bbbb3..c6b0fa2 100644 --- a/tests/test_cases.py +++ b/tests/test_cases.py @@ -1,6 +1,6 @@ ''' test_cases.py -Copyright (c) 2016-2021, Postgres Professional +Copyright (c) 2016-2023, Postgres Professional ''' import json diff --git a/tests/tpcds.py b/tests/tpcds.py index 8ac7183..944b799 100644 --- a/tests/tpcds.py +++ b/tests/tpcds.py @@ -1,6 +1,6 @@ ''' test_cases.py -Copyright (c) 2016-2020, Postgres Professional +Copyright (c) 2016-2023, Postgres Professional ''' import os From e8cde164f9dce99e6783c01dedbad20c6a8f75b2 Mon Sep 17 00:00:00 2001 From: Marina Polyakova Date: Mon, 4 Sep 2023 19:35:56 +0300 Subject: [PATCH 77/91] Rename some support functions for pgstat* views. See the commit 8dfa37b797843a83a5756ea3309055e8953e1a86 (Rename some support functions for pgstat* views.) in PostgreSQL 16. --- pg_query_state.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pg_query_state.c b/pg_query_state.c index a52e051..ab76f7f 100644 --- a/pg_query_state.c +++ b/pg_query_state.c @@ -365,7 +365,11 @@ search_be_status(int pid) for (beid = 1; beid <= pgstat_fetch_stat_numbackends(); beid++) { +#if PG_VERSION_NUM >= 160000 + PgBackendStatus *be_status = pgstat_get_beentry_by_backend_id(beid); +#else PgBackendStatus *be_status = pgstat_fetch_stat_beentry(beid); +#endif if (be_status && be_status->st_procpid == pid) return be_status; From 9597aca40a73a7e3fcfb0983446eaa4fa8d71c97 Mon Sep 17 00:00:00 2001 From: Ekaterina Sokolova Date: Thu, 22 Feb 2024 20:22:28 +0300 Subject: [PATCH 78/91] Module update for PostgreSQL 16. --- .travis.yml | 2 + LICENSE | 2 +- README.md | 15 +- patches/custom_signals_16.0.patch | 229 +++++++++++++++++++++++++ patches/runtime_explain_16.0.patch | 258 +++++++++++++++++++++++++++++ pg_query_state.c | 2 +- pg_query_state.h | 2 +- run_tests.sh | 2 +- signal_handler.c | 2 +- tests/common.py | 2 +- tests/pg_qs_test_runner.py | 2 +- tests/test_cases.py | 2 +- tests/tpcds.py | 2 +- 13 files changed, 512 insertions(+), 10 deletions(-) create mode 100644 patches/custom_signals_16.0.patch create mode 100644 patches/runtime_explain_16.0.patch diff --git a/.travis.yml b/.travis.yml index e311dbc..0983e07 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,6 +22,8 @@ notifications: on_failure: always env: + - PG_VERSION=16 LEVEL=hardcore USE_TPCDS=0 + - PG_VERSION=16 - PG_VERSION=15 LEVEL=hardcore USE_TPCDS=0 - PG_VERSION=15 - PG_VERSION=14 LEVEL=hardcore USE_TPCDS=0 diff --git a/LICENSE b/LICENSE index 5d50c25..7c10525 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ pg_query_state is released under the PostgreSQL License, a liberal Open Source license, similar to the BSD or MIT licenses. -Copyright (c) 2016-2023, Postgres Professional +Copyright (c) 2016-2024, Postgres Professional Portions Copyright (c) 1996-2019, PostgreSQL Global Development Group Portions Copyright (c) 1994, The Regents of the University of California diff --git a/README.md b/README.md index 3ba5eb2..fba15cd 100644 --- a/README.md +++ b/README.md @@ -15,12 +15,25 @@ Using this module there can help in the following things: - overwatch the query execution ## Installation -To install `pg_query_state`, please apply corresponding patches `custom_signal_(PG_VERSION).patch` and `runtime_explain_(PG_VERSION).patch` (or `runtime_explain.patch` for PG version <= 10.0) in `patches/` directory to reqired stable version of PostgreSQL and rebuild PostgreSQL. +To install `pg_query_state`, please apply corresponding patches `custom_signal_(PG_VERSION).patch` and `runtime_explain_(PG_VERSION).patch` (or `runtime_explain.patch` for PG version <= 10.0) from the `patches/` directory to reqired stable version of PostgreSQL and rebuild PostgreSQL. + +To do this, run the following commands from the postgresql directory: +``` +patch -p1 < path_to_pg_query_state_folder/patches/runtime_explain_(PG_VERSION).patch +patch -p1 < path_to_pg_query_state_folder/patches/custom_signal_(PG_VERSION).patch +``` Then execute this in the module's directory: ``` make install USE_PGXS=1 ``` +To execute the command correctly, make sure you have the PATH or PG_CONFIG variable set. +``` +export PATH=path_to_your_bin_folder:$PATH +# or +export PG_CONFIG=path_to_your_bin_folder/pg_config +``` + Add module name to the `shared_preload_libraries` parameter in `postgresql.conf`: ``` shared_preload_libraries = 'pg_query_state' diff --git a/patches/custom_signals_16.0.patch b/patches/custom_signals_16.0.patch new file mode 100644 index 0000000..3a2183f --- /dev/null +++ b/patches/custom_signals_16.0.patch @@ -0,0 +1,229 @@ +diff --git a/src/backend/storage/ipc/procsignal.c b/src/backend/storage/ipc/procsignal.c +index c85cb5cc18..37ae4b3759 100644 +--- a/src/backend/storage/ipc/procsignal.c ++++ b/src/backend/storage/ipc/procsignal.c +@@ -6,6 +6,7 @@ + * + * Portions Copyright (c) 1996-2023, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California ++ * Portions Copyright (c) 2024, Postgres Professional + * + * IDENTIFICATION + * src/backend/storage/ipc/procsignal.c +@@ -97,6 +98,13 @@ typedef struct + #define BARRIER_CLEAR_BIT(flags, type) \ + ((flags) &= ~(((uint32) 1) << (uint32) (type))) + ++#define IsCustomProcSignalReason(reason) \ ++ ((reason) >= PROCSIG_CUSTOM_1 && (reason) <= PROCSIG_CUSTOM_N) ++ ++static bool CustomSignalPendings[NUM_CUSTOM_PROCSIGNALS]; ++static bool CustomSignalProcessing[NUM_CUSTOM_PROCSIGNALS]; ++static ProcSignalHandler_type CustomInterruptHandlers[NUM_CUSTOM_PROCSIGNALS]; ++ + static ProcSignalHeader *ProcSignal = NULL; + static ProcSignalSlot *MyProcSignalSlot = NULL; + +@@ -104,6 +112,8 @@ static bool CheckProcSignal(ProcSignalReason reason); + static void CleanupProcSignalState(int status, Datum arg); + static void ResetProcSignalBarrierBits(uint32 flags); + ++static void CheckAndSetCustomSignalInterrupts(void); ++ + /* + * ProcSignalShmemSize + * Compute space needed for ProcSignal's shared memory +@@ -247,6 +257,36 @@ CleanupProcSignalState(int status, Datum arg) + slot->pss_pid = 0; + } + ++/* ++ * RegisterCustomProcSignalHandler ++ * Assign specific handler of custom process signal with new ++ * ProcSignalReason key. ++ * ++ * This function has to be called in _PG_init function of extensions at the ++ * stage of loading shared preloaded libraries. Otherwise it throws fatal error. ++ * ++ * Return INVALID_PROCSIGNAL if all slots for custom signals are occupied. ++ */ ++ProcSignalReason ++RegisterCustomProcSignalHandler(ProcSignalHandler_type handler) ++{ ++ ProcSignalReason reason; ++ ++ if (!process_shared_preload_libraries_in_progress) ++ ereport(FATAL, (errcode(ERRCODE_INTERNAL_ERROR), ++ errmsg("cannot register custom signal after startup"))); ++ ++ /* Iterate through custom signal slots to find a free one */ ++ for (reason = PROCSIG_CUSTOM_1; reason <= PROCSIG_CUSTOM_N; reason++) ++ if (!CustomInterruptHandlers[reason - PROCSIG_CUSTOM_1]) ++ { ++ CustomInterruptHandlers[reason - PROCSIG_CUSTOM_1] = handler; ++ return reason; ++ } ++ ++ return INVALID_PROCSIGNAL; ++} ++ + /* + * SendProcSignal + * Send a signal to a Postgres process +@@ -682,7 +722,72 @@ procsignal_sigusr1_handler(SIGNAL_ARGS) + if (CheckProcSignal(PROCSIG_RECOVERY_CONFLICT_BUFFERPIN)) + RecoveryConflictInterrupt(PROCSIG_RECOVERY_CONFLICT_BUFFERPIN); + ++ CheckAndSetCustomSignalInterrupts(); ++ + SetLatch(MyLatch); + + errno = save_errno; + } ++ ++/* ++ * Handle receipt of an interrupt indicating any of custom process signals. ++ */ ++static void ++CheckAndSetCustomSignalInterrupts() ++{ ++ ProcSignalReason reason; ++ ++ for (reason = PROCSIG_CUSTOM_1; reason <= PROCSIG_CUSTOM_N; reason++) ++ { ++ if (CheckProcSignal(reason)) ++ { ++ ++ /* set interrupt flags */ ++ InterruptPending = true; ++ CustomSignalPendings[reason - PROCSIG_CUSTOM_1] = true; ++ } ++ } ++ ++ SetLatch(MyLatch); ++} ++ ++/* ++ * CheckAndHandleCustomSignals ++ * Check custom signal flags and call handler assigned to that signal ++ * if it is not NULL ++ * ++ * This function is called within CHECK_FOR_INTERRUPTS if interrupt occurred. ++ */ ++void ++CheckAndHandleCustomSignals(void) ++{ ++ int i; ++ ++ /* ++ * This is invoked from ProcessInterrupts(), and since some of the ++ * functions it calls contain CHECK_FOR_INTERRUPTS(), there is a potential ++ * for recursive calls if more signals are received while this runs, so ++ * let's block interrupts until done. ++ */ ++ HOLD_INTERRUPTS(); ++ ++ /* Check on expiring of custom signals and call its handlers if exist */ ++ for (i = 0; i < NUM_CUSTOM_PROCSIGNALS; i++) ++ { ++ if (!CustomSignalProcessing[i] && CustomSignalPendings[i]) ++ { ++ ProcSignalHandler_type handler; ++ ++ CustomSignalPendings[i] = false; ++ handler = CustomInterruptHandlers[i]; ++ if (handler != NULL) ++ { ++ CustomSignalProcessing[i] = true; ++ handler(); ++ CustomSignalProcessing[i] = false; ++ } ++ } ++ } ++ ++ RESUME_INTERRUPTS(); ++} +diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c +index 36cc99ec9c..a3acce427a 100644 +--- a/src/backend/tcop/postgres.c ++++ b/src/backend/tcop/postgres.c +@@ -3442,6 +3442,8 @@ ProcessInterrupts(void) + if (ParallelMessagePending) + HandleParallelMessages(); + ++ CheckAndHandleCustomSignals(); ++ + if (LogMemoryContextPending) + ProcessLogMemoryContextInterrupt(); + +diff --git a/src/include/storage/procsignal.h b/src/include/storage/procsignal.h +index 2f52100b00..0e31a5771e 100644 +--- a/src/include/storage/procsignal.h ++++ b/src/include/storage/procsignal.h +@@ -6,6 +6,7 @@ + * + * Portions Copyright (c) 1996-2023, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California ++ * Portions Copyright (c) 2024, Postgres Professional + * + * src/include/storage/procsignal.h + * +@@ -17,6 +18,8 @@ + #include "storage/backendid.h" + + ++#define NUM_CUSTOM_PROCSIGNALS 64 ++ + /* + * Reasons for signaling a Postgres child process (a backend or an auxiliary + * process, like checkpointer). We can cope with concurrent signals for different +@@ -29,6 +32,8 @@ + */ + typedef enum + { ++ INVALID_PROCSIGNAL = -1, /* Must be first */ ++ + PROCSIG_CATCHUP_INTERRUPT, /* sinval catchup interrupt */ + PROCSIG_NOTIFY_INTERRUPT, /* listen/notify interrupt */ + PROCSIG_PARALLEL_MESSAGE, /* message from cooperating parallel backend */ +@@ -46,6 +51,14 @@ typedef enum + PROCSIG_RECOVERY_CONFLICT_BUFFERPIN, + PROCSIG_RECOVERY_CONFLICT_STARTUP_DEADLOCK, + ++ PROCSIG_CUSTOM_1, ++ /* ++ * PROCSIG_CUSTOM_2, ++ * ..., ++ * PROCSIG_CUSTOM_N-1, ++ */ ++ PROCSIG_CUSTOM_N = PROCSIG_CUSTOM_1 + NUM_CUSTOM_PROCSIGNALS - 1, ++ + NUM_PROCSIGNALS /* Must be last! */ + } ProcSignalReason; + +@@ -54,6 +67,9 @@ typedef enum + PROCSIGNAL_BARRIER_SMGRRELEASE /* ask smgr to close files */ + } ProcSignalBarrierType; + ++/* Handler of custom process signal */ ++typedef void (*ProcSignalHandler_type) (void); ++ + /* + * prototypes for functions in procsignal.c + */ +@@ -61,12 +77,15 @@ extern Size ProcSignalShmemSize(void); + extern void ProcSignalShmemInit(void); + + extern void ProcSignalInit(int pss_idx); ++extern ProcSignalReason ++ RegisterCustomProcSignalHandler(ProcSignalHandler_type handler); + extern int SendProcSignal(pid_t pid, ProcSignalReason reason, + BackendId backendId); + + extern uint64 EmitProcSignalBarrier(ProcSignalBarrierType type); + extern void WaitForProcSignalBarrier(uint64 generation); + extern void ProcessProcSignalBarrier(void); ++extern void CheckAndHandleCustomSignals(void); + + extern void procsignal_sigusr1_handler(SIGNAL_ARGS); + diff --git a/patches/runtime_explain_16.0.patch b/patches/runtime_explain_16.0.patch new file mode 100644 index 0000000..3d132ca --- /dev/null +++ b/patches/runtime_explain_16.0.patch @@ -0,0 +1,258 @@ +diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c +index 6c2e5c8a4f..74be3944d1 100644 +--- a/src/backend/commands/explain.c ++++ b/src/backend/commands/explain.c +@@ -1023,14 +1023,36 @@ report_triggers(ResultRelInfo *rInfo, bool show_relname, ExplainState *es) + char *relname; + char *conname = NULL; + ++ instr_time starttimespan; ++ double total; ++ double ntuples; ++ double ncalls; ++ ++ if (!es->runtime) ++ { + /* Must clean up instrumentation state */ + InstrEndLoop(instr); ++ } ++ ++ /* Collect statistic variables */ ++ if (!INSTR_TIME_IS_ZERO(instr->starttime)) ++ { ++ INSTR_TIME_SET_CURRENT(starttimespan); ++ INSTR_TIME_SUBTRACT(starttimespan, instr->starttime); ++ } ++ else ++ INSTR_TIME_SET_ZERO(starttimespan); ++ ++ total = instr->total + INSTR_TIME_GET_DOUBLE(instr->counter) ++ + INSTR_TIME_GET_DOUBLE(starttimespan); ++ ntuples = instr->ntuples + instr->tuplecount; ++ ncalls = ntuples + !INSTR_TIME_IS_ZERO(starttimespan); + + /* + * We ignore triggers that were never invoked; they likely aren't + * relevant to the current query type. + */ +- if (instr->ntuples == 0) ++ if (ncalls == 0) + continue; + + ExplainOpenGroup("Trigger", NULL, true, es); +@@ -1056,9 +1078,9 @@ report_triggers(ResultRelInfo *rInfo, bool show_relname, ExplainState *es) + appendStringInfo(es->str, " on %s", relname); + if (es->timing) + appendStringInfo(es->str, ": time=%.3f calls=%.0f\n", +- 1000.0 * instr->total, instr->ntuples); ++ 1000.0 * total, ncalls); + else +- appendStringInfo(es->str, ": calls=%.0f\n", instr->ntuples); ++ appendStringInfo(es->str, ": calls=%.0f\n", ncalls); + } + else + { +@@ -1067,9 +1089,8 @@ report_triggers(ResultRelInfo *rInfo, bool show_relname, ExplainState *es) + ExplainPropertyText("Constraint Name", conname, es); + ExplainPropertyText("Relation", relname, es); + if (es->timing) +- ExplainPropertyFloat("Time", "ms", 1000.0 * instr->total, 3, +- es); +- ExplainPropertyFloat("Calls", NULL, instr->ntuples, 0, es); ++ ExplainPropertyFloat("Time", "ms", 1000.0 * total, 3, es); ++ ExplainPropertyFloat("Calls", NULL, ncalls, 0, es); + } + + if (conname) +@@ -1645,8 +1666,11 @@ ExplainNode(PlanState *planstate, List *ancestors, + * instrumentation results the user didn't ask for. But we do the + * InstrEndLoop call anyway, if possible, to reduce the number of cases + * auto_explain has to contend with. ++ * ++ * If flag es->stateinfo is set, i.e. when printing the current execution ++ * state, this step of cleaning up is missed. + */ +- if (planstate->instrument) ++ if (planstate->instrument && !es->runtime) + InstrEndLoop(planstate->instrument); + + if (es->analyze && +@@ -1681,7 +1705,7 @@ ExplainNode(PlanState *planstate, List *ancestors, + ExplainPropertyFloat("Actual Loops", NULL, nloops, 0, es); + } + } +- else if (es->analyze) ++ else if (es->analyze && !es->runtime) + { + if (es->format == EXPLAIN_FORMAT_TEXT) + appendStringInfoString(es->str, " (never executed)"); +@@ -1697,6 +1721,75 @@ ExplainNode(PlanState *planstate, List *ancestors, + } + } + ++ /* ++ * Print the progress of node execution at current loop. ++ */ ++ if (planstate->instrument && es->analyze && es->runtime) ++ { ++ instr_time starttimespan; ++ double startup_sec; ++ double total_sec; ++ double rows; ++ double loop_num; ++ bool finished; ++ ++ if (!INSTR_TIME_IS_ZERO(planstate->instrument->starttime)) ++ { ++ INSTR_TIME_SET_CURRENT(starttimespan); ++ INSTR_TIME_SUBTRACT(starttimespan, planstate->instrument->starttime); ++ } ++ else ++ INSTR_TIME_SET_ZERO(starttimespan); ++ startup_sec = 1000.0 * planstate->instrument->firsttuple; ++ total_sec = 1000.0 * (INSTR_TIME_GET_DOUBLE(planstate->instrument->counter) ++ + INSTR_TIME_GET_DOUBLE(starttimespan)); ++ rows = planstate->instrument->tuplecount; ++ loop_num = planstate->instrument->nloops + 1; ++ ++ finished = planstate->instrument->nloops > 0 ++ && !planstate->instrument->running ++ && INSTR_TIME_IS_ZERO(starttimespan); ++ ++ if (!finished) ++ { ++ ExplainOpenGroup("Current loop", "Current loop", true, es); ++ if (es->format == EXPLAIN_FORMAT_TEXT) ++ { ++ if (es->timing) ++ { ++ if (planstate->instrument->running) ++ appendStringInfo(es->str, ++ " (Current loop: actual time=%.3f..%.3f rows=%.0f, loop number=%.0f)", ++ startup_sec, total_sec, rows, loop_num); ++ else ++ appendStringInfo(es->str, ++ " (Current loop: running time=%.3f actual rows=0, loop number=%.0f)", ++ total_sec, loop_num); ++ } ++ else ++ appendStringInfo(es->str, ++ " (Current loop: actual rows=%.0f, loop number=%.0f)", ++ rows, loop_num); ++ } ++ else ++ { ++ ExplainPropertyFloat("Actual Loop Number", NULL, loop_num, 0, es); ++ if (es->timing) ++ { ++ if (planstate->instrument->running) ++ { ++ ExplainPropertyFloat("Actual Startup Time", NULL, startup_sec, 3, es); ++ ExplainPropertyFloat("Actual Total Time", NULL, total_sec, 3, es); ++ } ++ else ++ ExplainPropertyFloat("Running Time", NULL, total_sec, 3, es); ++ } ++ ExplainPropertyFloat("Actual Rows", NULL, rows, 0, es); ++ } ++ ExplainCloseGroup("Current loop", "Current loop", true, es); ++ } ++ } ++ + /* in text format, first line ends here */ + if (es->format == EXPLAIN_FORMAT_TEXT) + appendStringInfoChar(es->str, '\n'); +@@ -2104,6 +2197,9 @@ ExplainNode(PlanState *planstate, List *ancestors, + + /* Prepare per-worker buffer/WAL usage */ + if (es->workers_state && (es->buffers || es->wal) && es->verbose) ++ /* Show worker detail after query execution */ ++ if (es->analyze && es->verbose && planstate->worker_instrument ++ && !es->runtime) + { + WorkerInstrumentation *w = planstate->worker_instrument; + +@@ -3068,6 +3164,11 @@ show_hash_info(HashState *hashstate, ExplainState *es) + memcpy(&hinstrument, hashstate->hinstrument, + sizeof(HashInstrumentation)); + ++ if (hashstate->hashtable) ++ { ++ ExecHashAccumInstrumentation(&hinstrument, hashstate->hashtable); ++ } ++ + /* + * Merge results from workers. In the parallel-oblivious case, the + * results from all participants should be identical, except where +@@ -3447,20 +3548,16 @@ show_instrumentation_count(const char *qlabel, int which, + if (!es->analyze || !planstate->instrument) + return; + ++ nloops = planstate->instrument->nloops; + if (which == 2) +- nfiltered = planstate->instrument->nfiltered2; ++ nfiltered = ((nloops > 0) ? planstate->instrument->nfiltered2 / nloops : 0); + else +- nfiltered = planstate->instrument->nfiltered1; ++ nfiltered = ((nloops > 0) ? planstate->instrument->nfiltered1 / nloops : 0); + nloops = planstate->instrument->nloops; + + /* In text mode, suppress zero counts; they're not interesting enough */ + if (nfiltered > 0 || es->format != EXPLAIN_FORMAT_TEXT) +- { +- if (nloops > 0) +- ExplainPropertyFloat(qlabel, NULL, nfiltered / nloops, 0, es); +- else +- ExplainPropertyFloat(qlabel, NULL, 0.0, 0, es); +- } ++ ExplainPropertyFloat(qlabel, NULL, nfiltered, 0, es); + } + + /* +@@ -4060,15 +4157,27 @@ show_modifytable_info(ModifyTableState *mtstate, List *ancestors, + double insert_path; + double other_path; + +- InstrEndLoop(outerPlanState(mtstate)->instrument); ++ if (!es->runtime) ++ InstrEndLoop(outerPlanState(mtstate)->instrument); + + /* count the number of source rows */ +- total = outerPlanState(mtstate)->instrument->ntuples; +- other_path = mtstate->ps.instrument->ntuples2; +- insert_path = total - other_path; ++ other_path = mtstate->ps.instrument->nfiltered2; ++ ++ /* ++ * Insert occurs after extracting row from subplan and in runtime mode ++ * we can appear between these two operations - situation when ++ * total > insert_path + other_path. Therefore we don't know exactly ++ * whether last row from subplan is inserted. ++ * We don't print inserted tuples in runtime mode in order to not print ++ * inconsistent data ++ */ ++ if (!es->runtime) ++ { ++ total = outerPlanState(mtstate)->instrument->ntuples; ++ insert_path = total - other_path; ++ ExplainPropertyFloat("Tuples Inserted", NULL, insert_path, 0, es); ++ } + +- ExplainPropertyFloat("Tuples Inserted", NULL, +- insert_path, 0, es); + ExplainPropertyFloat("Conflicting Tuples", NULL, + other_path, 0, es); + } +diff --git a/src/include/commands/explain.h b/src/include/commands/explain.h +index 3d3e632a0c..3eb7bf345d 100644 +--- a/src/include/commands/explain.h ++++ b/src/include/commands/explain.h +@@ -48,6 +48,8 @@ typedef struct ExplainState + bool settings; /* print modified settings */ + bool generic; /* generate a generic plan */ + ExplainFormat format; /* output format */ ++ bool runtime; /* print intermediate state of query execution, ++ not after completion */ + /* state for output formatting --- not reset for each new plan tree */ + int indent; /* current indentation level */ + List *grouping_stack; /* format-specific grouping state */ diff --git a/pg_query_state.c b/pg_query_state.c index ab76f7f..7d03c22 100644 --- a/pg_query_state.c +++ b/pg_query_state.c @@ -2,7 +2,7 @@ * pg_query_state.c * Extract information about query state from other backend * - * Copyright (c) 2016-2023, Postgres Professional + * Copyright (c) 2016-2024, Postgres Professional * * contrib/pg_query_state/pg_query_state.c * IDENTIFICATION diff --git a/pg_query_state.h b/pg_query_state.h index 9152560..f632008 100644 --- a/pg_query_state.h +++ b/pg_query_state.h @@ -2,7 +2,7 @@ * pg_query_state.h * Headers for pg_query_state extension. * - * Copyright (c) 2016-2023, Postgres Professional + * Copyright (c) 2016-2024, Postgres Professional * * IDENTIFICATION * contrib/pg_query_state/pg_query_state.h diff --git a/run_tests.sh b/run_tests.sh index 1c43847..7e3cf79 100644 --- a/run_tests.sh +++ b/run_tests.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2023, Postgres Professional +# Copyright (c) 2019-2024, Postgres Professional # # supported levels: # * standard diff --git a/signal_handler.c b/signal_handler.c index 7e4b602..dfe8780 100644 --- a/signal_handler.c +++ b/signal_handler.c @@ -2,7 +2,7 @@ * signal_handler.c * Collect current query state and send it to requestor in custom signal handler * - * Copyright (c) 2016-2023, Postgres Professional + * Copyright (c) 2016-2024, Postgres Professional * * IDENTIFICATION * contrib/pg_query_state/signal_handler.c diff --git a/tests/common.py b/tests/common.py index c83abb1..6dab69a 100644 --- a/tests/common.py +++ b/tests/common.py @@ -1,6 +1,6 @@ ''' common.py -Copyright (c) 2016-2023, Postgres Professional +Copyright (c) 2016-2024, Postgres Professional ''' import psycopg2 diff --git a/tests/pg_qs_test_runner.py b/tests/pg_qs_test_runner.py index a0df6a9..f4088a9 100644 --- a/tests/pg_qs_test_runner.py +++ b/tests/pg_qs_test_runner.py @@ -1,6 +1,6 @@ ''' pg_qs_test_runner.py -Copyright (c) 2016-2023, Postgres Professional +Copyright (c) 2016-2024, Postgres Professional ''' import argparse diff --git a/tests/test_cases.py b/tests/test_cases.py index c6b0fa2..f86641d 100644 --- a/tests/test_cases.py +++ b/tests/test_cases.py @@ -1,6 +1,6 @@ ''' test_cases.py -Copyright (c) 2016-2023, Postgres Professional +Copyright (c) 2016-2024, Postgres Professional ''' import json diff --git a/tests/tpcds.py b/tests/tpcds.py index 944b799..1f2b6da 100644 --- a/tests/tpcds.py +++ b/tests/tpcds.py @@ -1,6 +1,6 @@ ''' test_cases.py -Copyright (c) 2016-2023, Postgres Professional +Copyright (c) 2016-2024, Postgres Professional ''' import os From d099f6834bf166dc993e4858736760de972b6b76 Mon Sep 17 00:00:00 2001 From: Ekaterina Sokolova Date: Tue, 27 Feb 2024 22:16:15 +0300 Subject: [PATCH 79/91] Fix Dockerfile for Travis-CI. --- Dockerfile.tmpl | 2 +- docker-compose.yml | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Dockerfile.tmpl b/Dockerfile.tmpl index 1e512bc..93b9833 100644 --- a/Dockerfile.tmpl +++ b/Dockerfile.tmpl @@ -6,7 +6,7 @@ RUN apk add --no-cache \ perl perl-ipc-run \ make musl-dev gcc bison flex coreutils \ zlib-dev libedit-dev \ - clang clang-analyzer linux-headers \ + icu-dev clang clang-analyzer linux-headers \ python3 python3-dev py3-virtualenv; diff --git a/docker-compose.yml b/docker-compose.yml index 67f1cee..550e2be 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,2 +1,3 @@ -tests: +services: + tests: build: . \ No newline at end of file From 25bd499d95517dd05ac6af72e497a65c9965ee6a Mon Sep 17 00:00:00 2001 From: Marina Polyakova Date: Wed, 3 Apr 2024 10:59:35 +0300 Subject: [PATCH 80/91] Fix build with PostgreSQL 17devel at 7eb9a8201890f3b208fd4c109a5b08bf139b692a See the following commits in PostgreSQL 17devel: - ab355e3a88de745607f6dd4c21f0119b5c68f2ad Redefine backend ID to be an index into the proc array - 024c521117579a6d356050ad3d78fdc95e44eefa Replace BackendIds with 0-based ProcNumbers --- pg_query_state.c | 47 +++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 45 insertions(+), 2 deletions(-) diff --git a/pg_query_state.c b/pg_query_state.c index 7d03c22..adea7db 100644 --- a/pg_query_state.c +++ b/pg_query_state.c @@ -365,7 +365,9 @@ search_be_status(int pid) for (beid = 1; beid <= pgstat_fetch_stat_numbackends(); beid++) { -#if PG_VERSION_NUM >= 160000 +#if PG_VERSION_NUM >= 170000 + PgBackendStatus *be_status = pgstat_get_beentry_by_proc_number(beid); +#elif PG_VERSION_NUM >= 160000 PgBackendStatus *be_status = pgstat_get_beentry_by_backend_id(beid); #else PgBackendStatus *be_status = pgstat_fetch_stat_beentry(beid); @@ -505,7 +507,14 @@ pg_query_state(PG_FUNCTION_ARGS) errmsg("attempt to extract state of current process"))); proc = BackendPidGetProc(pid); - if (!proc || proc->backendId == InvalidBackendId || proc->databaseId == InvalidOid || proc->roleId == InvalidOid) + if (!proc || +#if PG_VERSION_NUM >= 170000 + proc->vxid.procNumber == INVALID_PROC_NUMBER || +#else + proc->backendId == InvalidBackendId || +#endif + proc->databaseId == InvalidOid || + proc->roleId == InvalidOid) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("backend with pid=%d not found", pid))); @@ -730,7 +739,12 @@ GetRemoteBackendUserId(PGPROC *proc) { Oid result; +#if PG_VERSION_NUM >= 170000 + Assert(proc && proc->vxid.procNumber != INVALID_PROC_NUMBER); +#else Assert(proc && proc->backendId != InvalidBackendId); +#endif + Assert(UserIdPollReason != INVALID_PROCSIGNAL); Assert(counterpart_userid); @@ -738,7 +752,12 @@ GetRemoteBackendUserId(PGPROC *proc) counterpart_userid->caller = MyLatch; pg_write_barrier(); +#if PG_VERSION_NUM >= 170000 + SendProcSignal(proc->pid, UserIdPollReason, proc->vxid.procNumber); +#else SendProcSignal(proc->pid, UserIdPollReason, proc->backendId); +#endif + for (;;) { SpinLockAcquire(&counterpart_userid->mutex); @@ -926,7 +945,12 @@ GetRemoteBackendWorkers(PGPROC *proc) List *result = NIL; LOCKTAG tag; +#if PG_VERSION_NUM >= 170000 + Assert(proc && proc->vxid.procNumber != INVALID_PROC_NUMBER); +#else Assert(proc && proc->backendId != InvalidBackendId); +#endif + Assert(WorkerPollReason != INVALID_PROCSIGNAL); Assert(mq); @@ -936,7 +960,12 @@ GetRemoteBackendWorkers(PGPROC *proc) shm_mq_set_receiver(mq, MyProc); UnlockShmem(&tag); +#if PG_VERSION_NUM >= 170000 + sig_result = SendProcSignal(proc->pid, WorkerPollReason, proc->vxid.procNumber); +#else sig_result = SendProcSignal(proc->pid, WorkerPollReason, proc->backendId); +#endif + if (sig_result == -1) goto signal_error; @@ -1088,9 +1117,16 @@ GetRemoteBackendQueryStates(PGPROC *leader, * send signal `QueryStatePollReason` to all processes and define all alive * ones */ +#if PG_VERSION_NUM >= 170000 + sig_result = SendProcSignal(leader->pid, + QueryStatePollReason, + leader->vxid.procNumber); +#else sig_result = SendProcSignal(leader->pid, QueryStatePollReason, leader->backendId); +#endif + if (sig_result == -1) goto signal_error; foreach(iter, pworkers) @@ -1101,9 +1137,16 @@ GetRemoteBackendQueryStates(PGPROC *leader, pg_atomic_add_fetch_u32(&counterpart_userid->n_peers, 1); +#if PG_VERSION_NUM >= 170000 + sig_result = SendProcSignal(proc->pid, + QueryStatePollReason, + proc->vxid.procNumber); +#else sig_result = SendProcSignal(proc->pid, QueryStatePollReason, proc->backendId); +#endif + if (sig_result == -1) { if (errno != ESRCH) From 1547e213cf366c7ee399d0e9a3bf1b9c5e6764c2 Mon Sep 17 00:00:00 2001 From: Ekaterina Sokolova Date: Fri, 12 Apr 2024 13:02:05 +0300 Subject: [PATCH 81/91] [PGPRO-10083] Set guc to false in python test. pg_query_state.enable_timing and enable_buffers default to false. But we should forcefully set guc variables to false in case of its true values in the configuration file. Tags: pg_query_state. --- tests/test_cases.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_cases.py b/tests/test_cases.py index f86641d..c866f58 100644 --- a/tests/test_cases.py +++ b/tests/test_cases.py @@ -379,6 +379,9 @@ def test_timing_buffers_conflicts(config): timing_pattern = '(?:running time=\d+.\d+)|(?:actual time=\d+.\d+..\d+.\d+)' buffers_pattern = 'Buffers:' + common.set_guc(acon, 'pg_query_state.enable_timing', 'off') + common.set_guc(acon, 'pg_query_state.enable_buffers', 'off') + qs, notices = common.onetime_query_state(config, acon, query, {'timing': True, 'buffers': False}) assert len(qs) == 1 and not re.search(timing_pattern, qs[0][3]) assert notices == ['WARNING: timing statistics disabled\n'] From aeabc16b73b6699360e653075afefbeacaf002a8 Mon Sep 17 00:00:00 2001 From: "Anton A. Melnikov" Date: Mon, 15 Apr 2024 18:38:51 +0300 Subject: [PATCH 82/91] When searching for backend status iterate over the index in the LocalPgBackendStatus array, not over BackendId for v16 or ProcNumber for 16+. BackendId may not be equal to the index +1 in the LocalPgBackendStatus array as well as ProcNumber may not be equal the index. --- pg_query_state.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/pg_query_state.c b/pg_query_state.c index adea7db..1949643 100644 --- a/pg_query_state.c +++ b/pg_query_state.c @@ -365,10 +365,16 @@ search_be_status(int pid) for (beid = 1; beid <= pgstat_fetch_stat_numbackends(); beid++) { -#if PG_VERSION_NUM >= 170000 - PgBackendStatus *be_status = pgstat_get_beentry_by_proc_number(beid); -#elif PG_VERSION_NUM >= 160000 - PgBackendStatus *be_status = pgstat_get_beentry_by_backend_id(beid); +#if PG_VERSION_NUM >= 160000 + LocalPgBackendStatus *lbe_status = pgstat_get_local_beentry_by_index(beid); + PgBackendStatus *be_status; + + Assert(lbe_status); + #ifndef PGPRO_STD + be_status = &lbe_status->backendStatus; + #else + be_status = lbe_status->backendStatus; + #endif #else PgBackendStatus *be_status = pgstat_fetch_stat_beentry(beid); #endif From a81edd4f7bd7eb063e58b579554f0698b51fdb42 Mon Sep 17 00:00:00 2001 From: Ekaterina Sokolova Date: Wed, 17 Apr 2024 14:09:48 +0300 Subject: [PATCH 83/91] Update TPC-DS tests 1) Enable TPC-DS tests 2) Added the ability to crash when calling function pg_query_state 3) Fix path to library in venv 4) Disable requiring Travis tests to be run for unsupported versions --- .travis.yml | 18 ++++++++---------- run_tests.sh | 15 ++++++++++----- tests/pg_qs_test_runner.py | 4 +++- tests/tpcds.py | 31 +++++++++++++++++++++++++++++-- 4 files changed, 50 insertions(+), 18 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0983e07..56b2783 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,24 +22,22 @@ notifications: on_failure: always env: - - PG_VERSION=16 LEVEL=hardcore USE_TPCDS=0 + - PG_VERSION=16 LEVEL=hardcore USE_TPCDS=1 - PG_VERSION=16 - - PG_VERSION=15 LEVEL=hardcore USE_TPCDS=0 + - PG_VERSION=15 LEVEL=hardcore USE_TPCDS=1 - PG_VERSION=15 - - PG_VERSION=14 LEVEL=hardcore USE_TPCDS=0 + - PG_VERSION=14 LEVEL=hardcore USE_TPCDS=1 - PG_VERSION=14 - - PG_VERSION=13 LEVEL=hardcore USE_TPCDS=0 + - PG_VERSION=13 LEVEL=hardcore USE_TPCDS=1 - PG_VERSION=13 - - PG_VERSION=12 LEVEL=hardcore USE_TPCDS=0 + - PG_VERSION=12 LEVEL=hardcore USE_TPCDS=1 - PG_VERSION=12 - - PG_VERSION=11 LEVEL=hardcore USE_TPCDS=0 - PG_VERSION=11 - - PG_VERSION=10 LEVEL=hardcore USE_TPCDS=0 - PG_VERSION=10 - - PG_VERSION=9.6 LEVEL=hardcore - PG_VERSION=9.6 matrix: allow_failures: - - env: PG_VERSION=10 LEVEL=nightmare - - env: PG_VERSION=9.6 LEVEL=nightmare + - env: PG_VERSION=11 + - env: PG_VERSION=10 + - env: PG_VERSION=9.6 diff --git a/run_tests.sh b/run_tests.sh index 7e3cf79..d330d1e 100644 --- a/run_tests.sh +++ b/run_tests.sh @@ -13,6 +13,9 @@ set -ux status=0 +venv_path=tmp/env +rm -rf "$venv_path" + # global exports export PGPORT=55435 export VIRTUAL_ENV_DISABLE_PROMPT=1 @@ -148,13 +151,14 @@ if [ -f regression.diffs ]; then cat regression.diffs; fi # run python tests set +x -e -python3 -m venv /tmp/env && source /tmp/env/bin/activate && -pip install -r ./tests/requirements.txt +python3 -m venv "$venv_path" && source "$venv_path/bin/activate" +pip3 install --upgrade -t "$venv_path" -r ./tests/requirements.txt +#pip3 install -e "./$venv_path" set -e #exit virtualenv with error code -python tests/pg_qs_test_runner.py --port $PGPORT +python3 tests/pg_qs_test_runner.py --port $PGPORT if [[ "$USE_TPCDS" == "1" ]]; then - python tests/pg_qs_test_runner.py --port $PGPORT --tpc-ds-setup - python tests/pg_qs_test_runner.py --port $PGPORT --tpc-ds-run + python3 tests/pg_qs_test_runner.py --port $PGPORT --tpc-ds-setup + python3 tests/pg_qs_test_runner.py --port $PGPORT --tpc-ds-run fi deactivate set -x @@ -179,4 +183,5 @@ gcov $CUSTOM_PG_SRC/contrib/pg_query_state/*.c $CUSTOM_PG_SRC/contrib/pg_query_s set +ux # send coverage stats to Codecov +export CODECOV_TOKEN=55ab7421-9277-45af-a329-d8b40db96b2a bash <(curl -s https://codecov.io/bash) diff --git a/tests/pg_qs_test_runner.py b/tests/pg_qs_test_runner.py index f4088a9..944f77f 100644 --- a/tests/pg_qs_test_runner.py +++ b/tests/pg_qs_test_runner.py @@ -8,9 +8,11 @@ import os import sys +sys.path.append(os.path.dirname(os.path.abspath(__file__))) +sys.path.append(os.path.abspath('tmp/env')) + import psycopg2 -sys.path.append(os.path.dirname(os.path.abspath(__file__))) from test_cases import * import tpcds diff --git a/tests/tpcds.py b/tests/tpcds.py index 1f2b6da..4e38002 100644 --- a/tests/tpcds.py +++ b/tests/tpcds.py @@ -8,6 +8,10 @@ import time import progressbar +# This actually imports progressbar2 but `import progressbar2' itself doesn't work. +# In case of problems with the progressbar/progressbar2, check that you have the +# progressbar2 installed and the path to it or venv is specified. + import psycopg2.extensions import common @@ -55,13 +59,13 @@ def run_tpcds(config): TPC_DS_STATEMENT_TIMEOUT = 20000 # statement_timeout in ms print('Preparing TPC-DS queries...') + err_count = 0 queries = [] for query_file in sorted(os.listdir('tmp_stress/tpcds-result-reproduction/query_qualification/')): with open('tmp_stress/tpcds-result-reproduction/query_qualification/%s' % query_file, 'r') as f: queries.append(f.read()) acon, = common.n_async_connect(config) - pid = acon.get_backend_pid() print('Starting TPC-DS queries...') timeout_list = [] @@ -84,8 +88,25 @@ def run_tpcds(config): PG_QS_DELAY, BEFORE_GETTING_QS_DELAY = 0.1, 0.1 BEFORE_GETTING_QS, GETTING_QS = range(2) state, n_first_getting_qs_retries = BEFORE_GETTING_QS, 0 + + pg_qs_args = { + 'config': config, + 'pid': acon.get_backend_pid() + } + while True: - result, notices = common.pg_query_state(config, pid) + try: + result, notices = common.pg_query_state(**pg_qs_args) + except Exception as e: + # do not consider the test failed if the "error in message + # queue data transmitting" is received, this may happen with + # some small probability, but if it happens too often it is + # a problem, we will handle this case after the loop + if "error in message queue data transmitting" in e.pgerror: + err_count += 1 + else: + raise e + # run state machine to determine the first getting of query state # and query finishing if state == BEFORE_GETTING_QS: @@ -109,6 +130,12 @@ def run_tpcds(config): except psycopg2.extensions.QueryCanceledError: timeout_list.append(i + 1) + if err_count > 2: + print("ERROR: error in message queue data transmitting") + raise + elif err_count > 0: + print(err_count, " times there was error in message queue data transmitting") + common.n_close((acon,)) if len(timeout_list) > 0: From 7801aa13e527bf1cc0b0bb7760e57127e7aba9a3 Mon Sep 17 00:00:00 2001 From: Ekaterina Sokolova Date: Fri, 14 Jun 2024 15:48:41 +0300 Subject: [PATCH 84/91] Improve error handling. --- tests/tpcds.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/tpcds.py b/tests/tpcds.py index 4e38002..bdeb408 100644 --- a/tests/tpcds.py +++ b/tests/tpcds.py @@ -26,7 +26,10 @@ def setup_tpcds(config): try: conn = psycopg2.connect(**config) cur = conn.cursor() + except Exception as e: + raise DataLoadException('Load failed: %s' % e) + try: # Create pg_query_state extension cur.execute('CREATE EXTENSION IF NOT EXISTS pg_query_state') @@ -131,8 +134,8 @@ def run_tpcds(config): timeout_list.append(i + 1) if err_count > 2: - print("ERROR: error in message queue data transmitting") - raise + print("\nERROR: error in message queue data transmitting") + raise Exception('error was received %d times'%err_count) elif err_count > 0: print(err_count, " times there was error in message queue data transmitting") From ec52e0ae8333b6d574d7a1449bc5d4c4781ecf2f Mon Sep 17 00:00:00 2001 From: Ekaterina Sokolova Date: Tue, 27 Aug 2024 12:12:45 +0300 Subject: [PATCH 85/91] Correcting confusion in variables. --- patches/runtime_explain_11.0.patch | 7 +++---- patches/runtime_explain_12.0.patch | 7 +++---- patches/runtime_explain_13.0.patch | 7 +++---- patches/runtime_explain_14.0.patch | 7 +++---- patches/runtime_explain_15.0.patch | 7 +++---- patches/runtime_explain_16.0.patch | 7 +++---- 6 files changed, 18 insertions(+), 24 deletions(-) diff --git a/patches/runtime_explain_11.0.patch b/patches/runtime_explain_11.0.patch index dddbcbe..9d12d5b 100644 --- a/patches/runtime_explain_11.0.patch +++ b/patches/runtime_explain_11.0.patch @@ -209,10 +209,9 @@ index 16a80a0ea1..b12906b005 100644 /* count the number of source rows */ - total = mtstate->mt_plans[0]->instrument->ntuples; -- other_path = mtstate->ps.instrument->ntuples2; + other_path = mtstate->ps.instrument->ntuples2; - insert_path = total - other_path; -+ other_path = mtstate->ps.instrument->nfiltered2; -+ + + /* + * Insert occurs after extracting row from subplan and in runtime mode + * we can appear between these two operations - situation when @@ -227,7 +226,7 @@ index 16a80a0ea1..b12906b005 100644 + insert_path = total - other_path; + ExplainPropertyFloat("Tuples Inserted", NULL, insert_path, 0, es); + } - ++ - ExplainPropertyFloat("Tuples Inserted", NULL, - insert_path, 0, es); ExplainPropertyFloat("Conflicting Tuples", NULL, diff --git a/patches/runtime_explain_12.0.patch b/patches/runtime_explain_12.0.patch index 1d105b2..9aa8397 100644 --- a/patches/runtime_explain_12.0.patch +++ b/patches/runtime_explain_12.0.patch @@ -222,10 +222,9 @@ index 92969636b75..fab4267a2c1 100644 /* count the number of source rows */ - total = mtstate->mt_plans[0]->instrument->ntuples; -- other_path = mtstate->ps.instrument->ntuples2; + other_path = mtstate->ps.instrument->ntuples2; - insert_path = total - other_path; -+ other_path = mtstate->ps.instrument->nfiltered2; -+ + + /* + * Insert occurs after extracting row from subplan and in runtime mode + * we can appear between these two operations - situation when @@ -240,7 +239,7 @@ index 92969636b75..fab4267a2c1 100644 + insert_path = total - other_path; + ExplainPropertyFloat("Tuples Inserted", NULL, insert_path, 0, es); + } - ++ - ExplainPropertyFloat("Tuples Inserted", NULL, - insert_path, 0, es); ExplainPropertyFloat("Conflicting Tuples", NULL, diff --git a/patches/runtime_explain_13.0.patch b/patches/runtime_explain_13.0.patch index 973ebd5..be29669 100644 --- a/patches/runtime_explain_13.0.patch +++ b/patches/runtime_explain_13.0.patch @@ -219,10 +219,9 @@ index 20708db9f12..866948bd0c1 100644 /* count the number of source rows */ - total = mtstate->mt_plans[0]->instrument->ntuples; -- other_path = mtstate->ps.instrument->ntuples2; + other_path = mtstate->ps.instrument->ntuples2; - insert_path = total - other_path; -+ other_path = mtstate->ps.instrument->nfiltered2; -+ + + /* + * Insert occurs after extracting row from subplan and in runtime mode + * we can appear between these two operations - situation when @@ -237,7 +236,7 @@ index 20708db9f12..866948bd0c1 100644 + insert_path = total - other_path; + ExplainPropertyFloat("Tuples Inserted", NULL, insert_path, 0, es); + } - ++ - ExplainPropertyFloat("Tuples Inserted", NULL, - insert_path, 0, es); ExplainPropertyFloat("Conflicting Tuples", NULL, diff --git a/patches/runtime_explain_14.0.patch b/patches/runtime_explain_14.0.patch index 7904cc2..b266b15 100644 --- a/patches/runtime_explain_14.0.patch +++ b/patches/runtime_explain_14.0.patch @@ -219,10 +219,9 @@ index 10644dfac4..7106ed4257 100644 /* count the number of source rows */ - total = outerPlanState(mtstate)->instrument->ntuples; -- other_path = mtstate->ps.instrument->ntuples2; + other_path = mtstate->ps.instrument->ntuples2; - insert_path = total - other_path; -+ other_path = mtstate->ps.instrument->nfiltered2; -+ + + /* + * Insert occurs after extracting row from subplan and in runtime mode + * we can appear between these two operations - situation when @@ -237,7 +236,7 @@ index 10644dfac4..7106ed4257 100644 + insert_path = total - other_path; + ExplainPropertyFloat("Tuples Inserted", NULL, insert_path, 0, es); + } - ++ - ExplainPropertyFloat("Tuples Inserted", NULL, - insert_path, 0, es); ExplainPropertyFloat("Conflicting Tuples", NULL, diff --git a/patches/runtime_explain_15.0.patch b/patches/runtime_explain_15.0.patch index adab6dc..d60cea8 100644 --- a/patches/runtime_explain_15.0.patch +++ b/patches/runtime_explain_15.0.patch @@ -219,10 +219,9 @@ index 10644dfac4..7106ed4257 100644 /* count the number of source rows */ - total = outerPlanState(mtstate)->instrument->ntuples; -- other_path = mtstate->ps.instrument->ntuples2; + other_path = mtstate->ps.instrument->ntuples2; - insert_path = total - other_path; -+ other_path = mtstate->ps.instrument->nfiltered2; -+ + + /* + * Insert occurs after extracting row from subplan and in runtime mode + * we can appear between these two operations - situation when @@ -237,7 +236,7 @@ index 10644dfac4..7106ed4257 100644 + insert_path = total - other_path; + ExplainPropertyFloat("Tuples Inserted", NULL, insert_path, 0, es); + } - ++ - ExplainPropertyFloat("Tuples Inserted", NULL, - insert_path, 0, es); ExplainPropertyFloat("Conflicting Tuples", NULL, diff --git a/patches/runtime_explain_16.0.patch b/patches/runtime_explain_16.0.patch index 3d132ca..2b955e9 100644 --- a/patches/runtime_explain_16.0.patch +++ b/patches/runtime_explain_16.0.patch @@ -219,10 +219,9 @@ index 6c2e5c8a4f..74be3944d1 100644 /* count the number of source rows */ - total = outerPlanState(mtstate)->instrument->ntuples; -- other_path = mtstate->ps.instrument->ntuples2; + other_path = mtstate->ps.instrument->ntuples2; - insert_path = total - other_path; -+ other_path = mtstate->ps.instrument->nfiltered2; -+ + + /* + * Insert occurs after extracting row from subplan and in runtime mode + * we can appear between these two operations - situation when @@ -237,7 +236,7 @@ index 6c2e5c8a4f..74be3944d1 100644 + insert_path = total - other_path; + ExplainPropertyFloat("Tuples Inserted", NULL, insert_path, 0, es); + } - ++ - ExplainPropertyFloat("Tuples Inserted", NULL, - insert_path, 0, es); ExplainPropertyFloat("Conflicting Tuples", NULL, From d01ea74a4b4c8f76f110badbcbc40b30308030f0 Mon Sep 17 00:00:00 2001 From: "Anton A. Melnikov" Date: Thu, 19 Sep 2024 22:05:49 +0300 Subject: [PATCH 86/91] PGPRO-10866: Add static decoration to avoid error: "no previous extern declaration for non-static variable [-Wmissing-variable-declarations]". Tags: pg_query_state --- pg_query_state.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pg_query_state.c b/pg_query_state.c index 1949643..635b967 100644 --- a/pg_query_state.c +++ b/pg_query_state.c @@ -101,8 +101,8 @@ static List *GetRemoteBackendQueryStates(PGPROC *leader, ExplainFormat format); /* Shared memory variables */ -shm_toc *toc = NULL; -RemoteUserIdResult *counterpart_userid = NULL; +static shm_toc *toc = NULL; +static RemoteUserIdResult *counterpart_userid = NULL; pg_qs_params *params = NULL; shm_mq *mq = NULL; From 1230ab03f36615c064c3adc6de0936f70b28e884 Mon Sep 17 00:00:00 2001 From: Ekaterina Sokolova Date: Tue, 15 Oct 2024 11:06:00 +0300 Subject: [PATCH 87/91] Updates for PostgreSQL 17. 1) Add patches for PostgreSQL 17; 2) Add alternative output to python test due to EXPLAIN output change. See fd0398fcb099. --- .travis.yml | 2 + patches/custom_signals_17.0.patch | 227 ++++++++++++++++++++++++ patches/runtime_explain_17.0.patch | 265 +++++++++++++++++++++++++++++ tests/test_cases.py | 13 +- 4 files changed, 506 insertions(+), 1 deletion(-) create mode 100644 patches/custom_signals_17.0.patch create mode 100644 patches/runtime_explain_17.0.patch diff --git a/.travis.yml b/.travis.yml index 56b2783..a9ae4c6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,6 +22,8 @@ notifications: on_failure: always env: + - PG_VERSION=17 LEVEL=hardcore USE_TPCDS=1 + - PG_VERSION=17 - PG_VERSION=16 LEVEL=hardcore USE_TPCDS=1 - PG_VERSION=16 - PG_VERSION=15 LEVEL=hardcore USE_TPCDS=1 diff --git a/patches/custom_signals_17.0.patch b/patches/custom_signals_17.0.patch new file mode 100644 index 0000000..d227104 --- /dev/null +++ b/patches/custom_signals_17.0.patch @@ -0,0 +1,227 @@ +diff --git a/src/backend/storage/ipc/procsignal.c b/src/backend/storage/ipc/procsignal.c +index 4ed9ced..6e70892 100644 +--- a/src/backend/storage/ipc/procsignal.c ++++ b/src/backend/storage/ipc/procsignal.c +@@ -6,6 +6,7 @@ + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California ++ * Portions Copyright (c) 2024, Postgres Professional + * + * IDENTIFICATION + * src/backend/storage/ipc/procsignal.c +@@ -96,6 +97,13 @@ typedef struct + #define BARRIER_CLEAR_BIT(flags, type) \ + ((flags) &= ~(((uint32) 1) << (uint32) (type))) + ++#define IsCustomProcSignalReason(reason) \ ++ ((reason) >= PROCSIG_CUSTOM_1 && (reason) <= PROCSIG_CUSTOM_N) ++ ++static bool CustomSignalPendings[NUM_CUSTOM_PROCSIGNALS]; ++static bool CustomSignalProcessing[NUM_CUSTOM_PROCSIGNALS]; ++static ProcSignalHandler_type CustomInterruptHandlers[NUM_CUSTOM_PROCSIGNALS]; ++ + static ProcSignalHeader *ProcSignal = NULL; + static ProcSignalSlot *MyProcSignalSlot = NULL; + +@@ -103,6 +111,8 @@ static bool CheckProcSignal(ProcSignalReason reason); + static void CleanupProcSignalState(int status, Datum arg); + static void ResetProcSignalBarrierBits(uint32 flags); + ++static void CheckAndSetCustomSignalInterrupts(void); ++ + /* + * ProcSignalShmemSize + * Compute space needed for ProcSignal's shared memory +@@ -242,6 +252,36 @@ CleanupProcSignalState(int status, Datum arg) + slot->pss_pid = 0; + } + ++/* ++ * RegisterCustomProcSignalHandler ++ * Assign specific handler of custom process signal with new ++ * ProcSignalReason key. ++ * ++ * This function has to be called in _PG_init function of extensions at the ++ * stage of loading shared preloaded libraries. Otherwise it throws fatal error. ++ * ++ * Return INVALID_PROCSIGNAL if all slots for custom signals are occupied. ++ */ ++ProcSignalReason ++RegisterCustomProcSignalHandler(ProcSignalHandler_type handler) ++{ ++ ProcSignalReason reason; ++ ++ if (!process_shared_preload_libraries_in_progress) ++ ereport(FATAL, (errcode(ERRCODE_INTERNAL_ERROR), ++ errmsg("cannot register custom signal after startup"))); ++ ++ /* Iterate through custom signal slots to find a free one */ ++ for (reason = PROCSIG_CUSTOM_1; reason <= PROCSIG_CUSTOM_N; reason++) ++ if (!CustomInterruptHandlers[reason - PROCSIG_CUSTOM_1]) ++ { ++ CustomInterruptHandlers[reason - PROCSIG_CUSTOM_1] = handler; ++ return reason; ++ } ++ ++ return INVALID_PROCSIGNAL; ++} ++ + /* + * SendProcSignal + * Send a signal to a Postgres process +@@ -676,5 +716,70 @@ procsignal_sigusr1_handler(SIGNAL_ARGS) + if (CheckProcSignal(PROCSIG_RECOVERY_CONFLICT_BUFFERPIN)) + HandleRecoveryConflictInterrupt(PROCSIG_RECOVERY_CONFLICT_BUFFERPIN); + ++ CheckAndSetCustomSignalInterrupts(); ++ + SetLatch(MyLatch); + } ++ ++/* ++ * Handle receipt of an interrupt indicating any of custom process signals. ++ */ ++static void ++CheckAndSetCustomSignalInterrupts() ++{ ++ ProcSignalReason reason; ++ ++ for (reason = PROCSIG_CUSTOM_1; reason <= PROCSIG_CUSTOM_N; reason++) ++ { ++ if (CheckProcSignal(reason)) ++ { ++ ++ /* set interrupt flags */ ++ InterruptPending = true; ++ CustomSignalPendings[reason - PROCSIG_CUSTOM_1] = true; ++ } ++ } ++ ++ SetLatch(MyLatch); ++} ++ ++/* ++ * CheckAndHandleCustomSignals ++ * Check custom signal flags and call handler assigned to that signal ++ * if it is not NULL ++ * ++ * This function is called within CHECK_FOR_INTERRUPTS if interrupt occurred. ++ */ ++void ++CheckAndHandleCustomSignals(void) ++{ ++ int i; ++ ++ /* ++ * This is invoked from ProcessInterrupts(), and since some of the ++ * functions it calls contain CHECK_FOR_INTERRUPTS(), there is a potential ++ * for recursive calls if more signals are received while this runs, so ++ * let's block interrupts until done. ++ */ ++ HOLD_INTERRUPTS(); ++ ++ /* Check on expiring of custom signals and call its handlers if exist */ ++ for (i = 0; i < NUM_CUSTOM_PROCSIGNALS; i++) ++ { ++ if (!CustomSignalProcessing[i] && CustomSignalPendings[i]) ++ { ++ ProcSignalHandler_type handler; ++ ++ CustomSignalPendings[i] = false; ++ handler = CustomInterruptHandlers[i]; ++ if (handler != NULL) ++ { ++ CustomSignalProcessing[i] = true; ++ handler(); ++ CustomSignalProcessing[i] = false; ++ } ++ } ++ } ++ ++ RESUME_INTERRUPTS(); ++} +diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c +index a750dc8..e1b0be5 100644 +--- a/src/backend/tcop/postgres.c ++++ b/src/backend/tcop/postgres.c +@@ -3492,6 +3492,8 @@ ProcessInterrupts(void) + if (ParallelMessagePending) + HandleParallelMessages(); + ++ CheckAndHandleCustomSignals(); ++ + if (LogMemoryContextPending) + ProcessLogMemoryContextInterrupt(); + +diff --git a/src/include/storage/procsignal.h b/src/include/storage/procsignal.h +index 7d290ea..f262f0c 100644 +--- a/src/include/storage/procsignal.h ++++ b/src/include/storage/procsignal.h +@@ -6,6 +6,7 @@ + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California ++ * Portions Copyright (c) 2024, Postgres Professional + * + * src/include/storage/procsignal.h + * +@@ -17,6 +18,8 @@ + #include "storage/procnumber.h" + + ++#define NUM_CUSTOM_PROCSIGNALS 64 ++ + /* + * Reasons for signaling a Postgres child process (a backend or an auxiliary + * process, like checkpointer). We can cope with concurrent signals for different +@@ -29,6 +32,8 @@ + */ + typedef enum + { ++ INVALID_PROCSIGNAL = -1, /* Must be first */ ++ + PROCSIG_CATCHUP_INTERRUPT, /* sinval catchup interrupt */ + PROCSIG_NOTIFY_INTERRUPT, /* listen/notify interrupt */ + PROCSIG_PARALLEL_MESSAGE, /* message from cooperating parallel backend */ +@@ -37,6 +42,14 @@ typedef enum + PROCSIG_LOG_MEMORY_CONTEXT, /* ask backend to log the memory contexts */ + PROCSIG_PARALLEL_APPLY_MESSAGE, /* Message from parallel apply workers */ + ++ PROCSIG_CUSTOM_1, ++ /* ++ * PROCSIG_CUSTOM_2, ++ * ..., ++ * PROCSIG_CUSTOM_N-1, ++ */ ++ PROCSIG_CUSTOM_N = PROCSIG_CUSTOM_1 + NUM_CUSTOM_PROCSIGNALS - 1, ++ + /* Recovery conflict reasons */ + PROCSIG_RECOVERY_CONFLICT_FIRST, + PROCSIG_RECOVERY_CONFLICT_DATABASE = PROCSIG_RECOVERY_CONFLICT_FIRST, +@@ -56,6 +69,9 @@ typedef enum + PROCSIGNAL_BARRIER_SMGRRELEASE, /* ask smgr to close files */ + } ProcSignalBarrierType; + ++/* Handler of custom process signal */ ++typedef void (*ProcSignalHandler_type) (void); ++ + /* + * prototypes for functions in procsignal.c + */ +@@ -63,12 +79,15 @@ extern Size ProcSignalShmemSize(void); + extern void ProcSignalShmemInit(void); + + extern void ProcSignalInit(void); ++extern ProcSignalReason ++ RegisterCustomProcSignalHandler(ProcSignalHandler_type handler); + extern int SendProcSignal(pid_t pid, ProcSignalReason reason, + ProcNumber procNumber); + + extern uint64 EmitProcSignalBarrier(ProcSignalBarrierType type); + extern void WaitForProcSignalBarrier(uint64 generation); + extern void ProcessProcSignalBarrier(void); ++extern void CheckAndHandleCustomSignals(void); + + extern void procsignal_sigusr1_handler(SIGNAL_ARGS); + diff --git a/patches/runtime_explain_17.0.patch b/patches/runtime_explain_17.0.patch new file mode 100644 index 0000000..65e22b8 --- /dev/null +++ b/patches/runtime_explain_17.0.patch @@ -0,0 +1,265 @@ +diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c +index 18a5af6b919..73d3d6171eb 100644 +--- a/src/backend/commands/explain.c ++++ b/src/backend/commands/explain.c +@@ -18,6 +18,7 @@ + #include "commands/createas.h" + #include "commands/defrem.h" + #include "commands/prepare.h" ++#include "executor/nodeHash.h" + #include "foreign/fdwapi.h" + #include "jit/jit.h" + #include "libpq/pqformat.h" +@@ -1233,14 +1234,36 @@ report_triggers(ResultRelInfo *rInfo, bool show_relname, ExplainState *es) + char *relname; + char *conname = NULL; + ++ instr_time starttimespan; ++ double total; ++ double ntuples; ++ double ncalls; ++ ++ if (!es->runtime) ++ { + /* Must clean up instrumentation state */ + InstrEndLoop(instr); ++ } ++ ++ /* Collect statistic variables */ ++ if (!INSTR_TIME_IS_ZERO(instr->starttime)) ++ { ++ INSTR_TIME_SET_CURRENT(starttimespan); ++ INSTR_TIME_SUBTRACT(starttimespan, instr->starttime); ++ } ++ else ++ INSTR_TIME_SET_ZERO(starttimespan); ++ ++ total = instr->total + INSTR_TIME_GET_DOUBLE(instr->counter) ++ + INSTR_TIME_GET_DOUBLE(starttimespan); ++ ntuples = instr->ntuples + instr->tuplecount; ++ ncalls = ntuples + !INSTR_TIME_IS_ZERO(starttimespan); + + /* + * We ignore triggers that were never invoked; they likely aren't + * relevant to the current query type. + */ +- if (instr->ntuples == 0) ++ if (ncalls == 0) + continue; + + ExplainOpenGroup("Trigger", NULL, true, es); +@@ -1266,9 +1289,9 @@ report_triggers(ResultRelInfo *rInfo, bool show_relname, ExplainState *es) + appendStringInfo(es->str, " on %s", relname); + if (es->timing) + appendStringInfo(es->str, ": time=%.3f calls=%.0f\n", +- 1000.0 * instr->total, instr->ntuples); ++ 1000.0 * total, ncalls); + else +- appendStringInfo(es->str, ": calls=%.0f\n", instr->ntuples); ++ appendStringInfo(es->str, ": calls=%.0f\n", ncalls); + } + else + { +@@ -1277,9 +1300,8 @@ report_triggers(ResultRelInfo *rInfo, bool show_relname, ExplainState *es) + ExplainPropertyText("Constraint Name", conname, es); + ExplainPropertyText("Relation", relname, es); + if (es->timing) +- ExplainPropertyFloat("Time", "ms", 1000.0 * instr->total, 3, +- es); +- ExplainPropertyFloat("Calls", NULL, instr->ntuples, 0, es); ++ ExplainPropertyFloat("Time", "ms", 1000.0 * total, 3, es); ++ ExplainPropertyFloat("Calls", NULL, ncalls, 0, es); + } + + if (conname) +@@ -1949,8 +1971,11 @@ ExplainNode(PlanState *planstate, List *ancestors, + * instrumentation results the user didn't ask for. But we do the + * InstrEndLoop call anyway, if possible, to reduce the number of cases + * auto_explain has to contend with. ++ * ++ * If flag es->stateinfo is set, i.e. when printing the current execution ++ * state, this step of cleaning up is missed. + */ +- if (planstate->instrument) ++ if (planstate->instrument && !es->runtime) + InstrEndLoop(planstate->instrument); + + if (es->analyze && +@@ -1985,7 +2010,7 @@ ExplainNode(PlanState *planstate, List *ancestors, + ExplainPropertyFloat("Actual Loops", NULL, nloops, 0, es); + } + } +- else if (es->analyze) ++ else if (es->analyze && !es->runtime) + { + if (es->format == EXPLAIN_FORMAT_TEXT) + appendStringInfoString(es->str, " (never executed)"); +@@ -2001,6 +2026,75 @@ ExplainNode(PlanState *planstate, List *ancestors, + } + } + ++ /* ++ * Print the progress of node execution at current loop. ++ */ ++ if (planstate->instrument && es->analyze && es->runtime) ++ { ++ instr_time starttimespan; ++ double startup_sec; ++ double total_sec; ++ double rows; ++ double loop_num; ++ bool finished; ++ ++ if (!INSTR_TIME_IS_ZERO(planstate->instrument->starttime)) ++ { ++ INSTR_TIME_SET_CURRENT(starttimespan); ++ INSTR_TIME_SUBTRACT(starttimespan, planstate->instrument->starttime); ++ } ++ else ++ INSTR_TIME_SET_ZERO(starttimespan); ++ startup_sec = 1000.0 * planstate->instrument->firsttuple; ++ total_sec = 1000.0 * (INSTR_TIME_GET_DOUBLE(planstate->instrument->counter) ++ + INSTR_TIME_GET_DOUBLE(starttimespan)); ++ rows = planstate->instrument->tuplecount; ++ loop_num = planstate->instrument->nloops + 1; ++ ++ finished = planstate->instrument->nloops > 0 ++ && !planstate->instrument->running ++ && INSTR_TIME_IS_ZERO(starttimespan); ++ ++ if (!finished) ++ { ++ ExplainOpenGroup("Current loop", "Current loop", true, es); ++ if (es->format == EXPLAIN_FORMAT_TEXT) ++ { ++ if (es->timing) ++ { ++ if (planstate->instrument->running) ++ appendStringInfo(es->str, ++ " (Current loop: actual time=%.3f..%.3f rows=%.0f, loop number=%.0f)", ++ startup_sec, total_sec, rows, loop_num); ++ else ++ appendStringInfo(es->str, ++ " (Current loop: running time=%.3f actual rows=0, loop number=%.0f)", ++ total_sec, loop_num); ++ } ++ else ++ appendStringInfo(es->str, ++ " (Current loop: actual rows=%.0f, loop number=%.0f)", ++ rows, loop_num); ++ } ++ else ++ { ++ ExplainPropertyFloat("Actual Loop Number", NULL, loop_num, 0, es); ++ if (es->timing) ++ { ++ if (planstate->instrument->running) ++ { ++ ExplainPropertyFloat("Actual Startup Time", NULL, startup_sec, 3, es); ++ ExplainPropertyFloat("Actual Total Time", NULL, total_sec, 3, es); ++ } ++ else ++ ExplainPropertyFloat("Running Time", NULL, total_sec, 3, es); ++ } ++ ExplainPropertyFloat("Actual Rows", NULL, rows, 0, es); ++ } ++ ExplainCloseGroup("Current loop", "Current loop", true, es); ++ } ++ } ++ + /* in text format, first line ends here */ + if (es->format == EXPLAIN_FORMAT_TEXT) + appendStringInfoChar(es->str, '\n'); +@@ -2416,6 +2510,9 @@ ExplainNode(PlanState *planstate, List *ancestors, + + /* Prepare per-worker buffer/WAL usage */ + if (es->workers_state && (es->buffers || es->wal) && es->verbose) ++ /* Show worker detail after query execution */ ++ if (es->analyze && es->verbose && planstate->worker_instrument ++ && !es->runtime) + { + WorkerInstrumentation *w = planstate->worker_instrument; + +@@ -3403,6 +3500,11 @@ show_hash_info(HashState *hashstate, ExplainState *es) + memcpy(&hinstrument, hashstate->hinstrument, + sizeof(HashInstrumentation)); + ++ if (hashstate->hashtable) ++ { ++ ExecHashAccumInstrumentation(&hinstrument, hashstate->hashtable); ++ } ++ + /* + * Merge results from workers. In the parallel-oblivious case, the + * results from all participants should be identical, except where +@@ -3937,20 +4039,16 @@ show_instrumentation_count(const char *qlabel, int which, + if (!es->analyze || !planstate->instrument) + return; + ++ nloops = planstate->instrument->nloops; + if (which == 2) +- nfiltered = planstate->instrument->nfiltered2; ++ nfiltered = ((nloops > 0) ? planstate->instrument->nfiltered2 / nloops : 0); + else +- nfiltered = planstate->instrument->nfiltered1; ++ nfiltered = ((nloops > 0) ? planstate->instrument->nfiltered1 / nloops : 0); + nloops = planstate->instrument->nloops; + + /* In text mode, suppress zero counts; they're not interesting enough */ + if (nfiltered > 0 || es->format != EXPLAIN_FORMAT_TEXT) +- { +- if (nloops > 0) +- ExplainPropertyFloat(qlabel, NULL, nfiltered / nloops, 0, es); +- else +- ExplainPropertyFloat(qlabel, NULL, 0.0, 0, es); +- } ++ ExplainPropertyFloat(qlabel, NULL, nfiltered, 0, es); + } + + /* +@@ -4617,15 +4715,27 @@ show_modifytable_info(ModifyTableState *mtstate, List *ancestors, + double insert_path; + double other_path; + +- InstrEndLoop(outerPlanState(mtstate)->instrument); ++ if (!es->runtime) ++ InstrEndLoop(outerPlanState(mtstate)->instrument); + + /* count the number of source rows */ +- total = outerPlanState(mtstate)->instrument->ntuples; + other_path = mtstate->ps.instrument->ntuples2; +- insert_path = total - other_path; + +- ExplainPropertyFloat("Tuples Inserted", NULL, +- insert_path, 0, es); ++ /* ++ * Insert occurs after extracting row from subplan and in runtime mode ++ * we can appear between these two operations - situation when ++ * total > insert_path + other_path. Therefore we don't know exactly ++ * whether last row from subplan is inserted. ++ * We don't print inserted tuples in runtime mode in order to not print ++ * inconsistent data ++ */ ++ if (!es->runtime) ++ { ++ total = outerPlanState(mtstate)->instrument->ntuples; ++ insert_path = total - other_path; ++ ExplainPropertyFloat("Tuples Inserted", NULL, insert_path, 0, es); ++ } ++ + ExplainPropertyFloat("Conflicting Tuples", NULL, + other_path, 0, es); + } +diff --git a/src/include/commands/explain.h b/src/include/commands/explain.h +index 3ab0aae78f7..3644c0db116 100644 +--- a/src/include/commands/explain.h ++++ b/src/include/commands/explain.h +@@ -57,6 +57,8 @@ typedef struct ExplainState + bool generic; /* generate a generic plan */ + ExplainSerializeOption serialize; /* serialize the query's output? */ + ExplainFormat format; /* output format */ ++ bool runtime; /* print intermediate state of query execution, ++ not after completion */ + /* state for output formatting --- not reset for each new plan tree */ + int indent; /* current indentation level */ + List *grouping_stack; /* format-specific grouping state */ diff --git a/tests/test_cases.py b/tests/test_cases.py index c866f58..498484b 100644 --- a/tests/test_cases.py +++ b/tests/test_cases.py @@ -110,6 +110,17 @@ def test_nested_call(config): expected = 'Function Scan on n_join_foo_bar (Current loop: actual rows=0, loop number=1)' expected_nested = r"""Result \(Current loop: actual rows=0, loop number=1\) InitPlan 1 \(returns \$0\) + -> Aggregate \(Current loop: actual rows=0, loop number=1\) + -> Hash Join \(Current loop: actual rows=\d+, loop number=1\) + Hash Cond: \(foo.c1 = bar.c1\) + Join Filter: \(unlock_if_eq_1\(foo.c1\) = bar.c1\) + -> Seq Scan on foo \(Current loop: actual rows=\d+, loop number=1\) + -> Hash \(Current loop: actual rows=500000, loop number=1\) + Buckets: \d+ Batches: \d+ Memory Usage: \d+kB + -> Seq Scan on bar \(Current loop: actual rows=\d+, loop number=1\)""" + + expected_nested_2 = r"""Result \(Current loop: actual rows=0, loop number=1\) + InitPlan 1 -> Aggregate \(Current loop: actual rows=0, loop number=1\) -> Hash Join \(Current loop: actual rows=\d+, loop number=1\) Hash Cond: \(foo.c1 = bar.c1\) @@ -136,7 +147,7 @@ def test_nested_call(config): assert qs[0][2] == call_function assert qs[0][3] == expected assert qs[1][2] == nested_query1 or qs[1][2] == nested_query2 - assert re.match(expected_nested, qs[1][3]) + assert re.match(expected_nested, qs[1][3]) or re.match(expected_nested_2, qs[1][3]) assert qs[0][4] == qs[1][4] == None assert len(notices) == 0 From 0c6e36aec1bde12a74326de9bb03685a6772b6b7 Mon Sep 17 00:00:00 2001 From: Ekaterina Sokolova Date: Thu, 31 Oct 2024 17:43:49 +0300 Subject: [PATCH 88/91] Remove checks with timeout problems. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a9ae4c6..06bae7b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,7 +22,6 @@ notifications: on_failure: always env: - - PG_VERSION=17 LEVEL=hardcore USE_TPCDS=1 - PG_VERSION=17 - PG_VERSION=16 LEVEL=hardcore USE_TPCDS=1 - PG_VERSION=16 @@ -40,6 +39,7 @@ env: matrix: allow_failures: + - env: PG_VERSION=13 LEVEL=hardcore USE_TPCDS=1 - env: PG_VERSION=11 - env: PG_VERSION=10 - env: PG_VERSION=9.6 From 154e5f5f6ed5ddb2f14b208e84d7d2247ea4669d Mon Sep 17 00:00:00 2001 From: Arseny Kositsin Date: Mon, 11 Nov 2024 13:54:17 +0300 Subject: [PATCH 89/91] [PGPRO-11597] Redesigned the launch of isolation tests "CREATE/DROP EXTENSION" has been removed from spec. Instead, the "ISOLATION_OPTS" variable is used. Tags: pg_query_state --- .travis.yml | 2 -- Makefile | 10 ++-------- README.md | 2 +- specs/corner_cases.spec | 2 -- 4 files changed, 3 insertions(+), 13 deletions(-) diff --git a/.travis.yml b/.travis.yml index 06bae7b..c272005 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,13 +33,11 @@ env: - PG_VERSION=13 - PG_VERSION=12 LEVEL=hardcore USE_TPCDS=1 - PG_VERSION=12 - - PG_VERSION=11 - PG_VERSION=10 - PG_VERSION=9.6 matrix: allow_failures: - env: PG_VERSION=13 LEVEL=hardcore USE_TPCDS=1 - - env: PG_VERSION=11 - env: PG_VERSION=10 - env: PG_VERSION=9.6 diff --git a/Makefile b/Makefile index 4468c51..c96aae2 100644 --- a/Makefile +++ b/Makefile @@ -13,14 +13,8 @@ EXTRA_CLEAN = ./isolation_output $(EXTENSION)--$(EXTVERSION).sql \ Dockerfile ./tests/*.pyc ./tmp_stress ISOLATION = corner_cases -# -# PG11 doesn't support ISOLATION_OPTS variable. We have to use -# "CREATE/DROP EXTENTION" command in spec. -# -# One day, when we'll get rid of PG11, it will be possible to uncomment this -# variable and remove "CREATE EXTENTION" from spec. -# -# ISOLATION_OPTS = --load-extension=pg_query_state + +ISOLATION_OPTS = --load-extension=pg_query_state ifdef USE_PGXS PG_CONFIG ?= pg_config diff --git a/README.md b/README.md index fba15cd..6c983c1 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ To install `pg_query_state`, please apply corresponding patches `custom_signal_( To do this, run the following commands from the postgresql directory: ``` patch -p1 < path_to_pg_query_state_folder/patches/runtime_explain_(PG_VERSION).patch -patch -p1 < path_to_pg_query_state_folder/patches/custom_signal_(PG_VERSION).patch +patch -p1 < path_to_pg_query_state_folder/patches/custom_signals_(PG_VERSION).patch ``` Then execute this in the module's directory: diff --git a/specs/corner_cases.spec b/specs/corner_cases.spec index c9f3fde..315b676 100644 --- a/specs/corner_cases.spec +++ b/specs/corner_cases.spec @@ -1,6 +1,5 @@ setup { - CREATE EXTENSION pg_query_state; CREATE ROLE alice; CREATE ROLE bob; CREATE ROLE super SUPERUSER; @@ -31,7 +30,6 @@ teardown DROP ROLE super; DROP ROLE bob; DROP ROLE alice; - DROP EXTENSION pg_query_state; } session "s1" From 5a0f75b68774bd23a92fc5612d48e837d0daf08b Mon Sep 17 00:00:00 2001 From: Zharkov Roman Date: Tue, 21 Jan 2025 15:55:49 +0300 Subject: [PATCH 90/91] Add meson.build file to support building from the contrib source tree. --- meson.build | 53 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 meson.build diff --git a/meson.build b/meson.build new file mode 100644 index 0000000..b2d4248 --- /dev/null +++ b/meson.build @@ -0,0 +1,53 @@ +# Copyright (c) 2025, Postgres Professional + +# Does not support the PGXS infrastructure at this time. Please, compile as part +# of the contrib source tree. + +pg_query_state_sources = files( + 'pg_query_state.c', + 'signal_handler.c', +) + +if host_system == 'windows' + pg_query_state_sources += rc_lib_gen.process(win32ver_rc, extra_args: [ + '--NAME', 'pg_query_state', + '--FILEDESC', 'pg_query_state - provides facility to know the current state of query execution on working backend.',]) +endif + +pg_query_state = shared_module('pg_query_state', + pg_query_state_sources, + kwargs: contrib_mod_args, +) +contrib_targets += pg_query_state + +extversion = '1.1' +output_name = 'pg_query_state--' + extversion + '.sql' + +configure_file( + input: 'init.sql', + output: output_name, + copy: true, + install: true, + install_dir: contrib_data_args['install_dir'], +) + +install_data( + 'pg_query_state.control', + 'pg_query_state--1.0--1.1.sql', + kwargs: contrib_data_args, +) + +tests += { + 'name': 'pg_query_state', + 'sd': meson.current_source_dir(), + 'bd': meson.current_build_dir(), + 'isolation': { + 'specs': [ + 'corner_cases', + ], + 'regress_args': [ + '--temp-config', files('test.conf'), + '--load-extension=pg_query_state', + ], + }, +} From ee0d4a809821a2a32215aa9f8dcaaa14279c90d8 Mon Sep 17 00:00:00 2001 From: Ekaterina Sokolova Date: Tue, 28 Jan 2025 15:48:16 +0300 Subject: [PATCH 91/91] Disable TPC-DS tests in Travis CI because of using an outdated third-party library. --- .travis.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index c272005..0812444 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,15 +23,15 @@ notifications: env: - PG_VERSION=17 - - PG_VERSION=16 LEVEL=hardcore USE_TPCDS=1 + - PG_VERSION=16 LEVEL=hardcore USE_TPCDS=0 - PG_VERSION=16 - - PG_VERSION=15 LEVEL=hardcore USE_TPCDS=1 + - PG_VERSION=15 LEVEL=hardcore USE_TPCDS=0 - PG_VERSION=15 - - PG_VERSION=14 LEVEL=hardcore USE_TPCDS=1 + - PG_VERSION=14 LEVEL=hardcore USE_TPCDS=0 - PG_VERSION=14 - - PG_VERSION=13 LEVEL=hardcore USE_TPCDS=1 + - PG_VERSION=13 LEVEL=hardcore USE_TPCDS=0 - PG_VERSION=13 - - PG_VERSION=12 LEVEL=hardcore USE_TPCDS=1 + - PG_VERSION=12 LEVEL=hardcore USE_TPCDS=0 - PG_VERSION=12 - PG_VERSION=10 - PG_VERSION=9.6