From ff6dcfb747759de5edf7775c970581b3ee03751a Mon Sep 17 00:00:00 2001 From: Andrew Sawyers Date: Mon, 21 Jan 2019 19:55:21 -0800 Subject: [PATCH 1/8] - fix bug looking at the port value in options --- rethinkdb/utils_common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rethinkdb/utils_common.py b/rethinkdb/utils_common.py index 59b2583f..3a11ed5a 100644 --- a/rethinkdb/utils_common.py +++ b/rethinkdb/utils_common.py @@ -47,7 +47,7 @@ def __init__(self, connect_options): connect_options['port'] = int(connect_options['port']) - if connect_options <= 0: + if connect_options['port'] <= 0: raise AssertionError('Port number can not be less than one') self.__connectOptions = copy.deepcopy(connect_options) From 66edbbbf3549a99207e6bb30987c709fc498ef91 Mon Sep 17 00:00:00 2001 From: Andrew Sawyers Date: Mon, 21 Jan 2019 20:02:09 -0800 Subject: [PATCH 2/8] - fix what is RethinkDB statement --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 45a38c4d..427a539f 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ ## Overview ### What is RethinkDB? -RethinkDB is the fork of RethinkDB which is the first open-source scalable database built for realtime applications. It exposes a new database access model -- instead of polling for changes, the developer can tell the database to continuously push updated query results to applications in realtime. RethinkDB allows developers to build scalable realtime apps in a fraction of the time with less effort. +RethinkDB is the first open-source scalable database built for realtime applications. It exposes a new database access model -- instead of polling for changes, the developer can tell the database to continuously push updated query results to applications in realtime. RethinkDB allows developers to build scalable realtime apps in a fraction of the time with less effort. ## Installation ```bash From d1daf23536e5f2bcf3e95e31f3a3a2e7fa8ce376 Mon Sep 17 00:00:00 2001 From: Andrew Sawyers Date: Tue, 22 Jan 2019 00:33:20 -0800 Subject: [PATCH 3/8] - multiprocessing SimpleQueue takes a context as a required arg lookup the context and pass it into class initalization of the SimpleQueue - for simplification, follow example patterns of importing multiprocessing as mp - multiprocessing.Queue can cause surprising results which are avoided using a queue manager Manager(). See https://docs.python.org/3.7/library/multiprocessing.html - optparse passes in self to the check_existing_file, so set as _ - optparse calls the callback with many more args then originally setup for. - fix env lookup variables which were missed when rebirthdb merged back with rethinkdb --- rethinkdb/_export.py | 28 +++++++++++++++------------- rethinkdb/_import.py | 34 ++++++++++++++++++---------------- rethinkdb/utils_common.py | 23 +++++++++++++---------- 3 files changed, 46 insertions(+), 39 deletions(-) diff --git a/rethinkdb/_export.py b/rethinkdb/_export.py index f6e50f80..0a546777 100755 --- a/rethinkdb/_export.py +++ b/rethinkdb/_export.py @@ -23,7 +23,7 @@ import ctypes import datetime import json -import multiprocessing +import multiprocessing as mp import numbers import optparse import os @@ -259,11 +259,12 @@ def export_table(db, table, directory, options, error_queue, progress_info, sind with sindex_counter.get_lock(): sindex_counter.value += len(table_info["indexes"]) # -- start the writer - task_queue = SimpleQueue() + ctx = mp.get_context(mp.get_start_method()) + task_queue = SimpleQueue(ctx=ctx) writer = None if options.format == "json": filename = directory + "/%s/%s.json" % (db, table) - writer = multiprocessing.Process( + writer = mp.Process( target=json_writer, args=( filename, @@ -273,7 +274,7 @@ def export_table(db, table, directory, options, error_queue, progress_info, sind options.format)) elif options.format == "csv": filename = directory + "/%s/%s.csv" % (db, table) - writer = multiprocessing.Process( + writer = mp.Process( target=csv_writer, args=( filename, @@ -283,7 +284,7 @@ def export_table(db, table, directory, options, error_queue, progress_info, sind error_queue)) elif options.format == "ndjson": filename = directory + "/%s/%s.ndjson" % (db, table) - writer = multiprocessing.Process( + writer = mp.Process( target=json_writer, args=( filename, @@ -388,12 +389,13 @@ def update_progress(progress_info, options): def run_clients(options, workingDir, db_table_set): # Spawn one client for each db.table, up to options.clients at a time - exit_event = multiprocessing.Event() + exit_event = mp.Event() processes = [] - error_queue = SimpleQueue() - interrupt_event = multiprocessing.Event() - sindex_counter = multiprocessing.Value(ctypes.c_longlong, 0) - hook_counter = multiprocessing.Value(ctypes.c_longlong, 0) + ctx = mp.get_context(mp.get_start_method()) + error_queue = SimpleQueue(ctx=ctx) + interrupt_event = mp.Event() + sindex_counter = mp.Value(ctypes.c_longlong, 0) + hook_counter = mp.Value(ctypes.c_longlong, 0) signal.signal(signal.SIGINT, lambda a, b: abort_export(a, b, exit_event, interrupt_event)) errors = [] @@ -405,8 +407,8 @@ def run_clients(options, workingDir, db_table_set): tableSize = int(options.retryQuery("count", query.db(db).table(table).info()['doc_count_estimates'].sum())) - progress_info.append((multiprocessing.Value(ctypes.c_longlong, 0), - multiprocessing.Value(ctypes.c_longlong, tableSize))) + progress_info.append((mp.Value(ctypes.c_longlong, 0), + mp.Value(ctypes.c_longlong, tableSize))) arg_lists.append((db, table, workingDir, options, @@ -428,7 +430,7 @@ def run_clients(options, workingDir, db_table_set): processes = [process for process in processes if process.is_alive()] if len(processes) < options.clients and len(arg_lists) > 0: - newProcess = multiprocessing.Process(target=export_table, args=arg_lists.pop(0)) + newProcess = mp.Process(target=export_table, args=arg_lists.pop(0)) newProcess.start() processes.append(newProcess) diff --git a/rethinkdb/_import.py b/rethinkdb/_import.py index b0fc57db..b118087d 100755 --- a/rethinkdb/_import.py +++ b/rethinkdb/_import.py @@ -26,7 +26,7 @@ import csv import ctypes import json -import multiprocessing +import multiprocessing as mp import optparse import os import signal @@ -110,12 +110,12 @@ def __init__( self.query_runner = query_runner # reporting information - self._bytes_size = multiprocessing.Value(ctypes.c_longlong, -1) - self._bytes_read = multiprocessing.Value(ctypes.c_longlong, -1) + self._bytes_size = mp.Value(ctypes.c_longlong, -1) + self._bytes_read = mp.Value(ctypes.c_longlong, -1) - self._total_rows = multiprocessing.Value(ctypes.c_longlong, -1) - self._rows_read = multiprocessing.Value(ctypes.c_longlong, 0) - self._rows_written = multiprocessing.Value(ctypes.c_longlong, 0) + self._total_rows = mp.Value(ctypes.c_longlong, -1) + self._rows_read = mp.Value(ctypes.c_longlong, 0) + self._rows_written = mp.Value(ctypes.c_longlong, 0) # source if hasattr(source, 'read'): @@ -957,7 +957,7 @@ def table_writer(tables, options, work_queue, error_queue, warning_queue, exit_e nesting_depth=MAX_NESTING_DEPTH), durability=options.durability, conflict=conflict_action, - ignore_write_hook=True)) + )) if res["errors"] > 0: raise RuntimeError("Error when importing into table '%s.%s': %s" % (db, table, res["first_error"])) @@ -1083,13 +1083,15 @@ def import_tables(options, sources, files_ignored=None): tables = dict(((x.db, x.table), x) for x in sources) # (db, table) => table - work_queue = Queue(options.clients * 3) - error_queue = SimpleQueue() - warning_queue = SimpleQueue() - exit_event = multiprocessing.Event() - interrupt_event = multiprocessing.Event() + ctx = mp.get_context(mp.get_start_method()) + max_queue_size = options.clients * 3 + work_queue = mp.Manager().Queue(max_queue_size) + error_queue = SimpleQueue(ctx=ctx) + warning_queue = SimpleQueue(ctx=ctx) + exit_event = mp.Event() + interrupt_event = mp.Event() - timing_queue = SimpleQueue() + timing_queue = SimpleQueue(ctx=ctx) errors = [] warnings = [] @@ -1166,7 +1168,7 @@ def drain_queues(): try: # - start the progress bar if not options.quiet: - progress_bar = multiprocessing.Process( + progress_bar = mp.Process( target=update_progress, name="progress bar", args=(sources, options.debug, exit_event, progress_bar_sleep) @@ -1178,7 +1180,7 @@ def drain_queues(): writers = [] pools.append(writers) for i in range(options.clients): - writer = multiprocessing.Process( + writer = mp.Process( target=table_writer, name="table writer %d" % i, @@ -1202,7 +1204,7 @@ def drain_queues(): # add a workers to fill up the readers pool while len(readers) < options.clients: table = next(file_iter) - reader = multiprocessing.Process( + reader = mp.Process( target=table.read_to_queue, name="table reader %s.%s" % (table.db, diff --git a/rethinkdb/utils_common.py b/rethinkdb/utils_common.py index 3a11ed5a..6414b128 100644 --- a/rethinkdb/utils_common.py +++ b/rethinkdb/utils_common.py @@ -151,7 +151,6 @@ def format_epilog(self, formatter): return self.epilog or '' def __init__(self, *args, **kwargs): - # -- Type Checkers def check_tls_option(opt_str, value): @@ -178,7 +177,7 @@ def check_positive_int(opt_str, value): return int(value) - def check_existing_file(opt_str, value): + def check_existing_file(_, opt_str, value): if not os.path.isfile(value): raise optparse.OptionValueError('%s value was not an existing file: %s' % (opt_str, value)) @@ -207,7 +206,10 @@ def file_contents(opt_str, value): # -- Callbacks - def combined_connect_action(value, parser): + def combined_connect_action(obj, opt, value, parser, *args, **kwargs): + """optparse.takeaction() calls the callback (which this is set as) + with the following args: self, opt, value, parser *args, **kwargs + """ res = self.__connectRegex.match(value) if not res: raise optparse.OptionValueError("Invalid 'host:port' format: %s" % value) @@ -295,7 +297,7 @@ def take_action(self, action, dest, opt, value, values, parser): help='driver port of a rethinkdb server', type='int', default=os.environ.get( - 'REBIRTHDB_DRIVER_PORT', + 'RETHINKDB_DRIVER_PORT', net.DEFAULT_PORT)) connection_group.add_option( '--host-name', @@ -303,7 +305,7 @@ def take_action(self, action, dest, opt, value, values, parser): metavar='HOST', help='host and driver port of a rethinkdb server', default=os.environ.get( - 'REBIRTHDB_HOSTNAME', + 'RETHINKDB_HOSTNAME', 'localhost')) connection_group.add_option( '-u', @@ -312,7 +314,7 @@ def take_action(self, action, dest, opt, value, values, parser): metavar='USERNAME', help='user name to connect as', default=os.environ.get( - 'REBIRTHDB_USER', + 'RETHINKDB_USER', 'admin')) connection_group.add_option( '-p', @@ -344,12 +346,13 @@ def parse_args(self, *args, **kwargs): # - validate ENV variables - if 'REBIRTHDB_DRIVER_PORT' in os.environ: - driver_port = os.environ['REBIRTHDB_DRIVER_PORT'] + if 'RETHINKDB_DRIVER_PORT' in os.environ: + driver_port = os.environ['RETHINKDB_DRIVER_PORT'] if not isinstance(driver_port, int) or driver_port < 1: - self.error('ENV variable REBIRTHDB_DRIVER_PORT is not a useable integer: %s' - % os.environ['REBIRTHDB_DRIVER_PORT']) + self.error('ENV variable RETHINKDB_DRIVER_PORT is not a useable ' + 'integer: %s' + % os.environ['RETHINKDB_DRIVER_PORT']) # - parse options From 1e7d612c4a40974c98ddd0bc0c967dce832bad63 Mon Sep 17 00:00:00 2001 From: Andrew Sawyers Date: Sun, 27 Jan 2019 01:22:40 -0800 Subject: [PATCH 4/8] remove more references to rebirth and rebirth assets --- Makefile | 14 +++++++------- scripts/install-db.sh | 8 +++++--- tests/helpers.py | 4 ++-- tests/integration/test_ping.py | 10 +++++----- 4 files changed, 19 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index 493869bc..1ca473f0 100644 --- a/Makefile +++ b/Makefile @@ -17,14 +17,14 @@ PACKAGE_NAME = rethinkdb PROTO_FILE_NAME = ql2.proto -PROTO_FILE_URL = https://raw.githubusercontent.com/RebirthDB/rebirthdb/next/src/rdb_protocol/${PROTO_FILE_NAME} +PROTO_FILE_URL = https://raw.githubusercontent.com/rethinkdb/rethinkdb/next/src/rdb_protocol/${PROTO_FILE_NAME} TARGET_PROTO_FILE = ${PACKAGE_NAME}/${PROTO_FILE_NAME} FILE_CONVERTER_NAME = convert_protofile.py -FILE_CONVERTER_URL = https://raw.githubusercontent.com/RebirthDB/rebirthdb/next/scripts/${FILE_CONVERTER_NAME} +FILE_CONVERTER_URL = https://raw.githubusercontent.com/rethinkdb/rethinkdb/next/scripts/${FILE_CONVERTER_NAME} REMOTE_TEST_SETUP_NAME = prepare_remote_test.py -REMOTE_TEST_SETUP_URL = https://raw.githubusercontent.com/RebirthDB/rebirthdb/next/scripts/${REMOTE_TEST_SETUP_NAME} +REMOTE_TEST_SETUP_URL = https://raw.githubusercontent.com/rethinkdb/rethinkdb/next/scripts/${REMOTE_TEST_SETUP_NAME} CONVERTED_PROTO_FILE_NAME = ql2_pb2.py TARGET_CONVERTED_PROTO_FILE = ${PACKAGE_NAME}/${CONVERTED_PROTO_FILE_NAME} @@ -48,14 +48,14 @@ test-unit: pytest -v -m unit test-integration: - @rebirthdb& + @rethinkdb& pytest -v -m integration - @killall rebirthdb + @killall rethinkdb test-ci: - @rebirthdb& + @rethinkdb& pytest -v --cov rethinkdb --cov-report xml - @killall rebirthdb + @killall rethinkdb test-remote: python ${REMOTE_TEST_SETUP_NAME} pytest -m integration diff --git a/scripts/install-db.sh b/scripts/install-db.sh index 3a78d83e..f01f947a 100755 --- a/scripts/install-db.sh +++ b/scripts/install-db.sh @@ -5,8 +5,10 @@ set -u export DISTRIB_CODENAME=$(lsb_release -sc) -echo "deb https://dl.bintray.com/rebirthdb/apt $DISTRIB_CODENAME main" | sudo tee /etc/apt/sources.list.d/rebirthdb.list -wget -qO- https://dl.bintray.com/rebirthdb/keys/pubkey.gpg | sudo apt-key add - +echo "This currently will not work for rethinkdb. It is in the process of being fixed." +exit 1 +echo "deb https://dl.bintray.com/rethinkdb/apt $DISTRIB_CODENAME main" | sudo tee /etc/apt/sources.list.d/rethinkdb.list +wget -qO- https://dl.bintray.com/rethinkdb/keys/pubkey.gpg | sudo apt-key add - sudo apt-get update --option Acquire::Retries=100 --option Acquire::http::Timeout="300" -sudo apt-get --allow-unauthenticated install rebirthdb --option Acquire::Retries=100 --option Acquire::http::Timeout="300" +sudo apt-get --allow-unauthenticated install rethinkdb --option Acquire::Retries=100 --option Acquire::http::Timeout="300" diff --git a/tests/helpers.py b/tests/helpers.py index 758784f6..61fef4a2 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -11,11 +11,11 @@ class IntegrationTestCaseBase(object): def connect(self): self.conn = self.r.connect( - host=self.rebirthdb_host + host=self.rethinkdb_host ) def setup_method(self): - self.rebirthdb_host=os.getenv('REBIRTHDB_HOST') + self.rethinkdb_host=os.getenv('RETHINKDB_HOST') self.connect() diff --git a/tests/integration/test_ping.py b/tests/integration/test_ping.py index f8b89532..5d26cbcf 100644 --- a/tests/integration/test_ping.py +++ b/tests/integration/test_ping.py @@ -10,7 +10,7 @@ @pytest.mark.integration class TestPing(IntegrationTestCaseBase): def teardown_method(self): - with self.r.connect(host=self.rebirthdb_host) as conn: + with self.r.connect(host=self.rethinkdb_host) as conn: self.r.db("rethinkdb").table("users").filter( self.r.row["id"].ne("admin") ).delete().run(conn) @@ -18,11 +18,11 @@ def teardown_method(self): def test_bad_password(self): with pytest.raises(self.r.ReqlAuthError): - self.r.connect(password=BAD_PASSWORD, host=self.rebirthdb_host) + self.r.connect(password=BAD_PASSWORD, host=self.rethinkdb_host) def test_password_connect(self): new_user = "user" - with self.r.connect(user="admin", password="", host=self.rebirthdb_host) as conn: + with self.r.connect(user="admin", password="", host=self.rethinkdb_host) as conn: curr = self.r.db("rethinkdb").table("users").insert( {"id": new_user, "password": BAD_PASSWORD} ).run(conn) @@ -40,7 +40,7 @@ def test_password_connect(self): { 'new_val': {'read': True}, 'old_val': None}]} - with self.r.connect(user=new_user, password=BAD_PASSWORD, host=self.rebirthdb_host) as conn: + with self.r.connect(user=new_user, password=BAD_PASSWORD, host=self.rethinkdb_host) as conn: curr = self.r.db("rethinkdb").table("users").get("admin").run(conn) assert curr == {'id': 'admin', 'password': False} with pytest.raises(self.r.ReqlPermissionError): @@ -49,6 +49,6 @@ def test_password_connect(self): ).run(conn) def test_context_manager(self): - with self.r.connect(host=self.rebirthdb_host) as conn: + with self.r.connect(host=self.rethinkdb_host) as conn: assert conn.is_open() is True assert conn.is_open() is False From 50aafdedc4ad236e752ec48abd228e4f008e964a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Boros?= Date: Sat, 20 Apr 2019 11:26:35 +0200 Subject: [PATCH 5/8] Do not use python3 only syntax --- setup.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index e9d5894c..22f99249 100644 --- a/setup.py +++ b/setup.py @@ -44,10 +44,11 @@ if MATCH.group("post"): VERSION += "." + MATCH.group("post") - with open("rethinkdb/version.py", "w") as ostream: - print("# Autogenerated version", file=ostream) - print(file=ostream) - print("VERSION", "=", repr(VERSION), file=ostream) + with open("rethinkdb/version.py", "w") as f: + f.writelines([ + "# Autogenerated version", + "VERSION = {0}".format(VERSION) + ]) else: raise RuntimeError("{!r} does not match version format {!r}".format( RETHINKDB_VERSION_DESCRIBE, VERSION_RE)) From f57c7031e9140db95629a89910155e1125bf66d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Boros?= Date: Sat, 20 Apr 2019 11:26:54 +0200 Subject: [PATCH 6/8] Fix smaller issues --- rethinkdb/_export.py | 34 ++++++++++++++++++++-------------- rethinkdb/utils_common.py | 8 ++++---- 2 files changed, 24 insertions(+), 18 deletions(-) diff --git a/rethinkdb/_export.py b/rethinkdb/_export.py index 0a546777..7c2e88d5 100755 --- a/rethinkdb/_export.py +++ b/rethinkdb/_export.py @@ -23,7 +23,7 @@ import ctypes import datetime import json -import multiprocessing as mp +import multiprocessing import numbers import optparse import os @@ -35,6 +35,8 @@ import traceback from multiprocessing.queues import SimpleQueue +import six + from rethinkdb import errors, query, utils_common from rethinkdb.logger import default_logger @@ -259,12 +261,16 @@ def export_table(db, table, directory, options, error_queue, progress_info, sind with sindex_counter.get_lock(): sindex_counter.value += len(table_info["indexes"]) # -- start the writer - ctx = mp.get_context(mp.get_start_method()) - task_queue = SimpleQueue(ctx=ctx) + if six.PY3: + ctx = multiprocessing.get_context(multiprocessing.get_start_method()) + task_queue = SimpleQueue(ctx=ctx) + else: + task_queue = SimpleQueue() + writer = None if options.format == "json": filename = directory + "/%s/%s.json" % (db, table) - writer = mp.Process( + writer = multiprocessing.Process( target=json_writer, args=( filename, @@ -274,7 +280,7 @@ def export_table(db, table, directory, options, error_queue, progress_info, sind options.format)) elif options.format == "csv": filename = directory + "/%s/%s.csv" % (db, table) - writer = mp.Process( + writer = multiprocessing.Process( target=csv_writer, args=( filename, @@ -284,7 +290,7 @@ def export_table(db, table, directory, options, error_queue, progress_info, sind error_queue)) elif options.format == "ndjson": filename = directory + "/%s/%s.ndjson" % (db, table) - writer = mp.Process( + writer = multiprocessing.Process( target=json_writer, args=( filename, @@ -389,13 +395,13 @@ def update_progress(progress_info, options): def run_clients(options, workingDir, db_table_set): # Spawn one client for each db.table, up to options.clients at a time - exit_event = mp.Event() + exit_event = multiprocessing.Event() processes = [] - ctx = mp.get_context(mp.get_start_method()) + ctx = multiprocessing.get_context(multiprocessing.get_start_method()) error_queue = SimpleQueue(ctx=ctx) - interrupt_event = mp.Event() - sindex_counter = mp.Value(ctypes.c_longlong, 0) - hook_counter = mp.Value(ctypes.c_longlong, 0) + interrupt_event = multiprocessing.Event() + sindex_counter = multiprocessing.Value(ctypes.c_longlong, 0) + hook_counter = multiprocessing.Value(ctypes.c_longlong, 0) signal.signal(signal.SIGINT, lambda a, b: abort_export(a, b, exit_event, interrupt_event)) errors = [] @@ -407,8 +413,8 @@ def run_clients(options, workingDir, db_table_set): tableSize = int(options.retryQuery("count", query.db(db).table(table).info()['doc_count_estimates'].sum())) - progress_info.append((mp.Value(ctypes.c_longlong, 0), - mp.Value(ctypes.c_longlong, tableSize))) + progress_info.append((multiprocessing.Value(ctypes.c_longlong, 0), + multiprocessing.Value(ctypes.c_longlong, tableSize))) arg_lists.append((db, table, workingDir, options, @@ -430,7 +436,7 @@ def run_clients(options, workingDir, db_table_set): processes = [process for process in processes if process.is_alive()] if len(processes) < options.clients and len(arg_lists) > 0: - newProcess = mp.Process(target=export_table, args=arg_lists.pop(0)) + newProcess = multiprocessing.Process(target=export_table, args=arg_lists.pop(0)) newProcess.start() processes.append(newProcess) diff --git a/rethinkdb/utils_common.py b/rethinkdb/utils_common.py index 8d3c9ac1..823e2fa9 100644 --- a/rethinkdb/utils_common.py +++ b/rethinkdb/utils_common.py @@ -129,7 +129,7 @@ def check_minimum_version(options, minimum_version='1.6'): version_string = options.retryQuery('get server version', query.db( 'rethinkdb').table('server_status')[0]['process']['version']) - matches = re.match(r'rethinkdb (?P(\d+)\.(\d+)\.(\d+)).*', version_string) + matches = re.match(r'(rethinkdb|rebirthdb) (?P(\d+)\.(\d+)\.(\d+)).*', version_string) if not matches: raise RuntimeError("invalid version string format: %s" % version_string) @@ -285,11 +285,11 @@ def take_action(self, action, dest, opt, value, values, parser): '--connect', dest='driver_port', metavar='HOST:PORT', - help='host and client port of a rethinkdb node to connect (default: localhost:%d)' % - net.DEFAULT_PORT, + help='host and client port of a rethinkdb node to connect (default: localhost:%d)' % net.DEFAULT_PORT, action='callback', callback=combined_connect_action, - type='string') + type='str' + ) connection_group.add_option( '--driver-port', dest='driver_port', From 2622bdaf82cb7e231efe2de6329126a1ff513367 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Boros?= Date: Sat, 20 Apr 2019 11:45:02 +0200 Subject: [PATCH 7/8] Fix python2 compatibility issues --- rethinkdb/_export.py | 7 +++++-- rethinkdb/_import.py | 39 +++++++++++++++++++++++---------------- 2 files changed, 28 insertions(+), 18 deletions(-) diff --git a/rethinkdb/_export.py b/rethinkdb/_export.py index 7c2e88d5..e25bef42 100755 --- a/rethinkdb/_export.py +++ b/rethinkdb/_export.py @@ -397,8 +397,11 @@ def run_clients(options, workingDir, db_table_set): # Spawn one client for each db.table, up to options.clients at a time exit_event = multiprocessing.Event() processes = [] - ctx = multiprocessing.get_context(multiprocessing.get_start_method()) - error_queue = SimpleQueue(ctx=ctx) + if six.PY3: + ctx = multiprocessing.get_context(multiprocessing.get_start_method()) + error_queue = SimpleQueue(ctx=ctx) + else: + error_queue = SimpleQueue() interrupt_event = multiprocessing.Event() sindex_counter = multiprocessing.Value(ctypes.c_longlong, 0) hook_counter = multiprocessing.Value(ctypes.c_longlong, 0) diff --git a/rethinkdb/_import.py b/rethinkdb/_import.py index b118087d..032c57c2 100755 --- a/rethinkdb/_import.py +++ b/rethinkdb/_import.py @@ -26,13 +26,14 @@ import csv import ctypes import json -import multiprocessing as mp +import multiprocessing import optparse import os import signal import sys import time import traceback +import six from multiprocessing.queues import Queue, SimpleQueue from rethinkdb import ast, errors, query, utils_common @@ -110,12 +111,12 @@ def __init__( self.query_runner = query_runner # reporting information - self._bytes_size = mp.Value(ctypes.c_longlong, -1) - self._bytes_read = mp.Value(ctypes.c_longlong, -1) + self._bytes_size = multiprocessing.Value(ctypes.c_longlong, -1) + self._bytes_read = multiprocessing.Value(ctypes.c_longlong, -1) - self._total_rows = mp.Value(ctypes.c_longlong, -1) - self._rows_read = mp.Value(ctypes.c_longlong, 0) - self._rows_written = mp.Value(ctypes.c_longlong, 0) + self._total_rows = multiprocessing.Value(ctypes.c_longlong, -1) + self._rows_read = multiprocessing.Value(ctypes.c_longlong, 0) + self._rows_written = multiprocessing.Value(ctypes.c_longlong, 0) # source if hasattr(source, 'read'): @@ -1083,15 +1084,21 @@ def import_tables(options, sources, files_ignored=None): tables = dict(((x.db, x.table), x) for x in sources) # (db, table) => table - ctx = mp.get_context(mp.get_start_method()) + if six.PY3: + ctx = multiprocessing.get_context(multiprocessing.get_start_method()) + error_queue = SimpleQueue(ctx=ctx) + warning_queue = SimpleQueue(ctx=ctx) + timing_queue = SimpleQueue(ctx=ctx) + else: + error_queue = SimpleQueue() + warning_queue = SimpleQueue() + timing_queue = SimpleQueue() + max_queue_size = options.clients * 3 - work_queue = mp.Manager().Queue(max_queue_size) - error_queue = SimpleQueue(ctx=ctx) - warning_queue = SimpleQueue(ctx=ctx) - exit_event = mp.Event() - interrupt_event = mp.Event() + work_queue = multiprocessing.Manager().Queue(max_queue_size) - timing_queue = SimpleQueue(ctx=ctx) + exit_event = multiprocessing.Event() + interrupt_event = multiprocessing.Event() errors = [] warnings = [] @@ -1168,7 +1175,7 @@ def drain_queues(): try: # - start the progress bar if not options.quiet: - progress_bar = mp.Process( + progress_bar = multiprocessing.Process( target=update_progress, name="progress bar", args=(sources, options.debug, exit_event, progress_bar_sleep) @@ -1180,7 +1187,7 @@ def drain_queues(): writers = [] pools.append(writers) for i in range(options.clients): - writer = mp.Process( + writer = multiprocessing.Process( target=table_writer, name="table writer %d" % i, @@ -1204,7 +1211,7 @@ def drain_queues(): # add a workers to fill up the readers pool while len(readers) < options.clients: table = next(file_iter) - reader = mp.Process( + reader = multiprocessing.Process( target=table.read_to_queue, name="table reader %s.%s" % (table.db, From b4d8125e551a3ae58f6dd6e3aa244469927dbc0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Boros?= Date: Sat, 20 Apr 2019 11:52:48 +0200 Subject: [PATCH 8/8] Fix variable naming --- rethinkdb/_export.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rethinkdb/_export.py b/rethinkdb/_export.py index e25bef42..e57ebd12 100755 --- a/rethinkdb/_export.py +++ b/rethinkdb/_export.py @@ -439,9 +439,9 @@ def run_clients(options, workingDir, db_table_set): processes = [process for process in processes if process.is_alive()] if len(processes) < options.clients and len(arg_lists) > 0: - newProcess = multiprocessing.Process(target=export_table, args=arg_lists.pop(0)) - newProcess.start() - processes.append(newProcess) + new_process = multiprocessing.Process(target=export_table, args=arg_lists.pop(0)) + new_process.start() + processes.append(new_process) update_progress(progress_info, options)