From f44cbbf22bbb182cd1d149677e03a5619f3a92a3 Mon Sep 17 00:00:00 2001 From: Jayson Reis Date: Sun, 11 Oct 2020 10:45:46 +0200 Subject: [PATCH 1/8] Update changelog --- CHANGELOG.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 3b10d29..c5b70a6 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,11 @@ CHANGELOG --------- +v0.31.1 +``````` + +- Implement Protocol.__str__ for real consistent hashing + test (#237) + v0.30.0 ``````` From 1fcf030eb8adede0247e14c6f99d8005a9cef909 Mon Sep 17 00:00:00 2001 From: Jayson Reis Date: Sun, 10 Oct 2021 00:06:00 +0200 Subject: [PATCH 2/8] Move CI to GH --- .ci-before-script.sh | 9 +++++ .travis-runs-tests.sh => .ci-runs-tests.sh | 10 +++-- .github/workflows/tests-and-lint.yml | 33 +++++++++++++++++ .travis-before-script.sh | 14 ------- .vscode/settings.json | 3 ++ setup.py | 43 +++++++++++++--------- 6 files changed, 77 insertions(+), 35 deletions(-) create mode 100755 .ci-before-script.sh rename .travis-runs-tests.sh => .ci-runs-tests.sh (73%) create mode 100644 .github/workflows/tests-and-lint.yml delete mode 100755 .travis-before-script.sh create mode 100644 .vscode/settings.json diff --git a/.ci-before-script.sh b/.ci-before-script.sh new file mode 100755 index 0000000..190aaf3 --- /dev/null +++ b/.ci-before-script.sh @@ -0,0 +1,9 @@ +#!/bin/bash +set -ex + +if [ "$STEP" != "tests" ]; then + exit 0 +fi + +sudo apt-get update +sudo apt install memcached \ No newline at end of file diff --git a/.travis-runs-tests.sh b/.ci-runs-tests.sh similarity index 73% rename from .travis-runs-tests.sh rename to .ci-runs-tests.sh index 70b06a6..3e6f4fe 100755 --- a/.travis-runs-tests.sh +++ b/.ci-runs-tests.sh @@ -1,14 +1,18 @@ #!/bin/bash -set -e -set -x +set -ex +env if [ "$STEP" = "tests" ]; then - sudo service memcached stop py.test --version export PYTHONPATH=. python setup.py develop py.test --cov=bmemcached + exit 0 fi if [ "$STEP" = "lint" ]; then flake8 + exit 0 fi + +echo "Unknown step: $STEP" +exit 1 diff --git a/.github/workflows/tests-and-lint.yml b/.github/workflows/tests-and-lint.yml new file mode 100644 index 0000000..367c0b3 --- /dev/null +++ b/.github/workflows/tests-and-lint.yml @@ -0,0 +1,33 @@ +name: Tests and Lint + +on: + push: + branches: [master] + pull_request: + branches: [master] + +jobs: + build: + runs-on: ubuntu-latest + strategy: + max-parallel: 4 + matrix: + python-version: [2.7, 3.6, 3.7, 3.8, 3.9] + step: + - tests + - lint + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install Dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements_test.txt && python setup.py develop + STEP="${{ matrix.step}}" ./.ci-before-script.sh + - name: Run tests or lint + run: | + STEP="${{ matrix.step}}" ./.ci-runs-tests.sh diff --git a/.travis-before-script.sh b/.travis-before-script.sh deleted file mode 100755 index d2fd9ff..0000000 --- a/.travis-before-script.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -set -ex - -if [ "$STEP" != "tests" ]; then - exit 0 -fi - -sudo apt-get update -sudo apt-get remove memcached -sudo apt-get build-dep memcached -sudo apt-get install libssl-dev -wget https://memcached.org/files/memcached-1.5.21.tar.gz -tar zxvf memcached-1.5.21.tar.gz -pushd memcached-1.5.21 && ./configure --prefix=/usr --enable-tls && make && sudo make install && popd diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..de288e1 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "python.formatting.provider": "black" +} \ No newline at end of file diff --git a/setup.py b/setup.py index 960eeb1..965d0b0 100644 --- a/setup.py +++ b/setup.py @@ -1,32 +1,39 @@ import os from setuptools import setup +import sys def read(filename): return open(os.path.join(os.path.dirname(__file__), filename)).read() +version_dependant_requirements = [ + "uhashring < 2" if sys.version_info < (3, 6) else "uhashring", # It uses f-strings +] + setup( - name='python-binary-memcached', - version='0.30.1', - author='Jayson Reis', - author_email='santosdosreis@gmail.com', - description='A pure python module to access memcached via its binary protocol with SASL auth support', - long_description='{0}\n{1}'.format(read('README.rst'), read('CHANGELOG.rst')), - url='https://github.com/jaysonsantos/python-binary-memcached', - packages=['bmemcached', 'bmemcached.client'], + name="python-binary-memcached", + version="0.30.1", + author="Jayson Reis", + author_email="santosdosreis@gmail.com", + description="A pure python module to access memcached via its binary protocol with SASL auth support", + long_description="{0}\n{1}".format(read("README.rst"), read("CHANGELOG.rst")), + url="https://github.com/jaysonsantos/python-binary-memcached", + packages=["bmemcached", "bmemcached.client"], classifiers=[ - 'Development Status :: 4 - Beta', - 'License :: OSI Approved :: MIT License', - 'Operating System :: OS Independent', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3.4', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', + "Development Status :: 4 - Beta", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3.4", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", ], install_requires=[ - 'six', - 'uhashring', + "six", ] + + version_dependant_requirements, ) From fb3d307528d74d1ea718f9e3af2f5feddacaeecd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 9 Oct 2021 22:52:45 +0000 Subject: [PATCH 3/8] Bump pygments from 2.2.0 to 2.7.4 in /docs Bumps [pygments](https://github.com/pygments/pygments) from 2.2.0 to 2.7.4. - [Release notes](https://github.com/pygments/pygments/releases) - [Changelog](https://github.com/pygments/pygments/blob/master/CHANGES) - [Commits](https://github.com/pygments/pygments/compare/2.2.0...2.7.4) Signed-off-by: dependabot[bot] --- docs/docs_requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docs_requirements.txt b/docs/docs_requirements.txt index 9929249..0c435ed 100644 --- a/docs/docs_requirements.txt +++ b/docs/docs_requirements.txt @@ -1,3 +1,3 @@ Sphinx==1.8.1 -Pygments==2.2.0 +Pygments==2.7.4 Sphinx==1.8.1 From 41ed6136a029eda65271055c88f1bbbb4e4f1aba Mon Sep 17 00:00:00 2001 From: bisho Date: Fri, 15 Oct 2021 14:29:31 -0700 Subject: [PATCH 4/8] feat: Expose incr/decr `default` and `time` protocol arguments in client class (#243) The protocol implements `default` and `time` arguments, but those are not exposed in the client class implementation. This adds those fields. --- bmemcached/client/distributed.py | 16 ++++++++++++---- bmemcached/client/replicating.py | 16 ++++++++++++---- test/test_simple_functions.py | 8 ++++++++ 3 files changed, 32 insertions(+), 8 deletions(-) diff --git a/bmemcached/client/distributed.py b/bmemcached/client/distributed.py index 66000e1..80cb242 100644 --- a/bmemcached/client/distributed.py +++ b/bmemcached/client/distributed.py @@ -204,7 +204,7 @@ def cas(self, key, value, cas, time=0, compress_level=-1): server = self._get_server(key) return server.cas(key, value, cas, time, compress_level) - def incr(self, key, value): + def incr(self, key, value, default=0, time=1000000): """ Increment a key, if it exists, returns it's actual value, if it don't, return 0. @@ -212,13 +212,17 @@ def incr(self, key, value): :type key: six.string_types :param value: Number to be incremented :type value: int + :param default: If key not set, initialize to this value + :type default: int + :param time: Time in seconds that your key will expire. + :type time: int :return: Actual value of the key on server :rtype: int """ server = self._get_server(key) - return server.incr(key, value) + return server.incr(key, value, default=default, time=time) - def decr(self, key, value): + def decr(self, key, value, default=0, time=1000000): """ Decrement a key, if it exists, returns it's actual value, if it don't, return 0. Minimum value of decrement return is 0. @@ -227,8 +231,12 @@ def decr(self, key, value): :type key: six.string_types :param value: Number to be decremented :type value: int + :param default: If key not set, initialize to this value + :type default: int + :param time: Time in seconds that your key will expire. + :type time: int :return: Actual value of the key on server :rtype: int """ server = self._get_server(key) - return server.decr(key, value) + return server.decr(key, value, default=default, time=time) diff --git a/bmemcached/client/replicating.py b/bmemcached/client/replicating.py index 5bd9e5c..2cd7ff7 100644 --- a/bmemcached/client/replicating.py +++ b/bmemcached/client/replicating.py @@ -232,7 +232,7 @@ def delete_multi(self, keys): return all(returns) - def incr(self, key, value): + def incr(self, key, value, default=0, time=1000000): """ Increment a key, if it exists, returns it's actual value, if it don't, return 0. @@ -240,16 +240,20 @@ def incr(self, key, value): :type key: six.string_types :param value: Number to be incremented :type value: int + :param default: If key not set, initialize to this value + :type default: int + :param time: Time in seconds that your key will expire. + :type time: int :return: Actual value of the key on server :rtype: int """ returns = [] for server in self.servers: - returns.append(server.incr(key, value)) + returns.append(server.incr(key, value, default=default, time=time)) return returns[0] - def decr(self, key, value): + def decr(self, key, value, default=0, time=1000000): """ Decrement a key, if it exists, returns it's actual value, if it don't, return 0. Minimum value of decrement return is 0. @@ -258,11 +262,15 @@ def decr(self, key, value): :type key: six.string_types :param value: Number to be decremented :type value: int + :param default: If key not set, initialize to this value + :type default: int + :param time: Time in seconds that your key will expire. + :type time: int :return: Actual value of the key on server :rtype: int """ returns = [] for server in self.servers: - returns.append(server.decr(key, value)) + returns.append(server.decr(key, value, default=default, time=time)) return returns[0] diff --git a/test/test_simple_functions.py b/test/test_simple_functions.py index c04c7e5..f69ce26 100644 --- a/test/test_simple_functions.py +++ b/test/test_simple_functions.py @@ -208,10 +208,18 @@ def testIncrement(self): self.assertEqual(0, self.client.incr('test_key', 1)) self.assertEqual(1, self.client.incr('test_key', 1)) + def testIncrementInitialize(self): + self.assertEqual(10, self.client.incr('test_key', 1, default=10)) + self.assertEqual(11, self.client.incr('test_key', 1, default=10)) + def testDecrement(self): self.assertEqual(0, self.client.decr('test_key', 1)) self.assertEqual(0, self.client.decr('test_key', 1)) + def testDecrementInitialize(self): + self.assertEqual(10, self.client.decr('test_key', 1, default=10)) + self.assertEqual(9, self.client.decr('test_key', 1, default=10)) + def testFlush(self): self.client.set('test_key', 'test') self.assertTrue(self.client.flush_all()) From f3fbae52957ef1ec2e036fac60df2fe5b99085ea Mon Sep 17 00:00:00 2001 From: Alex Vandiver Date: Tue, 28 Dec 2021 06:58:10 -0800 Subject: [PATCH 5/8] refactor: Use bytearrays for building up bytes for I/O. (#245) Bytes are immutable in python, which means that repeatedly appending to them is _quadratic_ in time. That is, we would like and expect that deleting 10k objects 10 times would take about as long as deleting 100k, but because we repeatedly concatenate to a bytes object in the loop, they are not: timeit.timeit(lambda: server.delete_multi(["foo"]*10000),number=10) # 0.5087270829999966 timeit.timeit(lambda: server.delete_multi(["foo"]*100000),number=1) # 10.650619775999985 Switch to using `bytearray` in all places which handle variable numbers of bytes. After this change: timeit.timeit(lambda: server.delete_multi(["foo"]*10000),number=10) # 0.1197161969999998 timeit.timeit(lambda: server.delete_multi(["foo"]*100000),number=1) # 0.1269589350000011 --- bmemcached/protocol.py | 40 ++++++++++++++++++---------------------- 1 file changed, 18 insertions(+), 22 deletions(-) diff --git a/bmemcached/protocol.py b/bmemcached/protocol.py index 1732f0f..e45e010 100644 --- a/bmemcached/protocol.py +++ b/bmemcached/protocol.py @@ -195,7 +195,7 @@ def _read_socket(self, size): :param size: Size in bytes to be read. :return: Data from socket """ - value = b'' + value = bytearray() while len(value) < size: data = self.connection.recv(size - len(value)) if not data: @@ -206,7 +206,7 @@ def _read_socket(self, size): if len(value) < size: raise socket.error() - return value + return bytes(value) def _get_response(self): """ @@ -501,7 +501,7 @@ def get_multi(self, keys): if n == 0: return {} - msg = b'' + msg = bytearray() for i, key in enumerate(keys): keybytes = str_to_bytes(key) command = self.COMMANDS['getk' if i == n - 1 else 'getkq'] @@ -690,7 +690,7 @@ def set_multi(self, mappings, time=100, compress_level=-1): :rtype: list """ mappings = list(mappings.items()) - msg = [] + msg = bytearray() for opaque, (key, value) in enumerate(mappings): if isinstance(key, tuple): @@ -707,23 +707,19 @@ def set_multi(self, mappings, time=100, compress_level=-1): keybytes = str_to_bytes(key) flags, value = self.serialize(value, compress_level=compress_level) - m = struct.pack(self.HEADER_STRUCT + - self.COMMANDS[command]['struct'] % (len(keybytes), len(value)), - self.MAGIC['request'], - self.COMMANDS[command]['command'], - len(keybytes), - 8, 0, 0, len(keybytes) + len(value) + 8, opaque, cas or 0, - flags, time, keybytes, value) - msg.append(m) - - m = struct.pack(self.HEADER_STRUCT + - self.COMMANDS['noop']['struct'], - self.MAGIC['request'], - self.COMMANDS['noop']['command'], - 0, 0, 0, 0, 0, 0, 0) - msg.append(m) - - msg = b''.join(msg) + msg += struct.pack(self.HEADER_STRUCT + + self.COMMANDS[command]['struct'] % (len(keybytes), len(value)), + self.MAGIC['request'], + self.COMMANDS[command]['command'], + len(keybytes), + 8, 0, 0, len(keybytes) + len(value) + 8, opaque, cas or 0, + flags, time, keybytes, value) + + msg += struct.pack(self.HEADER_STRUCT + + self.COMMANDS['noop']['struct'], + self.MAGIC['request'], + self.COMMANDS['noop']['command'], + 0, 0, 0, 0, 0, 0, 0) self._send(msg) @@ -854,7 +850,7 @@ def delete_multi(self, keys): :rtype: bool """ logger.debug('Deleting keys %r', keys) - msg = b'' + msg = bytearray() for key in keys: keybytes = str_to_bytes(key) msg += struct.pack( From 84d940d90f62e3ae1927c908f323a62c2edaba9d Mon Sep 17 00:00:00 2001 From: Jayson Reis Date: Tue, 28 Dec 2021 16:22:36 +0100 Subject: [PATCH 6/8] chore: Change from bumpversion to commitizen --- .bumpversion.cfg | 8 -------- .cz.yaml | 4 ++++ .pre-commit-config.yaml | 7 +++++++ 3 files changed, 11 insertions(+), 8 deletions(-) delete mode 100644 .bumpversion.cfg create mode 100644 .cz.yaml create mode 100644 .pre-commit-config.yaml diff --git a/.bumpversion.cfg b/.bumpversion.cfg deleted file mode 100644 index 7c8a004..0000000 --- a/.bumpversion.cfg +++ /dev/null @@ -1,8 +0,0 @@ -[bumpversion] -commit = True -tag = True -current_version = 0.30.1 - -[bumpversion:file:setup.py] - -[bumpversion:file:docs/conf.py] diff --git a/.cz.yaml b/.cz.yaml new file mode 100644 index 0000000..2bce1b3 --- /dev/null +++ b/.cz.yaml @@ -0,0 +1,4 @@ +commitizen: + name: cz_conventional_commits + tag_format: v$version + version: 0.30.1 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..ae10d80 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,7 @@ +repos: +- hooks: + - id: commitizen + stages: + - commit-msg + repo: https://github.com/commitizen-tools/commitizen + rev: v2.20.3 From faf5da4d4a30add552a55848cc500aab34d2e659 Mon Sep 17 00:00:00 2001 From: Jayson Reis Date: Tue, 28 Dec 2021 16:27:41 +0100 Subject: [PATCH 7/8] chore: Prepare changelog for commitizen --- CHANGELOG.rst => CHANGELOG.md | 0 setup.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename CHANGELOG.rst => CHANGELOG.md (100%) diff --git a/CHANGELOG.rst b/CHANGELOG.md similarity index 100% rename from CHANGELOG.rst rename to CHANGELOG.md diff --git a/setup.py b/setup.py index 965d0b0..3863778 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ def read(filename): author="Jayson Reis", author_email="santosdosreis@gmail.com", description="A pure python module to access memcached via its binary protocol with SASL auth support", - long_description="{0}\n{1}".format(read("README.rst"), read("CHANGELOG.rst")), + long_description="{0}\n{1}".format(read("README.rst"), read("CHANGELOG.md")), url="https://github.com/jaysonsantos/python-binary-memcached", packages=["bmemcached", "bmemcached.client"], classifiers=[ From b8375a4a614b518b046ea58dfa9af1590fe9d134 Mon Sep 17 00:00:00 2001 From: Jayson Reis Date: Tue, 28 Dec 2021 16:31:51 +0100 Subject: [PATCH 8/8] =?UTF-8?q?bump:=20version=200.30.1=20=E2=86=92=200.31?= =?UTF-8?q?.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .cz.yaml | 2 +- CHANGELOG.md | 41 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/.cz.yaml b/.cz.yaml index 2bce1b3..50ca519 100644 --- a/.cz.yaml +++ b/.cz.yaml @@ -1,4 +1,4 @@ commitizen: name: cz_conventional_commits tag_format: v$version - version: 0.30.1 + version: 0.31.0 diff --git a/CHANGELOG.md b/CHANGELOG.md index c5b70a6..0d46296 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,3 +22,44 @@ v0.28 - moved bmemcached.Client to bmemcached.ReplicantClient *but keeps backward compatibility* - added DistributedClient to distribute keys over servers using consistent hashing +## v0.31.0 (2021-12-28) + +### Refactor + +- Use bytearrays for building up bytes for I/O. (#245) + +### Feat + +- Expose incr/decr `default` and `time` protocol arguments in client class (#243) + +## v0.30.1 (2020-10-11) + +## v0.30.0 (2020-08-18) + +## v0.30 (2020-06-10) + +## v0.29.0 (2020-01-29) + +## v0.28.0 (2018-10-02) + +## v0.27.0 (2018-08-10) + +## 0.26.1 (2017-07-18) + +## 0.26.0 (2017-01-13) + +## 0.25.0 (2016-12-15) + +## 0.24.7 (2016-11-08) + +## 0.24.2 (2014-05-27) + +## 0.24.1 (2014-05-20) + +## 0.24 (2014-04-28) + +## 0.23 (2014-04-18) + +## v0.18 (2013-05-06) + +## v0.17 (2013-04-15)