Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit da988ec

Browse files
TinittoTinitto
committed
Close #20: Add pagination to select
1 parent 4221210 commit da988ec

File tree

14 files changed

+530
-106
lines changed

14 files changed

+530
-106
lines changed

‎CHANGELOG.md

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,16 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
77

88
## [Unreleased]
99

10+
## [0.4.0] - 2022-12-17
11+
12+
### Added
13+
14+
- Added pagination
15+
16+
### Changed
17+
18+
- Changed redis index to use sorted sets instead of ordinary sets
19+
1020
## [0.3.0] - 2022-12-15
1121

1222
### Added

‎README.md

Lines changed: 93 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,8 @@ from pydantic_redis import RedisConfig, Model, Store
3838

3939
class Author(Model):
4040
"""
41-
An Author model, just like a pydantic model with appropriate type annotations
41+
An Author model, just like a pydantic model with appropriate
42+
type annotations
4243
NOTE: The `_primary_key_field` is mandatory
4344
"""
4445
_primary_key_field: str = 'name'
@@ -51,18 +52,24 @@ class Book(Model):
5152
A Book model.
5253
5354
Models can have the following field types
54-
- The usual i.e. float, int, dict, list, date, str, dict, Optional etc as long as they are serializable by orjson
55+
- The usual i.e. float, int, dict, list, date, str, dict, Optional etc
56+
as long as they are serializable by orjson
5557
- Nested models e.g. `author: Author` or `author: Optional[Author]`
56-
- List of nested models e.g. `authors: List[Author]` or `authors: Optional[List[Author]]`
57-
- Tuples including nested models e.g. `access_log: Tuple[Author, date]` or `access_log: Optional[Tuple[Author, date]]]`
58+
- List of nested models e.g. `authors: List[Author]`
59+
or `authors: Optional[List[Author]]`
60+
- Tuples including nested models e.g. `access_log: Tuple[Author, date]`
61+
or `access_log: Optional[Tuple[Author, date]]]`
5862
59-
NOTE: 1. Any nested model whether plain or in a list or tuple will automatically inserted into the redis store
60-
when the parent model is inserted. e.g. a Book with an author field, when inserted, will also insert
61-
the author. The author can then be queried directly if that's something one wishes to do.
63+
NOTE: 1. Any nested model whether plain or in a list or tuple will automatically
64+
inserted into the redis store when the parent model is inserted.
65+
e.g. a Book with an author field, when inserted, will also insert
66+
the author. The author can then be queried directly if that's something
67+
one wishes to do.
6268
63-
2. When a parent model is inserted with a nested model instance that already exists, the older nested model
64-
instance is overwritten. This is one way of updating nested models. All parent models that contain that nested
65-
model instance will see the change.
69+
2. When a parent model is inserted with a nested model instance that
70+
already exists, the older nested model instance is overwritten.
71+
This is one way of updating nested models.
72+
All parent models that contain that nested model instance will see the change.
6673
"""
6774
_primary_key_field: str = 'title'
6875
title: str
@@ -81,8 +88,10 @@ class Library(Model):
8188
8289
About Nested Model Performance
8390
---
84-
To minimize the performance penalty for nesting models, we use REDIS EVALSHA to eagerly load the nested models
85-
before the response is returned to the client. This ensures that only ONE network call is made every time.
91+
To minimize the performance penalty for nesting models,
92+
we use REDIS EVALSHA to eagerly load the nested models
93+
before the response is returned to the client.
94+
This ensures that only ONE network call is made every time.
8695
"""
8796
_primary_key_field: str = 'name'
8897
name: str
@@ -180,15 +189,26 @@ print(all_books)
180189
# Prints [Book(title="Oliver Twist", author="Charles Dickens", published_on=date(year=1215, month=4, day=4),
181190
# in_stock=False), Book(...]
182191

192+
# or paginate i.e. skip some books and return only upto a given number
193+
paginated_books = Book.select(skip=2, limit=2)
194+
print(paginated_books)
195+
183196
# Get some, with all fields shown. Data returned is a list of models instances.
184197
some_books = Book.select(ids=["Oliver Twist", "Jane Eyre"])
185198
print(some_books)
186199

200+
# Note: Pagination does not work when ids are provided i.e.
201+
assert some_books == Book.select(ids=["Oliver Twist", "Jane Eyre"], skip=100, limit=10)
202+
187203
# Get all, with only a few fields shown. Data returned is a list of dictionaries.
188204
books_with_few_fields = Book.select(columns=["author", "in_stock"])
189205
print(books_with_few_fields)
190206
# Prints [{"author": "'Charles Dickens", "in_stock": "True"},...]
191207

208+
# or paginate i.e. skip some books and return only upto a given number
209+
paginated_books_with_few_fields = Book.select(columns=["author", "in_stock"], skip=2, limit=2)
210+
print(paginated_books_with_few_fields)
211+
192212
# Get some, with only some fields shown. Data returned is a list of dictionaries.
193213
some_books_with_few_fields = Book.select(ids=["Oliver Twist", "Jane Eyre"], columns=["author", "in_stock"])
194214
print(some_books_with_few_fields)
@@ -217,13 +237,15 @@ from datetime import date
217237
from typing import Tuple, List, Optional
218238
from pydantic_redis.asyncio import RedisConfig, Model, Store
219239

220-
# The features are exactly the same as the synchronous version, except for the ability
221-
# to return coroutines when `insert`, `update`, `select` or `delete` are called.
240+
# The features are exactly the same as the synchronous version,
241+
# except for the ability to return coroutines when `insert`,
242+
# `update`, `select` or `delete` are called.
222243

223244

224245
class Author(Model):
225246
"""
226-
An Author model, just like a pydantic model with appropriate type annotations
247+
An Author model, just like a pydantic model with appropriate
248+
type annotations
227249
NOTE: The `_primary_key_field` is mandatory
228250
"""
229251
_primary_key_field: str = 'name'
@@ -236,18 +258,24 @@ class Book(Model):
236258
A Book model.
237259
238260
Models can have the following field types
239-
- The usual i.e. float, int, dict, list, date, str, dict, Optional etc as long as they are serializable by orjson
261+
- The usual i.e. float, int, dict, list, date, str, dict, Optional etc
262+
as long as they are serializable by orjson
240263
- Nested models e.g. `author: Author` or `author: Optional[Author]`
241-
- List of nested models e.g. `authors: List[Author]` or `authors: Optional[List[Author]]`
242-
- Tuples including nested models e.g. `access_log: Tuple[Author, date]` or `access_log: Optional[Tuple[Author, date]]]`
264+
- List of nested models e.g. `authors: List[Author]`
265+
or `authors: Optional[List[Author]]`
266+
- Tuples including nested models e.g. `access_log: Tuple[Author, date]`
267+
or `access_log: Optional[Tuple[Author, date]]]`
243268
244-
NOTE: 1. Any nested model whether plain or in a list or tuple will automatically inserted into the redis store
245-
when the parent model is inserted. e.g. a Book with an author field, when inserted, will also insert
246-
the author. The author can then be queried directly if that's something one wishes to do.
269+
NOTE: 1. Any nested model whether plain or in a list or tuple will automatically
270+
inserted into the redis store when the parent model is inserted.
271+
e.g. a Book with an author field, when inserted, will also insert
272+
the author. The author can then be queried directly if that's something
273+
one wishes to do.
247274
248-
2. When a parent model is inserted with a nested model instance that already exists, the older nested model
249-
instance is overwritten. This is one way of updating nested models. All parent models that contain that nested
250-
model instance will see the change.
275+
2. When a parent model is inserted with a nested model instance that
276+
already exists, the older nested model instance is overwritten.
277+
This is one way of updating nested models.
278+
All parent models that contain that nested model instance will see the change.
251279
"""
252280
_primary_key_field: str = 'title'
253281
title: str
@@ -266,8 +294,10 @@ class Library(Model):
266294
267295
About Nested Model Performance
268296
---
269-
To minimize the performance penalty for nesting models, we use REDIS EVALSHA to eagerly load the nested models
270-
before the response is returned to the client. This ensures that only ONE network call is made every time.
297+
To minimize the performance penalty for nesting models,
298+
we use REDIS EVALSHA to eagerly load the nested models
299+
before the response is returned to the client.
300+
This ensures that only ONE network call is made every time.
271301
"""
272302
_primary_key_field: str = 'name'
273303
name: str
@@ -367,15 +397,26 @@ async def run_async():
367397
# Prints [Book(title="Oliver Twist", author="Charles Dickens", published_on=date(year=1215, month=4, day=4),
368398
# in_stock=False), Book(...]
369399

400+
# or paginate i.e. skip some books and return only upto a given number
401+
paginated_books = await Book.select(skip=2, limit=2)
402+
print(paginated_books)
403+
370404
# Get some, with all fields shown. Data returned is a list of models instances.
371405
some_books = await Book.select(ids=["Oliver Twist", "Jane Eyre"])
372406
print(some_books)
373407

408+
# Note: Pagination does not work when ids are provided i.e.
409+
assert some_books == await Book.select(ids=["Oliver Twist", "Jane Eyre"], skip=100, limit=10)
410+
374411
# Get all, with only a few fields shown. Data returned is a list of dictionaries.
375412
books_with_few_fields = await Book.select(columns=["author", "in_stock"])
376413
print(books_with_few_fields)
377414
# Prints [{"author": "'Charles Dickens", "in_stock": "True"},...]
378415

416+
# or paginate i.e. skip some books and return only upto a given number
417+
paginated_books_with_few_fields = await Book.select(columns=["author", "in_stock"], skip=2, limit=2)
418+
print(paginated_books_with_few_fields)
419+
379420
# Get some, with only some fields shown. Data returned is a list of dictionaries.
380421
some_books_with_few_fields = await Book.select(ids=["Oliver Twist", "Jane Eyre"], columns=["author", "in_stock"])
381422
print(some_books_with_few_fields)
@@ -440,30 +481,32 @@ asyncio.run(run_async())
440481
On an average PC ~16GB RAM, i7 Core
441482

442483
```
443-
------------------------------------------------- benchmark: 20 tests -------------------------------------------------
444-
Name (time in us) Mean Min Max
445-
-----------------------------------------------------------------------------------------------------------------------
446-
benchmark_select_columns_for_one_id[redis_store-book1] 143.5316 (1.08) 117.4340 (1.0) 347.5900 (1.0)
447-
benchmark_select_columns_for_one_id[redis_store-book3] 151.6032 (1.14) 117.6690 (1.00) 405.4620 (1.17)
448-
benchmark_select_columns_for_one_id[redis_store-book0] 133.0856 (1.0) 117.8720 (1.00) 403.9400 (1.16)
449-
benchmark_select_columns_for_one_id[redis_store-book2] 156.8152 (1.18) 118.7220 (1.01) 569.9800 (1.64)
450-
benchmark_select_columns_for_some_items[redis_store] 138.0488 (1.04) 120.1550 (1.02) 350.7040 (1.01)
451-
benchmark_delete[redis_store-Wuthering Heights] 199.9205 (1.50) 127.6990 (1.09) 1,092.2190 (3.14)
452-
benchmark_bulk_delete[redis_store] 178.4756 (1.34) 143.7480 (1.22) 647.6660 (1.86)
453-
benchmark_select_all_for_one_id[redis_store-book1] 245.7787 (1.85) 195.2030 (1.66) 528.9250 (1.52)
454-
benchmark_select_all_for_one_id[redis_store-book0] 239.1152 (1.80) 199.4360 (1.70) 767.2540 (2.21)
455-
benchmark_select_all_for_one_id[redis_store-book3] 243.8724 (1.83) 200.8060 (1.71) 535.3640 (1.54)
456-
benchmark_select_all_for_one_id[redis_store-book2] 256.1625 (1.92) 202.4630 (1.72) 701.3000 (2.02)
457-
benchmark_update[redis_store-Wuthering Heights-data0] 329.1363 (2.47) 266.9700 (2.27) 742.1360 (2.14)
458-
benchmark_select_some_items[redis_store] 301.0471 (2.26) 268.9410 (2.29) 551.1060 (1.59)
459-
benchmark_select_columns[redis_store] 313.4356 (2.36) 281.4460 (2.40) 578.7730 (1.67)
460-
benchmark_single_insert[redis_store-book2] 348.5624 (2.62) 297.3610 (2.53) 580.8780 (1.67)
461-
benchmark_single_insert[redis_store-book1] 342.1879 (2.57) 297.5410 (2.53) 650.5420 (1.87)
462-
benchmark_single_insert[redis_store-book0] 366.4513 (2.75) 310.1640 (2.64) 660.5380 (1.90)
463-
benchmark_single_insert[redis_store-book3] 377.6208 (2.84) 327.5290 (2.79) 643.4090 (1.85)
464-
benchmark_select_default[redis_store] 486.6931 (3.66) 428.8810 (3.65) 1,181.9620 (3.40)
465-
benchmark_bulk_insert[redis_store] 897.7862 (6.75) 848.7410 (7.23) 1,188.5160 (3.42)
466-
-----------------------------------------------------------------------------------------------------------------------
484+
-------------------------------------------------- benchmark: 22 tests --------------------------------------------------
485+
Name (time in us) Mean Min Max
486+
-------------------------------------------------------------------------------------------------------------------------
487+
benchmark_select_columns_for_one_id[redis_store-book2] 124.2687 (1.00) 115.4530 (1.0) 331.8030 (1.26)
488+
benchmark_select_columns_for_one_id[redis_store-book0] 123.7213 (1.0) 115.6680 (1.00) 305.7170 (1.16)
489+
benchmark_select_columns_for_one_id[redis_store-book3] 124.4495 (1.01) 115.9580 (1.00) 263.4370 (1.0)
490+
benchmark_select_columns_for_one_id[redis_store-book1] 124.8431 (1.01) 117.4770 (1.02) 310.3140 (1.18)
491+
benchmark_select_columns_for_some_items[redis_store] 128.0657 (1.04) 118.6380 (1.03) 330.2680 (1.25)
492+
benchmark_delete[redis_store-Wuthering Heights] 131.8713 (1.07) 125.9920 (1.09) 328.9660 (1.25)
493+
benchmark_bulk_delete[redis_store] 148.6963 (1.20) 142.3190 (1.23) 347.4750 (1.32)
494+
benchmark_select_all_for_one_id[redis_store-book3] 211.6941 (1.71) 195.6520 (1.69) 422.8840 (1.61)
495+
benchmark_select_all_for_one_id[redis_store-book2] 212.3612 (1.72) 195.9020 (1.70) 447.4910 (1.70)
496+
benchmark_select_all_for_one_id[redis_store-book1] 212.9524 (1.72) 197.7530 (1.71) 423.3030 (1.61)
497+
benchmark_select_all_for_one_id[redis_store-book0] 214.9924 (1.74) 198.8280 (1.72) 402.6310 (1.53)
498+
benchmark_select_columns_paginated[redis_store] 227.9248 (1.84) 211.0610 (1.83) 425.8390 (1.62)
499+
benchmark_select_some_items[redis_store] 297.5700 (2.41) 271.1510 (2.35) 572.1220 (2.17)
500+
benchmark_select_default_paginated[redis_store] 301.7495 (2.44) 282.6500 (2.45) 490.3450 (1.86)
501+
benchmark_select_columns[redis_store] 316.2119 (2.56) 290.6110 (2.52) 578.0310 (2.19)
502+
benchmark_update[redis_store-Wuthering Heights-data0] 346.5816 (2.80) 304.5420 (2.64) 618.0250 (2.35)
503+
benchmark_single_insert[redis_store-book2] 378.0613 (3.06) 337.8070 (2.93) 616.4930 (2.34)
504+
benchmark_single_insert[redis_store-book0] 396.6513 (3.21) 347.1000 (3.01) 696.1350 (2.64)
505+
benchmark_single_insert[redis_store-book3] 395.9082 (3.20) 361.0980 (3.13) 623.8630 (2.37)
506+
benchmark_single_insert[redis_store-book1] 401.1377 (3.24) 363.5890 (3.15) 610.4400 (2.32)
507+
benchmark_select_default[redis_store] 498.4673 (4.03) 428.1350 (3.71) 769.7640 (2.92)
508+
benchmark_bulk_insert[redis_store] 1,025.0436 (8.29) 962.2230 (8.33) 1,200.3840 (4.56)
509+
-------------------------------------------------------------------------------------------------------------------------
467510
```
468511

469512
## Contributions

‎pydantic_redis/__init__.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,4 +5,4 @@
55

66
__all__ = [Store, RedisConfig, Model, asyncio]
77

8-
__version__ = "0.3.0"
8+
__version__ = "0.4.0"

‎pydantic_redis/asyncio/model.py

Lines changed: 18 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
)
1616

1717
from .store import Store
18+
from ..shared.model.delete_utils import delete_on_pipeline
1819

1920

2021
class Model(AbstractModel):
@@ -92,41 +93,41 @@ async def delete(cls, ids: Union[Any, List[Any]]):
9293
store = cls.get_store()
9394

9495
async with store.redis_store.pipeline() as pipeline:
95-
primary_keys = []
96-
97-
if isinstance(ids, list):
98-
primary_keys = ids
99-
elif ids is not None:
100-
primary_keys = [ids]
101-
102-
names = [
103-
get_primary_key(model=cls, primary_key_value=primary_key_value)
104-
for primary_key_value in primary_keys
105-
]
106-
pipeline.delete(*names)
107-
# remove the primary keys from the index
108-
table_index_key = get_table_index_key(model=cls)
109-
pipeline.srem(table_index_key, *names)
96+
delete_on_pipeline(model=cls, pipeline=pipeline, ids=ids)
11097
return await pipeline.execute()
11198

11299
@classmethod
113100
async def select(
114101
cls,
115102
columns: Optional[List[str]] = None,
116103
ids: Optional[List[Any]] = None,
104+
skip: int = 0,
105+
limit: Optional[int] = None,
117106
**kwargs,
118107
):
119108
"""
120109
Selects given rows or sets of rows in the table
110+
111+
112+
However, if `limit` is set, the number of items
113+
returned will be less or equal to `limit`.
114+
`skip` defaults to 0. It is the number of items to skip.
115+
`skip` is only relevant when limit is specified.
116+
117+
`skip` and `limit` are irrelevant when `ids` are provided.
121118
"""
122119
if columns is None and ids is None:
123-
response = await select_all_fields_all_ids(model=cls)
120+
response = await select_all_fields_all_ids(
121+
model=cls, skip=skip, limit=limit
122+
)
124123

125124
elif columns is None and isinstance(ids, list):
126125
response = await select_all_fields_some_ids(model=cls, ids=ids)
127126

128127
elif isinstance(columns, list) and ids is None:
129-
response = await select_some_fields_all_ids(model=cls, fields=columns)
128+
response = await select_some_fields_all_ids(
129+
model=cls, fields=columns, skip=skip, limit=limit
130+
)
130131

131132
elif isinstance(columns, list) and isinstance(ids, list):
132133
response = await select_some_fields_some_ids(

0 commit comments

Comments
 (0)