From 4e74cfdb52c5490e9458461adc38fafa102ceeb9 Mon Sep 17 00:00:00 2001 From: Harry Date: Fri, 8 Feb 2019 11:00:16 +0000 Subject: [PATCH 001/165] makefile for running stuff --- Makefile | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..46cc16d9 --- /dev/null +++ b/Makefile @@ -0,0 +1,5 @@ +test: + pytest --tb=short + +watch-tests: + ls *.py | entr pytest --tb=short From bbeacc4eadc09a89c77888c948933b6d76e62f73 Mon Sep 17 00:00:00 2001 From: Harry Date: Fri, 8 Mar 2019 21:45:23 +0000 Subject: [PATCH 002/165] first test [first_test] --- test_batches.py | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 test_batches.py diff --git a/test_batches.py b/test_batches.py new file mode 100644 index 00000000..31a60a69 --- /dev/null +++ b/test_batches.py @@ -0,0 +1,10 @@ +from model import Batch, OrderLine +from datetime import date + +def test_allocating_to_a_batch_reduces_the_available_quantity(): + batch = Batch("batch-001", "SMALL-TABLE", qty=20, eta=date.today()) + line = OrderLine('order-ref', "SMALL-TABLE", 2) + + batch.allocate(line) + + assert batch.available_quantity == 18 From ba11b4a0855a173d6f7d9c5abf82f7b2c9992e48 Mon Sep 17 00:00:00 2001 From: Harry Date: Fri, 8 Mar 2019 21:47:37 +0000 Subject: [PATCH 003/165] first stab at a model [domain_model_1] --- model.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 model.py diff --git a/model.py b/model.py new file mode 100644 index 00000000..eaa03f83 --- /dev/null +++ b/model.py @@ -0,0 +1,24 @@ +from __future__ import annotations +from dataclasses import dataclass +from datetime import date +from typing import Optional + + +@dataclass(frozen=True) +class OrderLine: + orderid: str + sku: str + qty: int + + +class Batch: + def __init__( + self, ref: str, sku: str, qty: int, eta: Optional[date] + ): + self.reference = ref + self.sku = sku + self.eta = eta + self.available_quantity = qty + + def allocate(self, line: OrderLine): + self.available_quantity -= line.qty From 197b5bbdc83eb2d1cc5c430f81ed08cb9b9f2d77 Mon Sep 17 00:00:00 2001 From: Harry Date: Fri, 8 Mar 2019 22:22:09 +0000 Subject: [PATCH 004/165] more tests for can_allocate [test_can_allocate] --- test_batches.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/test_batches.py b/test_batches.py index 31a60a69..a4ef5689 100644 --- a/test_batches.py +++ b/test_batches.py @@ -1,3 +1,4 @@ +from datetime import date from model import Batch, OrderLine from datetime import date @@ -8,3 +9,27 @@ def test_allocating_to_a_batch_reduces_the_available_quantity(): batch.allocate(line) assert batch.available_quantity == 18 + +def make_batch_and_line(sku, batch_qty, line_qty): + return ( + Batch("batch-001", sku, batch_qty, eta=date.today()), + OrderLine("order-123", sku, line_qty) + ) + + +def test_can_allocate_if_available_greater_than_required(): + large_batch, small_line = make_batch_and_line("ELEGANT-LAMP", 20, 2) + assert large_batch.can_allocate(small_line) + +def test_cannot_allocate_if_available_smaller_than_required(): + small_batch, large_line = make_batch_and_line("ELEGANT-LAMP", 2, 20) + assert small_batch.can_allocate(large_line) is False + +def test_can_allocate_if_available_equal_to_required(): + batch, line = make_batch_and_line("ELEGANT-LAMP", 2, 2) + assert batch.can_allocate(line) + +def test_cannot_allocate_if_skus_do_not_match(): + batch = Batch("batch-001", 'UNCOMFORTABLE-CHAIR', 100, eta=None) + different_sku_line = OrderLine("order-123", 'EXPENSIVE-TOASTER', 10) + assert batch.can_allocate(different_sku_line) is False From f9072f7497823999bf654d3a387bc9aa6878b44b Mon Sep 17 00:00:00 2001 From: Harry Date: Fri, 8 Mar 2019 22:25:54 +0000 Subject: [PATCH 005/165] can_allocate fn [can_allocate] --- model.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/model.py b/model.py index eaa03f83..8e9428a6 100644 --- a/model.py +++ b/model.py @@ -22,3 +22,7 @@ def __init__( def allocate(self, line: OrderLine): self.available_quantity -= line.qty + + def can_allocate(self, line: OrderLine) -> bool: + return self.sku == line.sku and self.available_quantity >= line.qty + From 92dc6bf46111967c659555c6466418cb7c3b8fed Mon Sep 17 00:00:00 2001 From: Harry Date: Fri, 8 Mar 2019 22:45:38 +0000 Subject: [PATCH 006/165] simple deallocate test --- test_batches.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/test_batches.py b/test_batches.py index a4ef5689..43b8199f 100644 --- a/test_batches.py +++ b/test_batches.py @@ -30,6 +30,12 @@ def test_can_allocate_if_available_equal_to_required(): assert batch.can_allocate(line) def test_cannot_allocate_if_skus_do_not_match(): - batch = Batch("batch-001", 'UNCOMFORTABLE-CHAIR', 100, eta=None) - different_sku_line = OrderLine("order-123", 'EXPENSIVE-TOASTER', 10) + batch = Batch("batch-001", "UNCOMFORTABLE-CHAIR", 100, eta=None) + different_sku_line = OrderLine("order-123", "EXPENSIVE-TOASTER", 10) assert batch.can_allocate(different_sku_line) is False + +def test_deallocate(): + batch, line = make_batch_and_line("EXPENSIVE-FOOTSTOOL", 20, 2) + batch.allocate(line) + batch.deallocate(line) + assert batch.available_quantity == 20 From 5ef939a8de4d75919990e7aed3c01e786836518f Mon Sep 17 00:00:00 2001 From: Harry Date: Fri, 8 Mar 2019 22:46:50 +0000 Subject: [PATCH 007/165] simple deallocate function --- model.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/model.py b/model.py index 8e9428a6..c421a19a 100644 --- a/model.py +++ b/model.py @@ -23,6 +23,9 @@ def __init__( def allocate(self, line: OrderLine): self.available_quantity -= line.qty + def deallocate(self, line: OrderLine): + self.available_quantity += line.qty + def can_allocate(self, line: OrderLine) -> bool: return self.sku == line.sku and self.available_quantity >= line.qty From 68b8b4cefb42e7f11d145368d52e2e4606c7d9a8 Mon Sep 17 00:00:00 2001 From: Harry Date: Fri, 8 Mar 2019 22:47:53 +0000 Subject: [PATCH 008/165] test deallocate not allocated [test_deallocate_unallocated] --- test_batches.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test_batches.py b/test_batches.py index 43b8199f..da4644cd 100644 --- a/test_batches.py +++ b/test_batches.py @@ -39,3 +39,8 @@ def test_deallocate(): batch.allocate(line) batch.deallocate(line) assert batch.available_quantity == 20 + +def test_can_only_deallocate_allocated_lines(): + batch, unallocated_line = make_batch_and_line("DECORATIVE-TRINKET", 20, 2) + batch.deallocate(unallocated_line) + assert batch.available_quantity == 20 From 1b76c8abed3776f28c0c1fdbc5a18d39e1dd4096 Mon Sep 17 00:00:00 2001 From: Harry Date: Fri, 8 Mar 2019 22:48:39 +0000 Subject: [PATCH 009/165] model now tracks allocations [domain_model_complete] --- model.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/model.py b/model.py index c421a19a..7adbce60 100644 --- a/model.py +++ b/model.py @@ -1,7 +1,7 @@ from __future__ import annotations from dataclasses import dataclass from datetime import date -from typing import Optional +from typing import Optional, Set @dataclass(frozen=True) @@ -18,13 +18,24 @@ def __init__( self.reference = ref self.sku = sku self.eta = eta - self.available_quantity = qty + self._purchased_quantity = qty + self._allocations = set() # type: Set[OrderLine] def allocate(self, line: OrderLine): - self.available_quantity -= line.qty + if self.can_allocate(line): + self._allocations.add(line) def deallocate(self, line: OrderLine): - self.available_quantity += line.qty + if line in self._allocations: + self._allocations.remove(line) + + @property + def allocated_quantity(self) -> int: + return sum(line.qty for line in self._allocations) + + @property + def available_quantity(self) -> int: + return self._purchased_quantity - self.allocated_quantity def can_allocate(self, line: OrderLine) -> bool: return self.sku == line.sku and self.available_quantity >= line.qty From 531d999d825c47c148e07867bea233091c338532 Mon Sep 17 00:00:00 2001 From: Harry Date: Fri, 8 Mar 2019 22:49:55 +0000 Subject: [PATCH 010/165] test allocate twice [last_test] --- test_batches.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/test_batches.py b/test_batches.py index da4644cd..32100b2c 100644 --- a/test_batches.py +++ b/test_batches.py @@ -1,6 +1,6 @@ from datetime import date from model import Batch, OrderLine -from datetime import date + def test_allocating_to_a_batch_reduces_the_available_quantity(): batch = Batch("batch-001", "SMALL-TABLE", qty=20, eta=date.today()) @@ -34,6 +34,12 @@ def test_cannot_allocate_if_skus_do_not_match(): different_sku_line = OrderLine("order-123", "EXPENSIVE-TOASTER", 10) assert batch.can_allocate(different_sku_line) is False +def test_allocation_is_idempotent(): + batch, line = make_batch_and_line("ANGULAR-DESK", 20, 2) + batch.allocate(line) + batch.allocate(line) + assert batch.available_quantity == 18 + def test_deallocate(): batch, line = make_batch_and_line("EXPENSIVE-FOOTSTOOL", 20, 2) batch.allocate(line) @@ -44,3 +50,4 @@ def test_can_only_deallocate_allocated_lines(): batch, unallocated_line = make_batch_and_line("DECORATIVE-TRINKET", 20, 2) batch.deallocate(unallocated_line) assert batch.available_quantity == 20 + From 2238a824ffd151a9ebe6654ee46dab8cab04b794 Mon Sep 17 00:00:00 2001 From: Harry Date: Fri, 8 Mar 2019 22:58:03 +0000 Subject: [PATCH 011/165] equality and hash operators [equality_on_batches] --- model.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/model.py b/model.py index 7adbce60..7758e08f 100644 --- a/model.py +++ b/model.py @@ -21,6 +21,14 @@ def __init__( self._purchased_quantity = qty self._allocations = set() # type: Set[OrderLine] + def __eq__(self, other): + if not isinstance(other, Batch): + return False + return other.reference == self.reference + + def __hash__(self): + return hash(self.reference) + def allocate(self, line: OrderLine): if self.can_allocate(line): self._allocations.add(line) From b23c15ba0fd9cec0f576015bb04130d330a63488 Mon Sep 17 00:00:00 2001 From: Harry Date: Sat, 9 Mar 2019 08:40:39 +0000 Subject: [PATCH 012/165] new tests for allocate domain service [test_allocate] --- test_allocate.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 test_allocate.py diff --git a/test_allocate.py b/test_allocate.py new file mode 100644 index 00000000..8d9d8314 --- /dev/null +++ b/test_allocate.py @@ -0,0 +1,29 @@ +from datetime import date, timedelta +from model import allocate, OrderLine, Batch + +today = date.today() +tomorrow = today + timedelta(days=1) +later = tomorrow + timedelta(days=10) + +def test_prefers_current_stock_batches_to_shipments(): + in_stock_batch = Batch("in-warehouse-stock", "RETRO-CLOCK", 100, eta=None) + shipment_batch = Batch("shipment-batch", "RETRO-CLOCK", 100, eta=tomorrow) + line = OrderLine("oref", "RETRO-CLOCK", 10) + + allocate(line, [in_stock_batch, shipment_batch]) + + assert in_stock_batch.available_quantity == 90 + assert shipment_batch.available_quantity == 100 + + +def test_prefers_earlier_batches(): + earliest = Batch("speedy-batch", "MINIMALIST-SPOON", 100, eta=today) + medium = Batch("normal-batch", "MINIMALIST-SPOON", 100, eta=tomorrow) + latest = Batch("slow-batch", "MINIMALIST-SPOON", 100, eta=later) + line = OrderLine("order1", "MINIMALIST-SPOON", 10) + + allocate(line, [medium, earliest, latest]) + + assert earliest.available_quantity == 90 + assert medium.available_quantity == 100 + assert latest.available_quantity == 100 From 01d69441ce281949dd8cb9aeebe5b342e1eaf93e Mon Sep 17 00:00:00 2001 From: Harry Date: Sat, 9 Mar 2019 08:41:18 +0000 Subject: [PATCH 013/165] allocate fn, domain service [domain_service] --- model.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/model.py b/model.py index 7758e08f..4c776504 100644 --- a/model.py +++ b/model.py @@ -1,7 +1,15 @@ from __future__ import annotations from dataclasses import dataclass from datetime import date -from typing import Optional, Set +from typing import Optional, List, Set + + +def allocate(line: OrderLine, batches: List[Batch]) -> str: + batch = next( + b for b in sorted(batches) if b.can_allocate(line) + ) + batch.allocate(line) + return batch.reference @dataclass(frozen=True) From bb8402a538a18997be7fa0249e0f09648b297458 Mon Sep 17 00:00:00 2001 From: Harry Date: Thu, 18 Jul 2019 10:18:43 +0100 Subject: [PATCH 014/165] fixup a batchref --- test_allocate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_allocate.py b/test_allocate.py index 8d9d8314..1337c1b8 100644 --- a/test_allocate.py +++ b/test_allocate.py @@ -6,7 +6,7 @@ later = tomorrow + timedelta(days=10) def test_prefers_current_stock_batches_to_shipments(): - in_stock_batch = Batch("in-warehouse-stock", "RETRO-CLOCK", 100, eta=None) + in_stock_batch = Batch("in-stock-batch", "RETRO-CLOCK", 100, eta=None) shipment_batch = Batch("shipment-batch", "RETRO-CLOCK", 100, eta=tomorrow) line = OrderLine("oref", "RETRO-CLOCK", 10) From eb49ab43e97975b01ca96610d943082eb38bd2b4 Mon Sep 17 00:00:00 2001 From: Harry Date: Wed, 20 Mar 2019 14:26:35 +0000 Subject: [PATCH 015/165] change tests add one for return --- test_allocate.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test_allocate.py b/test_allocate.py index 1337c1b8..a6520453 100644 --- a/test_allocate.py +++ b/test_allocate.py @@ -27,3 +27,11 @@ def test_prefers_earlier_batches(): assert earliest.available_quantity == 90 assert medium.available_quantity == 100 assert latest.available_quantity == 100 + + +def test_returns_allocated_batch_ref(): + in_stock_batch = Batch("in-stock-batch-ref", "HIGHBROW-POSTER", 100, eta=None) + shipment_batch = Batch("shipment-batch-ref", "HIGHBROW-POSTER", 100, eta=tomorrow) + line = OrderLine("oref", "HIGHBROW-POSTER", 10) + allocation = allocate(line, [in_stock_batch, shipment_batch]) + assert allocation == "in-stock-batch" From e2e7bc573f35160e37a9ff5ad61c0cdadabaf786 Mon Sep 17 00:00:00 2001 From: Harry Date: Sat, 9 Mar 2019 08:41:31 +0000 Subject: [PATCH 016/165] make Batches sortable [dunder_gt] --- model.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/model.py b/model.py index 4c776504..c46c04df 100644 --- a/model.py +++ b/model.py @@ -29,6 +29,9 @@ def __init__( self._purchased_quantity = qty self._allocations = set() # type: Set[OrderLine] + def __repr__(self): + return f'' + def __eq__(self, other): if not isinstance(other, Batch): return False @@ -37,6 +40,13 @@ def __eq__(self, other): def __hash__(self): return hash(self.reference) + def __gt__(self, other): + if self.eta is None: + return False + if other.eta is None: + return True + return self.eta > other.eta + def allocate(self, line: OrderLine): if self.can_allocate(line): self._allocations.add(line) From 886d9b876b498c0a3b90883dfc677c4ed0d33054 Mon Sep 17 00:00:00 2001 From: Harry Date: Thu, 18 Jul 2019 10:23:10 +0100 Subject: [PATCH 017/165] fixup a sku --- test_allocate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_allocate.py b/test_allocate.py index a6520453..de903768 100644 --- a/test_allocate.py +++ b/test_allocate.py @@ -34,4 +34,4 @@ def test_returns_allocated_batch_ref(): shipment_batch = Batch("shipment-batch-ref", "HIGHBROW-POSTER", 100, eta=tomorrow) line = OrderLine("oref", "HIGHBROW-POSTER", 10) allocation = allocate(line, [in_stock_batch, shipment_batch]) - assert allocation == "in-stock-batch" + assert allocation == in_stock_batch.reference From 08681fb452ee3196e1dfd662e57035e802173333 Mon Sep 17 00:00:00 2001 From: Harry Date: Mon, 11 Mar 2019 15:37:22 +0000 Subject: [PATCH 018/165] test out of stock exception [test_out_of_stock] --- test_allocate.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/test_allocate.py b/test_allocate.py index de903768..b9df7ff7 100644 --- a/test_allocate.py +++ b/test_allocate.py @@ -1,5 +1,6 @@ from datetime import date, timedelta -from model import allocate, OrderLine, Batch +import pytest +from model import allocate, OrderLine, Batch, OutOfStock today = date.today() tomorrow = today + timedelta(days=1) @@ -35,3 +36,11 @@ def test_returns_allocated_batch_ref(): line = OrderLine("oref", "HIGHBROW-POSTER", 10) allocation = allocate(line, [in_stock_batch, shipment_batch]) assert allocation == in_stock_batch.reference + + +def test_raises_out_of_stock_exception_if_cannot_allocate(): + batch = Batch('batch1', 'SMALL-FORK', 10, eta=today) + allocate(OrderLine('order1', 'SMALL-FORK', 10), [batch]) + + with pytest.raises(OutOfStock, match='SMALL-FORK'): + allocate(OrderLine('order2', 'SMALL-FORK', 1), [batch]) From b91ab8912511efe497e6cf434c0e7ee84a05f963 Mon Sep 17 00:00:00 2001 From: Harry Date: Mon, 11 Mar 2019 15:37:37 +0000 Subject: [PATCH 019/165] raising out of stock exception [out_of_stock] --- model.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/model.py b/model.py index c46c04df..f3515db8 100644 --- a/model.py +++ b/model.py @@ -4,12 +4,19 @@ from typing import Optional, List, Set +class OutOfStock(Exception): + pass + + def allocate(line: OrderLine, batches: List[Batch]) -> str: - batch = next( - b for b in sorted(batches) if b.can_allocate(line) - ) - batch.allocate(line) - return batch.reference + try: + batch = next( + b for b in sorted(batches) if b.can_allocate(line) + ) + batch.allocate(line) + return batch.reference + except StopIteration: + raise OutOfStock(f'Out of stock for sku {line.sku}') @dataclass(frozen=True) From c1ffcf834be39d2dd5e49b51e197b9e9f525fbc3 Mon Sep 17 00:00:00 2001 From: Harry Date: Mon, 1 Jul 2019 11:18:46 +0100 Subject: [PATCH 020/165] add readme from master --- README.md | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 00000000..71d8d309 --- /dev/null +++ b/README.md @@ -0,0 +1,54 @@ +# Example application code for the python architecture book + +## Chapters + +Each chapter has its own branch which contains all the commits for that chapter, +so it has the state that corresponds to the _end_ of that chapter. If you want +to try and code along with a chapter, you'll want to check out the branch for the +previous chapter. + +https://github.com/python-leap/code/branches/all + + +## Exercises + +Branches for the exercises follow the convention `{chatper_name}_exercise`, eg +https://github.com/python-leap/code/tree/chapter_03_service_layer_exercise + + +## Requirements + +* docker with docker-compose +* for chapters 1 and 2, and optionally for the rest: a local python3.7 virtualenv + + +## Building the containers + +_(this is only required from chapter 3 onwards)_ + +```sh +make build +make up +# or +make all # builds, brings containers up, runs tests +``` + +## Running the tests + +```sh +make test +# or, to run individual test types +make unit +make integration +make e2e +# or, if you have a local virtualenv +make up +pytest tests/unit +pytest tests/integration +pytest tests/e2e + + +## Makefile + +There are more useful commands in the makefile, have a look and try them out. + From 207cb6c1d2a2c20d000d8c8a1333e1acdf26c8d0 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 26 Mar 2019 20:47:08 +0000 Subject: [PATCH 021/165] travis config. [chapter_01_domain_model_ends] --- .travis.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..732341c8 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,10 @@ +dist: xenial +language: python +python: 3.8 + +script: +- make test + +branches: + except: + - /.*_exercise$/ From 2f4f5301cc42f7fb146bd2867194f7f2b0ffffd5 Mon Sep 17 00:00:00 2001 From: Harry Date: Sun, 10 Mar 2019 23:50:34 +0000 Subject: [PATCH 022/165] first cut of orm, orderlines only [sqlalchemy_classical_mapper] --- .travis.yml | 3 +++ orm.py | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+) create mode 100644 orm.py diff --git a/.travis.yml b/.travis.yml index 732341c8..8a74f94b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,9 @@ dist: xenial language: python python: 3.8 +install: +- pip3 install sqlalchemy + script: - make test diff --git a/orm.py b/orm.py new file mode 100644 index 00000000..f7071e81 --- /dev/null +++ b/orm.py @@ -0,0 +1,18 @@ +from sqlalchemy import Table, MetaData, Column, Integer, String +from sqlalchemy.orm import mapper + +import model + + +metadata = MetaData() + +order_lines = Table( + 'order_lines', metadata, + Column('orderid', String(255), primary_key=True), + Column('sku', String(255), primary_key=True), + Column('qty', Integer), +) + + +def start_mappers(): + mapper(model.OrderLine, order_lines) From 37bebeadf48abed8b3f3cac6ff272bcd3c58c05c Mon Sep 17 00:00:00 2001 From: Harry Date: Sun, 10 Mar 2019 23:51:02 +0000 Subject: [PATCH 023/165] first tests of orm [orm_tests] --- conftest.py | 20 ++++++++++++++++++++ test_orm.py | 25 +++++++++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 conftest.py create mode 100644 test_orm.py diff --git a/conftest.py b/conftest.py new file mode 100644 index 00000000..b64099fd --- /dev/null +++ b/conftest.py @@ -0,0 +1,20 @@ +import pytest +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker, clear_mappers + +from orm import metadata, start_mappers + + +@pytest.fixture +def in_memory_db(): + engine = create_engine('sqlite:///:memory:') + metadata.create_all(engine) + return engine + + +@pytest.fixture +def session(in_memory_db): + start_mappers() + yield sessionmaker(bind=in_memory_db)() + clear_mappers() + diff --git a/test_orm.py b/test_orm.py new file mode 100644 index 00000000..65e0bada --- /dev/null +++ b/test_orm.py @@ -0,0 +1,25 @@ +import model + +def test_orderline_mapper_can_load_lines(session): + session.execute( + 'INSERT INTO order_lines (orderid, sku, qty) VALUES ' + '("order1", "RED-CHAIR", 12),' + '("order1", "RED-TABLE", 13),' + '("order2", "BLUE-LIPSTICK", 14)' + ) + expected = [ + model.OrderLine("order1", "RED-CHAIR", 12), + model.OrderLine("order1", "RED-TABLE", 13), + model.OrderLine("order2", "BLUE-LIPSTICK", 14), + ] + assert session.query(model.OrderLine).all() == expected + + +def test_orderline_mapper_can_save_lines(session): + new_line = model.OrderLine("order1", "DECORATIVE-WIDGET", 12) + session.add(new_line) + session.commit() + + rows = list(session.execute('SELECT orderid, sku, qty FROM "order_lines"')) + assert rows == [("order1", "DECORATIVE-WIDGET", 12)] + From 3b35504c6e43847752c12afa2528954fe9fbfb4a Mon Sep 17 00:00:00 2001 From: Harry Date: Sun, 10 Mar 2019 23:51:23 +0000 Subject: [PATCH 024/165] unfortunate hack on dataclass in model --- model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/model.py b/model.py index f3515db8..bbb3a3c3 100644 --- a/model.py +++ b/model.py @@ -19,7 +19,7 @@ def allocate(line: OrderLine, batches: List[Batch]) -> str: raise OutOfStock(f'Out of stock for sku {line.sku}') -@dataclass(frozen=True) +@dataclass(unsafe_hash=True) class OrderLine: orderid: str sku: str From 2a29db0a35ab484c2eda2884416e392cb745648d Mon Sep 17 00:00:00 2001 From: Harry Date: Sun, 10 Mar 2019 23:55:12 +0000 Subject: [PATCH 025/165] batches with no allocations --- orm.py | 11 ++++++++++- test_orm.py | 13 +++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/orm.py b/orm.py index f7071e81..7dcbac70 100644 --- a/orm.py +++ b/orm.py @@ -1,4 +1,4 @@ -from sqlalchemy import Table, MetaData, Column, Integer, String +from sqlalchemy import Table, MetaData, Column, Integer, String, Date from sqlalchemy.orm import mapper import model @@ -13,6 +13,15 @@ Column('qty', Integer), ) +batches = Table( + 'batches', metadata, + Column('reference', String(255), primary_key=True), + Column('sku', String(255), primary_key=True), + Column('_purchased_qty', Integer), + Column('eta', Date), +) + def start_mappers(): mapper(model.OrderLine, order_lines) + mapper(model.Batch, batches) diff --git a/test_orm.py b/test_orm.py index 65e0bada..fa81e752 100644 --- a/test_orm.py +++ b/test_orm.py @@ -1,4 +1,5 @@ import model +from datetime import date def test_orderline_mapper_can_load_lines(session): session.execute( @@ -23,3 +24,15 @@ def test_orderline_mapper_can_save_lines(session): rows = list(session.execute('SELECT orderid, sku, qty FROM "order_lines"')) assert rows == [("order1", "DECORATIVE-WIDGET", 12)] + + +def test_batches(session): + session.execute('INSERT INTO "batches" VALUES ("batch1", "sku1", 100, null)') + session.execute('INSERT INTO "batches" VALUES ("batch2", "sku2", 200, "2011-04-11")') + expected = [ + model.Batch("batch1", "sku1", 100, eta=None), + model.Batch("batch2", "sku2", 200, eta=date(2011, 4, 11)), + ] + + assert session.query(model.Batch).all() == expected + From eb08e7b738108cc4f51d2d21aab658d3020e36ef Mon Sep 17 00:00:00 2001 From: Harry Date: Mon, 11 Mar 2019 00:48:49 +0000 Subject: [PATCH 026/165] ORM for _allocations set on Batch --- orm.py | 40 ++++++++++++++++++++++++++---------- test_orm.py | 58 ++++++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 84 insertions(+), 14 deletions(-) diff --git a/orm.py b/orm.py index 7dcbac70..11d5a8c5 100644 --- a/orm.py +++ b/orm.py @@ -1,5 +1,8 @@ -from sqlalchemy import Table, MetaData, Column, Integer, String, Date -from sqlalchemy.orm import mapper +from sqlalchemy import ( + Table, MetaData, Column, Integer, String, Date, + ForeignKey +) +from sqlalchemy.orm import mapper, relationship import model @@ -8,20 +11,35 @@ order_lines = Table( 'order_lines', metadata, - Column('orderid', String(255), primary_key=True), - Column('sku', String(255), primary_key=True), - Column('qty', Integer), + Column('id', Integer, primary_key=True, autoincrement=True), + Column('sku', String(255)), + Column('qty', Integer, nullable=False), + Column('orderid', String(255)), ) batches = Table( 'batches', metadata, - Column('reference', String(255), primary_key=True), - Column('sku', String(255), primary_key=True), - Column('_purchased_qty', Integer), - Column('eta', Date), + Column('id', Integer, primary_key=True, autoincrement=True), + Column('reference', String(255)), + Column('sku', String(255)), + Column('_purchased_quantity', Integer, nullable=False), + Column('eta', Date, nullable=True), +) + +allocations = Table( + 'allocations', metadata, + Column('id', Integer, primary_key=True, autoincrement=True), + Column('orderline_id', ForeignKey('order_lines.id')), + Column('batch_id', ForeignKey('batches.id')), ) def start_mappers(): - mapper(model.OrderLine, order_lines) - mapper(model.Batch, batches) + lines_mapper = mapper(model.OrderLine, order_lines) + mapper(model.Batch, batches, properties={ + '_allocations': relationship( + lines_mapper, + secondary=allocations, + collection_class=set, + ) + }) diff --git a/test_orm.py b/test_orm.py index fa81e752..c7788a8c 100644 --- a/test_orm.py +++ b/test_orm.py @@ -26,9 +26,15 @@ def test_orderline_mapper_can_save_lines(session): -def test_batches(session): - session.execute('INSERT INTO "batches" VALUES ("batch1", "sku1", 100, null)') - session.execute('INSERT INTO "batches" VALUES ("batch2", "sku2", 200, "2011-04-11")') +def test_retrieving_batches(session): + session.execute( + 'INSERT INTO batches (reference, sku, _purchased_quantity, eta)' + ' VALUES ("batch1", "sku1", 100, null)' + ) + session.execute( + 'INSERT INTO batches (reference, sku, _purchased_quantity, eta)' + ' VALUES ("batch2", "sku2", 200, "2011-04-11")' + ) expected = [ model.Batch("batch1", "sku1", 100, eta=None), model.Batch("batch2", "sku2", 200, eta=date(2011, 4, 11)), @@ -36,3 +42,49 @@ def test_batches(session): assert session.query(model.Batch).all() == expected + +def test_saving_batches(session): + batch = model.Batch('batch1', 'sku1', 100, eta=None) + session.add(batch) + session.commit() + rows = list(session.execute( + 'SELECT reference, sku, _purchased_quantity, eta FROM "batches"' + )) + assert rows == [('batch1', 'sku1', 100, None)] + +def test_saving_allocations(session): + batch = model.Batch('batch1', 'sku1', 100, eta=None) + line = model.OrderLine('order1', 'sku1', 10) + batch.allocate(line) + session.add(batch) + session.commit() + rows = list(session.execute('SELECT orderline_id, batch_id FROM "allocations"')) + assert rows == [(batch.id, line.id)] + + +def test_retrieving_allocations(session): + session.execute( + 'INSERT INTO order_lines (orderid, sku, qty) VALUES ("order1", "sku1", 12)' + ) + [[olid]] = session.execute( + 'SELECT id FROM order_lines WHERE orderid=:orderid AND sku=:sku', + dict(orderid='order1', sku='sku1') + ) + session.execute( + 'INSERT INTO batches (reference, sku, _purchased_quantity, eta)' + ' VALUES ("batch1", "sku1", 100, null)' + ) + [[bid]] = session.execute( + 'SELECT id FROM batches WHERE reference=:ref AND sku=:sku', + dict(ref='batch1', sku='sku1') + ) + session.execute( + 'INSERT INTO allocations (orderline_id, batch_id) VALUES (:olid, :bid)', + dict(olid=olid, bid=bid) + ) + + batch = session.query(model.Batch).one() + + assert batch._allocations == { + model.OrderLine("order1", "sku1", 12) + } From ee1cb02f0444ceee3786719de187162b046c9ad3 Mon Sep 17 00:00:00 2001 From: Harry Date: Mon, 11 Mar 2019 01:10:30 +0000 Subject: [PATCH 027/165] repository tests --- test_repository.py | 64 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 test_repository.py diff --git a/test_repository.py b/test_repository.py new file mode 100644 index 00000000..c7cb5cb7 --- /dev/null +++ b/test_repository.py @@ -0,0 +1,64 @@ +# pylint: disable=protected-access +import model +import repository + +def test_repository_can_save_a_batch(session): + batch = model.Batch("batch1", "RUSTY-SOAPDISH", 100, eta=None) + + repo = repository.SqlAlchemyRepository(session) + repo.add(batch) + session.commit() + + rows = list(session.execute( + 'SELECT reference, sku, _purchased_quantity, eta FROM "batches"' + )) + assert rows == [("batch1", "RUSTY-SOAPDISH", 100, None)] + + +def insert_order_line(session): + session.execute( + 'INSERT INTO order_lines (orderid, sku, qty)' + ' VALUES ("order1", "GENERIC-SOFA", 12)' + ) + [[orderline_id]] = session.execute( + 'SELECT id FROM order_lines WHERE orderid=:orderid AND sku=:sku', + dict(orderid="order1", sku="GENERIC-SOFA") + ) + return orderline_id + +def insert_batch(session, batch_id): + session.execute( + 'INSERT INTO batches (reference, sku, _purchased_quantity, eta)' + ' VALUES (:batch_id, "GENERIC-SOFA", 100, null)', + dict(batch_id=batch_id) + ) + [[batch_id]] = session.execute( + 'SELECT id FROM batches WHERE reference=:batch_id AND sku="GENERIC-SOFA"', + dict(batch_id=batch_id) + ) + return batch_id + +def insert_allocation(session, orderline_id, batch_id): + session.execute( + 'INSERT INTO allocations (orderline_id, batch_id)' + ' VALUES (:orderline_id, :batch_id)', + dict(orderline_id=orderline_id, batch_id=batch_id) + ) + + +def test_repository_can_retrieve_a_batch_with_allocations(session): + orderline_id = insert_order_line(session) + batch1_id = insert_batch(session, "batch1") + insert_batch(session, "batch2") + insert_allocation(session, orderline_id, batch1_id) + + repo = repository.SqlAlchemyRepository(session) + retrieved = repo.get("batch1") + + expected = model.Batch("batch1", "GENERIC-SOFA", 100, eta=None) + assert retrieved == expected # Batch.__eq__ only compares reference + assert retrieved.sku == expected.sku + assert retrieved._purchased_quantity == expected._purchased_quantity + assert retrieved._allocations == { + model.OrderLine("order1", "GENERIC-SOFA", 12), + } From 95099b99d789ef559b7168014d0dc4acfcc47071 Mon Sep 17 00:00:00 2001 From: Harry Date: Mon, 11 Mar 2019 01:10:39 +0000 Subject: [PATCH 028/165] repository for batches [chapter_02_repository_ends] --- repository.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 repository.py diff --git a/repository.py b/repository.py new file mode 100644 index 00000000..074fe12e --- /dev/null +++ b/repository.py @@ -0,0 +1,29 @@ +import abc +import model + + +class AbstractRepository(abc.ABC): + + @abc.abstractmethod + def add(self, batch: model.Batch): + raise NotImplementedError + + @abc.abstractmethod + def get(self, reference) -> model.Batch: + raise NotImplementedError + + + +class SqlAlchemyRepository(AbstractRepository): + + def __init__(self, session): + self.session = session + + def add(self, batch): + self.session.add(batch) + + def get(self, reference): + return self.session.query(model.Batch).filter_by(reference=reference).one() + + def list(self): + return self.session.query(model.Batch).all() From 4089712ea973eec374dab48ffa26167098f4f59b Mon Sep 17 00:00:00 2001 From: Harry Date: Mon, 11 Mar 2019 04:10:52 +0000 Subject: [PATCH 029/165] first api tests [first_api_test] --- config.py | 15 +++++++++++++ conftest.py | 64 +++++++++++++++++++++++++++++++++++++++++++++++++++++ test_api.py | 35 +++++++++++++++++++++++++++++ 3 files changed, 114 insertions(+) create mode 100644 config.py create mode 100644 test_api.py diff --git a/config.py b/config.py new file mode 100644 index 00000000..7b1584bb --- /dev/null +++ b/config.py @@ -0,0 +1,15 @@ +import os + +def get_postgres_uri(): + host = os.environ.get('DB_HOST', 'localhost') + port = 54321 if host == 'localhost' else 5432 + password = os.environ.get('DB_PASSWORD', 'abc123') + user, db_name = 'allocation', 'allocation' + return f"postgresql://{user}:{password}@{host}:{port}/{db_name}" + + +def get_api_url(): + host = os.environ.get('API_HOST', 'localhost') + port = 5005 if host == 'localhost' else 80 + return f"http://{host}:{port}" + diff --git a/conftest.py b/conftest.py index b64099fd..36bbad31 100644 --- a/conftest.py +++ b/conftest.py @@ -1,8 +1,13 @@ +# pylint: disable=redefined-outer-name +import time +from pathlib import Path + import pytest from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker, clear_mappers from orm import metadata, start_mappers +from config import get_postgres_uri @pytest.fixture @@ -18,3 +23,62 @@ def session(in_memory_db): yield sessionmaker(bind=in_memory_db)() clear_mappers() + +@pytest.fixture(scope='session') +def postgres_db(): + engine = create_engine(get_postgres_uri()) + metadata.create_all(engine) + return engine + + +@pytest.fixture +def postgres_session(postgres_db): + start_mappers() + yield sessionmaker(bind=postgres_db)() + clear_mappers() + + +@pytest.fixture +def add_stock(postgres_session): + batches_added = set() + skus_added = set() + + def _add_stock(lines): + for ref, sku, qty, eta in lines: + postgres_session.execute( + 'INSERT INTO batches (reference, sku, _purchased_quantity, eta)' + ' VALUES (:ref, :sku, :qty, :eta)', + dict(ref=ref, sku=sku, qty=qty, eta=eta), + ) + [[batch_id]] = postgres_session.execute( + 'SELECT id FROM batches WHERE reference=:ref AND sku=:sku', + dict(ref=ref, sku=sku), + ) + batches_added.add(batch_id) + skus_added.add(sku) + postgres_session.commit() + + yield _add_stock + + for batch_id in batches_added: + postgres_session.execute( + 'DELETE FROM allocations WHERE batch_id=:batch_id', + dict(batch_id=batch_id), + ) + postgres_session.execute( + 'DELETE FROM batches WHERE id=:batch_id', + dict(batch_id=batch_id), + ) + for sku in skus_added: + postgres_session.execute( + 'DELETE FROM order_lines WHERE sku=:sku', + dict(sku=sku), + ) + postgres_session.commit() + + +@pytest.fixture +def restart_api(): + (Path(__file__).parent / 'flask_app.py').touch() + time.sleep(0.3) + diff --git a/test_api.py b/test_api.py new file mode 100644 index 00000000..88310309 --- /dev/null +++ b/test_api.py @@ -0,0 +1,35 @@ +import uuid +import pytest +import requests + +import config + +def random_suffix(): + return uuid.uuid4().hex[:6] + +def random_sku(name=''): + return f'sku-{name}-{random_suffix()}' + +def random_batchref(name=''): + return f'batch-{name}-{random_suffix()}' + +def random_orderid(name=''): + return f'order-{name}-{random_suffix()}' + + +@pytest.mark.usefixtures('restart_api') +def test_api_returns_allocation(add_stock): + sku, othersku = random_sku(), random_sku('other') + earlybatch = random_batchref(1) + laterbatch = random_batchref(2) + otherbatch = random_batchref(3) + add_stock([ + (laterbatch, sku, 100, '2011-01-02'), + (earlybatch, sku, 100, '2011-01-01'), + (otherbatch, othersku, 100, None), + ]) + data = {'orderid': random_orderid(), 'sku': sku, 'qty': 3} + url = config.get_api_url() + r = requests.post(f'{url}/allocate', json=data) + assert r.status_code == 201 + assert r.json()['batchref'] == earlybatch From bae4b779f155716217f5b628c09af7081a1b6289 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 19 Mar 2019 11:30:31 +0000 Subject: [PATCH 030/165] all the dockerfile gubbins --- Dockerfile | 15 +++++++++++++++ docker-compose.yml | 26 ++++++++++++++++++++++++++ requirements.txt | 4 ++++ 3 files changed, 45 insertions(+) create mode 100644 Dockerfile create mode 100644 docker-compose.yml create mode 100644 requirements.txt diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..13730e6e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,15 @@ +FROM python:3.8-alpine + +RUN apk add --no-cache --virtual .build-deps gcc postgresql-dev musl-dev python3-dev +RUN apk add libpq + +COPY requirements.txt /tmp +RUN pip install -r /tmp/requirements.txt + +RUN apk del --no-cache .build-deps + +RUN mkdir -p /code +COPY *.py /code/ +WORKDIR /code +ENV FLASK_APP=flask_app.py FLASK_DEBUG=1 PYTHONUNBUFFERED=1 +CMD flask run --host=0.0.0.0 --port=80 diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..313cf946 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,26 @@ +version: "3" +services: + + app: + build: + context: . + dockerfile: Dockerfile + depends_on: + - postgres + environment: + - DB_HOST=postgres + - DB_PASSWORD=abc123 + volumes: + - ./:/code + ports: + - "5005:80" + + + postgres: + image: postgres:9.6 + environment: + - POSTGRES_USER=allocation + - POSTGRES_PASSWORD=abc123 + ports: + - "54321:5432" + diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..dabc8617 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +pytest +sqlalchemy +flask +psycopg2 From d377597e87e3f34b996eecb926985b4e93f2fa9a Mon Sep 17 00:00:00 2001 From: Harry Date: Mon, 11 Mar 2019 04:21:51 +0000 Subject: [PATCH 031/165] first cut of flask app [first_cut_flask_app] --- flask_app.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 flask_app.py diff --git a/flask_app.py b/flask_app.py new file mode 100644 index 00000000..5841b78f --- /dev/null +++ b/flask_app.py @@ -0,0 +1,27 @@ +from flask import Flask, jsonify, request +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +import config +import model +import orm +import repository + + +orm.start_mappers() +get_session = sessionmaker(bind=create_engine(config.get_postgres_uri())) +app = Flask(__name__) + +@app.route("/allocate", methods=['POST']) +def allocate_endpoint(): + session = get_session() + batches = repository.SqlAlchemyRepository(session).list() + line = model.OrderLine( + request.json['orderid'], + request.json['sku'], + request.json['qty'], + ) + + batchref = model.allocate(line, batches) + + return jsonify({'batchref': batchref}), 201 From 2bd8dbc7ebb4760b8eef40b8be3d310a6825f2a7 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 19 Mar 2019 11:58:59 +0000 Subject: [PATCH 032/165] test persistence by double-allocating. [second_api_test] --- test_api.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/test_api.py b/test_api.py index 88310309..85b842e7 100644 --- a/test_api.py +++ b/test_api.py @@ -33,3 +33,27 @@ def test_api_returns_allocation(add_stock): r = requests.post(f'{url}/allocate', json=data) assert r.status_code == 201 assert r.json()['batchref'] == earlybatch + + +@pytest.mark.usefixtures('restart_api') +def test_allocations_are_persisted(add_stock): + sku = random_sku() + batch1, batch2 = random_batchref(1), random_batchref(2) + order1, order2 = random_orderid(1), random_orderid(2) + add_stock([ + (batch1, sku, 10, '2011-01-01'), + (batch2, sku, 10, '2011-01-02'), + ]) + line1 = {'orderid': order1, 'sku': sku, 'qty': 10} + line2 = {'orderid': order2, 'sku': sku, 'qty': 10} + url = config.get_api_url() + + # first order uses up all stock in batch 1 + r = requests.post(f'{url}/allocate', json=line1) + assert r.status_code == 201 + assert r.json()['batchref'] == batch1 + + # second order should go to batch 2 + r = requests.post(f'{url}/allocate', json=line2) + assert r.status_code == 201 + assert r.json()['batchref'] == batch2 From bf9fd5b9d3339e2d862c338a4e77334386becee2 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 19 Mar 2019 11:59:20 +0000 Subject: [PATCH 033/165] need to commit [flask_commit] --- flask_app.py | 1 + 1 file changed, 1 insertion(+) diff --git a/flask_app.py b/flask_app.py index 5841b78f..4b79f3ba 100644 --- a/flask_app.py +++ b/flask_app.py @@ -24,4 +24,5 @@ def allocate_endpoint(): batchref = model.allocate(line, batches) + session.commit() return jsonify({'batchref': batchref}), 201 From 213243a02dd7d0ac29698bd8199c6fd9c7c302db Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 19 Mar 2019 12:10:34 +0000 Subject: [PATCH 034/165] test some 400 error cases [test_error_cases] --- test_api.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/test_api.py b/test_api.py index 85b842e7..dc9e7823 100644 --- a/test_api.py +++ b/test_api.py @@ -57,3 +57,26 @@ def test_allocations_are_persisted(add_stock): r = requests.post(f'{url}/allocate', json=line2) assert r.status_code == 201 assert r.json()['batchref'] == batch2 + + +@pytest.mark.usefixtures('restart_api') +def test_400_message_for_out_of_stock(add_stock): + sku, smalL_batch, large_order = random_sku(), random_batchref(), random_orderid() + add_stock([ + (smalL_batch, sku, 10, '2011-01-01'), + ]) + data = {'orderid': large_order, 'sku': sku, 'qty': 20} + url = config.get_api_url() + r = requests.post(f'{url}/allocate', json=data) + assert r.status_code == 400 + assert r.json()['message'] == f'Out of stock for sku {sku}' + + +@pytest.mark.usefixtures('restart_api') +def test_400_message_for_invalid_sku(): + unknown_sku, orderid = random_sku(), random_orderid() + data = {'orderid': orderid, 'sku': unknown_sku, 'qty': 20} + url = config.get_api_url() + r = requests.post(f'{url}/allocate', json=data) + assert r.status_code == 400 + assert r.json()['message'] == f'Invalid sku {unknown_sku}' From 5b9b48e2dc634b18512ee60da2b9d867afe0c6ee Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 19 Mar 2019 12:17:15 +0000 Subject: [PATCH 035/165] flask now does error handling [flask_error_handling] --- flask_app.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/flask_app.py b/flask_app.py index 4b79f3ba..71d3461c 100644 --- a/flask_app.py +++ b/flask_app.py @@ -12,6 +12,9 @@ get_session = sessionmaker(bind=create_engine(config.get_postgres_uri())) app = Flask(__name__) +def is_valid_sku(sku, batches): + return sku in {b.sku for b in batches} + @app.route("/allocate", methods=['POST']) def allocate_endpoint(): session = get_session() @@ -22,7 +25,14 @@ def allocate_endpoint(): request.json['qty'], ) - batchref = model.allocate(line, batches) + if not is_valid_sku(line.sku, batches): + return jsonify({'message': f'Invalid sku {line.sku}'}), 400 + + try: + batchref = model.allocate(line, batches) + except model.OutOfStock as e: + return jsonify({'message': str(e)}), 400 session.commit() return jsonify({'batchref': batchref}), 201 + From 7f3fda2492406de8110ba1ffb8308d32ef36bd2b Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 19 Mar 2019 13:44:27 +0000 Subject: [PATCH 036/165] first tests for the services layer [first_services_tests] --- test_services.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 test_services.py diff --git a/test_services.py b/test_services.py new file mode 100644 index 00000000..92dcd517 --- /dev/null +++ b/test_services.py @@ -0,0 +1,22 @@ +import pytest + +import model +import services + + +def test_returns_allocation(): + line = model.OrderLine("o1", "COMPLICATED-LAMP", 10) + batch = model.Batch("b1", "COMPLICATED-LAMP", 100, eta=None) + repo = FakeRepository([batch]) + + result = services.allocate(line, repo, FakeSession()) + assert result == "b1" + + +def test_error_for_invalid_sku(): + line = model.OrderLine("o1", "NONEXISTENTSKU", 10) + batch = model.Batch("b1", "AREALSKU", 100, eta=None) + repo = FakeRepository([batch]) + + with pytest.raises(services.InvalidSku, match="Invalid sku NONEXISTENTSKU"): + services.allocate(line, repo, FakeSession()) From 08651a5104e84a92de593dc29e9853c98cabf21c Mon Sep 17 00:00:00 2001 From: Harry Date: Fri, 3 Jan 2020 00:54:06 +0000 Subject: [PATCH 037/165] FakeRepository [fake_repo] --- test_services.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/test_services.py b/test_services.py index 92dcd517..10d4ce20 100644 --- a/test_services.py +++ b/test_services.py @@ -1,9 +1,24 @@ import pytest - import model +import repository import services +class FakeRepository(repository.AbstractRepository): + + def __init__(self, batches): + self._batches = set(batches) + + def add(self, batch): + self._batches.add(batch) + + def get(self, reference): + return next(b for b in self._batches if b.reference == reference) + + def list(self): + return list(self._batches) + + def test_returns_allocation(): line = model.OrderLine("o1", "COMPLICATED-LAMP", 10) batch = model.Batch("b1", "COMPLICATED-LAMP", 100, eta=None) From c2e577a3546d1db5b886ca8ac4855ace1fc80ea4 Mon Sep 17 00:00:00 2001 From: Harry Date: Fri, 3 Jan 2020 00:54:48 +0000 Subject: [PATCH 038/165] FakeSession [fake_session] --- test_services.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test_services.py b/test_services.py index 10d4ce20..9a1cb587 100644 --- a/test_services.py +++ b/test_services.py @@ -19,6 +19,13 @@ def list(self): return list(self._batches) +class FakeSession(): + committed = False + + def commit(self): + self.committed = True + + def test_returns_allocation(): line = model.OrderLine("o1", "COMPLICATED-LAMP", 10) batch = model.Batch("b1", "COMPLICATED-LAMP", 100, eta=None) From 83a38a65564f92b4d05cb84f25566edbbda4de8c Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 23 Apr 2019 12:52:08 +0100 Subject: [PATCH 039/165] test commmits [second_services_test] --- test_services.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test_services.py b/test_services.py index 9a1cb587..e242e65b 100644 --- a/test_services.py +++ b/test_services.py @@ -42,3 +42,13 @@ def test_error_for_invalid_sku(): with pytest.raises(services.InvalidSku, match="Invalid sku NONEXISTENTSKU"): services.allocate(line, repo, FakeSession()) + + +def test_commits(): + line = model.OrderLine('o1', 'sku1', 10) + batch = model.Batch('b1', 'sku1', 100, eta=None) + repo = FakeRepository([batch]) + session = FakeSession() + + services.allocate(line, repo, session) + assert session.committed is True From 1fd431e2bce8b92e436e224fe2fe0e000dafd5ea Mon Sep 17 00:00:00 2001 From: Harry Date: Thu, 18 Jul 2019 13:21:43 +0100 Subject: [PATCH 040/165] Test for commits [second_services_test] --- test_services.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test_services.py b/test_services.py index e242e65b..921ba107 100644 --- a/test_services.py +++ b/test_services.py @@ -45,8 +45,8 @@ def test_error_for_invalid_sku(): def test_commits(): - line = model.OrderLine('o1', 'sku1', 10) - batch = model.Batch('b1', 'sku1', 100, eta=None) + line = model.OrderLine('o1', 'OMINOUS-MIRROR', 10) + batch = model.Batch('b1', 'OMINOUS-MIRROR', 100, eta=None) repo = FakeRepository([batch]) session = FakeSession() From 5fd89961a7a723b144532c58f7dd02a857a507ae Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 19 Mar 2019 13:44:44 +0000 Subject: [PATCH 041/165] services layer with valid-sku check [service_function] --- services.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 services.py diff --git a/services.py b/services.py new file mode 100644 index 00000000..92370944 --- /dev/null +++ b/services.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +import model +from model import OrderLine +from repository import AbstractRepository + +class InvalidSku(Exception): + pass + + +def is_valid_sku(sku, batches): + return sku in {b.sku for b in batches} + +def allocate(line: OrderLine, repo: AbstractRepository, session) -> str: + batches = repo.list() + if not is_valid_sku(line.sku, batches): + raise InvalidSku(f'Invalid sku {line.sku}') + batchref = model.allocate(line, batches) + session.commit() + return batchref From 9017372983487758b4312c0c4727e5cdbeb10492 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 19 Mar 2019 14:33:19 +0000 Subject: [PATCH 042/165] modify flask app to use service layer [flask_app_using_service_layer] --- flask_app.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/flask_app.py b/flask_app.py index 71d3461c..cf3173c8 100644 --- a/flask_app.py +++ b/flask_app.py @@ -6,33 +6,25 @@ import model import orm import repository +import services orm.start_mappers() get_session = sessionmaker(bind=create_engine(config.get_postgres_uri())) app = Flask(__name__) -def is_valid_sku(sku, batches): - return sku in {b.sku for b in batches} - @app.route("/allocate", methods=['POST']) def allocate_endpoint(): session = get_session() - batches = repository.SqlAlchemyRepository(session).list() + repo = repository.SqlAlchemyRepository(session) line = model.OrderLine( request.json['orderid'], request.json['sku'], request.json['qty'], ) - - if not is_valid_sku(line.sku, batches): - return jsonify({'message': f'Invalid sku {line.sku}'}), 400 - try: - batchref = model.allocate(line, batches) - except model.OutOfStock as e: + batchref = services.allocate(line, repo, session) + except (model.OutOfStock, services.InvalidSku) as e: return jsonify({'message': str(e)}), 400 - session.commit() return jsonify({'batchref': batchref}), 201 - From 3f6cec6df2f6f3af167a73c945e180b7ac1e9acf Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 19 Mar 2019 14:48:36 +0000 Subject: [PATCH 043/165] strip out unecessary tests from e2e layer [fewer_e2e_tests] --- test_api.py | 41 ++--------------------------------------- 1 file changed, 2 insertions(+), 39 deletions(-) diff --git a/test_api.py b/test_api.py index dc9e7823..8bb36297 100644 --- a/test_api.py +++ b/test_api.py @@ -18,7 +18,7 @@ def random_orderid(name=''): @pytest.mark.usefixtures('restart_api') -def test_api_returns_allocation(add_stock): +def test_happy_path_returns_201_and_allocated_batch(add_stock): sku, othersku = random_sku(), random_sku('other') earlybatch = random_batchref(1) laterbatch = random_batchref(2) @@ -36,44 +36,7 @@ def test_api_returns_allocation(add_stock): @pytest.mark.usefixtures('restart_api') -def test_allocations_are_persisted(add_stock): - sku = random_sku() - batch1, batch2 = random_batchref(1), random_batchref(2) - order1, order2 = random_orderid(1), random_orderid(2) - add_stock([ - (batch1, sku, 10, '2011-01-01'), - (batch2, sku, 10, '2011-01-02'), - ]) - line1 = {'orderid': order1, 'sku': sku, 'qty': 10} - line2 = {'orderid': order2, 'sku': sku, 'qty': 10} - url = config.get_api_url() - - # first order uses up all stock in batch 1 - r = requests.post(f'{url}/allocate', json=line1) - assert r.status_code == 201 - assert r.json()['batchref'] == batch1 - - # second order should go to batch 2 - r = requests.post(f'{url}/allocate', json=line2) - assert r.status_code == 201 - assert r.json()['batchref'] == batch2 - - -@pytest.mark.usefixtures('restart_api') -def test_400_message_for_out_of_stock(add_stock): - sku, smalL_batch, large_order = random_sku(), random_batchref(), random_orderid() - add_stock([ - (smalL_batch, sku, 10, '2011-01-01'), - ]) - data = {'orderid': large_order, 'sku': sku, 'qty': 20} - url = config.get_api_url() - r = requests.post(f'{url}/allocate', json=data) - assert r.status_code == 400 - assert r.json()['message'] == f'Out of stock for sku {sku}' - - -@pytest.mark.usefixtures('restart_api') -def test_400_message_for_invalid_sku(): +def test_unhappy_path_returns_400_and_error_message(): unknown_sku, orderid = random_sku(), random_orderid() data = {'orderid': orderid, 'sku': unknown_sku, 'qty': 20} url = config.get_api_url() From 3d1eb24c41ad8cc430a7ea592086057ef4fbb472 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 26 Mar 2019 21:21:17 +0000 Subject: [PATCH 044/165] fix conftest waits, requirements.txt and travis config --- .travis.yml | 4 ++-- Makefile | 9 +++++++-- conftest.py | 33 ++++++++++++++++++++++++++++++--- requirements.txt | 1 + 4 files changed, 40 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8a74f94b..f601680f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,10 +3,10 @@ language: python python: 3.8 install: -- pip3 install sqlalchemy +- pip3 install -r requirements.txt script: -- make test +- make all branches: except: diff --git a/Makefile b/Makefile index 46cc16d9..80c383c0 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,10 @@ +build: + docker-compose build + +up: + docker-compose up -d app + test: pytest --tb=short -watch-tests: - ls *.py | entr pytest --tb=short +all: build up test diff --git a/conftest.py b/conftest.py index 36bbad31..be7b0e18 100644 --- a/conftest.py +++ b/conftest.py @@ -3,11 +3,14 @@ from pathlib import Path import pytest +import requests +from requests.exceptions import ConnectionError +from sqlalchemy.exc import OperationalError from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker, clear_mappers from orm import metadata, start_mappers -from config import get_postgres_uri +import config @pytest.fixture @@ -24,9 +27,32 @@ def session(in_memory_db): clear_mappers() +def wait_for_postgres_to_come_up(engine): + deadline = time.time() + 10 + while time.time() < deadline: + try: + return engine.connect() + except OperationalError: + time.sleep(0.5) + pytest.fail('Postgres never came up') + + +def wait_for_webapp_to_come_up(): + deadline = time.time() + 10 + url = config.get_api_url() + while time.time() < deadline: + try: + return requests.get(url) + except ConnectionError: + time.sleep(0.5) + pytest.fail('API never came up') + + + @pytest.fixture(scope='session') def postgres_db(): - engine = create_engine(get_postgres_uri()) + engine = create_engine(config.get_postgres_uri()) + wait_for_postgres_to_come_up(engine) metadata.create_all(engine) return engine @@ -80,5 +106,6 @@ def _add_stock(lines): @pytest.fixture def restart_api(): (Path(__file__).parent / 'flask_app.py').touch() - time.sleep(0.3) + time.sleep(0.5) + wait_for_webapp_to_come_up() diff --git a/requirements.txt b/requirements.txt index dabc8617..2ff54bf8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,4 @@ pytest sqlalchemy flask psycopg2 +requests From 61d850ff9e298448757e18a2dbb819a04eb1d72d Mon Sep 17 00:00:00 2001 From: Harry Date: Wed, 24 Apr 2019 12:21:25 +0100 Subject: [PATCH 045/165] makefile: down and logs [chapter_04_service_layer_ends] --- Makefile | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 80c383c0..f62eef70 100644 --- a/Makefile +++ b/Makefile @@ -7,4 +7,10 @@ up: test: pytest --tb=short -all: build up test +logs: + docker-compose logs app | tail -100 + +down: + docker-compose down + +all: down build up test From 180f2327c95fa1d32a6ce11d6851ffc05611ba5f Mon Sep 17 00:00:00 2001 From: Harry Date: Mon, 23 Dec 2019 16:37:06 +0000 Subject: [PATCH 046/165] move to a more nested folder structure --- orm.py => adapters/orm.py | 0 repository.py => adapters/repository.py | 0 model.py => domain/model.py | 0 flask_app.py => entrypoints/flask_app.py | 0 services.py => service_layer/services.py | 0 5 files changed, 0 insertions(+), 0 deletions(-) rename orm.py => adapters/orm.py (100%) rename repository.py => adapters/repository.py (100%) rename model.py => domain/model.py (100%) rename flask_app.py => entrypoints/flask_app.py (100%) rename services.py => service_layer/services.py (100%) diff --git a/orm.py b/adapters/orm.py similarity index 100% rename from orm.py rename to adapters/orm.py diff --git a/repository.py b/adapters/repository.py similarity index 100% rename from repository.py rename to adapters/repository.py diff --git a/model.py b/domain/model.py similarity index 100% rename from model.py rename to domain/model.py diff --git a/flask_app.py b/entrypoints/flask_app.py similarity index 100% rename from flask_app.py rename to entrypoints/flask_app.py diff --git a/services.py b/service_layer/services.py similarity index 100% rename from services.py rename to service_layer/services.py From d5b5e9869096b82ba1e93f5cee964cee85379d0d Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 31 Dec 2019 13:18:04 +0000 Subject: [PATCH 047/165] nest the tests too --- test_api.py => tests/e2e/test_api.py | 0 test_orm.py => tests/integration/test_orm.py | 0 test_repository.py => tests/integration/test_repository.py | 0 test_allocate.py => tests/unit/test_allocate.py | 0 test_batches.py => tests/unit/test_batches.py | 0 test_services.py => tests/unit/test_services.py | 0 6 files changed, 0 insertions(+), 0 deletions(-) rename test_api.py => tests/e2e/test_api.py (100%) rename test_orm.py => tests/integration/test_orm.py (100%) rename test_repository.py => tests/integration/test_repository.py (100%) rename test_allocate.py => tests/unit/test_allocate.py (100%) rename test_batches.py => tests/unit/test_batches.py (100%) rename test_services.py => tests/unit/test_services.py (100%) diff --git a/test_api.py b/tests/e2e/test_api.py similarity index 100% rename from test_api.py rename to tests/e2e/test_api.py diff --git a/test_orm.py b/tests/integration/test_orm.py similarity index 100% rename from test_orm.py rename to tests/integration/test_orm.py diff --git a/test_repository.py b/tests/integration/test_repository.py similarity index 100% rename from test_repository.py rename to tests/integration/test_repository.py diff --git a/test_allocate.py b/tests/unit/test_allocate.py similarity index 100% rename from test_allocate.py rename to tests/unit/test_allocate.py diff --git a/test_batches.py b/tests/unit/test_batches.py similarity index 100% rename from test_batches.py rename to tests/unit/test_batches.py diff --git a/test_services.py b/tests/unit/test_services.py similarity index 100% rename from test_services.py rename to tests/unit/test_services.py From 25da595958eada5d994782d898315bc629a18718 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 31 Dec 2019 12:44:27 +0000 Subject: [PATCH 048/165] get all tests passing --- Dockerfile | 2 +- adapters/__init__.py | 0 adapters/orm.py | 2 +- adapters/repository.py | 2 +- domain/__init__.py | 0 entrypoints/__init__.py | 0 entrypoints/flask_app.py | 7 +++---- service_layer/__init__.py | 0 service_layer/services.py | 6 +++--- tests/__init__.py | 0 conftest.py => tests/conftest.py | 4 ++-- tests/integration/test_orm.py | 2 +- tests/integration/test_repository.py | 4 ++-- tests/unit/test_allocate.py | 2 +- tests/unit/test_batches.py | 2 +- tests/unit/test_services.py | 6 +++--- 16 files changed, 19 insertions(+), 20 deletions(-) create mode 100644 adapters/__init__.py create mode 100644 domain/__init__.py create mode 100644 entrypoints/__init__.py create mode 100644 service_layer/__init__.py create mode 100644 tests/__init__.py rename conftest.py => tests/conftest.py (95%) diff --git a/Dockerfile b/Dockerfile index 13730e6e..8c7c140f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,5 +11,5 @@ RUN apk del --no-cache .build-deps RUN mkdir -p /code COPY *.py /code/ WORKDIR /code -ENV FLASK_APP=flask_app.py FLASK_DEBUG=1 PYTHONUNBUFFERED=1 +ENV FLASK_APP=entrypoints/flask_app.py FLASK_DEBUG=1 PYTHONUNBUFFERED=1 CMD flask run --host=0.0.0.0 --port=80 diff --git a/adapters/__init__.py b/adapters/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/adapters/orm.py b/adapters/orm.py index 11d5a8c5..571240df 100644 --- a/adapters/orm.py +++ b/adapters/orm.py @@ -4,7 +4,7 @@ ) from sqlalchemy.orm import mapper, relationship -import model +from domain import model metadata = MetaData() diff --git a/adapters/repository.py b/adapters/repository.py index 074fe12e..e2aab99d 100644 --- a/adapters/repository.py +++ b/adapters/repository.py @@ -1,5 +1,5 @@ import abc -import model +from domain import model class AbstractRepository(abc.ABC): diff --git a/domain/__init__.py b/domain/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/entrypoints/__init__.py b/entrypoints/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/entrypoints/flask_app.py b/entrypoints/flask_app.py index cf3173c8..6d66d71f 100644 --- a/entrypoints/flask_app.py +++ b/entrypoints/flask_app.py @@ -3,10 +3,9 @@ from sqlalchemy.orm import sessionmaker import config -import model -import orm -import repository -import services +from domain import model +from adapters import orm, repository +from service_layer import services orm.start_mappers() diff --git a/service_layer/__init__.py b/service_layer/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/service_layer/services.py b/service_layer/services.py index 92370944..254f607f 100644 --- a/service_layer/services.py +++ b/service_layer/services.py @@ -1,8 +1,8 @@ from __future__ import annotations -import model -from model import OrderLine -from repository import AbstractRepository +from domain import model +from domain.model import OrderLine +from adapters.repository import AbstractRepository class InvalidSku(Exception): pass diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/conftest.py b/tests/conftest.py similarity index 95% rename from conftest.py rename to tests/conftest.py index be7b0e18..127bb20c 100644 --- a/conftest.py +++ b/tests/conftest.py @@ -9,7 +9,7 @@ from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker, clear_mappers -from orm import metadata, start_mappers +from adapters.orm import metadata, start_mappers import config @@ -105,7 +105,7 @@ def _add_stock(lines): @pytest.fixture def restart_api(): - (Path(__file__).parent / 'flask_app.py').touch() + (Path(__file__).parent / '../entrypoints/flask_app.py').touch() time.sleep(0.5) wait_for_webapp_to_come_up() diff --git a/tests/integration/test_orm.py b/tests/integration/test_orm.py index c7788a8c..3426d460 100644 --- a/tests/integration/test_orm.py +++ b/tests/integration/test_orm.py @@ -1,4 +1,4 @@ -import model +from domain import model from datetime import date def test_orderline_mapper_can_load_lines(session): diff --git a/tests/integration/test_repository.py b/tests/integration/test_repository.py index c7cb5cb7..f1d0c37a 100644 --- a/tests/integration/test_repository.py +++ b/tests/integration/test_repository.py @@ -1,6 +1,6 @@ # pylint: disable=protected-access -import model -import repository +from domain import model +from adapters import repository def test_repository_can_save_a_batch(session): batch = model.Batch("batch1", "RUSTY-SOAPDISH", 100, eta=None) diff --git a/tests/unit/test_allocate.py b/tests/unit/test_allocate.py index b9df7ff7..e2307072 100644 --- a/tests/unit/test_allocate.py +++ b/tests/unit/test_allocate.py @@ -1,6 +1,6 @@ from datetime import date, timedelta import pytest -from model import allocate, OrderLine, Batch, OutOfStock +from domain.model import allocate, OrderLine, Batch, OutOfStock today = date.today() tomorrow = today + timedelta(days=1) diff --git a/tests/unit/test_batches.py b/tests/unit/test_batches.py index 32100b2c..dd2f98a5 100644 --- a/tests/unit/test_batches.py +++ b/tests/unit/test_batches.py @@ -1,5 +1,5 @@ from datetime import date -from model import Batch, OrderLine +from domain.model import Batch, OrderLine def test_allocating_to_a_batch_reduces_the_available_quantity(): diff --git a/tests/unit/test_services.py b/tests/unit/test_services.py index 921ba107..9ab196f2 100644 --- a/tests/unit/test_services.py +++ b/tests/unit/test_services.py @@ -1,7 +1,7 @@ import pytest -import model -import repository -import services +from domain import model +from adapters import repository +from service_layer import services class FakeRepository(repository.AbstractRepository): From 8766fe93c6345ea812410c5c6ba9974f44f1f04c Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 23 Apr 2019 12:11:47 +0100 Subject: [PATCH 049/165] rewrite service layer to take primitives [service_takes_primitives] --- service_layer/services.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/service_layer/services.py b/service_layer/services.py index 254f607f..bdbf5cfb 100644 --- a/service_layer/services.py +++ b/service_layer/services.py @@ -11,7 +11,10 @@ class InvalidSku(Exception): def is_valid_sku(sku, batches): return sku in {b.sku for b in batches} -def allocate(line: OrderLine, repo: AbstractRepository, session) -> str: +def allocate( + orderid: str, sku: str, qty: int, repo: AbstractRepository, session +) -> str: + line = OrderLine(orderid, sku, qty) batches = repo.list() if not is_valid_sku(line.sku, batches): raise InvalidSku(f'Invalid sku {line.sku}') From 520e675e545cc40e9e5cea3773abc872bbbacd8d Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 23 Apr 2019 12:12:16 +0100 Subject: [PATCH 050/165] services tests partially converted to primitives [tests_call_with_primitives] --- tests/unit/test_services.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/tests/unit/test_services.py b/tests/unit/test_services.py index 9ab196f2..64c7a328 100644 --- a/tests/unit/test_services.py +++ b/tests/unit/test_services.py @@ -27,28 +27,25 @@ def commit(self): def test_returns_allocation(): - line = model.OrderLine("o1", "COMPLICATED-LAMP", 10) - batch = model.Batch("b1", "COMPLICATED-LAMP", 100, eta=None) + batch = model.Batch("batch1", "COMPLICATED-LAMP", 100, eta=None) repo = FakeRepository([batch]) - result = services.allocate(line, repo, FakeSession()) - assert result == "b1" + result = services.allocate("o1", "COMPLICATED-LAMP", 10, repo, FakeSession()) + assert result == "batch1" def test_error_for_invalid_sku(): - line = model.OrderLine("o1", "NONEXISTENTSKU", 10) batch = model.Batch("b1", "AREALSKU", 100, eta=None) repo = FakeRepository([batch]) with pytest.raises(services.InvalidSku, match="Invalid sku NONEXISTENTSKU"): - services.allocate(line, repo, FakeSession()) + services.allocate("o1", "NONEXISTENTSKU", 10, repo, FakeSession()) def test_commits(): - line = model.OrderLine('o1', 'OMINOUS-MIRROR', 10) - batch = model.Batch('b1', 'OMINOUS-MIRROR', 100, eta=None) + batch = model.Batch("b1", "OMINOUS-MIRROR", 100, eta=None) repo = FakeRepository([batch]) session = FakeSession() - services.allocate(line, repo, session) + services.allocate("o1", "OMINOUS-MIRROR", 10, repo, session) assert session.committed is True From c66d7460f4d6f3dd47823195eb0f9220f88d9d2f Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 23 Apr 2019 12:59:52 +0100 Subject: [PATCH 051/165] fixture function for batches [services_factory_function] --- tests/unit/test_services.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/unit/test_services.py b/tests/unit/test_services.py index 64c7a328..6048e96f 100644 --- a/tests/unit/test_services.py +++ b/tests/unit/test_services.py @@ -6,6 +6,12 @@ class FakeRepository(repository.AbstractRepository): + @staticmethod + def for_batch(ref, sku, qty, eta=None): + return FakeRepository([ + model.Batch(ref, sku, qty, eta), + ]) + def __init__(self, batches): self._batches = set(batches) @@ -27,25 +33,19 @@ def commit(self): def test_returns_allocation(): - batch = model.Batch("batch1", "COMPLICATED-LAMP", 100, eta=None) - repo = FakeRepository([batch]) - + repo = FakeRepository.for_batch("batch1", "COMPLICATED-LAMP", 100, eta=None) result = services.allocate("o1", "COMPLICATED-LAMP", 10, repo, FakeSession()) assert result == "batch1" def test_error_for_invalid_sku(): - batch = model.Batch("b1", "AREALSKU", 100, eta=None) - repo = FakeRepository([batch]) - + repo = FakeRepository.for_batch('b1', 'AREALSKU', 100, eta=None) with pytest.raises(services.InvalidSku, match="Invalid sku NONEXISTENTSKU"): services.allocate("o1", "NONEXISTENTSKU", 10, repo, FakeSession()) def test_commits(): - batch = model.Batch("b1", "OMINOUS-MIRROR", 100, eta=None) - repo = FakeRepository([batch]) + repo = FakeRepository.for_batch("b1", "OMINOUS-MIRROR", 100, eta=None) session = FakeSession() - services.allocate("o1", "OMINOUS-MIRROR", 10, repo, session) assert session.committed is True From 5e999fde19eba694b9079999fa12064c844f5336 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 23 Apr 2019 12:37:17 +0100 Subject: [PATCH 052/165] new service to add a batch [add_batch_service] --- service_layer/services.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/service_layer/services.py b/service_layer/services.py index bdbf5cfb..2922a6a5 100644 --- a/service_layer/services.py +++ b/service_layer/services.py @@ -1,4 +1,6 @@ from __future__ import annotations +from typing import Optional +from datetime import date from domain import model from domain.model import OrderLine @@ -11,6 +13,15 @@ class InvalidSku(Exception): def is_valid_sku(sku, batches): return sku in {b.sku for b in batches} + +def add_batch( + ref: str, sku: str, qty: int, eta: Optional[date], + repo: AbstractRepository, session, +): + repo.add(model.Batch(ref, sku, qty, eta)) + session.commit() + + def allocate( orderid: str, sku: str, qty: int, repo: AbstractRepository, session ) -> str: From 16c8dd0a12e954778da4b7bf72ab58816058ef95 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 23 Apr 2019 12:37:39 +0100 Subject: [PATCH 053/165] service-layer test for add batch [test_add_batch] --- tests/unit/test_services.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/unit/test_services.py b/tests/unit/test_services.py index 6048e96f..dc1c0cdd 100644 --- a/tests/unit/test_services.py +++ b/tests/unit/test_services.py @@ -32,6 +32,13 @@ def commit(self): self.committed = True +def test_add_batch(): + repo, session = FakeRepository([]), FakeSession() + services.add_batch('b1', 'CRUNCHY-ARMCHAIR', 100, None, repo, session) + assert repo.get('b1') is not None + assert session.committed + + def test_returns_allocation(): repo = FakeRepository.for_batch("batch1", "COMPLICATED-LAMP", 100, eta=None) result = services.allocate("o1", "COMPLICATED-LAMP", 10, repo, FakeSession()) From 39fbe8a132cd277c7929b83171aa2a9d2e65f2f1 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 23 Apr 2019 12:37:57 +0100 Subject: [PATCH 054/165] all service-layer tests now services [services_tests_all_services] --- tests/unit/test_services.py | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/tests/unit/test_services.py b/tests/unit/test_services.py index dc1c0cdd..1968e2d5 100644 --- a/tests/unit/test_services.py +++ b/tests/unit/test_services.py @@ -6,12 +6,6 @@ class FakeRepository(repository.AbstractRepository): - @staticmethod - def for_batch(ref, sku, qty, eta=None): - return FakeRepository([ - model.Batch(ref, sku, qty, eta), - ]) - def __init__(self, batches): self._batches = set(batches) @@ -34,25 +28,29 @@ def commit(self): def test_add_batch(): repo, session = FakeRepository([]), FakeSession() - services.add_batch('b1', 'CRUNCHY-ARMCHAIR', 100, None, repo, session) - assert repo.get('b1') is not None + services.add_batch("b1", "CRUNCHY-ARMCHAIR", 100, None, repo, session) + assert repo.get("b1") is not None assert session.committed -def test_returns_allocation(): - repo = FakeRepository.for_batch("batch1", "COMPLICATED-LAMP", 100, eta=None) - result = services.allocate("o1", "COMPLICATED-LAMP", 10, repo, FakeSession()) +def test_allocate_returns_allocation(): + repo, session = FakeRepository([]), FakeSession() + services.add_batch("batch1", "COMPLICATED-LAMP", 100, None, repo, session) + result = services.allocate("o1", "COMPLICATED-LAMP", 10, repo, session) assert result == "batch1" -def test_error_for_invalid_sku(): - repo = FakeRepository.for_batch('b1', 'AREALSKU', 100, eta=None) +def test_allocate_errors_for_invalid_sku(): + repo, session = FakeRepository([]), FakeSession() + services.add_batch("b1", "AREALSKU", 100, None, repo, session) + with pytest.raises(services.InvalidSku, match="Invalid sku NONEXISTENTSKU"): services.allocate("o1", "NONEXISTENTSKU", 10, repo, FakeSession()) def test_commits(): - repo = FakeRepository.for_batch("b1", "OMINOUS-MIRROR", 100, eta=None) + repo, session = FakeRepository([]), FakeSession() session = FakeSession() + services.add_batch("b1", "OMINOUS-MIRROR", 100, None, repo, session) services.allocate("o1", "OMINOUS-MIRROR", 10, repo, session) assert session.committed is True From 9624503f521b34ff1dc17f40ba198f015c4474b0 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 23 Apr 2019 12:44:09 +0100 Subject: [PATCH 055/165] modify flask app to use new service layer api [api_uses_modified_service] --- entrypoints/flask_app.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/entrypoints/flask_app.py b/entrypoints/flask_app.py index 6d66d71f..a0520d44 100644 --- a/entrypoints/flask_app.py +++ b/entrypoints/flask_app.py @@ -16,13 +16,13 @@ def allocate_endpoint(): session = get_session() repo = repository.SqlAlchemyRepository(session) - line = model.OrderLine( - request.json['orderid'], - request.json['sku'], - request.json['qty'], - ) try: - batchref = services.allocate(line, repo, session) + batchref = services.allocate( + request.json['orderid'], + request.json['sku'], + request.json['qty'], + repo, session + ) except (model.OutOfStock, services.InvalidSku) as e: return jsonify({'message': str(e)}), 400 From b310cfc15c79cf1fd6851220b2684531f6603290 Mon Sep 17 00:00:00 2001 From: Harry Date: Wed, 24 Apr 2019 12:21:59 +0100 Subject: [PATCH 056/165] add api endpoint for add_batch [api_for_add_batch] --- entrypoints/flask_app.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/entrypoints/flask_app.py b/entrypoints/flask_app.py index a0520d44..c9b6a446 100644 --- a/entrypoints/flask_app.py +++ b/entrypoints/flask_app.py @@ -1,3 +1,4 @@ +from datetime import datetime from flask import Flask, jsonify, request from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker @@ -7,11 +8,25 @@ from adapters import orm, repository from service_layer import services - orm.start_mappers() get_session = sessionmaker(bind=create_engine(config.get_postgres_uri())) app = Flask(__name__) + +@app.route("/add_batch", methods=['POST']) +def add_batch(): + session = get_session() + repo = repository.SqlAlchemyRepository(session) + eta = request.json['eta'] + if eta is not None: + eta = datetime.fromisoformat(eta).date() + services.add_batch( + request.json['ref'], request.json['sku'], request.json['qty'], eta, + repo, session + ) + return 'OK', 201 + + @app.route("/allocate", methods=['POST']) def allocate_endpoint(): session = get_session() From 9a4d80f29e86560f3b7056fc0fff80f1e1ff8af7 Mon Sep 17 00:00:00 2001 From: Harry Date: Wed, 24 Apr 2019 12:22:34 +0100 Subject: [PATCH 057/165] api tests no longer need hardcoded sql fixture [chapter_05_high_gear_low_gear_ends] --- tests/conftest.py | 39 --------------------------------------- tests/e2e/test_api.py | 21 +++++++++++++++------ 2 files changed, 15 insertions(+), 45 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 127bb20c..400e2d68 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -64,45 +64,6 @@ def postgres_session(postgres_db): clear_mappers() -@pytest.fixture -def add_stock(postgres_session): - batches_added = set() - skus_added = set() - - def _add_stock(lines): - for ref, sku, qty, eta in lines: - postgres_session.execute( - 'INSERT INTO batches (reference, sku, _purchased_quantity, eta)' - ' VALUES (:ref, :sku, :qty, :eta)', - dict(ref=ref, sku=sku, qty=qty, eta=eta), - ) - [[batch_id]] = postgres_session.execute( - 'SELECT id FROM batches WHERE reference=:ref AND sku=:sku', - dict(ref=ref, sku=sku), - ) - batches_added.add(batch_id) - skus_added.add(sku) - postgres_session.commit() - - yield _add_stock - - for batch_id in batches_added: - postgres_session.execute( - 'DELETE FROM allocations WHERE batch_id=:batch_id', - dict(batch_id=batch_id), - ) - postgres_session.execute( - 'DELETE FROM batches WHERE id=:batch_id', - dict(batch_id=batch_id), - ) - for sku in skus_added: - postgres_session.execute( - 'DELETE FROM order_lines WHERE sku=:sku', - dict(sku=sku), - ) - postgres_session.commit() - - @pytest.fixture def restart_api(): (Path(__file__).parent / '../entrypoints/flask_app.py').touch() diff --git a/tests/e2e/test_api.py b/tests/e2e/test_api.py index 8bb36297..8b07e246 100644 --- a/tests/e2e/test_api.py +++ b/tests/e2e/test_api.py @@ -17,17 +17,25 @@ def random_orderid(name=''): return f'order-{name}-{random_suffix()}' +def post_to_add_batch(ref, sku, qty, eta): + url = config.get_api_url() + r = requests.post( + f'{url}/add_batch', + json={'ref': ref, 'sku': sku, 'qty': qty, 'eta': eta} + ) + assert r.status_code == 201 + + +@pytest.mark.usefixtures('postgres_db') @pytest.mark.usefixtures('restart_api') -def test_happy_path_returns_201_and_allocated_batch(add_stock): +def test_happy_path_returns_201_and_allocated_batch(): sku, othersku = random_sku(), random_sku('other') earlybatch = random_batchref(1) laterbatch = random_batchref(2) otherbatch = random_batchref(3) - add_stock([ - (laterbatch, sku, 100, '2011-01-02'), - (earlybatch, sku, 100, '2011-01-01'), - (otherbatch, othersku, 100, None), - ]) + post_to_add_batch(laterbatch, sku, 100, '2011-01-02') + post_to_add_batch(earlybatch, sku, 100, '2011-01-01') + post_to_add_batch(otherbatch, othersku, 100, None) data = {'orderid': random_orderid(), 'sku': sku, 'qty': 3} url = config.get_api_url() r = requests.post(f'{url}/allocate', json=data) @@ -35,6 +43,7 @@ def test_happy_path_returns_201_and_allocated_batch(add_stock): assert r.json()['batchref'] == earlybatch +@pytest.mark.usefixtures('postgres_db') @pytest.mark.usefixtures('restart_api') def test_unhappy_path_returns_400_and_error_message(): unknown_sku, orderid = random_sku(), random_orderid() From ffd071cf7a0534eff8b8d2669b72fc5bb4feabe4 Mon Sep 17 00:00:00 2001 From: Harry Date: Wed, 27 Feb 2019 06:13:15 +0000 Subject: [PATCH 058/165] start moving files into src folder and add setup.py --- Dockerfile | 13 ++++++++----- {adapters => src/allocation}/__init__.py | 0 {domain => src/allocation/adapters}/__init__.py | 0 {adapters => src/allocation/adapters}/orm.py | 0 {adapters => src/allocation/adapters}/repository.py | 0 config.py => src/allocation/config.py | 0 {entrypoints => src/allocation/domain}/__init__.py | 0 {domain => src/allocation/domain}/model.py | 0 .../allocation/entrypoints}/__init__.py | 0 .../allocation/entrypoints}/flask_app.py | 0 src/allocation/service_layer/__init__.py | 0 .../allocation/service_layer}/services.py | 0 src/setup.py | 7 +++++++ 13 files changed, 15 insertions(+), 5 deletions(-) rename {adapters => src/allocation}/__init__.py (100%) rename {domain => src/allocation/adapters}/__init__.py (100%) rename {adapters => src/allocation/adapters}/orm.py (100%) rename {adapters => src/allocation/adapters}/repository.py (100%) rename config.py => src/allocation/config.py (100%) rename {entrypoints => src/allocation/domain}/__init__.py (100%) rename {domain => src/allocation/domain}/model.py (100%) rename {service_layer => src/allocation/entrypoints}/__init__.py (100%) rename {entrypoints => src/allocation/entrypoints}/flask_app.py (100%) create mode 100644 src/allocation/service_layer/__init__.py rename {service_layer => src/allocation/service_layer}/services.py (100%) create mode 100644 src/setup.py diff --git a/Dockerfile b/Dockerfile index 8c7c140f..04d923f2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,13 +3,16 @@ FROM python:3.8-alpine RUN apk add --no-cache --virtual .build-deps gcc postgresql-dev musl-dev python3-dev RUN apk add libpq -COPY requirements.txt /tmp +COPY requirements.txt /tmp/ RUN pip install -r /tmp/requirements.txt RUN apk del --no-cache .build-deps -RUN mkdir -p /code -COPY *.py /code/ -WORKDIR /code -ENV FLASK_APP=entrypoints/flask_app.py FLASK_DEBUG=1 PYTHONUNBUFFERED=1 +RUN mkdir -p /src +COPY src/ /src/ +RUN pip install -e /src +COPY tests/ /tests/ + +WORKDIR /src +ENV FLASK_APP=allocation/entrypoints/flask_app.py FLASK_DEBUG=1 PYTHONUNBUFFERED=1 CMD flask run --host=0.0.0.0 --port=80 diff --git a/adapters/__init__.py b/src/allocation/__init__.py similarity index 100% rename from adapters/__init__.py rename to src/allocation/__init__.py diff --git a/domain/__init__.py b/src/allocation/adapters/__init__.py similarity index 100% rename from domain/__init__.py rename to src/allocation/adapters/__init__.py diff --git a/adapters/orm.py b/src/allocation/adapters/orm.py similarity index 100% rename from adapters/orm.py rename to src/allocation/adapters/orm.py diff --git a/adapters/repository.py b/src/allocation/adapters/repository.py similarity index 100% rename from adapters/repository.py rename to src/allocation/adapters/repository.py diff --git a/config.py b/src/allocation/config.py similarity index 100% rename from config.py rename to src/allocation/config.py diff --git a/entrypoints/__init__.py b/src/allocation/domain/__init__.py similarity index 100% rename from entrypoints/__init__.py rename to src/allocation/domain/__init__.py diff --git a/domain/model.py b/src/allocation/domain/model.py similarity index 100% rename from domain/model.py rename to src/allocation/domain/model.py diff --git a/service_layer/__init__.py b/src/allocation/entrypoints/__init__.py similarity index 100% rename from service_layer/__init__.py rename to src/allocation/entrypoints/__init__.py diff --git a/entrypoints/flask_app.py b/src/allocation/entrypoints/flask_app.py similarity index 100% rename from entrypoints/flask_app.py rename to src/allocation/entrypoints/flask_app.py diff --git a/src/allocation/service_layer/__init__.py b/src/allocation/service_layer/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/service_layer/services.py b/src/allocation/service_layer/services.py similarity index 100% rename from service_layer/services.py rename to src/allocation/service_layer/services.py diff --git a/src/setup.py b/src/setup.py new file mode 100644 index 00000000..be04950f --- /dev/null +++ b/src/setup.py @@ -0,0 +1,7 @@ +from setuptools import setup + +setup( + name='allocation', + version='0.1', + packages=['allocation'], +) From 800ae376e72548ce7118bbff02a628ad14587820 Mon Sep 17 00:00:00 2001 From: Harry Date: Wed, 27 Feb 2019 06:29:22 +0000 Subject: [PATCH 059/165] fix all the imports, get it all working --- docker-compose.yml | 2 +- mypy.ini | 7 ++----- src/allocation/adapters/orm.py | 2 +- src/allocation/adapters/repository.py | 2 +- src/allocation/entrypoints/flask_app.py | 8 ++++---- src/allocation/service_layer/services.py | 6 +++--- tests/conftest.py | 7 +++---- tests/e2e/test_api.py | 2 +- tests/integration/test_orm.py | 2 +- tests/integration/test_repository.py | 4 ++-- tests/mypy.ini | 6 ++++++ tests/unit/test_allocate.py | 2 +- tests/unit/test_batches.py | 2 +- tests/unit/test_services.py | 6 +++--- 14 files changed, 30 insertions(+), 28 deletions(-) create mode 100644 tests/mypy.ini diff --git a/docker-compose.yml b/docker-compose.yml index 313cf946..cc8341f9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,7 +11,7 @@ services: - DB_HOST=postgres - DB_PASSWORD=abc123 volumes: - - ./:/code + - ./src:/src ports: - "5005:80" diff --git a/mypy.ini b/mypy.ini index ead5ef09..65ab939b 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,9 +1,6 @@ [mypy] ignore_missing_imports = False +mypy_path = ./src -[mypy-pytest.*] +[mypy-pytest.*,sqlalchemy.*] ignore_missing_imports = True - -[mypy-sqlalchemy.*] -ignore_missing_imports = True - diff --git a/src/allocation/adapters/orm.py b/src/allocation/adapters/orm.py index 571240df..cf245b2f 100644 --- a/src/allocation/adapters/orm.py +++ b/src/allocation/adapters/orm.py @@ -4,7 +4,7 @@ ) from sqlalchemy.orm import mapper, relationship -from domain import model +from allocation.domain import model metadata = MetaData() diff --git a/src/allocation/adapters/repository.py b/src/allocation/adapters/repository.py index e2aab99d..3c3128d3 100644 --- a/src/allocation/adapters/repository.py +++ b/src/allocation/adapters/repository.py @@ -1,5 +1,5 @@ import abc -from domain import model +from allocation.domain import model class AbstractRepository(abc.ABC): diff --git a/src/allocation/entrypoints/flask_app.py b/src/allocation/entrypoints/flask_app.py index c9b6a446..415bf159 100644 --- a/src/allocation/entrypoints/flask_app.py +++ b/src/allocation/entrypoints/flask_app.py @@ -3,10 +3,10 @@ from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker -import config -from domain import model -from adapters import orm, repository -from service_layer import services +from allocation import config +from allocation.domain import model +from allocation.adapters import orm, repository +from allocation.service_layer import services orm.start_mappers() get_session = sessionmaker(bind=create_engine(config.get_postgres_uri())) diff --git a/src/allocation/service_layer/services.py b/src/allocation/service_layer/services.py index 2922a6a5..9529f192 100644 --- a/src/allocation/service_layer/services.py +++ b/src/allocation/service_layer/services.py @@ -2,9 +2,9 @@ from typing import Optional from datetime import date -from domain import model -from domain.model import OrderLine -from adapters.repository import AbstractRepository +from allocation.domain import model +from allocation.domain.model import OrderLine +from allocation.adapters.repository import AbstractRepository class InvalidSku(Exception): pass diff --git a/tests/conftest.py b/tests/conftest.py index 400e2d68..622ec390 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,8 +9,8 @@ from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker, clear_mappers -from adapters.orm import metadata, start_mappers -import config +from allocation.adapters.orm import metadata, start_mappers +from allocation import config @pytest.fixture @@ -66,7 +66,6 @@ def postgres_session(postgres_db): @pytest.fixture def restart_api(): - (Path(__file__).parent / '../entrypoints/flask_app.py').touch() + (Path(__file__).parent / '../src/allocation/entrypoints/flask_app.py').touch() time.sleep(0.5) wait_for_webapp_to_come_up() - diff --git a/tests/e2e/test_api.py b/tests/e2e/test_api.py index 8b07e246..3d117ef6 100644 --- a/tests/e2e/test_api.py +++ b/tests/e2e/test_api.py @@ -2,7 +2,7 @@ import pytest import requests -import config +from allocation import config def random_suffix(): return uuid.uuid4().hex[:6] diff --git a/tests/integration/test_orm.py b/tests/integration/test_orm.py index 3426d460..83aa5520 100644 --- a/tests/integration/test_orm.py +++ b/tests/integration/test_orm.py @@ -1,4 +1,4 @@ -from domain import model +from allocation.domain import model from datetime import date def test_orderline_mapper_can_load_lines(session): diff --git a/tests/integration/test_repository.py b/tests/integration/test_repository.py index f1d0c37a..29dddc77 100644 --- a/tests/integration/test_repository.py +++ b/tests/integration/test_repository.py @@ -1,6 +1,6 @@ # pylint: disable=protected-access -from domain import model -from adapters import repository +from allocation.domain import model +from allocation.adapters import repository def test_repository_can_save_a_batch(session): batch = model.Batch("batch1", "RUSTY-SOAPDISH", 100, eta=None) diff --git a/tests/mypy.ini b/tests/mypy.ini new file mode 100644 index 00000000..39fadd90 --- /dev/null +++ b/tests/mypy.ini @@ -0,0 +1,6 @@ +[mypy] +ignore_missing_imports = False +mypy_path = ../src + +[mypy-pytest.*,sqlalchemy.*] +ignore_missing_imports = True diff --git a/tests/unit/test_allocate.py b/tests/unit/test_allocate.py index e2307072..88fa99c7 100644 --- a/tests/unit/test_allocate.py +++ b/tests/unit/test_allocate.py @@ -1,6 +1,6 @@ from datetime import date, timedelta import pytest -from domain.model import allocate, OrderLine, Batch, OutOfStock +from allocation.domain.model import allocate, OrderLine, Batch, OutOfStock today = date.today() tomorrow = today + timedelta(days=1) diff --git a/tests/unit/test_batches.py b/tests/unit/test_batches.py index dd2f98a5..099d37b4 100644 --- a/tests/unit/test_batches.py +++ b/tests/unit/test_batches.py @@ -1,5 +1,5 @@ from datetime import date -from domain.model import Batch, OrderLine +from allocation.domain.model import Batch, OrderLine def test_allocating_to_a_batch_reduces_the_available_quantity(): diff --git a/tests/unit/test_services.py b/tests/unit/test_services.py index 1968e2d5..0e4fc152 100644 --- a/tests/unit/test_services.py +++ b/tests/unit/test_services.py @@ -1,7 +1,7 @@ import pytest -from domain import model -from adapters import repository -from service_layer import services +from allocation.domain import model +from allocation.adapters import repository +from allocation.service_layer import services class FakeRepository(repository.AbstractRepository): From 587bb8d0c46ec72c2889031a66353a004665ad56 Mon Sep 17 00:00:00 2001 From: Harry Date: Wed, 27 Feb 2019 06:44:34 +0000 Subject: [PATCH 060/165] get tests working in docker container --- .travis.yml | 3 --- Makefile | 15 ++++++++++++--- docker-compose.yml | 3 +++ tests/__init__.py | 0 tests/mypy.ini | 6 ------ tests/pytest.ini | 2 ++ 6 files changed, 17 insertions(+), 12 deletions(-) delete mode 100644 tests/__init__.py delete mode 100644 tests/mypy.ini create mode 100644 tests/pytest.ini diff --git a/.travis.yml b/.travis.yml index f601680f..fcd3ceea 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,9 +2,6 @@ dist: xenial language: python python: 3.8 -install: -- pip3 install -r requirements.txt - script: - make all diff --git a/Makefile b/Makefile index f62eef70..4fb33031 100644 --- a/Makefile +++ b/Makefile @@ -4,13 +4,22 @@ build: up: docker-compose up -d app -test: - pytest --tb=short +test: up + docker-compose run --rm --no-deps --entrypoint=pytest app /tests/unit /tests/integration /tests/e2e + +unit-tests: + docker-compose run --rm --no-deps --entrypoint=pytest app /tests/unit + +integration-tests: up + docker-compose run --rm --no-deps --entrypoint=pytest app /tests/integration + +e2e-tests: up + docker-compose run --rm --no-deps --entrypoint=pytest app /tests/e2e logs: docker-compose logs app | tail -100 down: - docker-compose down + docker-compose down --remove-orphans all: down build up test diff --git a/docker-compose.yml b/docker-compose.yml index cc8341f9..039400e9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,8 +10,11 @@ services: environment: - DB_HOST=postgres - DB_PASSWORD=abc123 + - API_HOST=app + - PYTHONDONTWRITEBYTECODE=1 volumes: - ./src:/src + - ./tests:/tests ports: - "5005:80" diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/mypy.ini b/tests/mypy.ini deleted file mode 100644 index 39fadd90..00000000 --- a/tests/mypy.ini +++ /dev/null @@ -1,6 +0,0 @@ -[mypy] -ignore_missing_imports = False -mypy_path = ../src - -[mypy-pytest.*,sqlalchemy.*] -ignore_missing_imports = True diff --git a/tests/pytest.ini b/tests/pytest.ini new file mode 100644 index 00000000..bbd083ac --- /dev/null +++ b/tests/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +addopts = --tb=short From b8ce109e85c9dc9abee74df0b297c73c5cabf372 Mon Sep 17 00:00:00 2001 From: Harry Date: Fri, 24 May 2019 00:10:48 +0100 Subject: [PATCH 061/165] make mypy slightly stricter --- mypy.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/mypy.ini b/mypy.ini index 65ab939b..62194f35 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,6 +1,7 @@ [mypy] ignore_missing_imports = False mypy_path = ./src +check_untyped_defs = True [mypy-pytest.*,sqlalchemy.*] ignore_missing_imports = True From b90b24867408b5e3c524dc8aaa7367a812b67806 Mon Sep 17 00:00:00 2001 From: Harry Date: Mon, 11 Mar 2019 23:08:10 +0000 Subject: [PATCH 062/165] better requirements.txt [appendix_project_structure_ends] --- requirements.txt | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 2ff54bf8..a8906795 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,11 @@ -pytest +# app sqlalchemy flask -psycopg2 +psycopg2-binary + +# tests +pytest +pytest-icdiff +mypy requests + From de6a9e31aaeb32fe55b4602fc0c87011e4d21814 Mon Sep 17 00:00:00 2001 From: Harry Date: Mon, 11 Mar 2019 23:11:44 +0000 Subject: [PATCH 063/165] basic uow test, uow and conftest.py changes --- src/allocation/unit_of_work.py | 53 ++++++++++++++++++++++++++++++++++ tests/conftest.py | 9 ++++-- tests/integration/test_uow.py | 38 ++++++++++++++++++++++++ 3 files changed, 97 insertions(+), 3 deletions(-) create mode 100644 src/allocation/unit_of_work.py create mode 100644 tests/integration/test_uow.py diff --git a/src/allocation/unit_of_work.py b/src/allocation/unit_of_work.py new file mode 100644 index 00000000..39006f0d --- /dev/null +++ b/src/allocation/unit_of_work.py @@ -0,0 +1,53 @@ +# pylint: disable=attribute-defined-outside-init +from __future__ import annotations +import abc +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker +from sqlalchemy.orm.session import Session + +from allocation import config +from allocation import repository + + +class AbstractUnitOfWork(abc.ABC): + batches: repository.AbstractRepository + + def __enter__(self) -> AbstractUnitOfWork: + return self + + def __exit__(self, *args): + self.rollback() + + @abc.abstractmethod + def commit(self): + raise NotImplementedError + + @abc.abstractmethod + def rollback(self): + raise NotImplementedError + + + +DEFAULT_SESSION_FACTORY = sessionmaker(bind=create_engine( + config.get_postgres_uri(), +)) + +class SqlAlchemyUnitOfWork(AbstractUnitOfWork): + + def __init__(self, session_factory=DEFAULT_SESSION_FACTORY): + self.session_factory = session_factory + + def __enter__(self): + self.session = self.session_factory() # type: Session + self.batches = repository.SqlAlchemyRepository(self.session) + return super().__enter__() + + def __exit__(self, *args): + super().__exit__(*args) + self.session.close() + + def commit(self): + self.session.commit() + + def rollback(self): + self.session.rollback() diff --git a/tests/conftest.py b/tests/conftest.py index 622ec390..cfe2611b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -19,13 +19,16 @@ def in_memory_db(): metadata.create_all(engine) return engine - @pytest.fixture -def session(in_memory_db): +def session_factory(in_memory_db): start_mappers() - yield sessionmaker(bind=in_memory_db)() + yield sessionmaker(bind=in_memory_db) clear_mappers() +@pytest.fixture +def session(session_factory): + return session_factory() + def wait_for_postgres_to_come_up(engine): deadline = time.time() + 10 diff --git a/tests/integration/test_uow.py b/tests/integration/test_uow.py new file mode 100644 index 00000000..6f0de82b --- /dev/null +++ b/tests/integration/test_uow.py @@ -0,0 +1,38 @@ +import pytest +from allocation import model +from allocation import unit_of_work + +def insert_batch(session, ref, sku, qty, eta): + session.execute( + 'INSERT INTO batches (reference, sku, _purchased_quantity, eta)' + ' VALUES (:ref, :sku, :qty, :eta)', + dict(ref=ref, sku=sku, qty=qty, eta=eta) + ) + +def get_allocated_batch_ref(session, orderid, sku): + [[orderlineid]] = session.execute( + 'SELECT id FROM order_lines WHERE orderid=:orderid AND sku=:sku', + dict(orderid=orderid, sku=sku) + ) + [[batchref]] = session.execute( + 'SELECT b.reference FROM allocations JOIN batches AS b ON batch_id = b.id' + ' WHERE orderline_id=:orderlineid', + dict(orderlineid=orderlineid) + ) + return batchref + + +def test_uow_can_retrieve_a_batch_and_allocate_to_it(session_factory): + session = session_factory() + insert_batch(session, 'batch1', 'HIPSTER-WORKBENCH', 100, None) + session.commit() + + uow = unit_of_work.SqlAlchemyUnitOfWork(session_factory) + with uow: + batch = uow.batches.get(reference='batch1') + line = model.OrderLine('o1', 'HIPSTER-WORKBENCH', 10) + batch.allocate(line) + uow.commit() + + batchref = get_allocated_batch_ref(session, 'o1', 'HIPSTER-WORKBENCH') + assert batchref == 'batch1' From 56cb62e1e0d4f22ceb7f3c065b6f09117cfcddb6 Mon Sep 17 00:00:00 2001 From: Harry Date: Wed, 20 Mar 2019 10:53:14 +0000 Subject: [PATCH 064/165] use uow in services, flask app --- src/allocation/config.py | 1 + src/allocation/entrypoints/flask_app.py | 18 +++----- src/allocation/service_layer/services.py | 24 +++++----- .../{ => service_layer}/unit_of_work.py | 2 +- tests/integration/test_uow.py | 5 +-- tests/unit/test_services.py | 45 ++++++++++--------- 6 files changed, 48 insertions(+), 47 deletions(-) rename src/allocation/{ => service_layer}/unit_of_work.py (96%) diff --git a/src/allocation/config.py b/src/allocation/config.py index 7b1584bb..38296a14 100644 --- a/src/allocation/config.py +++ b/src/allocation/config.py @@ -1,5 +1,6 @@ import os + def get_postgres_uri(): host = os.environ.get('DB_HOST', 'localhost') port = 54321 if host == 'localhost' else 5432 diff --git a/src/allocation/entrypoints/flask_app.py b/src/allocation/entrypoints/flask_app.py index 415bf159..2b720763 100644 --- a/src/allocation/entrypoints/flask_app.py +++ b/src/allocation/entrypoints/flask_app.py @@ -1,42 +1,34 @@ from datetime import datetime from flask import Flask, jsonify, request -from sqlalchemy import create_engine -from sqlalchemy.orm import sessionmaker -from allocation import config from allocation.domain import model -from allocation.adapters import orm, repository -from allocation.service_layer import services +from allocation.adapters import orm +from allocation.service_layer import services, unit_of_work -orm.start_mappers() -get_session = sessionmaker(bind=create_engine(config.get_postgres_uri())) app = Flask(__name__) +orm.start_mappers() @app.route("/add_batch", methods=['POST']) def add_batch(): - session = get_session() - repo = repository.SqlAlchemyRepository(session) eta = request.json['eta'] if eta is not None: eta = datetime.fromisoformat(eta).date() services.add_batch( request.json['ref'], request.json['sku'], request.json['qty'], eta, - repo, session + unit_of_work.SqlAlchemyUnitOfWork(), ) return 'OK', 201 @app.route("/allocate", methods=['POST']) def allocate_endpoint(): - session = get_session() - repo = repository.SqlAlchemyRepository(session) try: batchref = services.allocate( request.json['orderid'], request.json['sku'], request.json['qty'], - repo, session + unit_of_work.SqlAlchemyUnitOfWork(), ) except (model.OutOfStock, services.InvalidSku) as e: return jsonify({'message': str(e)}), 400 diff --git a/src/allocation/service_layer/services.py b/src/allocation/service_layer/services.py index 9529f192..b94af210 100644 --- a/src/allocation/service_layer/services.py +++ b/src/allocation/service_layer/services.py @@ -4,7 +4,8 @@ from allocation.domain import model from allocation.domain.model import OrderLine -from allocation.adapters.repository import AbstractRepository +from allocation.service_layer import unit_of_work + class InvalidSku(Exception): pass @@ -16,19 +17,22 @@ def is_valid_sku(sku, batches): def add_batch( ref: str, sku: str, qty: int, eta: Optional[date], - repo: AbstractRepository, session, + uow: unit_of_work.AbstractUnitOfWork ): - repo.add(model.Batch(ref, sku, qty, eta)) - session.commit() + with uow: + uow.batches.add(model.Batch(ref, sku, qty, eta)) + uow.commit() def allocate( - orderid: str, sku: str, qty: int, repo: AbstractRepository, session + orderid: str, sku: str, qty: int, + uow: unit_of_work.AbstractUnitOfWork ) -> str: line = OrderLine(orderid, sku, qty) - batches = repo.list() - if not is_valid_sku(line.sku, batches): - raise InvalidSku(f'Invalid sku {line.sku}') - batchref = model.allocate(line, batches) - session.commit() + with uow: + batches = uow.batches.list() + if not is_valid_sku(line.sku, batches): + raise InvalidSku(f'Invalid sku {line.sku}') + batchref = model.allocate(line, batches) + uow.commit() return batchref diff --git a/src/allocation/unit_of_work.py b/src/allocation/service_layer/unit_of_work.py similarity index 96% rename from src/allocation/unit_of_work.py rename to src/allocation/service_layer/unit_of_work.py index 39006f0d..a777c344 100644 --- a/src/allocation/unit_of_work.py +++ b/src/allocation/service_layer/unit_of_work.py @@ -6,7 +6,7 @@ from sqlalchemy.orm.session import Session from allocation import config -from allocation import repository +from allocation.adapters import repository class AbstractUnitOfWork(abc.ABC): diff --git a/tests/integration/test_uow.py b/tests/integration/test_uow.py index 6f0de82b..0e05f18d 100644 --- a/tests/integration/test_uow.py +++ b/tests/integration/test_uow.py @@ -1,6 +1,5 @@ -import pytest -from allocation import model -from allocation import unit_of_work +from allocation.domain import model +from allocation.service_layer import unit_of_work def insert_batch(session, ref, sku, qty, eta): session.execute( diff --git a/tests/unit/test_services.py b/tests/unit/test_services.py index 0e4fc152..a9dc9503 100644 --- a/tests/unit/test_services.py +++ b/tests/unit/test_services.py @@ -1,7 +1,6 @@ import pytest -from allocation.domain import model from allocation.adapters import repository -from allocation.service_layer import services +from allocation.service_layer import services, unit_of_work class FakeRepository(repository.AbstractRepository): @@ -19,38 +18,44 @@ def list(self): return list(self._batches) -class FakeSession(): - committed = False +class FakeUnitOfWork(unit_of_work.AbstractUnitOfWork): + + def __init__(self): + self.batches = FakeRepository([]) + self.committed = False def commit(self): self.committed = True + def rollback(self): + pass + + def test_add_batch(): - repo, session = FakeRepository([]), FakeSession() - services.add_batch("b1", "CRUNCHY-ARMCHAIR", 100, None, repo, session) - assert repo.get("b1") is not None - assert session.committed + uow = FakeUnitOfWork() + services.add_batch("b1", "CRUNCHY-ARMCHAIR", 100, None, uow) + assert uow.batches.get("b1") is not None + assert uow.committed def test_allocate_returns_allocation(): - repo, session = FakeRepository([]), FakeSession() - services.add_batch("batch1", "COMPLICATED-LAMP", 100, None, repo, session) - result = services.allocate("o1", "COMPLICATED-LAMP", 10, repo, session) + uow = FakeUnitOfWork() + services.add_batch("batch1", "COMPLICATED-LAMP", 100, None, uow) + result = services.allocate("o1", "COMPLICATED-LAMP", 10, uow) assert result == "batch1" def test_allocate_errors_for_invalid_sku(): - repo, session = FakeRepository([]), FakeSession() - services.add_batch("b1", "AREALSKU", 100, None, repo, session) + uow = FakeUnitOfWork() + services.add_batch("b1", "AREALSKU", 100, None, uow) with pytest.raises(services.InvalidSku, match="Invalid sku NONEXISTENTSKU"): - services.allocate("o1", "NONEXISTENTSKU", 10, repo, FakeSession()) + services.allocate("o1", "NONEXISTENTSKU", 10, uow) -def test_commits(): - repo, session = FakeRepository([]), FakeSession() - session = FakeSession() - services.add_batch("b1", "OMINOUS-MIRROR", 100, None, repo, session) - services.allocate("o1", "OMINOUS-MIRROR", 10, repo, session) - assert session.committed is True +def test_allocate_commits(): + uow = FakeUnitOfWork() + services.add_batch("b1", "OMINOUS-MIRROR", 100, None, uow) + services.allocate("o1", "OMINOUS-MIRROR", 10, uow) + assert uow.committed From 7959ae6df225489eb3a08d6ff440e32727518195 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 12 Mar 2019 11:52:13 +0000 Subject: [PATCH 065/165] two more tests for rollback behaviour [chapter_06_uow_ends] --- tests/integration/test_uow.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/integration/test_uow.py b/tests/integration/test_uow.py index 0e05f18d..5ac1af5a 100644 --- a/tests/integration/test_uow.py +++ b/tests/integration/test_uow.py @@ -1,6 +1,8 @@ +import pytest from allocation.domain import model from allocation.service_layer import unit_of_work + def insert_batch(session, ref, sku, qty, eta): session.execute( 'INSERT INTO batches (reference, sku, _purchased_quantity, eta)' @@ -8,6 +10,7 @@ def insert_batch(session, ref, sku, qty, eta): dict(ref=ref, sku=sku, qty=qty, eta=eta) ) + def get_allocated_batch_ref(session, orderid, sku): [[orderlineid]] = session.execute( 'SELECT id FROM order_lines WHERE orderid=:orderid AND sku=:sku', @@ -35,3 +38,28 @@ def test_uow_can_retrieve_a_batch_and_allocate_to_it(session_factory): batchref = get_allocated_batch_ref(session, 'o1', 'HIPSTER-WORKBENCH') assert batchref == 'batch1' + + +def test_rolls_back_uncommitted_work_by_default(session_factory): + uow = unit_of_work.SqlAlchemyUnitOfWork(session_factory) + with uow: + insert_batch(uow.session, 'batch1', 'MEDIUM-PLINTH', 100, None) + + new_session = session_factory() + rows = list(new_session.execute('SELECT * FROM "batches"')) + assert rows == [] + + +def test_rolls_back_on_error(session_factory): + class MyException(Exception): + pass + + uow = unit_of_work.SqlAlchemyUnitOfWork(session_factory) + with pytest.raises(MyException): + with uow: + insert_batch(uow.session, 'batch1', 'LARGE-FORK', 100, None) + raise MyException() + + new_session = session_factory() + rows = list(new_session.execute('SELECT * FROM "batches"')) + assert rows == [] From c8853411b967cca792d7da7e68b980902e69ffaf Mon Sep 17 00:00:00 2001 From: Harry Date: Sat, 23 Mar 2019 10:26:16 +0000 Subject: [PATCH 066/165] start on a Product model with an allocate fn [product_aggregate] --- src/allocation/adapters/orm.py | 13 ++- src/allocation/adapters/repository.py | 16 ++-- src/allocation/domain/model.py | 24 +++-- src/allocation/service_layer/services.py | 16 ++-- src/allocation/service_layer/unit_of_work.py | 4 +- tests/integration/test_orm.py | 90 ------------------- tests/integration/test_repository.py | 64 ------------- tests/integration/test_uow.py | 8 +- .../{test_allocate.py => test_product.py} | 18 ++-- tests/unit/test_services.py | 28 +++--- 10 files changed, 75 insertions(+), 206 deletions(-) delete mode 100644 tests/integration/test_orm.py delete mode 100644 tests/integration/test_repository.py rename tests/unit/{test_allocate.py => test_product.py} (68%) diff --git a/src/allocation/adapters/orm.py b/src/allocation/adapters/orm.py index cf245b2f..edaa0c55 100644 --- a/src/allocation/adapters/orm.py +++ b/src/allocation/adapters/orm.py @@ -17,11 +17,17 @@ Column('orderid', String(255)), ) +products = Table( + 'products', metadata, + Column('sku', String(255), primary_key=True), + # Column('version_number', Integer, nullable=False, default=0), +) + batches = Table( 'batches', metadata, Column('id', Integer, primary_key=True, autoincrement=True), Column('reference', String(255)), - Column('sku', String(255)), + Column('sku', ForeignKey('products.sku')), Column('_purchased_quantity', Integer, nullable=False), Column('eta', Date, nullable=True), ) @@ -36,10 +42,13 @@ def start_mappers(): lines_mapper = mapper(model.OrderLine, order_lines) - mapper(model.Batch, batches, properties={ + batches_mapper = mapper(model.Batch, batches, properties={ '_allocations': relationship( lines_mapper, secondary=allocations, collection_class=set, ) }) + mapper(model.Product, products, properties={ + 'batches': relationship(batches_mapper) + }) diff --git a/src/allocation/adapters/repository.py b/src/allocation/adapters/repository.py index 3c3128d3..0459d70e 100644 --- a/src/allocation/adapters/repository.py +++ b/src/allocation/adapters/repository.py @@ -1,15 +1,14 @@ import abc from allocation.domain import model - class AbstractRepository(abc.ABC): @abc.abstractmethod - def add(self, batch: model.Batch): + def add(self, product: model.Product): raise NotImplementedError @abc.abstractmethod - def get(self, reference) -> model.Batch: + def get(self, sku) -> model.Product: raise NotImplementedError @@ -19,11 +18,8 @@ class SqlAlchemyRepository(AbstractRepository): def __init__(self, session): self.session = session - def add(self, batch): - self.session.add(batch) - - def get(self, reference): - return self.session.query(model.Batch).filter_by(reference=reference).one() + def add(self, product): + self.session.add(product) - def list(self): - return self.session.query(model.Batch).all() + def get(self, sku): + return self.session.query(model.Product).filter_by(sku=sku).first() diff --git a/src/allocation/domain/model.py b/src/allocation/domain/model.py index bbb3a3c3..4365fe20 100644 --- a/src/allocation/domain/model.py +++ b/src/allocation/domain/model.py @@ -8,15 +8,21 @@ class OutOfStock(Exception): pass -def allocate(line: OrderLine, batches: List[Batch]) -> str: - try: - batch = next( - b for b in sorted(batches) if b.can_allocate(line) - ) - batch.allocate(line) - return batch.reference - except StopIteration: - raise OutOfStock(f'Out of stock for sku {line.sku}') +class Product: + + def __init__(self, sku: str, batches: List[Batch]): + self.sku = sku + self.batches = batches + + def allocate(self, line: OrderLine) -> str: + try: + batch = next( + b for b in sorted(self.batches) if b.can_allocate(line) + ) + batch.allocate(line) + return batch.reference + except StopIteration: + raise OutOfStock(f'Out of stock for sku {line.sku}') @dataclass(unsafe_hash=True) diff --git a/src/allocation/service_layer/services.py b/src/allocation/service_layer/services.py index b94af210..d0e6c60c 100644 --- a/src/allocation/service_layer/services.py +++ b/src/allocation/service_layer/services.py @@ -11,16 +11,16 @@ class InvalidSku(Exception): pass -def is_valid_sku(sku, batches): - return sku in {b.sku for b in batches} - - def add_batch( ref: str, sku: str, qty: int, eta: Optional[date], uow: unit_of_work.AbstractUnitOfWork ): with uow: - uow.batches.add(model.Batch(ref, sku, qty, eta)) + product = uow.products.get(sku=sku) + if product is None: + product = model.Product(sku, batches=[]) + uow.products.add(product) + product.batches.append(model.Batch(ref, sku, qty, eta)) uow.commit() @@ -30,9 +30,9 @@ def allocate( ) -> str: line = OrderLine(orderid, sku, qty) with uow: - batches = uow.batches.list() - if not is_valid_sku(line.sku, batches): + product = uow.products.get(sku=line.sku) + if product is None: raise InvalidSku(f'Invalid sku {line.sku}') - batchref = model.allocate(line, batches) + batchref = product.allocate(line) uow.commit() return batchref diff --git a/src/allocation/service_layer/unit_of_work.py b/src/allocation/service_layer/unit_of_work.py index a777c344..af9d9117 100644 --- a/src/allocation/service_layer/unit_of_work.py +++ b/src/allocation/service_layer/unit_of_work.py @@ -10,7 +10,7 @@ class AbstractUnitOfWork(abc.ABC): - batches: repository.AbstractRepository + products: repository.AbstractRepository def __enter__(self) -> AbstractUnitOfWork: return self @@ -39,7 +39,7 @@ def __init__(self, session_factory=DEFAULT_SESSION_FACTORY): def __enter__(self): self.session = self.session_factory() # type: Session - self.batches = repository.SqlAlchemyRepository(self.session) + self.products = repository.SqlAlchemyRepository(self.session) return super().__enter__() def __exit__(self, *args): diff --git a/tests/integration/test_orm.py b/tests/integration/test_orm.py deleted file mode 100644 index 83aa5520..00000000 --- a/tests/integration/test_orm.py +++ /dev/null @@ -1,90 +0,0 @@ -from allocation.domain import model -from datetime import date - -def test_orderline_mapper_can_load_lines(session): - session.execute( - 'INSERT INTO order_lines (orderid, sku, qty) VALUES ' - '("order1", "RED-CHAIR", 12),' - '("order1", "RED-TABLE", 13),' - '("order2", "BLUE-LIPSTICK", 14)' - ) - expected = [ - model.OrderLine("order1", "RED-CHAIR", 12), - model.OrderLine("order1", "RED-TABLE", 13), - model.OrderLine("order2", "BLUE-LIPSTICK", 14), - ] - assert session.query(model.OrderLine).all() == expected - - -def test_orderline_mapper_can_save_lines(session): - new_line = model.OrderLine("order1", "DECORATIVE-WIDGET", 12) - session.add(new_line) - session.commit() - - rows = list(session.execute('SELECT orderid, sku, qty FROM "order_lines"')) - assert rows == [("order1", "DECORATIVE-WIDGET", 12)] - - - -def test_retrieving_batches(session): - session.execute( - 'INSERT INTO batches (reference, sku, _purchased_quantity, eta)' - ' VALUES ("batch1", "sku1", 100, null)' - ) - session.execute( - 'INSERT INTO batches (reference, sku, _purchased_quantity, eta)' - ' VALUES ("batch2", "sku2", 200, "2011-04-11")' - ) - expected = [ - model.Batch("batch1", "sku1", 100, eta=None), - model.Batch("batch2", "sku2", 200, eta=date(2011, 4, 11)), - ] - - assert session.query(model.Batch).all() == expected - - -def test_saving_batches(session): - batch = model.Batch('batch1', 'sku1', 100, eta=None) - session.add(batch) - session.commit() - rows = list(session.execute( - 'SELECT reference, sku, _purchased_quantity, eta FROM "batches"' - )) - assert rows == [('batch1', 'sku1', 100, None)] - -def test_saving_allocations(session): - batch = model.Batch('batch1', 'sku1', 100, eta=None) - line = model.OrderLine('order1', 'sku1', 10) - batch.allocate(line) - session.add(batch) - session.commit() - rows = list(session.execute('SELECT orderline_id, batch_id FROM "allocations"')) - assert rows == [(batch.id, line.id)] - - -def test_retrieving_allocations(session): - session.execute( - 'INSERT INTO order_lines (orderid, sku, qty) VALUES ("order1", "sku1", 12)' - ) - [[olid]] = session.execute( - 'SELECT id FROM order_lines WHERE orderid=:orderid AND sku=:sku', - dict(orderid='order1', sku='sku1') - ) - session.execute( - 'INSERT INTO batches (reference, sku, _purchased_quantity, eta)' - ' VALUES ("batch1", "sku1", 100, null)' - ) - [[bid]] = session.execute( - 'SELECT id FROM batches WHERE reference=:ref AND sku=:sku', - dict(ref='batch1', sku='sku1') - ) - session.execute( - 'INSERT INTO allocations (orderline_id, batch_id) VALUES (:olid, :bid)', - dict(olid=olid, bid=bid) - ) - - batch = session.query(model.Batch).one() - - assert batch._allocations == { - model.OrderLine("order1", "sku1", 12) - } diff --git a/tests/integration/test_repository.py b/tests/integration/test_repository.py deleted file mode 100644 index 29dddc77..00000000 --- a/tests/integration/test_repository.py +++ /dev/null @@ -1,64 +0,0 @@ -# pylint: disable=protected-access -from allocation.domain import model -from allocation.adapters import repository - -def test_repository_can_save_a_batch(session): - batch = model.Batch("batch1", "RUSTY-SOAPDISH", 100, eta=None) - - repo = repository.SqlAlchemyRepository(session) - repo.add(batch) - session.commit() - - rows = list(session.execute( - 'SELECT reference, sku, _purchased_quantity, eta FROM "batches"' - )) - assert rows == [("batch1", "RUSTY-SOAPDISH", 100, None)] - - -def insert_order_line(session): - session.execute( - 'INSERT INTO order_lines (orderid, sku, qty)' - ' VALUES ("order1", "GENERIC-SOFA", 12)' - ) - [[orderline_id]] = session.execute( - 'SELECT id FROM order_lines WHERE orderid=:orderid AND sku=:sku', - dict(orderid="order1", sku="GENERIC-SOFA") - ) - return orderline_id - -def insert_batch(session, batch_id): - session.execute( - 'INSERT INTO batches (reference, sku, _purchased_quantity, eta)' - ' VALUES (:batch_id, "GENERIC-SOFA", 100, null)', - dict(batch_id=batch_id) - ) - [[batch_id]] = session.execute( - 'SELECT id FROM batches WHERE reference=:batch_id AND sku="GENERIC-SOFA"', - dict(batch_id=batch_id) - ) - return batch_id - -def insert_allocation(session, orderline_id, batch_id): - session.execute( - 'INSERT INTO allocations (orderline_id, batch_id)' - ' VALUES (:orderline_id, :batch_id)', - dict(orderline_id=orderline_id, batch_id=batch_id) - ) - - -def test_repository_can_retrieve_a_batch_with_allocations(session): - orderline_id = insert_order_line(session) - batch1_id = insert_batch(session, "batch1") - insert_batch(session, "batch2") - insert_allocation(session, orderline_id, batch1_id) - - repo = repository.SqlAlchemyRepository(session) - retrieved = repo.get("batch1") - - expected = model.Batch("batch1", "GENERIC-SOFA", 100, eta=None) - assert retrieved == expected # Batch.__eq__ only compares reference - assert retrieved.sku == expected.sku - assert retrieved._purchased_quantity == expected._purchased_quantity - assert retrieved._allocations == { - model.OrderLine("order1", "GENERIC-SOFA", 12), - } diff --git a/tests/integration/test_uow.py b/tests/integration/test_uow.py index 5ac1af5a..482349eb 100644 --- a/tests/integration/test_uow.py +++ b/tests/integration/test_uow.py @@ -4,6 +4,10 @@ def insert_batch(session, ref, sku, qty, eta): + session.execute( + 'INSERT INTO products (sku) VALUES (:sku)', + dict(sku=sku), + ) session.execute( 'INSERT INTO batches (reference, sku, _purchased_quantity, eta)' ' VALUES (:ref, :sku, :qty, :eta)', @@ -31,9 +35,9 @@ def test_uow_can_retrieve_a_batch_and_allocate_to_it(session_factory): uow = unit_of_work.SqlAlchemyUnitOfWork(session_factory) with uow: - batch = uow.batches.get(reference='batch1') + product = uow.products.get(sku='HIPSTER-WORKBENCH') line = model.OrderLine('o1', 'HIPSTER-WORKBENCH', 10) - batch.allocate(line) + product.allocate(line) uow.commit() batchref = get_allocated_batch_ref(session, 'o1', 'HIPSTER-WORKBENCH') diff --git a/tests/unit/test_allocate.py b/tests/unit/test_product.py similarity index 68% rename from tests/unit/test_allocate.py rename to tests/unit/test_product.py index 88fa99c7..8c8771c5 100644 --- a/tests/unit/test_allocate.py +++ b/tests/unit/test_product.py @@ -1,17 +1,18 @@ from datetime import date, timedelta import pytest -from allocation.domain.model import allocate, OrderLine, Batch, OutOfStock +from allocation.domain.model import Product, OrderLine, Batch, OutOfStock today = date.today() tomorrow = today + timedelta(days=1) later = tomorrow + timedelta(days=10) -def test_prefers_current_stock_batches_to_shipments(): +def test_prefers_warehouse_batches_to_shipments(): in_stock_batch = Batch("in-stock-batch", "RETRO-CLOCK", 100, eta=None) shipment_batch = Batch("shipment-batch", "RETRO-CLOCK", 100, eta=tomorrow) + product = Product(sku="RETRO-CLOCK", batches=[in_stock_batch, shipment_batch]) line = OrderLine("oref", "RETRO-CLOCK", 10) - allocate(line, [in_stock_batch, shipment_batch]) + product.allocate(line) assert in_stock_batch.available_quantity == 90 assert shipment_batch.available_quantity == 100 @@ -21,9 +22,10 @@ def test_prefers_earlier_batches(): earliest = Batch("speedy-batch", "MINIMALIST-SPOON", 100, eta=today) medium = Batch("normal-batch", "MINIMALIST-SPOON", 100, eta=tomorrow) latest = Batch("slow-batch", "MINIMALIST-SPOON", 100, eta=later) + product = Product(sku="MINIMALIST-SPOON", batches=[medium, earliest, latest]) line = OrderLine("order1", "MINIMALIST-SPOON", 10) - allocate(line, [medium, earliest, latest]) + product.allocate(line) assert earliest.available_quantity == 90 assert medium.available_quantity == 100 @@ -34,13 +36,15 @@ def test_returns_allocated_batch_ref(): in_stock_batch = Batch("in-stock-batch-ref", "HIGHBROW-POSTER", 100, eta=None) shipment_batch = Batch("shipment-batch-ref", "HIGHBROW-POSTER", 100, eta=tomorrow) line = OrderLine("oref", "HIGHBROW-POSTER", 10) - allocation = allocate(line, [in_stock_batch, shipment_batch]) + product = Product(sku="HIGHBROW-POSTER", batches=[in_stock_batch, shipment_batch]) + allocation = product.allocate(line) assert allocation == in_stock_batch.reference def test_raises_out_of_stock_exception_if_cannot_allocate(): batch = Batch('batch1', 'SMALL-FORK', 10, eta=today) - allocate(OrderLine('order1', 'SMALL-FORK', 10), [batch]) + product = Product(sku="SMALL-FORK", batches=[batch]) + product.allocate(OrderLine('order1', 'SMALL-FORK', 10)) with pytest.raises(OutOfStock, match='SMALL-FORK'): - allocate(OrderLine('order2', 'SMALL-FORK', 1), [batch]) + product.allocate(OrderLine('order2', 'SMALL-FORK', 1)) diff --git a/tests/unit/test_services.py b/tests/unit/test_services.py index a9dc9503..bc22b157 100644 --- a/tests/unit/test_services.py +++ b/tests/unit/test_services.py @@ -5,23 +5,20 @@ class FakeRepository(repository.AbstractRepository): - def __init__(self, batches): - self._batches = set(batches) + def __init__(self, products): + self._products = set(products) - def add(self, batch): - self._batches.add(batch) + def add(self, product): + self._products.add(product) - def get(self, reference): - return next(b for b in self._batches if b.reference == reference) - - def list(self): - return list(self._batches) + def get(self, sku): + return next((p for p in self._products if p.sku == sku), None) class FakeUnitOfWork(unit_of_work.AbstractUnitOfWork): def __init__(self): - self.batches = FakeRepository([]) + self.products = FakeRepository([]) self.committed = False def commit(self): @@ -32,13 +29,20 @@ def rollback(self): -def test_add_batch(): +def test_add_batch_for_new_product(): uow = FakeUnitOfWork() services.add_batch("b1", "CRUNCHY-ARMCHAIR", 100, None, uow) - assert uow.batches.get("b1") is not None + assert uow.products.get('CRUNCHY-ARMCHAIR') is not None assert uow.committed +def test_add_batch_for_existing_product(): + uow = FakeUnitOfWork() + services.add_batch("b1", "GARISH-RUG", 100, None, uow) + services.add_batch("b2", "GARISH-RUG", 99, None, uow) + assert "b2" in [b.reference for b in uow.products.get("GARISH-RUG").batches] + + def test_allocate_returns_allocation(): uow = FakeUnitOfWork() services.add_batch("batch1", "COMPLICATED-LAMP", 100, None, uow) From ba250945ca40ce72fa46a8be1d29b03d4d0b0d44 Mon Sep 17 00:00:00 2001 From: Harry Date: Sat, 23 Mar 2019 22:59:22 +0000 Subject: [PATCH 067/165] first cut of a product with a version number --- src/allocation/domain/model.py | 4 +++- tests/unit/test_product.py | 8 ++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/allocation/domain/model.py b/src/allocation/domain/model.py index 4365fe20..3749cb48 100644 --- a/src/allocation/domain/model.py +++ b/src/allocation/domain/model.py @@ -10,9 +10,10 @@ class OutOfStock(Exception): class Product: - def __init__(self, sku: str, batches: List[Batch]): + def __init__(self, sku: str, batches: List[Batch], version_number: int = 0): self.sku = sku self.batches = batches + self.version_number = version_number def allocate(self, line: OrderLine) -> str: try: @@ -20,6 +21,7 @@ def allocate(self, line: OrderLine) -> str: b for b in sorted(self.batches) if b.can_allocate(line) ) batch.allocate(line) + self.version_number += 1 return batch.reference except StopIteration: raise OutOfStock(f'Out of stock for sku {line.sku}') diff --git a/tests/unit/test_product.py b/tests/unit/test_product.py index 8c8771c5..eb40e1be 100644 --- a/tests/unit/test_product.py +++ b/tests/unit/test_product.py @@ -48,3 +48,11 @@ def test_raises_out_of_stock_exception_if_cannot_allocate(): with pytest.raises(OutOfStock, match='SMALL-FORK'): product.allocate(OrderLine('order2', 'SMALL-FORK', 1)) + + +def test_increments_version_number(): + line = OrderLine('oref', "SCANDI-PEN", 10) + product = Product(sku="SCANDI-PEN", batches=[Batch('b1', "SCANDI-PEN", 100, eta=None)]) + product.version_number = 7 + product.allocate(line) + assert product.version_number == 8 From af5c38bb94705a5cc2382eb937565ae51fc001b4 Mon Sep 17 00:00:00 2001 From: Harry Date: Sat, 23 Mar 2019 23:39:49 +0000 Subject: [PATCH 068/165] start adding tests for consistency on version_number [data_integrity_test] --- src/allocation/adapters/orm.py | 2 +- src/allocation/adapters/repository.py | 1 + tests/__init__.py | 0 tests/conftest.py | 9 ++-- tests/e2e/__init__.py | 0 tests/e2e/test_api.py | 13 +----- tests/integration/__init__.py | 0 tests/integration/test_uow.py | 62 +++++++++++++++++++++++++-- tests/random_refs.py | 13 ++++++ 9 files changed, 81 insertions(+), 19 deletions(-) create mode 100644 tests/__init__.py create mode 100644 tests/e2e/__init__.py create mode 100644 tests/integration/__init__.py create mode 100644 tests/random_refs.py diff --git a/src/allocation/adapters/orm.py b/src/allocation/adapters/orm.py index edaa0c55..7df30b89 100644 --- a/src/allocation/adapters/orm.py +++ b/src/allocation/adapters/orm.py @@ -20,7 +20,7 @@ products = Table( 'products', metadata, Column('sku', String(255), primary_key=True), - # Column('version_number', Integer, nullable=False, default=0), + Column('version_number', Integer, nullable=False, server_default='0'), ) batches = Table( diff --git a/src/allocation/adapters/repository.py b/src/allocation/adapters/repository.py index 0459d70e..39eb0638 100644 --- a/src/allocation/adapters/repository.py +++ b/src/allocation/adapters/repository.py @@ -22,4 +22,5 @@ def add(self, product): self.session.add(product) def get(self, sku): + print(sku, type(sku)) return self.session.query(model.Product).filter_by(sku=sku).first() diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/conftest.py b/tests/conftest.py index cfe2611b..c34d61b4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -59,13 +59,16 @@ def postgres_db(): metadata.create_all(engine) return engine - @pytest.fixture -def postgres_session(postgres_db): +def postgres_session_factory(postgres_db): start_mappers() - yield sessionmaker(bind=postgres_db)() + yield sessionmaker(bind=postgres_db) clear_mappers() +@pytest.fixture +def postgres_session(postgres_session_factory): + return postgres_session_factory() + @pytest.fixture def restart_api(): diff --git a/tests/e2e/__init__.py b/tests/e2e/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/e2e/test_api.py b/tests/e2e/test_api.py index 3d117ef6..00aa906e 100644 --- a/tests/e2e/test_api.py +++ b/tests/e2e/test_api.py @@ -3,18 +3,7 @@ import requests from allocation import config - -def random_suffix(): - return uuid.uuid4().hex[:6] - -def random_sku(name=''): - return f'sku-{name}-{random_suffix()}' - -def random_batchref(name=''): - return f'batch-{name}-{random_suffix()}' - -def random_orderid(name=''): - return f'order-{name}-{random_suffix()}' +from ..random_refs import random_sku, random_batchref, random_orderid def post_to_add_batch(ref, sku, qty, eta): diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/integration/test_uow.py b/tests/integration/test_uow.py index 482349eb..1cc4fb06 100644 --- a/tests/integration/test_uow.py +++ b/tests/integration/test_uow.py @@ -1,12 +1,18 @@ +# pylint: disable=broad-except +import threading +import time +import traceback +from typing import List import pytest from allocation.domain import model from allocation.service_layer import unit_of_work +from ..random_refs import random_sku, random_batchref, random_orderid -def insert_batch(session, ref, sku, qty, eta): +def insert_batch(session, ref, sku, qty, eta, product_version=1): session.execute( - 'INSERT INTO products (sku) VALUES (:sku)', - dict(sku=sku), + 'INSERT INTO products (sku, version_number) VALUES (:sku, :version)', + dict(sku=sku, version=product_version), ) session.execute( 'INSERT INTO batches (reference, sku, _purchased_quantity, eta)' @@ -67,3 +73,53 @@ class MyException(Exception): new_session = session_factory() rows = list(new_session.execute('SELECT * FROM "batches"')) assert rows == [] + + +def try_to_allocate(orderid, sku, exceptions): + line = model.OrderLine(orderid, sku, 10) + try: + with unit_of_work.SqlAlchemyUnitOfWork() as uow: + product = uow.products.get(sku=sku) + product.allocate(line) + time.sleep(0.2) + uow.commit() + except Exception as e: + print(traceback.format_exc()) + exceptions.append(e) + + +def test_concurrent_updates_to_version_are_not_allowed(postgres_session_factory): + sku, batch = random_sku(), random_batchref() + session = postgres_session_factory() + insert_batch(session, batch, sku, 100, eta=None, product_version=1) + session.commit() + + order1, order2 = random_orderid(1), random_orderid(2) + exceptions = [] # type: List[Exception] + try_to_allocate_order1 = lambda: try_to_allocate(order1, sku, exceptions) + try_to_allocate_order2 = lambda: try_to_allocate(order2, sku, exceptions) + thread1 = threading.Thread(target=try_to_allocate_order1) + thread2 = threading.Thread(target=try_to_allocate_order2) + thread1.start() + thread2.start() + thread1.join() + thread2.join() + + [[version]] = session.execute( + "SELECT version_number FROM products WHERE sku=:sku", + dict(sku=sku), + ) + assert version == 2 + [exception] = exceptions + assert 'could not serialize access due to concurrent update' in str(exception) + + orders = list(session.execute( + "SELECT orderid FROM allocations" + " JOIN batches ON allocations.batch_id = batches.id" + " JOIN order_lines ON allocations.orderline_id = order_lines.id" + " WHERE order_lines.sku=:sku", + dict(sku=sku), + )) + assert len(orders) == 1 + with unit_of_work.SqlAlchemyUnitOfWork() as uow: + uow.session.execute('select 1') diff --git a/tests/random_refs.py b/tests/random_refs.py new file mode 100644 index 00000000..37259c98 --- /dev/null +++ b/tests/random_refs.py @@ -0,0 +1,13 @@ +import uuid + +def random_suffix(): + return uuid.uuid4().hex[:6] + +def random_sku(name=''): + return f'sku-{name}-{random_suffix()}' + +def random_batchref(name=''): + return f'batch-{name}-{random_suffix()}' + +def random_orderid(name=''): + return f'order-{name}-{random_suffix()}' From f05e4ab135fd1e6f91a178db7d527604d7bc782a Mon Sep 17 00:00:00 2001 From: Harry Date: Sat, 23 Mar 2019 23:40:06 +0000 Subject: [PATCH 069/165] select for update is one approach [with_for_update] --- src/allocation/adapters/repository.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/allocation/adapters/repository.py b/src/allocation/adapters/repository.py index 39eb0638..ca569c28 100644 --- a/src/allocation/adapters/repository.py +++ b/src/allocation/adapters/repository.py @@ -22,5 +22,7 @@ def add(self, product): self.session.add(product) def get(self, sku): - print(sku, type(sku)) - return self.session.query(model.Product).filter_by(sku=sku).first() + return self.session.query(model.Product) \ + .filter_by(sku=sku) \ + .with_for_update() \ + .first() From e6de3889167c8f076a48aeb761bf2d35ac8b4b75 Mon Sep 17 00:00:00 2001 From: Harry Date: Sat, 23 Mar 2019 23:53:32 +0000 Subject: [PATCH 070/165] alternative: isolation=SERIALIZABLE [chapter_07_aggregate_ends] --- src/allocation/adapters/repository.py | 5 +---- src/allocation/service_layer/unit_of_work.py | 1 + 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/allocation/adapters/repository.py b/src/allocation/adapters/repository.py index ca569c28..0459d70e 100644 --- a/src/allocation/adapters/repository.py +++ b/src/allocation/adapters/repository.py @@ -22,7 +22,4 @@ def add(self, product): self.session.add(product) def get(self, sku): - return self.session.query(model.Product) \ - .filter_by(sku=sku) \ - .with_for_update() \ - .first() + return self.session.query(model.Product).filter_by(sku=sku).first() diff --git a/src/allocation/service_layer/unit_of_work.py b/src/allocation/service_layer/unit_of_work.py index af9d9117..77fc8f5a 100644 --- a/src/allocation/service_layer/unit_of_work.py +++ b/src/allocation/service_layer/unit_of_work.py @@ -30,6 +30,7 @@ def rollback(self): DEFAULT_SESSION_FACTORY = sessionmaker(bind=create_engine( config.get_postgres_uri(), + isolation_level="REPEATABLE READ", )) class SqlAlchemyUnitOfWork(AbstractUnitOfWork): From c01843d65abfb905911bd66581530ab07ab8a8b8 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 9 Apr 2019 10:59:34 +0100 Subject: [PATCH 071/165] Mocky test for email [mocky_test_for_send_email] --- src/allocation/adapters/email.py | 2 ++ tests/unit/test_services.py | 17 ++++++++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 src/allocation/adapters/email.py diff --git a/src/allocation/adapters/email.py b/src/allocation/adapters/email.py new file mode 100644 index 00000000..85b33957 --- /dev/null +++ b/src/allocation/adapters/email.py @@ -0,0 +1,2 @@ +def send_mail(*args): + print('SENDING EMAIL:', *args) diff --git a/tests/unit/test_services.py b/tests/unit/test_services.py index bc22b157..b23a5db1 100644 --- a/tests/unit/test_services.py +++ b/tests/unit/test_services.py @@ -1,5 +1,7 @@ +from unittest import mock import pytest from allocation.adapters import repository +from allocation.domain.model import OutOfStock from allocation.service_layer import services, unit_of_work @@ -32,7 +34,7 @@ def rollback(self): def test_add_batch_for_new_product(): uow = FakeUnitOfWork() services.add_batch("b1", "CRUNCHY-ARMCHAIR", 100, None, uow) - assert uow.products.get('CRUNCHY-ARMCHAIR') is not None + assert uow.products.get("CRUNCHY-ARMCHAIR") is not None assert uow.committed @@ -63,3 +65,16 @@ def test_allocate_commits(): services.add_batch("b1", "OMINOUS-MIRROR", 100, None, uow) services.allocate("o1", "OMINOUS-MIRROR", 10, uow) assert uow.committed + + +def test_sends_email_on_out_of_stock_error(): + uow = FakeUnitOfWork() + services.add_batch("b1", "POPULAR-CURTAINS", 9, None, uow) + + with mock.patch("allocation.adapters.email.send_mail") as mock_send_mail: + with pytest.raises(OutOfStock): + services.allocate("o1", "POPULAR-CURTAINS", 10, uow) + assert mock_send_mail.call_args == mock.call( + "stock@made.com", + f"Out of stock for POPULAR-CURTAINS", + ) From 60c2dc0ab096e6ab0b1c63f912e1f31e1d2e2f17 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 9 Apr 2019 11:13:48 +0100 Subject: [PATCH 072/165] email in model [email_in_model] --- src/allocation/domain/model.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/allocation/domain/model.py b/src/allocation/domain/model.py index 3749cb48..c8e76367 100644 --- a/src/allocation/domain/model.py +++ b/src/allocation/domain/model.py @@ -2,6 +2,7 @@ from dataclasses import dataclass from datetime import date from typing import Optional, List, Set +from allocation.adapters import email class OutOfStock(Exception): @@ -24,6 +25,7 @@ def allocate(self, line: OrderLine) -> str: self.version_number += 1 return batch.reference except StopIteration: + email.send_mail('stock@made.com', f'Out of stock for {line.sku}') raise OutOfStock(f'Out of stock for sku {line.sku}') From 13da95f3dea7a876a28fee680b1d85dccfee5cff Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 9 Apr 2019 11:23:24 +0100 Subject: [PATCH 073/165] putting it in the services layer isn't lovely either [email_in_services] --- src/allocation/domain/model.py | 3 --- src/allocation/service_layer/services.py | 11 ++++++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/allocation/domain/model.py b/src/allocation/domain/model.py index c8e76367..bf558ef4 100644 --- a/src/allocation/domain/model.py +++ b/src/allocation/domain/model.py @@ -2,7 +2,6 @@ from dataclasses import dataclass from datetime import date from typing import Optional, List, Set -from allocation.adapters import email class OutOfStock(Exception): @@ -25,7 +24,6 @@ def allocate(self, line: OrderLine) -> str: self.version_number += 1 return batch.reference except StopIteration: - email.send_mail('stock@made.com', f'Out of stock for {line.sku}') raise OutOfStock(f'Out of stock for sku {line.sku}') @@ -82,4 +80,3 @@ def available_quantity(self) -> int: def can_allocate(self, line: OrderLine) -> bool: return self.sku == line.sku and self.available_quantity >= line.qty - diff --git a/src/allocation/service_layer/services.py b/src/allocation/service_layer/services.py index d0e6c60c..6a3a746b 100644 --- a/src/allocation/service_layer/services.py +++ b/src/allocation/service_layer/services.py @@ -2,6 +2,7 @@ from typing import Optional from datetime import date +from allocation.adapters import email from allocation.domain import model from allocation.domain.model import OrderLine from allocation.service_layer import unit_of_work @@ -33,6 +34,10 @@ def allocate( product = uow.products.get(sku=line.sku) if product is None: raise InvalidSku(f'Invalid sku {line.sku}') - batchref = product.allocate(line) - uow.commit() - return batchref + try: + batchref = product.allocate(line) + uow.commit() + return batchref + except model.OutOfStock: + email.send_mail('stock@made.com', f'Out of stock for {line.sku}') + raise From 068569b8e5dd9482923c679b8d77d08bdbb54733 Mon Sep 17 00:00:00 2001 From: Harry Date: Thu, 28 Mar 2019 15:45:20 +0000 Subject: [PATCH 074/165] first cut of out of stock event [domain_event] --- src/allocation/domain/events.py | 9 +++++++++ src/allocation/domain/model.py | 6 +++++- tests/unit/test_product.py | 12 ++++++++++++ 3 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 src/allocation/domain/events.py diff --git a/src/allocation/domain/events.py b/src/allocation/domain/events.py new file mode 100644 index 00000000..66741038 --- /dev/null +++ b/src/allocation/domain/events.py @@ -0,0 +1,9 @@ +from dataclasses import dataclass + +class Event: + pass + +@dataclass +class OutOfStock(Event): + sku: str + diff --git a/src/allocation/domain/model.py b/src/allocation/domain/model.py index bf558ef4..0c68ea2a 100644 --- a/src/allocation/domain/model.py +++ b/src/allocation/domain/model.py @@ -2,6 +2,7 @@ from dataclasses import dataclass from datetime import date from typing import Optional, List, Set +from . import events class OutOfStock(Exception): @@ -14,6 +15,7 @@ def __init__(self, sku: str, batches: List[Batch], version_number: int = 0): self.sku = sku self.batches = batches self.version_number = version_number + self.events = [] # type: List[events.Event] def allocate(self, line: OrderLine) -> str: try: @@ -24,7 +26,9 @@ def allocate(self, line: OrderLine) -> str: self.version_number += 1 return batch.reference except StopIteration: - raise OutOfStock(f'Out of stock for sku {line.sku}') + self.events.append(events.OutOfStock(line.sku)) + # raise OutOfStock(f'Out of stock for sku {line.sku}') + return None @dataclass(unsafe_hash=True) diff --git a/tests/unit/test_product.py b/tests/unit/test_product.py index eb40e1be..dd6db461 100644 --- a/tests/unit/test_product.py +++ b/tests/unit/test_product.py @@ -1,5 +1,7 @@ from datetime import date, timedelta import pytest + +from allocation.domain import events from allocation.domain.model import Product, OrderLine, Batch, OutOfStock today = date.today() @@ -50,6 +52,16 @@ def test_raises_out_of_stock_exception_if_cannot_allocate(): product.allocate(OrderLine('order2', 'SMALL-FORK', 1)) +def test_records_out_of_stock_event_if_cannot_allocate(): + sku1_batch = Batch('batch1', 'sku1', 100, eta=today) + sku2_line = OrderLine('oref', 'sku2', 10) + product = Product(sku='sku1', batches=[sku1_batch]) + + with pytest.raises(OutOfStock): + product.allocate(sku2_line) + assert product.events[-1] == events.OutOfStock(sku='sku2') + + def test_increments_version_number(): line = OrderLine('oref', "SCANDI-PEN", 10) product = Product(sku="SCANDI-PEN", batches=[Batch('b1', "SCANDI-PEN", 100, eta=None)]) From 3a6d3045e8802a80a984b5fb5805db4d922d7c5e Mon Sep 17 00:00:00 2001 From: Harry Date: Sun, 9 Jun 2019 10:14:18 +0100 Subject: [PATCH 075/165] remove commented-out line from model --- src/allocation/domain/model.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allocation/domain/model.py b/src/allocation/domain/model.py index 0c68ea2a..c46849e7 100644 --- a/src/allocation/domain/model.py +++ b/src/allocation/domain/model.py @@ -27,7 +27,6 @@ def allocate(self, line: OrderLine) -> str: return batch.reference except StopIteration: self.events.append(events.OutOfStock(line.sku)) - # raise OutOfStock(f'Out of stock for sku {line.sku}') return None From 0e80732990a8dd75eb70120440e82401e473f822 Mon Sep 17 00:00:00 2001 From: Harry Date: Sun, 9 Jun 2019 10:14:28 +0100 Subject: [PATCH 076/165] Adjust unit tests now we're no longer raising out of stock exception --- tests/unit/test_product.py | 22 ++++++---------------- tests/unit/test_services.py | 4 +--- 2 files changed, 7 insertions(+), 19 deletions(-) diff --git a/tests/unit/test_product.py b/tests/unit/test_product.py index dd6db461..22de7185 100644 --- a/tests/unit/test_product.py +++ b/tests/unit/test_product.py @@ -1,8 +1,7 @@ from datetime import date, timedelta -import pytest - from allocation.domain import events -from allocation.domain.model import Product, OrderLine, Batch, OutOfStock +from allocation.domain.model import Product, OrderLine, Batch + today = date.today() tomorrow = today + timedelta(days=1) @@ -43,23 +42,14 @@ def test_returns_allocated_batch_ref(): assert allocation == in_stock_batch.reference -def test_raises_out_of_stock_exception_if_cannot_allocate(): +def test_records_out_of_stock_event_if_cannot_allocate(): batch = Batch('batch1', 'SMALL-FORK', 10, eta=today) product = Product(sku="SMALL-FORK", batches=[batch]) product.allocate(OrderLine('order1', 'SMALL-FORK', 10)) - with pytest.raises(OutOfStock, match='SMALL-FORK'): - product.allocate(OrderLine('order2', 'SMALL-FORK', 1)) - - -def test_records_out_of_stock_event_if_cannot_allocate(): - sku1_batch = Batch('batch1', 'sku1', 100, eta=today) - sku2_line = OrderLine('oref', 'sku2', 10) - product = Product(sku='sku1', batches=[sku1_batch]) - - with pytest.raises(OutOfStock): - product.allocate(sku2_line) - assert product.events[-1] == events.OutOfStock(sku='sku2') + allocation = product.allocate(OrderLine('order2', 'SMALL-FORK', 1)) + assert product.events[-1] == events.OutOfStock(sku="SMALL-FORK") + assert allocation is None def test_increments_version_number(): diff --git a/tests/unit/test_services.py b/tests/unit/test_services.py index b23a5db1..e2265227 100644 --- a/tests/unit/test_services.py +++ b/tests/unit/test_services.py @@ -1,7 +1,6 @@ from unittest import mock import pytest from allocation.adapters import repository -from allocation.domain.model import OutOfStock from allocation.service_layer import services, unit_of_work @@ -72,8 +71,7 @@ def test_sends_email_on_out_of_stock_error(): services.add_batch("b1", "POPULAR-CURTAINS", 9, None, uow) with mock.patch("allocation.adapters.email.send_mail") as mock_send_mail: - with pytest.raises(OutOfStock): - services.allocate("o1", "POPULAR-CURTAINS", 10, uow) + services.allocate("o1", "POPULAR-CURTAINS", 10, uow) assert mock_send_mail.call_args == mock.call( "stock@made.com", f"Out of stock for POPULAR-CURTAINS", From 68ec8e2ecb4fc98a7da4df2415ec5f7b8e713950 Mon Sep 17 00:00:00 2001 From: Harry Date: Fri, 3 Jan 2020 02:01:07 +0000 Subject: [PATCH 077/165] first cut of message bus [service_talks_to_messagebus] --- src/allocation/service_layer/messagebus.py | 21 +++++++++++++++++++++ src/allocation/service_layer/services.py | 12 ++++++------ 2 files changed, 27 insertions(+), 6 deletions(-) create mode 100644 src/allocation/service_layer/messagebus.py diff --git a/src/allocation/service_layer/messagebus.py b/src/allocation/service_layer/messagebus.py new file mode 100644 index 00000000..bf04ebf0 --- /dev/null +++ b/src/allocation/service_layer/messagebus.py @@ -0,0 +1,21 @@ +from typing import List, Dict, Callable, Type +from allocation.adapters import email +from allocation.domain import events + + +def handle(event: events.Event): + for handler in HANDLERS[type(event)]: + handler(event) + + +def send_out_of_stock_notification(event: events.OutOfStock): + email.send_mail( + 'stock@made.com', + f'Out of stock for {event.sku}', + ) + + +HANDLERS = { + events.OutOfStock: [send_out_of_stock_notification], + +} # type: Dict[Type[events.Event], List[Callable]] diff --git a/src/allocation/service_layer/services.py b/src/allocation/service_layer/services.py index 6a3a746b..7aef8ad5 100644 --- a/src/allocation/service_layer/services.py +++ b/src/allocation/service_layer/services.py @@ -1,11 +1,12 @@ from __future__ import annotations -from typing import Optional +from typing import Optional, TYPE_CHECKING from datetime import date -from allocation.adapters import email from allocation.domain import model from allocation.domain.model import OrderLine -from allocation.service_layer import unit_of_work +from . import messagebus +if TYPE_CHECKING: + from . import unit_of_work class InvalidSku(Exception): @@ -38,6 +39,5 @@ def allocate( batchref = product.allocate(line) uow.commit() return batchref - except model.OutOfStock: - email.send_mail('stock@made.com', f'Out of stock for {line.sku}') - raise + finally: + messagebus.handle(product.events) From 8cb6e2d236986d8afc1c24318e491e7ba8dec22d Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 9 Apr 2019 12:57:11 +0100 Subject: [PATCH 078/165] uow now does messagebus magically. breaks tests [uow_has_messagebus] --- src/allocation/adapters/repository.py | 9 ++++++++- src/allocation/service_layer/services.py | 10 +++------- src/allocation/service_layer/unit_of_work.py | 17 +++++++++++++++-- tests/unit/test_services.py | 2 +- 4 files changed, 27 insertions(+), 11 deletions(-) diff --git a/src/allocation/adapters/repository.py b/src/allocation/adapters/repository.py index 0459d70e..d3c609d3 100644 --- a/src/allocation/adapters/repository.py +++ b/src/allocation/adapters/repository.py @@ -1,6 +1,8 @@ import abc +from typing import Set from allocation.domain import model + class AbstractRepository(abc.ABC): @abc.abstractmethod @@ -17,9 +19,14 @@ class SqlAlchemyRepository(AbstractRepository): def __init__(self, session): self.session = session + self.seen = set() # type: Set[model.Product] def add(self, product): + self.seen.add(product) self.session.add(product) def get(self, sku): - return self.session.query(model.Product).filter_by(sku=sku).first() + product = self.session.query(model.Product).filter_by(sku=sku).first() + if product: + self.seen.add(product) + return product diff --git a/src/allocation/service_layer/services.py b/src/allocation/service_layer/services.py index 7aef8ad5..1bbea8ea 100644 --- a/src/allocation/service_layer/services.py +++ b/src/allocation/service_layer/services.py @@ -4,7 +4,6 @@ from allocation.domain import model from allocation.domain.model import OrderLine -from . import messagebus if TYPE_CHECKING: from . import unit_of_work @@ -35,9 +34,6 @@ def allocate( product = uow.products.get(sku=line.sku) if product is None: raise InvalidSku(f'Invalid sku {line.sku}') - try: - batchref = product.allocate(line) - uow.commit() - return batchref - finally: - messagebus.handle(product.events) + batchref = product.allocate(line) + uow.commit() + return batchref diff --git a/src/allocation/service_layer/unit_of_work.py b/src/allocation/service_layer/unit_of_work.py index 77fc8f5a..616e7ec0 100644 --- a/src/allocation/service_layer/unit_of_work.py +++ b/src/allocation/service_layer/unit_of_work.py @@ -5,8 +5,11 @@ from sqlalchemy.orm import sessionmaker from sqlalchemy.orm.session import Session + from allocation import config from allocation.adapters import repository +from . import messagebus + class AbstractUnitOfWork(abc.ABC): @@ -18,8 +21,18 @@ def __enter__(self) -> AbstractUnitOfWork: def __exit__(self, *args): self.rollback() - @abc.abstractmethod def commit(self): + self._commit() + self.publish_events() + + def publish_events(self): + for product in self.products.seen: + while product.events: + event = product.events.pop(0) + messagebus.handle(event) + + @abc.abstractmethod + def _commit(self): raise NotImplementedError @abc.abstractmethod @@ -47,7 +60,7 @@ def __exit__(self, *args): super().__exit__(*args) self.session.close() - def commit(self): + def _commit(self): self.session.commit() def rollback(self): diff --git a/tests/unit/test_services.py b/tests/unit/test_services.py index e2265227..503bff7d 100644 --- a/tests/unit/test_services.py +++ b/tests/unit/test_services.py @@ -22,7 +22,7 @@ def __init__(self): self.products = FakeRepository([]) self.committed = False - def commit(self): + def _commit(self): self.committed = True def rollback(self): From ae7a75abbe83d04120bf213d2d5ab9fab02c6c35 Mon Sep 17 00:00:00 2001 From: Harry Date: Sun, 9 Jun 2019 09:46:47 +0100 Subject: [PATCH 079/165] implement .seen on repository [repository_tracks_seen] --- src/allocation/adapters/repository.py | 29 ++++++++++++++++++--------- tests/unit/test_services.py | 5 +++-- 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/src/allocation/adapters/repository.py b/src/allocation/adapters/repository.py index d3c609d3..33a2bc0f 100644 --- a/src/allocation/adapters/repository.py +++ b/src/allocation/adapters/repository.py @@ -5,12 +5,25 @@ class AbstractRepository(abc.ABC): - @abc.abstractmethod + def __init__(self): + self.seen = set() # type: Set[model.Product] + def add(self, product: model.Product): + self._add(product) + self.seen.add(product) + + def get(self, sku) -> model.Product: + product = self._get(sku) + if product: + self.seen.add(product) + return product + + @abc.abstractmethod + def _add(self, product: model.Product): raise NotImplementedError @abc.abstractmethod - def get(self, sku) -> model.Product: + def _get(self, sku) -> model.Product: raise NotImplementedError @@ -18,15 +31,11 @@ def get(self, sku) -> model.Product: class SqlAlchemyRepository(AbstractRepository): def __init__(self, session): + super().__init__() self.session = session - self.seen = set() # type: Set[model.Product] - def add(self, product): - self.seen.add(product) + def _add(self, product): self.session.add(product) - def get(self, sku): - product = self.session.query(model.Product).filter_by(sku=sku).first() - if product: - self.seen.add(product) - return product + def _get(self, sku): + return self.session.query(model.Product).filter_by(sku=sku).first() diff --git a/tests/unit/test_services.py b/tests/unit/test_services.py index 503bff7d..612fe6f5 100644 --- a/tests/unit/test_services.py +++ b/tests/unit/test_services.py @@ -7,12 +7,13 @@ class FakeRepository(repository.AbstractRepository): def __init__(self, products): + super().__init__() self._products = set(products) - def add(self, product): + def _add(self, product): self._products.add(product) - def get(self, sku): + def _get(self, sku): return next((p for p in self._products if p.sku == sku), None) From 320dbc198ebbda917741ba29df47f1976f1b8152 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 9 Apr 2019 13:07:55 +0100 Subject: [PATCH 080/165] a little hack in the orm so that events work --- src/allocation/adapters/orm.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/allocation/adapters/orm.py b/src/allocation/adapters/orm.py index 7df30b89..285fdd0b 100644 --- a/src/allocation/adapters/orm.py +++ b/src/allocation/adapters/orm.py @@ -1,6 +1,6 @@ from sqlalchemy import ( - Table, MetaData, Column, Integer, String, Date, - ForeignKey + Table, MetaData, Column, Integer, String, Date, ForeignKey, + event, ) from sqlalchemy.orm import mapper, relationship @@ -52,3 +52,8 @@ def start_mappers(): mapper(model.Product, products, properties={ 'batches': relationship(batches_mapper) }) + +@event.listens_for(model.Product, 'load') +def receive_load(product, _): + product.events = [] + From 7373b2e1d7335eecdad95721720b297bbc24176b Mon Sep 17 00:00:00 2001 From: Harry Date: Sun, 9 Jun 2019 10:21:30 +0100 Subject: [PATCH 081/165] remove now unused out-of-stock exception [chapter_08_events_and_message_bus_ends] --- src/allocation/domain/model.py | 4 ---- src/allocation/entrypoints/flask_app.py | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/src/allocation/domain/model.py b/src/allocation/domain/model.py index c46849e7..a6b84c4e 100644 --- a/src/allocation/domain/model.py +++ b/src/allocation/domain/model.py @@ -5,10 +5,6 @@ from . import events -class OutOfStock(Exception): - pass - - class Product: def __init__(self, sku: str, batches: List[Batch], version_number: int = 0): diff --git a/src/allocation/entrypoints/flask_app.py b/src/allocation/entrypoints/flask_app.py index 2b720763..cb7b55a8 100644 --- a/src/allocation/entrypoints/flask_app.py +++ b/src/allocation/entrypoints/flask_app.py @@ -30,7 +30,7 @@ def allocate_endpoint(): request.json['qty'], unit_of_work.SqlAlchemyUnitOfWork(), ) - except (model.OutOfStock, services.InvalidSku) as e: + except services.InvalidSku as e: return jsonify({'message': str(e)}), 400 return jsonify({'batchref': batchref}), 201 From a70c61a05e638e69d7ee0fcf9390f08e62ddd651 Mon Sep 17 00:00:00 2001 From: Harry Date: Sun, 26 May 2019 21:29:23 +0100 Subject: [PATCH 082/165] allocationrequired and batchcreated events [two_new_events] --- src/allocation/domain/events.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/allocation/domain/events.py b/src/allocation/domain/events.py index 66741038..d59e66b5 100644 --- a/src/allocation/domain/events.py +++ b/src/allocation/domain/events.py @@ -1,9 +1,24 @@ +# pylint: disable=too-few-public-methods from dataclasses import dataclass +from datetime import date +from typing import Optional class Event: pass @dataclass -class OutOfStock(Event): +class BatchCreated(Event): + ref: str + sku: str + qty: int + eta: Optional[date] = None + +@dataclass +class AllocationRequired(Event): + orderid: str sku: str + qty: int +@dataclass +class OutOfStock(Event): + sku: str From cdd3dd6c6f05b816c73fcfb54ea39325d6177f0c Mon Sep 17 00:00:00 2001 From: Harry Date: Fri, 3 Jan 2020 02:58:58 +0000 Subject: [PATCH 083/165] change exception import in flask app --- src/allocation/entrypoints/flask_app.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allocation/entrypoints/flask_app.py b/src/allocation/entrypoints/flask_app.py index cb7b55a8..69614824 100644 --- a/src/allocation/entrypoints/flask_app.py +++ b/src/allocation/entrypoints/flask_app.py @@ -1,9 +1,9 @@ from datetime import datetime from flask import Flask, jsonify, request -from allocation.domain import model from allocation.adapters import orm from allocation.service_layer import services, unit_of_work +from allocation.service_layer.services import InvalidSku app = Flask(__name__) orm.start_mappers() @@ -30,7 +30,7 @@ def allocate_endpoint(): request.json['qty'], unit_of_work.SqlAlchemyUnitOfWork(), ) - except services.InvalidSku as e: + except InvalidSku as e: return jsonify({'message': str(e)}), 400 return jsonify({'batchref': batchref}), 201 From 71645f5e1808d585a561507c3e263e1c0daf28a0 Mon Sep 17 00:00:00 2001 From: Harry Date: Sun, 26 May 2019 21:45:42 +0100 Subject: [PATCH 084/165] move services.py to handlers.py rename test services to test handlers --- src/allocation/service_layer/{services.py => handlers.py} | 0 tests/unit/{test_services.py => test_handlers.py} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename src/allocation/service_layer/{services.py => handlers.py} (100%) rename tests/unit/{test_services.py => test_handlers.py} (100%) diff --git a/src/allocation/service_layer/services.py b/src/allocation/service_layer/handlers.py similarity index 100% rename from src/allocation/service_layer/services.py rename to src/allocation/service_layer/handlers.py diff --git a/tests/unit/test_services.py b/tests/unit/test_handlers.py similarity index 100% rename from tests/unit/test_services.py rename to tests/unit/test_handlers.py From 2b0b2f57b33f60c373ecdf40b2f86256ea761717 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 6 Aug 2019 13:50:45 +0100 Subject: [PATCH 085/165] Start moving to handlers [services_to_handlers] --- src/allocation/adapters/email.py | 2 +- src/allocation/service_layer/handlers.py | 34 ++++++++++++++-------- src/allocation/service_layer/messagebus.py | 14 +++------ 3 files changed, 27 insertions(+), 23 deletions(-) diff --git a/src/allocation/adapters/email.py b/src/allocation/adapters/email.py index 85b33957..bd48f980 100644 --- a/src/allocation/adapters/email.py +++ b/src/allocation/adapters/email.py @@ -1,2 +1,2 @@ -def send_mail(*args): +def send(*args): print('SENDING EMAIL:', *args) diff --git a/src/allocation/service_layer/handlers.py b/src/allocation/service_layer/handlers.py index 1bbea8ea..8eaf4e3b 100644 --- a/src/allocation/service_layer/handlers.py +++ b/src/allocation/service_layer/handlers.py @@ -1,8 +1,6 @@ from __future__ import annotations -from typing import Optional, TYPE_CHECKING -from datetime import date - -from allocation.domain import model +from typing import TYPE_CHECKING +from allocation.domain import events, model from allocation.domain.model import OrderLine if TYPE_CHECKING: from . import unit_of_work @@ -12,24 +10,25 @@ class InvalidSku(Exception): pass + def add_batch( - ref: str, sku: str, qty: int, eta: Optional[date], - uow: unit_of_work.AbstractUnitOfWork + event: events.BatchCreated, uow: unit_of_work.AbstractUnitOfWork ): with uow: - product = uow.products.get(sku=sku) + product = uow.products.get(sku=event.sku) if product is None: - product = model.Product(sku, batches=[]) + product = model.Product(event.sku, batches=[]) uow.products.add(product) - product.batches.append(model.Batch(ref, sku, qty, eta)) + product.batches.append(model.Batch( + event.ref, event.sku, event.qty, event.eta + )) uow.commit() def allocate( - orderid: str, sku: str, qty: int, - uow: unit_of_work.AbstractUnitOfWork + event: events.AllocationRequired, uow: unit_of_work.AbstractUnitOfWork ) -> str: - line = OrderLine(orderid, sku, qty) + line = OrderLine(event.orderid, event.sku, event.qty) with uow: product = uow.products.get(sku=line.sku) if product is None: @@ -37,3 +36,14 @@ def allocate( batchref = product.allocate(line) uow.commit() return batchref + + +# pylint: disable=unused-argument + +def send_out_of_stock_notification( + event: events.OutOfStock, uow: unit_of_work.AbstractUnitOfWork, +): + email.send( + 'stock@made.com', + f'Out of stock for {event.sku}', + ) diff --git a/src/allocation/service_layer/messagebus.py b/src/allocation/service_layer/messagebus.py index bf04ebf0..d1d5997b 100644 --- a/src/allocation/service_layer/messagebus.py +++ b/src/allocation/service_layer/messagebus.py @@ -1,6 +1,6 @@ from typing import List, Dict, Callable, Type -from allocation.adapters import email from allocation.domain import events +from . import handlers def handle(event: events.Event): @@ -8,14 +8,8 @@ def handle(event: events.Event): handler(event) -def send_out_of_stock_notification(event: events.OutOfStock): - email.send_mail( - 'stock@made.com', - f'Out of stock for {event.sku}', - ) - - HANDLERS = { - events.OutOfStock: [send_out_of_stock_notification], - + events.BatchCreated: [handlers.add_batch], + events.AllocationRequired: [handlers.allocate], + events.OutOfStock: [handlers.send_out_of_stock_notification], } # type: Dict[Type[events.Event], List[Callable]] From 0182ccf6654f7f4b4ed05d149fa8c3fa3a288f4c Mon Sep 17 00:00:00 2001 From: Harry Date: Sun, 9 Feb 2020 12:02:51 +0000 Subject: [PATCH 086/165] use classes in services/handlers test [tests_use_classes] --- tests/unit/test_handlers.py | 73 ++++++++++++++++++++----------------- 1 file changed, 39 insertions(+), 34 deletions(-) diff --git a/tests/unit/test_handlers.py b/tests/unit/test_handlers.py index 612fe6f5..496704e6 100644 --- a/tests/unit/test_handlers.py +++ b/tests/unit/test_handlers.py @@ -1,3 +1,4 @@ +# pylint: disable=no-self-use from unittest import mock import pytest from allocation.adapters import repository @@ -31,49 +32,53 @@ def rollback(self): -def test_add_batch_for_new_product(): - uow = FakeUnitOfWork() - services.add_batch("b1", "CRUNCHY-ARMCHAIR", 100, None, uow) - assert uow.products.get("CRUNCHY-ARMCHAIR") is not None - assert uow.committed +class TestAddBatch: + def test_for_new_product(self): + uow = FakeUnitOfWork() + services.add_batch("b1", "CRUNCHY-ARMCHAIR", 100, None, uow) + assert uow.products.get("CRUNCHY-ARMCHAIR") is not None + assert uow.committed -def test_add_batch_for_existing_product(): - uow = FakeUnitOfWork() - services.add_batch("b1", "GARISH-RUG", 100, None, uow) - services.add_batch("b2", "GARISH-RUG", 99, None, uow) - assert "b2" in [b.reference for b in uow.products.get("GARISH-RUG").batches] + def test_for_existing_product(self): + uow = FakeUnitOfWork() + services.add_batch("b1", "GARISH-RUG", 100, None, uow) + services.add_batch("b2", "GARISH-RUG", 99, None, uow) + assert "b2" in [b.reference for b in uow.products.get("GARISH-RUG").batches] -def test_allocate_returns_allocation(): - uow = FakeUnitOfWork() - services.add_batch("batch1", "COMPLICATED-LAMP", 100, None, uow) - result = services.allocate("o1", "COMPLICATED-LAMP", 10, uow) - assert result == "batch1" +class TestAllocate: -def test_allocate_errors_for_invalid_sku(): - uow = FakeUnitOfWork() - services.add_batch("b1", "AREALSKU", 100, None, uow) + def test_returns_allocation(self): + uow = FakeUnitOfWork() + services.add_batch("batch1", "COMPLICATED-LAMP", 100, None, uow) + result = services.allocate("o1", "COMPLICATED-LAMP", 10, uow) + assert result == "batch1" - with pytest.raises(services.InvalidSku, match="Invalid sku NONEXISTENTSKU"): - services.allocate("o1", "NONEXISTENTSKU", 10, uow) + def test_errors_for_invalid_sku(self): + uow = FakeUnitOfWork() + services.add_batch("b1", "AREALSKU", 100, None, uow) -def test_allocate_commits(): - uow = FakeUnitOfWork() - services.add_batch("b1", "OMINOUS-MIRROR", 100, None, uow) - services.allocate("o1", "OMINOUS-MIRROR", 10, uow) - assert uow.committed + with pytest.raises(services.InvalidSku, match="Invalid sku NONEXISTENTSKU"): + services.allocate("o1", "NONEXISTENTSKU", 10, uow) -def test_sends_email_on_out_of_stock_error(): - uow = FakeUnitOfWork() - services.add_batch("b1", "POPULAR-CURTAINS", 9, None, uow) + def test_commits(self): + uow = FakeUnitOfWork() + services.add_batch("b1", "OMINOUS-MIRROR", 100, None, uow) + services.allocate("o1", "OMINOUS-MIRROR", 10, uow) + assert uow.committed - with mock.patch("allocation.adapters.email.send_mail") as mock_send_mail: - services.allocate("o1", "POPULAR-CURTAINS", 10, uow) - assert mock_send_mail.call_args == mock.call( - "stock@made.com", - f"Out of stock for POPULAR-CURTAINS", - ) + + def test_sends_email_on_out_of_stock_error(self): + uow = FakeUnitOfWork() + services.add_batch("b1", "POPULAR-CURTAINS", 9, None, uow) + + with mock.patch("allocation.adapters.email.send_mail") as mock_send_mail: + services.allocate("o1", "POPULAR-CURTAINS", 10, uow) + assert mock_send_mail.call_args == mock.call( + "stock@made.com", + f"Out of stock for POPULAR-CURTAINS", + ) From 0bc35ef48fcd071bf43bd803ce3995e43a82c5cd Mon Sep 17 00:00:00 2001 From: Harry Date: Sun, 9 Feb 2020 12:08:05 +0000 Subject: [PATCH 087/165] tests change to use bus [handler_tests] --- tests/unit/test_handlers.py | 51 ++++++++++++++++++++++++------------- 1 file changed, 34 insertions(+), 17 deletions(-) diff --git a/tests/unit/test_handlers.py b/tests/unit/test_handlers.py index 496704e6..9e967d59 100644 --- a/tests/unit/test_handlers.py +++ b/tests/unit/test_handlers.py @@ -1,8 +1,10 @@ # pylint: disable=no-self-use from unittest import mock import pytest + from allocation.adapters import repository -from allocation.service_layer import services, unit_of_work +from allocation.domain import events +from allocation.service_layer import handlers, messagebus, unit_of_work class FakeRepository(repository.AbstractRepository): @@ -36,49 +38,64 @@ class TestAddBatch: def test_for_new_product(self): uow = FakeUnitOfWork() - services.add_batch("b1", "CRUNCHY-ARMCHAIR", 100, None, uow) + messagebus.handle( + events.BatchCreated("b1", "CRUNCHY-ARMCHAIR", 100, None), uow + ) assert uow.products.get("CRUNCHY-ARMCHAIR") is not None assert uow.committed def test_for_existing_product(self): uow = FakeUnitOfWork() - services.add_batch("b1", "GARISH-RUG", 100, None, uow) - services.add_batch("b2", "GARISH-RUG", 99, None, uow) + messagebus.handle(events.BatchCreated("b1", "GARISH-RUG", 100, None), uow) + messagebus.handle(events.BatchCreated("b2", "GARISH-RUG", 99, None), uow) assert "b2" in [b.reference for b in uow.products.get("GARISH-RUG").batches] + class TestAllocate: def test_returns_allocation(self): uow = FakeUnitOfWork() - services.add_batch("batch1", "COMPLICATED-LAMP", 100, None, uow) - result = services.allocate("o1", "COMPLICATED-LAMP", 10, uow) + messagebus.handle( + events.BatchCreated("batch1", "COMPLICATED-LAMP", 100, None), uow + ) + result = messagebus.handle( + events.AllocationRequired("o1", "COMPLICATED-LAMP", 10), uow + ) assert result == "batch1" def test_errors_for_invalid_sku(self): uow = FakeUnitOfWork() - services.add_batch("b1", "AREALSKU", 100, None, uow) - - with pytest.raises(services.InvalidSku, match="Invalid sku NONEXISTENTSKU"): - services.allocate("o1", "NONEXISTENTSKU", 10, uow) + messagebus.handle(events.BatchCreated("b1", "AREALSKU", 100, None), uow) + with pytest.raises(handlers.InvalidSku, match="Invalid sku NONEXISTENTSKU"): + messagebus.handle( + events.AllocationRequired("o1", "NONEXISTENTSKU", 10), uow + ) def test_commits(self): uow = FakeUnitOfWork() - services.add_batch("b1", "OMINOUS-MIRROR", 100, None, uow) - services.allocate("o1", "OMINOUS-MIRROR", 10, uow) + messagebus.handle( + events.BatchCreated("b1", "OMINOUS-MIRROR", 100, None), uow + ) + messagebus.handle( + events.AllocationRequired("o1", "OMINOUS-MIRROR", 10), uow + ) assert uow.committed def test_sends_email_on_out_of_stock_error(self): uow = FakeUnitOfWork() - services.add_batch("b1", "POPULAR-CURTAINS", 9, None, uow) + messagebus.handle( + events.BatchCreated("b1", "POPULAR-CURTAINS", 9, None), uow + ) - with mock.patch("allocation.adapters.email.send_mail") as mock_send_mail: - services.allocate("o1", "POPULAR-CURTAINS", 10, uow) + with mock.patch("allocation.adapters.email.send") as mock_send_mail: + messagebus.handle( + events.AllocationRequired("o1", "POPULAR-CURTAINS", 10), uow + ) assert mock_send_mail.call_args == mock.call( - "stock@made.com", - f"Out of stock for POPULAR-CURTAINS", + "stock@made.com", f"Out of stock for POPULAR-CURTAINS" ) From 9716fe08d27cf46c85cac5b342e22ecb5bc4f89f Mon Sep 17 00:00:00 2001 From: Harry Date: Fri, 3 Jan 2020 02:46:32 +0000 Subject: [PATCH 088/165] messagebus takes a uow [handle_takes_uow] --- src/allocation/service_layer/messagebus.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/allocation/service_layer/messagebus.py b/src/allocation/service_layer/messagebus.py index d1d5997b..f61d9cc6 100644 --- a/src/allocation/service_layer/messagebus.py +++ b/src/allocation/service_layer/messagebus.py @@ -1,11 +1,14 @@ -from typing import List, Dict, Callable, Type +from __future__ import annotations +from typing import List, Dict, Callable, Type, TYPE_CHECKING from allocation.domain import events from . import handlers +if TYPE_CHECKING: + from allocation.service_layer import unit_of_work -def handle(event: events.Event): +def handle(event: events.Event, uow: unit_of_work.AbstractUnitOfWork): for handler in HANDLERS[type(event)]: - handler(event) + handler(event, uow=uow) HANDLERS = { From 75ee8c6d03cb1751692a3422db9b85d049d94eaa Mon Sep 17 00:00:00 2001 From: Harry Date: Mon, 6 Jan 2020 22:07:35 +0000 Subject: [PATCH 089/165] messagebus has uow, manages queue [handle_has_uow_and_queue] --- src/allocation/service_layer/messagebus.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/allocation/service_layer/messagebus.py b/src/allocation/service_layer/messagebus.py index f61d9cc6..61742b10 100644 --- a/src/allocation/service_layer/messagebus.py +++ b/src/allocation/service_layer/messagebus.py @@ -3,12 +3,16 @@ from allocation.domain import events from . import handlers if TYPE_CHECKING: - from allocation.service_layer import unit_of_work + from . import unit_of_work def handle(event: events.Event, uow: unit_of_work.AbstractUnitOfWork): - for handler in HANDLERS[type(event)]: - handler(event, uow=uow) + queue = [event] + while queue: + event = queue.pop(0) + for handler in HANDLERS[type(event)]: + handler(event, uow=uow) + queue.extend(uow.collect_new_events()) HANDLERS = { From 4b2000e81d9e2291b844046385f73ae64d2f945c Mon Sep 17 00:00:00 2001 From: Harry Date: Mon, 6 Jan 2020 22:08:03 +0000 Subject: [PATCH 090/165] Uow no longer puts events directly on the bus [uow_collect_new_events] --- src/allocation/service_layer/unit_of_work.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/allocation/service_layer/unit_of_work.py b/src/allocation/service_layer/unit_of_work.py index 616e7ec0..db55084f 100644 --- a/src/allocation/service_layer/unit_of_work.py +++ b/src/allocation/service_layer/unit_of_work.py @@ -8,8 +8,6 @@ from allocation import config from allocation.adapters import repository -from . import messagebus - class AbstractUnitOfWork(abc.ABC): @@ -23,13 +21,11 @@ def __exit__(self, *args): def commit(self): self._commit() - self.publish_events() - def publish_events(self): + def collect_new_events(self): for product in self.products.seen: while product.events: - event = product.events.pop(0) - messagebus.handle(event) + yield product.events.pop(0) @abc.abstractmethod def _commit(self): From c6d5aae3d5bdc4da9fa85c8698a84c7ecb3b19f7 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 6 Aug 2019 12:57:30 +0100 Subject: [PATCH 091/165] Ugly hack for messagebus to return results [hack_messagebus_results] --- src/allocation/service_layer/messagebus.py | 4 +++- tests/unit/test_handlers.py | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/allocation/service_layer/messagebus.py b/src/allocation/service_layer/messagebus.py index 61742b10..bef6d35a 100644 --- a/src/allocation/service_layer/messagebus.py +++ b/src/allocation/service_layer/messagebus.py @@ -7,12 +7,14 @@ def handle(event: events.Event, uow: unit_of_work.AbstractUnitOfWork): + results = [] queue = [event] while queue: event = queue.pop(0) for handler in HANDLERS[type(event)]: - handler(event, uow=uow) + results.append(handler(event, uow=uow)) queue.extend(uow.collect_new_events()) + return results HANDLERS = { diff --git a/tests/unit/test_handlers.py b/tests/unit/test_handlers.py index 9e967d59..4752cebc 100644 --- a/tests/unit/test_handlers.py +++ b/tests/unit/test_handlers.py @@ -60,10 +60,10 @@ def test_returns_allocation(self): messagebus.handle( events.BatchCreated("batch1", "COMPLICATED-LAMP", 100, None), uow ) - result = messagebus.handle( + results = messagebus.handle( events.AllocationRequired("o1", "COMPLICATED-LAMP", 10), uow ) - assert result == "batch1" + assert results.pop(0) == "batch1" def test_errors_for_invalid_sku(self): From 33c4089dd4fc7055368306cc2518e9f430b5ce37 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 6 Aug 2019 12:33:23 +0100 Subject: [PATCH 092/165] modify flask to use messagebus [flask_uses_messagebus] --- src/allocation/entrypoints/flask_app.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/allocation/entrypoints/flask_app.py b/src/allocation/entrypoints/flask_app.py index 69614824..bccf2d28 100644 --- a/src/allocation/entrypoints/flask_app.py +++ b/src/allocation/entrypoints/flask_app.py @@ -1,9 +1,10 @@ from datetime import datetime from flask import Flask, jsonify, request +from allocation.domain import events from allocation.adapters import orm -from allocation.service_layer import services, unit_of_work -from allocation.service_layer.services import InvalidSku +from allocation.service_layer import messagebus, unit_of_work +from allocation.service_layer.handlers import InvalidSku app = Flask(__name__) orm.start_mappers() @@ -14,22 +15,21 @@ def add_batch(): eta = request.json['eta'] if eta is not None: eta = datetime.fromisoformat(eta).date() - services.add_batch( + event = events.BatchCreated( request.json['ref'], request.json['sku'], request.json['qty'], eta, - unit_of_work.SqlAlchemyUnitOfWork(), ) + messagebus.handle(event, unit_of_work.SqlAlchemyUnitOfWork()) return 'OK', 201 @app.route("/allocate", methods=['POST']) def allocate_endpoint(): try: - batchref = services.allocate( - request.json['orderid'], - request.json['sku'], - request.json['qty'], - unit_of_work.SqlAlchemyUnitOfWork(), + event = events.AllocationRequired( + request.json['orderid'], request.json['sku'], request.json['qty'], ) + results = messagebus.handle(event, unit_of_work.SqlAlchemyUnitOfWork()) + batchref = results.pop(0) except InvalidSku as e: return jsonify({'message': str(e)}), 400 From a5f6c136e3b5fe500a5d53b76bb61f22c66dbde7 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 6 Aug 2019 13:27:32 +0100 Subject: [PATCH 093/165] new event for new input [batch_quantity_changed_event] --- src/allocation/domain/events.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/allocation/domain/events.py b/src/allocation/domain/events.py index d59e66b5..5d24ffc9 100644 --- a/src/allocation/domain/events.py +++ b/src/allocation/domain/events.py @@ -13,6 +13,11 @@ class BatchCreated(Event): qty: int eta: Optional[date] = None +@dataclass +class BatchQuantityChanged(Event): + ref: str + qty: int + @dataclass class AllocationRequired(Event): orderid: str From c4d64c381d78dafea164d1e27e85f7ac704a54f5 Mon Sep 17 00:00:00 2001 From: Harry Date: Sat, 25 May 2019 10:11:42 +0100 Subject: [PATCH 094/165] new test and put them into classes [test_change_batch_quantity_handler] --- tests/unit/test_handlers.py | 40 ++++++++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_handlers.py b/tests/unit/test_handlers.py index 4752cebc..173f6fb5 100644 --- a/tests/unit/test_handlers.py +++ b/tests/unit/test_handlers.py @@ -1,4 +1,5 @@ # pylint: disable=no-self-use +from datetime import date from unittest import mock import pytest @@ -52,7 +53,6 @@ def test_for_existing_product(self): assert "b2" in [b.reference for b in uow.products.get("GARISH-RUG").batches] - class TestAllocate: def test_returns_allocation(self): @@ -99,3 +99,41 @@ def test_sends_email_on_out_of_stock_error(self): assert mock_send_mail.call_args == mock.call( "stock@made.com", f"Out of stock for POPULAR-CURTAINS" ) + + + +class TestChangeBatchQuantity: + + def test_changes_available_quantity(self): + uow = FakeUnitOfWork() + messagebus.handle( + events.BatchCreated("batch1", "ADORABLE-SETTEE", 100, None), uow + ) + [batch] = uow.products.get(sku="ADORABLE-SETTEE").batches + assert batch.available_quantity == 100 + + messagebus.handle(events.BatchQuantityChanged("batch1", 50), uow) + + assert batch.available_quantity == 50 + + + def test_reallocates_if_necessary(self): + uow = FakeUnitOfWork() + event_history = [ + events.BatchCreated("batch1", "INDIFFERENT-TABLE", 50, None), + events.BatchCreated("batch2", "INDIFFERENT-TABLE", 50, date.today()), + events.AllocationRequired("order1", "INDIFFERENT-TABLE", 20), + events.AllocationRequired("order2", "INDIFFERENT-TABLE", 20), + ] + for e in event_history: + messagebus.handle(e, uow) + [batch1, batch2] = uow.products.get(sku="INDIFFERENT-TABLE").batches + assert batch1.available_quantity == 10 + assert batch2.available_quantity == 50 + + messagebus.handle(events.BatchQuantityChanged("batch1", 25), uow) + + # order1 or order2 will be deallocated, so we'll have 25 - 20 + assert batch1.available_quantity == 5 + # and 20 will be reallocated to the next batch + assert batch2.available_quantity == 30 From 7743a664b564e76586c2edf7041a3a73b52c9445 Mon Sep 17 00:00:00 2001 From: Harry Date: Sat, 25 May 2019 12:14:35 +0100 Subject: [PATCH 095/165] start on handler for change quantity [change_quantity_handler] --- src/allocation/service_layer/handlers.py | 12 +++++++++++- src/allocation/service_layer/messagebus.py | 2 ++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/allocation/service_layer/handlers.py b/src/allocation/service_layer/handlers.py index 8eaf4e3b..3e3bd361 100644 --- a/src/allocation/service_layer/handlers.py +++ b/src/allocation/service_layer/handlers.py @@ -1,5 +1,6 @@ from __future__ import annotations from typing import TYPE_CHECKING +from allocation.adapters import email from allocation.domain import events, model from allocation.domain.model import OrderLine if TYPE_CHECKING: @@ -10,7 +11,6 @@ class InvalidSku(Exception): pass - def add_batch( event: events.BatchCreated, uow: unit_of_work.AbstractUnitOfWork ): @@ -38,6 +38,16 @@ def allocate( return batchref + +def change_batch_quantity( + event: events.BatchQuantityChanged, uow: unit_of_work.AbstractUnitOfWork +): + with uow: + product = uow.products.get_by_batchref(batchref=event.ref) + product.change_batch_quantity(ref=event.ref, qty=event.qty) + uow.commit() + + # pylint: disable=unused-argument def send_out_of_stock_notification( diff --git a/src/allocation/service_layer/messagebus.py b/src/allocation/service_layer/messagebus.py index bef6d35a..7514816e 100644 --- a/src/allocation/service_layer/messagebus.py +++ b/src/allocation/service_layer/messagebus.py @@ -19,6 +19,8 @@ def handle(event: events.Event, uow: unit_of_work.AbstractUnitOfWork): HANDLERS = { events.BatchCreated: [handlers.add_batch], + events.BatchQuantityChanged: [handlers.change_batch_quantity], events.AllocationRequired: [handlers.allocate], events.OutOfStock: [handlers.send_out_of_stock_notification], + } # type: Dict[Type[events.Event], List[Callable]] From bca49d68b866cfa6fa31b08383392786201a1028 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 28 May 2019 10:40:46 +0100 Subject: [PATCH 096/165] get_by_batchref in abstract+real repo, and a new integration test for it [get_by_batchref] --- src/allocation/adapters/repository.py | 18 ++++++++++++++++++ tests/integration/test_repository.py | 14 ++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 tests/integration/test_repository.py diff --git a/src/allocation/adapters/repository.py b/src/allocation/adapters/repository.py index 33a2bc0f..bda90c07 100644 --- a/src/allocation/adapters/repository.py +++ b/src/allocation/adapters/repository.py @@ -1,8 +1,10 @@ import abc from typing import Set +from allocation.adapters import orm from allocation.domain import model + class AbstractRepository(abc.ABC): def __init__(self): @@ -18,6 +20,12 @@ def get(self, sku) -> model.Product: self.seen.add(product) return product + def get_by_batchref(self, batchref) -> model.Product: + product = self._get_by_batchref(batchref) + if product: + self.seen.add(product) + return product + @abc.abstractmethod def _add(self, product: model.Product): raise NotImplementedError @@ -26,6 +34,11 @@ def _add(self, product: model.Product): def _get(self, sku) -> model.Product: raise NotImplementedError + @abc.abstractmethod + def _get_by_batchref(self, batchref) -> model.Product: + raise NotImplementedError + + class SqlAlchemyRepository(AbstractRepository): @@ -39,3 +52,8 @@ def _add(self, product): def _get(self, sku): return self.session.query(model.Product).filter_by(sku=sku).first() + + def _get_by_batchref(self, batchref): + return self.session.query(model.Product).join(model.Batch).filter( + orm.batches.c.reference == batchref, + ).first() diff --git a/tests/integration/test_repository.py b/tests/integration/test_repository.py new file mode 100644 index 00000000..733346c3 --- /dev/null +++ b/tests/integration/test_repository.py @@ -0,0 +1,14 @@ +from allocation.adapters import repository +from allocation.domain import model + +def test_get_by_batchref(session): + repo = repository.SqlAlchemyRepository(session) + b1 = model.Batch(ref='b1', sku='sku1', qty=100, eta=None) + b2 = model.Batch(ref='b2', sku='sku1', qty=100, eta=None) + b3 = model.Batch(ref='b3', sku='sku2', qty=100, eta=None) + p1 = model.Product(sku='sku1', batches=[b1, b2]) + p2 = model.Product(sku='sku2', batches=[b3]) + repo.add(p1) + repo.add(p2) + assert repo.get_by_batchref('b2') == p1 + assert repo.get_by_batchref('b3') == p2 From cb293795c50732060071a32ab093135361f60fb5 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 28 May 2019 10:41:12 +0100 Subject: [PATCH 097/165] fake repo get_by_batchref [fakerepo_get_by_batchref] --- tests/unit/test_handlers.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/unit/test_handlers.py b/tests/unit/test_handlers.py index 173f6fb5..9373228d 100644 --- a/tests/unit/test_handlers.py +++ b/tests/unit/test_handlers.py @@ -20,6 +20,12 @@ def _add(self, product): def _get(self, sku): return next((p for p in self._products if p.sku == sku), None) + def _get_by_batchref(self, batchref): + return next(( + p for p in self._products for b in p.batches + if b.reference == batchref + ), None) + class FakeUnitOfWork(unit_of_work.AbstractUnitOfWork): From b41eca7afab3b40fd530c2842399494642162971 Mon Sep 17 00:00:00 2001 From: Harry Date: Sat, 25 May 2019 18:01:26 +0100 Subject: [PATCH 098/165] change_batch_quantity on product, needed change to batch, also emit allocated event. [change_batch_model_layer] --- src/allocation/domain/model.py | 13 ++++++++++--- tests/unit/test_batches.py | 12 ------------ 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/src/allocation/domain/model.py b/src/allocation/domain/model.py index a6b84c4e..dd1c4c22 100644 --- a/src/allocation/domain/model.py +++ b/src/allocation/domain/model.py @@ -25,6 +25,14 @@ def allocate(self, line: OrderLine) -> str: self.events.append(events.OutOfStock(line.sku)) return None + def change_batch_quantity(self, ref: str, qty: int): + batch = next(b for b in self.batches if b.reference == ref) + batch._purchased_quantity = qty + while batch.available_quantity < 0: + line = batch.deallocate_one() + self.events.append( + events.AllocationRequired(line.orderid, line.sku, line.qty) + ) @dataclass(unsafe_hash=True) class OrderLine: @@ -65,9 +73,8 @@ def allocate(self, line: OrderLine): if self.can_allocate(line): self._allocations.add(line) - def deallocate(self, line: OrderLine): - if line in self._allocations: - self._allocations.remove(line) + def deallocate_one(self) -> OrderLine: + return self._allocations.pop() @property def allocated_quantity(self) -> int: diff --git a/tests/unit/test_batches.py b/tests/unit/test_batches.py index 099d37b4..39917dd1 100644 --- a/tests/unit/test_batches.py +++ b/tests/unit/test_batches.py @@ -39,15 +39,3 @@ def test_allocation_is_idempotent(): batch.allocate(line) batch.allocate(line) assert batch.available_quantity == 18 - -def test_deallocate(): - batch, line = make_batch_and_line("EXPENSIVE-FOOTSTOOL", 20, 2) - batch.allocate(line) - batch.deallocate(line) - assert batch.available_quantity == 20 - -def test_can_only_deallocate_allocated_lines(): - batch, unallocated_line = make_batch_and_line("DECORATIVE-TRINKET", 20, 2) - batch.deallocate(unallocated_line) - assert batch.available_quantity == 20 - From 9faf06db567afdaec79a2c66012559acabdc0720 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 29 Oct 2019 12:37:46 +0000 Subject: [PATCH 099/165] sort-of fake messagebus on fake-uow. [fake_messagebus] --- tests/unit/test_handlers.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/unit/test_handlers.py b/tests/unit/test_handlers.py index 9373228d..cfe9a38c 100644 --- a/tests/unit/test_handlers.py +++ b/tests/unit/test_handlers.py @@ -1,6 +1,7 @@ # pylint: disable=no-self-use from datetime import date from unittest import mock +from typing import List import pytest from allocation.adapters import repository @@ -143,3 +144,15 @@ def test_reallocates_if_necessary(self): assert batch1.available_quantity == 5 # and 20 will be reallocated to the next batch assert batch2.available_quantity == 30 + + +class FakeUnitOfWorkWithFakeMessageBus(FakeUnitOfWork): + + def __init__(self): + super().__init__() + self.events_published = [] # type: List[events.Event] + + def publish_events(self): + for product in self.products.seen: + while product.events: + self.events_published.append(product.events.pop(0)) From 0cc6021911b9337377e47dd439acd569a305aefd Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 29 Oct 2019 12:39:06 +0000 Subject: [PATCH 100/165] isolated test for a handler [test_handler_in_isolation] --- tests/unit/test_handlers.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/unit/test_handlers.py b/tests/unit/test_handlers.py index cfe9a38c..b1f0c0a0 100644 --- a/tests/unit/test_handlers.py +++ b/tests/unit/test_handlers.py @@ -156,3 +156,29 @@ def publish_events(self): for product in self.products.seen: while product.events: self.events_published.append(product.events.pop(0)) + + +def test_reallocates_if_necessary_isolated(): + uow = FakeUnitOfWorkWithFakeMessageBus() + + # test setup as before + event_history = [ + events.BatchCreated("batch1", "INDIFFERENT-TABLE", 50, None), + events.BatchCreated("batch2", "INDIFFERENT-TABLE", 50, date.today()), + events.AllocationRequired("order1", "INDIFFERENT-TABLE", 20), + events.AllocationRequired("order2", "INDIFFERENT-TABLE", 20), + ] + for e in event_history: + messagebus.handle(e, uow) + [batch1, batch2] = uow.products.get(sku="INDIFFERENT-TABLE").batches + assert batch1.available_quantity == 10 + assert batch2.available_quantity == 50 + + messagebus.handle(events.BatchQuantityChanged("batch1", 25), uow) + + # assert on new events emitted rather than downstream side-effects + [reallocation_event] = uow.events_published + assert isinstance(reallocation_event, events.AllocationRequired) + assert reallocation_event.orderid in {'order1', 'order2'} + assert reallocation_event.sku == 'INDIFFERENT-TABLE' + From db8f9ae0fa0cadb575f5328e16e7487edb2a466a Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 29 Oct 2019 12:40:06 +0000 Subject: [PATCH 101/165] Revert fake messagebus stuff [chapter_09_all_messagebus_ends] This reverts commit 8f2c36f43fbbc9b4cfa42f9ee75acb4f486d4342. This reverts commit d3988118e478db2748d7b41d3ad70c6a22ac9f9f. --- tests/unit/test_handlers.py | 39 ------------------------------------- 1 file changed, 39 deletions(-) diff --git a/tests/unit/test_handlers.py b/tests/unit/test_handlers.py index b1f0c0a0..9373228d 100644 --- a/tests/unit/test_handlers.py +++ b/tests/unit/test_handlers.py @@ -1,7 +1,6 @@ # pylint: disable=no-self-use from datetime import date from unittest import mock -from typing import List import pytest from allocation.adapters import repository @@ -144,41 +143,3 @@ def test_reallocates_if_necessary(self): assert batch1.available_quantity == 5 # and 20 will be reallocated to the next batch assert batch2.available_quantity == 30 - - -class FakeUnitOfWorkWithFakeMessageBus(FakeUnitOfWork): - - def __init__(self): - super().__init__() - self.events_published = [] # type: List[events.Event] - - def publish_events(self): - for product in self.products.seen: - while product.events: - self.events_published.append(product.events.pop(0)) - - -def test_reallocates_if_necessary_isolated(): - uow = FakeUnitOfWorkWithFakeMessageBus() - - # test setup as before - event_history = [ - events.BatchCreated("batch1", "INDIFFERENT-TABLE", 50, None), - events.BatchCreated("batch2", "INDIFFERENT-TABLE", 50, date.today()), - events.AllocationRequired("order1", "INDIFFERENT-TABLE", 20), - events.AllocationRequired("order2", "INDIFFERENT-TABLE", 20), - ] - for e in event_history: - messagebus.handle(e, uow) - [batch1, batch2] = uow.products.get(sku="INDIFFERENT-TABLE").batches - assert batch1.available_quantity == 10 - assert batch2.available_quantity == 50 - - messagebus.handle(events.BatchQuantityChanged("batch1", 25), uow) - - # assert on new events emitted rather than downstream side-effects - [reallocation_event] = uow.events_published - assert isinstance(reallocation_event, events.AllocationRequired) - assert reallocation_event.orderid in {'order1', 'order2'} - assert reallocation_event.sku == 'INDIFFERENT-TABLE' - From 8f6cea3e3fe9992525e42f72f92203a9b7082b64 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 13 Aug 2019 17:12:54 +0100 Subject: [PATCH 102/165] add commands [commands_dot_py] --- src/allocation/domain/commands.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 src/allocation/domain/commands.py diff --git a/src/allocation/domain/commands.py b/src/allocation/domain/commands.py new file mode 100644 index 00000000..c4f8a25c --- /dev/null +++ b/src/allocation/domain/commands.py @@ -0,0 +1,25 @@ +# pylint: disable=too-few-public-methods +from datetime import date +from typing import Optional +from dataclasses import dataclass + +class Command: + pass + +@dataclass +class Allocate(Command): + orderid: str + sku: str + qty: int + +@dataclass +class CreateBatch(Command): + ref: str + sku: str + qty: int + eta: Optional[date] = None + +@dataclass +class ChangeBatchQuantity(Command): + ref: str + qty: int From ff19fe3a20e630c8b5c7f904539a30186bb257c4 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 7 Jan 2020 12:44:24 +0000 Subject: [PATCH 103/165] top-level messagebus.handle dispatches events vs comannds [messagebus_dispatches_differently] --- src/allocation/service_layer/messagebus.py | 27 +++++++++++++++------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/src/allocation/service_layer/messagebus.py b/src/allocation/service_layer/messagebus.py index 7514816e..76522ec7 100644 --- a/src/allocation/service_layer/messagebus.py +++ b/src/allocation/service_layer/messagebus.py @@ -1,19 +1,30 @@ +# pylint: disable=bare-except from __future__ import annotations -from typing import List, Dict, Callable, Type, TYPE_CHECKING -from allocation.domain import events +import logging +from typing import List, Dict, Callable, Type, Union, TYPE_CHECKING +from allocation.domain import commands, events from . import handlers + if TYPE_CHECKING: from . import unit_of_work +logger = logging.getLogger(__name__) + +Message = Union[commands.Command, events.Event] + -def handle(event: events.Event, uow: unit_of_work.AbstractUnitOfWork): +def handle(message: Message, uow: unit_of_work.AbstractUnitOfWork): results = [] - queue = [event] + queue = [message] while queue: - event = queue.pop(0) - for handler in HANDLERS[type(event)]: - results.append(handler(event, uow=uow)) - queue.extend(uow.collect_new_events()) + message = queue.pop(0) + if isinstance(message, events.Event): + handle_event(message, queue, uow) + elif isinstance(message, commands.Command): + cmd_result = handle_command(message, queue, uow) + results.append(cmd_result) + else: + raise Exception(f'{message} was not an Event or Command') return results From 8fa768c0ca0c08f3c84409a91487d6bce77b743c Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 7 Jan 2020 14:33:15 +0000 Subject: [PATCH 104/165] subhandler for events [handle_event] --- src/allocation/service_layer/messagebus.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/allocation/service_layer/messagebus.py b/src/allocation/service_layer/messagebus.py index 76522ec7..54a039a2 100644 --- a/src/allocation/service_layer/messagebus.py +++ b/src/allocation/service_layer/messagebus.py @@ -28,6 +28,21 @@ def handle(message: Message, uow: unit_of_work.AbstractUnitOfWork): return results +def handle_event( + event: events.Event, + queue: List[Message], + uow: unit_of_work.AbstractUnitOfWork +): + for handler in EVENT_HANDLERS[type(event)]: + try: + logger.debug('handling event %s with handler %s', event, handler) + handler(event, uow=uow) + queue.extend(uow.collect_new_events()) + except Exception: + logger.exception('Exception handling event %s', event) + continue + + HANDLERS = { events.BatchCreated: [handlers.add_batch], events.BatchQuantityChanged: [handlers.change_batch_quantity], From 53a3da050240238f749a25c58352e2f8d83cc8c5 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 7 Jan 2020 14:33:45 +0000 Subject: [PATCH 105/165] subhandler for commands [handle_command] --- src/allocation/service_layer/messagebus.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/allocation/service_layer/messagebus.py b/src/allocation/service_layer/messagebus.py index 54a039a2..da36560c 100644 --- a/src/allocation/service_layer/messagebus.py +++ b/src/allocation/service_layer/messagebus.py @@ -1,4 +1,4 @@ -# pylint: disable=bare-except +# pylint: disable=broad-except from __future__ import annotations import logging from typing import List, Dict, Callable, Type, Union, TYPE_CHECKING @@ -43,6 +43,22 @@ def handle_event( continue +def handle_command( + command: commands.Command, + queue: List[Message], + uow: unit_of_work.AbstractUnitOfWork +): + logger.debug('handling command %s', command) + try: + handler = COMMAND_HANDLERS[type(command)] + result = handler(command, uow=uow) + queue.extend(uow.collect_new_events()) + return result + except Exception: + logger.exception('Exception handling command %s', command) + raise + + HANDLERS = { events.BatchCreated: [handlers.add_batch], events.BatchQuantityChanged: [handlers.change_batch_quantity], From e1af6839c169f4708f11d7ad2d91c75e293bfed9 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 7 Jan 2020 12:45:59 +0000 Subject: [PATCH 106/165] handlers dicts split in two [two_hander_dicts] --- src/allocation/service_layer/messagebus.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/allocation/service_layer/messagebus.py b/src/allocation/service_layer/messagebus.py index da36560c..a0a674c4 100644 --- a/src/allocation/service_layer/messagebus.py +++ b/src/allocation/service_layer/messagebus.py @@ -59,10 +59,12 @@ def handle_command( raise -HANDLERS = { - events.BatchCreated: [handlers.add_batch], - events.BatchQuantityChanged: [handlers.change_batch_quantity], - events.AllocationRequired: [handlers.allocate], +EVENT_HANDLERS = { events.OutOfStock: [handlers.send_out_of_stock_notification], - } # type: Dict[Type[events.Event], List[Callable]] + +COMMAND_HANDLERS = { + commands.Allocate: handlers.allocate, + commands.CreateBatch: handlers.add_batch, + commands.ChangeBatchQuantity: handlers.change_batch_quantity, +} # type: Dict[Type[commands.Command], Callable] From ab409fdd75033e29991fefddf2e6f69e0cc18bf2 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 13 Aug 2019 17:32:38 +0100 Subject: [PATCH 107/165] remove old events --- src/allocation/domain/events.py | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/src/allocation/domain/events.py b/src/allocation/domain/events.py index 5d24ffc9..b00ad7e7 100644 --- a/src/allocation/domain/events.py +++ b/src/allocation/domain/events.py @@ -1,29 +1,9 @@ # pylint: disable=too-few-public-methods from dataclasses import dataclass -from datetime import date -from typing import Optional class Event: pass -@dataclass -class BatchCreated(Event): - ref: str - sku: str - qty: int - eta: Optional[date] = None - -@dataclass -class BatchQuantityChanged(Event): - ref: str - qty: int - -@dataclass -class AllocationRequired(Event): - orderid: str - sku: str - qty: int - @dataclass class OutOfStock(Event): sku: str From 53a99b3c5701a57ed955fa9f3e551ea0caf7620c Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 13 Aug 2019 17:37:40 +0100 Subject: [PATCH 108/165] handlers take commands now, modify tests too --- src/allocation/service_layer/handlers.py | 23 +++++------ tests/unit/test_handlers.py | 49 ++++++++++++------------ 2 files changed, 35 insertions(+), 37 deletions(-) diff --git a/src/allocation/service_layer/handlers.py b/src/allocation/service_layer/handlers.py index 3e3bd361..85fb5e03 100644 --- a/src/allocation/service_layer/handlers.py +++ b/src/allocation/service_layer/handlers.py @@ -1,7 +1,7 @@ from __future__ import annotations from typing import TYPE_CHECKING from allocation.adapters import email -from allocation.domain import events, model +from allocation.domain import commands, events, model from allocation.domain.model import OrderLine if TYPE_CHECKING: from . import unit_of_work @@ -12,23 +12,23 @@ class InvalidSku(Exception): def add_batch( - event: events.BatchCreated, uow: unit_of_work.AbstractUnitOfWork + cmd: commands.CreateBatch, uow: unit_of_work.AbstractUnitOfWork ): with uow: - product = uow.products.get(sku=event.sku) + product = uow.products.get(sku=cmd.sku) if product is None: - product = model.Product(event.sku, batches=[]) + product = model.Product(cmd.sku, batches=[]) uow.products.add(product) product.batches.append(model.Batch( - event.ref, event.sku, event.qty, event.eta + cmd.ref, cmd.sku, cmd.qty, cmd.eta )) uow.commit() def allocate( - event: events.AllocationRequired, uow: unit_of_work.AbstractUnitOfWork + cmd: commands.Allocate, uow: unit_of_work.AbstractUnitOfWork ) -> str: - line = OrderLine(event.orderid, event.sku, event.qty) + line = OrderLine(cmd.orderid, cmd.sku, cmd.qty) with uow: product = uow.products.get(sku=line.sku) if product is None: @@ -38,18 +38,15 @@ def allocate( return batchref - def change_batch_quantity( - event: events.BatchQuantityChanged, uow: unit_of_work.AbstractUnitOfWork + cmd: commands.ChangeBatchQuantity, uow: unit_of_work.AbstractUnitOfWork ): with uow: - product = uow.products.get_by_batchref(batchref=event.ref) - product.change_batch_quantity(ref=event.ref, qty=event.qty) + product = uow.products.get_by_batchref(batchref=cmd.ref) + product.change_batch_quantity(ref=cmd.ref, qty=cmd.qty) uow.commit() -# pylint: disable=unused-argument - def send_out_of_stock_notification( event: events.OutOfStock, uow: unit_of_work.AbstractUnitOfWork, ): diff --git a/tests/unit/test_handlers.py b/tests/unit/test_handlers.py index 9373228d..1fae8c47 100644 --- a/tests/unit/test_handlers.py +++ b/tests/unit/test_handlers.py @@ -2,9 +2,8 @@ from datetime import date from unittest import mock import pytest - from allocation.adapters import repository -from allocation.domain import events +from allocation.domain import commands, events from allocation.service_layer import handlers, messagebus, unit_of_work @@ -46,7 +45,7 @@ class TestAddBatch: def test_for_new_product(self): uow = FakeUnitOfWork() messagebus.handle( - events.BatchCreated("b1", "CRUNCHY-ARMCHAIR", 100, None), uow + commands.CreateBatch("b1", "CRUNCHY-ARMCHAIR", 100, None), uow ) assert uow.products.get("CRUNCHY-ARMCHAIR") is not None assert uow.committed @@ -54,40 +53,42 @@ def test_for_new_product(self): def test_for_existing_product(self): uow = FakeUnitOfWork() - messagebus.handle(events.BatchCreated("b1", "GARISH-RUG", 100, None), uow) - messagebus.handle(events.BatchCreated("b2", "GARISH-RUG", 99, None), uow) + messagebus.handle(commands.CreateBatch("b1", "GARISH-RUG", 100, None), uow) + messagebus.handle(commands.CreateBatch("b2", "GARISH-RUG", 99, None), uow) assert "b2" in [b.reference for b in uow.products.get("GARISH-RUG").batches] class TestAllocate: - def test_returns_allocation(self): + def test_allocates(self): uow = FakeUnitOfWork() messagebus.handle( - events.BatchCreated("batch1", "COMPLICATED-LAMP", 100, None), uow + commands.CreateBatch("batch1", "COMPLICATED-LAMP", 100, None), uow ) results = messagebus.handle( - events.AllocationRequired("o1", "COMPLICATED-LAMP", 10), uow + commands.Allocate("o1", "COMPLICATED-LAMP", 10), uow ) assert results.pop(0) == "batch1" + [batch] = uow.products.get("COMPLICATED-LAMP").batches + assert batch.available_quantity == 90 def test_errors_for_invalid_sku(self): uow = FakeUnitOfWork() - messagebus.handle(events.BatchCreated("b1", "AREALSKU", 100, None), uow) + messagebus.handle(commands.CreateBatch("b1", "AREALSKU", 100, None), uow) with pytest.raises(handlers.InvalidSku, match="Invalid sku NONEXISTENTSKU"): messagebus.handle( - events.AllocationRequired("o1", "NONEXISTENTSKU", 10), uow + commands.Allocate("o1", "NONEXISTENTSKU", 10), uow ) def test_commits(self): uow = FakeUnitOfWork() messagebus.handle( - events.BatchCreated("b1", "OMINOUS-MIRROR", 100, None), uow + commands.CreateBatch("b1", "OMINOUS-MIRROR", 100, None), uow ) messagebus.handle( - events.AllocationRequired("o1", "OMINOUS-MIRROR", 10), uow + commands.Allocate("o1", "OMINOUS-MIRROR", 10), uow ) assert uow.committed @@ -95,12 +96,12 @@ def test_commits(self): def test_sends_email_on_out_of_stock_error(self): uow = FakeUnitOfWork() messagebus.handle( - events.BatchCreated("b1", "POPULAR-CURTAINS", 9, None), uow + commands.CreateBatch("b1", "POPULAR-CURTAINS", 9, None), uow ) with mock.patch("allocation.adapters.email.send") as mock_send_mail: messagebus.handle( - events.AllocationRequired("o1", "POPULAR-CURTAINS", 10), uow + commands.Allocate("o1", "POPULAR-CURTAINS", 10), uow ) assert mock_send_mail.call_args == mock.call( "stock@made.com", f"Out of stock for POPULAR-CURTAINS" @@ -113,31 +114,31 @@ class TestChangeBatchQuantity: def test_changes_available_quantity(self): uow = FakeUnitOfWork() messagebus.handle( - events.BatchCreated("batch1", "ADORABLE-SETTEE", 100, None), uow + commands.CreateBatch("batch1", "ADORABLE-SETTEE", 100, None), uow ) [batch] = uow.products.get(sku="ADORABLE-SETTEE").batches assert batch.available_quantity == 100 - messagebus.handle(events.BatchQuantityChanged("batch1", 50), uow) + messagebus.handle(commands.ChangeBatchQuantity("batch1", 50), uow) assert batch.available_quantity == 50 def test_reallocates_if_necessary(self): uow = FakeUnitOfWork() - event_history = [ - events.BatchCreated("batch1", "INDIFFERENT-TABLE", 50, None), - events.BatchCreated("batch2", "INDIFFERENT-TABLE", 50, date.today()), - events.AllocationRequired("order1", "INDIFFERENT-TABLE", 20), - events.AllocationRequired("order2", "INDIFFERENT-TABLE", 20), + history = [ + commands.CreateBatch("batch1", "INDIFFERENT-TABLE", 50, None), + commands.CreateBatch("batch2", "INDIFFERENT-TABLE", 50, date.today()), + commands.Allocate("order1", "INDIFFERENT-TABLE", 20), + commands.Allocate("order2", "INDIFFERENT-TABLE", 20), ] - for e in event_history: - messagebus.handle(e, uow) + for msg in history: + messagebus.handle(msg, uow) [batch1, batch2] = uow.products.get(sku="INDIFFERENT-TABLE").batches assert batch1.available_quantity == 10 assert batch2.available_quantity == 50 - messagebus.handle(events.BatchQuantityChanged("batch1", 25), uow) + messagebus.handle(commands.ChangeBatchQuantity("batch1", 25), uow) # order1 or order2 will be deallocated, so we'll have 25 - 20 assert batch1.available_quantity == 5 From 734c05ef4bfcd0e285e9ea837a3b39c3bac0ce0c Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 13 Aug 2019 17:39:20 +0100 Subject: [PATCH 109/165] model method for change batch qty now raise commands hmmm --- src/allocation/domain/model.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allocation/domain/model.py b/src/allocation/domain/model.py index dd1c4c22..f797f6c0 100644 --- a/src/allocation/domain/model.py +++ b/src/allocation/domain/model.py @@ -2,7 +2,7 @@ from dataclasses import dataclass from datetime import date from typing import Optional, List, Set -from . import events +from . import commands, events class Product: @@ -31,7 +31,7 @@ def change_batch_quantity(self, ref: str, qty: int): while batch.available_quantity < 0: line = batch.deallocate_one() self.events.append( - events.AllocationRequired(line.orderid, line.sku, line.qty) + commands.Allocate(line.orderid, line.sku, line.qty) ) @dataclass(unsafe_hash=True) From c3da55bc7fc9031c0d7e4cbb9b9098b017cddb43 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 13 Aug 2019 17:39:28 +0100 Subject: [PATCH 110/165] flask now uses commands [chapter_10_commands_ends] --- src/allocation/entrypoints/flask_app.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/allocation/entrypoints/flask_app.py b/src/allocation/entrypoints/flask_app.py index bccf2d28..313544c3 100644 --- a/src/allocation/entrypoints/flask_app.py +++ b/src/allocation/entrypoints/flask_app.py @@ -1,7 +1,7 @@ from datetime import datetime from flask import Flask, jsonify, request -from allocation.domain import events +from allocation.domain import commands from allocation.adapters import orm from allocation.service_layer import messagebus, unit_of_work from allocation.service_layer.handlers import InvalidSku @@ -15,20 +15,22 @@ def add_batch(): eta = request.json['eta'] if eta is not None: eta = datetime.fromisoformat(eta).date() - event = events.BatchCreated( + cmd = commands.CreateBatch( request.json['ref'], request.json['sku'], request.json['qty'], eta, ) - messagebus.handle(event, unit_of_work.SqlAlchemyUnitOfWork()) + uow = unit_of_work.SqlAlchemyUnitOfWork() + messagebus.handle(cmd, uow) return 'OK', 201 @app.route("/allocate", methods=['POST']) def allocate_endpoint(): try: - event = events.AllocationRequired( + cmd = commands.Allocate( request.json['orderid'], request.json['sku'], request.json['qty'], ) - results = messagebus.handle(event, unit_of_work.SqlAlchemyUnitOfWork()) + uow = unit_of_work.SqlAlchemyUnitOfWork() + results = messagebus.handle(cmd, uow) batchref = results.pop(0) except InvalidSku as e: return jsonify({'message': str(e)}), 400 From 387ae088bea19d69115423018768b243aa5eb941 Mon Sep 17 00:00:00 2001 From: Harry Date: Fri, 26 Jul 2019 14:58:39 +0100 Subject: [PATCH 111/165] Add redis to docker-compose and config, conftest --- docker-compose.yml | 6 ++++++ mypy.ini | 2 +- requirements.txt | 1 + src/allocation/config.py | 5 +++++ tests/conftest.py | 29 +++++++++++++++++++++++++++-- 5 files changed, 40 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 039400e9..74ac28b6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,6 +7,7 @@ services: dockerfile: Dockerfile depends_on: - postgres + - redis environment: - DB_HOST=postgres - DB_PASSWORD=abc123 @@ -27,3 +28,8 @@ services: ports: - "54321:5432" + redis: + image: redis:alpine + ports: + - "63791:6379" + diff --git a/mypy.ini b/mypy.ini index 62194f35..601283d7 100644 --- a/mypy.ini +++ b/mypy.ini @@ -3,5 +3,5 @@ ignore_missing_imports = False mypy_path = ./src check_untyped_defs = True -[mypy-pytest.*,sqlalchemy.*] +[mypy-pytest.*,sqlalchemy.*,redis.*] ignore_missing_imports = True diff --git a/requirements.txt b/requirements.txt index a8906795..592da462 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,3 +9,4 @@ pytest-icdiff mypy requests +redis diff --git a/src/allocation/config.py b/src/allocation/config.py index 38296a14..efc65cf7 100644 --- a/src/allocation/config.py +++ b/src/allocation/config.py @@ -14,3 +14,8 @@ def get_api_url(): port = 5005 if host == 'localhost' else 80 return f"http://{host}:{port}" +def get_redis_host_and_port(): + host = os.environ.get('REDIS_HOST', 'localhost') + port = 63791 if host == 'localhost' else 6379 + return dict(host=host, port=port) + diff --git a/tests/conftest.py b/tests/conftest.py index c34d61b4..bd20a704 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,10 +1,14 @@ # pylint: disable=redefined-outer-name +import shutil +import subprocess import time from pathlib import Path import pytest +import redis import requests -from requests.exceptions import ConnectionError +from requests.exceptions import RequestException +from redis.exceptions import RedisError from sqlalchemy.exc import OperationalError from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker, clear_mappers @@ -46,11 +50,21 @@ def wait_for_webapp_to_come_up(): while time.time() < deadline: try: return requests.get(url) - except ConnectionError: + except RequestException: time.sleep(0.5) pytest.fail('API never came up') +def wait_for_redis_to_come_up(): + deadline = time.time() + 5 + r = redis.Redis(**config.get_redis_host_and_port()) + while time.time() < deadline: + try: + return r.ping() + except RedisError: + time.sleep(0.5) + pytest.fail('Redis never came up') + @pytest.fixture(scope='session') def postgres_db(): @@ -75,3 +89,14 @@ def restart_api(): (Path(__file__).parent / '../src/allocation/entrypoints/flask_app.py').touch() time.sleep(0.5) wait_for_webapp_to_come_up() + +@pytest.fixture +def restart_redis_pubsub(): + wait_for_redis_to_come_up() + if not shutil.which('docker-compose'): + print('skipping restart, assumes running in container') + return + subprocess.run( + ['docker-compose', 'restart', '-t', '0', 'redis_pubsub'], + check=True, + ) From 5adfbe96c3fdcb6c866b2775dac528fe57a58d74 Mon Sep 17 00:00:00 2001 From: Harry Date: Fri, 26 Jul 2019 15:02:06 +0100 Subject: [PATCH 112/165] refactor e2e tests, move random_refs and api_client out --- tests/e2e/api_client.py | 21 +++++++++++++++++++ tests/e2e/test_api.py | 45 +++++++++++++++++------------------------ 2 files changed, 39 insertions(+), 27 deletions(-) create mode 100644 tests/e2e/api_client.py diff --git a/tests/e2e/api_client.py b/tests/e2e/api_client.py new file mode 100644 index 00000000..94c0ef99 --- /dev/null +++ b/tests/e2e/api_client.py @@ -0,0 +1,21 @@ +import requests +from allocation import config + + +def post_to_add_batch(ref, sku, qty, eta): + url = config.get_api_url() + r = requests.post( + f'{url}/add_batch', + json={'ref': ref, 'sku': sku, 'qty': qty, 'eta': eta} + ) + assert r.status_code == 201 + + +def post_to_allocate(orderid, sku, qty, expect_success=True): + url = config.get_api_url() + r = requests.post(f'{url}/allocate', json={ + 'orderid': orderid, 'sku': sku, 'qty': qty, + }) + if expect_success: + assert r.status_code == 201 + return r diff --git a/tests/e2e/test_api.py b/tests/e2e/test_api.py index 00aa906e..d6cbca1b 100644 --- a/tests/e2e/test_api.py +++ b/tests/e2e/test_api.py @@ -1,18 +1,6 @@ -import uuid import pytest -import requests - -from allocation import config -from ..random_refs import random_sku, random_batchref, random_orderid - - -def post_to_add_batch(ref, sku, qty, eta): - url = config.get_api_url() - r = requests.post( - f'{url}/add_batch', - json={'ref': ref, 'sku': sku, 'qty': qty, 'eta': eta} - ) - assert r.status_code == 201 +from ..random_refs import random_batchref, random_orderid, random_sku +from . import api_client @pytest.mark.usefixtures('postgres_db') @@ -22,22 +10,25 @@ def test_happy_path_returns_201_and_allocated_batch(): earlybatch = random_batchref(1) laterbatch = random_batchref(2) otherbatch = random_batchref(3) - post_to_add_batch(laterbatch, sku, 100, '2011-01-02') - post_to_add_batch(earlybatch, sku, 100, '2011-01-01') - post_to_add_batch(otherbatch, othersku, 100, None) - data = {'orderid': random_orderid(), 'sku': sku, 'qty': 3} - url = config.get_api_url() - r = requests.post(f'{url}/allocate', json=data) - assert r.status_code == 201 - assert r.json()['batchref'] == earlybatch + api_client.post_to_add_batch(laterbatch, sku, 100, '2011-01-02') + api_client.post_to_add_batch(earlybatch, sku, 100, '2011-01-01') + api_client.post_to_add_batch(otherbatch, othersku, 100, None) + + response = api_client.post_to_allocate(random_orderid(), sku, qty=3) + + assert response.status_code == 201 + assert response.json()['batchref'] == earlybatch + @pytest.mark.usefixtures('postgres_db') @pytest.mark.usefixtures('restart_api') def test_unhappy_path_returns_400_and_error_message(): unknown_sku, orderid = random_sku(), random_orderid() - data = {'orderid': orderid, 'sku': unknown_sku, 'qty': 20} - url = config.get_api_url() - r = requests.post(f'{url}/allocate', json=data) - assert r.status_code == 400 - assert r.json()['message'] == f'Invalid sku {unknown_sku}' + + response = api_client.post_to_allocate( + orderid, unknown_sku, qty=20, expect_success=False, + ) + + assert response.status_code == 400 + assert response.json()['message'] == f'Invalid sku {unknown_sku}' From 316621869b78bc6b81984bf5a5a026d995e317a7 Mon Sep 17 00:00:00 2001 From: Harry Date: Fri, 26 Jul 2019 15:02:41 +0100 Subject: [PATCH 113/165] redis client for tests [redis_client_for_tests] --- tests/e2e/redis_client.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 tests/e2e/redis_client.py diff --git a/tests/e2e/redis_client.py b/tests/e2e/redis_client.py new file mode 100644 index 00000000..d776baea --- /dev/null +++ b/tests/e2e/redis_client.py @@ -0,0 +1,18 @@ +import json +import redis + +from allocation import config + +r = redis.Redis(**config.get_redis_host_and_port()) + + +def subscribe_to(channel): + pubsub = r.pubsub() + pubsub.subscribe(channel) + confirmation = pubsub.get_message(timeout=3) + assert confirmation['type'] == 'subscribe' + return pubsub + + +def publish_message(channel, message): + r.publish(channel, json.dumps(message)) From 72b56b6477b413eb9e2d4f3dcdfa817e64f7302b Mon Sep 17 00:00:00 2001 From: Harry Date: Fri, 26 Jul 2019 15:07:14 +0100 Subject: [PATCH 114/165] Test for our external events [redis_e2e_test] --- requirements.txt | 5 ++-- tests/e2e/test_external_events.py | 38 +++++++++++++++++++++++++++++++ tests/pytest.ini | 2 ++ 3 files changed, 43 insertions(+), 2 deletions(-) create mode 100644 tests/e2e/test_external_events.py diff --git a/requirements.txt b/requirements.txt index 592da462..abada9a3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,10 +3,11 @@ sqlalchemy flask psycopg2-binary -# tests +# dev/tests pytest pytest-icdiff mypy +pylint requests - redis +tenacity diff --git a/tests/e2e/test_external_events.py b/tests/e2e/test_external_events.py new file mode 100644 index 00000000..33f52307 --- /dev/null +++ b/tests/e2e/test_external_events.py @@ -0,0 +1,38 @@ +import json +import pytest +from tenacity import Retrying, RetryError, stop_after_delay +from . import api_client, redis_client +from ..random_refs import random_batchref, random_orderid, random_sku + + + +@pytest.mark.usefixtures('postgres_db') +@pytest.mark.usefixtures('restart_api') +@pytest.mark.usefixtures('restart_redis_pubsub') +def test_change_batch_quantity_leading_to_reallocation(): + # start with two batches and an order allocated to one of them + orderid, sku = random_orderid(), random_sku() + earlier_batch, later_batch = random_batchref('old'), random_batchref('newer') + api_client.post_to_add_batch(earlier_batch, sku, qty=10, eta='2011-01-02') + api_client.post_to_add_batch(later_batch, sku, qty=10, eta='2011-01-02') + response = api_client.post_to_allocate(orderid, sku, 10) + assert response.json()['batchref'] == earlier_batch + + subscription = redis_client.subscribe_to('line_allocated') + + # change quantity on allocated batch so it's less than our order + redis_client.publish_message('change_batch_quantity', { + 'batchref': earlier_batch, 'qty': 5 + }) + + # wait until we see a message saying the order has been reallocated + messages = [] + for attempt in Retrying(stop=stop_after_delay(3), reraise=True): + with attempt: + message = subscription.get_message(timeout=1) + if message: + messages.append(message) + print(messages) + data = json.loads(messages[-1]['data']) + assert data['orderid'] == orderid + assert data['batchref'] == later_batch diff --git a/tests/pytest.ini b/tests/pytest.ini index bbd083ac..3fd8685e 100644 --- a/tests/pytest.ini +++ b/tests/pytest.ini @@ -1,2 +1,4 @@ [pytest] addopts = --tb=short +filterwarnings = + ignore::DeprecationWarning From fca6380df3429e2bb9204d9ef1bd73db1c3b9e65 Mon Sep 17 00:00:00 2001 From: Harry Date: Fri, 15 Nov 2019 14:46:04 +0000 Subject: [PATCH 115/165] use tenacity in conftest --- tests/conftest.py | 32 +++++++------------------------- 1 file changed, 7 insertions(+), 25 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index bd20a704..352f7bd8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,11 +7,9 @@ import pytest import redis import requests -from requests.exceptions import RequestException -from redis.exceptions import RedisError -from sqlalchemy.exc import OperationalError from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker, clear_mappers +from tenacity import retry, stop_after_delay from allocation.adapters.orm import metadata, start_mappers from allocation import config @@ -34,36 +32,20 @@ def session(session_factory): return session_factory() +@retry(stop=stop_after_delay(10)) def wait_for_postgres_to_come_up(engine): - deadline = time.time() + 10 - while time.time() < deadline: - try: - return engine.connect() - except OperationalError: - time.sleep(0.5) - pytest.fail('Postgres never came up') + return engine.connect() +@retry(stop=stop_after_delay(10)) def wait_for_webapp_to_come_up(): - deadline = time.time() + 10 - url = config.get_api_url() - while time.time() < deadline: - try: - return requests.get(url) - except RequestException: - time.sleep(0.5) - pytest.fail('API never came up') + return requests.get(config.get_api_url()) +@retry(stop=stop_after_delay(10)) def wait_for_redis_to_come_up(): - deadline = time.time() + 5 r = redis.Redis(**config.get_redis_host_and_port()) - while time.time() < deadline: - try: - return r.ping() - except RedisError: - time.sleep(0.5) - pytest.fail('Redis never came up') + return r.ping() @pytest.fixture(scope='session') From 7ea2a3b7c537ade0c48857b01cb7dc14dd8ee543 Mon Sep 17 00:00:00 2001 From: Harry Date: Fri, 26 Jul 2019 15:07:35 +0100 Subject: [PATCH 116/165] Docker infrastructure for new redis event listener container --- Dockerfile | 2 -- Makefile | 12 ++++++------ docker-compose.yml | 32 +++++++++++++++++++++++++++++--- 3 files changed, 35 insertions(+), 11 deletions(-) diff --git a/Dockerfile b/Dockerfile index 04d923f2..a52f27b6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,5 +14,3 @@ RUN pip install -e /src COPY tests/ /tests/ WORKDIR /src -ENV FLASK_APP=allocation/entrypoints/flask_app.py FLASK_DEBUG=1 PYTHONUNBUFFERED=1 -CMD flask run --host=0.0.0.0 --port=80 diff --git a/Makefile b/Makefile index 4fb33031..968a3be5 100644 --- a/Makefile +++ b/Makefile @@ -2,22 +2,22 @@ build: docker-compose build up: - docker-compose up -d app + docker-compose up -d test: up - docker-compose run --rm --no-deps --entrypoint=pytest app /tests/unit /tests/integration /tests/e2e + docker-compose run --rm --no-deps --entrypoint=pytest api /tests/unit /tests/integration /tests/e2e unit-tests: - docker-compose run --rm --no-deps --entrypoint=pytest app /tests/unit + docker-compose run --rm --no-deps --entrypoint=pytest api /tests/unit integration-tests: up - docker-compose run --rm --no-deps --entrypoint=pytest app /tests/integration + docker-compose run --rm --no-deps --entrypoint=pytest api /tests/integration e2e-tests: up - docker-compose run --rm --no-deps --entrypoint=pytest app /tests/e2e + docker-compose run --rm --no-deps --entrypoint=pytest api /tests/e2e logs: - docker-compose logs app | tail -100 + docker-compose logs --tail=25 api redis_pubsub down: docker-compose down --remove-orphans diff --git a/docker-compose.yml b/docker-compose.yml index 74ac28b6..dc2cc369 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,25 +1,51 @@ version: "3" + services: - app: + redis_pubsub: build: context: . dockerfile: Dockerfile + image: allocation-image depends_on: - postgres - redis environment: - DB_HOST=postgres - DB_PASSWORD=abc123 - - API_HOST=app + - REDIS_HOST=redis - PYTHONDONTWRITEBYTECODE=1 volumes: - ./src:/src - ./tests:/tests + entrypoint: + - python + - /src/allocation/entrypoints/redis_eventconsumer.py + + api: + image: allocation-image + depends_on: + - redis_pubsub + environment: + - DB_HOST=postgres + - DB_PASSWORD=abc123 + - API_HOST=api + - REDIS_HOST=redis + - PYTHONDONTWRITEBYTECODE=1 + - FLASK_APP=allocation/entrypoints/flask_app.py + - FLASK_DEBUG=1 + - PYTHONUNBUFFERED=1 + volumes: + - ./src:/src + - ./tests:/tests + entrypoint: + - flask + - run + - --host=0.0.0.0 + - --port=80 ports: - "5005:80" - postgres: image: postgres:9.6 environment: From 5bb705582c4becd8605df9e686daaf5988f3e3f0 Mon Sep 17 00:00:00 2001 From: Harry Date: Wed, 14 Aug 2019 15:02:48 +0100 Subject: [PATCH 117/165] add Allocated event [allocated_event] --- src/allocation/domain/events.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/allocation/domain/events.py b/src/allocation/domain/events.py index b00ad7e7..4a376291 100644 --- a/src/allocation/domain/events.py +++ b/src/allocation/domain/events.py @@ -4,6 +4,13 @@ class Event: pass +@dataclass +class Allocated(Event): + orderid: str + sku: str + qty: int + batchref: str + @dataclass class OutOfStock(Event): sku: str From 1b4c664f4e53b0f9729b2cf9f91ddce1bc250933 Mon Sep 17 00:00:00 2001 From: Harry Date: Fri, 3 Jan 2020 19:11:39 +0000 Subject: [PATCH 118/165] redis eventconsumer first cut [redis_eventconsumer_first_cut] --- .../entrypoints/redis_eventconsumer.py | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 src/allocation/entrypoints/redis_eventconsumer.py diff --git a/src/allocation/entrypoints/redis_eventconsumer.py b/src/allocation/entrypoints/redis_eventconsumer.py new file mode 100644 index 00000000..a27cd02b --- /dev/null +++ b/src/allocation/entrypoints/redis_eventconsumer.py @@ -0,0 +1,31 @@ +import json +import logging +import redis + +from allocation import config +from allocation.domain import commands +from allocation.adapters import orm +from allocation.service_layer import messagebus, unit_of_work + +logger = logging.getLogger(__name__) + +r = redis.Redis(**config.get_redis_host_and_port()) + + +def main(): + orm.start_mappers() + pubsub = r.pubsub(ignore_subscribe_messages=True) + pubsub.subscribe('change_batch_quantity') + + for m in pubsub.listen(): + handle_change_batch_quantity(m) + + +def handle_change_batch_quantity(m): + logging.debug('handling %s', m) + data = json.loads(m['data']) + cmd = commands.ChangeBatchQuantity(ref=data['batchref'], qty=data['qty']) + messagebus.handle(cmd, uow=unit_of_work.SqlAlchemyUnitOfWork()) + +if __name__ == '__main__': + main() From 49dba972c3df397bd44a6c60b70eba557fc0039a Mon Sep 17 00:00:00 2001 From: Harry Date: Fri, 3 Jan 2020 19:12:02 +0000 Subject: [PATCH 119/165] redis eventpublisher first cut [redis_eventpublisher_first_cut] --- src/allocation/adapters/redis_eventpublisher.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 src/allocation/adapters/redis_eventpublisher.py diff --git a/src/allocation/adapters/redis_eventpublisher.py b/src/allocation/adapters/redis_eventpublisher.py new file mode 100644 index 00000000..2afac458 --- /dev/null +++ b/src/allocation/adapters/redis_eventpublisher.py @@ -0,0 +1,16 @@ +import json +import logging +from dataclasses import asdict +import redis + +from allocation import config +from allocation.domain import events + +logger = logging.getLogger(__name__) + +r = redis.Redis(**config.get_redis_host_and_port()) + + +def publish(channel, event: events.Event): + logging.debug('publishing: channel=%s, event=%s', channel, event) + r.publish(channel, json.dumps(asdict(event))) From 8fd9276a8e658aff9c3e816c35927e9b7ad13abf Mon Sep 17 00:00:00 2001 From: Harry Date: Thu, 18 Jul 2019 22:29:20 +0100 Subject: [PATCH 120/165] sneak in a redis patch so unit test dont need redis --- tests/unit/test_handlers.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/unit/test_handlers.py b/tests/unit/test_handlers.py index 1fae8c47..c8ca46bd 100644 --- a/tests/unit/test_handlers.py +++ b/tests/unit/test_handlers.py @@ -58,6 +58,14 @@ def test_for_existing_product(self): assert "b2" in [b.reference for b in uow.products.get("GARISH-RUG").batches] + +@pytest.fixture(autouse=True) +def fake_redis_publish(): + with mock.patch("allocation.adapters.redis_eventpublisher.publish"): + yield + + + class TestAllocate: def test_allocates(self): From e34840fc698dff70733f777af89667eed0fc05ac Mon Sep 17 00:00:00 2001 From: Harry Date: Wed, 14 Aug 2019 15:20:18 +0100 Subject: [PATCH 121/165] test and Product change to emit event [model_emits_allocated_event] --- src/allocation/domain/model.py | 4 ++++ tests/unit/test_product.py | 11 +++++++++++ 2 files changed, 15 insertions(+) diff --git a/src/allocation/domain/model.py b/src/allocation/domain/model.py index f797f6c0..34f580d6 100644 --- a/src/allocation/domain/model.py +++ b/src/allocation/domain/model.py @@ -20,6 +20,10 @@ def allocate(self, line: OrderLine) -> str: ) batch.allocate(line) self.version_number += 1 + self.events.append(events.Allocated( + orderid=line.orderid, sku=line.sku, qty=line.qty, + batchref=batch.reference, + )) return batch.reference except StopIteration: self.events.append(events.OutOfStock(line.sku)) diff --git a/tests/unit/test_product.py b/tests/unit/test_product.py index 22de7185..c1beb274 100644 --- a/tests/unit/test_product.py +++ b/tests/unit/test_product.py @@ -42,6 +42,17 @@ def test_returns_allocated_batch_ref(): assert allocation == in_stock_batch.reference +def test_outputs_allocated_event(): + batch = Batch("batchref", "RETRO-LAMPSHADE", 100, eta=None) + line = OrderLine("oref", "RETRO-LAMPSHADE", 10) + product = Product(sku="RETRO-LAMPSHADE", batches=[batch]) + product.allocate(line) + expected = events.Allocated( + orderid="oref", sku="RETRO-LAMPSHADE", qty=10, batchref=batch.reference + ) + assert product.events[-1] == expected + + def test_records_out_of_stock_event_if_cannot_allocate(): batch = Batch('batch1', 'SMALL-FORK', 10, eta=today) product = Product(sku="SMALL-FORK", batches=[batch]) From 0b3f74a948b2599dea0dc21a926c684ec8d5216b Mon Sep 17 00:00:00 2001 From: Harry Date: Wed, 14 Aug 2019 15:22:23 +0100 Subject: [PATCH 122/165] add handler for Allocated [chapter_11_external_events_ends] --- src/allocation/service_layer/handlers.py | 10 +++++++++- src/allocation/service_layer/messagebus.py | 1 + 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/allocation/service_layer/handlers.py b/src/allocation/service_layer/handlers.py index 85fb5e03..75170c97 100644 --- a/src/allocation/service_layer/handlers.py +++ b/src/allocation/service_layer/handlers.py @@ -1,6 +1,6 @@ from __future__ import annotations from typing import TYPE_CHECKING -from allocation.adapters import email +from allocation.adapters import email, redis_eventpublisher from allocation.domain import commands, events, model from allocation.domain.model import OrderLine if TYPE_CHECKING: @@ -47,6 +47,8 @@ def change_batch_quantity( uow.commit() +#pylint: disable=unused-argument + def send_out_of_stock_notification( event: events.OutOfStock, uow: unit_of_work.AbstractUnitOfWork, ): @@ -54,3 +56,9 @@ def send_out_of_stock_notification( 'stock@made.com', f'Out of stock for {event.sku}', ) + + +def publish_allocated_event( + event: events.Allocated, uow: unit_of_work.AbstractUnitOfWork, +): + redis_eventpublisher.publish('line_allocated', event) diff --git a/src/allocation/service_layer/messagebus.py b/src/allocation/service_layer/messagebus.py index a0a674c4..cb3be902 100644 --- a/src/allocation/service_layer/messagebus.py +++ b/src/allocation/service_layer/messagebus.py @@ -60,6 +60,7 @@ def handle_command( EVENT_HANDLERS = { + events.Allocated: [handlers.publish_allocated_event], events.OutOfStock: [handlers.send_out_of_stock_notification], } # type: Dict[Type[events.Event], List[Callable]] From cff6a669d47f7fbb153ca55ed42f8ffea4a40f98 Mon Sep 17 00:00:00 2001 From: Harry Date: Mon, 10 Jun 2019 21:22:46 +0100 Subject: [PATCH 123/165] modify api tests to try and do a get after a post [get_after_post] --- tests/conftest.py | 1 + tests/e2e/api_client.py | 6 +++++- tests/e2e/test_api.py | 24 ++++++++++++++---------- 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 352f7bd8..0b1bc71a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -14,6 +14,7 @@ from allocation.adapters.orm import metadata, start_mappers from allocation import config +pytest.register_assert_rewrite('tests.e2e.api_client') @pytest.fixture def in_memory_db(): diff --git a/tests/e2e/api_client.py b/tests/e2e/api_client.py index 94c0ef99..82dbd1b0 100644 --- a/tests/e2e/api_client.py +++ b/tests/e2e/api_client.py @@ -17,5 +17,9 @@ def post_to_allocate(orderid, sku, qty, expect_success=True): 'orderid': orderid, 'sku': sku, 'qty': qty, }) if expect_success: - assert r.status_code == 201 + assert r.status_code == 202 return r + +def get_allocation(orderid): + url = config.get_api_url() + return requests.get(f'{url}/allocations/{orderid}') diff --git a/tests/e2e/test_api.py b/tests/e2e/test_api.py index d6cbca1b..d5dfc670 100644 --- a/tests/e2e/test_api.py +++ b/tests/e2e/test_api.py @@ -2,10 +2,10 @@ from ..random_refs import random_batchref, random_orderid, random_sku from . import api_client - @pytest.mark.usefixtures('postgres_db') @pytest.mark.usefixtures('restart_api') -def test_happy_path_returns_201_and_allocated_batch(): +def test_happy_path_returns_202_and_batch_is_allocated(): + orderid = random_orderid() sku, othersku = random_sku(), random_sku('other') earlybatch = random_batchref(1) laterbatch = random_batchref(2) @@ -14,21 +14,25 @@ def test_happy_path_returns_201_and_allocated_batch(): api_client.post_to_add_batch(earlybatch, sku, 100, '2011-01-01') api_client.post_to_add_batch(otherbatch, othersku, 100, None) - response = api_client.post_to_allocate(random_orderid(), sku, qty=3) - - assert response.status_code == 201 - assert response.json()['batchref'] == earlybatch + r = api_client.post_to_allocate(orderid, sku, qty=3) + assert r.status_code == 202 + r = api_client.get_allocation(orderid) + assert r.ok + assert r.json() == [ + {'sku': sku, 'batchref': earlybatch}, + ] @pytest.mark.usefixtures('postgres_db') @pytest.mark.usefixtures('restart_api') def test_unhappy_path_returns_400_and_error_message(): unknown_sku, orderid = random_sku(), random_orderid() - - response = api_client.post_to_allocate( + r = api_client.post_to_allocate( orderid, unknown_sku, qty=20, expect_success=False, ) + assert r.status_code == 400 + assert r.json()['message'] == f'Invalid sku {unknown_sku}' - assert response.status_code == 400 - assert response.json()['message'] == f'Invalid sku {unknown_sku}' + r = api_client.get_allocation(orderid) + assert r.status_code == 404 From 40148afd59e01df9c85a040863c8b6a7e0b67332 Mon Sep 17 00:00:00 2001 From: Harry Date: Mon, 10 Jun 2019 21:23:22 +0100 Subject: [PATCH 124/165] modify allocate handler to no longer return anything --- src/allocation/service_layer/handlers.py | 5 ++--- tests/unit/test_handlers.py | 3 +-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/allocation/service_layer/handlers.py b/src/allocation/service_layer/handlers.py index 75170c97..2d9975aa 100644 --- a/src/allocation/service_layer/handlers.py +++ b/src/allocation/service_layer/handlers.py @@ -27,15 +27,14 @@ def add_batch( def allocate( cmd: commands.Allocate, uow: unit_of_work.AbstractUnitOfWork -) -> str: +): line = OrderLine(cmd.orderid, cmd.sku, cmd.qty) with uow: product = uow.products.get(sku=line.sku) if product is None: raise InvalidSku(f'Invalid sku {line.sku}') - batchref = product.allocate(line) + product.allocate(line) uow.commit() - return batchref def change_batch_quantity( diff --git a/tests/unit/test_handlers.py b/tests/unit/test_handlers.py index c8ca46bd..4aaee189 100644 --- a/tests/unit/test_handlers.py +++ b/tests/unit/test_handlers.py @@ -73,10 +73,9 @@ def test_allocates(self): messagebus.handle( commands.CreateBatch("batch1", "COMPLICATED-LAMP", 100, None), uow ) - results = messagebus.handle( + messagebus.handle( commands.Allocate("o1", "COMPLICATED-LAMP", 10), uow ) - assert results.pop(0) == "batch1" [batch] = uow.products.get("COMPLICATED-LAMP").batches assert batch.available_quantity == 90 From 7ac1cf4965489a8df92128c9610bc526f45c3788 Mon Sep 17 00:00:00 2001 From: Harry Date: Mon, 10 Jun 2019 21:50:01 +0100 Subject: [PATCH 125/165] modify flask to add new view endpoint and return 202s --- src/allocation/entrypoints/flask_app.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/allocation/entrypoints/flask_app.py b/src/allocation/entrypoints/flask_app.py index 313544c3..815d3f9c 100644 --- a/src/allocation/entrypoints/flask_app.py +++ b/src/allocation/entrypoints/flask_app.py @@ -5,6 +5,7 @@ from allocation.adapters import orm from allocation.service_layer import messagebus, unit_of_work from allocation.service_layer.handlers import InvalidSku +from allocation import views app = Flask(__name__) orm.start_mappers() @@ -35,4 +36,13 @@ def allocate_endpoint(): except InvalidSku as e: return jsonify({'message': str(e)}), 400 - return jsonify({'batchref': batchref}), 201 + return 'OK', 202 + + +@app.route("/allocations/", methods=['GET']) +def allocations_view_endpoint(orderid): + uow = unit_of_work.SqlAlchemyUnitOfWork() + result = views.allocations(orderid, uow) + if not result: + return 'not found', 404 + return jsonify(result), 200 From ce6f584a78270287b5816156b7a547b6cc2acbd6 Mon Sep 17 00:00:00 2001 From: Harry Date: Wed, 12 Jun 2019 21:57:35 +0100 Subject: [PATCH 126/165] session_factory -> sqlite_session_factory (needs backport) --- tests/conftest.py | 6 +++--- tests/integration/test_repository.py | 4 ++-- tests/integration/test_uow.py | 18 +++++++++--------- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 0b1bc71a..455428dd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -23,14 +23,14 @@ def in_memory_db(): return engine @pytest.fixture -def session_factory(in_memory_db): +def sqlite_session_factory(in_memory_db): start_mappers() yield sessionmaker(bind=in_memory_db) clear_mappers() @pytest.fixture -def session(session_factory): - return session_factory() +def sqlite_session(sqlite_session_factory): + return sqlite_session_factory() @retry(stop=stop_after_delay(10)) diff --git a/tests/integration/test_repository.py b/tests/integration/test_repository.py index 733346c3..d3b936df 100644 --- a/tests/integration/test_repository.py +++ b/tests/integration/test_repository.py @@ -1,8 +1,8 @@ from allocation.adapters import repository from allocation.domain import model -def test_get_by_batchref(session): - repo = repository.SqlAlchemyRepository(session) +def test_get_by_batchref(sqlite_session): + repo = repository.SqlAlchemyRepository(sqlite_session) b1 = model.Batch(ref='b1', sku='sku1', qty=100, eta=None) b2 = model.Batch(ref='b2', sku='sku1', qty=100, eta=None) b3 = model.Batch(ref='b3', sku='sku2', qty=100, eta=None) diff --git a/tests/integration/test_uow.py b/tests/integration/test_uow.py index 1cc4fb06..0c7264ef 100644 --- a/tests/integration/test_uow.py +++ b/tests/integration/test_uow.py @@ -34,12 +34,12 @@ def get_allocated_batch_ref(session, orderid, sku): return batchref -def test_uow_can_retrieve_a_batch_and_allocate_to_it(session_factory): - session = session_factory() +def test_uow_can_retrieve_a_batch_and_allocate_to_it(sqlite_session_factory): + session = sqlite_session_factory() insert_batch(session, 'batch1', 'HIPSTER-WORKBENCH', 100, None) session.commit() - uow = unit_of_work.SqlAlchemyUnitOfWork(session_factory) + uow = unit_of_work.SqlAlchemyUnitOfWork(sqlite_session_factory) with uow: product = uow.products.get(sku='HIPSTER-WORKBENCH') line = model.OrderLine('o1', 'HIPSTER-WORKBENCH', 10) @@ -50,27 +50,27 @@ def test_uow_can_retrieve_a_batch_and_allocate_to_it(session_factory): assert batchref == 'batch1' -def test_rolls_back_uncommitted_work_by_default(session_factory): - uow = unit_of_work.SqlAlchemyUnitOfWork(session_factory) +def test_rolls_back_uncommitted_work_by_default(sqlite_session_factory): + uow = unit_of_work.SqlAlchemyUnitOfWork(sqlite_session_factory) with uow: insert_batch(uow.session, 'batch1', 'MEDIUM-PLINTH', 100, None) - new_session = session_factory() + new_session = sqlite_session_factory() rows = list(new_session.execute('SELECT * FROM "batches"')) assert rows == [] -def test_rolls_back_on_error(session_factory): +def test_rolls_back_on_error(sqlite_session_factory): class MyException(Exception): pass - uow = unit_of_work.SqlAlchemyUnitOfWork(session_factory) + uow = unit_of_work.SqlAlchemyUnitOfWork(sqlite_session_factory) with pytest.raises(MyException): with uow: insert_batch(uow.session, 'batch1', 'LARGE-FORK', 100, None) raise MyException() - new_session = session_factory() + new_session = sqlite_session_factory() rows = list(new_session.execute('SELECT * FROM "batches"')) assert rows == [] From bd3501df3c15bc5f87a43003a20748a5b6aba7b4 Mon Sep 17 00:00:00 2001 From: Harry Date: Mon, 10 Jun 2019 21:50:10 +0100 Subject: [PATCH 127/165] integration test for our view --- tests/integration/test_views.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 tests/integration/test_views.py diff --git a/tests/integration/test_views.py b/tests/integration/test_views.py new file mode 100644 index 00000000..78e3e3df --- /dev/null +++ b/tests/integration/test_views.py @@ -0,0 +1,23 @@ +from datetime import date +from allocation import views +from allocation.domain import commands +from allocation.service_layer import messagebus, unit_of_work + +today = date.today() + + +def test_allocations_view(sqlite_session_factory): + uow = unit_of_work.SqlAlchemyUnitOfWork(sqlite_session_factory) + messagebus.handle(commands.CreateBatch('sku1batch', 'sku1', 50, None), uow) + messagebus.handle(commands.CreateBatch('sku2batch', 'sku2', 50, today), uow) + messagebus.handle(commands.Allocate('order1', 'sku1', 20), uow) + messagebus.handle(commands.Allocate('order1', 'sku2', 20), uow) + # add a spurious batch and order to make sure we're getting the right ones + messagebus.handle(commands.CreateBatch('sku1batch-later', 'sku1', 50, today), uow) + messagebus.handle(commands.Allocate('otherorder', 'sku1', 30), uow) + messagebus.handle(commands.Allocate('otherorder', 'sku2', 10), uow) + + assert views.allocations('order1', uow) == [ + {'sku': 'sku1', 'batchref': 'sku1batch'}, + {'sku': 'sku2', 'batchref': 'sku2batch'}, + ] From 43c2df7105a1976601f2e8f72eaab1be515e4946 Mon Sep 17 00:00:00 2001 From: Harry Date: Mon, 10 Jun 2019 21:50:25 +0100 Subject: [PATCH 128/165] first cut of a view with raw sql [views_dot_py] --- src/allocation/views.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 src/allocation/views.py diff --git a/src/allocation/views.py b/src/allocation/views.py new file mode 100644 index 00000000..e00e8980 --- /dev/null +++ b/src/allocation/views.py @@ -0,0 +1,13 @@ +from allocation.service_layer import unit_of_work + +def allocations(orderid: str, uow: unit_of_work.SqlAlchemyUnitOfWork): + with uow: + results = list(uow.session.execute( + 'SELECT ol.sku, b.reference' + ' FROM allocations AS a' + ' JOIN batches AS b ON a.batch_id = b.id' + ' JOIN order_lines AS ol ON a.orderline_id = ol.id' + ' WHERE ol.orderid = :orderid', + dict(orderid=orderid) + )) + return [{'sku': sku, 'batchref': batchref} for sku, batchref in results] From ccbaa5a231dea210feabc7ffbe8e722900fdf071 Mon Sep 17 00:00:00 2001 From: Harry Date: Wed, 6 Nov 2019 13:26:44 +0000 Subject: [PATCH 129/165] use repository and go via Product [view_using_repo] --- src/allocation/views.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/allocation/views.py b/src/allocation/views.py index e00e8980..e19b5e40 100644 --- a/src/allocation/views.py +++ b/src/allocation/views.py @@ -2,12 +2,10 @@ def allocations(orderid: str, uow: unit_of_work.SqlAlchemyUnitOfWork): with uow: - results = list(uow.session.execute( - 'SELECT ol.sku, b.reference' - ' FROM allocations AS a' - ' JOIN batches AS b ON a.batch_id = b.id' - ' JOIN order_lines AS ol ON a.orderline_id = ol.id' - ' WHERE ol.orderid = :orderid', - dict(orderid=orderid) - )) - return [{'sku': sku, 'batchref': batchref} for sku, batchref in results] + products = uow.products.for_order(orderid=orderid) + batches = [b for p in products for b in p.batches] + return [ + {'sku': b.sku, 'batchref': b.reference} + for b in batches + if orderid in b.orderids + ] From 4e0b85211d58875003a9c33aacae68e0eced3779 Mon Sep 17 00:00:00 2001 From: Harry Date: Wed, 6 Nov 2019 13:27:20 +0000 Subject: [PATCH 130/165] arguably-unnecessary helper property on model. [orderids_property] --- src/allocation/domain/model.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/allocation/domain/model.py b/src/allocation/domain/model.py index 34f580d6..3e38122b 100644 --- a/src/allocation/domain/model.py +++ b/src/allocation/domain/model.py @@ -73,6 +73,10 @@ def __gt__(self, other): return True return self.eta > other.eta + @property + def orderids(self): + return {l.orderid for l in self._allocations} + def allocate(self, line: OrderLine): if self.can_allocate(line): self._allocations.add(line) From 6e9dcbf85da743b48626a697416ec68af0c02e69 Mon Sep 17 00:00:00 2001 From: Harry Date: Wed, 6 Nov 2019 13:27:47 +0000 Subject: [PATCH 131/165] finder method on repo [for_order_method] --- src/allocation/adapters/repository.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/allocation/adapters/repository.py b/src/allocation/adapters/repository.py index bda90c07..0d432655 100644 --- a/src/allocation/adapters/repository.py +++ b/src/allocation/adapters/repository.py @@ -57,3 +57,11 @@ def _get_by_batchref(self, batchref): return self.session.query(model.Product).join(model.Batch).filter( orm.batches.c.reference == batchref, ).first() + + + def for_order(self, orderid): + order_lines = self.session.query(model.OrderLine).filter_by(orderid=orderid) + skus = {l.sku for l in order_lines} + return self.session.query(model.Product).join(model.Batch).filter( + model.Batch.sku.in_(skus) + ) From 3e70f131ea5f286a7c3c8ad5aaee269fdcba7d68 Mon Sep 17 00:00:00 2001 From: Harry Date: Wed, 6 Nov 2019 13:29:39 +0000 Subject: [PATCH 132/165] Use the ORM instead [view_using_orm] --- src/allocation/adapters/repository.py | 8 -------- src/allocation/domain/model.py | 4 ---- src/allocation/views.py | 14 +++++++------- 3 files changed, 7 insertions(+), 19 deletions(-) diff --git a/src/allocation/adapters/repository.py b/src/allocation/adapters/repository.py index 0d432655..bda90c07 100644 --- a/src/allocation/adapters/repository.py +++ b/src/allocation/adapters/repository.py @@ -57,11 +57,3 @@ def _get_by_batchref(self, batchref): return self.session.query(model.Product).join(model.Batch).filter( orm.batches.c.reference == batchref, ).first() - - - def for_order(self, orderid): - order_lines = self.session.query(model.OrderLine).filter_by(orderid=orderid) - skus = {l.sku for l in order_lines} - return self.session.query(model.Product).join(model.Batch).filter( - model.Batch.sku.in_(skus) - ) diff --git a/src/allocation/domain/model.py b/src/allocation/domain/model.py index 3e38122b..34f580d6 100644 --- a/src/allocation/domain/model.py +++ b/src/allocation/domain/model.py @@ -73,10 +73,6 @@ def __gt__(self, other): return True return self.eta > other.eta - @property - def orderids(self): - return {l.orderid for l in self._allocations} - def allocate(self, line: OrderLine): if self.can_allocate(line): self._allocations.add(line) diff --git a/src/allocation/views.py b/src/allocation/views.py index e19b5e40..30069b3a 100644 --- a/src/allocation/views.py +++ b/src/allocation/views.py @@ -1,11 +1,11 @@ +from allocation.domain import model from allocation.service_layer import unit_of_work def allocations(orderid: str, uow: unit_of_work.SqlAlchemyUnitOfWork): with uow: - products = uow.products.for_order(orderid=orderid) - batches = [b for p in products for b in p.batches] - return [ - {'sku': b.sku, 'batchref': b.reference} - for b in batches - if orderid in b.orderids - ] + batches = uow.session.query(model.Batch).join( + model.OrderLine, model.Batch._allocations + ).filter( + model.OrderLine.orderid == orderid + ) + return [{'sku': b.sku, 'batchref': b.reference} for b in batches] From 93f12b7c66bddbae6652f8e4256e479a897df274 Mon Sep 17 00:00:00 2001 From: Harry Date: Mon, 10 Jun 2019 22:33:33 +0100 Subject: [PATCH 133/165] Simpler view based on a new read model table --- src/allocation/views.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/allocation/views.py b/src/allocation/views.py index 30069b3a..4cd07397 100644 --- a/src/allocation/views.py +++ b/src/allocation/views.py @@ -1,11 +1,9 @@ -from allocation.domain import model from allocation.service_layer import unit_of_work def allocations(orderid: str, uow: unit_of_work.SqlAlchemyUnitOfWork): with uow: - batches = uow.session.query(model.Batch).join( - model.OrderLine, model.Batch._allocations - ).filter( - model.OrderLine.orderid == orderid - ) - return [{'sku': b.sku, 'batchref': b.reference} for b in batches] + results = list(uow.session.execute( + 'SELECT sku, batchref FROM allocations_view WHERE orderid = :orderid', + dict(orderid=orderid) + )) + return [dict(r) for r in results] From 0ef4dfc94538443dcc051d8defdda73f47ed79d1 Mon Sep 17 00:00:00 2001 From: Harry Date: Mon, 10 Jun 2019 22:33:43 +0100 Subject: [PATCH 134/165] new table in orm --- src/allocation/adapters/orm.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/allocation/adapters/orm.py b/src/allocation/adapters/orm.py index 285fdd0b..16ebdcfb 100644 --- a/src/allocation/adapters/orm.py +++ b/src/allocation/adapters/orm.py @@ -39,6 +39,13 @@ Column('batch_id', ForeignKey('batches.id')), ) +allocations_view = Table( + 'allocations_view', metadata, + Column('orderid', String(255)), + Column('sku', String(255)), + Column('batchref', String(255)), +) + def start_mappers(): lines_mapper = mapper(model.OrderLine, order_lines) From 8d0d567d6f5143192ae78796f55a49c43102adbc Mon Sep 17 00:00:00 2001 From: Harry Date: Mon, 10 Jun 2019 22:33:58 +0100 Subject: [PATCH 135/165] handler for view model update --- src/allocation/service_layer/handlers.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/allocation/service_layer/handlers.py b/src/allocation/service_layer/handlers.py index 2d9975aa..ef8483e1 100644 --- a/src/allocation/service_layer/handlers.py +++ b/src/allocation/service_layer/handlers.py @@ -61,3 +61,15 @@ def publish_allocated_event( event: events.Allocated, uow: unit_of_work.AbstractUnitOfWork, ): redis_eventpublisher.publish('line_allocated', event) + + +def add_allocation_to_read_model( + event: events.Allocated, uow: unit_of_work.SqlAlchemyUnitOfWork, +): + with uow: + uow.session.execute( + 'INSERT INTO allocations_view (orderid, sku, batchref)' + ' VALUES (:orderid, :sku, :batchref)', + dict(orderid=event.orderid, sku=event.sku, batchref=event.batchref) + ) + uow.commit() From 3bf05382dd123f2e4b5171132cca25df8d6db2c5 Mon Sep 17 00:00:00 2001 From: Harry Date: Mon, 10 Jun 2019 22:34:34 +0100 Subject: [PATCH 136/165] add handler for allocated --- src/allocation/service_layer/messagebus.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/allocation/service_layer/messagebus.py b/src/allocation/service_layer/messagebus.py index cb3be902..20ad1eac 100644 --- a/src/allocation/service_layer/messagebus.py +++ b/src/allocation/service_layer/messagebus.py @@ -60,7 +60,10 @@ def handle_command( EVENT_HANDLERS = { - events.Allocated: [handlers.publish_allocated_event], + events.Allocated: [ + handlers.publish_allocated_event, + handlers.add_allocation_to_read_model + ], events.OutOfStock: [handlers.send_out_of_stock_notification], } # type: Dict[Type[events.Event], List[Callable]] From 4f3f4a7c20b6b1ef13bffb7d65ad069d488d4154 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 13 Aug 2019 06:21:27 +0100 Subject: [PATCH 137/165] handle_command no longer returns --- src/allocation/entrypoints/flask_app.py | 3 +-- src/allocation/service_layer/messagebus.py | 8 ++------ 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/src/allocation/entrypoints/flask_app.py b/src/allocation/entrypoints/flask_app.py index 815d3f9c..434a32f8 100644 --- a/src/allocation/entrypoints/flask_app.py +++ b/src/allocation/entrypoints/flask_app.py @@ -31,8 +31,7 @@ def allocate_endpoint(): request.json['orderid'], request.json['sku'], request.json['qty'], ) uow = unit_of_work.SqlAlchemyUnitOfWork() - results = messagebus.handle(cmd, uow) - batchref = results.pop(0) + messagebus.handle(cmd, uow) except InvalidSku as e: return jsonify({'message': str(e)}), 400 diff --git a/src/allocation/service_layer/messagebus.py b/src/allocation/service_layer/messagebus.py index 20ad1eac..cf30c3d3 100644 --- a/src/allocation/service_layer/messagebus.py +++ b/src/allocation/service_layer/messagebus.py @@ -14,18 +14,15 @@ def handle(message: Message, uow: unit_of_work.AbstractUnitOfWork): - results = [] queue = [message] while queue: message = queue.pop(0) if isinstance(message, events.Event): handle_event(message, queue, uow) elif isinstance(message, commands.Command): - cmd_result = handle_command(message, queue, uow) - results.append(cmd_result) + handle_command(message, queue, uow) else: raise Exception(f'{message} was not an Event or Command') - return results def handle_event( @@ -51,9 +48,8 @@ def handle_command( logger.debug('handling command %s', command) try: handler = COMMAND_HANDLERS[type(command)] - result = handler(command, uow=uow) + handler(command, uow=uow) queue.extend(uow.collect_new_events()) - return result except Exception: logger.exception('Exception handling command %s', command) raise From 98c9e0cd790717c08028d2ef92c7cb25c954bb04 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 13 Aug 2019 06:21:35 +0100 Subject: [PATCH 138/165] pylint thing --- src/allocation/service_layer/handlers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/allocation/service_layer/handlers.py b/src/allocation/service_layer/handlers.py index ef8483e1..2e42a829 100644 --- a/src/allocation/service_layer/handlers.py +++ b/src/allocation/service_layer/handlers.py @@ -1,3 +1,4 @@ +#pylint: disable=unused-argument from __future__ import annotations from typing import TYPE_CHECKING from allocation.adapters import email, redis_eventpublisher From b4d7a02b7ae9b3130c0d550f6932e7632877d21d Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 13 Aug 2019 06:20:52 +0100 Subject: [PATCH 139/165] fix redis e2e test --- tests/e2e/test_external_events.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/e2e/test_external_events.py b/tests/e2e/test_external_events.py index 33f52307..762731eb 100644 --- a/tests/e2e/test_external_events.py +++ b/tests/e2e/test_external_events.py @@ -14,9 +14,11 @@ def test_change_batch_quantity_leading_to_reallocation(): orderid, sku = random_orderid(), random_sku() earlier_batch, later_batch = random_batchref('old'), random_batchref('newer') api_client.post_to_add_batch(earlier_batch, sku, qty=10, eta='2011-01-02') - api_client.post_to_add_batch(later_batch, sku, qty=10, eta='2011-01-02') - response = api_client.post_to_allocate(orderid, sku, 10) - assert response.json()['batchref'] == earlier_batch + api_client.post_to_add_batch(later_batch, sku, qty=10, eta='2011-01-03') + r = api_client.post_to_allocate(orderid, sku, 10) + assert r.ok + response = api_client.get_allocation(orderid) + assert response.json()[0]['batchref'] == earlier_batch subscription = redis_client.subscribe_to('line_allocated') From 780efcbb8968f6e1848ddff39923ac8cac4446f8 Mon Sep 17 00:00:00 2001 From: Harry Date: Mon, 10 Jun 2019 22:43:35 +0100 Subject: [PATCH 140/165] make sure deallocation fixes view model too [deallocation_to_readmodel] --- src/allocation/domain/events.py | 6 ++++++ src/allocation/domain/model.py | 2 +- src/allocation/service_layer/handlers.py | 19 +++++++++++++++++++ src/allocation/service_layer/messagebus.py | 4 ++++ tests/integration/test_views.py | 12 ++++++++++++ 5 files changed, 42 insertions(+), 1 deletion(-) diff --git a/src/allocation/domain/events.py b/src/allocation/domain/events.py index 4a376291..3790fbbe 100644 --- a/src/allocation/domain/events.py +++ b/src/allocation/domain/events.py @@ -11,6 +11,12 @@ class Allocated(Event): qty: int batchref: str +@dataclass +class Deallocated(Event): + orderid: str + sku: str + qty: int + @dataclass class OutOfStock(Event): sku: str diff --git a/src/allocation/domain/model.py b/src/allocation/domain/model.py index 34f580d6..a33679c9 100644 --- a/src/allocation/domain/model.py +++ b/src/allocation/domain/model.py @@ -35,7 +35,7 @@ def change_batch_quantity(self, ref: str, qty: int): while batch.available_quantity < 0: line = batch.deallocate_one() self.events.append( - commands.Allocate(line.orderid, line.sku, line.qty) + events.Deallocated(line.orderid, line.sku, line.qty) ) @dataclass(unsafe_hash=True) diff --git a/src/allocation/service_layer/handlers.py b/src/allocation/service_layer/handlers.py index 2e42a829..520d9140 100644 --- a/src/allocation/service_layer/handlers.py +++ b/src/allocation/service_layer/handlers.py @@ -1,5 +1,6 @@ #pylint: disable=unused-argument from __future__ import annotations +from dataclasses import asdict from typing import TYPE_CHECKING from allocation.adapters import email, redis_eventpublisher from allocation.domain import commands, events, model @@ -37,6 +38,13 @@ def allocate( product.allocate(line) uow.commit() +def reallocate( + event: events.Deallocated, uow: unit_of_work.AbstractUnitOfWork +): + with uow: + product = uow.products.get(sku=event.sku) + product.events.append(commands.Allocate(**asdict(event))) + uow.commit() def change_batch_quantity( cmd: commands.ChangeBatchQuantity, uow: unit_of_work.AbstractUnitOfWork @@ -74,3 +82,14 @@ def add_allocation_to_read_model( dict(orderid=event.orderid, sku=event.sku, batchref=event.batchref) ) uow.commit() + +def remove_allocation_from_read_model( + event: events.Deallocated, uow: unit_of_work.SqlAlchemyUnitOfWork, +): + with uow: + uow.session.execute( + 'DELETE FROM allocations_view ' + ' WHERE orderid = :orderid AND sku = :sku', + dict(orderid=event.orderid, sku=event.sku) + ) + uow.commit() diff --git a/src/allocation/service_layer/messagebus.py b/src/allocation/service_layer/messagebus.py index cf30c3d3..97dacb3d 100644 --- a/src/allocation/service_layer/messagebus.py +++ b/src/allocation/service_layer/messagebus.py @@ -60,6 +60,10 @@ def handle_command( handlers.publish_allocated_event, handlers.add_allocation_to_read_model ], + events.Deallocated: [ + handlers.remove_allocation_from_read_model, + handlers.reallocate, + ], events.OutOfStock: [handlers.send_out_of_stock_notification], } # type: Dict[Type[events.Event], List[Callable]] diff --git a/tests/integration/test_views.py b/tests/integration/test_views.py index 78e3e3df..c157e89f 100644 --- a/tests/integration/test_views.py +++ b/tests/integration/test_views.py @@ -21,3 +21,15 @@ def test_allocations_view(sqlite_session_factory): {'sku': 'sku1', 'batchref': 'sku1batch'}, {'sku': 'sku2', 'batchref': 'sku2batch'}, ] + + +def test_deallocation(sqlite_session_factory): + uow = unit_of_work.SqlAlchemyUnitOfWork(sqlite_session_factory) + messagebus.handle(commands.CreateBatch('b1', 'sku1', 50, None), uow) + messagebus.handle(commands.CreateBatch('b2', 'sku1', 50, today), uow) + messagebus.handle(commands.Allocate('o1', 'sku1', 40), uow) + messagebus.handle(commands.ChangeBatchQuantity('b1', 10), uow) + + assert views.allocations('o1', uow) == [ + {'sku': 'sku1', 'batchref': 'b2'}, + ] From 7f1868cd0d2d923209d98ef637db793b78fc7f1d Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 26 Nov 2019 12:22:25 +0000 Subject: [PATCH 141/165] handlers talk to redis [redis_readmodel_handlers] --- src/allocation/service_layer/handlers.py | 24 ++++-------------------- 1 file changed, 4 insertions(+), 20 deletions(-) diff --git a/src/allocation/service_layer/handlers.py b/src/allocation/service_layer/handlers.py index 520d9140..bff56857 100644 --- a/src/allocation/service_layer/handlers.py +++ b/src/allocation/service_layer/handlers.py @@ -72,24 +72,8 @@ def publish_allocated_event( redis_eventpublisher.publish('line_allocated', event) -def add_allocation_to_read_model( - event: events.Allocated, uow: unit_of_work.SqlAlchemyUnitOfWork, -): - with uow: - uow.session.execute( - 'INSERT INTO allocations_view (orderid, sku, batchref)' - ' VALUES (:orderid, :sku, :batchref)', - dict(orderid=event.orderid, sku=event.sku, batchref=event.batchref) - ) - uow.commit() +def add_allocation_to_read_model(event: events.Allocated, _): + redis_eventpublisher.update_readmodel(event.orderid, event.sku, event.batchref) -def remove_allocation_from_read_model( - event: events.Deallocated, uow: unit_of_work.SqlAlchemyUnitOfWork, -): - with uow: - uow.session.execute( - 'DELETE FROM allocations_view ' - ' WHERE orderid = :orderid AND sku = :sku', - dict(orderid=event.orderid, sku=event.sku) - ) - uow.commit() +def remove_allocation_from_read_model(event: events.Deallocated, _): + redis_eventpublisher.update_readmodel(event.orderid, event.sku, None) From 51c336e2e4b71cc6cbc4445933779ecd2933f2de Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 26 Nov 2019 16:16:46 +0000 Subject: [PATCH 142/165] new helpers to update read model [redis_readmodel_client] --- src/allocation/adapters/redis_eventpublisher.py | 8 ++++++++ src/allocation/entrypoints/redis_eventconsumer.py | 1 + 2 files changed, 9 insertions(+) diff --git a/src/allocation/adapters/redis_eventpublisher.py b/src/allocation/adapters/redis_eventpublisher.py index 2afac458..2291ab1f 100644 --- a/src/allocation/adapters/redis_eventpublisher.py +++ b/src/allocation/adapters/redis_eventpublisher.py @@ -14,3 +14,11 @@ def publish(channel, event: events.Event): logging.debug('publishing: channel=%s, event=%s', channel, event) r.publish(channel, json.dumps(asdict(event))) + + +def update_readmodel(orderid, sku, batchref): + r.hset(orderid, sku, batchref) + + +def get_readmodel(orderid): + return r.hgetall(orderid) diff --git a/src/allocation/entrypoints/redis_eventconsumer.py b/src/allocation/entrypoints/redis_eventconsumer.py index a27cd02b..f334fb99 100644 --- a/src/allocation/entrypoints/redis_eventconsumer.py +++ b/src/allocation/entrypoints/redis_eventconsumer.py @@ -27,5 +27,6 @@ def handle_change_batch_quantity(m): cmd = commands.ChangeBatchQuantity(ref=data['batchref'], qty=data['qty']) messagebus.handle(cmd, uow=unit_of_work.SqlAlchemyUnitOfWork()) + if __name__ == '__main__': main() From 85e585d0c3279d95209bfa87ddebc755474fb040 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 26 Nov 2019 16:17:08 +0000 Subject: [PATCH 143/165] view now users redis, tweak tests+app. [redis_readmodel_view] --- src/allocation/entrypoints/flask_app.py | 3 +-- src/allocation/views.py | 14 +++++++------- tests/integration/test_views.py | 19 ++++++++++++++++--- 3 files changed, 24 insertions(+), 12 deletions(-) diff --git a/src/allocation/entrypoints/flask_app.py b/src/allocation/entrypoints/flask_app.py index 434a32f8..da5ef6de 100644 --- a/src/allocation/entrypoints/flask_app.py +++ b/src/allocation/entrypoints/flask_app.py @@ -40,8 +40,7 @@ def allocate_endpoint(): @app.route("/allocations/", methods=['GET']) def allocations_view_endpoint(orderid): - uow = unit_of_work.SqlAlchemyUnitOfWork() - result = views.allocations(orderid, uow) + result = views.allocations(orderid) if not result: return 'not found', 404 return jsonify(result), 200 diff --git a/src/allocation/views.py b/src/allocation/views.py index 4cd07397..f5dcc1ac 100644 --- a/src/allocation/views.py +++ b/src/allocation/views.py @@ -1,9 +1,9 @@ +from allocation.adapters import redis_eventpublisher from allocation.service_layer import unit_of_work -def allocations(orderid: str, uow: unit_of_work.SqlAlchemyUnitOfWork): - with uow: - results = list(uow.session.execute( - 'SELECT sku, batchref FROM allocations_view WHERE orderid = :orderid', - dict(orderid=orderid) - )) - return [dict(r) for r in results] +def allocations(orderid): + batches = redis_eventpublisher.get_readmodel(orderid) + return [ + {'batchref': b.decode(), 'sku': s.decode()} + for s, b in batches.items() + ] diff --git a/tests/integration/test_views.py b/tests/integration/test_views.py index c157e89f..08a26aae 100644 --- a/tests/integration/test_views.py +++ b/tests/integration/test_views.py @@ -1,11 +1,24 @@ from datetime import date -from allocation import views +import pytest +import redis +from allocation import config, views from allocation.domain import commands from allocation.service_layer import messagebus, unit_of_work today = date.today() +@pytest.fixture +def cleanup_redis(): + r = redis.Redis(**config.get_redis_host_and_port()) + yield + for k in r.keys(): + print('cleaning up redis key', k) + r.delete(k) + +pytestmark = pytest.mark.usefixtures('cleanup_redis') + + def test_allocations_view(sqlite_session_factory): uow = unit_of_work.SqlAlchemyUnitOfWork(sqlite_session_factory) messagebus.handle(commands.CreateBatch('sku1batch', 'sku1', 50, None), uow) @@ -17,7 +30,7 @@ def test_allocations_view(sqlite_session_factory): messagebus.handle(commands.Allocate('otherorder', 'sku1', 30), uow) messagebus.handle(commands.Allocate('otherorder', 'sku2', 10), uow) - assert views.allocations('order1', uow) == [ + assert views.allocations('order1') == [ {'sku': 'sku1', 'batchref': 'sku1batch'}, {'sku': 'sku2', 'batchref': 'sku2batch'}, ] @@ -30,6 +43,6 @@ def test_deallocation(sqlite_session_factory): messagebus.handle(commands.Allocate('o1', 'sku1', 40), uow) messagebus.handle(commands.ChangeBatchQuantity('b1', 10), uow) - assert views.allocations('o1', uow) == [ + assert views.allocations('o1') == [ {'sku': 'sku1', 'batchref': 'b2'}, ] From 164ba3e01a0537a42f8123da784398d672a3f833 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 26 Nov 2019 16:17:16 +0000 Subject: [PATCH 144/165] Revert "view now users redis, tweak tests+app. [...]" [chapter_12_cqrs_ends] This reverts commit 6dd38db04ea7c637f2d2510e7f587ac2009dd2ac. Revert "new helpers to update read model ..." This reverts commit dbaebdf249d952f2c3915526fd4d1bc6fe3cd18f. Revert "handlers talk to redis ..." This reverts commit 00658e2181de3e118579fa0a0da9b7ebb9e081f9. --- .../adapters/redis_eventpublisher.py | 8 ------- src/allocation/entrypoints/flask_app.py | 3 ++- src/allocation/service_layer/handlers.py | 24 +++++++++++++++---- src/allocation/views.py | 14 +++++------ tests/integration/test_views.py | 19 +++------------ 5 files changed, 32 insertions(+), 36 deletions(-) diff --git a/src/allocation/adapters/redis_eventpublisher.py b/src/allocation/adapters/redis_eventpublisher.py index 2291ab1f..2afac458 100644 --- a/src/allocation/adapters/redis_eventpublisher.py +++ b/src/allocation/adapters/redis_eventpublisher.py @@ -14,11 +14,3 @@ def publish(channel, event: events.Event): logging.debug('publishing: channel=%s, event=%s', channel, event) r.publish(channel, json.dumps(asdict(event))) - - -def update_readmodel(orderid, sku, batchref): - r.hset(orderid, sku, batchref) - - -def get_readmodel(orderid): - return r.hgetall(orderid) diff --git a/src/allocation/entrypoints/flask_app.py b/src/allocation/entrypoints/flask_app.py index da5ef6de..434a32f8 100644 --- a/src/allocation/entrypoints/flask_app.py +++ b/src/allocation/entrypoints/flask_app.py @@ -40,7 +40,8 @@ def allocate_endpoint(): @app.route("/allocations/", methods=['GET']) def allocations_view_endpoint(orderid): - result = views.allocations(orderid) + uow = unit_of_work.SqlAlchemyUnitOfWork() + result = views.allocations(orderid, uow) if not result: return 'not found', 404 return jsonify(result), 200 diff --git a/src/allocation/service_layer/handlers.py b/src/allocation/service_layer/handlers.py index bff56857..520d9140 100644 --- a/src/allocation/service_layer/handlers.py +++ b/src/allocation/service_layer/handlers.py @@ -72,8 +72,24 @@ def publish_allocated_event( redis_eventpublisher.publish('line_allocated', event) -def add_allocation_to_read_model(event: events.Allocated, _): - redis_eventpublisher.update_readmodel(event.orderid, event.sku, event.batchref) +def add_allocation_to_read_model( + event: events.Allocated, uow: unit_of_work.SqlAlchemyUnitOfWork, +): + with uow: + uow.session.execute( + 'INSERT INTO allocations_view (orderid, sku, batchref)' + ' VALUES (:orderid, :sku, :batchref)', + dict(orderid=event.orderid, sku=event.sku, batchref=event.batchref) + ) + uow.commit() -def remove_allocation_from_read_model(event: events.Deallocated, _): - redis_eventpublisher.update_readmodel(event.orderid, event.sku, None) +def remove_allocation_from_read_model( + event: events.Deallocated, uow: unit_of_work.SqlAlchemyUnitOfWork, +): + with uow: + uow.session.execute( + 'DELETE FROM allocations_view ' + ' WHERE orderid = :orderid AND sku = :sku', + dict(orderid=event.orderid, sku=event.sku) + ) + uow.commit() diff --git a/src/allocation/views.py b/src/allocation/views.py index f5dcc1ac..4cd07397 100644 --- a/src/allocation/views.py +++ b/src/allocation/views.py @@ -1,9 +1,9 @@ -from allocation.adapters import redis_eventpublisher from allocation.service_layer import unit_of_work -def allocations(orderid): - batches = redis_eventpublisher.get_readmodel(orderid) - return [ - {'batchref': b.decode(), 'sku': s.decode()} - for s, b in batches.items() - ] +def allocations(orderid: str, uow: unit_of_work.SqlAlchemyUnitOfWork): + with uow: + results = list(uow.session.execute( + 'SELECT sku, batchref FROM allocations_view WHERE orderid = :orderid', + dict(orderid=orderid) + )) + return [dict(r) for r in results] diff --git a/tests/integration/test_views.py b/tests/integration/test_views.py index 08a26aae..c157e89f 100644 --- a/tests/integration/test_views.py +++ b/tests/integration/test_views.py @@ -1,24 +1,11 @@ from datetime import date -import pytest -import redis -from allocation import config, views +from allocation import views from allocation.domain import commands from allocation.service_layer import messagebus, unit_of_work today = date.today() -@pytest.fixture -def cleanup_redis(): - r = redis.Redis(**config.get_redis_host_and_port()) - yield - for k in r.keys(): - print('cleaning up redis key', k) - r.delete(k) - -pytestmark = pytest.mark.usefixtures('cleanup_redis') - - def test_allocations_view(sqlite_session_factory): uow = unit_of_work.SqlAlchemyUnitOfWork(sqlite_session_factory) messagebus.handle(commands.CreateBatch('sku1batch', 'sku1', 50, None), uow) @@ -30,7 +17,7 @@ def test_allocations_view(sqlite_session_factory): messagebus.handle(commands.Allocate('otherorder', 'sku1', 30), uow) messagebus.handle(commands.Allocate('otherorder', 'sku2', 10), uow) - assert views.allocations('order1') == [ + assert views.allocations('order1', uow) == [ {'sku': 'sku1', 'batchref': 'sku1batch'}, {'sku': 'sku2', 'batchref': 'sku2batch'}, ] @@ -43,6 +30,6 @@ def test_deallocation(sqlite_session_factory): messagebus.handle(commands.Allocate('o1', 'sku1', 40), uow) messagebus.handle(commands.ChangeBatchQuantity('b1', 10), uow) - assert views.allocations('o1') == [ + assert views.allocations('o1', uow) == [ {'sku': 'sku1', 'batchref': 'b2'}, ] From 8474e8dae6cf7e24b741d2488cdc271da143cf67 Mon Sep 17 00:00:00 2001 From: Harry Date: Mon, 15 Jul 2019 16:57:14 +0100 Subject: [PATCH 145/165] handlers now have all and only explicit dependencies [handler_with_explicit_dependency] --- src/allocation/service_layer/handlers.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/allocation/service_layer/handlers.py b/src/allocation/service_layer/handlers.py index 520d9140..5355fe6c 100644 --- a/src/allocation/service_layer/handlers.py +++ b/src/allocation/service_layer/handlers.py @@ -1,8 +1,7 @@ #pylint: disable=unused-argument from __future__ import annotations from dataclasses import asdict -from typing import TYPE_CHECKING -from allocation.adapters import email, redis_eventpublisher +from typing import Callable, TYPE_CHECKING from allocation.domain import commands, events, model from allocation.domain.model import OrderLine if TYPE_CHECKING: @@ -38,6 +37,7 @@ def allocate( product.allocate(line) uow.commit() + def reallocate( event: events.Deallocated, uow: unit_of_work.AbstractUnitOfWork ): @@ -46,6 +46,7 @@ def reallocate( product.events.append(commands.Allocate(**asdict(event))) uow.commit() + def change_batch_quantity( cmd: commands.ChangeBatchQuantity, uow: unit_of_work.AbstractUnitOfWork ): @@ -58,18 +59,18 @@ def change_batch_quantity( #pylint: disable=unused-argument def send_out_of_stock_notification( - event: events.OutOfStock, uow: unit_of_work.AbstractUnitOfWork, + event: events.OutOfStock, send_mail: Callable, ): - email.send( + send_mail( 'stock@made.com', f'Out of stock for {event.sku}', ) def publish_allocated_event( - event: events.Allocated, uow: unit_of_work.AbstractUnitOfWork, + event: events.Allocated, publish: Callable, ): - redis_eventpublisher.publish('line_allocated', event) + publish('line_allocated', event) def add_allocation_to_read_model( @@ -83,6 +84,7 @@ def add_allocation_to_read_model( ) uow.commit() + def remove_allocation_from_read_model( event: events.Deallocated, uow: unit_of_work.SqlAlchemyUnitOfWork, ): From 9139c9f73ff664d83aa8453d138cdd734c2746d0 Mon Sep 17 00:00:00 2001 From: Harry Date: Mon, 6 Jan 2020 07:47:34 +0000 Subject: [PATCH 146/165] change reallocate handler to avoid cmd/event clash --- src/allocation/service_layer/handlers.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/allocation/service_layer/handlers.py b/src/allocation/service_layer/handlers.py index 5355fe6c..dcde0b34 100644 --- a/src/allocation/service_layer/handlers.py +++ b/src/allocation/service_layer/handlers.py @@ -41,10 +41,7 @@ def allocate( def reallocate( event: events.Deallocated, uow: unit_of_work.AbstractUnitOfWork ): - with uow: - product = uow.products.get(sku=event.sku) - product.events.append(commands.Allocate(**asdict(event))) - uow.commit() + allocate(commands.Allocate(**asdict(event)), uow=uow) def change_batch_quantity( From 36c411069ba4c191cc8593956e520cb371e2705a Mon Sep 17 00:00:00 2001 From: Harry Date: Mon, 6 Jan 2020 07:47:56 +0000 Subject: [PATCH 147/165] bring static handlers dicts across from messagebus --- src/allocation/service_layer/handlers.py | 15 ++++++++++++++- src/allocation/service_layer/messagebus.py | 21 +-------------------- 2 files changed, 15 insertions(+), 21 deletions(-) diff --git a/src/allocation/service_layer/handlers.py b/src/allocation/service_layer/handlers.py index dcde0b34..a48b8496 100644 --- a/src/allocation/service_layer/handlers.py +++ b/src/allocation/service_layer/handlers.py @@ -1,7 +1,7 @@ #pylint: disable=unused-argument from __future__ import annotations from dataclasses import asdict -from typing import Callable, TYPE_CHECKING +from typing import List, Dict, Callable, Type, TYPE_CHECKING from allocation.domain import commands, events, model from allocation.domain.model import OrderLine if TYPE_CHECKING: @@ -92,3 +92,16 @@ def remove_allocation_from_read_model( dict(orderid=event.orderid, sku=event.sku) ) uow.commit() + + +EVENT_HANDLERS = { + events.Allocated: [publish_allocated_event, add_allocation_to_read_model], + events.Deallocated: [remove_allocation_from_read_model, reallocate], + events.OutOfStock: [send_out_of_stock_notification], +} # type: Dict[Type[events.Event], List[Callable]] + +COMMAND_HANDLERS = { + commands.Allocate: allocate, + commands.CreateBatch: add_batch, + commands.ChangeBatchQuantity: change_batch_quantity, +} # type: Dict[Type[commands.Command], Callable] diff --git a/src/allocation/service_layer/messagebus.py b/src/allocation/service_layer/messagebus.py index 97dacb3d..36269ead 100644 --- a/src/allocation/service_layer/messagebus.py +++ b/src/allocation/service_layer/messagebus.py @@ -1,7 +1,7 @@ # pylint: disable=broad-except from __future__ import annotations import logging -from typing import List, Dict, Callable, Type, Union, TYPE_CHECKING +from typing import Union, TYPE_CHECKING from allocation.domain import commands, events from . import handlers @@ -53,22 +53,3 @@ def handle_command( except Exception: logger.exception('Exception handling command %s', command) raise - - -EVENT_HANDLERS = { - events.Allocated: [ - handlers.publish_allocated_event, - handlers.add_allocation_to_read_model - ], - events.Deallocated: [ - handlers.remove_allocation_from_read_model, - handlers.reallocate, - ], - events.OutOfStock: [handlers.send_out_of_stock_notification], -} # type: Dict[Type[events.Event], List[Callable]] - -COMMAND_HANDLERS = { - commands.Allocate: handlers.allocate, - commands.CreateBatch: handlers.add_batch, - commands.ChangeBatchQuantity: handlers.change_batch_quantity, -} # type: Dict[Type[commands.Command], Callable] From 05177d02b72a472e9007975293f875ed7d2361bd Mon Sep 17 00:00:00 2001 From: Harry Date: Sun, 5 Jan 2020 17:18:11 +0000 Subject: [PATCH 148/165] messagebus becomes a class, requires handlers [messagebus_as_class] --- src/allocation/service_layer/messagebus.py | 35 ++++++++++++++-------- 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/src/allocation/service_layer/messagebus.py b/src/allocation/service_layer/messagebus.py index 36269ead..345b360f 100644 --- a/src/allocation/service_layer/messagebus.py +++ b/src/allocation/service_layer/messagebus.py @@ -1,9 +1,8 @@ # pylint: disable=broad-except from __future__ import annotations import logging -from typing import Union, TYPE_CHECKING +from typing import Callable, Dict, List, Union, Type, TYPE_CHECKING from allocation.domain import commands, events -from . import handlers if TYPE_CHECKING: from . import unit_of_work @@ -13,16 +12,28 @@ Message = Union[commands.Command, events.Event] -def handle(message: Message, uow: unit_of_work.AbstractUnitOfWork): - queue = [message] - while queue: - message = queue.pop(0) - if isinstance(message, events.Event): - handle_event(message, queue, uow) - elif isinstance(message, commands.Command): - handle_command(message, queue, uow) - else: - raise Exception(f'{message} was not an Event or Command') +class MessageBus: + + def __init__( + self, + uow: unit_of_work.AbstractUnitOfWork, + event_handlers: Dict[Type[events.Event], List[Callable]], + command_handlers: Dict[Type[commands.Command], Callable], + ): + self.uow = uow + self.event_handlers = event_handlers + self.command_handlers = command_handlers + + def handle(self, message: Message): + self.queue = [message] + while self.queue: + message = self.queue.pop(0) + if isinstance(message, events.Event): + self.handle_event(message) + elif isinstance(message, commands.Command): + self.handle_command(message) + else: + raise Exception(f'{message} was not an Event or Command') def handle_event( From 77461e1f9d029f2265f343cc7ba6bcdd43673051 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 7 Jan 2020 14:48:53 +0000 Subject: [PATCH 149/165] use self.handlers for handle_event and handle_command [messagebus_handlers_change] --- src/allocation/service_layer/messagebus.py | 46 +++++++++------------- 1 file changed, 19 insertions(+), 27 deletions(-) diff --git a/src/allocation/service_layer/messagebus.py b/src/allocation/service_layer/messagebus.py index 345b360f..e6ccae66 100644 --- a/src/allocation/service_layer/messagebus.py +++ b/src/allocation/service_layer/messagebus.py @@ -1,4 +1,4 @@ -# pylint: disable=broad-except +# pylint: disable=broad-except, attribute-defined-outside-init from __future__ import annotations import logging from typing import Callable, Dict, List, Union, Type, TYPE_CHECKING @@ -36,31 +36,23 @@ def handle(self, message: Message): raise Exception(f'{message} was not an Event or Command') -def handle_event( - event: events.Event, - queue: List[Message], - uow: unit_of_work.AbstractUnitOfWork -): - for handler in EVENT_HANDLERS[type(event)]: - try: - logger.debug('handling event %s with handler %s', event, handler) - handler(event, uow=uow) - queue.extend(uow.collect_new_events()) - except Exception: - logger.exception('Exception handling event %s', event) - continue + def handle_event(self, event: events.Event): + for handler in self.event_handlers[type(event)]: + try: + logger.debug('handling event %s with handler %s', event, handler) + handler(event) + self.queue.extend(self.uow.collect_new_events()) + except Exception: + logger.exception('Exception handling event %s', event) + continue -def handle_command( - command: commands.Command, - queue: List[Message], - uow: unit_of_work.AbstractUnitOfWork -): - logger.debug('handling command %s', command) - try: - handler = COMMAND_HANDLERS[type(command)] - handler(command, uow=uow) - queue.extend(uow.collect_new_events()) - except Exception: - logger.exception('Exception handling command %s', command) - raise + def handle_command(self, command: commands.Command): + logger.debug('handling command %s', command) + try: + handler = self.command_handlers[type(command)] + handler(command) + self.queue.extend(self.uow.collect_new_events()) + except Exception: + logger.exception('Exception handling command %s', command) + raise From 12c60e0d61d3889e2726716f57dbd65b873203af Mon Sep 17 00:00:00 2001 From: Harry Date: Mon, 15 Jul 2019 17:08:54 +0100 Subject: [PATCH 150/165] conftest change to backport, session_factory -> sqlite_session_factory --- tests/conftest.py | 10 +++------- tests/integration/test_repository.py | 5 +++-- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 455428dd..5a83e224 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -17,21 +17,17 @@ pytest.register_assert_rewrite('tests.e2e.api_client') @pytest.fixture -def in_memory_db(): +def in_memory_sqlite_db(): engine = create_engine('sqlite:///:memory:') metadata.create_all(engine) return engine @pytest.fixture -def sqlite_session_factory(in_memory_db): +def sqlite_session_factory(in_memory_sqlite_db): start_mappers() - yield sessionmaker(bind=in_memory_db) + yield sessionmaker(bind=in_memory_sqlite_db) clear_mappers() -@pytest.fixture -def sqlite_session(sqlite_session_factory): - return sqlite_session_factory() - @retry(stop=stop_after_delay(10)) def wait_for_postgres_to_come_up(engine): diff --git a/tests/integration/test_repository.py b/tests/integration/test_repository.py index d3b936df..d49efb98 100644 --- a/tests/integration/test_repository.py +++ b/tests/integration/test_repository.py @@ -1,8 +1,9 @@ from allocation.adapters import repository from allocation.domain import model -def test_get_by_batchref(sqlite_session): - repo = repository.SqlAlchemyRepository(sqlite_session) +def test_get_by_batchref(sqlite_session_factory): + session = sqlite_session_factory() + repo = repository.SqlAlchemyRepository(session) b1 = model.Batch(ref='b1', sku='sku1', qty=100, eta=None) b2 = model.Batch(ref='b2', sku='sku1', qty=100, eta=None) b3 = model.Batch(ref='b3', sku='sku2', qty=100, eta=None) From 57faacc5be033328ed21bae5966909f053ad43f7 Mon Sep 17 00:00:00 2001 From: Harry Date: Mon, 15 Jul 2019 17:09:18 +0100 Subject: [PATCH 151/165] conftest change to backport to ch 5, isolation serializable in all pg tests --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 5a83e224..a9ed1703 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -47,7 +47,7 @@ def wait_for_redis_to_come_up(): @pytest.fixture(scope='session') def postgres_db(): - engine = create_engine(config.get_postgres_uri()) + engine = create_engine(config.get_postgres_uri(), isolation_level='SERIALIZABLE') wait_for_postgres_to_come_up(engine) metadata.create_all(engine) return engine From 3c6f6adda78c243ffe3dac46d0acb3c8b01c0939 Mon Sep 17 00:00:00 2001 From: Harry Date: Mon, 15 Jul 2019 17:09:59 +0100 Subject: [PATCH 152/165] uow tests maybe backport, pass explicit uow to threads --- tests/integration/test_uow.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/tests/integration/test_uow.py b/tests/integration/test_uow.py index 0c7264ef..84b43c6e 100644 --- a/tests/integration/test_uow.py +++ b/tests/integration/test_uow.py @@ -1,4 +1,4 @@ -# pylint: disable=broad-except +# pylint: disable=broad-except, too-many-arguments import threading import time import traceback @@ -75,15 +75,15 @@ class MyException(Exception): assert rows == [] -def try_to_allocate(orderid, sku, exceptions): +def try_to_allocate(orderid, sku, exceptions, session_factory): line = model.OrderLine(orderid, sku, 10) try: - with unit_of_work.SqlAlchemyUnitOfWork() as uow: + with unit_of_work.SqlAlchemyUnitOfWork(session_factory) as uow: product = uow.products.get(sku=sku) product.allocate(line) time.sleep(0.2) uow.commit() - except Exception as e: + except Exception as e: # pylint: disable=broad-except print(traceback.format_exc()) exceptions.append(e) @@ -96,8 +96,12 @@ def test_concurrent_updates_to_version_are_not_allowed(postgres_session_factory) order1, order2 = random_orderid(1), random_orderid(2) exceptions = [] # type: List[Exception] - try_to_allocate_order1 = lambda: try_to_allocate(order1, sku, exceptions) - try_to_allocate_order2 = lambda: try_to_allocate(order2, sku, exceptions) + try_to_allocate_order1 = lambda: try_to_allocate( + order1, sku, exceptions, postgres_session_factory + ) + try_to_allocate_order2 = lambda: try_to_allocate( + order2, sku, exceptions, postgres_session_factory + ) thread1 = threading.Thread(target=try_to_allocate_order1) thread2 = threading.Thread(target=try_to_allocate_order2) thread1.start() @@ -121,5 +125,5 @@ def test_concurrent_updates_to_version_are_not_allowed(postgres_session_factory) dict(sku=sku), )) assert len(orders) == 1 - with unit_of_work.SqlAlchemyUnitOfWork() as uow: + with unit_of_work.SqlAlchemyUnitOfWork(postgres_session_factory) as uow: uow.session.execute('select 1') From b3e08bca36073808c24eb6b9713dc4afb77ef560 Mon Sep 17 00:00:00 2001 From: Harry Date: Mon, 7 Oct 2019 14:29:59 +0100 Subject: [PATCH 153/165] bootstrap script preps DI'd handlers and start orm [bootstrap_script] --- src/allocation/bootstrap.py | 44 +++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 src/allocation/bootstrap.py diff --git a/src/allocation/bootstrap.py b/src/allocation/bootstrap.py new file mode 100644 index 00000000..fc65d34a --- /dev/null +++ b/src/allocation/bootstrap.py @@ -0,0 +1,44 @@ +import inspect +from typing import Callable +from allocation.adapters import email, orm, redis_eventpublisher +from allocation.service_layer import handlers, messagebus, unit_of_work + + +def bootstrap( + start_orm: bool = True, + uow: unit_of_work.AbstractUnitOfWork = unit_of_work.SqlAlchemyUnitOfWork(), + send_mail: Callable = email.send, + publish: Callable = redis_eventpublisher.publish, +) -> messagebus.MessageBus: + + if start_orm: + orm.start_mappers() + + dependencies = {'uow': uow, 'send_mail': send_mail, 'publish': publish} + injected_event_handlers = { + event_type: [ + inject_dependencies(handler, dependencies) + for handler in event_handlers + ] + for event_type, event_handlers in handlers.EVENT_HANDLERS.items() + } + injected_command_handlers = { + command_type: inject_dependencies(handler, dependencies) + for command_type, handler in handlers.COMMAND_HANDLERS.items() + } + + return messagebus.MessageBus( + uow=uow, + event_handlers=injected_event_handlers, + command_handlers=injected_command_handlers, + ) + + +def inject_dependencies(handler, dependencies): + params = inspect.signature(handler).parameters + deps = { + name: dependency + for name, dependency in dependencies.items() + if name in params + } + return lambda message: handler(message, **deps) From 96e76ec52cb526abc4fe80974500eb7ed04cc11b Mon Sep 17 00:00:00 2001 From: Harry Date: Mon, 15 Jul 2019 17:03:43 +0100 Subject: [PATCH 154/165] use bootstrap in service layer tests [bootstrap_tests] --- tests/unit/test_handlers.py | 114 ++++++++++++++++-------------------- 1 file changed, 52 insertions(+), 62 deletions(-) diff --git a/tests/unit/test_handlers.py b/tests/unit/test_handlers.py index 4aaee189..e50a8548 100644 --- a/tests/unit/test_handlers.py +++ b/tests/unit/test_handlers.py @@ -1,10 +1,12 @@ # pylint: disable=no-self-use +from __future__ import annotations from datetime import date from unittest import mock import pytest +from allocation import bootstrap from allocation.adapters import repository -from allocation.domain import commands, events -from allocation.service_layer import handlers, messagebus, unit_of_work +from allocation.domain import commands +from allocation.service_layer import handlers, unit_of_work class FakeRepository(repository.AbstractRepository): @@ -39,100 +41,88 @@ def rollback(self): pass +def bootstrap_test_app(): + return bootstrap.bootstrap( + start_orm=False, + uow=FakeUnitOfWork(), + send_mail=lambda *args: None, + publish=lambda *args: None, + ) + class TestAddBatch: def test_for_new_product(self): - uow = FakeUnitOfWork() - messagebus.handle( - commands.CreateBatch("b1", "CRUNCHY-ARMCHAIR", 100, None), uow - ) - assert uow.products.get("CRUNCHY-ARMCHAIR") is not None - assert uow.committed + bus = bootstrap_test_app() + bus.handle(commands.CreateBatch("b1", "CRUNCHY-ARMCHAIR", 100, None)) + assert bus.uow.products.get("CRUNCHY-ARMCHAIR") is not None + assert bus.uow.committed def test_for_existing_product(self): - uow = FakeUnitOfWork() - messagebus.handle(commands.CreateBatch("b1", "GARISH-RUG", 100, None), uow) - messagebus.handle(commands.CreateBatch("b2", "GARISH-RUG", 99, None), uow) - assert "b2" in [b.reference for b in uow.products.get("GARISH-RUG").batches] - - - -@pytest.fixture(autouse=True) -def fake_redis_publish(): - with mock.patch("allocation.adapters.redis_eventpublisher.publish"): - yield + bus = bootstrap_test_app() + bus.handle(commands.CreateBatch("b1", "GARISH-RUG", 100, None)) + bus.handle(commands.CreateBatch("b2", "GARISH-RUG", 99, None)) + assert "b2" in [b.reference for b in bus.uow.products.get("GARISH-RUG").batches] class TestAllocate: def test_allocates(self): - uow = FakeUnitOfWork() - messagebus.handle( - commands.CreateBatch("batch1", "COMPLICATED-LAMP", 100, None), uow - ) - messagebus.handle( - commands.Allocate("o1", "COMPLICATED-LAMP", 10), uow - ) - [batch] = uow.products.get("COMPLICATED-LAMP").batches + bus = bootstrap_test_app() + bus.handle(commands.CreateBatch("batch1", "COMPLICATED-LAMP", 100, None)) + bus.handle(commands.Allocate("o1", "COMPLICATED-LAMP", 10)) + [batch] = bus.uow.products.get("COMPLICATED-LAMP").batches assert batch.available_quantity == 90 def test_errors_for_invalid_sku(self): - uow = FakeUnitOfWork() - messagebus.handle(commands.CreateBatch("b1", "AREALSKU", 100, None), uow) + bus = bootstrap_test_app() + bus.handle(commands.CreateBatch("b1", "AREALSKU", 100, None)) with pytest.raises(handlers.InvalidSku, match="Invalid sku NONEXISTENTSKU"): - messagebus.handle( - commands.Allocate("o1", "NONEXISTENTSKU", 10), uow - ) + bus.handle(commands.Allocate("o1", "NONEXISTENTSKU", 10)) def test_commits(self): - uow = FakeUnitOfWork() - messagebus.handle( - commands.CreateBatch("b1", "OMINOUS-MIRROR", 100, None), uow - ) - messagebus.handle( - commands.Allocate("o1", "OMINOUS-MIRROR", 10), uow - ) - assert uow.committed + bus = bootstrap_test_app() + bus.handle(commands.CreateBatch("b1", "OMINOUS-MIRROR", 100, None)) + bus.handle(commands.Allocate("o1", "OMINOUS-MIRROR", 10)) + assert bus.uow.committed def test_sends_email_on_out_of_stock_error(self): - uow = FakeUnitOfWork() - messagebus.handle( - commands.CreateBatch("b1", "POPULAR-CURTAINS", 9, None), uow + emails = [] + def fake_send_mail(*args): + emails.append(args) + bus = bootstrap.bootstrap( + start_orm=False, + uow=FakeUnitOfWork(), + send_mail=fake_send_mail, + publish=lambda *args: None, ) - - with mock.patch("allocation.adapters.email.send") as mock_send_mail: - messagebus.handle( - commands.Allocate("o1", "POPULAR-CURTAINS", 10), uow - ) - assert mock_send_mail.call_args == mock.call( - "stock@made.com", f"Out of stock for POPULAR-CURTAINS" - ) + bus.handle(commands.CreateBatch("b1", "POPULAR-CURTAINS", 9, None)) + bus.handle(commands.Allocate("o1", "POPULAR-CURTAINS", 10)) + assert emails == [ + ("stock@made.com", f"Out of stock for POPULAR-CURTAINS"), + ] class TestChangeBatchQuantity: def test_changes_available_quantity(self): - uow = FakeUnitOfWork() - messagebus.handle( - commands.CreateBatch("batch1", "ADORABLE-SETTEE", 100, None), uow - ) - [batch] = uow.products.get(sku="ADORABLE-SETTEE").batches + bus = bootstrap_test_app() + bus.handle(commands.CreateBatch("batch1", "ADORABLE-SETTEE", 100, None)) + [batch] = bus.uow.products.get(sku="ADORABLE-SETTEE").batches assert batch.available_quantity == 100 - messagebus.handle(commands.ChangeBatchQuantity("batch1", 50), uow) - + bus.handle(commands.ChangeBatchQuantity("batch1", 50)) assert batch.available_quantity == 50 def test_reallocates_if_necessary(self): - uow = FakeUnitOfWork() + bus = bootstrap_test_app() history = [ commands.CreateBatch("batch1", "INDIFFERENT-TABLE", 50, None), commands.CreateBatch("batch2", "INDIFFERENT-TABLE", 50, date.today()), @@ -140,12 +130,12 @@ def test_reallocates_if_necessary(self): commands.Allocate("order2", "INDIFFERENT-TABLE", 20), ] for msg in history: - messagebus.handle(msg, uow) - [batch1, batch2] = uow.products.get(sku="INDIFFERENT-TABLE").batches + bus.handle(msg) + [batch1, batch2] = bus.uow.products.get(sku="INDIFFERENT-TABLE").batches assert batch1.available_quantity == 10 assert batch2.available_quantity == 50 - messagebus.handle(commands.ChangeBatchQuantity("batch1", 25), uow) + bus.handle(commands.ChangeBatchQuantity("batch1", 25)) # order1 or order2 will be deallocated, so we'll have 25 - 20 assert batch1.available_quantity == 5 From 6a8afb6e59b0a766ec176d19ade8daeeb02cf233 Mon Sep 17 00:00:00 2001 From: Harry Date: Mon, 6 Jan 2020 07:32:55 +0000 Subject: [PATCH 155/165] fixture to start mappers explicitly, use in repo and uow tests --- tests/conftest.py | 9 ++++++--- tests/integration/test_repository.py | 4 ++++ tests/integration/test_uow.py | 3 +++ 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index a9ed1703..14e6dcf9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -24,11 +24,16 @@ def in_memory_sqlite_db(): @pytest.fixture def sqlite_session_factory(in_memory_sqlite_db): - start_mappers() yield sessionmaker(bind=in_memory_sqlite_db) + +@pytest.fixture +def mappers(): + start_mappers() + yield clear_mappers() + @retry(stop=stop_after_delay(10)) def wait_for_postgres_to_come_up(engine): return engine.connect() @@ -54,9 +59,7 @@ def postgres_db(): @pytest.fixture def postgres_session_factory(postgres_db): - start_mappers() yield sessionmaker(bind=postgres_db) - clear_mappers() @pytest.fixture def postgres_session(postgres_session_factory): diff --git a/tests/integration/test_repository.py b/tests/integration/test_repository.py index d49efb98..2d0c87ab 100644 --- a/tests/integration/test_repository.py +++ b/tests/integration/test_repository.py @@ -1,6 +1,10 @@ +import pytest from allocation.adapters import repository from allocation.domain import model +pytestmark = pytest.mark.usefixtures('mappers') + + def test_get_by_batchref(sqlite_session_factory): session = sqlite_session_factory() repo = repository.SqlAlchemyRepository(session) diff --git a/tests/integration/test_uow.py b/tests/integration/test_uow.py index 84b43c6e..04aff1c1 100644 --- a/tests/integration/test_uow.py +++ b/tests/integration/test_uow.py @@ -3,11 +3,14 @@ import time import traceback from typing import List +from unittest.mock import Mock import pytest from allocation.domain import model from allocation.service_layer import unit_of_work from ..random_refs import random_sku, random_batchref, random_orderid +pytestmark = pytest.mark.usefixtures('mappers') + def insert_batch(session, ref, sku, qty, eta, product_version=1): session.execute( From f954daec39591d215962efe55be004a4cc7e296d Mon Sep 17 00:00:00 2001 From: Harry Date: Mon, 6 Jan 2020 05:07:03 +0000 Subject: [PATCH 156/165] fix view tests to use bootstrap. [bootstrap_view_tests] --- tests/integration/test_views.py | 50 ++++++++++++++++++++------------- 1 file changed, 31 insertions(+), 19 deletions(-) diff --git a/tests/integration/test_views.py b/tests/integration/test_views.py index c157e89f..01b6884c 100644 --- a/tests/integration/test_views.py +++ b/tests/integration/test_views.py @@ -1,35 +1,47 @@ +# pylint: disable=redefined-outer-name from datetime import date -from allocation import views +from sqlalchemy.orm import clear_mappers +import pytest +from allocation import bootstrap, views from allocation.domain import commands -from allocation.service_layer import messagebus, unit_of_work +from allocation.service_layer import unit_of_work today = date.today() -def test_allocations_view(sqlite_session_factory): - uow = unit_of_work.SqlAlchemyUnitOfWork(sqlite_session_factory) - messagebus.handle(commands.CreateBatch('sku1batch', 'sku1', 50, None), uow) - messagebus.handle(commands.CreateBatch('sku2batch', 'sku2', 50, today), uow) - messagebus.handle(commands.Allocate('order1', 'sku1', 20), uow) - messagebus.handle(commands.Allocate('order1', 'sku2', 20), uow) +@pytest.fixture +def sqlite_bus(sqlite_session_factory): + bus = bootstrap.bootstrap( + start_orm=True, + uow=unit_of_work.SqlAlchemyUnitOfWork(sqlite_session_factory), + send_mail=lambda *args: None, + publish=lambda *args: None, + ) + yield bus + clear_mappers() + +def test_allocations_view(sqlite_bus): + sqlite_bus.handle(commands.CreateBatch('sku1batch', 'sku1', 50, None)) + sqlite_bus.handle(commands.CreateBatch('sku2batch', 'sku2', 50, today)) + sqlite_bus.handle(commands.Allocate('order1', 'sku1', 20)) + sqlite_bus.handle(commands.Allocate('order1', 'sku2', 20)) # add a spurious batch and order to make sure we're getting the right ones - messagebus.handle(commands.CreateBatch('sku1batch-later', 'sku1', 50, today), uow) - messagebus.handle(commands.Allocate('otherorder', 'sku1', 30), uow) - messagebus.handle(commands.Allocate('otherorder', 'sku2', 10), uow) + sqlite_bus.handle(commands.CreateBatch('sku1batch-later', 'sku1', 50, today)) + sqlite_bus.handle(commands.Allocate('otherorder', 'sku1', 30)) + sqlite_bus.handle(commands.Allocate('otherorder', 'sku2', 10)) - assert views.allocations('order1', uow) == [ + assert views.allocations('order1', sqlite_bus.uow) == [ {'sku': 'sku1', 'batchref': 'sku1batch'}, {'sku': 'sku2', 'batchref': 'sku2batch'}, ] -def test_deallocation(sqlite_session_factory): - uow = unit_of_work.SqlAlchemyUnitOfWork(sqlite_session_factory) - messagebus.handle(commands.CreateBatch('b1', 'sku1', 50, None), uow) - messagebus.handle(commands.CreateBatch('b2', 'sku1', 50, today), uow) - messagebus.handle(commands.Allocate('o1', 'sku1', 40), uow) - messagebus.handle(commands.ChangeBatchQuantity('b1', 10), uow) +def test_deallocation(sqlite_bus): + sqlite_bus.handle(commands.CreateBatch('b1', 'sku1', 50, None)) + sqlite_bus.handle(commands.CreateBatch('b2', 'sku1', 50, today)) + sqlite_bus.handle(commands.Allocate('o1', 'sku1', 40)) + sqlite_bus.handle(commands.ChangeBatchQuantity('b1', 10)) - assert views.allocations('o1', uow) == [ + assert views.allocations('o1', sqlite_bus.uow) == [ {'sku': 'sku1', 'batchref': 'b2'}, ] From d0c9333be0a0500f6c95e13189723ef902df3706 Mon Sep 17 00:00:00 2001 From: Harry Date: Mon, 7 Oct 2019 14:44:15 +0100 Subject: [PATCH 157/165] use bootstrap in flask [flask_calls_bootstrap] --- src/allocation/entrypoints/flask_app.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/src/allocation/entrypoints/flask_app.py b/src/allocation/entrypoints/flask_app.py index 434a32f8..2d3f76ac 100644 --- a/src/allocation/entrypoints/flask_app.py +++ b/src/allocation/entrypoints/flask_app.py @@ -1,14 +1,11 @@ from datetime import datetime from flask import Flask, jsonify, request - from allocation.domain import commands -from allocation.adapters import orm -from allocation.service_layer import messagebus, unit_of_work from allocation.service_layer.handlers import InvalidSku -from allocation import views +from allocation import bootstrap, views app = Flask(__name__) -orm.start_mappers() +bus = bootstrap.bootstrap() @app.route("/add_batch", methods=['POST']) @@ -19,8 +16,7 @@ def add_batch(): cmd = commands.CreateBatch( request.json['ref'], request.json['sku'], request.json['qty'], eta, ) - uow = unit_of_work.SqlAlchemyUnitOfWork() - messagebus.handle(cmd, uow) + bus.handle(cmd) return 'OK', 201 @@ -30,8 +26,7 @@ def allocate_endpoint(): cmd = commands.Allocate( request.json['orderid'], request.json['sku'], request.json['qty'], ) - uow = unit_of_work.SqlAlchemyUnitOfWork() - messagebus.handle(cmd, uow) + bus.handle(cmd) except InvalidSku as e: return jsonify({'message': str(e)}), 400 @@ -40,8 +35,7 @@ def allocate_endpoint(): @app.route("/allocations/", methods=['GET']) def allocations_view_endpoint(orderid): - uow = unit_of_work.SqlAlchemyUnitOfWork() - result = views.allocations(orderid, uow) + result = views.allocations(orderid, bus.uow) if not result: return 'not found', 404 return jsonify(result), 200 From a113c8d95bd763b772af2b199de837afe64bdd9d Mon Sep 17 00:00:00 2001 From: Harry Date: Mon, 6 Jan 2020 06:59:59 +0000 Subject: [PATCH 158/165] use bootstrap for redis --- src/allocation/entrypoints/redis_eventconsumer.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/allocation/entrypoints/redis_eventconsumer.py b/src/allocation/entrypoints/redis_eventconsumer.py index f334fb99..87eea8cc 100644 --- a/src/allocation/entrypoints/redis_eventconsumer.py +++ b/src/allocation/entrypoints/redis_eventconsumer.py @@ -2,30 +2,29 @@ import logging import redis -from allocation import config +from allocation import bootstrap, config from allocation.domain import commands -from allocation.adapters import orm -from allocation.service_layer import messagebus, unit_of_work logger = logging.getLogger(__name__) r = redis.Redis(**config.get_redis_host_and_port()) + def main(): - orm.start_mappers() + bus = bootstrap.bootstrap() pubsub = r.pubsub(ignore_subscribe_messages=True) pubsub.subscribe('change_batch_quantity') for m in pubsub.listen(): - handle_change_batch_quantity(m) + handle_change_batch_quantity(m, bus) -def handle_change_batch_quantity(m): +def handle_change_batch_quantity(m, bus): logging.debug('handling %s', m) data = json.loads(m['data']) cmd = commands.ChangeBatchQuantity(ref=data['batchref'], qty=data['qty']) - messagebus.handle(cmd, uow=unit_of_work.SqlAlchemyUnitOfWork()) + bus.handle(cmd) if __name__ == '__main__': From 44fe427d27a44b7dda389c0e5a4c9cc87607f8ed Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 7 Jan 2020 22:32:52 +0000 Subject: [PATCH 159/165] experiment with nonmagic DI zzzz [nomagic_di] --- src/allocation/bootstrap.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/src/allocation/bootstrap.py b/src/allocation/bootstrap.py index fc65d34a..897c86e3 100644 --- a/src/allocation/bootstrap.py +++ b/src/allocation/bootstrap.py @@ -1,5 +1,5 @@ -import inspect from typing import Callable +from allocation.domain import commands, events from allocation.adapters import email, orm, redis_eventpublisher from allocation.service_layer import handlers, messagebus, unit_of_work @@ -14,17 +14,25 @@ def bootstrap( if start_orm: orm.start_mappers() - dependencies = {'uow': uow, 'send_mail': send_mail, 'publish': publish} injected_event_handlers = { - event_type: [ - inject_dependencies(handler, dependencies) - for handler in event_handlers + events.Allocated: [ + lambda e: handlers.publish_allocated_event(e, publish), + lambda e: handlers.add_allocation_to_read_model(e, uow), + ], + events.Deallocated: [ + lambda e: handlers.remove_allocation_from_read_model(e, uow), + lambda e: handlers.reallocate(e, uow), + ], + events.OutOfStock: [ + lambda e: handlers.send_out_of_stock_notification(e, send_mail) ] - for event_type, event_handlers in handlers.EVENT_HANDLERS.items() } injected_command_handlers = { - command_type: inject_dependencies(handler, dependencies) - for command_type, handler in handlers.COMMAND_HANDLERS.items() + commands.Allocate: lambda c: handlers.allocate(c, uow), + commands.CreateBatch: \ + lambda c: handlers.add_batch(c, uow), + commands.ChangeBatchQuantity: \ + lambda c: handlers.change_batch_quantity(c, uow), } return messagebus.MessageBus( From 358339413dfe6d8f0628d7c4ab8855a5bc56f2ec Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 7 Jan 2020 22:36:23 +0000 Subject: [PATCH 160/165] Revert "experiment with nonmagic DI zzzz" This reverts commit f42b193857eec8560443767de9fd7ada9f2acb96. --- src/allocation/bootstrap.py | 24 ++++++++---------------- 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/src/allocation/bootstrap.py b/src/allocation/bootstrap.py index 897c86e3..fc65d34a 100644 --- a/src/allocation/bootstrap.py +++ b/src/allocation/bootstrap.py @@ -1,5 +1,5 @@ +import inspect from typing import Callable -from allocation.domain import commands, events from allocation.adapters import email, orm, redis_eventpublisher from allocation.service_layer import handlers, messagebus, unit_of_work @@ -14,25 +14,17 @@ def bootstrap( if start_orm: orm.start_mappers() + dependencies = {'uow': uow, 'send_mail': send_mail, 'publish': publish} injected_event_handlers = { - events.Allocated: [ - lambda e: handlers.publish_allocated_event(e, publish), - lambda e: handlers.add_allocation_to_read_model(e, uow), - ], - events.Deallocated: [ - lambda e: handlers.remove_allocation_from_read_model(e, uow), - lambda e: handlers.reallocate(e, uow), - ], - events.OutOfStock: [ - lambda e: handlers.send_out_of_stock_notification(e, send_mail) + event_type: [ + inject_dependencies(handler, dependencies) + for handler in event_handlers ] + for event_type, event_handlers in handlers.EVENT_HANDLERS.items() } injected_command_handlers = { - commands.Allocate: lambda c: handlers.allocate(c, uow), - commands.CreateBatch: \ - lambda c: handlers.add_batch(c, uow), - commands.ChangeBatchQuantity: \ - lambda c: handlers.change_batch_quantity(c, uow), + command_type: inject_dependencies(handler, dependencies) + for command_type, handler in handlers.COMMAND_HANDLERS.items() } return messagebus.MessageBus( From cd7ba9d7def752b020e35283234aafca0b3d3f72 Mon Sep 17 00:00:00 2001 From: Harry Date: Sun, 14 Jul 2019 02:55:46 +0100 Subject: [PATCH 161/165] switch to a notifications class [notifications_class] --- src/allocation/adapters/email.py | 2 -- src/allocation/adapters/notifications.py | 31 ++++++++++++++++++++++++ src/allocation/bootstrap.py | 12 ++++++--- src/allocation/config.py | 5 ++++ src/allocation/service_layer/handlers.py | 6 +++-- 5 files changed, 49 insertions(+), 7 deletions(-) delete mode 100644 src/allocation/adapters/email.py create mode 100644 src/allocation/adapters/notifications.py diff --git a/src/allocation/adapters/email.py b/src/allocation/adapters/email.py deleted file mode 100644 index bd48f980..00000000 --- a/src/allocation/adapters/email.py +++ /dev/null @@ -1,2 +0,0 @@ -def send(*args): - print('SENDING EMAIL:', *args) diff --git a/src/allocation/adapters/notifications.py b/src/allocation/adapters/notifications.py new file mode 100644 index 00000000..1d398d16 --- /dev/null +++ b/src/allocation/adapters/notifications.py @@ -0,0 +1,31 @@ +#pylint: disable=too-few-public-methods +import abc +import smtplib +from allocation import config + + +class AbstractNotifications(abc.ABC): + + @abc.abstractmethod + def send(self, destination, message): + raise NotImplementedError + + +DEFAULT_HOST = config.get_email_host_and_port()['host'] +DEFAULT_PORT = config.get_email_host_and_port()['port'] + + + +class EmailNotifications(AbstractNotifications): + + def __init__(self, smtp_host=DEFAULT_HOST, port=DEFAULT_PORT): + self.server = smtplib.SMTP(smtp_host, port=port) + self.server.noop() + + def send(self, destination, message): + msg = f'Subject: allocation service notification\n{message}' + self.server.sendmail( + from_addr='allocations@example.com', + to_addrs=[destination], + msg=msg + ) diff --git a/src/allocation/bootstrap.py b/src/allocation/bootstrap.py index fc65d34a..842c24ce 100644 --- a/src/allocation/bootstrap.py +++ b/src/allocation/bootstrap.py @@ -1,20 +1,26 @@ import inspect from typing import Callable -from allocation.adapters import email, orm, redis_eventpublisher +from allocation.adapters import orm, redis_eventpublisher +from allocation.adapters.notifications import ( + AbstractNotifications, EmailNotifications +) from allocation.service_layer import handlers, messagebus, unit_of_work def bootstrap( start_orm: bool = True, uow: unit_of_work.AbstractUnitOfWork = unit_of_work.SqlAlchemyUnitOfWork(), - send_mail: Callable = email.send, + notifications: AbstractNotifications = None, publish: Callable = redis_eventpublisher.publish, ) -> messagebus.MessageBus: + if notifications is None: + notifications = EmailNotifications() + if start_orm: orm.start_mappers() - dependencies = {'uow': uow, 'send_mail': send_mail, 'publish': publish} + dependencies = {'uow': uow, 'notifications': notifications, 'publish': publish} injected_event_handlers = { event_type: [ inject_dependencies(handler, dependencies) diff --git a/src/allocation/config.py b/src/allocation/config.py index efc65cf7..13d23b4d 100644 --- a/src/allocation/config.py +++ b/src/allocation/config.py @@ -19,3 +19,8 @@ def get_redis_host_and_port(): port = 63791 if host == 'localhost' else 6379 return dict(host=host, port=port) +def get_email_host_and_port(): + host = os.environ.get('EMAIL_HOST', 'localhost') + port = 11025 if host == 'localhost' else 1025 + http_port = 18025 if host == 'localhost' else 8025 + return dict(host=host, port=port, http_port=http_port) diff --git a/src/allocation/service_layer/handlers.py b/src/allocation/service_layer/handlers.py index a48b8496..75995f7b 100644 --- a/src/allocation/service_layer/handlers.py +++ b/src/allocation/service_layer/handlers.py @@ -5,6 +5,7 @@ from allocation.domain import commands, events, model from allocation.domain.model import OrderLine if TYPE_CHECKING: + from allocation.adapters import notifications from . import unit_of_work @@ -12,6 +13,7 @@ class InvalidSku(Exception): pass + def add_batch( cmd: commands.CreateBatch, uow: unit_of_work.AbstractUnitOfWork ): @@ -56,9 +58,9 @@ def change_batch_quantity( #pylint: disable=unused-argument def send_out_of_stock_notification( - event: events.OutOfStock, send_mail: Callable, + event: events.OutOfStock, notifications: notifications.AbstractNotifications, ): - send_mail( + notifications.send( 'stock@made.com', f'Out of stock for {event.sku}', ) From b3aa844138221b9ccbc581d05edac981dcc59406 Mon Sep 17 00:00:00 2001 From: Harry Date: Sun, 14 Jul 2019 02:56:00 +0100 Subject: [PATCH 162/165] tests for notifcations [notifications_unit_tests] --- tests/integration/test_views.py | 3 ++- tests/unit/test_handlers.py | 30 ++++++++++++++++++++---------- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/tests/integration/test_views.py b/tests/integration/test_views.py index 01b6884c..b125870f 100644 --- a/tests/integration/test_views.py +++ b/tests/integration/test_views.py @@ -1,6 +1,7 @@ # pylint: disable=redefined-outer-name from datetime import date from sqlalchemy.orm import clear_mappers +from unittest import mock import pytest from allocation import bootstrap, views from allocation.domain import commands @@ -14,7 +15,7 @@ def sqlite_bus(sqlite_session_factory): bus = bootstrap.bootstrap( start_orm=True, uow=unit_of_work.SqlAlchemyUnitOfWork(sqlite_session_factory), - send_mail=lambda *args: None, + notifications=mock.Mock(), publish=lambda *args: None, ) yield bus diff --git a/tests/unit/test_handlers.py b/tests/unit/test_handlers.py index e50a8548..49124dc5 100644 --- a/tests/unit/test_handlers.py +++ b/tests/unit/test_handlers.py @@ -1,12 +1,14 @@ # pylint: disable=no-self-use from __future__ import annotations +from collections import defaultdict from datetime import date -from unittest import mock +from typing import Dict, List import pytest from allocation import bootstrap -from allocation.adapters import repository from allocation.domain import commands -from allocation.service_layer import handlers, unit_of_work +from allocation.service_layer import handlers +from allocation.adapters import notifications, repository +from allocation.service_layer import unit_of_work class FakeRepository(repository.AbstractRepository): @@ -41,11 +43,21 @@ def rollback(self): pass +class FakeNotifications(notifications.AbstractNotifications): + + def __init__(self): + self.sent = defaultdict(list) # type: Dict[str, List[str]] + + def send(self, destination, message): + self.sent[destination].append(message) + + + def bootstrap_test_app(): return bootstrap.bootstrap( start_orm=False, uow=FakeUnitOfWork(), - send_mail=lambda *args: None, + notifications=FakeNotifications(), publish=lambda *args: None, ) @@ -92,19 +104,17 @@ def test_commits(self): def test_sends_email_on_out_of_stock_error(self): - emails = [] - def fake_send_mail(*args): - emails.append(args) + fake_notifs = FakeNotifications() bus = bootstrap.bootstrap( start_orm=False, uow=FakeUnitOfWork(), - send_mail=fake_send_mail, + notifications=fake_notifs, publish=lambda *args: None, ) bus.handle(commands.CreateBatch("b1", "POPULAR-CURTAINS", 9, None)) bus.handle(commands.Allocate("o1", "POPULAR-CURTAINS", 10)) - assert emails == [ - ("stock@made.com", f"Out of stock for POPULAR-CURTAINS"), + assert fake_notifs.sent['stock@made.com'] == [ + f"Out of stock for POPULAR-CURTAINS", ] From a99562da4e156a478fc444f556e02a4fa509e20f Mon Sep 17 00:00:00 2001 From: Harry Date: Sun, 14 Jul 2019 03:23:07 +0100 Subject: [PATCH 163/165] add a mailhog fake email server to docker-compose --- docker-compose.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index dc2cc369..f964ab74 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,10 +10,12 @@ services: depends_on: - postgres - redis + - mailhog environment: - DB_HOST=postgres - DB_PASSWORD=abc123 - REDIS_HOST=redis + - EMAIL_HOST=mailhog - PYTHONDONTWRITEBYTECODE=1 volumes: - ./src:/src @@ -26,11 +28,13 @@ services: image: allocation-image depends_on: - redis_pubsub + - mailhog environment: - DB_HOST=postgres - DB_PASSWORD=abc123 - API_HOST=api - REDIS_HOST=redis + - EMAIL_HOST=mailhog - PYTHONDONTWRITEBYTECODE=1 - FLASK_APP=allocation/entrypoints/flask_app.py - FLASK_DEBUG=1 @@ -59,3 +63,8 @@ services: ports: - "63791:6379" + mailhog: + image: mailhog/mailhog + ports: + - "11025:1025" + - "18025:8025" From 24b9c2f70438e89b0d402a11c490aba08c8f0e35 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 8 Oct 2019 12:56:58 +0100 Subject: [PATCH 164/165] logging.info needs backport --- src/allocation/adapters/orm.py | 4 +++- src/allocation/adapters/redis_eventpublisher.py | 2 +- src/allocation/entrypoints/redis_eventconsumer.py | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/allocation/adapters/orm.py b/src/allocation/adapters/orm.py index 16ebdcfb..c9b7a5d0 100644 --- a/src/allocation/adapters/orm.py +++ b/src/allocation/adapters/orm.py @@ -1,3 +1,4 @@ +import logging from sqlalchemy import ( Table, MetaData, Column, Integer, String, Date, ForeignKey, event, @@ -6,6 +7,7 @@ from allocation.domain import model +logger = logging.getLogger(__name__) metadata = MetaData() @@ -48,6 +50,7 @@ def start_mappers(): + logger.info("Starting mappers") lines_mapper = mapper(model.OrderLine, order_lines) batches_mapper = mapper(model.Batch, batches, properties={ '_allocations': relationship( @@ -63,4 +66,3 @@ def start_mappers(): @event.listens_for(model.Product, 'load') def receive_load(product, _): product.events = [] - diff --git a/src/allocation/adapters/redis_eventpublisher.py b/src/allocation/adapters/redis_eventpublisher.py index 2afac458..8e37ab26 100644 --- a/src/allocation/adapters/redis_eventpublisher.py +++ b/src/allocation/adapters/redis_eventpublisher.py @@ -12,5 +12,5 @@ def publish(channel, event: events.Event): - logging.debug('publishing: channel=%s, event=%s', channel, event) + logging.info('publishing: channel=%s, event=%s', channel, event) r.publish(channel, json.dumps(asdict(event))) diff --git a/src/allocation/entrypoints/redis_eventconsumer.py b/src/allocation/entrypoints/redis_eventconsumer.py index 87eea8cc..4c15e541 100644 --- a/src/allocation/entrypoints/redis_eventconsumer.py +++ b/src/allocation/entrypoints/redis_eventconsumer.py @@ -12,6 +12,7 @@ def main(): + logger.info('Redis pubsub starting') bus = bootstrap.bootstrap() pubsub = r.pubsub(ignore_subscribe_messages=True) pubsub.subscribe('change_batch_quantity') @@ -21,7 +22,7 @@ def main(): def handle_change_batch_quantity(m, bus): - logging.debug('handling %s', m) + logger.info('handling %s', m) data = json.loads(m['data']) cmd = commands.ChangeBatchQuantity(ref=data['batchref'], qty=data['qty']) bus.handle(cmd) From e3de63c922c2094d63949949b4cdf776fb5462dc Mon Sep 17 00:00:00 2001 From: Harry Date: Sun, 14 Jul 2019 03:23:44 +0100 Subject: [PATCH 165/165] integration test for emails [chapter_13_dependency_injection_ends] --- tests/integration/test_email.py | 37 +++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 tests/integration/test_email.py diff --git a/tests/integration/test_email.py b/tests/integration/test_email.py new file mode 100644 index 00000000..56b04d9a --- /dev/null +++ b/tests/integration/test_email.py @@ -0,0 +1,37 @@ +#pylint: disable=redefined-outer-name +import pytest +import requests +from sqlalchemy.orm import clear_mappers +from allocation import bootstrap, config +from allocation.domain import commands +from allocation.adapters import notifications +from allocation.service_layer import unit_of_work +from ..random_refs import random_sku + + +@pytest.fixture +def bus(sqlite_session_factory): + bus = bootstrap.bootstrap( + start_orm=True, + uow=unit_of_work.SqlAlchemyUnitOfWork(sqlite_session_factory), + notifications=notifications.EmailNotifications(), + publish=lambda *args: None, + ) + yield bus + clear_mappers() + + +def get_email_from_mailhog(sku): + host, port = map(config.get_email_host_and_port().get, ['host', 'http_port']) + all_emails = requests.get(f'http://{host}:{port}/api/v2/messages').json() + return next(m for m in all_emails['items'] if sku in str(m)) + + +def test_out_of_stock_email(bus): + sku = random_sku() + bus.handle(commands.CreateBatch('batch1', sku, 9, None)) + bus.handle(commands.Allocate('order1', sku, 10)) + email = get_email_from_mailhog(sku) + assert email['Raw']['From'] == 'allocations@example.com' + assert email['Raw']['To'] == ['stock@made.com'] + assert f'Out of stock for {sku}' in email['Raw']['Data']