DynamoDB scan with filter expression returns different results from AWS #9560
Description
Hi,
While investigating something else, I found two DynamoDB scans using a filter expression that return different results in moto compared to AWS. Here's the simplest example I could boil it down to:
import boto3
TABLE_NAME = "repro_table"
USE_MOTO = True
def create_table(client):
client.create_table(
TableName=TABLE_NAME,
AttributeDefinitions=[
{
'AttributeName': 'model_id',
'AttributeType': 'S'
},
],
KeySchema=[
{
'AttributeName': 'model_id',
'KeyType': 'HASH'
},
],
BillingMode="PROVISIONED",
ProvisionedThroughput={
"ReadCapacityUnits": 1,
"WriteCapacityUnits": 1,
},
)
print("Created table")
def scan_primary_boto(client):
for i, (filter_expr, attr_values) in enumerate((
("(attribute_exists (is_visible) AND is_visible <> :0)", {':0': {'NULL': True}}),
("(attribute_exists (is_visible) AND is_visible = :0)", {':0': {'BOOL': True}}),
)):
resp = client.scan(
TableName=TABLE_NAME,
FilterExpression=filter_expr,
ExpressionAttributeValues=attr_values,
)
ids = [item["model_id"]["S"] for item in resp["Items"]]
print(f"query {i} returned {ids}")
def repro() -> None:
client = boto3.client("dynamodb")
if USE_MOTO:
create_table(client)
client.put_item(
TableName=TABLE_NAME,
Item={"model_id": {"S": "NoBoolean"}}
)
client.put_item(
TableName=TABLE_NAME,
Item={"model_id": {"S": "NullBoolean"}, "is_visible": {"NULL": True}}
)
client.put_item(
TableName=TABLE_NAME,
Item={"model_id": {"S": "BooleanTrue"}, "is_visible": {"BOOL": True}}
)
client.put_item(
TableName=TABLE_NAME,
Item={"model_id": {"S": "BooleanFalse"}, "is_visible": {"BOOL": False}}
)
scan_primary_boto(client)
def main():
if USE_MOTO:
print("Using Moto")
from moto import mock_aws
with mock_aws():
repro()
else:
print("Using AWS")
repro()
if __name__ == "__main__":
main()When run against AWS, this outputs:
Using AWS
query 0 returned ['BooleanTrue', 'BooleanFalse']
query 1 returned ['BooleanTrue']
but with moto, it returns
Using Moto
Created table
query 0 returned ['BooleanFalse']
query 1 returned ['BooleanTrue', 'NullBoolean']
The filter expressions originally came from PynamoDB, which might be a little easier to understand:
filter0 = (MyModel.is_visible.exists()) & (MyModel.is_visible != None)
filter1 = (MyModel.is_visible.exists()) & (MyModel.is_visible == True)I believe the AWS results are correct:
- In the first query, moto misses the item with
is_visible=True - In the second query, moto incorrectly returns the item with
is_visible=None
I'll try to submit a patch for this when I have time (probably over holiday break).