Thanks to visit codestin.com
Credit goes to github.com

Skip to content

MOD-7293 fix AOF commands duplicated in JSON.MSET #1212

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 8 commits into from
Jul 3, 2024
Merged

MOD-7293 fix AOF commands duplicated in JSON.MSET #1212

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
6d6bb57
1. Write to AOF outside the iterator
eranhd Jun 23, 2024
fbb9288
skip on cluster mode and enforce useAof
eranhd Jun 24, 2024
eb99f24
call to replicate_verbatim in seperate to apply_changes.
eranhd Jun 25, 2024
a1b7c90
create new apply_changes API in the manager to call him while we want…
eranhd Jul 2, 2024
8bfaf8c
Merge remote-tracking branch 'origin/master' into eranhd_aof_duplicat…
eranhd Jul 3, 2024
9dfdb7a
lint
eranhd Jul 3, 2024
b635bbd
fix
eranhd Jul 3, 2024
fd5adba
lint
eranhd Jul 3, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 29 additions & 3 deletions redis_json/src/commands.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,7 @@ pub fn json_set<M: Manager>(manager: M, ctx: &Context, args: Vec<RedisString>) -
if op != SetOptions::NotExists {
redis_key.set_value(Vec::new(), val)?;
redis_key.apply_changes(ctx, "json.set")?;
ctx.replicate_verbatim();
REDIS_OK
} else {
Ok(RedisValue::Null)
Expand All @@ -203,6 +204,7 @@ pub fn json_set<M: Manager>(manager: M, ctx: &Context, args: Vec<RedisString>) -
let updated = apply_updates::<M>(&mut redis_key, val, update_info);
if updated {
redis_key.apply_changes(ctx, "json.set")?;
ctx.replicate_verbatim();
REDIS_OK
} else {
Ok(RedisValue::Null)
Expand All @@ -217,6 +219,7 @@ pub fn json_set<M: Manager>(manager: M, ctx: &Context, args: Vec<RedisString>) -
if path.get_path() == JSON_ROOT_PATH {
redis_key.set_value(Vec::new(), val)?;
redis_key.apply_changes(ctx, "json.set")?;
ctx.replicate_verbatim();
REDIS_OK
} else {
Err(RedisError::Str(
Expand Down Expand Up @@ -258,6 +261,7 @@ pub fn json_merge<M: Manager>(manager: M, ctx: &Context, args: Vec<RedisString>)
if path.get_path() == JSON_ROOT_PATH {
redis_key.merge_value(Vec::new(), val)?;
redis_key.apply_changes(ctx, "json.merge")?;
ctx.replicate_verbatim();
REDIS_OK
} else {
let mut update_info =
Expand All @@ -283,6 +287,7 @@ pub fn json_merge<M: Manager>(manager: M, ctx: &Context, args: Vec<RedisString>)
}
if res {
redis_key.apply_changes(ctx, "json.merge")?;
ctx.replicate_verbatim();
REDIS_OK
} else {
Ok(RedisValue::Null)
Expand All @@ -297,6 +302,7 @@ pub fn json_merge<M: Manager>(manager: M, ctx: &Context, args: Vec<RedisString>)
// Nothing to merge with it's a new doc
redis_key.set_value(Vec::new(), val)?;
redis_key.apply_changes(ctx, "json.merge")?;
ctx.replicate_verbatim();
REDIS_OK
} else {
Err(RedisError::Str(
Expand Down Expand Up @@ -344,7 +350,7 @@ pub fn json_mset<M: Manager>(manager: M, ctx: &Context, args: Vec<RedisString>)
actions.push((redis_key, update_info, value));
}

actions
let res = actions
.into_iter()
.fold(REDIS_OK, |res, (mut redis_key, update_info, value)| {
let updated = if let Some(update_info) = update_info {
Expand All @@ -354,10 +360,13 @@ pub fn json_mset<M: Manager>(manager: M, ctx: &Context, args: Vec<RedisString>)
redis_key.set_value(Vec::new(), value)?
};
if updated {
redis_key.apply_changes(ctx, "json.mset")?
redis_key.notify_keyspace_event(ctx, "json.mset")?
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think the separation you are doing here is right, manager should probably not handle the replication. But I believe we should make this separation all over the code, WDYT?
In addition, maybe we should find a more general name to it? Because on HDT it also need to apply the changes on the key and fire the effects.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

how it's managed in redis? there are calling to replication and notify in the commands? or it's something that happened in the behind the scenes?

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Notify is per command.
Replication is a totally different mechanism.

}
res
})
});

ctx.replicate_verbatim();
res
}

fn apply_updates<M: Manager>(
Expand Down Expand Up @@ -582,6 +591,7 @@ pub fn json_del<M: Manager>(manager: M, ctx: &Context, args: Vec<RedisString>) -
};
if res > 0 {
redis_key.apply_changes(ctx, "json.del")?;
ctx.replicate_verbatim();
}
res
}
Expand Down Expand Up @@ -778,6 +788,7 @@ where
}
if need_notify {
redis_key.apply_changes(ctx, cmd)?;
ctx.replicate_verbatim();
}
Ok(res)
}
Expand Down Expand Up @@ -809,6 +820,7 @@ where
});
}
redis_key.apply_changes(ctx, cmd)?;
ctx.replicate_verbatim();
Ok(res.unwrap().to_string().into())
} else {
Err(RedisError::String(
Expand Down Expand Up @@ -895,6 +907,7 @@ where
}
if need_notify {
redis_key.apply_changes(ctx, "json.toggle")?;
ctx.replicate_verbatim();
}
Ok(res.into())
}
Expand All @@ -917,6 +930,7 @@ where
res = redis_key.bool_toggle(p)?;
}
redis_key.apply_changes(ctx, "json.toggle")?;
ctx.replicate_verbatim();
Ok(res.to_string().into())
} else {
Err(RedisError::String(
Expand Down Expand Up @@ -987,6 +1001,7 @@ where
}
if need_notify {
redis_key.apply_changes(ctx, "json.strappend")?;
ctx.replicate_verbatim();
}
Ok(res.into())
}
Expand All @@ -1011,6 +1026,7 @@ where
res = Some(redis_key.str_append(p, json.to_string())?);
}
redis_key.apply_changes(ctx, "json.strappend")?;
ctx.replicate_verbatim();
Ok(res.unwrap().into())
} else {
Err(RedisError::String(
Expand Down Expand Up @@ -1117,13 +1133,15 @@ where
} else if paths.len() == 1 {
let res = redis_key.arr_append(paths.pop().unwrap(), args)?;
redis_key.apply_changes(ctx, "json.arrappend")?;
ctx.replicate_verbatim();
Ok(res.into())
} else {
let mut res = 0;
for p in paths {
res = redis_key.arr_append(p, args.clone())?;
}
redis_key.apply_changes(ctx, "json.arrappend")?;
ctx.replicate_verbatim();
Ok(res.into())
}
}
Expand Down Expand Up @@ -1155,6 +1173,7 @@ where
}
if need_notify {
redis_key.apply_changes(ctx, "json.arrappend")?;
ctx.replicate_verbatim();
}
Ok(res.into())
}
Expand Down Expand Up @@ -1283,6 +1302,7 @@ where

if need_notify {
redis_key.apply_changes(ctx, "json.arrinsert")?;
ctx.replicate_verbatim();
}
Ok(res.into())
}
Expand Down Expand Up @@ -1312,6 +1332,7 @@ where
res = Some(redis_key.arr_insert(p, &args, index)?);
}
redis_key.apply_changes(ctx, "json.arrinsert")?;
ctx.replicate_verbatim();
Ok(res.unwrap().into())
}
}
Expand Down Expand Up @@ -1464,6 +1485,7 @@ where
}
if need_notify {
redis_key.apply_changes(ctx, "json.arrpop")?;
ctx.replicate_verbatim();
}
Ok(res.into())
}
Expand Down Expand Up @@ -1491,6 +1513,7 @@ where
})?);
}
redis_key.apply_changes(ctx, "json.arrpop")?;
ctx.replicate_verbatim();
res
} else {
Err(RedisError::String(
Expand Down Expand Up @@ -1546,6 +1569,7 @@ where
}
if need_notify {
redis_key.apply_changes(ctx, "json.arrtrim")?;
ctx.replicate_verbatim();
}
Ok(res.into())
}
Expand Down Expand Up @@ -1575,6 +1599,7 @@ where
res = Some(redis_key.arr_trim(p, start, stop)?);
}
redis_key.apply_changes(ctx, "json.arrtrim")?;
ctx.replicate_verbatim();
Ok(res.unwrap().into())
}
}
Expand Down Expand Up @@ -1734,6 +1759,7 @@ pub fn json_clear<M: Manager>(manager: M, ctx: &Context, args: Vec<RedisString>)
}
if cleared > 0 {
redis_key.apply_changes(ctx, "json.clear")?;
ctx.replicate_verbatim();
}
Ok(cleared.into())
}
Expand Down
5 changes: 4 additions & 1 deletion redis_json/src/ivalue_manager.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -282,10 +282,13 @@ impl<'a> IValueKeyHolderWrite<'a> {

impl<'a> WriteHolder<IValue, IValue> for IValueKeyHolderWrite<'a> {
fn apply_changes(&mut self, ctx: &Context, command: &str) -> Result<(), RedisError> {
self.notify_keyspace_event(ctx, command)
}

fn notify_keyspace_event(&mut self, ctx: &Context, command: &str) -> Result<(), RedisError> {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If we're splitting the functionality of apply_changes then we should have it call notify_keyspace_event to reduce duplication. eg something like

fn apply_changes(&mut self, ctx: &Context, command: &str) -> Result<(), RedisError> {
    self
        .notify_keyspace_event(ctx, command)
        .inspect(|_| ctx.replicate_verbatim())
}

if ctx.notify_keyspace_event(NotifyEvent::MODULE, command, &self.key_name) != Status::Ok {
Err(RedisError::Str("failed notify key space event"))
} else {
ctx.replicate_verbatim();
Ok(())
}
}
Expand Down
1 change: 1 addition & 0 deletions redis_json/src/manager.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ pub trait WriteHolder<O: Clone, V: SelectValue> {
fn arr_trim(&mut self, path: Vec<String>, start: i64, stop: i64) -> Result<usize, RedisError>;
fn clear(&mut self, path: Vec<String>) -> Result<usize, RedisError>;
fn apply_changes(&mut self, ctx: &Context, command: &str) -> Result<(), RedisError>;
fn notify_keyspace_event(&mut self, ctx: &Context, command: &str) -> Result<(), RedisError>;
}

pub trait Manager {
Expand Down
43 changes: 43 additions & 0 deletions tests/pytest/test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import os
import redis
import json
import time
from RLTest import Env
from includes import *
from redis.client import NEVER_DECODE
Expand Down Expand Up @@ -1487,6 +1488,48 @@ def test_promote_u64_to_f64(env):
r.assertEqual(val, float(2 * i64max + 3)) # u64 + u64 promotes to f64. as prior, not breaking
r.expect('JSON.TYPE', 'num', '$').equal(['number']) # promoted


def test_mset_replication_in_aof(env):
env.skipOnCluster()
env = Env(useAof=True)
with env.getClusterConnectionIfNeeded() as r:
r.execute_command('config', 'set', 'notify-keyspace-events', 'KEA')

pubsub = r.pubsub()
pubsub.psubscribe('__key*')

time.sleep(1)
env.assertEqual('psubscribe', pubsub.get_message(timeout=1)['type'])

command = [b'JSON.MSET']
data = 'a' * 100
num_params = 5
for i in range(0, num_params):
key = f'k:{i}'
value = json.dumps({"data": data})
command.append(key.encode())
command.append(b'$')
command.append(value.encode())
env.expect(*command).ok()

for i in range(0, num_params):
msg = pubsub.get_message(timeout=1)
env.assertEqual(msg['data'], 'json.mset')
env.assertEqual(msg['type'], 'pmessage')
msg = pubsub.get_message(timeout=1)
env.assertEqual(msg['data'], f'k:{i}')
env.assertEqual(msg['type'], 'pmessage')

# verify JSON.MSET is appearing in the AOF once only
role = 'master'
aof_fn = env.envRunner._getFileName(role, '.aof.1.incr.aof')
with open(f'{env.logDir}/appendonlydir/{aof_fn}', 'r') as fd:
aof_content = [l for l in fd.readlines() if 'JSON.MSET' in l]
assert(len(aof_content) == 1)




# class CacheTestCase(BaseReJSONTest):
# @property
# def module_args(env):
Expand Down