From 29e99963dee1856cd881186df4dbb8b69b189b9c Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Thu, 13 Apr 2017 13:44:21 +0300 Subject: [PATCH 1/4] group: reset all partitions if invoked with -partition -1 --- group.go | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/group.go b/group.go index 4a060047..4917cb32 100644 --- a/group.go +++ b/group.go @@ -43,6 +43,9 @@ type groupOffset struct { Lag int64 `json:"lag"` } +const fetchAllPartitions = -23 +const resetAllPartitions = -1 + func (cmd *groupCmd) run(args []string, q chan struct{}) { var err error @@ -114,7 +117,7 @@ func (cmd *groupCmd) printGroupTopicOffset(grp, top string) { results := make(chan groupOffsetResult) done := make(chan struct{}) parts := []int32{cmd.partition} - if cmd.partition == -23 { + if cmd.partition == fetchAllPartitions || cmd.partition == resetAllPartitions { parts = cmd.fetchPartitions(top) } wg := &sync.WaitGroup{} @@ -369,8 +372,8 @@ func (cmd *groupCmd) parseArgs(as []string) { failf("filter regexp invalid err=%v", err) } - if args.reset != "" && (args.topic == "" || args.partition == -23 || args.group == "") { - failf("group, topic, partition are required to reset offsets") + if args.reset != "" && (args.topic == "" || args.partition == fetchAllPartitions || args.group == "") { + failf("group and topic are required to reset offsets. for all partitions, run with -partition -1") } switch args.reset { @@ -429,7 +432,7 @@ func (cmd *groupCmd) parseFlags(as []string) groupArgs { flags.StringVar(&args.reset, "reset", "", "Target offset to reset for consumer group (newest, oldest, or specific offset)") flags.BoolVar(&args.verbose, "verbose", false, "More verbose logging to stderr.") flags.StringVar(&args.version, "version", "", "Kafka protocol version") - flags.IntVar(&args.partition, "partition", -23, "Partition to limit offsets to") + flags.IntVar(&args.partition, "partition", fetchAllPartitions, "Partition to limit offsets to") flags.BoolVar(&args.offsets, "offsets", true, "Controls if offsets should be fetched (defauls to true)") flags.Usage = func() { @@ -468,4 +471,8 @@ kt group -topic fav-topic To reset a consumer group's offset: kt group -reset 23 -topic fav-topic -group specials -partition 2 + +To reset a consumer group's offset for all partitions: + +kt group -reset newest -topic fav-topic -group specials -partition -1 ` From a92724772a4df42278a4506db0ebe3821e488ede Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Thu, 13 Apr 2017 14:42:57 +0300 Subject: [PATCH 2/4] group: fix race resetting multiple partitions --- group.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/group.go b/group.go index 4917cb32..192d790c 100644 --- a/group.go +++ b/group.go @@ -180,19 +180,20 @@ func (cmd *groupCmd) fetchGroupOffset(wg *sync.WaitGroup, grp, top string, part } if cmd.reset >= 0 || cmd.reset == sarama.OffsetNewest || cmd.reset == sarama.OffsetOldest { - switch cmd.reset { + rst := cmd.reset + switch rst { case sarama.OffsetNewest, sarama.OffsetOldest: off, err := cmd.client.GetOffset(top, part, cmd.reset) if err != nil { failf("failed to get offset to reset to err=%v", err) } - cmd.reset = off + rst = off if cmd.verbose { - fmt.Fprintf(os.Stderr, "resolved reset offset to %v", cmd.reset) + fmt.Fprintf(os.Stderr, "resolved reset offset to %v", rst) } } - groupOff = cmd.reset - pom.MarkOffset(cmd.reset, "") + groupOff = rst + pom.MarkOffset(rst, "") } if partOff, err = cmd.client.GetOffset(top, part, sarama.OffsetNewest); err != nil { From db906c6dc6054d21b665759b739d774c12c403bc Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Mon, 17 Apr 2017 10:54:21 +0300 Subject: [PATCH 3/4] group: better param name in reset, include partition id in error message and verbose log --- group.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/group.go b/group.go index 192d790c..437c1fcd 100644 --- a/group.go +++ b/group.go @@ -180,20 +180,20 @@ func (cmd *groupCmd) fetchGroupOffset(wg *sync.WaitGroup, grp, top string, part } if cmd.reset >= 0 || cmd.reset == sarama.OffsetNewest || cmd.reset == sarama.OffsetOldest { - rst := cmd.reset - switch rst { + resolvedOff := cmd.reset + switch resolvedOff { case sarama.OffsetNewest, sarama.OffsetOldest: off, err := cmd.client.GetOffset(top, part, cmd.reset) if err != nil { - failf("failed to get offset to reset to err=%v", err) + failf("failed to get offset to reset to for partition=%d err=%v", part, err) } - rst = off + resolvedOff = off if cmd.verbose { - fmt.Fprintf(os.Stderr, "resolved reset offset to %v", rst) + fmt.Fprintf(os.Stderr, "resolved reset offset for partition=%d to %v", part, resolvedOff) } } - groupOff = rst - pom.MarkOffset(rst, "") + groupOff = resolvedOff + pom.MarkOffset(resolvedOff, "") } if partOff, err = cmd.client.GetOffset(top, part, sarama.OffsetNewest); err != nil { From b97201e253359c7abb389696d93fc1f1faa038ce Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Mon, 17 Apr 2017 11:34:50 +0300 Subject: [PATCH 4/4] group: Add -partition all param --- group.go | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/group.go b/group.go index 437c1fcd..cc7ea4c4 100644 --- a/group.go +++ b/group.go @@ -43,8 +43,9 @@ type groupOffset struct { Lag int64 `json:"lag"` } -const fetchAllPartitions = -23 -const resetAllPartitions = -1 +const allPartitions = -1 +const allPartitionsHuman = "all" +const resetNotSpecified = -23 func (cmd *groupCmd) run(args []string, q chan struct{}) { var err error @@ -117,7 +118,7 @@ func (cmd *groupCmd) printGroupTopicOffset(grp, top string) { results := make(chan groupOffsetResult) done := make(chan struct{}) parts := []int32{cmd.partition} - if cmd.partition == fetchAllPartitions || cmd.partition == resetAllPartitions { + if cmd.partition == allPartitions { parts = cmd.fetchPartitions(top) } wg := &sync.WaitGroup{} @@ -367,14 +368,23 @@ func (cmd *groupCmd) parseArgs(as []string) { cmd.verbose = args.verbose cmd.offsets = args.offsets cmd.version = kafkaVersion(args.version) - cmd.partition = int32(args.partition) + + if strings.ToLower(args.partition) == allPartitionsHuman { + cmd.partition = allPartitions + } else { + p, err := strconv.ParseInt(args.partition, 10, 32) + if err != nil { + failf("partition id invalid err=%v", err) + } + cmd.partition = int32(p) + } if cmd.filter, err = regexp.Compile(args.filter); err != nil { failf("filter regexp invalid err=%v", err) } - if args.reset != "" && (args.topic == "" || args.partition == fetchAllPartitions || args.group == "") { - failf("group and topic are required to reset offsets. for all partitions, run with -partition -1") + if args.reset != "" && (args.topic == "" || args.group == "") { + failf("group and topic are required to reset offsets.") } switch args.reset { @@ -384,7 +394,7 @@ func (cmd *groupCmd) parseArgs(as []string) { cmd.reset = sarama.OffsetOldest case "": // optional flag - cmd.reset = -23 + cmd.reset = resetNotSpecified default: cmd.reset, err = strconv.ParseInt(args.reset, 10, 64) if err != nil { @@ -414,7 +424,7 @@ func (cmd *groupCmd) parseArgs(as []string) { type groupArgs struct { topic string brokers string - partition int + partition string group string filter string reset string @@ -433,7 +443,7 @@ func (cmd *groupCmd) parseFlags(as []string) groupArgs { flags.StringVar(&args.reset, "reset", "", "Target offset to reset for consumer group (newest, oldest, or specific offset)") flags.BoolVar(&args.verbose, "verbose", false, "More verbose logging to stderr.") flags.StringVar(&args.version, "version", "", "Kafka protocol version") - flags.IntVar(&args.partition, "partition", fetchAllPartitions, "Partition to limit offsets to") + flags.StringVar(&args.partition, "partition", allPartitionsHuman, "Partition to limit offsets to, or all") flags.BoolVar(&args.offsets, "offsets", true, "Controls if offsets should be fetched (defauls to true)") flags.Usage = func() { @@ -475,5 +485,5 @@ kt group -reset 23 -topic fav-topic -group specials -partition 2 To reset a consumer group's offset for all partitions: -kt group -reset newest -topic fav-topic -group specials -partition -1 +kt group -reset newest -topic fav-topic -group specials -partition all `