Add `remove_duplicates` ignore_comments and merge_comments logic (#132)
Added `DuplicateResolution` enum to handle advanced deduplication modes.
The new mores are `ignore_comments` to keep only the first comment for
deduplicated items and `merge_comments` to merge comments from all
identical items when deduplicated (identical comment lines are also
deduplicated when merging).
diff --git a/README.md b/README.md
index 063b31b..be0e95e 100644
--- a/README.md
+++ b/README.md
@@ -899,6 +899,11 @@
The duplicate handling can be changed with the switch `remove_duplicates`:
+* `yes` (default): Deduplicates based on both code and comments.
+* `no`: Leaves duplicates untouched.
+* `keep_first_comment`: Deduplicates based on code lines only, retaining only the first non-empty comment found among duplicate occurrences.
+* `merge_comments`: Deduplicates based on code lines only, merging distinct comment blocks from all occurrences into the single remaining entry.
+
```diff
+# keep-sorted start remove_duplicates=no
rotation: bar
diff --git a/goldens/duplicates_keep_first_comment.in b/goldens/duplicates_keep_first_comment.in
new file mode 100644
index 0000000..5c4dccc
--- /dev/null
+++ b/goldens/duplicates_keep_first_comment.in
@@ -0,0 +1,18 @@
+Remove duplicates keeping first comment:
+// keep-sorted-test start remove_duplicates=keep_first_comment sticky_comments=yes
+// First foo
+foo
+bar
+// Second foo
+foo
+// Third foo
+foo
+// keep-sorted-test end
+
+Remove duplicates keeping first comment, when the first item has no comment:
+// keep-sorted-test start remove_duplicates=keep_first_comment sticky_comments=yes
+foo
+bar
+// Second foo
+foo
+// keep-sorted-test end
diff --git a/goldens/duplicates_keep_first_comment.out b/goldens/duplicates_keep_first_comment.out
new file mode 100644
index 0000000..bd881f2
--- /dev/null
+++ b/goldens/duplicates_keep_first_comment.out
@@ -0,0 +1,13 @@
+Remove duplicates keeping first comment:
+// keep-sorted-test start remove_duplicates=keep_first_comment sticky_comments=yes
+bar
+// First foo
+foo
+// keep-sorted-test end
+
+Remove duplicates keeping first comment, when the first item has no comment:
+// keep-sorted-test start remove_duplicates=keep_first_comment sticky_comments=yes
+bar
+// Second foo
+foo
+// keep-sorted-test end
diff --git a/goldens/duplicates_merge_comments.in b/goldens/duplicates_merge_comments.in
new file mode 100644
index 0000000..c6261ca
--- /dev/null
+++ b/goldens/duplicates_merge_comments.in
@@ -0,0 +1,34 @@
+Merge duplicate comments:
+// keep-sorted-test start remove_duplicates=merge_comments sticky_comments=yes
+// First foo
+foo
+bar
+// Second foo
+foo
+// Third foo
+foo
+// keep-sorted-test end
+
+Merge duplicate comments where some are identical:
+// keep-sorted-test start remove_duplicates=merge_comments sticky_comments=yes
+// Common foo flag
+foo
+bar
+// Common foo flag
+foo
+// Different foo flag
+foo
+// keep-sorted-test end
+
+Merge multi-line duplicate comments with paragraph breaks:
+// keep-sorted-test start remove_duplicates=merge_comments sticky_comments=yes
+// First foo
+//
+// Shared footer
+foo
+bar
+// Second foo
+//
+// Shared footer
+foo
+// keep-sorted-test end
diff --git a/goldens/duplicates_merge_comments.out b/goldens/duplicates_merge_comments.out
new file mode 100644
index 0000000..4b73b36
--- /dev/null
+++ b/goldens/duplicates_merge_comments.out
@@ -0,0 +1,28 @@
+Merge duplicate comments:
+// keep-sorted-test start remove_duplicates=merge_comments sticky_comments=yes
+bar
+// First foo
+// Second foo
+// Third foo
+foo
+// keep-sorted-test end
+
+Merge duplicate comments where some are identical:
+// keep-sorted-test start remove_duplicates=merge_comments sticky_comments=yes
+bar
+// Common foo flag
+// Different foo flag
+foo
+// keep-sorted-test end
+
+Merge multi-line duplicate comments with paragraph breaks:
+// keep-sorted-test start remove_duplicates=merge_comments sticky_comments=yes
+bar
+// First foo
+//
+// Shared footer
+// Second foo
+//
+// Shared footer
+foo
+// keep-sorted-test end
diff --git a/keepsorted/block.go b/keepsorted/block.go
index 9ff284b..63145ff 100644
--- a/keepsorted/block.go
+++ b/keepsorted/block.go
@@ -259,15 +259,48 @@
}
removedDuplicate := false
- if b.metadata.opts.RemoveDuplicates {
- seen := map[string]bool{}
+ if b.metadata.opts.RemoveDuplicates != DuplicateResolutionFalse {
+ seenStrings := map[string]bool{}
+ seenLines := map[string]*lineGroup{}
+ seenComments := map[string]map[string]bool{}
var deduped []*lineGroup
for _, lg := range groups {
- if s := lg.String(); !seen[s] {
- seen[s] = true
- deduped = append(deduped, lg)
+ if b.metadata.opts.RemoveDuplicates == DuplicateResolutionTrue {
+ if s := lg.String(); !seenStrings[s] {
+ seenStrings[s] = true
+ deduped = append(deduped, lg)
+ } else {
+ removedDuplicate = true
+ }
} else {
- removedDuplicate = true
+ codeMapKey := strings.Join(lg.lines, "\n")
+
+ if firstLg, ok := seenLines[codeMapKey]; !ok {
+ seenLines[codeMapKey] = lg
+ deduped = append(deduped, lg)
+ if b.metadata.opts.RemoveDuplicates == DuplicateResolutionMergeComments && len(lg.comment) > 0 {
+ seenComments[codeMapKey] = map[string]bool{strings.Join(lg.comment, "\n"): true}
+ }
+ } else {
+ removedDuplicate = true
+
+ if b.metadata.opts.RemoveDuplicates == DuplicateResolutionMergeComments {
+ if len(lg.comment) > 0 {
+ commentKey := strings.Join(lg.comment, "\n")
+ if sc, ok := seenComments[codeMapKey]; !ok {
+ seenComments[codeMapKey] = map[string]bool{commentKey: true}
+ firstLg.comment = slices.Clone(lg.comment)
+ } else if !sc[commentKey] {
+ sc[commentKey] = true
+ firstLg.comment = append(slices.Clone(firstLg.comment), lg.comment...)
+ }
+ }
+ } else if b.metadata.opts.RemoveDuplicates == DuplicateResolutionKeepFirstComment {
+ if len(firstLg.comment) == 0 && len(lg.comment) > 0 {
+ firstLg.comment = lg.comment
+ }
+ }
+ }
}
}
groups = deduped
diff --git a/keepsorted/keep_sorted_test.go b/keepsorted/keep_sorted_test.go
index b90cd0b..fb397f2 100644
--- a/keepsorted/keep_sorted_test.go
+++ b/keepsorted/keep_sorted_test.go
@@ -838,7 +838,7 @@
name: "AlreadySorted_ExceptForDuplicate",
opts: blockOptions{
- RemoveDuplicates: true,
+ RemoveDuplicates: DuplicateResolutionTrue,
},
in: []string{
"Bar",
@@ -1020,7 +1020,7 @@
opts: func() blockOptions {
opts := blockOptions{
- RemoveDuplicates: true,
+ RemoveDuplicates: DuplicateResolutionTrue,
StickyComments: true,
}
opts.setCommentMarker("//")
@@ -1048,7 +1048,7 @@
name: "RemoveDuplicates_IgnoresTraliningCommas",
opts: blockOptions{
- RemoveDuplicates: true,
+ RemoveDuplicates: DuplicateResolutionTrue,
},
in: []string{
"foo,",
@@ -1065,7 +1065,7 @@
name: "RemoveDuplicates_IgnoresTrailingCommas_RemovesCommaIfLastElement",
opts: blockOptions{
- RemoveDuplicates: true,
+ RemoveDuplicates: DuplicateResolutionTrue,
},
in: []string{
"foo,",
@@ -1082,7 +1082,7 @@
name: "RemoveDuplicates_IgnoresTrailingCommas_RemovesCommaIfOnlyElement",
opts: blockOptions{
- RemoveDuplicates: true,
+ RemoveDuplicates: DuplicateResolutionTrue,
},
in: []string{
"foo,",
@@ -1097,7 +1097,7 @@
name: "RemoveDuplicates_Keep",
opts: blockOptions{
- RemoveDuplicates: false,
+ RemoveDuplicates: DuplicateResolutionFalse,
},
in: []string{
"foo",
diff --git a/keepsorted/options.go b/keepsorted/options.go
index 4dfb978..f0e7a64 100644
--- a/keepsorted/options.go
+++ b/keepsorted/options.go
@@ -35,6 +35,22 @@
// true is unmarshaled as 1, false as 0.
type IntOrBool int
+// DuplicateResolution defines how duplicates are handled during sorting.
+type DuplicateResolution int
+
+const (
+ // DuplicateResolutionFalse keeps duplicate entries untouched.
+ DuplicateResolutionFalse DuplicateResolution = iota
+ // DuplicateResolutionTrue deduplicates entries based on both code and attached comments.
+ DuplicateResolutionTrue
+ // DuplicateResolutionKeepFirstComment deduplicates based on code lines only,
+ // retaining the first non-empty comment found among duplicate occurrences.
+ DuplicateResolutionKeepFirstComment
+ // DuplicateResolutionMergeComments deduplicates based on code lines only,
+ // merging distinct comment blocks from all occurrences into the single remaining entry.
+ DuplicateResolutionMergeComments
+)
+
type ByRegexOption struct {
Pattern *regexp.Regexp
Template *string
@@ -147,7 +163,7 @@
// Any other positive integer specifies the number of newlines to separate the groups.
NewlineSeparated IntOrBool `key:"newline_separated"`
// RemoveDuplicates determines whether we drop lines that are an exact duplicate.
- RemoveDuplicates bool `key:"remove_duplicates"`
+ RemoveDuplicates DuplicateResolution `key:"remove_duplicates"`
// Syntax used to start a comment for keep-sorted annotation, e.g. "//".
commentMarker string
@@ -161,7 +177,7 @@
StickyPrefixes: nil, // Will be populated with the comment marker of the start directive.
Order: OrderAsc,
CaseSensitive: true,
- RemoveDuplicates: true,
+ RemoveDuplicates: DuplicateResolutionTrue,
}
fieldIndexByKey map[string]int
@@ -250,6 +266,19 @@
default:
return strconv.Itoa(i), nil
}
+ case reflect.TypeFor[DuplicateResolution]():
+ switch val.Interface().(DuplicateResolution) {
+ case DuplicateResolutionFalse:
+ return "no", nil
+ case DuplicateResolutionTrue:
+ return "yes", nil
+ case DuplicateResolutionKeepFirstComment:
+ return "keep_first_comment", nil
+ case DuplicateResolutionMergeComments:
+ return "merge_comments", nil
+ default:
+ panic(fmt.Errorf("unhandled DuplicateResolution value: %v", val))
+ }
case reflect.TypeFor[int]():
return strconv.Itoa(int(val.Int())), nil
case reflect.TypeFor[[]int]():
diff --git a/keepsorted/options_parser.go b/keepsorted/options_parser.go
index 7848ab7..8c502b4 100644
--- a/keepsorted/options_parser.go
+++ b/keepsorted/options_parser.go
@@ -59,6 +59,9 @@
case reflect.TypeFor[int]():
val, err := p.popInt()
return reflect.ValueOf(val), err
+ case reflect.TypeFor[DuplicateResolution]():
+ val, err := p.popDuplicateResolution()
+ return reflect.ValueOf(val), err
case reflect.TypeFor[[]int]():
val, err := p.popIntList()
return reflect.ValueOf(val), err
@@ -98,6 +101,23 @@
return b, nil
}
+func (p *parser) popDuplicateResolution() (DuplicateResolution, error) {
+ val, rest, _ := strings.Cut(p.line, " ")
+ p.line = rest
+ switch val {
+ case "yes", "true":
+ return DuplicateResolutionTrue, nil
+ case "no", "false":
+ return DuplicateResolutionFalse, nil
+ case "keep_first_comment":
+ return DuplicateResolutionKeepFirstComment, nil
+ case "merge_comments":
+ return DuplicateResolutionMergeComments, nil
+ default:
+ return DuplicateResolutionFalse, fmt.Errorf("unrecognized remove_duplicates value %q", val)
+ }
+}
+
func (p *parser) popInt() (int, error) {
val, rest, _ := strings.Cut(p.line, " ")
p.line = rest
diff --git a/keepsorted/options_parser_test.go b/keepsorted/options_parser_test.go
index b23e18d..b6f144d 100644
--- a/keepsorted/options_parser_test.go
+++ b/keepsorted/options_parser_test.go
@@ -284,6 +284,32 @@
want: IntOrBool(0),
wantErr: true,
},
+ {
+ name: "DuplicateResolution_True",
+ input: "yes",
+ want: DuplicateResolutionTrue,
+ },
+ {
+ name: "DuplicateResolution_False",
+ input: "no",
+ want: DuplicateResolutionFalse,
+ },
+ {
+ name: "DuplicateResolution_KeepFirstComment",
+ input: "keep_first_comment",
+ want: DuplicateResolutionKeepFirstComment,
+ },
+ {
+ name: "DuplicateResolution_MergeComments",
+ input: "merge_comments",
+ want: DuplicateResolutionMergeComments,
+ },
+ {
+ name: "DuplicateResolution_Invalid",
+ input: "foo",
+ want: DuplicateResolutionFalse,
+ wantErr: true,
+ },
} {
t.Run(tc.name, func(t *testing.T) {
suffix := "trailing content..."
diff --git a/keepsorted/options_test.go b/keepsorted/options_test.go
index f7e2a2d..2bb8c58 100644
--- a/keepsorted/options_test.go
+++ b/keepsorted/options_test.go
@@ -290,6 +290,31 @@
GroupStartRegex: []*regexp.Regexp{regexp.MustCompile("^CREATE"), regexp.MustCompile("b")},
},
},
+ {
+ name: "RemoveDuplicates_Yes",
+ in: "remove_duplicates=yes",
+ want: blockOptions{RemoveDuplicates: DuplicateResolutionTrue},
+ },
+ {
+ name: "RemoveDuplicates_No",
+ in: "remove_duplicates=no",
+ want: blockOptions{RemoveDuplicates: DuplicateResolutionFalse},
+ },
+ {
+ name: "RemoveDuplicates_KeepFirstComment",
+ in: "remove_duplicates=keep_first_comment",
+ want: blockOptions{RemoveDuplicates: DuplicateResolutionKeepFirstComment},
+ },
+ {
+ name: "RemoveDuplicates_MergeComments",
+ in: "remove_duplicates=merge_comments",
+ want: blockOptions{RemoveDuplicates: DuplicateResolutionMergeComments},
+ },
+ {
+ name: "RemoveDuplicates_Invalid",
+ in: "remove_duplicates=nah",
+ wantErr: `while parsing option "remove_duplicates": unrecognized remove_duplicates value "nah"`,
+ },
} {
t.Run(tc.name, func(t *testing.T) {
initZerolog(t)