| // Copyright 2020 The Fuchsia Authors. All rights reserved. | |
| // Use of this source code is governed by a BSD-style license that can be | |
| // found in the LICENSE file. | |
| package covargs | |
| import ( | |
| "bytes" | |
| "compress/zlib" | |
| "fmt" | |
| "math" | |
| "os" | |
| "os/exec" | |
| "path/filepath" | |
| "runtime" | |
| "sort" | |
| "strconv" | |
| "strings" | |
| "sync" | |
| "go.fuchsia.dev/fuchsia/tools/debug/covargs/api/llvm" | |
| "go.fuchsia.dev/fuchsia/tools/debug/covargs/api/third_party/codecoverage" | |
| "golang.org/x/sync/errgroup" | |
| "google.golang.org/protobuf/encoding/protojson" | |
| ) | |
| // DiffMapping represents a source transformation done by a diff (i.e. patch), | |
| // where for each file it has a mapping from the old line number to the new one. | |
| type DiffMapping map[string]LineMapping | |
| // LineMapping maps the old line number to the new one within a single file. | |
| type LineMapping map[int]int | |
| type line struct { | |
| line, count int | |
| } | |
| type lineData []line | |
| type block struct { | |
| first, last int | |
| } | |
| type blockData map[int][]block | |
| func extractData(segments []llvm.Segment) (lineData, blockData) { | |
| var ld lineData | |
| bd := blockData{} | |
| // The most recent segment that starts from a previous line. | |
| var w *llvm.Segment | |
| var lineSegments []llvm.Segment | |
| nextSegmentIndex := 0 | |
| for i := 1; i <= segments[len(segments)-1].LineNumber; i += 1 { | |
| // Calculate the execution count for each line. Follow the logic in llvm-cov: | |
| // https://github.com/llvm-mirror/llvm/blob/3b35e17b21e388832d7b560a06a4f9eeaeb35330/lib/ProfileData/Coverage/CoverageMapping.cpp#L686 | |
| if len(lineSegments) > 0 { | |
| w = &lineSegments[len(lineSegments)-1] | |
| } | |
| lineSegments = nil | |
| for nextSegmentIndex < len(segments) && segments[nextSegmentIndex].LineNumber == i { | |
| lineSegments = append(lineSegments, segments[nextSegmentIndex]) | |
| nextSegmentIndex += 1 | |
| } | |
| lineStartsNewRegion := false | |
| for _, s := range lineSegments { | |
| if s.HasCount && s.IsRegionEntry { | |
| lineStartsNewRegion = true | |
| break | |
| } | |
| } | |
| startOfSkippedRegion := (len(lineSegments) > 0 && !lineSegments[0].HasCount) && lineSegments[0].IsRegionEntry | |
| coverable := !startOfSkippedRegion && ((w != nil && w.HasCount) || lineStartsNewRegion) | |
| if !coverable { | |
| continue | |
| } | |
| count := 0 | |
| if w != nil { | |
| count = w.Count | |
| } | |
| for _, s := range lineSegments { | |
| if s.HasCount && s.IsRegionEntry { | |
| if s.Count > count { | |
| count = s.Count | |
| } | |
| } | |
| } | |
| ld = append(ld, line{i, count}) | |
| // Calculate the uncovered blocks within the current line. Follow the logic in llvm-cov: | |
| // https://github.com/llvm-mirror/llvm/blob/993ef0ca960f8ffd107c33bfbf1fd603bcf5c66c/tools/llvm-cov/SourceCoverageViewText.cpp#L114 | |
| if count == 0 { | |
| // Skips calculating uncovered blocks if the whole line is not covered. | |
| continue | |
| } | |
| columnStart := 1 | |
| isBlockNotCovered := (w != nil && w.HasCount) && w.Count == 0 | |
| for _, s := range lineSegments { | |
| columnEnd := s.ColumnNumber | |
| if isBlockNotCovered { | |
| bd[s.LineNumber] = append(bd[s.LineNumber], block{columnStart, columnEnd - 1}) | |
| } | |
| isBlockNotCovered = s.HasCount && s.Count == 0 | |
| columnStart = columnEnd | |
| } | |
| if isBlockNotCovered { | |
| lastSegment := lineSegments[len(lineSegments)-1] | |
| bd[lastSegment.LineNumber] = append(bd[lastSegment.LineNumber], block{columnStart, -1}) | |
| } | |
| } | |
| return ld, bd | |
| } | |
| func rebaseData(lines lineData, blocks blockData, mapping LineMapping) (lineData, blockData) { | |
| var ld lineData | |
| for _, l := range lines { | |
| if _, ok := mapping[l.line]; !ok { | |
| continue | |
| } | |
| ld = append(ld, line{mapping[l.line], l.count}) | |
| } | |
| var bd blockData | |
| for i, b := range blocks { | |
| if _, ok := mapping[i]; !ok { | |
| continue | |
| } | |
| bd[mapping[i]] = b | |
| } | |
| return ld, bd | |
| } | |
| func compressData(lines lineData, blocks blockData) ([]*codecoverage.LineRange, []*codecoverage.ColumnRanges) { | |
| // Aggregate contiguous blocks of lines with the exact same hit count. | |
| var lr []*codecoverage.LineRange | |
| last := 0 | |
| for i := 1; i <= len(lines); i++ { | |
| isContinuous := i < len(lines) && lines[i].line == lines[i-1].line+1 | |
| hasSameCount := i < len(lines) && lines[i].count == lines[i-1].count | |
| // Merge two lines iff they have continuous line number and exactly the same count, e.g. (101, 10) and (102, 10). | |
| if isContinuous && hasSameCount { | |
| continue | |
| } | |
| lr = append(lr, &codecoverage.LineRange{ | |
| First: int32(lines[last].line), | |
| Last: int32(lines[i-1].line), | |
| Count: int64(lines[last].count), | |
| }) | |
| last = i | |
| } | |
| var cr []*codecoverage.ColumnRanges | |
| for i, block := range blocks { | |
| var ranges []*codecoverage.ColumnRange | |
| for _, b := range block { | |
| ranges = append(ranges, &codecoverage.ColumnRange{ | |
| First: int32(b.first), | |
| Last: int32(b.last), | |
| }) | |
| } | |
| cr = append(cr, &codecoverage.ColumnRanges{ | |
| Line: int32(i), | |
| Ranges: ranges, | |
| }) | |
| } | |
| return lr, cr | |
| } | |
| func getRevision(path string) (string, int64, error) { | |
| if _, err := os.Stat(path); os.IsNotExist(err) { | |
| return "", 0, nil | |
| } | |
| var stdout bytes.Buffer | |
| rel, err := filepath.Rel(filepath.Dir(path), path) | |
| if err != nil { | |
| return "", 0, err | |
| } | |
| cmd := exec.Command("git", "--literal-pathspecs", "log", "-n", "1", `--pretty=format:%H:%ct`, rel) | |
| cmd.Dir = filepath.Dir(path) | |
| cmd.Stdout = &stdout | |
| if err := cmd.Run(); err != nil { | |
| return "", 0, fmt.Errorf("failed to obtain revision: %w", err) | |
| } | |
| out := strings.TrimSpace(stdout.String()) | |
| if out == "" { | |
| return "", 0, nil | |
| } | |
| parts := strings.Split(out, ":") | |
| if len(parts) != 2 { | |
| return "", 0, fmt.Errorf("not in hash:timestamp format: %s", out) | |
| } | |
| hash := parts[0] | |
| timestamp, err := strconv.ParseInt(parts[1], 10, 64) | |
| if err != nil { | |
| return "", 0, fmt.Errorf("invalid timestamp %q: %w", parts[1], err) | |
| } | |
| return hash, timestamp, nil | |
| } | |
| func convertFile(file llvm.File, base string, mapping *DiffMapping) (*codecoverage.File, error) { | |
| if file.Segments == nil { | |
| return nil, nil | |
| } | |
| // The filename is expected to be relative to the current working | |
| // directory. However, in the report, we need to make it relative to the | |
| // base directory of the source tree. | |
| abs, err := filepath.Abs(file.Filename) | |
| if err != nil { | |
| return nil, err | |
| } | |
| rel, err := filepath.Rel(base, abs) | |
| if err != nil { | |
| return nil, err | |
| } | |
| ld, bd := extractData(file.Segments) | |
| sort.Slice(ld, func(i, j int) bool { | |
| return ld[i].line < ld[j].line | |
| }) | |
| var revision string | |
| var timestamp int64 | |
| if mapping != nil { | |
| if _, ok := (*mapping)[rel]; ok { | |
| ld, bd = rebaseData(ld, bd, (*mapping)[rel]) | |
| } | |
| } else { | |
| revision, timestamp, err = getRevision(file.Filename) | |
| if err != nil { | |
| return nil, err | |
| } | |
| } | |
| lr, cr := compressData(ld, bd) | |
| return &codecoverage.File{ | |
| Path: "//" + rel, | |
| Lines: lr, | |
| UncoveredBlocks: cr, | |
| Summaries: []*codecoverage.Metric{ | |
| { | |
| Name: "function", | |
| Covered: int32(file.Summary.Functions.Covered), | |
| Total: int32(file.Summary.Functions.Count), | |
| }, | |
| { | |
| Name: "region", | |
| Covered: int32(file.Summary.Regions.Covered), | |
| Total: int32(file.Summary.Regions.Count), | |
| }, | |
| { | |
| Name: "line", | |
| Covered: int32(file.Summary.Lines.Covered), | |
| Total: int32(file.Summary.Lines.Count), | |
| }, | |
| { | |
| Name: "branch", | |
| Covered: int32(file.Summary.Branches.Covered), | |
| Total: int32(file.Summary.Branches.Count), | |
| }, | |
| }, | |
| Revision: revision, | |
| Timestamp: timestamp, | |
| }, nil | |
| } | |
| func mergeSummaries(summaries ...[]*codecoverage.Metric) []*codecoverage.Metric { | |
| s := []*codecoverage.Metric{ | |
| { | |
| Name: "function", | |
| Covered: 0, | |
| Total: 0, | |
| }, | |
| { | |
| Name: "region", | |
| Covered: 0, | |
| Total: 0, | |
| }, | |
| { | |
| Name: "line", | |
| Covered: 0, | |
| Total: 0, | |
| }, | |
| { | |
| Name: "branch", | |
| Covered: 0, | |
| Total: 0, | |
| }, | |
| } | |
| d := map[string]int{} | |
| for i, m := range s { | |
| d[m.Name] = i | |
| } | |
| for _, m := range summaries { | |
| if m != nil { | |
| for _, n := range m { | |
| s[d[n.Name]].Covered += n.Covered | |
| s[d[n.Name]].Total += n.Total | |
| } | |
| } | |
| } | |
| return s | |
| } | |
| // ComputeSummaries calculates aggregate summaries for all directories. | |
| func ComputeSummaries(files []*codecoverage.File) ([]*codecoverage.GroupCoverageSummary, []*codecoverage.Metric) { | |
| summaries := map[string][]*codecoverage.Metric{} | |
| for _, f := range files { | |
| // We can't use filepath.Dir and filepath.Base because they normalize '//'. | |
| dir, _ := filepath.Split(f.Path) | |
| for dir != "//" { | |
| // In the coverage data format, dirs end with '/' except for root. | |
| summaries[dir] = mergeSummaries(summaries[dir], f.Summaries) | |
| dir, _ = filepath.Split(dir[:len(dir)-1]) | |
| } | |
| summaries["//"] = mergeSummaries(summaries["//"], f.Summaries) | |
| } | |
| fileSummaries := map[string][]*codecoverage.CoverageSummary{} | |
| for _, f := range files { | |
| dir, file := filepath.Split(f.Path) | |
| fileSummaries[dir] = append(fileSummaries[dir], &codecoverage.CoverageSummary{ | |
| Name: file, | |
| Path: f.Path, | |
| Summaries: f.Summaries, | |
| }) | |
| } | |
| dirSummaries := map[string][]*codecoverage.CoverageSummary{} | |
| for p, s := range summaries { | |
| if p == "//" { | |
| continue | |
| } | |
| // The path already ends with '/' so we have to omit it. | |
| dir, file := filepath.Split(p[:len(p)-1]) | |
| dirSummaries[dir] = append(dirSummaries[dir], &codecoverage.CoverageSummary{ | |
| Name: file + "/", | |
| Path: p, | |
| Summaries: s, | |
| }) | |
| } | |
| var groupSummaries []*codecoverage.GroupCoverageSummary | |
| for p, s := range summaries { | |
| groupSummaries = append(groupSummaries, &codecoverage.GroupCoverageSummary{ | |
| Path: p, | |
| Dirs: dirSummaries[p], | |
| Files: fileSummaries[p], | |
| Summaries: s, | |
| }) | |
| } | |
| sort.Slice(groupSummaries, func(i, j int) bool { | |
| return groupSummaries[i].Path < groupSummaries[j].Path | |
| }) | |
| return groupSummaries, summaries["//"] | |
| } | |
| // ConvertFiles converts the data in LLVM coverage JSON format into the | |
| // compressed coverage format used by Chromium coverage service. | |
| func ConvertFiles(export *llvm.Export, base string, mapping *DiffMapping) ([]*codecoverage.File, error) { | |
| var files []*codecoverage.File | |
| var g errgroup.Group | |
| var mu sync.Mutex | |
| s := make(chan struct{}, runtime.NumCPU()) | |
| for _, d := range export.Data { | |
| for _, f := range d.Files { | |
| f := f | |
| s <- struct{}{} | |
| g.Go(func() error { | |
| defer func() { <-s }() | |
| file, err := convertFile(f, base, mapping) | |
| if err != nil { | |
| return err | |
| } | |
| mu.Lock() | |
| files = append(files, file) | |
| mu.Unlock() | |
| return nil | |
| }) | |
| } | |
| } | |
| if err := g.Wait(); err != nil { | |
| return nil, err | |
| } | |
| return files, nil | |
| } | |
| func saveReport(report *codecoverage.CoverageReport, filename string) error { | |
| f, err := os.Create(filename) | |
| if err != nil { | |
| return fmt.Errorf("cannot open file %q: %v", filename, err) | |
| } | |
| defer f.Close() | |
| w := zlib.NewWriter(f) | |
| defer w.Close() | |
| b, err := protojson.MarshalOptions{ | |
| UseProtoNames: true, | |
| EmitUnpopulated: true, | |
| }.Marshal(report) | |
| if err != nil { | |
| return fmt.Errorf("cannot marshal report: %w", err) | |
| } | |
| if _, err := w.Write(b); err != nil { | |
| return fmt.Errorf("cannot emit report: %w", err) | |
| } | |
| return nil | |
| } | |
| // SaveReport saves compresses coverage data to disk, optionally sharding the | |
| // data into multiple files each of the same size. | |
| func SaveReport(files []*codecoverage.File, shardSize int, dir string) (*codecoverage.CoverageReport, error) { | |
| dirs, summaries := ComputeSummaries(files) | |
| report := &codecoverage.CoverageReport{ | |
| Dirs: dirs, | |
| Summaries: summaries, | |
| } | |
| if numFiles := len(files); numFiles > shardSize { | |
| const filename = "files%0*d.json.gz" | |
| numShards := int(math.Ceil(float64(numFiles) / float64(shardSize))) | |
| width := 1 + int(math.Log10(float64(numShards))) | |
| fileShards := make([]string, numShards) | |
| // TODO(phosek): Use goroutines to process slices in parallel. | |
| for i := 0; i < numShards; i++ { | |
| from := i * shardSize | |
| to := min((i+1)*shardSize, numFiles) | |
| report := codecoverage.CoverageReport{Files: files[from:to]} | |
| filename := fmt.Sprintf(filename, width, i+1) | |
| if err := saveReport(&report, filepath.Join(dir, filename)); err != nil { | |
| return nil, fmt.Errorf("failed to save report %q: %w", filename, err) | |
| } | |
| fileShards[i] = filename | |
| } | |
| report.FileShards = fileShards | |
| } else { | |
| report.Files = files | |
| } | |
| const filename = "all.json.gz" | |
| if err := saveReport(report, filepath.Join(dir, filename)); err != nil { | |
| return nil, fmt.Errorf("failed to save report %q: %w", filename, err) | |
| } | |
| return report, nil | |
| } |