blob: 660f9302b6fa27a89414887afc46adef806426c1 [file] [log] [blame]
// Copyright 2023 The Shac Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package reporting
import (
"bytes"
"context"
"errors"
"fmt"
"os"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/mattn/go-colorable"
"go.fuchsia.dev/shac-project/shac/internal/engine"
"go.fuchsia.dev/shac-project/shac/internal/sarif"
"google.golang.org/protobuf/encoding/protojson"
"google.golang.org/protobuf/testing/protocmp"
)
func TestGet(t *testing.T) {
r, err := Get(context.Background())
if err != nil {
t.Fatal(err)
}
if err := r.Close(); err != nil {
t.Fatal(err)
}
}
func TestBasic(t *testing.T) {
buf := bytes.Buffer{}
r := basic{out: &buf}
ctx := context.Background()
// No context.
if err := r.EmitFinding(ctx, "mycheck", engine.Notice, "message1", "", "", engine.Span{}, nil); err != nil {
t.Fatal(err)
}
// Only a file.
if err := r.EmitFinding(ctx, "mycheck", engine.Notice, "message2", "", "testdata/file.txt", engine.Span{}, nil); err != nil {
t.Fatal(err)
}
// File and line number. More than this is ignored.
if err := r.EmitFinding(ctx, "mycheck", engine.Notice, "message3", "", "testdata/file.txt", engine.Span{Start: engine.Cursor{Line: 10}}, nil); err != nil {
t.Fatal(err)
}
if err := r.EmitArtifact(ctx, "mycheck", "", "testdata/file.txt", []byte("content")); err == nil {
t.Fatal("expected failure")
}
start := time.Now()
r.CheckCompleted(ctx, "mycheck", start, time.Millisecond, engine.Notice, nil)
r.CheckCompleted(ctx, "badcheck", start, time.Millisecond, engine.Notice, errors.New("bad"))
r.Print(ctx, "", "src.star", 12, "debugmsg")
r.Print(ctx, "mycheck", "src.star", 12, "debugmsg")
if err := r.Close(); err != nil {
t.Fatal(err)
}
want := "[mycheck/notice] message1\n" +
"[mycheck/notice] testdata/file.txt: message2\n" +
"[mycheck/notice] testdata/file.txt(10): message3\n" +
"- mycheck (success in 1ms)\n" +
"- badcheck (error in 1ms): bad\n" +
"[src.star:12] debugmsg\n" +
"- mycheck [src.star:12] debugmsg\n"
if diff := cmp.Diff(want, buf.String()); diff != "" {
t.Fatalf("mismatch (-want +got):\n%s", diff)
}
}
func TestGitHub(t *testing.T) {
buf := bytes.Buffer{}
r := github{out: &buf}
ctx := context.Background()
// No context.
if err := r.EmitFinding(ctx, "mycheck", engine.Notice, "message1", "", "", engine.Span{}, nil); err != nil {
t.Fatal(err)
}
// Only a file.
if err := r.EmitFinding(ctx, "mycheck", engine.Notice, "message2", "", "testdata/file.txt", engine.Span{}, nil); err != nil {
t.Fatal(err)
}
// File and line number.
if err := r.EmitFinding(ctx, "mycheck", engine.Notice, "message3", "", "testdata/file.txt", engine.Span{Start: engine.Cursor{Line: 10}}, nil); err != nil {
t.Fatal(err)
}
// File, line number and column.
if err := r.EmitFinding(ctx, "mycheck", engine.Notice, "message4", "", "testdata/file.txt", engine.Span{Start: engine.Cursor{Line: 10, Col: 1}}, nil); err != nil {
t.Fatal(err)
}
// File, two line numbers.
if err := r.EmitFinding(ctx, "mycheck", engine.Notice, "message5", "", "testdata/file.txt", engine.Span{Start: engine.Cursor{Line: 10}, End: engine.Cursor{Line: 12}}, nil); err != nil {
t.Fatal(err)
}
// file, start and end span on separate lines.
if err := r.EmitFinding(ctx, "mycheck", engine.Notice, "message6", "", "testdata/file.txt", engine.Span{Start: engine.Cursor{Line: 10, Col: 1}, End: engine.Cursor{Line: 12, Col: 2}}, nil); err != nil {
t.Fatal(err)
}
if err := r.EmitArtifact(ctx, "mycheck", "", "testdata/file.txt", []byte("content")); err == nil {
t.Fatal("expected failure")
}
start := time.Now()
r.CheckCompleted(ctx, "mycheck", start, time.Millisecond, engine.Notice, nil)
r.CheckCompleted(ctx, "badcheck", start, time.Millisecond, engine.Notice, errors.New("bad"))
r.Print(ctx, "", "src.star", 12, "debugmsg")
r.Print(ctx, "mycheck", "src.star", 12, "debugmsg")
if err := r.Close(); err != nil {
t.Fatal(err)
}
want := "::notice ::title=mycheck::message1\n" +
"::notice ::file=testdata/file.txttitle=mycheck::message2\n" +
"::notice ::file=testdata/file.txt,line=10,title=mycheck::message3\n" +
"::notice ::file=testdata/file.txt,line=10,col=1,title=mycheck::message4\n" +
"::notice ::file=testdata/file.txt,line=10,endLine=12,title=mycheck::message5\n" +
"::notice ::file=testdata/file.txt,line=10,col=1,endLine=12,endCol=2,title=mycheck::message6\n" +
"::debug::[src.star:12] debugmsg\n" +
"::debug::mycheck [src.star:12] debugmsg\n"
if diff := cmp.Diff(want, buf.String()); diff != "" {
t.Fatalf("mismatch (-want +got):\n%s", diff)
}
}
func TestInteractive_Finding(t *testing.T) {
t.Parallel()
data := []struct {
name string
l engine.Level
filepath string
span engine.Span
want string
}{
{
"no context/notice",
engine.Notice,
"",
engine.Span{},
"<R>[<Hc>mycheck<R>/<G>notice<R>] message1\n",
},
{
"no context/warning",
engine.Warning,
"",
engine.Span{},
"<R>[<Hc>mycheck<R>/<Y>warning<R>] message1\n",
},
{
"no context/error",
engine.Error,
"",
engine.Span{},
"<R>[<Hc>mycheck<R>/<Re>error<R>] message1\n",
},
{
"file only",
engine.Notice,
"file.txt",
engine.Span{},
"<R>[<Hc>mycheck<R>/<G>notice<R>] file.txt: message1\n",
},
{
"start line first",
engine.Notice,
"file.txt",
engine.Span{Start: engine.Cursor{Line: 1}},
"<R>[<Hc>mycheck<R>/<G>notice<R>] file.txt(1): message1\n" +
"\n" +
" <G>This<R>\n" +
" File\n" +
"\n",
},
{
"start line middle",
engine.Notice,
"file.txt",
engine.Span{Start: engine.Cursor{Line: 3}},
"<R>[<Hc>mycheck<R>/<G>notice<R>] file.txt(3): message1\n" +
"\n" +
" File\n" +
" <G>Has<R>\n" +
" A\n" +
"\n",
},
{
"start line last",
engine.Notice,
"file.txt",
engine.Span{Start: engine.Cursor{Line: 9}},
"<R>[<Hc>mycheck<R>/<G>notice<R>] file.txt(9): message1\n" +
"\n" +
" Some\n" +
" <G>Content<R>\n" +
" \n" +
"\n",
},
{
"start line past end",
engine.Notice,
"file.txt",
engine.Span{Start: engine.Cursor{Line: 10}},
"<R>[<Hc>mycheck<R>/<G>notice<R>] file.txt(10): message1\n",
},
{
"end line same",
engine.Notice,
"file.txt",
engine.Span{Start: engine.Cursor{Line: 3}, End: engine.Cursor{Line: 3}},
"<R>[<Hc>mycheck<R>/<G>notice<R>] file.txt(3): message1\n" +
"\n" +
" File\n" +
" <G>Has<R>\n" +
" A\n" +
"\n",
},
{
"end line next",
engine.Notice,
"file.txt",
engine.Span{Start: engine.Cursor{Line: 3}, End: engine.Cursor{Line: 4}},
"<R>[<Hc>mycheck<R>/<G>notice<R>] file.txt(3): message1\n" +
"\n" +
" File\n" +
" <G>Has<R>\n" +
" <G>A<R>\n" +
" Little\n" +
"\n",
},
{
"end line 3",
engine.Notice,
"file.txt",
engine.Span{Start: engine.Cursor{Line: 3}, End: engine.Cursor{Line: 5}},
"<R>[<Hc>mycheck<R>/<G>notice<R>] file.txt(3): message1\n" +
"\n" +
" File\n" +
" <G>Has<R>\n" +
" <G>A<R>\n" +
" <G>Little<R>\n" +
" Bit\n" +
"\n",
},
{
"span 2 lines",
engine.Notice,
"file.txt",
engine.Span{Start: engine.Cursor{Line: 1, Col: 2}, End: engine.Cursor{Line: 2, Col: 3}},
"<R>[<Hc>mycheck<R>/<G>notice<R>] file.txt(1): message1\n" +
"\n" +
" T<G>his<R>\n" +
" <G>Fi<R>le\n" +
" Has\n" +
"\n",
},
{
"span end col overflow",
engine.Notice,
"file.txt",
engine.Span{Start: engine.Cursor{Line: 4, Col: 1}, End: engine.Cursor{Line: 5, Col: 11}},
"<R>[<Hc>mycheck<R>/<G>notice<R>] file.txt(4): message1\n" +
"\n" +
" Has\n" +
" <G>A<R>\n" +
" <G>Little<R>\n" +
" Bit\n" +
"\n",
},
{
"span intra-line 1 char",
engine.Notice,
"file.txt",
engine.Span{Start: engine.Cursor{Line: 5, Col: 2}, End: engine.Cursor{Line: 5, Col: 3}},
"<R>[<Hc>mycheck<R>/<G>notice<R>] file.txt(5): message1\n" +
"\n" +
" A\n" +
" L<G>i<R>ttle\n" +
" Bit\n" +
"\n",
},
{
"span intra-line 2 chars",
engine.Notice,
"file.txt",
engine.Span{Start: engine.Cursor{Line: 5, Col: 2}, End: engine.Cursor{Line: 5, Col: 4}},
"<R>[<Hc>mycheck<R>/<G>notice<R>] file.txt(5): message1\n" +
"\n" +
" A\n" +
" L<G>it<R>tle\n" +
" Bit\n" +
"\n",
},
{
"span intra-line EOL",
engine.Notice,
"file.txt",
engine.Span{Start: engine.Cursor{Line: 5, Col: 1}, End: engine.Cursor{Line: 5, Col: 7}},
"<R>[<Hc>mycheck<R>/<G>notice<R>] file.txt(5): message1\n" +
"\n" +
" A\n" +
" <G>Little<R>\n" +
" Bit\n" +
"\n",
},
{
"span intra-line EOL overflow",
engine.Notice,
"file.txt",
engine.Span{Start: engine.Cursor{Line: 5, Col: 1}, End: engine.Cursor{Line: 5, Col: 11}},
"<R>[<Hc>mycheck<R>/<G>notice<R>] file.txt(5): message1\n" +
"\n" +
" A\n" +
" <G>Little<R>\n" +
" Bit\n" +
"\n",
},
}
for i, l := range data {
l := l
t.Run(fmt.Sprintf("%d-%s", i, l.name), func(t *testing.T) {
t.Parallel()
buf := bytes.Buffer{}
// Strip the ANSI codes for now, otherwise it makes the test fairly messy.
// Note that many of the ANSI code are hacked out in ansi_test.go.
r := interactive{out: colorable.NewNonColorable(&buf)}
ctx := context.Background()
if err := r.EmitFinding(ctx, "mycheck", l.l, "message1", "testdata", l.filepath, l.span, nil); err != nil {
t.Fatal(err)
}
if err := r.Close(); err != nil {
t.Fatal(err)
}
if diff := cmp.Diff(l.want, buf.String()); diff != "" {
t.Fatalf("mismatch (-want +got):\n%s", diff)
}
})
}
}
func TestInteractive(t *testing.T) {
t.Parallel()
buf := bytes.Buffer{}
// Strip the ANSI codes for now, otherwise it makes the test fairly messy.
// Note that many of the ANSI code are hacked out in ansi_test.go.
r := interactive{out: colorable.NewNonColorable(&buf)}
ctx := context.Background()
if err := r.EmitArtifact(ctx, "mycheck", "", "testdata/file.txt", []byte("content")); err == nil {
t.Fatal("expected failure")
}
start := time.Now()
r.CheckCompleted(ctx, "mycheck", start, time.Millisecond, engine.Notice, nil)
r.CheckCompleted(ctx, "badcheck", start, time.Millisecond, engine.Notice, errors.New("bad"))
r.Print(ctx, "", "src.star", 12, "debugmsg")
r.Print(ctx, "mycheck", "src.star", 12, "debugmsg")
if err := r.Close(); err != nil {
t.Fatal(err)
}
want := "<R>- <G>mycheck<R> (success in 1ms)\n" +
"<R>- <Re>badcheck<R> (error in 1ms): bad\n" +
"<R>[src.star:12<R>] <B>debugmsg<R>\n" +
"<R>- <Y>mycheck <R>[src.star:12<R>] <B>debugmsg<R>\n"
if diff := cmp.Diff(want, buf.String()); diff != "" {
t.Fatalf("mismatch (-want +got):\n%s", diff)
}
}
func TestSARIF(t *testing.T) {
t.Parallel()
var buf bytes.Buffer
r := SarifReport{Out: &buf}
ctx := context.Background()
root := t.TempDir()
if err := r.EmitFinding(
ctx,
"check1",
engine.Error,
"Found an issue",
root,
"foo/bar.c",
engine.Span{},
nil,
); err != nil {
t.Fatal(err)
}
if err := r.EmitFinding(
ctx,
"check1",
engine.Warning,
"Found another issue",
root,
"foo/baz.c",
engine.Span{
Start: engine.Cursor{Line: 5, Col: 4},
End: engine.Cursor{Line: 6, Col: 2},
},
[]string{"blah", "meh"},
); err != nil {
t.Fatal(err)
}
if err := r.EmitFinding(
ctx,
"check2",
engine.Notice,
"Notice from check2",
root,
"path/to/another_file.rs",
engine.Span{
Start: engine.Cursor{Line: 2, Col: 3},
},
nil,
); err != nil {
t.Fatal(err)
}
if err := r.Close(); err != nil {
t.Fatal(err)
}
var got sarif.Document
if err := protojson.Unmarshal(buf.Bytes(), &got); err != nil {
t.Fatal(err)
}
want := sarif.Document{
Version: sarif.Version,
Runs: []*sarif.Run{
{
Tool: &sarif.Tool{Driver: &sarif.ToolComponent{Name: "check1"}},
Results: []*sarif.Result{
{
Level: sarif.Error,
Message: &sarif.Message{
Text: "Found an issue",
},
Locations: []*sarif.Location{
{
PhysicalLocation: &sarif.PhysicalLocation{
ArtifactLocation: &sarif.ArtifactLocation{
Uri: "foo/bar.c",
},
Region: &sarif.Region{},
},
},
},
},
{
Level: sarif.Warning,
Message: &sarif.Message{
Text: "Found another issue",
},
Locations: []*sarif.Location{
{
PhysicalLocation: &sarif.PhysicalLocation{
ArtifactLocation: &sarif.ArtifactLocation{
Uri: "foo/baz.c",
},
Region: &sarif.Region{
StartLine: 5,
StartColumn: 4,
EndLine: 6,
EndColumn: 2,
},
},
},
},
Fixes: []*sarif.Fix{
{
ArtifactChanges: []*sarif.ArtifactChange{
{
ArtifactLocation: &sarif.ArtifactLocation{
Uri: "foo/baz.c",
},
Replacements: []*sarif.Replacement{
{
DeletedRegion: &sarif.Region{
StartLine: 5,
StartColumn: 4,
EndLine: 6,
EndColumn: 2,
},
InsertedContent: &sarif.ArtifactContent{
Text: "blah",
},
},
},
},
},
},
{
ArtifactChanges: []*sarif.ArtifactChange{
{
ArtifactLocation: &sarif.ArtifactLocation{
Uri: "foo/baz.c",
},
Replacements: []*sarif.Replacement{
{
DeletedRegion: &sarif.Region{
StartLine: 5,
StartColumn: 4,
EndLine: 6,
EndColumn: 2,
},
InsertedContent: &sarif.ArtifactContent{
Text: "meh",
},
},
},
},
},
},
},
},
},
},
{
Tool: &sarif.Tool{Driver: &sarif.ToolComponent{Name: "check2"}},
Results: []*sarif.Result{
{Level: sarif.Note,
Message: &sarif.Message{Text: "Notice from check2"},
Locations: []*sarif.Location{
{
PhysicalLocation: &sarif.PhysicalLocation{
ArtifactLocation: &sarif.ArtifactLocation{
Uri: "path/to/another_file.rs",
},
Region: &sarif.Region{
StartLine: 2,
StartColumn: 3,
},
},
},
},
},
},
},
},
}
if diff := cmp.Diff(&want, &got, protocmp.Transform()); diff != "" {
t.Errorf("SARIF diff (-want +got):\n%s", diff)
}
}
func init() {
// Mutate the running environment to make the test deterministic.
os.Unsetenv("LUCI_CONTEXT")
os.Unsetenv("GITHUB_RUN_ID")
os.Unsetenv("VSCODE_GIT_IPC_HANDLE")
os.Setenv("TERM", "dumb")
}