stringclassifier/classifier_test.go - third_party/github.com/google/licenseclassifier - Git at Google

 // Copyright 2017 Google Inc.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
 //     http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
 package stringclassifier

 import (
 	"reflect"
 	"regexp"
 	"sort"
 	"testing"

 	"github.com/sergi/go-diff/diffmatchpatch"
 )

 var (
 	gettysburg = `Four score and seven years ago our fathers brought forth
 on this continent, a new nation, conceived in Liberty, and dedicated to the
 proposition that all men are created equal.`
 	modifiedGettysburg = `Four score and seven years ago our fathers brought forth
 on this continent, a nation that was new and improved, conceived in Liberty, and
 dedicated to the proposition that all men are created equal.`
 	gettysburgExtraWord = `Four score and seven years ago our fathers brought forth
 on this continent, a new nation, conceived in Liberty, and dedicated to the
 proposition that all men are created equal.Foobar`

 	declaration = `When in the Course of human events, it becomes necessary
 for one people to dissolve the political bands which have connected them with
 another, and to assume among the powers of the earth, the separate and equal
 station to which the Laws of Nature and of Nature's God entitle them, a decent
 respect to the opinions of mankind requires that they should declare the causes
 which impel them to the separation.`

 	loremipsum = `Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nulla
 varius enim mattis, rhoncus lectus id, aliquet sem. Phasellus eget ex in dolor
 feugiat ultricies. Etiam interdum sit amet nisl in placerat.  Sed vitae enim
 vulputate, tempus leo commodo, accumsan nulla.`
 	modifiedLorem = `Lorem ipsum dolor amet, consectetur adipiscing elit. Nulla
 varius enim mattis, lectus id, aliquet rhoncus  sem. Phasellus eget ex in dolor
 feugiat ultricies. Etiam interdum sit amet sit  nisl in placerat.  Sed vitae enim
 vulputate, tempus leo commodo, accumsan nulla.`
 	lessModifiedLorem = `Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nulla
 varius enim mattis, rhoncus lectus id, aliquet. Phasellus eget ex in dolor
 feugiat ultricies. Etiam interdum sit amet nisl in placerat.  Sed vitae enim
 vulputate, tempus leo commodo, accumsan nulla.`

 	postmodernThesisCollapse = `1. Expressions of collapse

 If one examines postcultural Marxism, one is faced with a choice: either
 reject capitalist submodern theory or conclude that the purpose of the reader
 is significant form. Bataille uses the term ‘capitalist construction’ to denote
 not, in fact, discourse, but prediscourse.

 Therefore, in Stardust, Gaiman analyses postcultural Marxism; in
 The Books of Magic, although, he denies capitalist submodern theory. If
 capitalist construction holds, we have to choose between capitalist submodern
 theory and Baudrillardist simulacra.

 However, conceptualist socialism implies that narrativity may be used to
 oppress the proletariat, given that sexuality is distinct from art. The subject
 is interpolated into a capitalist construction that includes language as a
 paradox.
 `
 	postmodernThesisNarratives = `1. Narratives of failure

 The main theme of the works of Joyce is the defining characteristic, and some
 would say the economy, of neocultural class. But Bataille promotes the use of
 socialist realism to deconstruct sexual identity.

 The subject is interpolated into a Baudrillardist simulation that includes
 consciousness as a whole. Thus, the primary theme of Pickett's[1] model of
 socialist realism is the role of the reader as artist.

 The subject is contextualised into a postcapitalist discourse that includes
 language as a paradox. It could be said that if Baudrillardist simulation
 holds, the works of Gibson are postmodern. The characteristic theme of the
 works of Gibson is the common ground between society and narrativity. However,
 Sartre uses the term 'postcapitalist discourse' to denote not, in fact,
 narrative, but postnarrative.
 `
 	postmodernThesisFatalFlaw = `1. Contexts of fatal flaw

 "Narrativity is part of the dialectic of culture," says Marx; however,
 according to Hamburger[1] , it is not so much narrativity that is part of the
 dialectic of culture, but rather the stasis, and hence the defining
 characteristic, of narrativity. Bataille promotes the use of Batailleist
 'powerful communication' to modify society.

 If one examines the presemioticist paradigm of reality, one is faced with a
 choice: either reject Batailleist 'powerful communication' or conclude that
 concensus must come from the masses. Therefore, Baudrillard uses the term 'the
 presemioticist paradigm of reality' to denote the difference between class and
 society. The subject is interpolated into a subtextual capitalist theory that
 includes consciousness as a whole.

 However, Pickett[2] implies that we have to choose between neotextual feminism
 and dialectic appropriation. Debord suggests the use of subtextual capitalist
 theory to deconstruct the status quo.
 `
 	nullifiable = `[[ , _ , _ , _
 ? _ : _
 ? _ : _
 ? _ : _
 ]}
 `
 	nonWords = regexp.MustCompile("[[:punct:]]+")
 )

 // removeNonWords removes non-words from the string, replacing them with empty
 // string. (This is meant to exercise tokenization problems.)
 func removeNonWords(s string) string {
 	return nonWords.ReplaceAllString(s, "")
 }

 func TestClassify_NearestMatch(t *testing.T) {
 	c := New(DefaultConfidenceThreshold, FlattenWhitespace)
 	c.AddValue("gettysburg", gettysburg)
 	c.AddValue("declaration", declaration)
 	c.AddValue("loremipsum", loremipsum)

 	tests := []struct {
 		description string
 		input       string  // input string to match
 		name        string  // name of expected nearest match
 		minConf     float64 // the lowest confidence accepted for the match
 		maxConf     float64 // the highest confidence we expect for this match
 	}{
 		{
 			description: "Full Declaration",
 			input:       declaration,
 			name:        "declaration",
 			minConf:     1.0,
 			maxConf:     1.0,
 		},
 		{
 			description: "Modified Lorem",
 			input:       modifiedLorem,
 			name:        "loremipsum",
 			minConf:     0.90,
 			maxConf:     0.91,
 		},
 		{
 			description: "Modified Gettysburg",
 			input:       modifiedGettysburg,
 			name:        "gettysburg",
 			minConf:     0.86,
 			maxConf:     0.87,
 		},
 	}

 	for _, tt := range tests {
 		m := c.NearestMatch(tt.input)

 		if got, want := m.Name, tt.name; got != want {
 			t.Errorf("NearestMatch(%q) = %q, want %q", tt.description, got, want)
 		}
 		if got, want := m.Confidence, tt.minConf; got < want {
 			t.Errorf("NearestMatch(%q) returned confidence %v, want minimum of %v", tt.description, got, want)
 		}
 		if got, want := m.Confidence, tt.maxConf; got > want {
 			t.Errorf("NearestMatch(%q) = %v, want maxiumum of %v", tt.description, got, want)
 		}
 	}
 }

 type result struct {
 	key    string // key of expected nearest match
 	offset int    // offset of match in unknown string

 	// The confidence values are retrieved by simply running the classifier
 	// and noting the output. A value greater than the "max" is fine and
 	// the tests can be adjusted to account for it. A value less than "min"
 	// should be carefully scrutinzed before adjusting the tests.
 	minConf float64 // the lowest confidence accepted for the match
 	maxConf float64 // the highest confidence we expect for this match
 }

 func TestClassify_MultipleMatch(t *testing.T) {
 	c := New(DefaultConfidenceThreshold, FlattenWhitespace)
 	c.AddValue("gettysburg", gettysburg)
 	c.AddValue("declaration", declaration)
 	c.AddValue("declaration-close", declaration[:len(declaration)/2-1]+"_"+declaration[len(declaration)/2:])
 	c.AddValue("loremipsum", loremipsum)

 	cNormalize := New(DefaultConfidenceThreshold, FlattenWhitespace, removeNonWords)
 	cNormalize.AddValue("gettysburg", gettysburg)

 	tests := []struct {
 		description string
 		c           *Classifier
 		input       string // input string to match
 		want        []result
 	}{
 		{
 			description: "Exact text match",
 			c:           c,
 			input:       postmodernThesisNarratives + declaration + postmodernThesisCollapse,
 			want: []result{
 				{
 					key:     "declaration",
 					offset:  842,
 					minConf: 1.0,
 					maxConf: 1.0,
 				},
 			},
 		},
 		{
 			description: "Partial text match",
 			c:           c,
 			input:       postmodernThesisNarratives + modifiedLorem + postmodernThesisCollapse,
 			want: []result{
 				{
 					key:     "loremipsum",
 					offset:  842,
 					minConf: 0.90,
 					maxConf: 0.91,
 				},
 			},
 		},
 		{
 			description: "Two partial matches",
 			c:           c,
 			input:       postmodernThesisNarratives + modifiedLorem + postmodernThesisCollapse + modifiedGettysburg + postmodernThesisFatalFlaw,
 			want: []result{
 				{
 					key:     "loremipsum",
 					offset:  842,
 					minConf: 0.90,
 					maxConf: 0.91,
 				},
 				{
 					key:     "gettysburg",
 					offset:  1900,
 					minConf: 0.86,
 					maxConf: 0.87,
 				},
 			},
 		},
 		{
 			description: "Partial matches of similar text",
 			c:           c,
 			input:       postmodernThesisNarratives + modifiedLorem + postmodernThesisCollapse + lessModifiedLorem + postmodernThesisFatalFlaw,
 			want: []result{
 				{
 					key:     "loremipsum",
 					offset:  1900,
 					minConf: 0.98,
 					maxConf: 0.99,
 				},
 				{
 					key:     "loremipsum",
 					offset:  842,
 					minConf: 0.90,
 					maxConf: 0.91,
 				},
 			},
 		},
 		{
 			description: "Nullifiable text",
 			c:           c,
 			input:       nullifiable,
 			want:        nil,
 		},
 		{
 			description: "No match",
 			c:           c,
 			input:       postmodernThesisNarratives + postmodernThesisCollapse,
 			want:        nil,
 		},
 		{
 			description: "Exact text match, with extra word and non-word normalizer",
 			c:           cNormalize,
 			input:       postmodernThesisNarratives + gettysburgExtraWord + postmodernThesisCollapse,
 			want: []result{
 				{
 					key:     "gettysburg",
 					offset:  820,
 					minConf: 1.0,
 					maxConf: 1.0,
 				},
 			},
 		},
 	}

 	for _, tt := range tests {
 		matches := tt.c.MultipleMatch(tt.input)
 		if len(matches) != len(tt.want) {
 			t.Errorf("MultipleMatch(%q) not enough matches = %v, want %v", tt.description, len(matches), len(tt.want))
 		}

 		for i := 0; i < len(matches); i++ {
 			m := matches[i]
 			w := tt.want[i]
 			if got, want := m.Name, w.key; got != want {
 				t.Errorf("MultipleMatch(%q) = %q, want %q", tt.description, got, want)
 			}
 			if got, want := m.Confidence, w.minConf; got < want {
 				t.Errorf("MultipleMatch(%q) %q = %v, want minimum of %v", tt.description, w.key, got, want)
 			}
 			if got, want := m.Confidence, w.maxConf; got > want {
 				t.Errorf("MultipleMatch(%q) %q = %v, want maximum of %v", tt.description, w.key, got, want)
 			}
 			if got, want := m.Offset, w.offset; got != want {
 				t.Errorf("MultipleMatch(%q) %q = %v, want offset of %v", tt.description, w.key, got, want)
 			}
 		}
 	}
 }

 func TestClassify_DiffRatio(t *testing.T) {
 	tests := []struct {
 		x, y string
 		want float64
 	}{
 		{"", "", 1.0},
 		{"a", "b", 1.0},
 		{"", "abc", 0},
 		{"ab", "c", 0.5},
 		{"a", "bc", 0.5},
 		{"a", "bcde", 0.25},
 	}

 	for _, tt := range tests {
 		if got, want := diffRatio(tt.x, tt.y), tt.want; got != want {
 			t.Errorf("diffRatio(%q, %q) = %f, want %f", tt.x, tt.y, got, want)
 		}
 	}
 }

 func TestClassify_Matches(t *testing.T) {
 	tests := []struct {
 		description string
 		matches     Matches
 		want        Matches
 	}{
 		{
 			description: "Different names, same confidences, same offset",
 			matches: Matches{
 				&Match{
 					Name:       "b",
 					Confidence: 0.42,
 					Offset:     0,
 				},
 				&Match{
 					Name:       "a",
 					Confidence: 0.42,
 					Offset:     0,
 				},
 			},
 			want: Matches{
 				&Match{
 					Name:       "a",
 					Confidence: 0.42,
 					Offset:     0,
 				},
 				&Match{
 					Name:       "b",
 					Confidence: 0.42,
 					Offset:     0,
 				},
 			},
 		},
 		{
 			description: "Same names, different confidences, same offset",
 			matches: Matches{
 				&Match{
 					Name:       "b",
 					Confidence: 0.42,
 					Offset:     0,
 				},
 				&Match{
 					Name:       "b",
 					Confidence: 0.90,
 					Offset:     0,
 				},
 			},
 			want: Matches{
 				&Match{
 					Name:       "b",
 					Confidence: 0.90,
 					Offset:     0,
 				},
 				&Match{
 					Name:       "b",
 					Confidence: 0.42,
 					Offset:     0,
 				},
 			},
 		},
 		{
 			description: "Same names, same confidences, different offsets",
 			matches: Matches{
 				&Match{
 					Name:       "b",
 					Confidence: 0.42,
 					Offset:     42,
 				},
 				&Match{
 					Name:       "b",
 					Confidence: 0.42,
 					Offset:     0,
 				},
 			},
 			want: Matches{
 				&Match{
 					Name:       "b",
 					Confidence: 0.42,
 					Offset:     0,
 				},
 				&Match{
 					Name:       "b",
 					Confidence: 0.42,
 					Offset:     42,
 				},
 			},
 		},

 		{
 			description: "Different names, different confidences, same offset",
 			matches: Matches{
 				&Match{
 					Name:       "b",
 					Confidence: 0.42,
 					Offset:     0,
 				},
 				&Match{
 					Name:       "a",
 					Confidence: 0.90,
 					Offset:     0,
 				},
 			},
 			want: Matches{
 				&Match{
 					Name:       "a",
 					Confidence: 0.90,
 					Offset:     0,
 				},
 				&Match{
 					Name:       "b",
 					Confidence: 0.42,
 					Offset:     0,
 				},
 			},
 		},
 		{
 			description: "Different names, same confidences, different offset",
 			matches: Matches{
 				&Match{
 					Name:       "b",
 					Confidence: 0.42,
 					Offset:     37,
 				},
 				&Match{
 					Name:       "a",
 					Confidence: 0.42,
 					Offset:     0,
 				},
 			},
 			want: Matches{
 				&Match{
 					Name:       "a",
 					Confidence: 0.42,
 					Offset:     0,
 				},
 				&Match{
 					Name:       "b",
 					Confidence: 0.42,
 					Offset:     37,
 				},
 			},
 		},
 		{
 			description: "Different names, different confidences, different offset",
 			matches: Matches{
 				&Match{
 					Name:       "a",
 					Confidence: 0.42,
 					Offset:     0,
 				},
 				&Match{
 					Name:       "b",
 					Confidence: 0.90,
 					Offset:     37,
 				},
 			},
 			want: Matches{
 				&Match{
 					Name:       "b",
 					Confidence: 0.90,
 					Offset:     37,
 				},
 				&Match{
 					Name:       "a",
 					Confidence: 0.42,
 					Offset:     0,
 				},
 			},
 		},
 	}

 	for _, tt := range tests {
 		sort.Sort(tt.matches)
 		if !reflect.DeepEqual(tt.matches, tt.want) {
 			for _, x := range tt.matches {
 				t.Errorf("got: %v", x)
 			}
 			for _, x := range tt.want {
 				t.Errorf("want: %v", x)
 			}
 			t.Errorf("MatchesSort(%q) = %v, want %v", tt.description, tt.matches, tt.want)
 		}
 	}
 }

 func TestClassify_DiffRangeEnd(t *testing.T) {
 	dmp := diffmatchpatch.New()
 	tests := []struct {
 		description string
 		unknown     string
 		known       string
 		end         int
 	}{
 		{
 			description: "identical",
 			unknown:     declaration,
 			known:       declaration,
 			end:         1,
 		},
 		{
 			description: "lorem",
 			unknown:     lessModifiedLorem,
 			known:       loremipsum,
 			end:         3,
 		},
 		{
 			description: "gettysburg",
 			unknown:     modifiedGettysburg,
 			known:       gettysburg,
 			end:         19,
 		},
 	}

 	for _, tt := range tests {
 		diffs := dmp.DiffMain(tt.unknown, tt.known, true)
 		if e := diffRangeEnd(tt.known, diffs); e != tt.end {
 			t.Errorf("DiffRangeEnd(%q) = end %v, want %v", tt.description, e, tt.end)
 		}
 	}
 }

 func BenchmarkClassifier(b *testing.B) {
 	c := New(DefaultConfidenceThreshold, FlattenWhitespace)
 	c.AddValue("gettysburg", gettysburg)
 	c.AddValue("declaration", declaration)
 	c.AddValue("loremipsum", loremipsum)

 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		c.NearestMatch(modifiedLorem)
 	}
 }
	// Copyright 2017 Google Inc.
	//
	// Licensed under the Apache License, Version 2.0 (the "License");
	// you may not use this file except in compliance with the License.
	// You may obtain a copy of the License at
	//
	// http://www.apache.org/licenses/LICENSE-2.0
	//
	// Unless required by applicable law or agreed to in writing, software
	// distributed under the License is distributed on an "AS IS" BASIS,
	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	// See the License for the specific language governing permissions and
	// limitations under the License.
	package stringclassifier

	import (
	"reflect"
	"regexp"
	"sort"
	"testing"

	"github.com/sergi/go-diff/diffmatchpatch"
	)

	var (
	gettysburg = `Four score and seven years ago our fathers brought forth
	on this continent, a new nation, conceived in Liberty, and dedicated to the
	proposition that all men are created equal.`
	modifiedGettysburg = `Four score and seven years ago our fathers brought forth
	on this continent, a nation that was new and improved, conceived in Liberty, and
	dedicated to the proposition that all men are created equal.`
	gettysburgExtraWord = `Four score and seven years ago our fathers brought forth
	on this continent, a new nation, conceived in Liberty, and dedicated to the
	proposition that all men are created equal.Foobar`

	declaration = `When in the Course of human events, it becomes necessary
	for one people to dissolve the political bands which have connected them with
	another, and to assume among the powers of the earth, the separate and equal
	station to which the Laws of Nature and of Nature's God entitle them, a decent
	respect to the opinions of mankind requires that they should declare the causes
	which impel them to the separation.`

	loremipsum = `Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nulla
	varius enim mattis, rhoncus lectus id, aliquet sem. Phasellus eget ex in dolor
	feugiat ultricies. Etiam interdum sit amet nisl in placerat. Sed vitae enim
	vulputate, tempus leo commodo, accumsan nulla.`
	modifiedLorem = `Lorem ipsum dolor amet, consectetur adipiscing elit. Nulla
	varius enim mattis, lectus id, aliquet rhoncus sem. Phasellus eget ex in dolor
	feugiat ultricies. Etiam interdum sit amet sit nisl in placerat. Sed vitae enim
	vulputate, tempus leo commodo, accumsan nulla.`
	lessModifiedLorem = `Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nulla
	varius enim mattis, rhoncus lectus id, aliquet. Phasellus eget ex in dolor
	feugiat ultricies. Etiam interdum sit amet nisl in placerat. Sed vitae enim
	vulputate, tempus leo commodo, accumsan nulla.`

	postmodernThesisCollapse = `1. Expressions of collapse

	If one examines postcultural Marxism, one is faced with a choice: either
	reject capitalist submodern theory or conclude that the purpose of the reader
	is significant form. Bataille uses the term ‘capitalist construction’ to denote
	not, in fact, discourse, but prediscourse.

	Therefore, in Stardust, Gaiman analyses postcultural Marxism; in
	The Books of Magic, although, he denies capitalist submodern theory. If
	capitalist construction holds, we have to choose between capitalist submodern
	theory and Baudrillardist simulacra.

	However, conceptualist socialism implies that narrativity may be used to
	oppress the proletariat, given that sexuality is distinct from art. The subject
	is interpolated into a capitalist construction that includes language as a
	paradox.
	`
	postmodernThesisNarratives = `1. Narratives of failure

	The main theme of the works of Joyce is the defining characteristic, and some
	would say the economy, of neocultural class. But Bataille promotes the use of
	socialist realism to deconstruct sexual identity.

	The subject is interpolated into a Baudrillardist simulation that includes
	consciousness as a whole. Thus, the primary theme of Pickett's[1] model of
	socialist realism is the role of the reader as artist.

	The subject is contextualised into a postcapitalist discourse that includes
	language as a paradox. It could be said that if Baudrillardist simulation
	holds, the works of Gibson are postmodern. The characteristic theme of the
	works of Gibson is the common ground between society and narrativity. However,
	Sartre uses the term 'postcapitalist discourse' to denote not, in fact,
	narrative, but postnarrative.
	`
	postmodernThesisFatalFlaw = `1. Contexts of fatal flaw

	"Narrativity is part of the dialectic of culture," says Marx; however,
	according to Hamburger[1] , it is not so much narrativity that is part of the
	dialectic of culture, but rather the stasis, and hence the defining
	characteristic, of narrativity. Bataille promotes the use of Batailleist
	'powerful communication' to modify society.

	If one examines the presemioticist paradigm of reality, one is faced with a
	choice: either reject Batailleist 'powerful communication' or conclude that
	concensus must come from the masses. Therefore, Baudrillard uses the term 'the
	presemioticist paradigm of reality' to denote the difference between class and
	society. The subject is interpolated into a subtextual capitalist theory that
	includes consciousness as a whole.

	However, Pickett[2] implies that we have to choose between neotextual feminism
	and dialectic appropriation. Debord suggests the use of subtextual capitalist
	theory to deconstruct the status quo.
	`
	nullifiable = `[[ , _ , _ , _
	? _ : _
	? _ : _
	? _ : _
	]}
	`
	nonWords = regexp.MustCompile("[[:punct:]]+")
	)

	// removeNonWords removes non-words from the string, replacing them with empty
	// string. (This is meant to exercise tokenization problems.)
	func removeNonWords(s string) string {
	return nonWords.ReplaceAllString(s, "")
	}

	func TestClassify_NearestMatch(t *testing.T) {
	c := New(DefaultConfidenceThreshold, FlattenWhitespace)
	c.AddValue("gettysburg", gettysburg)
	c.AddValue("declaration", declaration)
	c.AddValue("loremipsum", loremipsum)

	tests := []struct {
	description string
	input string // input string to match
	name string // name of expected nearest match
	minConf float64 // the lowest confidence accepted for the match
	maxConf float64 // the highest confidence we expect for this match
	}{
	{
	description: "Full Declaration",
	input: declaration,
	name: "declaration",
	minConf: 1.0,
	maxConf: 1.0,
	},
	{
	description: "Modified Lorem",
	input: modifiedLorem,
	name: "loremipsum",
	minConf: 0.90,
	maxConf: 0.91,
	},
	{
	description: "Modified Gettysburg",
	input: modifiedGettysburg,
	name: "gettysburg",
	minConf: 0.86,
	maxConf: 0.87,
	},
	}

	for _, tt := range tests {
	m := c.NearestMatch(tt.input)

	if got, want := m.Name, tt.name; got != want {
	t.Errorf("NearestMatch(%q) = %q, want %q", tt.description, got, want)
	}
	if got, want := m.Confidence, tt.minConf; got < want {
	t.Errorf("NearestMatch(%q) returned confidence %v, want minimum of %v", tt.description, got, want)
	}
	if got, want := m.Confidence, tt.maxConf; got > want {
	t.Errorf("NearestMatch(%q) = %v, want maxiumum of %v", tt.description, got, want)
	}
	}
	}

	type result struct {
	key string // key of expected nearest match
	offset int // offset of match in unknown string

	// The confidence values are retrieved by simply running the classifier
	// and noting the output. A value greater than the "max" is fine and
	// the tests can be adjusted to account for it. A value less than "min"
	// should be carefully scrutinzed before adjusting the tests.
	minConf float64 // the lowest confidence accepted for the match
	maxConf float64 // the highest confidence we expect for this match
	}

	func TestClassify_MultipleMatch(t *testing.T) {
	c := New(DefaultConfidenceThreshold, FlattenWhitespace)
	c.AddValue("gettysburg", gettysburg)
	c.AddValue("declaration", declaration)
	c.AddValue("declaration-close", declaration[:len(declaration)/2-1]+"_"+declaration[len(declaration)/2:])
	c.AddValue("loremipsum", loremipsum)

	cNormalize := New(DefaultConfidenceThreshold, FlattenWhitespace, removeNonWords)
	cNormalize.AddValue("gettysburg", gettysburg)

	tests := []struct {
	description string
	c *Classifier
	input string // input string to match
	want []result
	}{
	{
	description: "Exact text match",
	c: c,
	input: postmodernThesisNarratives + declaration + postmodernThesisCollapse,
	want: []result{
	{
	key: "declaration",
	offset: 842,
	minConf: 1.0,
	maxConf: 1.0,
	},
	},
	},
	{
	description: "Partial text match",
	c: c,
	input: postmodernThesisNarratives + modifiedLorem + postmodernThesisCollapse,
	want: []result{
	{
	key: "loremipsum",
	offset: 842,
	minConf: 0.90,
	maxConf: 0.91,
	},
	},
	},
	{
	description: "Two partial matches",
	c: c,
	input: postmodernThesisNarratives + modifiedLorem + postmodernThesisCollapse + modifiedGettysburg + postmodernThesisFatalFlaw,
	want: []result{
	{
	key: "loremipsum",
	offset: 842,
	minConf: 0.90,
	maxConf: 0.91,
	},
	{
	key: "gettysburg",
	offset: 1900,
	minConf: 0.86,
	maxConf: 0.87,
	},
	},
	},
	{
	description: "Partial matches of similar text",
	c: c,
	input: postmodernThesisNarratives + modifiedLorem + postmodernThesisCollapse + lessModifiedLorem + postmodernThesisFatalFlaw,
	want: []result{
	{
	key: "loremipsum",
	offset: 1900,
	minConf: 0.98,
	maxConf: 0.99,
	},
	{
	key: "loremipsum",
	offset: 842,
	minConf: 0.90,
	maxConf: 0.91,
	},
	},
	},
	{
	description: "Nullifiable text",
	c: c,
	input: nullifiable,
	want: nil,
	},
	{
	description: "No match",
	c: c,
	input: postmodernThesisNarratives + postmodernThesisCollapse,
	want: nil,
	},
	{
	description: "Exact text match, with extra word and non-word normalizer",
	c: cNormalize,
	input: postmodernThesisNarratives + gettysburgExtraWord + postmodernThesisCollapse,
	want: []result{
	{
	key: "gettysburg",
	offset: 820,
	minConf: 1.0,
	maxConf: 1.0,
	},
	},
	},
	}

	for _, tt := range tests {
	matches := tt.c.MultipleMatch(tt.input)
	if len(matches) != len(tt.want) {
	t.Errorf("MultipleMatch(%q) not enough matches = %v, want %v", tt.description, len(matches), len(tt.want))
	}

	for i := 0; i < len(matches); i++ {
	m := matches[i]
	w := tt.want[i]
	if got, want := m.Name, w.key; got != want {
	t.Errorf("MultipleMatch(%q) = %q, want %q", tt.description, got, want)
	}
	if got, want := m.Confidence, w.minConf; got < want {
	t.Errorf("MultipleMatch(%q) %q = %v, want minimum of %v", tt.description, w.key, got, want)
	}
	if got, want := m.Confidence, w.maxConf; got > want {
	t.Errorf("MultipleMatch(%q) %q = %v, want maximum of %v", tt.description, w.key, got, want)
	}
	if got, want := m.Offset, w.offset; got != want {
	t.Errorf("MultipleMatch(%q) %q = %v, want offset of %v", tt.description, w.key, got, want)
	}
	}
	}
	}

	func TestClassify_DiffRatio(t *testing.T) {
	tests := []struct {
	x, y string
	want float64
	}{
	{"", "", 1.0},
	{"a", "b", 1.0},
	{"", "abc", 0},
	{"ab", "c", 0.5},
	{"a", "bc", 0.5},
	{"a", "bcde", 0.25},
	}

	for _, tt := range tests {
	if got, want := diffRatio(tt.x, tt.y), tt.want; got != want {
	t.Errorf("diffRatio(%q, %q) = %f, want %f", tt.x, tt.y, got, want)
	}
	}
	}

	func TestClassify_Matches(t *testing.T) {
	tests := []struct {
	description string
	matches Matches
	want Matches
	}{
	{
	description: "Different names, same confidences, same offset",
	matches: Matches{
	&Match{
	Name: "b",
	Confidence: 0.42,
	Offset: 0,
	},
	&Match{
	Name: "a",
	Confidence: 0.42,
	Offset: 0,
	},
	},
	want: Matches{
	&Match{
	Name: "a",
	Confidence: 0.42,
	Offset: 0,
	},
	&Match{
	Name: "b",
	Confidence: 0.42,
	Offset: 0,
	},
	},
	},
	{
	description: "Same names, different confidences, same offset",
	matches: Matches{
	&Match{
	Name: "b",
	Confidence: 0.42,
	Offset: 0,
	},
	&Match{
	Name: "b",
	Confidence: 0.90,
	Offset: 0,
	},
	},
	want: Matches{
	&Match{
	Name: "b",
	Confidence: 0.90,
	Offset: 0,
	},
	&Match{
	Name: "b",
	Confidence: 0.42,
	Offset: 0,
	},
	},
	},
	{
	description: "Same names, same confidences, different offsets",
	matches: Matches{
	&Match{
	Name: "b",
	Confidence: 0.42,
	Offset: 42,
	},
	&Match{
	Name: "b",
	Confidence: 0.42,
	Offset: 0,
	},
	},
	want: Matches{
	&Match{
	Name: "b",
	Confidence: 0.42,
	Offset: 0,
	},
	&Match{
	Name: "b",
	Confidence: 0.42,
	Offset: 42,
	},
	},
	},

	{
	description: "Different names, different confidences, same offset",
	matches: Matches{
	&Match{
	Name: "b",
	Confidence: 0.42,
	Offset: 0,
	},
	&Match{
	Name: "a",
	Confidence: 0.90,
	Offset: 0,
	},
	},
	want: Matches{
	&Match{
	Name: "a",
	Confidence: 0.90,
	Offset: 0,
	},
	&Match{
	Name: "b",
	Confidence: 0.42,
	Offset: 0,
	},
	},
	},
	{
	description: "Different names, same confidences, different offset",
	matches: Matches{
	&Match{
	Name: "b",
	Confidence: 0.42,
	Offset: 37,
	},
	&Match{
	Name: "a",
	Confidence: 0.42,
	Offset: 0,
	},
	},
	want: Matches{
	&Match{
	Name: "a",
	Confidence: 0.42,
	Offset: 0,
	},
	&Match{
	Name: "b",
	Confidence: 0.42,
	Offset: 37,
	},
	},
	},
	{
	description: "Different names, different confidences, different offset",
	matches: Matches{
	&Match{
	Name: "a",
	Confidence: 0.42,
	Offset: 0,
	},
	&Match{
	Name: "b",
	Confidence: 0.90,
	Offset: 37,
	},
	},
	want: Matches{
	&Match{
	Name: "b",
	Confidence: 0.90,
	Offset: 37,
	},
	&Match{
	Name: "a",
	Confidence: 0.42,
	Offset: 0,
	},
	},
	},
	}

	for _, tt := range tests {
	sort.Sort(tt.matches)
	if !reflect.DeepEqual(tt.matches, tt.want) {
	for _, x := range tt.matches {
	t.Errorf("got: %v", x)
	}
	for _, x := range tt.want {
	t.Errorf("want: %v", x)
	}
	t.Errorf("MatchesSort(%q) = %v, want %v", tt.description, tt.matches, tt.want)
	}
	}
	}

	func TestClassify_DiffRangeEnd(t *testing.T) {
	dmp := diffmatchpatch.New()
	tests := []struct {
	description string
	unknown string
	known string
	end int
	}{
	{
	description: "identical",
	unknown: declaration,
	known: declaration,
	end: 1,
	},
	{
	description: "lorem",
	unknown: lessModifiedLorem,
	known: loremipsum,
	end: 3,
	},
	{
	description: "gettysburg",
	unknown: modifiedGettysburg,
	known: gettysburg,
	end: 19,
	},
	}

	for _, tt := range tests {
	diffs := dmp.DiffMain(tt.unknown, tt.known, true)
	if e := diffRangeEnd(tt.known, diffs); e != tt.end {
	t.Errorf("DiffRangeEnd(%q) = end %v, want %v", tt.description, e, tt.end)
	}
	}
	}

	func BenchmarkClassifier(b *testing.B) {
	c := New(DefaultConfidenceThreshold, FlattenWhitespace)
	c.AddValue("gettysburg", gettysburg)
	c.AddValue("declaration", declaration)
	c.AddValue("loremipsum", loremipsum)

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
	c.NearestMatch(modifiedLorem)
	}
	}