vendor/github.com/flynn-archive/go-shlex/shlex.go - third_party/github.com/moby/moby - Git at Google

 /*
 Copyright 2012 Google Inc. All Rights Reserved.

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */

 package shlex

 /*
 Package shlex implements a simple lexer which splits input in to tokens using
 shell-style rules for quoting and commenting.
 */
 import (
 	"bufio"
 	"errors"
 	"fmt"
 	"io"
 	"strings"
 )

 /*
 A TokenType is a top-level token; a word, space, comment, unknown.
 */
 type TokenType int

 /*
 A RuneTokenType is the type of a UTF-8 character; a character, quote, space, escape.
 */
 type RuneTokenType int

 type lexerState int

 type Token struct {
 	tokenType TokenType
 	value     string
 }

 /*
 Two tokens are equal if both their types and values are equal. A nil token can
 never equal another token.
 */
 func (a *Token) Equal(b *Token) bool {
 	if a == nil || b == nil {
 		return false
 	}
 	if a.tokenType != b.tokenType {
 		return false
 	}
 	return a.value == b.value
 }

 const (
 	RUNE_CHAR              string = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789._-,/@$*()+=><:;&^%~|!?[]{}"
 	RUNE_SPACE             string = " \t\r\n"
 	RUNE_ESCAPING_QUOTE    string = "\""
 	RUNE_NONESCAPING_QUOTE string = "'"
 	RUNE_ESCAPE                   = "\\"
 	RUNE_COMMENT                  = "#"

 	RUNETOKEN_UNKNOWN           RuneTokenType = 0
 	RUNETOKEN_CHAR              RuneTokenType = 1
 	RUNETOKEN_SPACE             RuneTokenType = 2
 	RUNETOKEN_ESCAPING_QUOTE    RuneTokenType = 3
 	RUNETOKEN_NONESCAPING_QUOTE RuneTokenType = 4
 	RUNETOKEN_ESCAPE            RuneTokenType = 5
 	RUNETOKEN_COMMENT           RuneTokenType = 6
 	RUNETOKEN_EOF               RuneTokenType = 7

 	TOKEN_UNKNOWN TokenType = 0
 	TOKEN_WORD    TokenType = 1
 	TOKEN_SPACE   TokenType = 2
 	TOKEN_COMMENT TokenType = 3

 	STATE_START           lexerState = 0
 	STATE_INWORD          lexerState = 1
 	STATE_ESCAPING        lexerState = 2
 	STATE_ESCAPING_QUOTED lexerState = 3
 	STATE_QUOTED_ESCAPING lexerState = 4
 	STATE_QUOTED          lexerState = 5
 	STATE_COMMENT         lexerState = 6

 	INITIAL_TOKEN_CAPACITY int = 100
 )

 /*
 A type for classifying characters. This allows for different sorts of
 classifiers - those accepting extended non-ascii chars, or strict posix
 compatibility, for example.
 */
 type TokenClassifier struct {
 	typeMap map[int32]RuneTokenType
 }

 func addRuneClass(typeMap *map[int32]RuneTokenType, runes string, tokenType RuneTokenType) {
 	for _, rune := range runes {
 		(*typeMap)[int32(rune)] = tokenType
 	}
 }

 /*
 Create a new classifier for basic ASCII characters.
 */
 func NewDefaultClassifier() *TokenClassifier {
 	typeMap := map[int32]RuneTokenType{}
 	addRuneClass(&typeMap, RUNE_CHAR, RUNETOKEN_CHAR)
 	addRuneClass(&typeMap, RUNE_SPACE, RUNETOKEN_SPACE)
 	addRuneClass(&typeMap, RUNE_ESCAPING_QUOTE, RUNETOKEN_ESCAPING_QUOTE)
 	addRuneClass(&typeMap, RUNE_NONESCAPING_QUOTE, RUNETOKEN_NONESCAPING_QUOTE)
 	addRuneClass(&typeMap, RUNE_ESCAPE, RUNETOKEN_ESCAPE)
 	addRuneClass(&typeMap, RUNE_COMMENT, RUNETOKEN_COMMENT)
 	return &TokenClassifier{
 		typeMap: typeMap}
 }

 func (classifier *TokenClassifier) ClassifyRune(rune int32) RuneTokenType {
 	return classifier.typeMap[rune]
 }

 /*
 A type for turning an input stream in to a sequence of strings. Whitespace and
 comments are skipped.
 */
 type Lexer struct {
 	tokenizer *Tokenizer
 }

 /*
 Create a new lexer.
 */
 func NewLexer(r io.Reader) (*Lexer, error) {

 	tokenizer, err := NewTokenizer(r)
 	if err != nil {
 		return nil, err
 	}
 	lexer := &Lexer{tokenizer: tokenizer}
 	return lexer, nil
 }

 /*
 Return the next word, and an error value. If there are no more words, the error
 will be io.EOF.
 */
 func (l *Lexer) NextWord() (string, error) {
 	var token *Token
 	var err error
 	for {
 		token, err = l.tokenizer.NextToken()
 		if err != nil {
 			return "", err
 		}
 		switch token.tokenType {
 		case TOKEN_WORD:
 			{
 				return token.value, nil
 			}
 		case TOKEN_COMMENT:
 			{
 				// skip comments
 			}
 		default:
 			{
 				panic(fmt.Sprintf("Unknown token type: %v", token.tokenType))
 			}
 		}
 	}
 	return "", io.EOF
 }

 /*
 A type for turning an input stream in to a sequence of typed tokens.
 */
 type Tokenizer struct {
 	input      *bufio.Reader
 	classifier *TokenClassifier
 }

 /*
 Create a new tokenizer.
 */
 func NewTokenizer(r io.Reader) (*Tokenizer, error) {
 	input := bufio.NewReader(r)
 	classifier := NewDefaultClassifier()
 	tokenizer := &Tokenizer{
 		input:      input,
 		classifier: classifier}
 	return tokenizer, nil
 }

 /*
 Scan the stream for the next token.

 This uses an internal state machine. It will panic if it encounters a character
 which it does not know how to handle.
 */
 func (t *Tokenizer) scanStream() (*Token, error) {
 	state := STATE_START
 	var tokenType TokenType
 	value := make([]int32, 0, INITIAL_TOKEN_CAPACITY)
 	var (
 		nextRune     int32
 		nextRuneType RuneTokenType
 		err          error
 	)
 SCAN:
 	for {
 		nextRune, _, err = t.input.ReadRune()
 		nextRuneType = t.classifier.ClassifyRune(nextRune)
 		if err != nil {
 			if err == io.EOF {
 				nextRuneType = RUNETOKEN_EOF
 				err = nil
 			} else {
 				return nil, err
 			}
 		}
 		switch state {
 		case STATE_START: // no runes read yet
 			{
 				switch nextRuneType {
 				case RUNETOKEN_EOF:
 					{
 						return nil, io.EOF
 					}
 				case RUNETOKEN_CHAR:
 					{
 						tokenType = TOKEN_WORD
 						value = append(value, nextRune)
 						state = STATE_INWORD
 					}
 				case RUNETOKEN_SPACE:
 					{
 					}
 				case RUNETOKEN_ESCAPING_QUOTE:
 					{
 						tokenType = TOKEN_WORD
 						state = STATE_QUOTED_ESCAPING
 					}
 				case RUNETOKEN_NONESCAPING_QUOTE:
 					{
 						tokenType = TOKEN_WORD
 						state = STATE_QUOTED
 					}
 				case RUNETOKEN_ESCAPE:
 					{
 						tokenType = TOKEN_WORD
 						state = STATE_ESCAPING
 					}
 				case RUNETOKEN_COMMENT:
 					{
 						tokenType = TOKEN_COMMENT
 						state = STATE_COMMENT
 					}
 				default:
 					{
 						return nil, errors.New(fmt.Sprintf("Unknown rune: %v", nextRune))
 					}
 				}
 			}
 		case STATE_INWORD: // in a regular word
 			{
 				switch nextRuneType {
 				case RUNETOKEN_EOF:
 					{
 						break SCAN
 					}
 				case RUNETOKEN_CHAR, RUNETOKEN_COMMENT:
 					{
 						value = append(value, nextRune)
 					}
 				case RUNETOKEN_SPACE:
 					{
 						t.input.UnreadRune()
 						break SCAN
 					}
 				case RUNETOKEN_ESCAPING_QUOTE:
 					{
 						state = STATE_QUOTED_ESCAPING
 					}
 				case RUNETOKEN_NONESCAPING_QUOTE:
 					{
 						state = STATE_QUOTED
 					}
 				case RUNETOKEN_ESCAPE:
 					{
 						state = STATE_ESCAPING
 					}
 				default:
 					{
 						return nil, errors.New(fmt.Sprintf("Uknown rune: %v", nextRune))
 					}
 				}
 			}
 		case STATE_ESCAPING: // the next rune after an escape character
 			{
 				switch nextRuneType {
 				case RUNETOKEN_EOF:
 					{
 						err = errors.New("EOF found after escape character")
 						break SCAN
 					}
 				case RUNETOKEN_CHAR, RUNETOKEN_SPACE, RUNETOKEN_ESCAPING_QUOTE, RUNETOKEN_NONESCAPING_QUOTE, RUNETOKEN_ESCAPE, RUNETOKEN_COMMENT:
 					{
 						state = STATE_INWORD
 						value = append(value, nextRune)
 					}
 				default:
 					{
 						return nil, errors.New(fmt.Sprintf("Uknown rune: %v", nextRune))
 					}
 				}
 			}
 		case STATE_ESCAPING_QUOTED: // the next rune after an escape character, in double quotes
 			{
 				switch nextRuneType {
 				case RUNETOKEN_EOF:
 					{
 						err = errors.New("EOF found after escape character")
 						break SCAN
 					}
 				case RUNETOKEN_CHAR, RUNETOKEN_SPACE, RUNETOKEN_ESCAPING_QUOTE, RUNETOKEN_NONESCAPING_QUOTE, RUNETOKEN_ESCAPE, RUNETOKEN_COMMENT:
 					{
 						state = STATE_QUOTED_ESCAPING
 						value = append(value, nextRune)
 					}
 				default:
 					{
 						return nil, errors.New(fmt.Sprintf("Uknown rune: %v", nextRune))
 					}
 				}
 			}
 		case STATE_QUOTED_ESCAPING: // in escaping double quotes
 			{
 				switch nextRuneType {
 				case RUNETOKEN_EOF:
 					{
 						err = errors.New("EOF found when expecting closing quote.")
 						break SCAN
 					}
 				case RUNETOKEN_CHAR, RUNETOKEN_UNKNOWN, RUNETOKEN_SPACE, RUNETOKEN_NONESCAPING_QUOTE, RUNETOKEN_COMMENT:
 					{
 						value = append(value, nextRune)
 					}
 				case RUNETOKEN_ESCAPING_QUOTE:
 					{
 						state = STATE_INWORD
 					}
 				case RUNETOKEN_ESCAPE:
 					{
 						state = STATE_ESCAPING_QUOTED
 					}
 				default:
 					{
 						return nil, errors.New(fmt.Sprintf("Uknown rune: %v", nextRune))
 					}
 				}
 			}
 		case STATE_QUOTED: // in non-escaping single quotes
 			{
 				switch nextRuneType {
 				case RUNETOKEN_EOF:
 					{
 						err = errors.New("EOF found when expecting closing quote.")
 						break SCAN
 					}
 				case RUNETOKEN_CHAR, RUNETOKEN_UNKNOWN, RUNETOKEN_SPACE, RUNETOKEN_ESCAPING_QUOTE, RUNETOKEN_ESCAPE, RUNETOKEN_COMMENT:
 					{
 						value = append(value, nextRune)
 					}
 				case RUNETOKEN_NONESCAPING_QUOTE:
 					{
 						state = STATE_INWORD
 					}
 				default:
 					{
 						return nil, errors.New(fmt.Sprintf("Uknown rune: %v", nextRune))
 					}
 				}
 			}
 		case STATE_COMMENT:
 			{
 				switch nextRuneType {
 				case RUNETOKEN_EOF:
 					{
 						break SCAN
 					}
 				case RUNETOKEN_CHAR, RUNETOKEN_UNKNOWN, RUNETOKEN_ESCAPING_QUOTE, RUNETOKEN_ESCAPE, RUNETOKEN_COMMENT, RUNETOKEN_NONESCAPING_QUOTE:
 					{
 						value = append(value, nextRune)
 					}
 				case RUNETOKEN_SPACE:
 					{
 						if nextRune == '\n' {
 							state = STATE_START
 							break SCAN
 						} else {
 							value = append(value, nextRune)
 						}
 					}
 				default:
 					{
 						return nil, errors.New(fmt.Sprintf("Uknown rune: %v", nextRune))
 					}
 				}
 			}
 		default:
 			{
 				panic(fmt.Sprintf("Unexpected state: %v", state))
 			}
 		}
 	}
 	token := &Token{
 		tokenType: tokenType,
 		value:     string(value)}
 	return token, err
 }

 /*
 Return the next token in the stream, and an error value. If there are no more
 tokens available, the error value will be io.EOF.
 */
 func (t *Tokenizer) NextToken() (*Token, error) {
 	return t.scanStream()
 }

 /*
 Split a string in to a slice of strings, based upon shell-style rules for
 quoting, escaping, and spaces.
 */
 func Split(s string) ([]string, error) {
 	l, err := NewLexer(strings.NewReader(s))
 	if err != nil {
 		return nil, err
 	}
 	subStrings := []string{}
 	for {
 		word, err := l.NextWord()
 		if err != nil {
 			if err == io.EOF {
 				return subStrings, nil
 			}
 			return subStrings, err
 		}
 		subStrings = append(subStrings, word)
 	}
 	return subStrings, nil
 }
	/*
	Copyright 2012 Google Inc. All Rights Reserved.

	Licensed under the Apache License, Version 2.0 (the "License");
	you may not use this file except in compliance with the License.
	You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

	Unless required by applicable law or agreed to in writing, software
	distributed under the License is distributed on an "AS IS" BASIS,
	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	See the License for the specific language governing permissions and
	limitations under the License.
	*/

	package shlex

	/*
	Package shlex implements a simple lexer which splits input in to tokens using
	shell-style rules for quoting and commenting.
	*/
	import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"strings"
	)

	/*
	A TokenType is a top-level token; a word, space, comment, unknown.
	*/
	type TokenType int

	/*
	A RuneTokenType is the type of a UTF-8 character; a character, quote, space, escape.
	*/
	type RuneTokenType int

	type lexerState int

	type Token struct {
	tokenType TokenType
	value string
	}

	/*
	Two tokens are equal if both their types and values are equal. A nil token can
	never equal another token.
	*/
	func (a Token) Equal(b Token) bool {
	if a == nil \|\| b == nil {
	return false
	}
	if a.tokenType != b.tokenType {
	return false
	}
	return a.value == b.value
	}

	const (
	RUNE_CHAR string = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789._-,/@$*()+=><:;&^%~\|!?[]{}"
	RUNE_SPACE string = " \t\r\n"
	RUNE_ESCAPING_QUOTE string = "\""
	RUNE_NONESCAPING_QUOTE string = "'"
	RUNE_ESCAPE = "\\"
	RUNE_COMMENT = "#"

	RUNETOKEN_UNKNOWN RuneTokenType = 0
	RUNETOKEN_CHAR RuneTokenType = 1
	RUNETOKEN_SPACE RuneTokenType = 2
	RUNETOKEN_ESCAPING_QUOTE RuneTokenType = 3
	RUNETOKEN_NONESCAPING_QUOTE RuneTokenType = 4
	RUNETOKEN_ESCAPE RuneTokenType = 5
	RUNETOKEN_COMMENT RuneTokenType = 6
	RUNETOKEN_EOF RuneTokenType = 7

	TOKEN_UNKNOWN TokenType = 0
	TOKEN_WORD TokenType = 1
	TOKEN_SPACE TokenType = 2
	TOKEN_COMMENT TokenType = 3

	STATE_START lexerState = 0
	STATE_INWORD lexerState = 1
	STATE_ESCAPING lexerState = 2
	STATE_ESCAPING_QUOTED lexerState = 3
	STATE_QUOTED_ESCAPING lexerState = 4
	STATE_QUOTED lexerState = 5
	STATE_COMMENT lexerState = 6

	INITIAL_TOKEN_CAPACITY int = 100
	)

	/*
	A type for classifying characters. This allows for different sorts of
	classifiers - those accepting extended non-ascii chars, or strict posix
	compatibility, for example.
	*/
	type TokenClassifier struct {
	typeMap map[int32]RuneTokenType
	}

	func addRuneClass(typeMap *map[int32]RuneTokenType, runes string, tokenType RuneTokenType) {
	for _, rune := range runes {
	(*typeMap)[int32(rune)] = tokenType
	}
	}

	/*
	Create a new classifier for basic ASCII characters.
	*/
	func NewDefaultClassifier() *TokenClassifier {
	typeMap := map[int32]RuneTokenType{}
	addRuneClass(&typeMap, RUNE_CHAR, RUNETOKEN_CHAR)
	addRuneClass(&typeMap, RUNE_SPACE, RUNETOKEN_SPACE)
	addRuneClass(&typeMap, RUNE_ESCAPING_QUOTE, RUNETOKEN_ESCAPING_QUOTE)
	addRuneClass(&typeMap, RUNE_NONESCAPING_QUOTE, RUNETOKEN_NONESCAPING_QUOTE)
	addRuneClass(&typeMap, RUNE_ESCAPE, RUNETOKEN_ESCAPE)
	addRuneClass(&typeMap, RUNE_COMMENT, RUNETOKEN_COMMENT)
	return &TokenClassifier{
	typeMap: typeMap}
	}

	func (classifier *TokenClassifier) ClassifyRune(rune int32) RuneTokenType {
	return classifier.typeMap[rune]
	}

	/*
	A type for turning an input stream in to a sequence of strings. Whitespace and
	comments are skipped.
	*/
	type Lexer struct {
	tokenizer *Tokenizer
	}

	/*
	Create a new lexer.
	*/
	func NewLexer(r io.Reader) (*Lexer, error) {

	tokenizer, err := NewTokenizer(r)
	if err != nil {
	return nil, err
	}
	lexer := &Lexer{tokenizer: tokenizer}
	return lexer, nil
	}

	/*
	Return the next word, and an error value. If there are no more words, the error
	will be io.EOF.
	*/
	func (l *Lexer) NextWord() (string, error) {
	var token *Token
	var err error
	for {
	token, err = l.tokenizer.NextToken()
	if err != nil {
	return "", err
	}
	switch token.tokenType {
	case TOKEN_WORD:
	{
	return token.value, nil
	}
	case TOKEN_COMMENT:
	{
	// skip comments
	}
	default:
	{
	panic(fmt.Sprintf("Unknown token type: %v", token.tokenType))
	}
	}
	}
	return "", io.EOF
	}

	/*
	A type for turning an input stream in to a sequence of typed tokens.
	*/
	type Tokenizer struct {
	input *bufio.Reader
	classifier *TokenClassifier
	}

	/*
	Create a new tokenizer.
	*/
	func NewTokenizer(r io.Reader) (*Tokenizer, error) {
	input := bufio.NewReader(r)
	classifier := NewDefaultClassifier()
	tokenizer := &Tokenizer{
	input: input,
	classifier: classifier}
	return tokenizer, nil
	}

	/*
	Scan the stream for the next token.

	This uses an internal state machine. It will panic if it encounters a character
	which it does not know how to handle.
	*/
	func (t Tokenizer) scanStream() (Token, error) {
	state := STATE_START
	var tokenType TokenType
	value := make([]int32, 0, INITIAL_TOKEN_CAPACITY)
	var (
	nextRune int32
	nextRuneType RuneTokenType
	err error
	)
	SCAN:
	for {
	nextRune, _, err = t.input.ReadRune()
	nextRuneType = t.classifier.ClassifyRune(nextRune)
	if err != nil {
	if err == io.EOF {
	nextRuneType = RUNETOKEN_EOF
	err = nil
	} else {
	return nil, err
	}
	}
	switch state {
	case STATE_START: // no runes read yet
	{
	switch nextRuneType {
	case RUNETOKEN_EOF:
	{
	return nil, io.EOF
	}
	case RUNETOKEN_CHAR:
	{
	tokenType = TOKEN_WORD
	value = append(value, nextRune)
	state = STATE_INWORD
	}
	case RUNETOKEN_SPACE:
	{
	}
	case RUNETOKEN_ESCAPING_QUOTE:
	{
	tokenType = TOKEN_WORD
	state = STATE_QUOTED_ESCAPING
	}
	case RUNETOKEN_NONESCAPING_QUOTE:
	{
	tokenType = TOKEN_WORD
	state = STATE_QUOTED
	}
	case RUNETOKEN_ESCAPE:
	{
	tokenType = TOKEN_WORD
	state = STATE_ESCAPING
	}
	case RUNETOKEN_COMMENT:
	{
	tokenType = TOKEN_COMMENT
	state = STATE_COMMENT
	}
	default:
	{
	return nil, errors.New(fmt.Sprintf("Unknown rune: %v", nextRune))
	}
	}
	}
	case STATE_INWORD: // in a regular word
	{
	switch nextRuneType {
	case RUNETOKEN_EOF:
	{
	break SCAN
	}
	case RUNETOKEN_CHAR, RUNETOKEN_COMMENT:
	{
	value = append(value, nextRune)
	}
	case RUNETOKEN_SPACE:
	{
	t.input.UnreadRune()
	break SCAN
	}
	case RUNETOKEN_ESCAPING_QUOTE:
	{
	state = STATE_QUOTED_ESCAPING
	}
	case RUNETOKEN_NONESCAPING_QUOTE:
	{
	state = STATE_QUOTED
	}
	case RUNETOKEN_ESCAPE:
	{
	state = STATE_ESCAPING
	}
	default:
	{
	return nil, errors.New(fmt.Sprintf("Uknown rune: %v", nextRune))
	}
	}
	}
	case STATE_ESCAPING: // the next rune after an escape character
	{
	switch nextRuneType {
	case RUNETOKEN_EOF:
	{
	err = errors.New("EOF found after escape character")
	break SCAN
	}
	case RUNETOKEN_CHAR, RUNETOKEN_SPACE, RUNETOKEN_ESCAPING_QUOTE, RUNETOKEN_NONESCAPING_QUOTE, RUNETOKEN_ESCAPE, RUNETOKEN_COMMENT:
	{
	state = STATE_INWORD
	value = append(value, nextRune)
	}
	default:
	{
	return nil, errors.New(fmt.Sprintf("Uknown rune: %v", nextRune))
	}
	}
	}
	case STATE_ESCAPING_QUOTED: // the next rune after an escape character, in double quotes
	{
	switch nextRuneType {
	case RUNETOKEN_EOF:
	{
	err = errors.New("EOF found after escape character")
	break SCAN
	}
	case RUNETOKEN_CHAR, RUNETOKEN_SPACE, RUNETOKEN_ESCAPING_QUOTE, RUNETOKEN_NONESCAPING_QUOTE, RUNETOKEN_ESCAPE, RUNETOKEN_COMMENT:
	{
	state = STATE_QUOTED_ESCAPING
	value = append(value, nextRune)
	}
	default:
	{
	return nil, errors.New(fmt.Sprintf("Uknown rune: %v", nextRune))
	}
	}
	}
	case STATE_QUOTED_ESCAPING: // in escaping double quotes
	{
	switch nextRuneType {
	case RUNETOKEN_EOF:
	{
	err = errors.New("EOF found when expecting closing quote.")
	break SCAN
	}
	case RUNETOKEN_CHAR, RUNETOKEN_UNKNOWN, RUNETOKEN_SPACE, RUNETOKEN_NONESCAPING_QUOTE, RUNETOKEN_COMMENT:
	{
	value = append(value, nextRune)
	}
	case RUNETOKEN_ESCAPING_QUOTE:
	{
	state = STATE_INWORD
	}
	case RUNETOKEN_ESCAPE:
	{
	state = STATE_ESCAPING_QUOTED
	}
	default:
	{
	return nil, errors.New(fmt.Sprintf("Uknown rune: %v", nextRune))
	}
	}
	}
	case STATE_QUOTED: // in non-escaping single quotes
	{
	switch nextRuneType {
	case RUNETOKEN_EOF:
	{
	err = errors.New("EOF found when expecting closing quote.")
	break SCAN
	}
	case RUNETOKEN_CHAR, RUNETOKEN_UNKNOWN, RUNETOKEN_SPACE, RUNETOKEN_ESCAPING_QUOTE, RUNETOKEN_ESCAPE, RUNETOKEN_COMMENT:
	{
	value = append(value, nextRune)
	}
	case RUNETOKEN_NONESCAPING_QUOTE:
	{
	state = STATE_INWORD
	}
	default:
	{
	return nil, errors.New(fmt.Sprintf("Uknown rune: %v", nextRune))
	}
	}
	}
	case STATE_COMMENT:
	{
	switch nextRuneType {
	case RUNETOKEN_EOF:
	{
	break SCAN
	}
	case RUNETOKEN_CHAR, RUNETOKEN_UNKNOWN, RUNETOKEN_ESCAPING_QUOTE, RUNETOKEN_ESCAPE, RUNETOKEN_COMMENT, RUNETOKEN_NONESCAPING_QUOTE:
	{
	value = append(value, nextRune)
	}
	case RUNETOKEN_SPACE:
	{
	if nextRune == '\n' {
	state = STATE_START
	break SCAN
	} else {
	value = append(value, nextRune)
	}
	}
	default:
	{
	return nil, errors.New(fmt.Sprintf("Uknown rune: %v", nextRune))
	}
	}
	}
	default:
	{
	panic(fmt.Sprintf("Unexpected state: %v", state))
	}
	}
	}
	token := &Token{
	tokenType: tokenType,
	value: string(value)}
	return token, err
	}

	/*
	Return the next token in the stream, and an error value. If there are no more
	tokens available, the error value will be io.EOF.
	*/
	func (t Tokenizer) NextToken() (Token, error) {
	return t.scanStream()
	}

	/*
	Split a string in to a slice of strings, based upon shell-style rules for
	quoting, escaping, and spaces.
	*/
	func Split(s string) ([]string, error) {
	l, err := NewLexer(strings.NewReader(s))
	if err != nil {
	return nil, err
	}
	subStrings := []string{}
	for {
	word, err := l.NextWord()
	if err != nil {
	if err == io.EOF {
	return subStrings, nil
	}
	return subStrings, err
	}
	subStrings = append(subStrings, word)
	}
	return subStrings, nil
	}