use syntax package hash

Author: soypat

src/cmd/compile/internal/syntax/parser_test.go

@@ -13,6 +13,7 @@ import (
 	"path/filepath"
 	"regexp"
 	"runtime"
+	"slices"
 	"strings"
 	"sync"
 	"testing"
@@ -56,7 +57,6 @@ func BenchmarkParseStdLib(b *testing.B) {
 		base *PosBase
 		data []byte
 	}
-	var largestfile *file
 	var files []file
 	goroot := testenv.GOROOT(b)
 	dirs := []string{
@@ -80,18 +80,35 @@ func BenchmarkParseStdLib(b *testing.B) {
 				data: data,
 				base: NewFileBase(filename),
 			})
-			f := &files[len(files)-1]
-			if largestfile == nil || len(f.data) > len(largestfile.data) {
-				largestfile = f
-			}
 		})
 	}
+	slices.SortStableFunc(files, func(a, b file) int {
+		return len(a.data) - len(b.data)
+	})
 	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		var buf bytes.Reader
-		buf.Reset(largestfile.data)
-		Parse(largestfile.base, &buf, nil, nil, 0)
+	const numberOfFiles = 10
+	if len(files) < numberOfFiles*2 {
+		b.Error("too few files matched to run")
 	}
+	b.Run(fmt.Sprintf("longest %d files", numberOfFiles), func(b *testing.B) {
+		var buf bytes.Reader
+		for i := 0; i < b.N; i++ {
+			for _, file := range files[len(files)-numberOfFiles:] {
+				buf.Reset(file.data)
+				Parse(file.base, &buf, nil, nil, 0)
+			}
+		}
+	})
+
+	b.Run(fmt.Sprintf("shortest %d files", numberOfFiles), func(b *testing.B) {
+		var buf bytes.Reader
+		for i := 0; i < b.N; i++ {
+			for _, file := range files[:numberOfFiles] {
+				buf.Reset(file.data)
+				Parse(file.base, &buf, nil, nil, 0)
+			}
+		}
+	})
 }
 
 func TestStdLib(t *testing.T) {

src/cmd/compile/internal/syntax/scanner.go

@@ -381,10 +381,7 @@ func (s *scanner) ident() {
 	// possibly a keyword
 	lit := s.segment()
 	if len(lit) >= 2 {
-		// tok := keywordMap[hash(lit)]
-		// tok := keywords[keywordsIndex(lit)]
-		tok := keywordRuntimeMap[string(lit)]
-		if tok != 0 && tokStrFast(tok) == string(lit) {
+		if tok := keywordMap[hash(lit)]; tok != 0 && tokStrFast(tok) == string(lit) {
 			s.nlsemi = contains(1<<_Break|1<<_Continue|1<<_Fallthrough|1<<_Return, tok)
 			s.tok = tok
 			return
@@ -425,47 +422,15 @@ func hash(s []byte) uint {
 }
 
 var keywordMap [1 << 6]token // size must be power of two
-var keywordRuntimeMap = make(map[string]token)
-var keywords [256]Token
-
-// keywordsIndex maps an identifier to an index in keywords array.
-func keywordsIndex(maybeKeyword []byte) uint8 {
-	if len(maybeKeyword) <= 3 {
-		return maybeKeyword[0]
-	}
-	// This hash was adjusted by hand. Finding the working combinations
-	// for this hash is quite straightforward, even when restricting all
-	// operations to power-of-two multiplications and addition/subtractions
-	// for performance reasons since multiplication of an integer by a power-of-two
-	// can be optimized to a bitshift which is faster on some architectures.
-	//
-	// Here is a list of hashes that also works for current keyword set:
-	// h = v0 + v1*2 + v2*4 + v3*8
-	// h = v0 + v1*4 + v2*8 + v3
-	// h = v0 + v1*2 + (v2+v3)*2
-	// h = v0*4 + v1*2 + v2*2 + v3*2
-	// h = v0*4 + v1*2 + v2*v3
-	v0 := maybeKeyword[0]
-	v1 := maybeKeyword[1]
-	v2 := maybeKeyword[2]
-	v3 := maybeKeyword[3]
-	h := v0 + v1*8 + v2 - v3
-	return h
-}
 
 func init() {
 	// populate keywordMap
 	for tok := _Break; tok <= _Var; tok++ {
-		kws := tok.String()
-		kw := []byte(kws)
-		i := keywordsIndex(kw)
-		h := hash(kw)
-		if keywordMap[h] != 0 || keywords[i] != 0 {
+		h := hash([]byte(tok.String()))
+		if keywordMap[h] != 0 {
 			panic("imperfect hash")
 		}
-		keywords[i] = tok
 		keywordMap[h] = tok
-		keywordRuntimeMap[kws] = tok
 	}
 }
 

src/go/token/token.go

@@ -279,47 +279,31 @@ func (op Token) Precedence() int {
 	return LowestPrec
 }
 
-var keywords [256]Token
-
-func init() {
-	for i := keyword_beg + 1; i < keyword_end; i++ {
-		keywords[keywordsIndex(i.String())] = i
-	}
+// hash is a perfect hash function for keywords.
+// It assumes that s has at least length 2.
+func hash(s string) uint {
+	return (uint(s[0])<<4 ^ uint(s[1]) + uint(len(s))) & uint(len(keywordMap)-1)
 }
 
-// keywordsIndex maps an identifier to an index in keywords array.
-func keywordsIndex(maybeKeyword string) uint8 {
-	if len(maybeKeyword) <= 3 {
-		// If adding a 2 or 3 letter keyword that starts with `i`(if),`f`(for) or `g`(go)
-		// you'd need to add logic to this if statement to differentiate between them.
-		if len(maybeKeyword) == 0 {
-			return 0
+var keywordMap [1 << 6]Token // size must be power of two
+
+func init() {
+	// populate keywordMap
+	for tok := keyword_beg + 1; tok < keyword_end; tok++ {
+		h := hash(tok.String())
+		if keywordMap[h] != 0 {
+			panic("imperfect hash")
 		}
-		return maybeKeyword[0]
+		keywordMap[h] = tok
 	}
-	// This hash was adjusted by hand. Finding the working combinations
-	// for this hash is quite straightforward, even when restricting all
-	// operations to power-of-two multiplications and addition/subtractions
-	// for performance reasons since multiplication of an integer by a power-of-two
-	// can be optimized to a bitshift which is faster on some architectures.
-	//
-	// Here is a list of hashes that also works for current keyword set:
-	// h = v0 + v1*2 + v2*4 + v3*8
-	// h = v0 + v1*4 + v2*8 + v3
-	// h = v0 + v1*2 + (v2+v3)*2
-	// h = v0*4 + v1*2 + v2*2 + v3*2
-	// h = v0*4 + v1*2 + v2*v3
-	v0 := maybeKeyword[0]
-	v1 := maybeKeyword[1]
-	v2 := maybeKeyword[2]
-	v3 := maybeKeyword[3]
-	h := v0 + v1*8 + v2 - v3
-	return h
 }
 
 // Lookup maps an identifier to its keyword token or [IDENT] (if not a keyword).
 func Lookup(ident string) Token {
-	maybeMatch := keywords[keywordsIndex(ident)]
+	if len(ident) < 2 {
+		return IDENT
+	}
+	maybeMatch := keywordMap[hash(ident)]
 	if maybeMatch != 0 && maybeMatch.String() == ident {
 		return maybeMatch
 	}
@@ -350,7 +334,10 @@ func IsExported(name string) bool {
 
 // IsKeyword reports whether name is a Go keyword, such as "func" or "return".
 func IsKeyword(ident string) bool {
-	tok := keywords[keywordsIndex(ident)]
+	if len(ident) < 2 {
+		return false
+	}
+	tok := keywordMap[hash(ident)]
 	return tok != 0 && tok.String() == ident
 }