go#package

@takufukushima

go!package!hierarchy• ast

• build

• doc

• format

• parser

• printer

• scanner

• token

go!package:!Sta+c!analysis

• ast

• build

• doc

• format

• parser

• printer

• scanner

• token

go!package:!U*li*es• ast

• build

• doc

• format

• parser

• printer

• scanner

• token

Sta$c&analysis

Typical(compiler

1. Lexical(analysis((scanning;(tokenizing)

2. Parsing

• Concrete,syntax,tree,(CST,,parse,tree)

3. Seman8c(analysis

• Abstract,syntax,tree,(AST)

4. Code(genera8on

5. Code,op=miza=on

go!package

1. Lexical(analysis((scanning;(tokenizing)

2. Parsing

• Concrete,syntax,tree,(CST,,parse,tree)

3. Seman8c(analysis,(Included,in,the,parser)

• Abstract,syntax,tree,(AST)

4. ___(genera8on

___"genera(on• Documenta+on,from,the,comment

• String,representa+on,of,AST

• Forma8ed,code

• Package,informa+on,for,the,build

Sta$c&analysis

1. token

2. scanner

3. ast

4. parser

token

1. token

2. scanner

3. ast

4. parser

Token

// Token is the set of lexical tokens of the Go programming language.type Token int// The list of tokens.const ( // Special tokens ILLEGAL Token = iota EOF COMMENT

literal_beg // Identifiers and basic type literals // (these tokens stand for classes of literals) IDENT // main INT // 12345 FLOAT // 123.45 IMAG // 123.45i CHAR // 'a' STRING // "abc" literal_end ...

Tokensvar tokens = [...]string{ ILLEGAL: "ILLEGAL",

EOF: "EOF", COMMENT: "COMMENT",

IDENT: "IDENT", INT: "INT", FLOAT: "FLOAT", IMAG: "IMAG", CHAR: "CHAR", STRING: "STRING",

ADD: "+", SUB: "-", MUL: "*", QUO: "/", REM: "%", ...

Tokens'in'BNF

AST

Expression = UnaryExpr | Expression binary_op UnaryExpr .UnaryExpr = PrimaryExpr | unary_op UnaryExpr .

Tokens

binary_op = "||" | "&&" | rel_op | add_op | mul_op .rel_op = "==" | "!=" | "<" | "<=" | ">" | ">=" .add_op = "+" | "-" | "|" | "^" .mul_op = "*" | "/" | "%" | "<<" | ">>" | "&" | "&^" .

unary_op = "+" | "-" | "!" | "^" | "*" | "&" | "<-" .

token!package!provides!basic!data!structures!for!scanner

• Position

• Pos

• File

• FileSet

Position!and!Pos// Position describes an arbitrary source position// including the file, line, and column location.// A Position is valid if the line number is > 0.type Position struct { // filename, if any Filename string // offset, starting at 0 Offset int // line number, starting at 1 Line int // column number, starting at 1 (character count) Column int}// Offset in FileSettype Pos int

File

// A File is a handle for a file belonging to a FileSet.// A File has a name, size, and line offset table.type File struct { set *FileSet // file name as provided to AddFile name string // Pos value range for this file is // [base...base+size] base int // file size as provided to AddFile size int

// lines and infos are protected by set.mutex // lines contains the offset of the first character // for each line (the first entry is always 0) lines []int infos []lineInfo}

FileSet

// A FileSet represents a set of source files.// Methods of file sets are synchronized; multiple// goroutines may invoke them concurrently.type FileSet struct { // protects the file set mutex sync.RWMutex // base offset for the next file base int // list of files in the order added to the set files []*File // cache of last file looked up last *File}

FileSet!methods// AddFile adds a new file with a given filename, base offset,// and file size to the file set s and returns the file.func (s *FileSet) AddFile(filename string, base, size int) *File// Read calls decode to deserialize a file set into s; s must// not be nil.func (s *FileSet) Read(decode func(interface{}) error) error// Write calls encode to serialize the file set s.func (s *FileSet) Write(encode func(interface{}) error) error

scanner

1. token

2. scanner

3. ast

4. parser

Scanner

type Scanner struct { // immutable state file *token.File // source file handle dir string // directory portion of file.Name() src []byte // source err ErrorHandler // error reporting; or nil mode Mode // scanning mode

// scanning state ch rune // current character offset int // character offset rdOffset int // reading offset (position after current character) lineOffset int // current line offset insertSemi bool // insert a semicolon before next newline

// public state - ok to modify ErrorCount int // number of errors encountered}

Scanner!methodsfunc (s *Scanner) Init(file *token.File, src []byte, err ErrorHandler, mode Mode)

// Scan scans the next token and returns the token// position, the token, and its literal string if// applicable. The source end is indicated by// token.EOF.func (s *Scanner) Scan() (pos token.Pos, tok token.Token, lit string)

Mode

type Mode uint

const ( // return comments as COMMENT tokens ScanComments Mode = 1 << iota // do not automatically insert semicolons // - for testing only dontInsertSemis )

ErrorHandler,"Error"and"ErrorList

// An ErrorHandler may be provided to Scanner.Init. If a// syntax error is encountered and a handler was installed,// the handler is called with a position and an error// message. The position points to the beginning of the// offending token.type ErrorHandler func(pos token.Position, msg string)

type Error struct { Pos token.Position Msg string}type ErrorList []*Error

Usage&example&in&scanner_test.go

func ExampleScanner_Scan() { // src is the input that we want to tokenize. src := []byte("cos(x) + 1i*sin(x) // Euler")

// Initialize the scanner. var s scanner.Scanner fset := token.NewFileSet() // positions are relative to fset file := fset.AddFile("", fset.Base(), len(src)) // register input "file" s.Init(file, src, nil /* no error handler */, scanner.ScanComments) // Repeated calls to Scan yield the token sequence found in the input.

for { pos, tok, lit := s.Scan() if tok == token.EOF { break } fmt.Printf("%s\t%s\t%q\n", fset.Position(pos), tok, lit) }}

Result'of'the'usage'example// output:// 1:1 IDENT "cos"// 1:4 ( ""// 1:5 IDENT "x"// 1:6 ) ""// 1:8 + ""// 1:10 IMAG "1i"// 1:12 * ""// 1:13 IDENT "sin"// 1:16 ( ""// 1:17 IDENT "x"// 1:18 ) ""// 1:20 ; "\n"// 1:20 COMMENT "// Euler"

ast

1. token

2. scanner

3. ast

4. parser

Syntax

Read%the%spec:"h$p://golang.org/ref/spec

Interfaces// All node types implement the Node interface.type Node interface { Pos() token.Pos // position of first character belonging to the node End() token.Pos // position of first character immediately after the node}// All expression nodes implement the Expr interface.type Expr interface { Node exprNode()}// All statement nodes implement the Stmt interface.type Stmt interface { Node stmtNode()}// All declaration nodes implement the Decl interface.type Decl interface { Node declNode()}

Interfaces• Node

• Expression

• Statement

• Declara4on

• Comment6and6CommentGroup

• File6and6Package

U"lity'func"onsfunc Walk(v Visitor, node Node)

func Fprint(w io.Writer, fset *token.FileSet, x interface{}, f FieldFilter) (err error)func Print(fset *token.FileSet, x interface{}) error

func FilterDecl(decl Decl, f Filter) boolfunc FilterFile(src *File, f Filter) boolfunc FilterPackage(pkg *Package, f Filter) boolfunc NotNilFilter(_ string, v reflect.Value) boolfunc Inspect(node Node, f func(Node) bool)

func FileExports(src *File) boolfunc IsExported(name string) boolfunc PackageExports(pkg *Package) boolfunc SortImports(fset *token.FileSet, f *File)

Node%methods

For$node$N$(e.g.,$N$=$Comment,$Field,$IdentExpr,$SwitchStmt$and$so$on)

// position of first character belonging to the nodefunc (x *N) End() token.Pos// position of first character immediately after the nodefunc (x *N) Pos() token.Pos

and$for$CommentGroup

// Text returns the text of the comment.func (g *CommentGroup) Text() string

Comment'and'CommentGroup• Node

• Expression

• Statement

• Declara4on

• Comment'and'CommentGroup

• File6and6Package

Comment!and!CommentGroup// A Comment node represents a single //-style or /*-style comment.type Comment struct { // position of "/" starting the comment Slash token.Pos // comment text (excluding '\n' for //-style comments) Text string }

// A CommentGroup represents a sequence of comments// with no other tokens and no empty lines between.//type CommentGroup struct { List []*Comment // len(List) > 0}

Expression*and*types• Node

• Expression

• Statement

• Declara/on

• Comment1and1CommentGroup

• File1and1Package

Filed!and!FieldList// A Field represents a Field declaration list in a struct type,// a method list in an interface type, or a parameter/result// declaration in a signature.type Field struct { Doc *CommentGroup // associated documentation; or nil Names []*Ident // field/method/parameter names; or nil if anonymous field Type Expr // field/method/parameter type Tag *BasicLit // field tag; or nil Comment *CommentGroup // line comments; or nil}// A FieldList represents a list of Fields, enclosed // by parentheses or braces.type FieldList struct { Opening token.Pos // position of opening parenthesis/brace, if any List []*Field // field list; or nil Closing token.Pos // position of closing parenthesis/brace, if any}

Expressionstype ( BadExpr struct { ... } Ident struct { ... } Ellipsis struct { ... } BasicLit struct { ... } FuncLit struct { ... } CompositeLit struct { ... } ParenExpr struct { ... } SelectorExpr struct { ... } IndexExpr struct { ... } SliceExpr struct { ... } TypeAssertExpr struct { ... } CallExpr struct { .... } StarExpr struct { ... } UnaryExpr struct { ... } BinaryExpr struct { ... } KeyValueExpr struct { ... })

Types// The direction of a channel type is indicated by one// of the following constants.type ChanDir int

const ( SEND ChanDir = 1 << iota RECV)

type ( ArrayType struct { ... } StructType struct { ... } FuncType struct { ... } InterfaceType struct { ... } MapType struct { ... } ChanType struct { ... })

Statement• Node

• Expression

• Statement

• Declara1on

• Comment5and5CommentGroup

• File5and5Package

Statementstype ( BadStmt struct { ... } DeclStmt struct { ... } EmptyStmt struct { ... } LabeledStmt struct { ... } ExprStmt struct { ... } SendStmt struct { ... } IncDecStmt struct { ... } AssignStmt struct { ... } GoStmt struct { ... } DeferStmt struct { ... } ReturnStmt struct { ... } BranchStmt struct { ... } BlockStmt struct {... } IfStmt struct { ... } CaseClause struct { ... } SwitchStmt struct { ... } TypeSwitchStmt struct { ... } CommClause struct { ... } SelectStmt struct { ... } ForStmt struct { ... } RangeStmt struct { ... })

Declara'on• Node

• Expression

• Statement

• Declara'on

• Comment2and2CommentGroup

• File2and2Package

Declara'ontype ( // The Spec type stands for any of *ImportSpec, // *ValueSpec, and *TypeSpec. Spec interface { ... } ImportSpec struct { ... } ValueSpec struct { ... } TypeSpec struct { ... })

type ( BadDecl struct { ... } GenDecl struct { ... } // General declaration node FuncDecl struct { ... })

File%and%Package• Node

• Expression

• Statement

• Declara4on

• Comment6and6CommentGroup

• File%and%Package

File!and!Packagetype File struct { Doc *CommentGroup // associated documentation; or nil Package token.Pos // position of "package" keyword Name *Ident // package name Decls []Decl // top-level declarations; or nil Scope *Scope // package scope (this file only) Imports []*ImportSpec // imports in this file Unresolved []*Ident // unresolved identifiers in this file Comments []*CommentGroup // list of all comments in the source file}

type Package struct { Name string // package name Scope *Scope // package scope across all files Imports map[string]*Object // map of package id -> package object Files map[string]*File // Go source files by filename}

parser

1. token

2. scanner

3. ast

4. parser

parser

// The parser structure holds the parser's internal state.type parser struct { file *token.File errors scanner.ErrorList scanner scanner.Scanner ... // Next token pos token.Pos // token position tok token.Token // one token look-ahead lit string // token literal ... // Ordinary identifier scopes pkgScope *ast.Scope // pkgScope.Outer == nil topScope *ast.Scope // top-most scope; may be pkgScope unresolved []*ast.Ident // unresolved identifiers imports []*ast.ImportSpec // list of imports

// Label scopes // (maintained by open/close LabelScope) labelScope *ast.Scope // label scope for current function targetStack [][]*ast.Ident // stack of unresolved labels}

Parsing(func,onsfunc ParseFile(fset *token.FileSet, filename string, src interface{}, mode Mode) (f *ast.File, err error)func ParseDir(fset *token.FileSet, path string, filter func(os.FileInfo) bool, mode Mode) (pkgs map[string]*ast.Package, first error)

// ParseExpr is a convenience function for obtaining// the AST of an expression x.func ParseExpr(x string) (ast.Expr, error)

Mode

type Mode uint

const ( // stop parsing after package clause PackageClauseOnly Mode = 1 << iota // stop parsing after import declarations ImportsOnly // parse comments and add them to AST ParseComments // print a trace of parsed productions Trace // report declaration errors DeclarationErrors // same as AllErrors, for backward-compatibility SpuriousErrors // report all errors (not just the first 10 on different lines) AllErrors = SpuriousErrors)

Usage&example&in&example_test.go

var fset = token.NewFileSet()

var validFiles = []string{ "parser.go", "parser_test.go", "error_test.go", "short_test.go",}

func TestParse(t *testing.T) { for _, filename := range validFiles { _, err := ParseFile(fset, filename, nil, DeclarationErrors) if err != nil { t.Fatalf("ParseFile(%s): %v", filename, err) } }}

Usage&example&in&example_test.go

// This example shows what an AST looks like when printed for debugging.func ExamplePrint() { // src is the input for which we want to print the AST. src := `package mainfunc main() { println("Hello, World!")}` // Create the AST by parsing src. fset := token.NewFileSet() // positions are relative to fset f, err := parser.ParseFile(fset, "", src, 0) if err != nil { panic(err) } // Print the AST. ast.Print(fset, f)}

U"li"es

U"li"es

1. printer

2. doc

3. format

4. build

printer

1. printer

2. doc

3. format

4. build

Fprint

// Fprint "pretty-prints" an AST node to output.// It calls Config.Fprint with default settings.func Fprint(output io.Writer, fset *token.FileSet, node interface{}) error

Config!and!Mode// A Config node controls the output of Fprint.type Config struct { Mode Mode // default: 0 Tabwidth int // default: 8 // default: 0 (all code is indented at least by this much) Indent int}func (cfg *Config) Fprint(output io.Writer, fset *token.FileSet, node interface{}) error

// A Mode value is a set of flags (or 0). They control printing.type Mode uintconst ( // do not use a tabwriter; if set, UseSpaces is ignored RawFormat Mode = 1 << iota // use tabs for indentation independent of UseSpaces TabIndent // use spaces instead of tabs for alignment UseSpaces // emit //line comments to preserve original source positions SourcePos)

CommentedNode

// A CommentedNode bundles an AST node and corresponding comments.// It may be provided as argument to any of the Fprint functions.type CommentedNode struct { // *ast.File, or ast.Expr, ast.Decl, ast.Spec, or ast.Stmt Node interface{} Comments []*ast.CommentGroup}

doc

1. printer

2. doc

3. format

4. build

doc

Types&of&doc

type Func struct { ... } // func foo() { ... }type Note struct { ... } // TODO(tfukushima): ...type Package struct { ... } // package bartype Type struct { ... } // typetype Value struct { ... } // var, const

Package!methodsfunc New(pkg *ast.Package, importPath string, mode Mode) *Packagefunc (p *Package) Filter(f Filter)

type Filter func(string) bool

// Mode values control the operation of New.type Mode int

const ( // extract documentation for all package-level // declarations, not just exported ones AllDecls Mode = 1 << iota

// show all embedded methods, not just the ones of // invisible (unexported) anonymous fields AllMethods)

Every&exported&(capitalized)&name&in&a&program&should&have&a&doc&comment

Python

def _this_is_an_unexported_function(): ...

def this_is_an_exported_function(): """This is a Docstring. Lisp has it, too. """

Gofunc thisIsAnUnexportedFunction() { ... }

// This should have a doc comment.func ThisIsAnExportedFunction() { ... }

Example

// An Example represents an example function found in a source files.type Example struct { Name string // name of the item being exemplified Doc string // example function doc string Code ast.Node Play *ast.File // a whole program version of the example Comments []*ast.CommentGroup Output string // expected output EmptyOutput bool // expect empty output Order int // original source code order}// Examples returns the examples found in the files, sorted by// Name field.func Examples(files ...*ast.File) []*Example

Synopsis

// Synopsis returns a cleaned version of the first sentence in s.func Synopsis(s string) stringvar IllegalPrefixes = []string{ "copyright", "all rights", "author",}

ToHTML!and!ToText// ToHTML converts comment text to formatted HTML.func ToHTML(w io.Writer, text string, words map[string]string)// ToText prepares comment text for presentation in textual// output.func ToText(w io.Writer, text string, indent, preIndent string, width int)

format

1. printer

2. doc

3. format

4. build

Node!and!Source// Node formats node in canonical gofmt style and writes the result// to dst.func Node(dst io.Writer, fset *token.FileSet, node interface{}) error

// Source formats src in canonical gofmt style and returns the result// or an (I/O or syntax) error.func Source(src []byte) ([]byte, error)

Usage&Example:&My&go fmtimport ( "go/format" "io/ioutil" "os")

const fileName = "hello.go"

func formatFile(fileName string) { src, err := ioutil.ReadFile(fileName) if err != nil { panic(err) } res, err := format.Source(src) if err != nil { panic(err) } ioutil.WriteFile(fileName, res, os.ModeType)}

func main() { formatFile(fileName)}

Resulf'of'the'usage'example/Users/tfukushima/go/src/github.com/tfukushima/mygofmt% cat hello.go package mainimport ("fmt""github.com/tfukushima/string")funcmain(){fmt.Println(string.Reverse("Hello, new gopher!"))}/Users/tfukushima/go/src/github.com/tfukushima/mygofmt% go run mygofmt.go/Users/tfukushima/go/src/github.com/tfukushima/mygofmt% cat hello.gopackage main

import ( "fmt" "github.com/tfukushima/string")

func main() { fmt.Println(string.Reverse("Hello, new gopher!"))}

build

1. printer

2. doc

3. format

4. build

Go#PathGOPATH=/home/user/gocode

/home/user/gocode/ src/ foo/ bar/ (go code in package bar) x.go quux/ (go code in package main) y.go bin/ quux (installed command) pkg/ linux_amd64/ foo/ bar.a (installed package object)

Build&constraints

To#build#only#on#the#specific#OS#and/or#arch:

// +build linux,386 darwin,!cgo(linux AND 386) OR (darwin AND (NOT cgo))

// +build linux darwin// +build 386(linux OR darwin) AND 386

To#keep#a#file#from#being#considered#for#the#build:

// +build ignore// +build unsatisfied_word

Build&constraints

During'a'par*cular'build,'the'following'words'are'sa*sfied:

- the target operating system, as spelled by runtime.GOOS- the target architecture, as spelled by runtime.GOARCH- the compiler being used, either "gc" or "gccgo"- "cgo", if ctxt.CgoEnabled is true- "go1.1", from Go version 1.1 onward- "go1.2", from Go version 1.2 onward- "go1.3", from Go version 1.3 onward- any additional words listed in ctxt.BuildTags

Build&constraints&by&file&names

- source_windows_arm64.go- source_windows_arm64_test.go

GOOS=windowsGOARCH=arm64

Package

// A Package describes the Go package found in a directory.type Package struct { Dir string // directory containing package sources Name string // package name Doc string // documentation synopsis ImportPath string // import path of package ("" if unknown) Root string // root of Go tree where this package lives SrcRoot string // package source root directory ("" if unknown) PkgRoot string // package install root directory ("" if unknown) BinDir string // command install directory ("" if unknown) Goroot bool // package found in Go root PkgObj string // installed .a file AllTags []string // tags that can influence file selection in this directory ConflictDir string // this directory shadows Dir in $GOPATH ...}// IsCommand reports whether the package is considered a command to be installed// (not just a library). Packages named "main" are treated as commands.func (p *Package) IsCommand() bool { return p.Name == "main"}

Context

// A Context specifies the supporting context for a build.type Context struct { GOARCH string // target architecture GOOS string // target operating system GOROOT string // Go root GOPATH string // Go path CgoEnabled bool // whether cgo can be used UseAllFiles bool // use files regardless of +build lines, file names Compiler string // compiler to assume when computing target paths ...}

// Default is the default Context for builds. It uses the GOARCH, GOOS,// GOROOT, and GOPATH environment variables if set, or else the compiled// code's GOARCH, GOOS, and GOROOT.var Default Context = defaultContext()

Context!methodsfunc (ctxt *Context) Import(path string, srcDir string, mode ImportMode) (*Package, error)func (ctxt *Context) ImportDir(dir string, mode ImportMode) (*Package, error)func (ctxt *Context) MatchFile(dir, name string) (match bool, err error)

// SrcDirs returns a list of package source root directories. It// draws from the current Go root and Go path but omits directories// that do not exist.func (ctxt *Context) SrcDirs() []string

ImportMode

type ImportMode uint

const ( // If FindOnly is set, Import stops after locating the directory // that should contain the sources for a package. It does not // read any files in the directory. FindOnly ImportMode = 1 << iota

// If AllowBinary is set, Import can be satisfied by a compiled // package object without corresponding sources. AllowBinary)

NoGoError

// NoGoError is the error used by Import to describe a directory// containing no buildable Go source files. (It may still contain// test files, files hidden by build tags, and so on.)type NoGoError struct { Dir string}

func (e *NoGoError) Error() string { return "no buildable Go source files in " + e.Dir}

Import!and!ImportDir// Import is shorthand for Default.Import.func Import(path, srcDir string, mode ImportMode) (*Package, error)

// Import is shorthand for Default.ImportDir.func ImportDir(dir string, mode ImportMode) (*Package, error)

Misc// ToolDir is the directory containing build tools.var ToolDir = filepath.Join(runtime.GOROOT(), "pkg/tool/"+runtime.GOOS+"_"+runtime.GOARCH)

// ArchChar returns the architecture character for the given goarch.// For example, ArchChar("amd64") returns "6".func ArchChar(goarch string) (string, error)

// IsLocalImport reports whether the import path is// a local import path, like ".", "..", "./foo", or "../foo".func IsLocalImport(path string) bool

Wrap%up

go!package!hierarchy• ast

• build

• doc

• format

• parser

• printer

• scanner

• token

go!package:!Sta+c!analysis

• ast

• build

• doc

• format

• parser

• printer

• scanner

• token

go!package:!U*li*es• ast

• build

• doc

• format

• parser

• printer

• scanner

• token

The$end$of$slides.$Any$ques1on?