Generation

generate functionSat, 12 Nov 2022

Tokenizer

func Tokenizer(s string) []string { words := []string{} var prev rune var word []rune for _, char := range s { if char == ' ' { if prev != ' ' && len(word) > 0 { words = append(words, string(word)) word = []rune{} } } else { word = append(word, char) } prev = char } if len(word) > 0 { words = append(words, string(word)) } return words }

Questions about programming?Chat with your personal AI assistant