Rewrite PO headers parsing and handling. Implement correct GNU gettext headers format. Fix tests. Fixes #10

gotext_test.go

@@ -3,6 +3,7 @@ package gotext
 import (
 	"os"
 	"path"
+	"sync"
 	"testing"
 )
 
@@ -32,10 +33,12 @@ func TestGettersSetters(t *testing.T) {
 func TestPackageFunctions(t *testing.T) {
 	// Set PO content
 	str := `
-# msgid ""
+msgid   ""
-# msgstr ""
+msgstr  "Project-Id-Version: %s\n"
+        "Report-Msgid-Bugs-To: %s\n"
+        
 # Initial comment
-# Headers below
+# More Headers below
 "Language: en\n"
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"
@@ -55,14 +58,6 @@ msgstr[0] "This one is the singular: %s"
 msgstr[1] "This one is the plural: %s"
 msgstr[2] "And this is the second plural form: %s"
 
-msgid "This one has invalid syntax translations"
-msgid_plural "Plural index"
-msgstr[abc] "Wrong index"
-msgstr[1 "Forgot to close brackets"
-msgstr[0] "Badly formatted string'
-
-msgid "Invalid formatted id[] with no translations
-
 msgctxt "Ctx"
 msgid "One with var: %s"
 msgid_plural "Several with vars: %s"
@@ -149,8 +144,8 @@ msgstr[1] ""
 func TestUntranslated(t *testing.T) {
 	// Set PO content
 	str := `
-# msgid ""
+msgid ""
-# msgstr ""
+msgstr ""
 # Initial comment
 # Headers below
 "Language: en\n"
@@ -258,22 +253,29 @@ msgstr[2] "And this is the second plural form: %s"
 		t.Fatalf("Can't write to test file: %s", err.Error())
 	}
 
-	// Init sync channels
+	var wg sync.WaitGroup
-	c1 := make(chan bool)
-	c2 := make(chan bool)
 
 	for i := 0; i < 100; i++ {
+		wg.Add(1)
 		// Test translations
-		go func(done chan bool) {
+		go func() {
-			Get("My text")
+			defer wg.Done()
-			done <- true
-		}(c1)
-
-		go func(done chan bool) {
-			Get("My text")
-			done <- true
-		}(c2)
 
 			Get("My text")
+			GetN("One with var: %s", "Several with vars: %s", 0, "test")
+		}()
+
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+
+			Get("My text")
+			GetN("One with var: %s", "Several with vars: %s", 1, "test")
+		}()
+
+		Get("My text")
+		GetN("One with var: %s", "Several with vars: %s", 2, "test")
 	}
+
+	wg.Wait()
 }

locale_test.go

@@ -9,8 +9,8 @@ import (
 func TestLocale(t *testing.T) {
 	// Set PO content
 	str := `
-# msgid ""
+msgid ""
-# msgstr ""
+msgstr ""
 # Initial comment
 # Headers below
 "Language: en\n"
@@ -38,8 +38,6 @@ msgstr[abc] "Wrong index"
 msgstr[1 "Forgot to close brackets"
 msgstr[0] "Badly formatted string'
 
-msgid "Invalid formatted id[] with no translations
-
 msgctxt "Ctx"
 msgid "One with var: %s"
 msgid_plural "Several with vars: %s"

po.go

@@ -79,8 +79,8 @@ Example:
 
 */
 type Po struct {
-	// Headers
+	// Headers storage
-	RawHeaders string
+	Headers textproto.MIMEHeader
 
 	// Language header
 	Language string
@@ -140,7 +140,6 @@ func (po *Po) ParseFile(f string) {
 func (po *Po) Parse(str string) {
 	// Lock while parsing
 	po.Lock()
-	defer po.Unlock()
 
 	// Init storage
 	if po.translations == nil {
@@ -195,7 +194,7 @@ func (po *Po) Parse(str string) {
 
 		// Multi line strings and headers
 		if strings.HasPrefix(l, "\"") && strings.HasSuffix(l, "\"") {
-			state = po.parseString(l, state)
+			po.parseString(l, state)
 			continue
 		}
 	}
@@ -203,6 +202,9 @@ func (po *Po) Parse(str string) {
 	// Save last translation buffer.
 	po.saveBuffer()
 
+	// Unlock to parse headers
+	po.Unlock()
+
 	// Parse headers
 	po.parseHeaders()
 }
@@ -210,8 +212,6 @@ func (po *Po) Parse(str string) {
 // saveBuffer takes the context and translation buffers
 // and saves it on the translations collection
 func (po *Po) saveBuffer() {
-	// If we have something to save...
-	if po.trBuffer.id != "" {
 	// With no context...
 	if po.ctxBuffer == "" {
 		po.translations[po.trBuffer.id] = po.trBuffer
@@ -221,14 +221,17 @@ func (po *Po) saveBuffer() {
 			po.contexts[po.ctxBuffer] = make(map[string]*translation)
 		}
 		po.contexts[po.ctxBuffer][po.trBuffer.id] = po.trBuffer
-		}
 
-		// Flush buffer
+		// Cleanup current context buffer if needed
-		po.trBuffer = newTranslation()
+		if po.trBuffer.id != "" {
 			po.ctxBuffer = ""
 		}
 	}
 
+	// Flush translation buffer
+	po.trBuffer = newTranslation()
+}
+
 // parseContext takes a line starting with "msgctxt",
 // saves the current translation buffer and creates a new context.
 func (po *Po) parseContext(l string) {
@@ -286,70 +289,72 @@ func (po *Po) parseMessage(l string) {
 
 // parseString takes a well formatted string without prefix
 // and creates headers or attach multi-line strings when corresponding
-func (po *Po) parseString(l string, state parseState) parseState {
+func (po *Po) parseString(l string, state parseState) {
+	clean, _ := strconv.Unquote(l)
+
 	switch state {
 	case msgStr:
-		// Check for multiline from previously set msgid
-		if po.trBuffer.id != "" {
 		// Append to last translation found
-			uq, _ := strconv.Unquote(l)
+		po.trBuffer.trs[len(po.trBuffer.trs)-1] += clean
-			po.trBuffer.trs[len(po.trBuffer.trs)-1] += uq
 
-		}
 	case msgID:
 		// Multiline msgid - Append to current id
-		uq, _ := strconv.Unquote(l)
+		po.trBuffer.id += clean
-		po.trBuffer.id += uq
+
 	case msgIDPlural:
 		// Multiline msgid - Append to current id
-		uq, _ := strconv.Unquote(l)
+		po.trBuffer.pluralID += clean
-		po.trBuffer.pluralID += uq
+
 	case msgCtxt:
 		// Multiline context - Append to current context
-		ctxt, _ := strconv.Unquote(l)
+		po.ctxBuffer += clean
-		po.ctxBuffer += ctxt
-	default:
-		// Otherwise is a header
-		h, _ := strconv.Unquote(strings.TrimSpace(l))
-		po.RawHeaders += h
-		return head
-	}
 
-	return state
+	}
 }
 
 // isValidLine checks for line prefixes to detect valid syntax.
 func (po *Po) isValidLine(l string) bool {
-	// Skip empty lines
-	if l == "" {
-		return false
-	}
-
 	// Check prefix
-	if !strings.HasPrefix(l, "\"") && !strings.HasPrefix(l, "msgctxt") && !strings.HasPrefix(l, "msgid") && !strings.HasPrefix(l, "msgid_plural") && !strings.HasPrefix(l, "msgstr") {
+	valid := []string{
-		return false
+		"\"",
+		"msgctxt",
+		"msgid",
+		"msgid_plural",
+		"msgstr",
 	}
 
+	for _, v := range valid {
+		if strings.HasPrefix(l, v) {
 			return true
 		}
+	}
+
+	return false
+}
 
 // parseHeaders retrieves data from previously parsed headers
 func (po *Po) parseHeaders() {
 	// Make sure we end with 2 carriage returns.
-	po.RawHeaders += "\n\n"
+	raw := po.Get("") + "\n\n"
 
 	// Read
-	reader := bufio.NewReader(strings.NewReader(po.RawHeaders))
+	reader := bufio.NewReader(strings.NewReader(raw))
 	tp := textproto.NewReader(reader)
 
-	mimeHeader, err := tp.ReadMIMEHeader()
+	var err error
+
+	// Sync Headers write.
+	po.Lock()
+	defer po.Unlock()
+
+	po.Headers, err = tp.ReadMIMEHeader()
 	if err != nil {
 		return
 	}
 
 	// Get/save needed headers
-	po.Language = mimeHeader.Get("Language")
+	po.Language = po.Headers.Get("Language")
-	po.PluralForms = mimeHeader.Get("Plural-Forms")
+	po.PluralForms = po.Headers.Get("Plural-Forms")
 
 	// Parse Plural-Forms formula
 	if po.PluralForms == "" {
@@ -422,12 +427,12 @@ func (po *Po) Get(str string, vars ...interface{}) string {
 
 	if po.translations != nil {
 		if _, ok := po.translations[str]; ok {
-			return fmt.Sprintf(po.translations[str].get(), vars...)
+			return po.printf(po.translations[str].get(), vars...)
 		}
 	}
 
 	// Return the same we received by default
-	return fmt.Sprintf(str, vars...)
+	return po.printf(str, vars...)
 }
 
 // GetN retrieves the (N)th plural form of translation for the given string.
@@ -439,15 +444,14 @@ func (po *Po) GetN(str, plural string, n int, vars ...interface{}) string {
 
 	if po.translations != nil {
 		if _, ok := po.translations[str]; ok {
-			return fmt.Sprintf(po.translations[str].getN(po.pluralForm(n)), vars...)
+			return po.printf(po.translations[str].getN(po.pluralForm(n)), vars...)
 		}
 	}
 
 	if n == 1 {
-		return fmt.Sprintf(str, vars...)
+		return po.printf(str, vars...)
 	}
-
+	return po.printf(plural, vars...)
-	return fmt.Sprintf(plural, vars...)
 }
 
 // GetC retrieves the corresponding translation for a given string in the given context.
@@ -461,14 +465,14 @@ func (po *Po) GetC(str, ctx string, vars ...interface{}) string {
 		if _, ok := po.contexts[ctx]; ok {
 			if po.contexts[ctx] != nil {
 				if _, ok := po.contexts[ctx][str]; ok {
-					return fmt.Sprintf(po.contexts[ctx][str].get(), vars...)
+					return po.printf(po.contexts[ctx][str].get(), vars...)
 				}
 			}
 		}
 	}
 
 	// Return the string we received by default
-	return fmt.Sprintf(str, vars...)
+	return po.printf(str, vars...)
 }
 
 // GetNC retrieves the (N)th plural form of translation for the given string in the given context.
@@ -482,14 +486,23 @@ func (po *Po) GetNC(str, plural string, n int, ctx string, vars ...interface{})
 		if _, ok := po.contexts[ctx]; ok {
 			if po.contexts[ctx] != nil {
 				if _, ok := po.contexts[ctx][str]; ok {
-					return fmt.Sprintf(po.contexts[ctx][str].getN(po.pluralForm(n)), vars...)
+					return po.printf(po.contexts[ctx][str].getN(po.pluralForm(n)), vars...)
 				}
 			}
 		}
 	}
 
 	if n == 1 {
+		return po.printf(str, vars...)
+	}
+	return po.printf(plural, vars...)
+}
+
+// printf applies text formatting only when needed to parse variables.
+func (po *Po) printf(str string, vars ...interface{}) string {
+	if len(vars) > 0 {
 		return fmt.Sprintf(str, vars...)
 	}
-	return fmt.Sprintf(plural, vars...)
+
+	return str
 }

po_test.go

@@ -9,6 +9,9 @@ import (
 func TestPo(t *testing.T) {
 	// Set PO content
 	str := `
+msgid   ""
+msgstr  ""
+
 # Initial comment
 # Headers below
 "Language: en\n"
@@ -48,14 +51,6 @@ msgstr[0] "This one is the singular: %s"
 msgstr[1] "This one is the plural: %s"
 msgstr[2] "And this is the second plural form: %s"
 
-msgid "This one has invalid syntax translations"
-msgid_plural "Plural index"
-msgstr[abc] "Wrong index"
-msgstr[1 "Forgot to close brackets"
-msgstr[0] "Badly formatted string'
-
-msgid "Invalid formatted id[] with no translations
-
 msgctxt "Ctx"
 msgid "One with var: %s"
 msgid_plural "Several with vars: %s"
@@ -75,11 +70,11 @@ msgstr ""
 msgid "Empty plural form singular"
 msgid_plural "Empty plural form"
 msgstr[0] "Singular translated"
-msgstr[1] "
+msgstr[1] ""
 
 msgid "More"
 msgstr "More translation"
-"
+
 	`
 
 	// Write PO content to file
@@ -152,17 +147,6 @@ msgstr "More translation"
 		t.Errorf("Expected 'This are tests' but got '%s'", tr)
 	}
 
-	// Test syntax error parsed translations
-	tr = po.Get("This one has invalid syntax translations")
-	if tr != "This one has invalid syntax translations" {
-		t.Errorf("Expected 'This one has invalid syntax translations' but got '%s'", tr)
-	}
-
-	tr = po.GetN("This one has invalid syntax translations", "This are tests", 4)
-	if tr != "Plural index" {
-		t.Errorf("Expected 'Plural index' but got '%s'", tr)
-	}
-
 	// Test context translations
 	v = "Test"
 	tr = po.GetC("One with var: %s", "Ctx", v)
@@ -214,9 +198,42 @@ msgstr "More translation"
 	}
 }
 
+func TestPlural(t *testing.T) {
+	// Set PO content
+	str := `
+msgid   ""
+msgstr  ""
+"Plural-Forms: nplurals=2; plural=(n != 1);\n"
+
+msgid "Singular: %s"
+msgid_plural "Plural: %s"
+msgstr[0] "TR Singular: %s"
+msgstr[1] "TR Plural: %s"
+msgstr[2] "TR Plural 2: %s"
+
+	
+`
+	// Create po object
+	po := new(Po)
+	po.Parse(str)
+
+	v := "Var"
+	tr := po.GetN("Singular: %s", "Plural: %s", 2, v)
+	if tr != "TR Plural: Var" {
+		t.Errorf("Expected 'TR Plural: Var' but got '%s'", tr)
+	}
+
+	tr = po.GetN("Singular: %s", "Plural: %s", 1, v)
+	if tr != "TR Singular: Var" {
+		t.Errorf("Expected 'TR Singular: Var' but got '%s'", tr)
+	}
+}
+
 func TestPoHeaders(t *testing.T) {
 	// Set PO content
 	str := `
+msgid   ""
+msgstr  ""
 # Initial comment
 # Headers below
 "Language: en\n"
@@ -246,9 +263,30 @@ msgstr "Translated example"
 	}
 }
 
+func TestMissingPoHeadersSupport(t *testing.T) {
+	// Set PO content
+	str := `
+msgid "Example"
+msgstr "Translated example"
+	`
+
+	// Create po object
+	po := new(Po)
+
+	// Parse
+	po.Parse(str)
+
+	// Check translation expected
+	if po.Get("Example") != "Translated example" {
+		t.Errorf("Expected 'Translated example' but got '%s'", po.Get("Example"))
+	}
+}
+
 func TestPluralFormsSingle(t *testing.T) {
 	// Single form
 	str := `
+msgid   ""
+msgstr  ""
 "Plural-Forms: nplurals=1; plural=0;"
 
 # Some comment
@@ -292,6 +330,8 @@ msgstr[3] "Plural form 3"
 func TestPluralForms2(t *testing.T) {
 	// 2 forms
 	str := `
+msgid   ""
+msgstr  ""
 "Plural-Forms: nplurals=2; plural=n != 1;"
 
 # Some comment
@@ -331,6 +371,8 @@ msgstr[3] "Plural form 3"
 func TestPluralForms3(t *testing.T) {
 	// 3 forms
 	str := `
+msgid   ""
+msgstr  ""
 "Plural-Forms: nplurals=3; plural=n%10==1 && n%100!=11 ? 0 : n != 0 ? 1 : 2;"
 
 # Some comment
@@ -378,6 +420,8 @@ msgstr[3] "Plural form 3"
 func TestPluralFormsSpecial(t *testing.T) {
 	// 3 forms special
 	str := `
+msgid   ""
+msgstr  ""
 "Plural-Forms: nplurals=3;"
 "plural=(n==1) ? 0 : (n>=2 && n<=4) ? 1 : 2;"