From 3beb4001424a34a36d130e1a1db84e85326b2c76 Mon Sep 17 00:00:00 2001 From: Cristiano Lopes Date: Thu, 4 Sep 2014 18:39:29 +0100 Subject: [PATCH 1/4] added translate-context extraction capabilities (filters not parsed yet) --- lib/extract.js | 46 +++++++++-- test/extract.coffee | 81 ++++++++++++++++++- test/fixtures/filter.html | 2 + test/fixtures/homonyms.html | 6 ++ test/fixtures/homonyms.js | 8 ++ test/fixtures/homonyms_corrupt.js | 8 ++ .../homonyms_corrupt_with_msgctxt_first.html | 6 ++ ...omonyms_corrupt_without_msgctxt_first.html | 6 ++ test/fixtures/multifilter.html | 2 + 9 files changed, 155 insertions(+), 10 deletions(-) create mode 100644 test/fixtures/homonyms.html create mode 100644 test/fixtures/homonyms.js create mode 100644 test/fixtures/homonyms_corrupt.js create mode 100644 test/fixtures/homonyms_corrupt_with_msgctxt_first.html create mode 100644 test/fixtures/homonyms_corrupt_without_msgctxt_first.html diff --git a/lib/extract.js b/lib/extract.js index a5f26f4..7728b93 100644 --- a/lib/extract.js +++ b/lib/extract.js @@ -70,15 +70,35 @@ var Extractor = (function () { Extractor.mkAttrRegex = mkAttrRegex; - Extractor.prototype.addString = function (file, string, plural, extractedComment) { + Extractor.prototype.addString = function (file, string, plural, extractedComment, translateContext) { string = string.trim(); + if (translateContext){ + translateContext = translateContext.trim(); + } - if (!this.strings[string]) { - this.strings[string] = new Po.Item(); + if (_.isEmpty(this.strings[string])){ + this.strings[string] = [new Po.Item()]; + } else { + var errorMessage = 'Found, for the same msgid '; + errorMessage += string; + errorMessage += ', translations with and without msgctxt'; + if (translateContext){ + if (_.all(this.strings[string], 'msgctxt')){ + this.strings[string].push(new Po.Item()); + } else { + throw new Error(errorMessage); + } + } else { + if (_.any(this.strings[string], 'msgctxt')){ + throw new Error(errorMessage); + } + } } - var item = this.strings[string]; + + var item = _.last(this.strings[string]); item.msgid = string; + item.msgctxt = translateContext; if (item.references.indexOf(file) < 0) { item.references.push(file); } @@ -150,11 +170,18 @@ var Extractor = (function () { var singular; var plural; var extractedComments = []; + var translateContext; if (isGettext(node) || isGetString(node)) { str = getJSExpression(node.arguments[0]); + if (node.arguments.length === 3) { + translateContext = getJSExpression(node.arguments[2]); + } } else if (isGetPlural(node)) { singular = getJSExpression(node.arguments[1]); plural = getJSExpression(node.arguments[2]); + if (node.arguments.length === 5) { + translateContext = getJSExpression(node.arguments[4]); + } } if (str || singular) { if (parentComment) { @@ -165,9 +192,9 @@ var Extractor = (function () { }); } if (str) { - self.addString(filename, str, plural, extractedComments); + self.addString(filename, str, plural, extractedComments, translateContext); } else if (singular) { - self.addString(filename, singular, plural, extractedComments); + self.addString(filename, singular, plural, extractedComments, translateContext); } } }); @@ -189,6 +216,7 @@ var Extractor = (function () { var str; var plural; var extractedComment; + var translateContext; node = $(n); for (var attr in node.attr()) { @@ -196,7 +224,8 @@ var Extractor = (function () { str = node.html(); plural = node.attr('translate-plural'); extractedComment = node.attr('translate-comment'); - self.addString(filename, str, plural, extractedComment); + translateContext = node.attr('translate-context'); + self.addString(filename, str, plural, extractedComment, translateContext); } else if (matches = noDelimRegex.exec(node.attr(attr))) { str = matches[2].replace(/\\\'/g, '\''); self.addString(filename, str); @@ -206,6 +235,7 @@ var Extractor = (function () { if (typeof node.attr('data-translate') !== 'undefined') { str = node.html(); plural = node.attr('data-translate-plural'); + translateContext = node.attr('data-translate-context'); self.addString(filename, str, plural); } }); @@ -241,7 +271,7 @@ var Extractor = (function () { }; for (var key in this.strings) { - catalog.items.push(this.strings[key]); + catalog.items.push.apply(catalog.items, this.strings[key]); } catalog.items.sort(function (a, b) { diff --git a/test/extract.coffee b/test/extract.coffee index 100ad60..e54418a 100644 --- a/test/extract.coffee +++ b/test/extract.coffee @@ -133,7 +133,7 @@ describe 'Extract', -> ] catalog = testExtract(files) - assert.equal(catalog.items.length, 2) + assert.equal(catalog.items.length, 4) assert.equal(catalog.items[0].msgid, 'Hello') assert.equal(catalog.items[0].msgstr, '') assert.equal(catalog.items[0].references.length, 1) @@ -143,6 +143,16 @@ describe 'Extract', -> assert.equal(catalog.items[1].msgstr, '') assert.equal(catalog.items[1].references.length, 1) assert.equal(catalog.items[1].references[0], 'test/fixtures/filter.html') + assert.equal(catalog.items[2].msgid, 'Bird') + assert.equal(catalog.items[2].msgstr, '') + assert.equal(catalog.items[2].msgctxt, 'cage') + assert.equal(catalog.items[2].references.length, 1) + assert.equal(catalog.items[2].references[0], 'test/fixtures/filter.html') + assert.equal(catalog.items[3].msgid, 'Bird') + assert.equal(catalog.items[3].msgstr, '') + assert.equal(catalog.items[3].msgctxt, 'free') + assert.equal(catalog.items[3].references.length, 1) + assert.equal(catalog.items[2].references[0], 'test/fixtures/filter.html') it 'Extracts concatenated filter strings', -> files = [ @@ -150,7 +160,7 @@ describe 'Extract', -> ] catalog = testExtract(files) - assert.equal(catalog.items.length, 2) + assert.equal(catalog.items.length, 4) assert.equal(catalog.items[0].msgid, 'Hello') assert.equal(catalog.items[0].msgstr, '') assert.equal(catalog.items[0].references.length, 1) @@ -160,6 +170,16 @@ describe 'Extract', -> assert.equal(catalog.items[1].msgstr, '') assert.equal(catalog.items[1].references.length, 1) assert.equal(catalog.items[1].references[0], 'test/fixtures/multifilter.html') + assert.equal(catalog.items[2].msgid, 'Bird') + assert.equal(catalog.items[2].msgstr, '') + assert.equal(catalog.items[2].msgctxt, 'cage') + assert.equal(catalog.items[2].references.length, 1) + assert.equal(catalog.items[2].references[0], 'test/fixtures/multifilter.html') + assert.equal(catalog.items[3].msgid, 'Bird') + assert.equal(catalog.items[3].msgstr, '') + assert.equal(catalog.items[3].msgctxt, 'free') + assert.equal(catalog.items[3].references.length, 1) + assert.equal(catalog.items[2].references[0], 'test/fixtures/multifilter.html') it 'Extracts flagged strings from JavaScript source', -> files = [ @@ -476,3 +496,60 @@ describe 'Extract', -> assert.equal(catalog.items[0].msgstr, '') assert.equal(catalog.items[0].references.length, 1) assert.equal(catalog.items[0].references[0], 'test/fixtures/tapestry.tml') + + it 'Can extract context aware translations from html', () -> + files = [ + 'test/fixtures/homonyms.html' + ] + catalog = testExtract(files) + + assert.equal(catalog.items.length, 2) + assert.equal(catalog.items[0].msgid, 'Hello!') + assert.equal(catalog.items[0].msgstr, '') + assert.equal(catalog.items[0].msgctxt, 'male') + assert.equal(catalog.items[0].references.length, 1) + assert.equal(catalog.items[0].references[0], 'test/fixtures/homonyms.html') + assert.equal(catalog.items[1].msgid, 'Hello!') + assert.equal(catalog.items[1].msgstr, '') + assert.equal(catalog.items[1].msgctxt, 'female') + assert.equal(catalog.items[1].references.length, 1) + assert.equal(catalog.items[1].references[0], 'test/fixtures/homonyms.html') + + it 'Can extract context aware translations from javascript', () -> + files = [ + 'test/fixtures/homonyms.js' + ] + catalog = testExtract(files) + + assert.equal(catalog.items.length, 4) + assert.equal(catalog.items[0].msgid, 'Bird') + assert.equal(catalog.items[0].msgid_plural, 'Birds') + assert.equal(catalog.items[0].msgstr.length, 2) + assert.equal(catalog.items[0].msgctxt, 'cage') + assert.equal(catalog.items[0].references.length, 1) + assert.equal(catalog.items[0].references[0], 'test/fixtures/homonyms.js') + assert.equal(catalog.items[1].msgid, 'Bird') + assert.equal(catalog.items[1].msgid_plural, 'Birds') + assert.equal(catalog.items[1].msgstr.length, 2) + assert.equal(catalog.items[1].msgctxt, 'free') + assert.equal(catalog.items[1].references.length, 1) + assert.equal(catalog.items[1].references[0], 'test/fixtures/homonyms.js') + assert.equal(catalog.items[2].msgid, 'Hello!') + assert.equal(catalog.items[2].msgstr, '') + assert.equal(catalog.items[2].msgctxt, 'male') + assert.equal(catalog.items[2].references.length, 1) + assert.equal(catalog.items[2].references[0], 'test/fixtures/homonyms.js') + assert.equal(catalog.items[3].msgid, 'Hello!') + assert.equal(catalog.items[3].msgstr, '') + assert.equal(catalog.items[3].msgctxt, 'female') + assert.equal(catalog.items[3].references.length, 1) + assert.equal(catalog.items[3].references[0], 'test/fixtures/homonyms.js') + + it 'Warns for msgid with and without msgctxt in the same code base', -> + files = [ + 'test/fixtures/homonyms_corrupt_with_msgctxt_first.html', + 'test/fixtures/homonyms_corrupt_without_msgctxt_first.html', + 'test/fixtures/homonyms_corrupt.js', + ] + assert.throws -> + testExtract(files) diff --git a/test/fixtures/filter.html b/test/fixtures/filter.html index 08fae33..8354868 100644 --- a/test/fixtures/filter.html +++ b/test/fixtures/filter.html @@ -2,5 +2,7 @@ {{'Second'|translate}} + {{'Bird'|translate:'cage'}} + {{'Bird'|translate:'free'}} diff --git a/test/fixtures/homonyms.html b/test/fixtures/homonyms.html new file mode 100644 index 0000000..bbba3f6 --- /dev/null +++ b/test/fixtures/homonyms.html @@ -0,0 +1,6 @@ + + +

Hello!

+

Hello!

+ + diff --git a/test/fixtures/homonyms.js b/test/fixtures/homonyms.js new file mode 100644 index 0000000..d82d939 --- /dev/null +++ b/test/fixtures/homonyms.js @@ -0,0 +1,8 @@ +angular.module("myApp").controller("helloController", function (gettext) { + gettext(); // Should be ignored. + var myStringContext1 = gettextCatalog.getString("Hello!", null, "male"); + var myStringContext2 = gettextCatalog.getString("Hello!", null, "female"); + var myString2Context1 = gettextCatalog.getPlural(3, "Bird", "Birds", null, "cage"); + var myString2Context2 = gettextCatalog.getPlural(3, "Bird", "Birds", null, "free"); + +}); diff --git a/test/fixtures/homonyms_corrupt.js b/test/fixtures/homonyms_corrupt.js new file mode 100644 index 0000000..ef07585 --- /dev/null +++ b/test/fixtures/homonyms_corrupt.js @@ -0,0 +1,8 @@ +angular.module("myApp").controller("helloController", function (gettext) { + gettext(); // Should be ignored. + var myString = gettextCatalog.getString("Hello!"); + var myStringContext2 = gettextCatalog.getString("Hello!", null, "female"); + var myString2Context1 = gettextCatalog.getPlural(3, "Bird", "Birds", null, "cage"); + var myString2Context2 = gettextCatalog.getPlural(3, "Bird", "Birds", null, "free"); + +}); diff --git a/test/fixtures/homonyms_corrupt_with_msgctxt_first.html b/test/fixtures/homonyms_corrupt_with_msgctxt_first.html new file mode 100644 index 0000000..5dc8c47 --- /dev/null +++ b/test/fixtures/homonyms_corrupt_with_msgctxt_first.html @@ -0,0 +1,6 @@ + + +

Hello!

+

Hello!

+ + diff --git a/test/fixtures/homonyms_corrupt_without_msgctxt_first.html b/test/fixtures/homonyms_corrupt_without_msgctxt_first.html new file mode 100644 index 0000000..b046a5c --- /dev/null +++ b/test/fixtures/homonyms_corrupt_without_msgctxt_first.html @@ -0,0 +1,6 @@ + + +

Hello!

+

Hello!

+ + diff --git a/test/fixtures/multifilter.html b/test/fixtures/multifilter.html index dadb3a3..07698dc 100644 --- a/test/fixtures/multifilter.html +++ b/test/fixtures/multifilter.html @@ -2,5 +2,7 @@ {{'Second'|translate|lowercase}} + {{'Bird'|translate:'free'|lowercase}} + {{'Bird'|translate:'cage'|uppercase}} From 28479f04c84f9af3642d4021bff81b7868455105 Mon Sep 17 00:00:00 2001 From: Cristiano Lopes Date: Sat, 6 Sep 2014 17:30:39 +0100 Subject: [PATCH 2/4] implemented msgctxt compilation --- lib/compile.js | 16 +++++++++++++++- test/compile.js | 24 ++++++++++++++++++++++++ test/fixtures/pt.po | 37 +++++++++++++++++++++++++++++++++++++ 3 files changed, 76 insertions(+), 1 deletion(-) create mode 100644 test/fixtures/pt.po diff --git a/lib/compile.js b/lib/compile.js index 1a7bc28..40f890e 100644 --- a/lib/compile.js +++ b/lib/compile.js @@ -71,7 +71,21 @@ var Compiler = (function () { for (var i = 0; i < catalog.items.length; i++) { var item = catalog.items[i]; if (item.msgstr[0].length > 0 && !item.flags.fuzzy) { - strings[item.msgid] = item.msgstr.length === 1 ? item.msgstr[0] : item.msgstr; + var translation = item.msgstr.length === 1 ? item.msgstr[0] : item.msgstr; + if (item.msgctxt || + _.isPlainObject(strings[item.msgid])){ + var translationContext = {}; + if (item.msgctxt){ + translationContext[item.msgctxt] = translation; + } + if (_.isPlainObject(strings[item.msgid])){ + strings[item.msgid] = _.extend(strings[item.msgid], translationContext); + } else { + strings[item.msgid] = translationContext; + } + } else { + strings[item.msgid] = translation; + } } } diff --git a/test/compile.js b/test/compile.js index ff18341..2c00daa 100644 --- a/test/compile.js +++ b/test/compile.js @@ -73,6 +73,30 @@ describe('Compile', function () { assert(catalog.called); }); + it('Adds objects to the catalog if translations have a msgctxt', function () { + var files = ['test/fixtures/pt.po']; + var output = testCompile(files); + console.log(output); + var catalog = { + called: false, + setStrings: function (language, strings) { + this.called = true; + assert.equal(language, 'pt-BR'); + assert.deepEqual(strings['Hello!'], { + male:'Olá!', + female:'Olá Olá!' + }); + assert.deepEqual(strings.Bird, { + cage:['Pássaro Oprimido', 'Pássaros Oprimidos'], + free:['Pássaro Livre', 'Pássaros Livres'] + }); + } + }; + + var context = vm.createContext(makeEnv('gettext', catalog)); + vm.runInContext(output, context); + }); + it('Accepts a module parameter', function () { var files = ['test/fixtures/nl.po']; var output = testCompile(files, { diff --git a/test/fixtures/pt.po b/test/fixtures/pt.po new file mode 100644 index 0000000..63022c3 --- /dev/null +++ b/test/fixtures/pt.po @@ -0,0 +1,37 @@ +msgid "" +msgstr "" +"Language: pt-BR\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=2; plural=n>1;\n" +"Project-Id-Version: \n" +"POT-Creation-Date: \n" +"PO-Revision-Date: \n" +"Last-Translator: \n" +"Language-Team: \n" +"X-Generator: Poedit 1.5.7\n" + +#: test/fixtures/homonyms.js +msgctxt "cage" +msgid "Bird" +msgid_plural "Birds" +msgstr[0] "Pássaro Oprimido" +msgstr[1] "Pássaros Oprimidos" + +#: test/fixtures/homonyms.js +msgctxt "free" +msgid "Bird" +msgid_plural "Birds" +msgstr[0] "Pássaro Livre" +msgstr[1] "Pássaros Livres" + +#: test/fixtures/homonyms.js +msgctxt "male" +msgid "Hello!" +msgstr "Olá!" + +#: test/fixtures/homonyms.js +msgctxt "female" +msgid "Hello!" +msgstr "Olá Olá!" From 4eb1ab34288150dc3769f17e28ea7b76c8fe1ef7 Mon Sep 17 00:00:00 2001 From: Cristiano Lopes Date: Sun, 7 Sep 2014 08:45:57 +0100 Subject: [PATCH 3/4] always wrap the context bound translations in an array even if only one exists --- lib/compile.js | 2 +- test/compile.js | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/compile.js b/lib/compile.js index 40f890e..d995cab 100644 --- a/lib/compile.js +++ b/lib/compile.js @@ -76,7 +76,7 @@ var Compiler = (function () { _.isPlainObject(strings[item.msgid])){ var translationContext = {}; if (item.msgctxt){ - translationContext[item.msgctxt] = translation; + translationContext[item.msgctxt] = _.isArray(translation) ? translation : [translation]; } if (_.isPlainObject(strings[item.msgid])){ strings[item.msgid] = _.extend(strings[item.msgid], translationContext); diff --git a/test/compile.js b/test/compile.js index 2c00daa..b0f9358 100644 --- a/test/compile.js +++ b/test/compile.js @@ -83,8 +83,8 @@ describe('Compile', function () { this.called = true; assert.equal(language, 'pt-BR'); assert.deepEqual(strings['Hello!'], { - male:'Olá!', - female:'Olá Olá!' + male:['Olá!'], + female:['Olá Olá!'] }); assert.deepEqual(strings.Bird, { cage:['Pássaro Oprimido', 'Pássaros Oprimidos'], From cdae2c09e0305cce9b9414c5be06bed6797f32a9 Mon Sep 17 00:00:00 2001 From: Cristiano Lopes Date: Sun, 7 Sep 2014 09:35:26 +0100 Subject: [PATCH 4/4] always wrapping context bound translations in an array and fixed duplication of entries with same msgctxt and msgid in the extracted pot file --- lib/extract.js | 25 +++++++++++++++++-------- test/extract.coffee | 12 ++++++------ test/fixtures/homonyms.html | 1 + 3 files changed, 24 insertions(+), 14 deletions(-) diff --git a/lib/extract.js b/lib/extract.js index 7728b93..7397c44 100644 --- a/lib/extract.js +++ b/lib/extract.js @@ -75,33 +75,42 @@ var Extractor = (function () { if (translateContext){ translateContext = translateContext.trim(); } + var item; + var missingContextError = false; if (_.isEmpty(this.strings[string])){ this.strings[string] = [new Po.Item()]; } else { - var errorMessage = 'Found, for the same msgid '; - errorMessage += string; - errorMessage += ', translations with and without msgctxt'; if (translateContext){ if (_.all(this.strings[string], 'msgctxt')){ - this.strings[string].push(new Po.Item()); + item = _.head(_.remove(this.strings[string], { + msgid: string, + msgctxt: translateContext + })) || new Po.Item(); + this.strings[string].push(item); } else { - throw new Error(errorMessage); + missingContextError = true; } } else { if (_.any(this.strings[string], 'msgctxt')){ - throw new Error(errorMessage); + missingContextError = true; } } } - - var item = _.last(this.strings[string]); + item = _.last(this.strings[string]); item.msgid = string; item.msgctxt = translateContext; if (item.references.indexOf(file) < 0) { item.references.push(file); } + if (missingContextError){ + var errorMessage = 'Found, for the same msgid '; + errorMessage += string; + errorMessage += ', translations with and without msgctxt'; + errorMessage += ' (in: ' + item.references.join(', '); + throw new Error(errorMessage); + } if (plural && plural !== '') { if (item.msgid_plural && item.msgid_plural !== plural) { throw new Error('Incompatible plural definitions for ' + string + ': ' + item.msgid_plural + ' / ' + plural + ' (in: ' + (item.references.join(', ')) + ')'); diff --git a/test/extract.coffee b/test/extract.coffee index e54418a..63f6003 100644 --- a/test/extract.coffee +++ b/test/extract.coffee @@ -504,16 +504,16 @@ describe 'Extract', -> catalog = testExtract(files) assert.equal(catalog.items.length, 2) - assert.equal(catalog.items[0].msgid, 'Hello!') - assert.equal(catalog.items[0].msgstr, '') - assert.equal(catalog.items[0].msgctxt, 'male') - assert.equal(catalog.items[0].references.length, 1) - assert.equal(catalog.items[0].references[0], 'test/fixtures/homonyms.html') assert.equal(catalog.items[1].msgid, 'Hello!') assert.equal(catalog.items[1].msgstr, '') - assert.equal(catalog.items[1].msgctxt, 'female') + assert.equal(catalog.items[1].msgctxt, 'male') assert.equal(catalog.items[1].references.length, 1) assert.equal(catalog.items[1].references[0], 'test/fixtures/homonyms.html') + assert.equal(catalog.items[0].msgid, 'Hello!') + assert.equal(catalog.items[0].msgstr, '') + assert.equal(catalog.items[0].msgctxt, 'female') + assert.equal(catalog.items[0].references.length, 1) + assert.equal(catalog.items[0].references[0], 'test/fixtures/homonyms.html') it 'Can extract context aware translations from javascript', () -> files = [ diff --git a/test/fixtures/homonyms.html b/test/fixtures/homonyms.html index bbba3f6..77a6531 100644 --- a/test/fixtures/homonyms.html +++ b/test/fixtures/homonyms.html @@ -2,5 +2,6 @@

Hello!

Hello!

+

Hello!