#61 Support for PHP's gettext()

Refactorings
nyavro · Oct 29, 2020 · 2d0274d · 2d0274d
1 parent a2118fd
commit 2d0274d
Show file tree

Hide file tree

Showing 30 changed files with 428 additions and 209 deletions.
diff --git a/src/main/kotlin/com/eny/i18n/plugin/ide/actions/KeyRequest.kt b/src/main/kotlin/com/eny/i18n/plugin/ide/actions/KeyRequest.kt
@@ -2,7 +2,7 @@ package com.eny.i18n.plugin.ide.actions
 
 import com.eny.i18n.plugin.ide.settings.Settings
 import com.eny.i18n.plugin.key.FullKey
-import com.eny.i18n.plugin.key.parser.KeyParser
+import com.eny.i18n.plugin.key.parser.KeyParserBuilder
 import com.eny.i18n.plugin.utils.KeyElement
 import com.eny.i18n.plugin.utils.PluginBundle
 import com.intellij.openapi.project.Project
@@ -19,8 +19,6 @@ class KeyRequestResult(val key: FullKey?, val isCancelled: Boolean)
  */
 class KeyRequest {
 
-    private val parser = KeyParser()
-
     /**
      * Requests key
      */
@@ -37,12 +35,11 @@ class KeyRequest {
             KeyRequestResult(null, true)
         } else {
             KeyRequestResult(
-                parser.parse(
-                    Pair(listOf(KeyElement.literal(keyStr)), null),
-                    nsSeparator = config.nsSeparator,
-                    keySeparator = config.keySeparator,
-                    emptyNamespace = config.vue
-                ),
+                (if(config.gettext) KeyParserBuilder.withoutTokenizer() else KeyParserBuilder.withSeparators(config.nsSeparator, config.keySeparator)).build()
+                    .parse2(
+                        Pair(listOf(KeyElement.literal(keyStr)), null),
+                        emptyNamespace = config.vue || config.gettext
+                    ),
                 false
             )
         }

diff --git a/src/main/kotlin/com/eny/i18n/plugin/ide/annotator/CompositeKeyAnnotatorBase.kt b/src/main/kotlin/com/eny/i18n/plugin/ide/annotator/CompositeKeyAnnotatorBase.kt
@@ -45,7 +45,6 @@ abstract class CompositeKeyAnnotatorBase(private val keyExtractor: FullKeyExtrac
             when {
                 mostResolvedReference.unresolved.isEmpty() && mostResolvedReference.element?.isLeaf() ?: false -> annotationHelper.annotateResolved(fullKey)
                 mostResolvedReference.unresolved.isEmpty() -> annotationHelper.annotateReferenceToObject(fullKey)
-                mostResolvedReference.isTemplateUnresolved() -> annotationHelper.annotateResolved(fullKey)
                 else -> annotationHelper.unresolvedKey(fullKey, mostResolvedReference)
             }
         }

diff --git a/src/main/kotlin/com/eny/i18n/plugin/ide/completion/CompositeKeyCompletionContributor.kt b/src/main/kotlin/com/eny/i18n/plugin/ide/completion/CompositeKeyCompletionContributor.kt
@@ -30,10 +30,9 @@ abstract class CompositeKeyCompletionContributor(private val callContext: CallCo
     override fun fillCompletionVariants(parameters: CompletionParameters, result: CompletionResultSet) {
 //        super.fillCompletionVariants(parameters, result)
         if(parameters.position.text.unQuote().substringAfter(DUMMY_KEY).trim().isNotBlank()) return
-        val isInTranslationContext = callContext.accepts(parameters.position.parent)
         val fullKey = keyExtractor.extractFullKey(parameters.position)
         if (fullKey == null) {
-            if (isInTranslationContext) {
+            if (callContext.accepts(parameters.position.parent)) {
                 val prefix = parameters.position.text.replace(DUMMY_KEY, "").unQuote().trim()
                 val emptyKeyCompletions = emptyKeyCompletions(parameters.position.project, prefix, parameters.position)
                 result.addAllElements(emptyKeyCompletions)

diff --git a/src/main/kotlin/com/eny/i18n/plugin/ide/folding/FoldingBuilderBase.kt b/src/main/kotlin/com/eny/i18n/plugin/ide/folding/FoldingBuilderBase.kt
@@ -4,13 +4,17 @@ import com.eny.i18n.plugin.factory.LanguageFactory
 import com.eny.i18n.plugin.ide.settings.Config
 import com.eny.i18n.plugin.ide.settings.Settings
 import com.eny.i18n.plugin.key.FullKey
+import com.eny.i18n.plugin.key.lexer.NoTokenizer
+import com.eny.i18n.plugin.key.lexer.NsKeyTokenizer
 import com.eny.i18n.plugin.key.parser.KeyParser
+import com.eny.i18n.plugin.key.parser.KeyParserBuilder
+import com.eny.i18n.plugin.parser.DummyTextNormalizer
+import com.eny.i18n.plugin.parser.ExpressionNormalizer
+import com.eny.i18n.plugin.parser.KeyNormalizer
 import com.eny.i18n.plugin.tree.CompositeKeyResolver
 import com.eny.i18n.plugin.tree.PropertyReference
 import com.eny.i18n.plugin.tree.PsiElementTree
-import com.eny.i18n.plugin.utils.LocalizationSourceSearch
-import com.eny.i18n.plugin.utils.ellipsis
-import com.eny.i18n.plugin.utils.unQuote
+import com.eny.i18n.plugin.utils.*
 import com.intellij.lang.ASTNode
 import com.intellij.lang.folding.FoldingBuilderEx
 import com.intellij.lang.folding.FoldingDescriptor
@@ -28,21 +32,22 @@ abstract class FoldingBuilderBase(private val languageFactory: LanguageFactory)
 
     private val group = FoldingGroup.newGroup("i18n")
 
-    private val parser: KeyParser = KeyParser()
-
     override fun getPlaceholderText(node: ASTNode): String? = ""
 
     override fun buildFoldRegions(root: PsiElement, document: Document, quick: Boolean): Array<FoldingDescriptor> {
         val config = Settings.getInstance(root.project).config()
+        val parser = (
+            if (config.gettext) KeyParserBuilder.withoutTokenizer()
+            else KeyParserBuilder.withSeparators(config.nsSeparator, config.keySeparator).withTemplateNormalizer()
+        ).build()
         if (!config.foldingEnabled) return arrayOf()
         val search = LocalizationSourceSearch(root.project)
         val foldingProvider = languageFactory.foldingProvider()
         return foldingProvider.collectContainers(root)
             .flatMap { container ->
                 val (literals, offset) = foldingProvider.collectLiterals(container)
                 literals.mapNotNull { literal ->
-                    parser
-                        .parse(literal.text.unQuote(), config.nsSeparator, config.keySeparator, config.vue)
+                    parser.parse1(literal.text.unQuote(), config.vue || config.gettext)
                         ?.let { key -> resolve(container, literal, search, config, key) }
                         ?.let { resolved ->
                             FoldingDescriptor(

diff --git a/src/main/kotlin/com/eny/i18n/plugin/ide/settings/Config.kt b/src/main/kotlin/com/eny/i18n/plugin/ide/settings/Config.kt
@@ -23,7 +23,9 @@ data class Config (
     val foldingMaxLength: Int = 20,
     val jsonContentGenerationEnabled: Boolean = true,
     val yamlContentGenerationEnabled: Boolean = true,
-    val extractSorted: Boolean = false
+    val extractSorted: Boolean = false,
+    val gettext: Boolean = false,
+    val gettextAliases: String = "gettext,_,__"
 ) {
 
     private val MAX_DEFAULT_NAMESPACES = 100

diff --git a/src/main/kotlin/com/eny/i18n/plugin/ide/settings/Settings.kt b/src/main/kotlin/com/eny/i18n/plugin/ide/settings/Settings.kt
@@ -53,6 +53,10 @@ class Settings : PersistentStateComponent<Settings> {
 
     internal var extractSorted = default.extractSorted
 
+    internal var gettext = default.gettext
+
+    internal var gettextAliases = default.gettextAliases
+
     /**
      * Returns plugin configuration
      */
@@ -81,7 +85,9 @@ class Settings : PersistentStateComponent<Settings> {
         foldingMaxLength = foldingMaxLength,
         jsonContentGenerationEnabled = jsonContentGenerationEnabled,
         yamlContentGenerationEnabled = yamlContentGenerationEnabled,
-        extractSorted = extractSorted
+        extractSorted = extractSorted,
+        gettext = gettext,
+        gettextAliases = gettextAliases
     )
 
     fun setConfig(config: Config) {
@@ -111,6 +117,8 @@ class Settings : PersistentStateComponent<Settings> {
         jsonContentGenerationEnabled = config.jsonContentGenerationEnabled
         yamlContentGenerationEnabled = config.yamlContentGenerationEnabled
         extractSorted = config.extractSorted
+        gettext = config.gettext
+        gettextAliases = config.gettextAliases
     }
 
     override fun loadState(state: Settings) = XmlSerializerUtil.copyBean(state, this)

diff --git a/src/main/kotlin/com/eny/i18n/plugin/ide/settings/SettingsPanel.kt b/src/main/kotlin/com/eny/i18n/plugin/ide/settings/SettingsPanel.kt
@@ -120,6 +120,16 @@ class SettingsPanel(val settings: Settings, val project: Project) {
         return panel
     }
 
+    private fun gettext(): JPanel {
+        val panel = JPanel()
+        panel.layout = BorderLayout()
+        panel.preferredSize = Dimension(350, 30)
+        val gettextMode = JCheckBox(PluginBundle.getMessage("settings.gettext.enabled"), settings.gettext)
+        gettextMode.addItemListener { _ -> settings.gettext = gettextMode.isSelected}
+        panel.add(gettextMode, BorderLayout.WEST)
+        return panel
+    }
+
     private fun settingsPanel(): JPanel {
         val root = JPanel()
         val panel = JPanel()
@@ -136,6 +146,7 @@ class SettingsPanel(val settings: Settings, val project: Project) {
         panel.add(checkbox(PluginBundle.getMessage("settings.extraction.sorted"), settings::extractSorted))
         panel.add(vue())
         panel.add(textInput("Vue locales directory", settings::vueDirectory))
+        panel.add(gettext())
         root.add(panel, BorderLayout.PAGE_START)
         return root
     }

diff --git a/src/main/kotlin/com/eny/i18n/plugin/key/FullKey.kt b/src/main/kotlin/com/eny/i18n/plugin/key/FullKey.kt
@@ -10,7 +10,6 @@ data class FullKey(
     val source: String,
     val ns: Literal?,
     val compositeKey:List<Literal>,
-    val isTemplate: Boolean = false,
     val namespaces: List<String>? = null
 ) {
     fun allNamespaces(): List<String> {

diff --git a/src/main/kotlin/com/eny/i18n/plugin/key/lexer/Tokenizer.kt b/src/main/kotlin/com/eny/i18n/plugin/key/lexer/Tokenizer.kt
@@ -1,7 +1,9 @@
 package com.eny.i18n.plugin.key.lexer
 
+import com.eny.i18n.plugin.parser.KeyNormalizer
 import com.eny.i18n.plugin.utils.KeyElement
 import com.eny.i18n.plugin.utils.KeyElementType
+import com.eny.i18n.plugin.utils.foldWhileAccum
 import java.util.*
 
 /**
@@ -27,22 +29,49 @@ object KeySeparator: Separator
 /**
  * Represents key literal
  */
-data class Literal(val text: String, val length: Int = text.length, val isTemplate: Boolean = false): Token {
+data class Literal(val text: String, val length: Int = text.length, val isTemp: Boolean = false): Token {
     /**
      * Merges two tokens
      */
     fun merge(token: Literal): Literal = Literal(text + token.text, length + token.length)
 }
 
+/**
+ * Tokenizer interface
+ */
+interface Tokenizer {
+    fun tokenize(elements: List<KeyElement>): Pair<String, List<Token>>
+}
+
+/**
+ * Class connects Tokenizer and Normalizers
+ */
+class NormalizingTokenizer(private val tokenizer: Tokenizer, private val normalizers: List<KeyNormalizer>): Tokenizer {
+    private fun normalize(element: KeyElement): KeyElement? = normalizers.foldWhileAccum(element, { acc, item -> item.normalize(acc) })
+    private fun normalize(elements: List<KeyElement>) = elements.mapNotNull {normalize(it)}
+    override fun tokenize(elements: List<KeyElement>): Pair<String, List<Token>> = tokenizer.tokenize(normalize(elements))
+}
+
+/**
+ * Does no tokenization
+ */
+class NoTokenizer: Tokenizer {
+    override fun tokenize(elements: List<KeyElement>): Pair<String, List<Token>> = Pair(
+        elements.joinToString(""){it.text}, elements.map {Literal(it.text)}
+    )
+}
+
 /**
  * Tokenizer of translation keys
  */
-class Tokenizer(private val nsSeparator: String, private val keySeparator: String) {
+class NsKeyTokenizer(private val nsSeparator: String, private val keySeparator: String): Tokenizer {
 
     /**
      * Tokenize list of key elements into list of tokens
      */
-    fun tokenizeAll(elements: List<KeyElement>): List<Token> = elements.flatMap (::tokenize)
+    override fun tokenize(elements: List<KeyElement>): Pair<String, List<Token>> {
+        return Pair(elements.joinToString(""){it.text}, elements.flatMap (::tokenize))
+    }
 
     /**
      * Tokenizes single key element into list of tokens
@@ -54,7 +83,7 @@ class Tokenizer(private val nsSeparator: String, private val keySeparator: Strin
                 Literal(
                     "*",
                     element.text.length,
-                    true
+                    false
                 )
             )
         }

diff --git a/src/main/kotlin/com/eny/i18n/plugin/key/parser/KeyParser.kt b/src/main/kotlin/com/eny/i18n/plugin/key/parser/KeyParser.kt
@@ -3,43 +3,36 @@ package com.eny.i18n.plugin.key.parser
 import com.eny.i18n.plugin.key.FullKey
 import com.eny.i18n.plugin.key.lexer.*
 import com.eny.i18n.plugin.parser.KeyNormalizer
-import com.eny.i18n.plugin.parser.KeyNormalizerImpl
 import com.eny.i18n.plugin.utils.KeyElement
-import com.eny.i18n.plugin.utils.KeyElementType
+import com.eny.i18n.plugin.utils.foldWhileAccum
 
 /**
  * Parses list of normalized key elements into FullKey
  */
-class KeyParser(private val normalizer: KeyNormalizer = KeyNormalizerImpl()) {
+class KeyParser(private val tokenizer: Tokenizer) {
 
     /**
      * Parses text to i18n key
      */
-    fun parse(text: String, nsSeparator: String, keySeparator: String, emptyNamespace: Boolean) =
-        parse(Pair(listOf(KeyElement.literal(text)), null), nsSeparator, keySeparator, emptyNamespace)
+    fun parse1(text: String, emptyNamespace: Boolean) =
+        parse2(Pair(listOf(KeyElement.literal(text)), null), emptyNamespace)
 
     /**
      * Parses list of key elements into i18n key
      */
-    fun parse(
+    fun parse2(
         pair: Pair<List<KeyElement>, List<String>?>,
-        nsSeparator: String = ":",
-        keySeparator: String = ".",
         emptyNamespace: Boolean = false
     ): FullKey? {
-        val normalized = normalizer.normalize(pair.first)
         val startState = if (emptyNamespace) {
             WaitingLiteral(file = null, key = emptyList())
         } else {
             Start(null)
         }
-        val isTemplate = normalized.any { it.type == KeyElementType.TEMPLATE }
-        return Tokenizer(nsSeparator, keySeparator)
-            .tokenizeAll(normalized)
-            .fold(startState) { state, token ->
-                state.next(token)
-            }
-            .fullKey(isTemplate, normalized.fold(""){ acc, item -> acc + item.text }, pair.second)
+        val (source, tokenized) = tokenizer.tokenize(pair.first)
+        return tokenized
+            .fold(startState) { state, token -> state.next(token) }
+            .fullKey(source, pair.second)
     }
 }
 
@@ -55,7 +48,7 @@ private interface State {
     /**
      * Get current parsed key
      */
-    fun fullKey(isTemplate: Boolean, source: String, namespaces: List<String>?): FullKey? = null
+    fun fullKey(source: String, namespaces: List<String>?): FullKey? = null
 }
 
 /**
@@ -76,8 +69,8 @@ private class Start(private val init: Literal?) : State {
             token is Literal -> Start(init?.merge(token) ?: token)
             else -> Error("Invalid ns separator position (0)") // Never get here
         }
-    override fun fullKey(isTemplate: Boolean, source: String, namespaces: List<String>?): FullKey? =
-        init?.let {FullKey(source, null, listOf(it), isTemplate, namespaces)}
+    override fun fullKey(source: String, namespaces: List<String>?): FullKey? =
+        init?.let {FullKey(source, null, listOf(it), namespaces)}
 }
 
 /**
@@ -89,8 +82,9 @@ private class WaitingLiteral(private val file: Literal?, val key: List<Literal>)
             is Literal -> WaitingLiteralOrSeparator(file, key + token)
             else -> Error("Invalid token $token")
         }
-    override fun fullKey(isTemplate: Boolean, source: String, namespaces: List<String>?): FullKey? =
-        FullKey(source, file, key + Literal("", 0), isTemplate, namespaces)
+    override fun fullKey(source: String, namespaces: List<String>?): FullKey? {
+        return FullKey(source, file, key + Literal("", 0), namespaces)
+    }
 }
 
 /**
@@ -103,6 +97,7 @@ private class WaitingLiteralOrSeparator(val file: Literal?, val key: List<Litera
             is KeySeparator -> WaitingLiteral(file, key)
             else -> Error("Invalid token $token")
         }
-    override fun fullKey(isTemplate: Boolean, source: String, namespaces: List<String>?): FullKey? =
-        FullKey(source, file, key, isTemplate, namespaces)
+    override fun fullKey(source: String, namespaces: List<String>?): FullKey? {
+        return FullKey(source, file, key, namespaces)
+    }
 }
diff --git a/src/main/kotlin/com/eny/i18n/plugin/key/parser/KeyParserBuilder.kt b/src/main/kotlin/com/eny/i18n/plugin/key/parser/KeyParserBuilder.kt
@@ -0,0 +1,50 @@
+package com.eny.i18n.plugin.key.parser
+
+import com.eny.i18n.plugin.key.lexer.NoTokenizer
+import com.eny.i18n.plugin.key.lexer.NormalizingTokenizer
+import com.eny.i18n.plugin.key.lexer.NsKeyTokenizer
+import com.eny.i18n.plugin.key.lexer.Tokenizer
+import com.eny.i18n.plugin.parser.DummyTextNormalizer
+import com.eny.i18n.plugin.parser.ExpressionNormalizer
+import com.eny.i18n.plugin.parser.KeyNormalizer
+
+/**
+ * Builds Key parser by given options
+ */
+class KeyParserBuilder private constructor (private val tokenizer: Tokenizer, private val normalizers: List<KeyNormalizer>) {
+
+    /**
+     * Builds one more step to KeyParser: adds dummy text normalizer
+     */
+    fun withDummyNormalizer(): KeyParserBuilder {
+        return KeyParserBuilder(tokenizer, normalizers + DummyTextNormalizer())
+    }
+
+    /**
+     * Builds one more step to KeyParser: adds dummy text normalizer
+     */
+    fun withTemplateNormalizer(): KeyParserBuilder {
+        return KeyParserBuilder(tokenizer, normalizers + ExpressionNormalizer())
+    }
+
+    /**
+     * Builds final KeyParser
+     */
+    fun build(): KeyParser = KeyParser(NormalizingTokenizer(tokenizer, normalizers))
+
+    companion object {
+        /**
+         * Builds one step of key parser: sets tokenizer with ns and key separators
+         */
+        fun withSeparators(ns: String, key: String): KeyParserBuilder {
+            return KeyParserBuilder(NsKeyTokenizer(ns, key), listOf())
+        }
+
+        /**
+         * Builds one step of key parser: sets NoTokenizer
+         */
+        fun withoutTokenizer(): KeyParserBuilder {
+            return KeyParserBuilder(NoTokenizer(), listOf())
+        }
+    }
+}