diff --git a/Service/PhraseApp.php b/Service/PhraseApp.php
index f037045..e1d0929 100644
--- a/Service/PhraseApp.php
+++ b/Service/PhraseApp.php
@@ -14,8 +14,6 @@
 use RuntimeException;
 use Symfony\Bundle\FrameworkBundle\Translation\TranslationLoader;
 use Symfony\Component\EventDispatcher\EventDispatcherInterface;
-use Symfony\Component\Finder\Finder;
-use Symfony\Component\Finder\SplFileInfo;
 use Symfony\Component\Translation\MessageCatalogue;
 use Symfony\Component\Translation\Writer\TranslationWriter;
 
@@ -305,13 +303,14 @@ protected function generateTmpPath()
 
     protected function cleanUp()
     {
-        $finder = new Finder();
-        $files  = $finder->files()->name('*.*.yml')->in($this->getTmpPath());
-        /** @var SplFileInfo $file */
-        foreach ($files as $file) {
-            if (false === @unlink($file)) {
-                $this->logger->warning(strtr('Could not delete the translation file "{file}".', ['{file}' => $file]));
+        foreach ($this->downloadedLocales as $locale => $files) {
+            foreach ($files as $file) {
+                if (false === @unlink($file)) {
+                    $this->logger->warning(strtr('Could not delete the translation file "{file}".', ['{file}' => $file]));
+                }
             }
+
+            unset($this->downloadedLocales[$locale]);
         }
     }
 }