diff --git a/bin/get-translatable-strings b/bin/get-translatable-strings
index 2b0e7452e8ca70ade6b3ff3fff598777bb98025b..3848ced109f5ad1c3f209f30c6072e52eee00f9d 100755
--- a/bin/get-translatable-strings
+++ b/bin/get-translatable-strings
@@ -1,7 +1,11 @@
 #!/usr/bin/php -q
 <?php
+
+declare(strict_types=1);
+
 use SimpleSAML\Module;
 use SimpleSAML\Utils;
+use Symfony\Component\Filesystem\Filesystem;
 
 // This is the base directory of the SimpleSAMLphp installation
 $baseDir = dirname(dirname(__FILE__));
@@ -11,7 +15,12 @@ require_once($baseDir . '/lib/_autoload.php');
 
 $transUtils = new Utils\Translate();
 $sysUtils = new Utils\System();
+$filesystem = new Filesystem();
+
 $tempDirBase = $sysUtils->getTempDir() . "/temptemplatestemp";
+// Ensure no leftover from any previous invocation
+$filesystem->remove($tempDirBase);
+
 $outputSuffix = '/locales/en/LC_MESSAGES';
 
 $modules = ['', 'core', 'admin', 'cron', 'exampleauth', 'multiauth', 'saml'];
@@ -25,5 +34,5 @@ foreach($modules as $module) {
     print `xgettext --default-domain=$domain -p $outputDir --from-code=UTF-8 -j --no-location -ktrans -L PHP $tempDir/*/*.php`;
 }
 
-// TODO: clean up workdir before/after run
+$filesystem->remove($tempDirBase);
 // TODO: merge new strings into translated languages catalogs
diff --git a/lib/SimpleSAML/Utils/Translate.php b/lib/SimpleSAML/Utils/Translate.php
index 7d6b72707c4d56cffea76a6db504afd080e06f85..4dc2ec8a349c9ad4cc1ef529edc73eabb8907c49 100644
--- a/lib/SimpleSAML/Utils/Translate.php
+++ b/lib/SimpleSAML/Utils/Translate.php
@@ -22,9 +22,9 @@ class Translate
     {
         $config = Configuration::loadFromArray(['template.cache' => $outputDir, 'module.enable' => [$module => true]]);
         $baseDir = $config->getBaseDir();
-        $tplSuffix = '/templates/';
+        $tplSuffix = DIRECTORY_SEPARATOR . 'templates' . DIRECTORY_SEPARATOR;
 
-        $tplDir = $baseDir . ($module === '' ? '' : 'modules/' . $module) . $tplSuffix;
+        $tplDir = $baseDir . ($module === '' ? '' : 'modules' . DIRECTORY_SEPARATOR . $module) . $tplSuffix;
         $templateprefix = ($module === '' ? '' : $module . ":");
 
         foreach (
diff --git a/tests/lib/SimpleSAML/Utils/SystemTest.php b/tests/lib/SimpleSAML/Utils/SystemTest.php
index 93c95aec3480100e27d8bc94dd5f6e59220e43d4..e7ce0a5915539b99fb555080f116bb721a3429ad 100644
--- a/tests/lib/SimpleSAML/Utils/SystemTest.php
+++ b/tests/lib/SimpleSAML/Utils/SystemTest.php
@@ -16,7 +16,7 @@ use SimpleSAML\Utils;
 /**
  * Tests for SimpleSAML\Utils\System.
  *
- * @covers \SimpleSAML\Utils\Random
+ * @covers \SimpleSAML\Utils\System
  */
 class SystemTest extends TestCase
 {
diff --git a/tests/lib/SimpleSAML/Utils/TranslateTest.php b/tests/lib/SimpleSAML/Utils/TranslateTest.php
new file mode 100644
index 0000000000000000000000000000000000000000..76dcae19f266c4dbbb1e60d1f6004391e9408918
--- /dev/null
+++ b/tests/lib/SimpleSAML/Utils/TranslateTest.php
@@ -0,0 +1,83 @@
+<?php
+
+declare(strict_types=1);
+
+namespace SimpleSAML\Test\Utils;
+
+use InvalidArgumentException;
+use org\bovigo\vfs\vfsStream;
+use org\bovigo\vfs\vfsStreamDirectory;
+use PHPUnit\Framework\TestCase;
+use ReflectionClass;
+use SimpleSAML\Configuration;
+use SimpleSAML\Error;
+use SimpleSAML\Utils;
+use Symfony\Component\Filesystem\Filesystem;
+use Symfony\Component\Finder\Finder;
+
+/**
+ * Tests for SimpleSAML\Utils\Translate.
+ *
+ * @covers \SimpleSAML\Utils\Translate
+ */
+class TranslateTest extends TestCase
+{
+    protected Utils\System $sysUtils;
+
+    protected Utils\Translate $translate;
+
+    protected Filesystem $filesystem;
+
+    protected string $tempdir;
+
+    public function setUp(): void
+    {
+        $this->filesystem = new Filesystem();
+        $this->sysUtils = new Utils\System();
+
+        do {
+            $tmp = sys_get_temp_dir() . DIRECTORY_SEPARATOR . mt_rand();
+        } while (!@mkdir($tmp, 0700));
+        $this->tempdir = $tmp;
+
+        $this->translate = new Utils\Translate();
+    }
+
+    /**
+     * @test
+     */
+    public function testCompileAllTemplatesMain(): void
+    {
+        $workdir = $this->tempdir . DIRECTORY_SEPARATOR . "testall";
+        $this->translate->compileAllTemplates('', $workdir);
+        $this->checkAllFilesAreCompiledTemplates($workdir);
+    }
+
+    /**
+     * @test
+     */
+    public function testCompileAllTemplatesModule(): void
+    {
+        $workdir = $this->tempdir . DIRECTORY_SEPARATOR . "testadmin";
+        $this->translate->compileAllTemplates('admin', $workdir);
+        $this->checkAllFilesAreCompiledTemplates($workdir);
+    }
+
+    public function tearDown(): void
+    {
+        $this->filesystem->remove($this->tempdir);
+    }
+
+    protected function checkAllFilesAreCompiledTemplates(string $dir)
+    {
+        $finder = new Finder();
+        $finder->files()->in($dir);
+
+        $this->assertTrue($finder->hasResults());
+
+        foreach ($finder as $file) {
+            $this->assertEquals('php', $file->getExtension());
+            $this->assertStringContainsString('class __TwigTemplate', $file->getContents());
+        }
+    }
+}