diff --git a/mypy/semanal.py b/mypy/semanal.py
index a66e55a78e22..c59448bf62e4 100644
--- a/mypy/semanal.py
+++ b/mypy/semanal.py
@@ -265,8 +265,8 @@ def visit_file(self, file_node: MypyFile, fnam: str, options: Options) -> None:
 
         if self.cur_mod_id == 'builtins':
             remove_imported_names_from_symtable(self.globals, 'builtins')
-            for alias_name in ['List', 'Dict', 'Set']:
-                self.globals.pop(alias_name, None)
+            for alias_name in type_aliases:
+                self.globals.pop(alias_name.split('.')[-1], None)
 
         if '__all__' in self.globals:
             for name, g in self.globals.items():
diff --git a/test-data/unit/pythoneval.test b/test-data/unit/pythoneval.test
index 8411fbd638dd..c7e24e05bfd3 100644
--- a/test-data/unit/pythoneval.test
+++ b/test-data/unit/pythoneval.test
@@ -747,8 +747,10 @@ print('>', list(b))
 [case testInternalBuiltinDefinition]
 import typing
 def f(x: _T) -> None: pass
+s: FrozenSet
 [out]
 _program.py:2: error: Name '_T' is not defined
+_program.py:3: error: Name 'FrozenSet' is not defined
 
 [case testVarArgsFunctionSubtyping]
 import typing