SONARPY-2243 Populate project V1 symbols out of descriptors collected by the V2 type inference

python-checks/src/test/resources/checks/incorrectExceptionType/incorrectExceptionTypeWithRegularImport.py

@@ -38,7 +38,7 @@ def raise_nested_non_exception_class():
   raise Enclsoing.Nested() # FN as only top-level imported symbols are considered
 
 def raise_RedefinedBaseExceptionChild():
-  raise RedefinedBaseExceptionChild() # FN
+  raise RedefinedBaseExceptionChild() # Noncompliant
 
 def raise_ChildOfActualException():
   raise ChildOfActualException() # OK

python-frontend/src/main/java/org/sonar/python/semantic/ProjectLevelSymbolTable.java

@@ -24,30 +24,35 @@
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 import javax.annotation.CheckForNull;
 import javax.annotation.Nullable;
 import org.sonar.plugins.python.api.PythonFile;
-import org.sonar.plugins.python.api.symbols.AmbiguousSymbol;
 import org.sonar.plugins.python.api.symbols.Symbol;
-import org.sonar.plugins.python.api.symbols.Usage;
 import org.sonar.plugins.python.api.tree.BaseTreeVisitor;
 import org.sonar.plugins.python.api.tree.CallExpression;
 import org.sonar.plugins.python.api.tree.FileInput;
 import org.sonar.plugins.python.api.tree.RegularArgument;
 import org.sonar.python.index.AmbiguousDescriptor;
 import org.sonar.python.index.Descriptor;
 import org.sonar.python.index.DescriptorUtils;
-import org.sonar.python.index.VariableDescriptor;
+import org.sonar.python.semantic.v2.BasicTypeTable;
+import org.sonar.python.semantic.v2.SymbolTableBuilderV2;
+import org.sonar.python.semantic.v2.TypeInferenceV2;
+import org.sonar.python.semantic.v2.UsageV2;
+import org.sonar.python.semantic.v2.converter.PythonTypeToDescriptorConverter;
 import org.sonar.python.semantic.v2.typeshed.TypeShedDescriptorsProvider;
+import org.sonar.python.types.v2.UnknownType;
 
 import static org.sonar.python.tree.TreeUtils.getSymbolFromTree;
 import static org.sonar.python.tree.TreeUtils.nthArgumentOrKeyword;
 
 public class ProjectLevelSymbolTable {
 
+  private final PythonTypeToDescriptorConverter pythonTypeToDescriptorConverter = new PythonTypeToDescriptorConverter();
   private final Map<String, Set<Descriptor>> globalDescriptorsByModuleName;
   private Map<String, Descriptor> globalDescriptorsByFQN;
   private final Set<String> djangoViewsFQN = new HashSet<>();
@@ -88,41 +93,17 @@ public void addModule(FileInput fileInput, String packageName, PythonFile python
     SymbolTableBuilder symbolTableBuilder = new SymbolTableBuilder(packageName, pythonFile);
     String fullyQualifiedModuleName = SymbolUtils.fullyQualifiedModuleName(packageName, pythonFile.fileName());
     fileInput.accept(symbolTableBuilder);
-    Set<Descriptor> globalDescriptors = new HashSet<>();
     importsByModule.put(fullyQualifiedModuleName, symbolTableBuilder.importedModulesFQN());
-    for (Symbol globalVariable : fileInput.globalVariables()) {
-      String fullyQualifiedVariableName = globalVariable.fullyQualifiedName();
-      if (((fullyQualifiedVariableName != null) && !fullyQualifiedVariableName.startsWith(fullyQualifiedModuleName)) ||
-        globalVariable.usages().stream().anyMatch(u -> u.kind().equals(Usage.Kind.IMPORT))) {
-        // TODO: We don't put builtin or imported names in global symbol table to avoid duplicate FQNs in project level symbol table (to fix with SONARPY-647)
-        continue;
-      }
-      if (globalVariable.is(Symbol.Kind.CLASS, Symbol.Kind.FUNCTION)) {
-        globalDescriptors.add(DescriptorUtils.descriptor(globalVariable));
-      } else {
-        String fullyQualifiedName = fullyQualifiedModuleName + "." + globalVariable.name();
-        if (globalVariable.is(Symbol.Kind.AMBIGUOUS)) {
-          globalDescriptors.add(DescriptorUtils.ambiguousDescriptor((AmbiguousSymbol) globalVariable, fullyQualifiedName));
-        } else {
-          globalDescriptors.add(new VariableDescriptor(globalVariable.name(), fullyQualifiedName, globalVariable.annotatedTypeName()));
-        }
-      }
-    }
-    globalDescriptorsByModuleName.put(fullyQualifiedModuleName, globalDescriptors);
-    if (globalDescriptorsByFQN != null) {
-      // TODO: build globalSymbolsByFQN incrementally
-      addModuleToGlobalSymbolsByFQN(globalDescriptors);
-    }
     DjangoViewsVisitor djangoViewsVisitor = new DjangoViewsVisitor();
     fileInput.accept(djangoViewsVisitor);
+    addModuleV2(fileInput, packageName, pythonFile);
   }
 
   private void addModuleToGlobalSymbolsByFQN(Set<Descriptor> descriptors) {
     Map<String, Descriptor> moduleDescriptorsByFQN = descriptors.stream()
       .filter(d -> d.fullyQualifiedName() != null)
       .collect(Collectors.toMap(Descriptor::fullyQualifiedName, Function.identity(), AmbiguousDescriptor::create));
-    globalDescriptorsByFQN.putAll(moduleDescriptorsByFQN);
-
+    globalDescriptorsByFQN().putAll(moduleDescriptorsByFQN);
   }
 
   private Map<String, Descriptor> globalDescriptorsByFQN() {
@@ -169,6 +150,11 @@ public Set<Descriptor> getDescriptorsFromModule(@Nullable String moduleName) {
     return globalDescriptorsByModuleName.get(moduleName);
   }
 
+  @CheckForNull
+  public Set<Descriptor> getDescriptorsFromModuleV2(@Nullable String moduleName) {
+    return globalDescriptorsByModuleName.get(moduleName);
+  }
+
   public Map<String, Set<String>> importsByModule() {
     return Collections.unmodifiableMap(importsByModule);
   }
@@ -201,6 +187,28 @@ public TypeShedDescriptorsProvider typeShedDescriptorsProvider() {
     return typeShedDescriptorsProvider;
   }
 
+
+  private void addModuleV2(FileInput astRoot, String packageName, PythonFile pythonFile) {
+    var fullyQualifiedModuleName = SymbolUtils.fullyQualifiedModuleName(packageName, pythonFile.fileName());
+    var symbolTable = new SymbolTableBuilderV2(astRoot).build();
+    var typeInferenceV2 = new TypeInferenceV2(new BasicTypeTable(), pythonFile, symbolTable);
+    var typesBySymbol = typeInferenceV2.inferTypes(astRoot);
+    var moduleDescriptors = typesBySymbol.entrySet()
+      .stream()
+      .filter(entry -> entry.getValue().stream().noneMatch(UnknownType.UnresolvedImportType.class::isInstance))
+      .map(entry -> {
+          var descriptor = pythonTypeToDescriptorConverter.convert(fullyQualifiedModuleName, entry.getKey(), entry.getValue());
+          return Map.entry(entry.getKey(), descriptor);
+        }
+      )
+      .filter(entry -> !(!Objects.requireNonNull(entry.getValue().fullyQualifiedName()).startsWith(fullyQualifiedModuleName)
+                         || entry.getKey().usages().stream().anyMatch(u -> u.kind().equals(UsageV2.Kind.IMPORT))))
+      .map(Map.Entry::getValue)
+      .collect(Collectors.toSet());
+    globalDescriptorsByModuleName.put(fullyQualifiedModuleName, moduleDescriptors);
+    addModuleToGlobalSymbolsByFQN(moduleDescriptors);
+  }
+
   private class DjangoViewsVisitor extends BaseTreeVisitor {
     @Override
     public void visitCallExpression(CallExpression callExpression) {

python-frontend/src/main/java/org/sonar/python/semantic/v2/ClassTypeBuilder.java

@@ -65,7 +65,7 @@ public ClassTypeBuilder withDefinitionLocation(@Nullable LocationInFile definiti
   }
 
   public ClassTypeBuilder addSuperClass(PythonType type) {
-    superClasses.add(new LazyTypeWrapper(type));
+    superClasses.add(TypeWrapper.of(type));
     return this;
   }
 

python-frontend/src/main/java/org/sonar/python/semantic/v2/SymbolsModuleTypeProvider.java

@@ -79,7 +79,7 @@ private static String getModuleFqnString(List<String> moduleFqn) {
   }
 
   private Optional<ModuleType> createModuleTypeFromProjectLevelSymbolTable(String moduleName, String moduleFqn, ModuleType parent) {
-    var retrieved = projectLevelSymbolTable.getDescriptorsFromModule(moduleFqn);
+    var retrieved = projectLevelSymbolTable.getDescriptorsFromModuleV2(moduleFqn);
     if (retrieved == null) {
       return Optional.empty();
     }

python-frontend/src/main/java/org/sonar/python/semantic/v2/converter/ClassDescriptorToPythonTypeConverter.java

@@ -25,6 +25,7 @@
 import org.sonar.python.types.v2.LazyTypeWrapper;
 import org.sonar.python.types.v2.Member;
 import org.sonar.python.types.v2.PythonType;
+import org.sonar.python.types.v2.TypeWrapper;
 
 public class ClassDescriptorToPythonTypeConverter implements DescriptorToPythonTypeConverter {
 
@@ -34,8 +35,13 @@ private static PythonType convert(ConversionContext ctx, ClassDescriptor from) {
       .withDefinitionLocation(from.definitionLocation());
 
     from.superClasses().stream()
-      .map(fqn -> ctx.lazyTypesContext().getOrCreateLazyType(fqn))
-      .map(LazyTypeWrapper::new)
+      .map(fqn -> {
+        if (fqn != null) {
+          return ctx.lazyTypesContext().getOrCreateLazyType(fqn);
+        }
+        return PythonType.UNKNOWN;
+      })
+      .map(TypeWrapper::of)
       .forEach(typeBuilder::addSuperClass);
 
     var type = typeBuilder.build();

python-frontend/src/main/java/org/sonar/python/semantic/v2/converter/PythonTypeToDescriptorConverter.java

@@ -0,0 +1,173 @@
+/*
+ * SonarQube Python Plugin
+ * Copyright (C) 2011-2024 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.python.semantic.v2.converter;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import javax.annotation.CheckForNull;
+import org.sonar.python.index.AmbiguousDescriptor;
+import org.sonar.python.index.ClassDescriptor;
+import org.sonar.python.index.Descriptor;
+import org.sonar.python.index.FunctionDescriptor;
+import org.sonar.python.index.VariableDescriptor;
+import org.sonar.python.semantic.v2.SymbolV2;
+import org.sonar.python.types.v2.ClassType;
+import org.sonar.python.types.v2.FunctionType;
+import org.sonar.python.types.v2.ParameterV2;
+import org.sonar.python.types.v2.PythonType;
+import org.sonar.python.types.v2.UnionType;
+import org.sonar.python.types.v2.UnknownType;
+
+public class PythonTypeToDescriptorConverter {
+
+  public Descriptor convert(String moduleFqn, SymbolV2 symbol, Set<PythonType> types) {
+    var candidates = types.stream()
+      .map(type -> convert(moduleFqn, moduleFqn, symbol.name(), type))
+      .flatMap(candidate -> {
+        if (candidate instanceof AmbiguousDescriptor ambiguousDescriptor) {
+          return ambiguousDescriptor.alternatives().stream();
+        } else {
+          return Stream.of(candidate);
+        }
+      })
+      .collect(Collectors.toSet());
+
+    if (candidates.size() == 1) {
+      return candidates.iterator().next();
+    }
+
+    return new AmbiguousDescriptor(symbol.name(), symbolFqn(moduleFqn, symbol.name()), candidates);
+  }
+
+  private static Descriptor convert(String moduleFqn, String parentFqn, String symbolName, PythonType type) {
+    if (type instanceof FunctionType functionType) {
+      return convert(moduleFqn, parentFqn, symbolName, functionType);
+    }
+    if (type instanceof ClassType classType) {
+      return convert(moduleFqn, parentFqn, symbolName, classType);
+    }
+    if (type instanceof UnionType unionType) {
+      return convert(moduleFqn, parentFqn, symbolName, unionType);
+    }
+    if (type instanceof UnknownType.UnresolvedImportType unresolvedImportType) {
+      return convert(parentFqn, symbolName, unresolvedImportType);
+    }
+    return new VariableDescriptor(symbolName, symbolFqn(parentFqn, symbolName), null);
+  }
+
+  private static Descriptor convert(String moduleFqn, String parentFqn, String symbolName, FunctionType type) {
+
+    var parameters = type.parameters()
+      .stream()
+      .map(parameter -> convert(moduleFqn, parameter))
+      .toList();
+
+    return new FunctionDescriptor(symbolName, symbolFqn(parentFqn, symbolName),
+      parameters,
+      type.isAsynchronous(),
+      type.isInstanceMethod(),
+      List.of(),
+      type.hasDecorators(),
+      type.definitionLocation().orElse(null),
+      null,
+      null
+    );
+  }
+
+  private static Descriptor convert(String moduleFqn, String parentFqn, String symbolName, ClassType type) {
+    var symbolFqn = symbolFqn(parentFqn, symbolName);
+    var memberDescriptors = type.members()
+      .stream()
+      .map(m -> convert(moduleFqn, symbolFqn, m.name(), m.type()))
+      .collect(Collectors.toSet());
+
+    var hasSuperClassWithoutDescriptor = false;
+    var superClasses = new ArrayList<String>();
+    for (var superClassWrapper : type.superClasses()) {
+      var superClass = superClassWrapper.type();
+      if (superClass != PythonType.UNKNOWN) {
+        var superClassFqn = typeFqn(moduleFqn, superClass);
+        superClasses.add(superClassFqn);
+      } else {
+        hasSuperClassWithoutDescriptor = true;
+      }
+    }
+
+    return new ClassDescriptor(symbolName, symbolFqn,
+      superClasses,
+      memberDescriptors,
+      type.hasDecorators(),
+      type.definitionLocation().orElse(null),
+      hasSuperClassWithoutDescriptor,
+      type.hasMetaClass(),
+      null,
+      false
+    );
+  }
+
+  private static Descriptor convert(String moduleFqn, String parentFqn, String symbolName, UnionType type) {
+    var candidates = type.candidates().stream()
+      .map(candidateType -> convert(moduleFqn, parentFqn, symbolName, candidateType))
+      .collect(Collectors.toSet());
+    return new AmbiguousDescriptor(symbolName,
+      symbolFqn(moduleFqn, symbolName),
+      candidates
+    );
+  }
+
+  private static Descriptor convert(String parentFqn, String symbolName, UnknownType.UnresolvedImportType type) {
+    return new VariableDescriptor(symbolName,
+      symbolFqn(parentFqn, symbolName),
+      type.importPath()
+    );
+  }
+
+  private static FunctionDescriptor.Parameter convert(String moduleFqn, ParameterV2 parameter) {
+    var type = parameter.declaredType().type().unwrappedType();
+    var annotatedType = typeFqn(moduleFqn, type);
+
+    return new FunctionDescriptor.Parameter(parameter.name(),
+      annotatedType,
+      parameter.hasDefaultValue(),
+      parameter.isKeywordOnly(),
+      parameter.isPositionalOnly(),
+      parameter.isPositionalVariadic(),
+      parameter.isKeywordVariadic(),
+      parameter.location());
+  }
+
+  @CheckForNull
+  private static String typeFqn(String moduleFqn, PythonType type) {
+    if (type instanceof UnknownType.UnresolvedImportType importType) {
+      return importType.importPath();
+    } else if (type instanceof ClassType classType) {
+      return moduleFqn + "." + classType.name();
+    }
+    return null;
+  }
+
+  private static String symbolFqn(String moduleFqn, String symbolName) {
+    return moduleFqn + "." + symbolName;
+  }
+
+}

python-frontend/src/main/java/org/sonar/python/semantic/v2/types/TrivialTypeInferenceVisitor.java

@@ -424,7 +424,7 @@ public void visitParameter(Parameter parameter) {
   }
 
   private static PythonType resolveTypeAnnotationExpressionType(Expression expression) {
-    if (expression instanceof Name name && !(name.typeV2() instanceof UnknownType)) {
+    if (expression instanceof Name name && name.typeV2() != PythonType.UNKNOWN) {
       return new ObjectType(name.typeV2(), TypeSource.TYPE_HINT);
     } else if (expression instanceof SubscriptionExpression subscriptionExpression && !(subscriptionExpression.object().typeV2() instanceof UnknownType)) {
       var candidateTypes = subscriptionExpression.subscripts()