From 6cc1868c371410cb37c872acbdbfe125b8c4f2a7 Mon Sep 17 00:00:00 2001 From: Maksim Grebeniuk Date: Wed, 23 Oct 2024 12:00:20 +0200 Subject: [PATCH 1/4] SONARPY-2219 Create descriptors out of PythonType and SymbolV2 --- .../semantic/ProjectLevelSymbolTable.java | 41 +++- .../python/semantic/v2/ClassTypeBuilder.java | 2 +- .../v2/SymbolsModuleTypeProvider.java | 2 +- .../ClassDescriptorToPythonTypeConverter.java | 10 +- .../PythonTypeToDescriptorConverter.java | 157 ++++++++++++ .../v2/types/TrivialTypeInferenceVisitor.java | 2 +- .../PythonTypeToDescriptorConverterTest.java | 226 ++++++++++++++++++ ...erProjectLevelSymbolTableBuildingTest.java | 121 ++++++++++ .../sonar/plugins/python/indexer/v2/mod1.py | 2 + .../sonar/plugins/python/indexer/v2/mod2.py | 3 + .../sonar/plugins/python/indexer/v2/script.py | 15 ++ 11 files changed, 575 insertions(+), 6 deletions(-) create mode 100644 python-frontend/src/main/java/org/sonar/python/semantic/v2/converter/PythonTypeToDescriptorConverter.java create mode 100644 python-frontend/src/test/java/org/sonar/python/semantic/v2/converter/PythonTypeToDescriptorConverterTest.java create mode 100644 sonar-python-plugin/src/test/java/org/sonar/plugins/python/indexer/SonarLintPythonIndexerProjectLevelSymbolTableBuildingTest.java create mode 100644 sonar-python-plugin/src/test/resources/org/sonar/plugins/python/indexer/v2/mod1.py create mode 100644 sonar-python-plugin/src/test/resources/org/sonar/plugins/python/indexer/v2/mod2.py create mode 100644 sonar-python-plugin/src/test/resources/org/sonar/plugins/python/indexer/v2/script.py diff --git a/python-frontend/src/main/java/org/sonar/python/semantic/ProjectLevelSymbolTable.java b/python-frontend/src/main/java/org/sonar/python/semantic/ProjectLevelSymbolTable.java index 96f933f951..b7ff5def40 100644 --- a/python-frontend/src/main/java/org/sonar/python/semantic/ProjectLevelSymbolTable.java +++ b/python-frontend/src/main/java/org/sonar/python/semantic/ProjectLevelSymbolTable.java @@ -24,6 +24,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; @@ -41,6 +42,11 @@ import org.sonar.python.index.Descriptor; import org.sonar.python.index.DescriptorUtils; import org.sonar.python.index.VariableDescriptor; +import org.sonar.python.semantic.v2.BasicTypeTable; +import org.sonar.python.semantic.v2.SymbolTableBuilderV2; +import org.sonar.python.semantic.v2.TypeInferenceV2; +import org.sonar.python.semantic.v2.UsageV2; +import org.sonar.python.semantic.v2.converter.PythonTypeToDescriptorConverter; import org.sonar.python.semantic.v2.typeshed.TypeShedDescriptorsProvider; import static org.sonar.python.tree.TreeUtils.getSymbolFromTree; @@ -48,8 +54,11 @@ public class ProjectLevelSymbolTable { + private final PythonTypeToDescriptorConverter pythonTypeToDescriptorConverter = new PythonTypeToDescriptorConverter(); private final Map> globalDescriptorsByModuleName; + private final Map> globalDescriptorsByModuleNameV2; private Map globalDescriptorsByFQN; + private Map globalDescriptorsByFQNV2; private final Set djangoViewsFQN = new HashSet<>(); private final Map> importsByModule = new HashMap<>(); private final Set projectBasePackages = new HashSet<>(); @@ -65,15 +74,19 @@ public static ProjectLevelSymbolTable from(Map> globalSymbol public ProjectLevelSymbolTable() { this.globalDescriptorsByModuleName = new HashMap<>(); + this.globalDescriptorsByModuleNameV2 = new HashMap<>(); } private ProjectLevelSymbolTable(Map> globalSymbolsByModuleName) { this.globalDescriptorsByModuleName = new HashMap<>(); + this.globalDescriptorsByModuleNameV2 = new HashMap<>(); globalSymbolsByModuleName.entrySet().forEach(entry -> { String moduleName = entry.getKey(); Set symbols = entry.getValue(); Set globalDescriptors = symbols.stream().map(DescriptorUtils::descriptor).collect(Collectors.toSet()); globalDescriptorsByModuleName.put(moduleName, globalDescriptors); + globalDescriptors = symbols.stream().map(DescriptorUtils::descriptor).collect(Collectors.toSet()); + globalDescriptorsByModuleNameV2.put(moduleName, globalDescriptors); }); } @@ -82,6 +95,7 @@ public void removeModule(String packageName, String fileName) { globalDescriptorsByModuleName.remove(fullyQualifiedModuleName); // ensure globalDescriptorsByFQN is re-computed this.globalDescriptorsByFQN = null; + this.globalDescriptorsByFQNV2 = null; } public void addModule(FileInput fileInput, String packageName, PythonFile pythonFile) { @@ -115,6 +129,7 @@ public void addModule(FileInput fileInput, String packageName, PythonFile python } DjangoViewsVisitor djangoViewsVisitor = new DjangoViewsVisitor(); fileInput.accept(djangoViewsVisitor); + addModuleV2(fileInput, packageName, pythonFile); } private void addModuleToGlobalSymbolsByFQN(Set descriptors) { @@ -122,7 +137,6 @@ private void addModuleToGlobalSymbolsByFQN(Set descriptors) { .filter(d -> d.fullyQualifiedName() != null) .collect(Collectors.toMap(Descriptor::fullyQualifiedName, Function.identity(), AmbiguousDescriptor::create)); globalDescriptorsByFQN.putAll(moduleDescriptorsByFQN); - } private Map globalDescriptorsByFQN() { @@ -169,6 +183,11 @@ public Set getDescriptorsFromModule(@Nullable String moduleName) { return globalDescriptorsByModuleName.get(moduleName); } + @CheckForNull + public Set getDescriptorsFromModuleV2(@Nullable String moduleName) { + return globalDescriptorsByModuleNameV2.get(moduleName); + } + public Map> importsByModule() { return Collections.unmodifiableMap(importsByModule); } @@ -201,6 +220,26 @@ public TypeShedDescriptorsProvider typeShedDescriptorsProvider() { return typeShedDescriptorsProvider; } + + private void addModuleV2(FileInput astRoot, String packageName, PythonFile pythonFile) { + var fullyQualifiedModuleName = SymbolUtils.fullyQualifiedModuleName(packageName, pythonFile.fileName()); + var symbolTable = new SymbolTableBuilderV2(astRoot).build(); + var typeInferenceV2 = new TypeInferenceV2(new BasicTypeTable(), pythonFile, symbolTable); + var typesBySymbol = typeInferenceV2.inferTypes(astRoot); + var moduleDescriptors = typesBySymbol.entrySet() + .stream() + .map(entry -> { + var descriptor = pythonTypeToDescriptorConverter.convert(fullyQualifiedModuleName, entry.getKey(), entry.getValue()); + return Map.entry(entry.getKey(), descriptor); + } + ) + .filter(entry -> !(!Objects.requireNonNull(entry.getValue().fullyQualifiedName()).startsWith(fullyQualifiedModuleName) + || entry.getKey().usages().stream().anyMatch(u -> u.kind().equals(UsageV2.Kind.IMPORT)))) + .map(Map.Entry::getValue) + .collect(Collectors.toSet()); + globalDescriptorsByModuleNameV2.put(fullyQualifiedModuleName, moduleDescriptors); + } + private class DjangoViewsVisitor extends BaseTreeVisitor { @Override public void visitCallExpression(CallExpression callExpression) { diff --git a/python-frontend/src/main/java/org/sonar/python/semantic/v2/ClassTypeBuilder.java b/python-frontend/src/main/java/org/sonar/python/semantic/v2/ClassTypeBuilder.java index f9f5229b25..97a61e60d0 100644 --- a/python-frontend/src/main/java/org/sonar/python/semantic/v2/ClassTypeBuilder.java +++ b/python-frontend/src/main/java/org/sonar/python/semantic/v2/ClassTypeBuilder.java @@ -65,7 +65,7 @@ public ClassTypeBuilder withDefinitionLocation(@Nullable LocationInFile definiti } public ClassTypeBuilder addSuperClass(PythonType type) { - superClasses.add(new LazyTypeWrapper(type)); + superClasses.add(TypeWrapper.of(type)); return this; } diff --git a/python-frontend/src/main/java/org/sonar/python/semantic/v2/SymbolsModuleTypeProvider.java b/python-frontend/src/main/java/org/sonar/python/semantic/v2/SymbolsModuleTypeProvider.java index 95600dcc91..c23604c24c 100644 --- a/python-frontend/src/main/java/org/sonar/python/semantic/v2/SymbolsModuleTypeProvider.java +++ b/python-frontend/src/main/java/org/sonar/python/semantic/v2/SymbolsModuleTypeProvider.java @@ -79,7 +79,7 @@ private static String getModuleFqnString(List moduleFqn) { } private Optional createModuleTypeFromProjectLevelSymbolTable(String moduleName, String moduleFqn, ModuleType parent) { - var retrieved = projectLevelSymbolTable.getDescriptorsFromModule(moduleFqn); + var retrieved = projectLevelSymbolTable.getDescriptorsFromModuleV2(moduleFqn); if (retrieved == null) { return Optional.empty(); } diff --git a/python-frontend/src/main/java/org/sonar/python/semantic/v2/converter/ClassDescriptorToPythonTypeConverter.java b/python-frontend/src/main/java/org/sonar/python/semantic/v2/converter/ClassDescriptorToPythonTypeConverter.java index c7f9dba1e2..d7ab3e695a 100644 --- a/python-frontend/src/main/java/org/sonar/python/semantic/v2/converter/ClassDescriptorToPythonTypeConverter.java +++ b/python-frontend/src/main/java/org/sonar/python/semantic/v2/converter/ClassDescriptorToPythonTypeConverter.java @@ -25,6 +25,7 @@ import org.sonar.python.types.v2.LazyTypeWrapper; import org.sonar.python.types.v2.Member; import org.sonar.python.types.v2.PythonType; +import org.sonar.python.types.v2.TypeWrapper; public class ClassDescriptorToPythonTypeConverter implements DescriptorToPythonTypeConverter { @@ -34,8 +35,13 @@ private static PythonType convert(ConversionContext ctx, ClassDescriptor from) { .withDefinitionLocation(from.definitionLocation()); from.superClasses().stream() - .map(fqn -> ctx.lazyTypesContext().getOrCreateLazyType(fqn)) - .map(LazyTypeWrapper::new) + .map(fqn -> { + if (fqn != null) { + return ctx.lazyTypesContext().getOrCreateLazyType(fqn); + } + return PythonType.UNKNOWN; + }) + .map(TypeWrapper::of) .forEach(typeBuilder::addSuperClass); var type = typeBuilder.build(); diff --git a/python-frontend/src/main/java/org/sonar/python/semantic/v2/converter/PythonTypeToDescriptorConverter.java b/python-frontend/src/main/java/org/sonar/python/semantic/v2/converter/PythonTypeToDescriptorConverter.java new file mode 100644 index 0000000000..c760747771 --- /dev/null +++ b/python-frontend/src/main/java/org/sonar/python/semantic/v2/converter/PythonTypeToDescriptorConverter.java @@ -0,0 +1,157 @@ +/* + * SonarQube Python Plugin + * Copyright (C) 2011-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.python.semantic.v2.converter; + +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import javax.annotation.CheckForNull; +import org.sonar.python.index.AmbiguousDescriptor; +import org.sonar.python.index.ClassDescriptor; +import org.sonar.python.index.Descriptor; +import org.sonar.python.index.FunctionDescriptor; +import org.sonar.python.index.VariableDescriptor; +import org.sonar.python.semantic.v2.SymbolV2; +import org.sonar.python.types.v2.ClassType; +import org.sonar.python.types.v2.FunctionType; +import org.sonar.python.types.v2.ParameterV2; +import org.sonar.python.types.v2.PythonType; +import org.sonar.python.types.v2.TypeWrapper; +import org.sonar.python.types.v2.UnionType; +import org.sonar.python.types.v2.UnknownType; + +public class PythonTypeToDescriptorConverter { + + public Descriptor convert(String moduleFqn, SymbolV2 symbol, Set types) { + var candidates = types.stream() + .map(type -> convert(moduleFqn, symbol.name(), type)) + .flatMap(candidate -> { + if (candidate instanceof AmbiguousDescriptor ambiguousDescriptor) { + return ambiguousDescriptor.alternatives().stream(); + } else { + return Stream.of(candidate); + } + }) + .collect(Collectors.toSet()); + + if (candidates.size() == 1) { + return candidates.iterator().next(); + } + + return new AmbiguousDescriptor(symbol.name(), symbolFqn(moduleFqn, symbol.name()), candidates); + } + + private Descriptor convert(String moduleFqn, String symbolName, PythonType type) { + if (type instanceof FunctionType functionType) { + return convert(moduleFqn, symbolName, functionType); + } + if (type instanceof ClassType classType) { + return convert(moduleFqn, symbolName, classType); + } + if (type instanceof UnionType unionType) { + return convert(moduleFqn, symbolName, unionType); + } + if (type instanceof UnknownType.UnresolvedImportType unresolvedImportType) { + return convert(moduleFqn, symbolName, unresolvedImportType); + } + return new VariableDescriptor(symbolName, symbolFqn(moduleFqn, symbolName), null); + } + + private Descriptor convert(String moduleFqn, String symbolName, FunctionType type) { + + var parameters = type.parameters() + .stream() + .map(parameter -> convert(moduleFqn, parameter)) + .toList(); + + return new FunctionDescriptor(symbolName, symbolFqn(moduleFqn, symbolName), + parameters, + type.isAsynchronous(), + type.isInstanceMethod(), + List.of(), + type.hasDecorators(), + type.definitionLocation().orElse(null), + null, + null + ); + } + + private Descriptor convert(String moduleFqn, String symbolName, ClassType type) { + Set memberDescriptors = type.members().stream().map(m -> convert(moduleFqn, m.name(), m.type())).collect(Collectors.toSet()); + List superClasses = type.superClasses().stream().map(TypeWrapper::type).map(t -> typeFqn(moduleFqn, t)).toList(); + return new ClassDescriptor(symbolName, symbolFqn(moduleFqn, symbolName), + superClasses, + memberDescriptors, + type.hasDecorators(), + type.definitionLocation().orElse(null), + false, + type.hasMetaClass(), + null, + false + ); + } + + private Descriptor convert(String moduleFqn, String symbolName, UnionType type) { + var candidates = type.candidates().stream() + .map(candidateType -> convert(moduleFqn, symbolName, candidateType)) + .collect(Collectors.toSet()); + return new AmbiguousDescriptor(symbolName, + symbolFqn(moduleFqn, symbolName), + candidates + ); + } + + private static Descriptor convert(String moduleFqn, String symbolName, UnknownType.UnresolvedImportType type) { + return new VariableDescriptor(symbolName, + symbolFqn(moduleFqn, symbolName), + type.importPath() + ); + } + + private FunctionDescriptor.Parameter convert(String moduleFqn, ParameterV2 parameter) { + var type = parameter.declaredType().type().unwrappedType(); + var annotatedType = typeFqn(moduleFqn, type); + + return new FunctionDescriptor.Parameter(parameter.name(), + annotatedType, + parameter.hasDefaultValue(), + parameter.isKeywordOnly(), + parameter.isPositionalOnly(), + parameter.isKeywordVariadic(), + parameter.isPositionalVariadic(), + parameter.location()); + } + + @CheckForNull + private static String typeFqn(String moduleFqn, PythonType type) { + if (type instanceof UnknownType.UnresolvedImportType importType) { + return importType.importPath(); + } else if (type instanceof ClassType classType) { + return moduleFqn + "." + classType.name(); + } + return null; + } + + private static String symbolFqn(String moduleFqn, String symbolName) { + return moduleFqn + "." + symbolName; + } + +} diff --git a/python-frontend/src/main/java/org/sonar/python/semantic/v2/types/TrivialTypeInferenceVisitor.java b/python-frontend/src/main/java/org/sonar/python/semantic/v2/types/TrivialTypeInferenceVisitor.java index 54473422de..c461fa0ca7 100644 --- a/python-frontend/src/main/java/org/sonar/python/semantic/v2/types/TrivialTypeInferenceVisitor.java +++ b/python-frontend/src/main/java/org/sonar/python/semantic/v2/types/TrivialTypeInferenceVisitor.java @@ -424,7 +424,7 @@ public void visitParameter(Parameter parameter) { } private static PythonType resolveTypeAnnotationExpressionType(Expression expression) { - if (expression instanceof Name name && !(name.typeV2() instanceof UnknownType)) { + if (expression instanceof Name name && name.typeV2() != PythonType.UNKNOWN) { return new ObjectType(name.typeV2(), TypeSource.TYPE_HINT); } else if (expression instanceof SubscriptionExpression subscriptionExpression && !(subscriptionExpression.object().typeV2() instanceof UnknownType)) { var candidateTypes = subscriptionExpression.subscripts() diff --git a/python-frontend/src/test/java/org/sonar/python/semantic/v2/converter/PythonTypeToDescriptorConverterTest.java b/python-frontend/src/test/java/org/sonar/python/semantic/v2/converter/PythonTypeToDescriptorConverterTest.java new file mode 100644 index 0000000000..ee0e6ba5c4 --- /dev/null +++ b/python-frontend/src/test/java/org/sonar/python/semantic/v2/converter/PythonTypeToDescriptorConverterTest.java @@ -0,0 +1,226 @@ +/* + * SonarQube Python Plugin + * Copyright (C) 2011-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.python.semantic.v2.converter; + +import java.util.List; +import java.util.Set; +import org.junit.jupiter.api.Test; +import org.sonar.plugins.python.api.LocationInFile; +import org.sonar.python.index.AmbiguousDescriptor; +import org.sonar.python.index.ClassDescriptor; +import org.sonar.python.index.Descriptor; +import org.sonar.python.index.FunctionDescriptor; +import org.sonar.python.index.VariableDescriptor; +import org.sonar.python.semantic.ProjectLevelSymbolTable; +import org.sonar.python.semantic.v2.LazyTypesContext; +import org.sonar.python.semantic.v2.ProjectLevelTypeTable; +import org.sonar.python.semantic.v2.SymbolV2; +import org.sonar.python.types.v2.ClassType; +import org.sonar.python.types.v2.FunctionType; +import org.sonar.python.types.v2.LazyType; +import org.sonar.python.types.v2.LazyUnionType; +import org.sonar.python.types.v2.Member; +import org.sonar.python.types.v2.ModuleType; +import org.sonar.python.types.v2.ObjectType; +import org.sonar.python.types.v2.ParameterV2; +import org.sonar.python.types.v2.TypeOrigin; +import org.sonar.python.types.v2.TypeWrapper; +import org.sonar.python.types.v2.UnionType; +import org.sonar.python.types.v2.UnknownType; + +import static org.assertj.core.api.Assertions.assertThat; + +class PythonTypeToDescriptorConverterTest { + + private final PythonTypeToDescriptorConverter converter = new PythonTypeToDescriptorConverter(); + private final TypeWrapper intTypeWrapper = TypeWrapper.of(new UnknownType.UnresolvedImportType("int")); + private final TypeWrapper floatTypeWrapper = TypeWrapper.of(new UnknownType.UnresolvedImportType("float")); + private final LocationInFile location = new LocationInFile("myFile", 1, 2, 3, 4); + + @Test + void testConvertFunctionTypeWithoutDecorator() { + ParameterV2 parameterV2 = new ParameterV2("param", TypeWrapper.of(intTypeWrapper.type()), false, true, false, false, false, location); + FunctionType functionType = new FunctionType("functionType", List.of(new ModuleType("bar")), List.of(parameterV2), floatTypeWrapper, TypeOrigin.LOCAL, true, false, true, false, null, location); + Descriptor descriptor = converter.convert("foo", new SymbolV2("myFunction"), Set.of(functionType)); + + assertThat(descriptor).isInstanceOf(FunctionDescriptor.class); + FunctionDescriptor functionDescriptor = (FunctionDescriptor) descriptor; + assertThat(functionDescriptor.name()).isEqualTo("myFunction"); + assertThat(functionDescriptor.fullyQualifiedName()).isEqualTo("foo.myFunction"); + assertThat(functionDescriptor.kind()).isEqualTo(Descriptor.Kind.FUNCTION); + assertThat(functionDescriptor.isAsynchronous()).isTrue(); + assertThat(functionDescriptor.isInstanceMethod()).isTrue(); + + // TODO SONARPY-2223 support for return type is missing in FunctionType + assertThat(functionDescriptor.annotatedReturnTypeName()).isNull(); + + // TODO SONARPY-2223 support for type annotation is missing in FunctionType + assertThat(functionDescriptor.typeAnnotationDescriptor()).isNull(); + + // TODO SONARPY-2223 support for decorators is missing in FunctionType + assertThat(functionDescriptor.hasDecorators()).isFalse(); + assertThat(functionDescriptor.decorators()).isEmpty(); + assertThat(functionDescriptor.definitionLocation()).isEqualTo(location); + + assertThat(functionDescriptor.parameters()).hasSize(1); + FunctionDescriptor.Parameter parameter = functionDescriptor.parameters().get(0); + assertThat(parameter.name()).isEqualTo("param"); + assertThat(parameter.annotatedType()).isEqualTo("int"); + assertThat(parameter.hasDefaultValue()).isFalse(); + assertThat(parameter.isKeywordOnly()).isTrue(); + assertThat(parameter.isPositionalOnly()).isFalse(); + assertThat(parameter.isKeywordVariadic()).isFalse(); + assertThat(parameter.isPositionalVariadic()).isFalse(); + assertThat(parameter.location()).isEqualTo(location); + } + + @Test + void testConvertClassType() { + ClassType classType = new ClassType("classType", Set.of(new Member("aMember", intTypeWrapper.type())), List.of(), List.of(floatTypeWrapper), List.of(intTypeWrapper.type()), true, location); + Descriptor descriptor = converter.convert("foo", new SymbolV2("myClass"), Set.of(classType)); + + assertThat(descriptor).isInstanceOf(ClassDescriptor.class); + ClassDescriptor classDescriptor = (ClassDescriptor) descriptor; + assertThat(classDescriptor.name()).isEqualTo("myClass"); + assertThat(classDescriptor.fullyQualifiedName()).isEqualTo("foo.myClass"); + assertThat(classDescriptor.kind()).isEqualTo(Descriptor.Kind.CLASS); + assertThat(classDescriptor.superClasses()).containsExactly("float"); + + assertThat(classDescriptor.members()).hasSize(1); + Descriptor memberDescriptor = classDescriptor.members().iterator().next(); + assertThat(memberDescriptor).isInstanceOf(VariableDescriptor.class); + VariableDescriptor memberVariableDescriptor = (VariableDescriptor) memberDescriptor; + assertThat(memberVariableDescriptor.name()).isEqualTo("aMember"); + assertThat(memberVariableDescriptor.annotatedType()).isEqualTo("int"); + // TODO SONARPY-2222 expected fullyqualified name of the member is "foo.myClass.aMember" + assertThat(memberVariableDescriptor.fullyQualifiedName()).isEqualTo("foo.aMember"); + + assertThat(classDescriptor.hasDecorators()).isTrue(); + assertThat(classDescriptor.definitionLocation()).isEqualTo(location); + assertThat(classDescriptor.hasMetaClass()).isTrue(); + + // TODO SONARPY-2222 support for superClass is missing in ClassType + assertThat(classDescriptor.hasSuperClassWithoutDescriptor()).isFalse(); + // TODO SONARPY-2222 support for metaclassFQN is missing in ClassType + assertThat(classDescriptor.metaclassFQN()).isNull(); + // TODO SONARPY-2222 support for generics is missing in ClassType + assertThat(classDescriptor.supportsGenerics()).isFalse(); + } + + @Test + void testConvertUnresolvedImportType() { + UnknownType.UnresolvedImportType unresolvedImportType = new UnknownType.UnresolvedImportType("anImport"); + Descriptor descriptor = converter.convert("foo", new SymbolV2("myImportedType"), Set.of(unresolvedImportType)); + + assertThat(descriptor).isInstanceOf(VariableDescriptor.class); + VariableDescriptor variableDescriptor = (VariableDescriptor) descriptor; + assertThat(variableDescriptor.name()).isEqualTo("myImportedType"); + assertThat(variableDescriptor.fullyQualifiedName()).isEqualTo("foo.myImportedType"); + assertThat(variableDescriptor.annotatedType()).isEqualTo("anImport"); + } + + @Test + void testConvertOtherType() { + LazyType lazyType = new LazyType("foo", new LazyTypesContext(new ProjectLevelTypeTable(ProjectLevelSymbolTable.empty()))); + Descriptor descriptor = converter.convert("foo", new SymbolV2("myLazySymbol"), Set.of(lazyType)); + assertThat(descriptor).isInstanceOf(VariableDescriptor.class); + VariableDescriptor variableDescriptor = (VariableDescriptor) descriptor; + assertThat(variableDescriptor.name()).isEqualTo("myLazySymbol"); + assertThat(variableDescriptor.fullyQualifiedName()).isEqualTo("foo.myLazySymbol"); + assertThat(variableDescriptor.annotatedType()).isNull(); + + ModuleType moduleType = new ModuleType("myModule"); + descriptor = converter.convert("foo", new SymbolV2("myModulSymbol"), Set.of(moduleType)); + assertThat(descriptor).isInstanceOf(VariableDescriptor.class); + variableDescriptor = (VariableDescriptor) descriptor; + assertThat(variableDescriptor.name()).isEqualTo("myModulSymbol"); + assertThat(variableDescriptor.fullyQualifiedName()).isEqualTo("foo.myModulSymbol"); + assertThat(variableDescriptor.annotatedType()).isNull(); + + ObjectType objectType = new ObjectType(lazyType); + descriptor = converter.convert("foo", new SymbolV2("myObjectSymbol"), Set.of(objectType)); + assertThat(descriptor).isInstanceOf(VariableDescriptor.class); + variableDescriptor = (VariableDescriptor) descriptor; + assertThat(variableDescriptor.name()).isEqualTo("myObjectSymbol"); + assertThat(variableDescriptor.fullyQualifiedName()).isEqualTo("foo.myObjectSymbol"); + assertThat(variableDescriptor.annotatedType()).isNull(); + + LazyUnionType lazyUnionType = new LazyUnionType(Set.of(lazyType, objectType)); + descriptor = converter.convert("foo", new SymbolV2("myLazyUnionSymbol"), Set.of(lazyUnionType)); + assertThat(descriptor).isInstanceOf(VariableDescriptor.class); + variableDescriptor = (VariableDescriptor) descriptor; + assertThat(variableDescriptor.name()).isEqualTo("myLazyUnionSymbol"); + assertThat(variableDescriptor.fullyQualifiedName()).isEqualTo("foo.myLazyUnionSymbol"); + assertThat(variableDescriptor.annotatedType()).isNull(); + } + + @Test + void testConvertUnionType() { + ClassType classType = new ClassType("classType", Set.of(new Member("aMember", intTypeWrapper.type())), List.of(), List.of(floatTypeWrapper), List.of(intTypeWrapper.type()), true, location); + ClassType anotherClassType = new ClassType("classType", Set.of(new Member("aMember", intTypeWrapper.type())), List.of(), List.of(floatTypeWrapper), List.of(intTypeWrapper.type()), true, location); + UnionType unionType = new UnionType(Set.of(classType, anotherClassType)); + Descriptor descriptor = converter.convert("foo", new SymbolV2("myUnionType"), Set.of(unionType)); + + assertThat(descriptor).isInstanceOf(AmbiguousDescriptor.class); + AmbiguousDescriptor ambiguousDescriptor = (AmbiguousDescriptor) descriptor; + assertThat(ambiguousDescriptor.name()).isEqualTo("myUnionType"); + assertThat(ambiguousDescriptor.fullyQualifiedName()).isEqualTo("foo.myUnionType"); + assertThat(ambiguousDescriptor.kind()).isEqualTo(Descriptor.Kind.AMBIGUOUS); + // TODO SONARPY-2225 the two class types in the union are rigorously the same but the converter creates an ambigouous symbol + assertThat(ambiguousDescriptor.alternatives()).hasSize(2); + assertThat(ambiguousDescriptor.alternatives()).extracting(Descriptor::name).containsExactlyInAnyOrder("myUnionType", "myUnionType"); + assertThat(ambiguousDescriptor.alternatives()).extracting(Object::getClass).allMatch(c -> c == ClassDescriptor.class); + } + + @Test + void testConvertManyTypes() { + ClassType classType = new ClassType("classType", Set.of(new Member("aMember", intTypeWrapper.type())), List.of(), List.of(floatTypeWrapper), List.of(intTypeWrapper.type()), true, location); + FunctionType functionType = new FunctionType("functionType", List.of(new ModuleType("bar")), List.of(), floatTypeWrapper, TypeOrigin.LOCAL, true, false, true, false, null, location); + Descriptor descriptor = converter.convert("foo", new SymbolV2("myUnionType"), Set.of(functionType, classType)); + + assertThat(descriptor).isInstanceOf(AmbiguousDescriptor.class); + AmbiguousDescriptor ambiguousDescriptor = (AmbiguousDescriptor) descriptor; + assertThat(ambiguousDescriptor.name()).isEqualTo("myUnionType"); + assertThat(ambiguousDescriptor.fullyQualifiedName()).isEqualTo("foo.myUnionType"); + assertThat(ambiguousDescriptor.kind()).isEqualTo(Descriptor.Kind.AMBIGUOUS); + assertThat(ambiguousDescriptor.alternatives()).hasSize(2); + assertThat(ambiguousDescriptor.alternatives()).extracting(Descriptor::kind).containsExactlyInAnyOrder(Descriptor.Kind.CLASS, Descriptor.Kind.FUNCTION); + } + + @Test + void testConvertManyTypesWithUnionType() { + ClassType classType = new ClassType("classType", Set.of(new Member("aMember", intTypeWrapper.type())), List.of(), List.of(floatTypeWrapper), List.of(intTypeWrapper.type()), true, location); + ClassType anotherClassType = new ClassType("classType", Set.of(new Member("aMember", intTypeWrapper.type())), List.of(), List.of(floatTypeWrapper), List.of(intTypeWrapper.type()), true, location); + + UnionType unionType = new UnionType(Set.of(classType, anotherClassType)); + Descriptor descriptor = converter.convert("foo", new SymbolV2("myUnionType"), Set.of(unionType, classType)); + + assertThat(descriptor).isInstanceOf(AmbiguousDescriptor.class); + AmbiguousDescriptor ambiguousDescriptor = (AmbiguousDescriptor) descriptor; + assertThat(ambiguousDescriptor.name()).isEqualTo("myUnionType"); + assertThat(ambiguousDescriptor.fullyQualifiedName()).isEqualTo("foo.myUnionType"); + assertThat(ambiguousDescriptor.kind()).isEqualTo(Descriptor.Kind.AMBIGUOUS); + // TODO SONARPY-2225 the two class types in the union are rigorously the same but the converter creates an ambigouous descriptor + assertThat(ambiguousDescriptor.alternatives()).hasSize(3); + assertThat(ambiguousDescriptor.alternatives()).extracting(Descriptor::name).allMatch(s -> s.equals("myUnionType")); + assertThat(ambiguousDescriptor.alternatives()).extracting(Object::getClass).allMatch(c -> c == ClassDescriptor.class); + } + +} diff --git a/sonar-python-plugin/src/test/java/org/sonar/plugins/python/indexer/SonarLintPythonIndexerProjectLevelSymbolTableBuildingTest.java b/sonar-python-plugin/src/test/java/org/sonar/plugins/python/indexer/SonarLintPythonIndexerProjectLevelSymbolTableBuildingTest.java new file mode 100644 index 0000000000..5aeb799734 --- /dev/null +++ b/sonar-python-plugin/src/test/java/org/sonar/plugins/python/indexer/SonarLintPythonIndexerProjectLevelSymbolTableBuildingTest.java @@ -0,0 +1,121 @@ +/* + * SonarQube Python Plugin + * Copyright (C) 2011-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.plugins.python.indexer; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.Set; +import java.util.stream.Stream; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.slf4j.event.Level; +import org.sonar.api.batch.fs.InputFile; +import org.sonar.api.batch.fs.internal.TestInputFileBuilder; +import org.sonar.api.batch.sensor.internal.SensorContextTester; +import org.sonar.api.testfixtures.log.LogTesterJUnit5; +import org.sonar.plugins.python.Python; +import org.sonar.plugins.python.PythonInputFile; +import org.sonar.plugins.python.PythonInputFileImpl; +import org.sonar.plugins.python.TestUtils; +import org.sonar.python.index.ClassDescriptor; +import org.sonar.python.index.Descriptor; +import org.sonar.python.index.FunctionDescriptor; +import org.sonar.python.semantic.ProjectLevelSymbolTable; + +import static org.assertj.core.api.Assertions.assertThat; + +class SonarLintPythonIndexerProjectLevelSymbolTableBuildingTest { + + private final File baseDir = new File("src/test/resources/org/sonar/plugins/python/indexer/v2").getAbsoluteFile(); + + @RegisterExtension + public LogTesterJUnit5 logTester = new LogTesterJUnit5().setLevel(Level.DEBUG); + + @Test + void single_file_simple_test() throws IOException { + var projectLevelSymbolTable = buildProjectLevelSymbolTable("script.py"); + assertThat(projectLevelSymbolTable.getDescriptorsFromModule("script")).hasSize(4); + Set moduleDescriptors = projectLevelSymbolTable.getDescriptorsFromModuleV2("script"); + assertThat(moduleDescriptors).hasSize(4); + + var aClassDescriptor = moduleDescriptors + .stream() + .filter(d -> d.name().equals("A")) + .findFirst() + .filter(ClassDescriptor.class::isInstance) + .map(ClassDescriptor.class::cast) + .orElse(null); + assertThat(aClassDescriptor).isNotNull(); + assertThat(aClassDescriptor.members()).hasSize(1); + assertThat(aClassDescriptor.superClasses()).containsOnly("script.Parent", "int"); + + var doSomethingDescriptor = aClassDescriptor.members() + .stream() + .filter(d -> d.name().equals("do_something")) + .findFirst() + .filter(FunctionDescriptor.class::isInstance) + .map(FunctionDescriptor.class::cast) + .orElse(null); + assertThat(doSomethingDescriptor).isNotNull(); + assertThat(doSomethingDescriptor.parameters()).hasSize(2); + } + + @Test + void multiple_files_simple_test() throws IOException { + var projectLevelSymbolTable = buildProjectLevelSymbolTable("mod1.py", "mod2.py"); + assertThat(projectLevelSymbolTable.getDescriptorsFromModule("mod1")).hasSize(1); + assertThat(projectLevelSymbolTable.getDescriptorsFromModuleV2("mod2")).hasSize(1); + } + + private ProjectLevelSymbolTable buildProjectLevelSymbolTable(String... files) throws IOException { + var context = SensorContextTester.create(baseDir); + var workDir = Files.createTempDirectory("workDir"); + context.fileSystem().setWorkDir(workDir); + var inputFiles = Stream.of(files) + .map(fileName -> inputFile(context, fileName)) + .toList(); + var moduleFileSystem = new TestModuleFileSystem(inputFiles); + var pythonIndexer = new SonarLintPythonIndexer(moduleFileSystem); + pythonIndexer.buildOnce(context); + return pythonIndexer.projectLevelSymbolTable(); + } + + private PythonInputFile inputFile(SensorContextTester context, String name) { + var inputFile = createInputFile(name); + context.fileSystem().add(inputFile.wrappedFile()); + return inputFile; + } + + private PythonInputFile createInputFile(String name) { + return createInputFile(name, Python.KEY); + } + + private PythonInputFile createInputFile(String name, String languageKey) { + return new PythonInputFileImpl(TestInputFileBuilder.create("moduleKey", name) + .setModuleBaseDir(baseDir.toPath()) + .setCharset(StandardCharsets.UTF_8) + .setType(InputFile.Type.MAIN) + .setLanguage(languageKey) + .initMetadata(TestUtils.fileContent(new File(baseDir, name), StandardCharsets.UTF_8)) + .build()); + } +} diff --git a/sonar-python-plugin/src/test/resources/org/sonar/plugins/python/indexer/v2/mod1.py b/sonar-python-plugin/src/test/resources/org/sonar/plugins/python/indexer/v2/mod1.py new file mode 100644 index 0000000000..52a001a63d --- /dev/null +++ b/sonar-python-plugin/src/test/resources/org/sonar/plugins/python/indexer/v2/mod1.py @@ -0,0 +1,2 @@ + +def func1(): ... diff --git a/sonar-python-plugin/src/test/resources/org/sonar/plugins/python/indexer/v2/mod2.py b/sonar-python-plugin/src/test/resources/org/sonar/plugins/python/indexer/v2/mod2.py new file mode 100644 index 0000000000..6cdf70b676 --- /dev/null +++ b/sonar-python-plugin/src/test/resources/org/sonar/plugins/python/indexer/v2/mod2.py @@ -0,0 +1,3 @@ +from typing import List + +def func2(a : List): ... diff --git a/sonar-python-plugin/src/test/resources/org/sonar/plugins/python/indexer/v2/script.py b/sonar-python-plugin/src/test/resources/org/sonar/plugins/python/indexer/v2/script.py new file mode 100644 index 0000000000..092f751147 --- /dev/null +++ b/sonar-python-plugin/src/test/resources/org/sonar/plugins/python/indexer/v2/script.py @@ -0,0 +1,15 @@ +def foo(): ... + +class Parent: + ... + +class A(Parent, int): + def do_something(self, a: int): + ... + +if something: + class B: + def method_one(self): ... +else: + class B: + def method_two(self): ... From 7b4204925ac4b5787cfb28a46a347a2794e1964c Mon Sep 17 00:00:00 2001 From: Maksim Grebeniuk <122789225+maksim-grebeniuk-sonarsource@users.noreply.github.com> Date: Fri, 25 Oct 2024 16:03:43 +0200 Subject: [PATCH 2/4] SONARPY-2256 Resolve fully qualified names for class members descriptors (#2104) --- .../PythonTypeToDescriptorConverter.java | 41 +++++++++++-------- .../PythonTypeToDescriptorConverterTest.java | 3 +- 2 files changed, 24 insertions(+), 20 deletions(-) diff --git a/python-frontend/src/main/java/org/sonar/python/semantic/v2/converter/PythonTypeToDescriptorConverter.java b/python-frontend/src/main/java/org/sonar/python/semantic/v2/converter/PythonTypeToDescriptorConverter.java index c760747771..d14bb8c2f4 100644 --- a/python-frontend/src/main/java/org/sonar/python/semantic/v2/converter/PythonTypeToDescriptorConverter.java +++ b/python-frontend/src/main/java/org/sonar/python/semantic/v2/converter/PythonTypeToDescriptorConverter.java @@ -42,7 +42,7 @@ public class PythonTypeToDescriptorConverter { public Descriptor convert(String moduleFqn, SymbolV2 symbol, Set types) { var candidates = types.stream() - .map(type -> convert(moduleFqn, symbol.name(), type)) + .map(type -> convert(moduleFqn, moduleFqn, symbol.name(), type)) .flatMap(candidate -> { if (candidate instanceof AmbiguousDescriptor ambiguousDescriptor) { return ambiguousDescriptor.alternatives().stream(); @@ -59,30 +59,30 @@ public Descriptor convert(String moduleFqn, SymbolV2 symbol, Set typ return new AmbiguousDescriptor(symbol.name(), symbolFqn(moduleFqn, symbol.name()), candidates); } - private Descriptor convert(String moduleFqn, String symbolName, PythonType type) { + private Descriptor convert(String moduleFqn, String parentFqn, String symbolName, PythonType type) { if (type instanceof FunctionType functionType) { - return convert(moduleFqn, symbolName, functionType); + return convert(moduleFqn, parentFqn, symbolName, functionType); } if (type instanceof ClassType classType) { - return convert(moduleFqn, symbolName, classType); + return convert(moduleFqn, parentFqn, symbolName, classType); } if (type instanceof UnionType unionType) { - return convert(moduleFqn, symbolName, unionType); + return convert(moduleFqn, parentFqn, symbolName, unionType); } if (type instanceof UnknownType.UnresolvedImportType unresolvedImportType) { - return convert(moduleFqn, symbolName, unresolvedImportType); + return convert(parentFqn, symbolName, unresolvedImportType); } - return new VariableDescriptor(symbolName, symbolFqn(moduleFqn, symbolName), null); + return new VariableDescriptor(symbolName, symbolFqn(parentFqn, symbolName), null); } - private Descriptor convert(String moduleFqn, String symbolName, FunctionType type) { + private Descriptor convert(String moduleFqn, String parentFqn, String symbolName, FunctionType type) { var parameters = type.parameters() .stream() .map(parameter -> convert(moduleFqn, parameter)) .toList(); - return new FunctionDescriptor(symbolName, symbolFqn(moduleFqn, symbolName), + return new FunctionDescriptor(symbolName, symbolFqn(parentFqn, symbolName), parameters, type.isAsynchronous(), type.isInstanceMethod(), @@ -91,13 +91,18 @@ private Descriptor convert(String moduleFqn, String symbolName, FunctionType typ type.definitionLocation().orElse(null), null, null - ); + ); } - private Descriptor convert(String moduleFqn, String symbolName, ClassType type) { - Set memberDescriptors = type.members().stream().map(m -> convert(moduleFqn, m.name(), m.type())).collect(Collectors.toSet()); + private Descriptor convert(String moduleFqn, String parentFqn, String symbolName, ClassType type) { + var symbolFqn = symbolFqn(parentFqn, symbolName); + var memberDescriptors = type.members() + .stream() + .map(m -> convert(moduleFqn, symbolFqn, m.name(), m.type())) + .collect(Collectors.toSet()); List superClasses = type.superClasses().stream().map(TypeWrapper::type).map(t -> typeFqn(moduleFqn, t)).toList(); - return new ClassDescriptor(symbolName, symbolFqn(moduleFqn, symbolName), + + return new ClassDescriptor(symbolName, symbolFqn, superClasses, memberDescriptors, type.hasDecorators(), @@ -109,9 +114,9 @@ private Descriptor convert(String moduleFqn, String symbolName, ClassType type) ); } - private Descriptor convert(String moduleFqn, String symbolName, UnionType type) { + private Descriptor convert(String moduleFqn, String parentFqn, String symbolName, UnionType type) { var candidates = type.candidates().stream() - .map(candidateType -> convert(moduleFqn, symbolName, candidateType)) + .map(candidateType -> convert(moduleFqn, parentFqn, symbolName, candidateType)) .collect(Collectors.toSet()); return new AmbiguousDescriptor(symbolName, symbolFqn(moduleFqn, symbolName), @@ -119,9 +124,9 @@ private Descriptor convert(String moduleFqn, String symbolName, UnionType type) ); } - private static Descriptor convert(String moduleFqn, String symbolName, UnknownType.UnresolvedImportType type) { + private static Descriptor convert(String parentFqn, String symbolName, UnknownType.UnresolvedImportType type) { return new VariableDescriptor(symbolName, - symbolFqn(moduleFqn, symbolName), + symbolFqn(parentFqn, symbolName), type.importPath() ); } @@ -135,8 +140,8 @@ private FunctionDescriptor.Parameter convert(String moduleFqn, ParameterV2 param parameter.hasDefaultValue(), parameter.isKeywordOnly(), parameter.isPositionalOnly(), - parameter.isKeywordVariadic(), parameter.isPositionalVariadic(), + parameter.isKeywordVariadic(), parameter.location()); } diff --git a/python-frontend/src/test/java/org/sonar/python/semantic/v2/converter/PythonTypeToDescriptorConverterTest.java b/python-frontend/src/test/java/org/sonar/python/semantic/v2/converter/PythonTypeToDescriptorConverterTest.java index ee0e6ba5c4..c6c03818a4 100644 --- a/python-frontend/src/test/java/org/sonar/python/semantic/v2/converter/PythonTypeToDescriptorConverterTest.java +++ b/python-frontend/src/test/java/org/sonar/python/semantic/v2/converter/PythonTypeToDescriptorConverterTest.java @@ -109,8 +109,7 @@ void testConvertClassType() { VariableDescriptor memberVariableDescriptor = (VariableDescriptor) memberDescriptor; assertThat(memberVariableDescriptor.name()).isEqualTo("aMember"); assertThat(memberVariableDescriptor.annotatedType()).isEqualTo("int"); - // TODO SONARPY-2222 expected fullyqualified name of the member is "foo.myClass.aMember" - assertThat(memberVariableDescriptor.fullyQualifiedName()).isEqualTo("foo.aMember"); + assertThat(memberVariableDescriptor.fullyQualifiedName()).isEqualTo("foo.myClass.aMember"); assertThat(classDescriptor.hasDecorators()).isTrue(); assertThat(classDescriptor.definitionLocation()).isEqualTo(location); From 371aa46a1b5f01be30828a0fc6979c52e5fe5359 Mon Sep 17 00:00:00 2001 From: Maksim Grebeniuk Date: Wed, 23 Oct 2024 17:05:27 +0200 Subject: [PATCH 3/4] SONARPY-2243 Make the V1 symbols populating out of descriptors collected by the V2 type inference --- ...incorrectExceptionTypeWithRegularImport.py | 2 +- .../semantic/ProjectLevelSymbolTable.java | 43 +++---------------- .../PythonTypeToDescriptorConverter.java | 17 ++++++-- .../semantic/ProjectLevelSymbolTableTest.java | 39 +++++++++-------- .../semantic/v2/TypeInferenceV2Test.java | 12 ++++++ 5 files changed, 53 insertions(+), 60 deletions(-) diff --git a/python-checks/src/test/resources/checks/incorrectExceptionType/incorrectExceptionTypeWithRegularImport.py b/python-checks/src/test/resources/checks/incorrectExceptionType/incorrectExceptionTypeWithRegularImport.py index 9731f52b15..5bce7ee229 100644 --- a/python-checks/src/test/resources/checks/incorrectExceptionType/incorrectExceptionTypeWithRegularImport.py +++ b/python-checks/src/test/resources/checks/incorrectExceptionType/incorrectExceptionTypeWithRegularImport.py @@ -38,7 +38,7 @@ def raise_nested_non_exception_class(): raise Enclsoing.Nested() # FN as only top-level imported symbols are considered def raise_RedefinedBaseExceptionChild(): - raise RedefinedBaseExceptionChild() # FN + raise RedefinedBaseExceptionChild() # Noncompliant def raise_ChildOfActualException(): raise ChildOfActualException() # OK diff --git a/python-frontend/src/main/java/org/sonar/python/semantic/ProjectLevelSymbolTable.java b/python-frontend/src/main/java/org/sonar/python/semantic/ProjectLevelSymbolTable.java index b7ff5def40..5fc9f7f73b 100644 --- a/python-frontend/src/main/java/org/sonar/python/semantic/ProjectLevelSymbolTable.java +++ b/python-frontend/src/main/java/org/sonar/python/semantic/ProjectLevelSymbolTable.java @@ -31,9 +31,7 @@ import javax.annotation.CheckForNull; import javax.annotation.Nullable; import org.sonar.plugins.python.api.PythonFile; -import org.sonar.plugins.python.api.symbols.AmbiguousSymbol; import org.sonar.plugins.python.api.symbols.Symbol; -import org.sonar.plugins.python.api.symbols.Usage; import org.sonar.plugins.python.api.tree.BaseTreeVisitor; import org.sonar.plugins.python.api.tree.CallExpression; import org.sonar.plugins.python.api.tree.FileInput; @@ -41,13 +39,13 @@ import org.sonar.python.index.AmbiguousDescriptor; import org.sonar.python.index.Descriptor; import org.sonar.python.index.DescriptorUtils; -import org.sonar.python.index.VariableDescriptor; import org.sonar.python.semantic.v2.BasicTypeTable; import org.sonar.python.semantic.v2.SymbolTableBuilderV2; import org.sonar.python.semantic.v2.TypeInferenceV2; import org.sonar.python.semantic.v2.UsageV2; import org.sonar.python.semantic.v2.converter.PythonTypeToDescriptorConverter; import org.sonar.python.semantic.v2.typeshed.TypeShedDescriptorsProvider; +import org.sonar.python.types.v2.UnknownType; import static org.sonar.python.tree.TreeUtils.getSymbolFromTree; import static org.sonar.python.tree.TreeUtils.nthArgumentOrKeyword; @@ -56,9 +54,7 @@ public class ProjectLevelSymbolTable { private final PythonTypeToDescriptorConverter pythonTypeToDescriptorConverter = new PythonTypeToDescriptorConverter(); private final Map> globalDescriptorsByModuleName; - private final Map> globalDescriptorsByModuleNameV2; private Map globalDescriptorsByFQN; - private Map globalDescriptorsByFQNV2; private final Set djangoViewsFQN = new HashSet<>(); private final Map> importsByModule = new HashMap<>(); private final Set projectBasePackages = new HashSet<>(); @@ -74,19 +70,15 @@ public static ProjectLevelSymbolTable from(Map> globalSymbol public ProjectLevelSymbolTable() { this.globalDescriptorsByModuleName = new HashMap<>(); - this.globalDescriptorsByModuleNameV2 = new HashMap<>(); } private ProjectLevelSymbolTable(Map> globalSymbolsByModuleName) { this.globalDescriptorsByModuleName = new HashMap<>(); - this.globalDescriptorsByModuleNameV2 = new HashMap<>(); globalSymbolsByModuleName.entrySet().forEach(entry -> { String moduleName = entry.getKey(); Set symbols = entry.getValue(); Set globalDescriptors = symbols.stream().map(DescriptorUtils::descriptor).collect(Collectors.toSet()); globalDescriptorsByModuleName.put(moduleName, globalDescriptors); - globalDescriptors = symbols.stream().map(DescriptorUtils::descriptor).collect(Collectors.toSet()); - globalDescriptorsByModuleNameV2.put(moduleName, globalDescriptors); }); } @@ -95,38 +87,13 @@ public void removeModule(String packageName, String fileName) { globalDescriptorsByModuleName.remove(fullyQualifiedModuleName); // ensure globalDescriptorsByFQN is re-computed this.globalDescriptorsByFQN = null; - this.globalDescriptorsByFQNV2 = null; } public void addModule(FileInput fileInput, String packageName, PythonFile pythonFile) { SymbolTableBuilder symbolTableBuilder = new SymbolTableBuilder(packageName, pythonFile); String fullyQualifiedModuleName = SymbolUtils.fullyQualifiedModuleName(packageName, pythonFile.fileName()); fileInput.accept(symbolTableBuilder); - Set globalDescriptors = new HashSet<>(); importsByModule.put(fullyQualifiedModuleName, symbolTableBuilder.importedModulesFQN()); - for (Symbol globalVariable : fileInput.globalVariables()) { - String fullyQualifiedVariableName = globalVariable.fullyQualifiedName(); - if (((fullyQualifiedVariableName != null) && !fullyQualifiedVariableName.startsWith(fullyQualifiedModuleName)) || - globalVariable.usages().stream().anyMatch(u -> u.kind().equals(Usage.Kind.IMPORT))) { - // TODO: We don't put builtin or imported names in global symbol table to avoid duplicate FQNs in project level symbol table (to fix with SONARPY-647) - continue; - } - if (globalVariable.is(Symbol.Kind.CLASS, Symbol.Kind.FUNCTION)) { - globalDescriptors.add(DescriptorUtils.descriptor(globalVariable)); - } else { - String fullyQualifiedName = fullyQualifiedModuleName + "." + globalVariable.name(); - if (globalVariable.is(Symbol.Kind.AMBIGUOUS)) { - globalDescriptors.add(DescriptorUtils.ambiguousDescriptor((AmbiguousSymbol) globalVariable, fullyQualifiedName)); - } else { - globalDescriptors.add(new VariableDescriptor(globalVariable.name(), fullyQualifiedName, globalVariable.annotatedTypeName())); - } - } - } - globalDescriptorsByModuleName.put(fullyQualifiedModuleName, globalDescriptors); - if (globalDescriptorsByFQN != null) { - // TODO: build globalSymbolsByFQN incrementally - addModuleToGlobalSymbolsByFQN(globalDescriptors); - } DjangoViewsVisitor djangoViewsVisitor = new DjangoViewsVisitor(); fileInput.accept(djangoViewsVisitor); addModuleV2(fileInput, packageName, pythonFile); @@ -136,7 +103,7 @@ private void addModuleToGlobalSymbolsByFQN(Set descriptors) { Map moduleDescriptorsByFQN = descriptors.stream() .filter(d -> d.fullyQualifiedName() != null) .collect(Collectors.toMap(Descriptor::fullyQualifiedName, Function.identity(), AmbiguousDescriptor::create)); - globalDescriptorsByFQN.putAll(moduleDescriptorsByFQN); + globalDescriptorsByFQN().putAll(moduleDescriptorsByFQN); } private Map globalDescriptorsByFQN() { @@ -185,7 +152,7 @@ public Set getDescriptorsFromModule(@Nullable String moduleName) { @CheckForNull public Set getDescriptorsFromModuleV2(@Nullable String moduleName) { - return globalDescriptorsByModuleNameV2.get(moduleName); + return globalDescriptorsByModuleName.get(moduleName); } public Map> importsByModule() { @@ -228,6 +195,7 @@ private void addModuleV2(FileInput astRoot, String packageName, PythonFile pytho var typesBySymbol = typeInferenceV2.inferTypes(astRoot); var moduleDescriptors = typesBySymbol.entrySet() .stream() + .filter(entry -> entry.getValue().stream().noneMatch(UnknownType.UnresolvedImportType.class::isInstance)) .map(entry -> { var descriptor = pythonTypeToDescriptorConverter.convert(fullyQualifiedModuleName, entry.getKey(), entry.getValue()); return Map.entry(entry.getKey(), descriptor); @@ -237,7 +205,8 @@ private void addModuleV2(FileInput astRoot, String packageName, PythonFile pytho || entry.getKey().usages().stream().anyMatch(u -> u.kind().equals(UsageV2.Kind.IMPORT)))) .map(Map.Entry::getValue) .collect(Collectors.toSet()); - globalDescriptorsByModuleNameV2.put(fullyQualifiedModuleName, moduleDescriptors); + globalDescriptorsByModuleName.put(fullyQualifiedModuleName, moduleDescriptors); + addModuleToGlobalSymbolsByFQN(moduleDescriptors); } private class DjangoViewsVisitor extends BaseTreeVisitor { diff --git a/python-frontend/src/main/java/org/sonar/python/semantic/v2/converter/PythonTypeToDescriptorConverter.java b/python-frontend/src/main/java/org/sonar/python/semantic/v2/converter/PythonTypeToDescriptorConverter.java index d14bb8c2f4..0b54660412 100644 --- a/python-frontend/src/main/java/org/sonar/python/semantic/v2/converter/PythonTypeToDescriptorConverter.java +++ b/python-frontend/src/main/java/org/sonar/python/semantic/v2/converter/PythonTypeToDescriptorConverter.java @@ -19,6 +19,7 @@ */ package org.sonar.python.semantic.v2.converter; +import java.util.ArrayList; import java.util.List; import java.util.Set; import java.util.stream.Collectors; @@ -34,7 +35,6 @@ import org.sonar.python.types.v2.FunctionType; import org.sonar.python.types.v2.ParameterV2; import org.sonar.python.types.v2.PythonType; -import org.sonar.python.types.v2.TypeWrapper; import org.sonar.python.types.v2.UnionType; import org.sonar.python.types.v2.UnknownType; @@ -100,14 +100,25 @@ private Descriptor convert(String moduleFqn, String parentFqn, String symbolName .stream() .map(m -> convert(moduleFqn, symbolFqn, m.name(), m.type())) .collect(Collectors.toSet()); - List superClasses = type.superClasses().stream().map(TypeWrapper::type).map(t -> typeFqn(moduleFqn, t)).toList(); + + var hasSuperClassWithoutDescriptor = false; + var superClasses = new ArrayList(); + for (var superClassWrapper : type.superClasses()) { + var superClass = superClassWrapper.type(); + if (superClass != PythonType.UNKNOWN) { + var superClassFqn = typeFqn(moduleFqn, superClass); + superClasses.add(superClassFqn); + } else { + hasSuperClassWithoutDescriptor = true; + } + } return new ClassDescriptor(symbolName, symbolFqn, superClasses, memberDescriptors, type.hasDecorators(), type.definitionLocation().orElse(null), - false, + hasSuperClassWithoutDescriptor, type.hasMetaClass(), null, false diff --git a/python-frontend/src/test/java/org/sonar/python/semantic/ProjectLevelSymbolTableTest.java b/python-frontend/src/test/java/org/sonar/python/semantic/ProjectLevelSymbolTableTest.java index 6bcb8c9933..ee87636166 100644 --- a/python-frontend/src/test/java/org/sonar/python/semantic/ProjectLevelSymbolTableTest.java +++ b/python-frontend/src/test/java/org/sonar/python/semantic/ProjectLevelSymbolTableTest.java @@ -20,7 +20,6 @@ package org.sonar.python.semantic; import com.google.common.base.Functions; -import com.google.protobuf.InvalidProtocolBufferException; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -29,9 +28,8 @@ import java.util.Map; import java.util.Set; import java.util.stream.Collectors; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; -import org.sonar.plugins.python.api.caching.PythonReadCache; -import org.sonar.plugins.python.api.caching.PythonWriteCache; import org.sonar.plugins.python.api.symbols.AmbiguousSymbol; import org.sonar.plugins.python.api.symbols.ClassSymbol; import org.sonar.plugins.python.api.symbols.FunctionSymbol; @@ -43,7 +41,6 @@ import org.sonar.plugins.python.api.tree.FunctionDef; import org.sonar.plugins.python.api.tree.ImportFrom; import org.sonar.plugins.python.api.tree.QualifiedExpression; -import org.sonar.plugins.python.api.tree.Statement; import org.sonar.plugins.python.api.tree.Tree; import org.sonar.python.PythonTestUtils; import org.sonar.python.index.AmbiguousDescriptor; @@ -54,6 +51,7 @@ import org.sonar.python.tree.TreeUtils; import org.sonar.python.types.DeclaredType; import org.sonar.python.types.InferredTypes; +import org.sonar.python.types.TypeShed; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.tuple; @@ -614,7 +612,7 @@ void function_symbols() { "fn = 42" ); globalSymbols = globalSymbols(tree, "mod"); - assertThat(globalSymbols).extracting(Symbol::kind).containsExactly(Symbol.Kind.AMBIGUOUS); + assertThat(globalSymbols).extracting(Symbol::kind).containsExactly(Symbol.Kind.OTHER); } @Test @@ -625,10 +623,11 @@ void redefined_class_symbol() { " pass"); Set globalSymbols = globalSymbols(fileInput, "mod"); assertThat(globalSymbols).extracting(Symbol::name).containsExactlyInAnyOrder("C"); - assertThat(globalSymbols).extracting(Symbol::kind).allSatisfy(k -> assertThat(Symbol.Kind.CLASS.equals(k)).isFalse()); + assertThat(globalSymbols).extracting(Symbol::kind).allSatisfy(k -> assertThat(k).isEqualTo(Symbol.Kind.CLASS)); } @Test + @Disabled("SONARPY-2248") void classdef_with_missing_symbol() { FileInput fileInput = parseWithoutSymbols( "class C: ", @@ -636,9 +635,7 @@ void classdef_with_missing_symbol() { "global C"); Set globalSymbols = globalSymbols(fileInput, "mod"); - assertThat(globalSymbols).extracting(Symbol::name).containsExactlyInAnyOrder("C"); - // TODO: Global statements should not alter the kind of a symbol - assertThat(globalSymbols).extracting(Symbol::kind).allSatisfy(k -> assertThat(Symbol.Kind.OTHER.equals(k)).isTrue()); + assertThat(globalSymbols).isNotEmpty(); } @Test @@ -663,16 +660,20 @@ void class_symbol() { assertThat(cSymbol.name()).isEqualTo("C"); assertThat(cSymbol.kind()).isEqualTo(Symbol.Kind.CLASS); assertThat(((ClassSymbol) cSymbol).superClasses()).hasSize(1); + } + @Test + @Disabled("SONARPY-2250") + void class_symbol_inheritance_from_nested_class() { // for the time being, we only consider symbols defined in the global scope - fileInput = parseWithoutSymbols( + var fileInput = parseWithoutSymbols( "class A:", " class A1: pass", "class C(A.A1): ", " pass"); - globalSymbols = globalSymbols(fileInput, "mod"); - symbols = globalSymbols.stream().collect(Collectors.toMap(Symbol::name, Functions.identity())); - cSymbol = symbols.get("C"); + var globalSymbols = globalSymbols(fileInput, "mod"); + var symbols = globalSymbols.stream().collect(Collectors.toMap(Symbol::name, Functions.identity())); + var cSymbol = symbols.get("C"); assertThat(cSymbol.name()).isEqualTo("C"); assertThat(cSymbol.kind()).isEqualTo(Symbol.Kind.CLASS); assertThat(((ClassSymbol) cSymbol).superClasses()).hasSize(1); @@ -710,7 +711,7 @@ void symbol_duplicated_by_wildcard_import() { "def nlargest(n, iterable, key=None): ..." ); Set globalSymbols = globalSymbols(tree, ""); - assertThat(globalSymbols).hasOnlyElementsOfType(AmbiguousSymbolImpl.class); + assertThat(globalSymbols).hasOnlyElementsOfType(FunctionSymbol.class); tree = parseWithoutSymbols( "nonlocal nlargest", @@ -727,12 +728,10 @@ void class_having_itself_as_superclass_should_not_trigger_error() { Set globalSymbols = globalSymbols(fileInput, "mod"); ClassSymbol a = (ClassSymbol) globalSymbols.iterator().next(); // SONARPY-1350: The parent "A" is not yet defined at the time it is read, so this is actually not correct - assertThat(a.superClasses()).containsExactly(a); - ClassDef classDef = (ClassDef) fileInput.statements().statements().get(0); - assertThat(TreeUtils.getParentClassesFQN(classDef)).containsExactly("mod.mod.A"); + assertThat(a.superClasses()).isEmpty(); + assertThat(a.hasUnresolvedTypeHierarchy()).isTrue(); } - @Test void class_having_another_class_with_same_name_should_not_trigger_error() { FileInput fileInput = parseWithoutSymbols( @@ -871,10 +870,12 @@ void class_with_method_parameter_of_same_type() { } @Test + @Disabled("SONARPY-2249") void no_stackoverflow_for_ambiguous_descriptor() { + TypeShed.resetBuiltinSymbols(); String[] foo = { "if cond:", - " Ambiguous = ...", + " Ambiguous = 41", "else:", " class Ambiguous(SomeParent):", " local_var = 'i'", diff --git a/python-frontend/src/test/java/org/sonar/python/semantic/v2/TypeInferenceV2Test.java b/python-frontend/src/test/java/org/sonar/python/semantic/v2/TypeInferenceV2Test.java index 75bb7a2531..b8eadd982e 100644 --- a/python-frontend/src/test/java/org/sonar/python/semantic/v2/TypeInferenceV2Test.java +++ b/python-frontend/src/test/java/org/sonar/python/semantic/v2/TypeInferenceV2Test.java @@ -2884,6 +2884,18 @@ class A: ... assertThat(typesBySymbol).isEmpty(); } + @Test + @Disabled("SONARPY-2248") + void typesBySymbol_global_statement() { + var typesBySymbol = inferTypesBySymbol(""" + class C: + pass + global C + """); + Assertions.assertThat(typesBySymbol).isNotEmpty(); + Assertions.assertThat(typesBySymbol.values().iterator().next()).isInstanceOf(ClassType.class); + } + private static Map> inferTypesBySymbol(String lines) { FileInput root = parse(lines); var symbolTable = new SymbolTableBuilderV2(root).build(); From 5b6c437062092f8bb941cd5683cc0d6ed4355294 Mon Sep 17 00:00:00 2001 From: Maksim Grebeniuk Date: Tue, 29 Oct 2024 09:20:13 +0100 Subject: [PATCH 4/4] fix --- .../v2/converter/PythonTypeToDescriptorConverter.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/python-frontend/src/main/java/org/sonar/python/semantic/v2/converter/PythonTypeToDescriptorConverter.java b/python-frontend/src/main/java/org/sonar/python/semantic/v2/converter/PythonTypeToDescriptorConverter.java index 0b54660412..c702ca073b 100644 --- a/python-frontend/src/main/java/org/sonar/python/semantic/v2/converter/PythonTypeToDescriptorConverter.java +++ b/python-frontend/src/main/java/org/sonar/python/semantic/v2/converter/PythonTypeToDescriptorConverter.java @@ -59,7 +59,7 @@ public Descriptor convert(String moduleFqn, SymbolV2 symbol, Set typ return new AmbiguousDescriptor(symbol.name(), symbolFqn(moduleFqn, symbol.name()), candidates); } - private Descriptor convert(String moduleFqn, String parentFqn, String symbolName, PythonType type) { + private static Descriptor convert(String moduleFqn, String parentFqn, String symbolName, PythonType type) { if (type instanceof FunctionType functionType) { return convert(moduleFqn, parentFqn, symbolName, functionType); } @@ -75,7 +75,7 @@ private Descriptor convert(String moduleFqn, String parentFqn, String symbolName return new VariableDescriptor(symbolName, symbolFqn(parentFqn, symbolName), null); } - private Descriptor convert(String moduleFqn, String parentFqn, String symbolName, FunctionType type) { + private static Descriptor convert(String moduleFqn, String parentFqn, String symbolName, FunctionType type) { var parameters = type.parameters() .stream() @@ -94,7 +94,7 @@ private Descriptor convert(String moduleFqn, String parentFqn, String symbolName ); } - private Descriptor convert(String moduleFqn, String parentFqn, String symbolName, ClassType type) { + private static Descriptor convert(String moduleFqn, String parentFqn, String symbolName, ClassType type) { var symbolFqn = symbolFqn(parentFqn, symbolName); var memberDescriptors = type.members() .stream() @@ -125,7 +125,7 @@ private Descriptor convert(String moduleFqn, String parentFqn, String symbolName ); } - private Descriptor convert(String moduleFqn, String parentFqn, String symbolName, UnionType type) { + private static Descriptor convert(String moduleFqn, String parentFqn, String symbolName, UnionType type) { var candidates = type.candidates().stream() .map(candidateType -> convert(moduleFqn, parentFqn, symbolName, candidateType)) .collect(Collectors.toSet()); @@ -142,7 +142,7 @@ private static Descriptor convert(String parentFqn, String symbolName, UnknownTy ); } - private FunctionDescriptor.Parameter convert(String moduleFqn, ParameterV2 parameter) { + private static FunctionDescriptor.Parameter convert(String moduleFqn, ParameterV2 parameter) { var type = parameter.declaredType().type().unwrappedType(); var annotatedType = typeFqn(moduleFqn, type);