Add an option to bypass the 10K Defect Limit during Report Generation

README.md

@@ -41,6 +41,7 @@ Generate editable reports for SonarQube projects.
  -e,--disable-spreadsheet          Disable spreadsheet generation.
  -f,--disable-csv                  Disable csv generation.
  -h,--help                         Display this message.
+ -i,--enableIssuesMultiRequests    Workaround SonarQube 10'000 issues limitation, by multiple requests.
  -l,--language <arg>               Language of the report. Values: en_US, fr_FR. Default: en_US.
  -m,--disable-markdown             Disable markdown generation.
  -n,--template-markdown <arg>      Path to the report template in markdown. Default: usage of internal template.
@@ -49,11 +50,11 @@ Generate editable reports for SonarQube projects.
  -r,--template-report <arg>        Path to the report template. Default: usage of internal template.
  -s,--server <arg>                 Complete URL of the targeted SonarQube server.
  -t,--token <arg>                  SonarQube "User token" of the SonarQube user who has permissions on the project.
+ -u,--maxUrlSize  <arg>            SonarQube WebAPI max URL text-size.
  -v,--version                      Display current version.
  -w,--disable-report               Disable report generation.
  -x,--template-spreadsheet <arg>   Path to the spreadsheet template. Default: usage of internal template.
 
-
 Please report issues at https://github.com/cnescatlab/sonar-cnes-report/issues
 ````
 

src/main/java/fr/cnes/sonar/plugin/ws/ExportTask.java

@@ -26,6 +26,7 @@
 import fr.cnes.sonar.report.exceptions.BadSonarQubeRequestException;
 import fr.cnes.sonar.report.exceptions.SonarQubeException;
 import fr.cnes.sonar.report.exceptions.UnknownQualityGateException;
+import fr.cnes.sonar.report.exceptions.UnsupportedSonarqubeResponseException;
 import fr.cnes.sonar.report.factory.ReportFactory;
 import fr.cnes.sonar.report.utils.StringManager;
 import org.apache.commons.io.FileUtils;
@@ -80,7 +81,8 @@ public class ExportTask implements RequestHandler {
      */
     @Override
     public void handle(Request request, Response response) throws BadExportationDataTypeException, IOException,
-            UnknownQualityGateException, OpenXML4JException, XmlException, SonarQubeException, ParseException {
+            UnknownQualityGateException, OpenXML4JException, XmlException, SonarQubeException, ParseException,
+            UnsupportedSonarqubeResponseException, Exception {
 
         // Get project key
         String projectKey = request.getParam(PluginStringManager.getProperty("api.report.args.key")).getValue();
@@ -116,6 +118,12 @@ public void handle(Request request, Response response) throws BadExportationData
 
             final Request.StringParam pEnableConf =
                     request.getParam(PluginStringManager.getProperty("api.report.args.enableConf"));
+
+            final Request.StringParam pEnableIssuesMultiRequests =
+                    request.getParam(PluginStringManager.getProperty("api.report.args.enableIssuesMultiRequests"));
+
+            final Request.StringParam pMaxUrlSize =
+                    request.getParam(PluginStringManager.getProperty("api.report.args.maxUrlSize"));
 
             // Build SonarQube local URL
             String port = config.get("sonar.web.port").orElse(PluginStringManager.getProperty("plugin.defaultPort"));
@@ -178,6 +186,20 @@ public void handle(Request request, Response response) throws BadExportationData
             if(pEnableConfValue != null && (pEnableConfValue.equals(FALSE) || pEnableConfValue.equals(NO))) {
                 reportParams.add("-c");
             }
+
+            // add param for Workaround SonarQube 10'000 issues limitation
+            {
+	            // use multiple requests
+            	final String pEnableIssuesMultiRequestsValue = pEnableIssuesMultiRequests.getValue();
+	            if((null != pEnableIssuesMultiRequestsValue) && (!pEnableIssuesMultiRequestsValue.equals(FALSE) && !pEnableIssuesMultiRequestsValue.equals(NO))) {
+	            	reportParams.add("-i");
+	            }
+
+	            // SonarQube WebAPI max URL text-size
+	            final String pMaxUrlSizeValue = pMaxUrlSize.getValue();
+	            reportParams.add("-u");
+	            reportParams.add(pMaxUrlSizeValue);
+            }
 
             // Execute report generation
             ReportCommandLine.execute(reportParams.toArray(new String[reportParams.size()]));
@@ -199,7 +221,7 @@ public void handle(Request request, Response response) throws BadExportationData
                 JsonWriter jsonWriter = new JsonWriter(writer);
             ) {
                 jsonWriter.beginObject();
-                jsonWriter.name("error").value(PluginStringManager.getProperty("api.tokenerror"));
+                jsonWriter.name("error").value(PluginStringManager.getProperty("api.exportError") +" "+ e.getMessage());
                 jsonWriter.endObject();
                 jsonWriter.flush();
             }

src/main/java/fr/cnes/sonar/plugin/ws/ReportWs.java

@@ -126,6 +126,20 @@ private void reportAction(final WebService.NewController controller){
         enableConfParam.setRequired(false);
         enableConfParam.setBooleanPossibleValues();
         enableConfParam.setDefaultValue(PluginStringManager.getProperty("api.report.args.defaultValue.enableConf"));
+
+        // Adding enableIssuesMultiRequests argument
+        WebService.NewParam enableIssuesMultiRequests = report.createParam(PluginStringManager.getProperty("api.report.args.enableIssuesMultiRequests"));
+        enableIssuesMultiRequests.setDescription(PluginStringManager.getProperty("api.report.args.description.enableIssuesMultiRequests"));
+        enableIssuesMultiRequests.setRequired(false);
+        enableIssuesMultiRequests.setBooleanPossibleValues();
+        enableIssuesMultiRequests.setDefaultValue(PluginStringManager.getProperty("api.report.args.defaultValue.enableIssuesMultiRequests"));
+
+        // Adding maxUrlSize argument
+        WebService.NewParam maxUrlSize = report.createParam(PluginStringManager.getProperty("api.report.args.maxUrlSize"));
+        maxUrlSize.setDescription(PluginStringManager.getProperty("api.report.args.description.maxUrlSize"));
+        maxUrlSize.setRequired(false);
+        maxUrlSize.setDefaultValue(PluginStringManager.getProperty("api.report.args.defaultValue.maxUrlSize"));
+        maxUrlSize.setExampleValue(PluginStringManager.getProperty("api.report.args.exampleValue.maxUrlSize"));
     }
 }
 

src/main/java/fr/cnes/sonar/report/ReportCommandLine.java

@@ -32,6 +32,7 @@
 import fr.cnes.sonar.report.exceptions.BadSonarQubeRequestException;
 import fr.cnes.sonar.report.exceptions.SonarQubeException;
 import fr.cnes.sonar.report.exceptions.UnknownQualityGateException;
+import fr.cnes.sonar.report.exceptions.UnsupportedSonarqubeResponseException;
 import fr.cnes.sonar.report.factory.ProviderFactory;
 import fr.cnes.sonar.report.factory.ReportFactory;
 import fr.cnes.sonar.report.factory.ReportModelFactory;
@@ -80,17 +81,15 @@ public static void main(final String[] args)  {
             // We use different method because it can be called outside main (for example, in from ReportSonarPlugin)
             execute(args);
 
-        } catch (BadExportationDataTypeException | BadSonarQubeRequestException | IOException |
-                UnknownQualityGateException | OpenXML4JException | XmlException | SonarQubeException |
-                IllegalStateException | IllegalArgumentException | ParseException e) {
+        } catch (Exception e) {
             // it logs all the stack trace
             LOGGER.log(Level.SEVERE, e.getMessage(), e);
             System.exit(-1);
         }
     }
 
     public static void execute(final String[] args) throws BadExportationDataTypeException , BadSonarQubeRequestException , IOException,
-    UnknownQualityGateException, OpenXML4JException, XmlException, SonarQubeException, ParseException {
+    UnknownQualityGateException, OpenXML4JException, XmlException, SonarQubeException, ParseException, UnsupportedSonarqubeResponseException, Exception {
         // Log message.
         String message;
 
@@ -116,8 +115,9 @@ public static void execute(final String[] args) throws BadExportationDataTypeExc
 
         // Instantiate a ProviderFactory depending on the execution mode of the application
         ProviderFactory providerFactory;
-        providerFactory = new StandaloneProviderFactory(url, conf.getToken(), conf.getProject(), conf.getBranch());
-
+        providerFactory = new StandaloneProviderFactory(url, conf.getToken(), 
+        		conf.getProject(), conf.getBranch(), 
+        		conf.isEnableIssuesMultiRequests(), conf.getMaxUrlSize());
 
         // Initialize connexion with SonarQube and retrieve primitive information
         final SonarQubeServer server = new ServerFactory(url, providerFactory).create();

src/main/java/fr/cnes/sonar/report/exceptions/UnsupportedSonarqubeResponseException.java

@@ -0,0 +1,32 @@
+/*
+ * This file is part of cnesreport.
+ *
+ * cnesreport is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * cnesreport is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with cnesreport.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package fr.cnes.sonar.report.exceptions;
+
+/**
+ * Thrown when a request is not recognize by SonarQube
+ */
+public class UnsupportedSonarqubeResponseException extends Exception {
+
+    /**
+     * Constructor
+     * @param message the text to print (exception's details)
+     */
+    public UnsupportedSonarqubeResponseException(final String message) {
+        super(message);
+    }
+}

src/main/java/fr/cnes/sonar/report/factory/ReportModelFactory.java

@@ -17,10 +17,16 @@
 
 package fr.cnes.sonar.report.factory;
 
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
 import fr.cnes.sonar.report.exceptions.BadSonarQubeRequestException;
 import fr.cnes.sonar.report.exceptions.SonarQubeException;
 import fr.cnes.sonar.report.exceptions.UnknownQualityGateException;
+import fr.cnes.sonar.report.exceptions.UnsupportedSonarqubeResponseException;
 import fr.cnes.sonar.report.model.Components;
+import fr.cnes.sonar.report.model.Issue;
 import fr.cnes.sonar.report.model.Report;
 import fr.cnes.sonar.report.providers.component.ComponentProvider;
 import fr.cnes.sonar.report.providers.facets.FacetsProvider;
@@ -79,8 +85,10 @@ public ReportModelFactory(final String project, final String branch, final Strin
      * @throws BadSonarQubeRequestException when a request to the server is not well-formed
      * @throws UnknownQualityGateException a quality gate is not correct
      * @throws SonarQubeException When an error occurred from SonarQube server.
+     * @throws Exception 
+     * @throws UnsupportedSonarqubeResponseException
      */
-    public Report create() throws BadSonarQubeRequestException, UnknownQualityGateException, SonarQubeException {
+    public Report create() throws BadSonarQubeRequestException, UnknownQualityGateException, SonarQubeException, UnsupportedSonarqubeResponseException, Exception {
         // the new report to return
         final Report report = new Report();
 
@@ -118,9 +126,18 @@ public Report create() throws BadSonarQubeRequestException, UnknownQualityGateEx
         report.setAnalysisDate(report.getProject().getAnalysisDate());
         report.truncateAnalysisDate();
         // formatted issues, unconfirmed issues and raw issues' setting
-        report.setIssues(issuesProvider.getIssues());
-        report.setUnconfirmed(issuesProvider.getUnconfirmedIssues());
-        report.setRawIssues(issuesProvider.getRawIssues());
+        {
+        	final List<Issue> listOfConfirmedIssues = new ArrayList<Issue>();
+        	final List<Issue> listOfUnconfirmedIssues = new ArrayList<Issue>();
+        	final List<Map<String, String>> listOfMapOfIssues = new ArrayList<Map<String, String>>();
+
+        	issuesProvider.getIssuesStructures("false", listOfConfirmedIssues, null                   , listOfMapOfIssues);
+        	issuesProvider.getIssuesStructures("true",  null                 , listOfUnconfirmedIssues, null);
+
+        	report.setIssues(listOfConfirmedIssues);
+        	report.setUnconfirmed(listOfUnconfirmedIssues);
+        	report.setRawIssues(listOfMapOfIssues);
+        }
         // facets's setting
         report.setFacets(facetsProvider.getFacets());
         report.setTimeFacets(facetsProvider.getTimeFacets());

src/main/java/fr/cnes/sonar/report/factory/StandaloneProviderFactory.java

@@ -58,35 +58,49 @@ public class StandaloneProviderFactory implements ProviderFactory {
     /**
      * Branch of the project
      */
-    private String branch;    
+    private String branch;
+    /**
+     * workaround SonarQube 10'000 issues limitation, by multiple requests
+     */
+    private boolean enableIssuesMultiRequests;
+    /**
+     * SonarQube WebAPI max URL text-size
+     */
+    private int maxUrlSize;
 
     /**
      * Constructor.
      * @param server SonarQube server.
      * @param token User's token.
      * @param project Project's id.
      * @param branch Project's branch.
+     * @param pEnableIssuesMultiRequests Workaround SonarQube 10'000 issues limitation, by multiple requests.
+     * @param pMaxUrlSize                SonarQube WebAPI max URL text-size.
     */
-	public StandaloneProviderFactory(String server, String token, String project, String branch){
+	public StandaloneProviderFactory(String server, String token, 
+			String project, String branch, 
+			boolean enableIssuesMultiRequests, int maxUrlSize){
 		this.server = server;
 		this.token = token;
 		this.project = project;
 		this.branch = branch;
+		this.enableIssuesMultiRequests = enableIssuesMultiRequests;
+		this.maxUrlSize = maxUrlSize;
 	}
 
     @Override
     public ComponentProvider createComponentProvider() {
-        return new ComponentProviderStandalone(this.server, this.token, this.project, this.branch);
+        return new ComponentProviderStandalone(this.server, this.token, this.project, this.branch, this.enableIssuesMultiRequests, this.maxUrlSize);
     }
 
     @Override
     public FacetsProvider createFacetsProvider() {
-        return new FacetsProviderStandalone(this.server, this.token, this.project, this.branch);
+        return new FacetsProviderStandalone(this.server, this.token, this.project, this.branch, this.enableIssuesMultiRequests, this.maxUrlSize);
     }
 
     @Override
     public IssuesProvider createIssuesProvider() {
-        return new IssuesProviderStandalone(this.server, this.token, this.project, this.branch);
+        return new IssuesProviderStandalone(this.server, this.token, this.project, this.branch, this.enableIssuesMultiRequests, this.maxUrlSize);
     }
 
     @Override
@@ -96,17 +110,17 @@ public LanguageProvider createLanguageProvider() {
 
     @Override
     public MeasureProvider createMeasureProvider() {
-        return new MeasureProviderStandalone(this.server, this.token, this.project, this.branch);
+        return new MeasureProviderStandalone(this.server, this.token, this.project, this.branch, this.enableIssuesMultiRequests, this.maxUrlSize);
     }
 
     @Override
     public ProjectProvider createProjectProvider() {
-        return new ProjectProviderStandalone(this.server, this.token, this.project, this.branch, createLanguageProvider());
+        return new ProjectProviderStandalone(this.server, this.token, this.project, this.branch, this.enableIssuesMultiRequests, this.maxUrlSize, createLanguageProvider());
     }
 
     @Override
     public QualityGateProvider createQualityGateProvider() {
-        return new QualityGateProviderStandalone(this.server, this.token, this.project, this.branch);
+        return new QualityGateProviderStandalone(this.server, this.token, this.project, this.branch, this.enableIssuesMultiRequests, this.maxUrlSize);
     }
 
     @Override
@@ -116,7 +130,7 @@ public QualityProfileProvider createQualityProfileProvider() {
 
     @Override
     public SecurityHotspotsProvider createSecurityHotspotsProvider() {
-        return new SecurityHotspotsProviderStandalone(this.server, this.token, this.project, this.branch);
+        return new SecurityHotspotsProviderStandalone(this.server, this.token, this.project, this.branch, this.enableIssuesMultiRequests, this.maxUrlSize);
     }
 
     @Override

src/main/java/fr/cnes/sonar/report/providers/AbstractDataProvider.java

@@ -54,6 +54,10 @@ public abstract class AbstractDataProvider {
      *  Name of the request for getting a specific project
      */
     protected static final String GET_PROJECT_REQUEST = "GET_PROJECT_REQUEST";
+    /**
+     * Parameter corresponding to request all Files, when using WEB-API
+     */
+    protected static final String ALLFILES = "";
     /**
      * Field to search in json to get the paging section
      */
@@ -115,6 +119,16 @@ public abstract class AbstractDataProvider {
      */
     protected String branch;
 
+    /**
+     * workaround SonarQube 10'000 issues limitation, by multiple requests
+     */
+    protected boolean enableIssuesMultiRequests;
+
+    /**
+     * SonarQube WebAPI max URL text-size
+     */
+    protected int maxUrlSize;
+
     /**
      * Client to talk with sonarqube's services
      */
@@ -159,8 +173,12 @@ public abstract class AbstractDataProvider {
      * @param token String representing the user token.
      * @param project The id of the project to report.
      * @param branch The branch of the project to report.
+     * @param pEnableIssuesMultiRequests Workaround SonarQube 10'000 issues limitation, by multiple requests.
+     * @param pMaxUrlSize                SonarQube WebAPI max URL text-size.
      */
-    protected AbstractDataProvider(final String server, final String token, final String project, final String branch) {
+    protected AbstractDataProvider(final String server, final String token, 
+    		                       final String project, final String branch, 
+    		                       final boolean enableIssuesMultiRequests, final int maxUrlSize) {
         // json tool
         this.gson = new Gson();
         // get sonar server
@@ -171,6 +189,9 @@ protected AbstractDataProvider(final String server, final String token, final St
         this.projectKey = project;
         // get branch
         this.branch = branch;
+
+        this.enableIssuesMultiRequests = enableIssuesMultiRequests;
+        this.maxUrlSize = maxUrlSize;
     }
 
     /**
@@ -281,7 +302,8 @@ private void isErrorFree(final JsonObject jsonObject) throws BadSonarQubeRequest
      */
     public JsonObject request(final String request)
             throws BadSonarQubeRequestException, SonarQubeException {
-        // do the request to the server and return a string answer
+
+    	// do the request to the server and return a string answer
         final String raw = stringRequest(request);
 
         // prepare json

src/main/java/fr/cnes/sonar/report/providers/component/AbstractComponentProvider.java

@@ -51,10 +51,13 @@ public abstract class AbstractComponentProvider extends AbstractDataProvider {
      * @param pToken String representing the user token.
      * @param pProject The id of the project to report.
      * @param pBranch The branch of the project to report.
+     * @param pEnableIssuesMultiRequests Workaround SonarQube 10'000 issues limitation, by multiple requests.
+     * @param pMaxUrlSize                SonarQube WebAPI max URL text-size.
      */
-    protected AbstractComponentProvider(final String pServer, final String pToken, final String pProject,
-            final String pBranch) {
-        super(pServer, pToken, pProject, pBranch);
+    protected AbstractComponentProvider(final String pServer, final String pToken, 
+    		final String pProject, final String pBranch, 
+            final boolean pEnableIssuesMultiRequests, final int pMaxUrlSize) {
+        super(pServer, pToken, pProject, pBranch, pEnableIssuesMultiRequests, pMaxUrlSize);
     }
 
     /**