diff --git a/docs/docs/en/guide/task/java.md b/docs/docs/en/guide/task/java.md index cf6702ce3b98..6ff23d9d4357 100644 --- a/docs/docs/en/guide/task/java.md +++ b/docs/docs/en/guide/task/java.md @@ -1,6 +1,6 @@ # Overview -This node is for executing java-type tasks and supports using files and jar packages as program entries. +This node is used to execute tasks of the `Java` type and supports running `jar` packages of the `FAT_JAR` and `NORMAL_JAR` types. # Create Tasks @@ -20,24 +20,38 @@ This node is for executing java-type tasks and supports using files and jar pack | Module Path | pick Java 9 + 's modularity feature, put all resources into-module-path, and require that the JDK version in your worker supports modularity. | | Main Parameter | Java program main method entry parameter. | | Java VM Parameters | JVM startup parameters. | -| Script | You need to write Java code if you use the Java run type. The public class must exist in the code without writing a package statement. | +| Main Class Name | Fully Qualified Name of the Main Class (Optional) | +| Main Package | Select the main program package to run the application. | | Resources | External JAR packages or other resource files that are added to the classpath or module path and can be easily retrieved in your JAVA script. | ## Example -Java type tasks have two modes of execution, here is a demonstration of executing tasks in Java mode. +There are two execution modes for Java task types, which will be demonstrated separately here. The main configuration parameters are as follows: - Run Type - Module Path - Main Parameters - Java VM Parameters -- Script +- Main Package +- Resources -![java_task](../../../../img/tasks/demo/java_task02.png) +As shown in the figure. + +- FAT_JAR + +![java_task](../../../../img/tasks/demo/java_fat.png) + +`FAT_JAR` is also known as `uber-jar`, where the dependencies and code are contained within the same `jar`. You only need to select this one `jar`. + +- NORMAL_JAR + +![java_task](../../../../img/tasks/demo/java_normal.png) + +`normal1.jar` is the entry point of the program, and `normal2.jar` contains the required dependencies. The entry point of the program must be specified in the main program package, and all the dependencies along with the program entry `jar` files should be selected in the resource file to ensure correct execution. ## Note -When you run the task in JAVA execution mode, the public class must exist in the code, and you could omit writing a package statement. +1. For security reasons, when executing JAVA tasks, please use the environment management module to configure the JDK environment, such as `JAVA_HOME` and other environment variables. +2. NORMAL_JAR should provide the main class name (optional), while FAT_JAR does not need to provide the main class name. -For security reasons, when executing JAVA tasks, please use the environment management module to configure the JDK environment, such as `JAVA_HOME` and other environment variables. diff --git a/docs/docs/zh/guide/task/java.md b/docs/docs/zh/guide/task/java.md index 3f29741acff9..36d9d155ed59 100644 --- a/docs/docs/zh/guide/task/java.md +++ b/docs/docs/zh/guide/task/java.md @@ -2,7 +2,7 @@ ## 综述 -该节点用于执行 java 类型的任务,支持使用单文件和jar包作为程序入口。 +该节点用于执行`Java`类型的任务,支持使用`FAT_JAR`类型和`NORMAL_JAR`类型的`jar`包运行。 ## 创建任务 @@ -22,12 +22,13 @@ | 模块路径 | 开启使用JAVA9+的模块化特性,把所有资源放入--module-path中,要求您的worker中的JDK版本支持模块化 | | 主程序参数 | 作为普通Java程序main方法入口参数 | | 虚拟机参数 | 配置启动虚拟机参数 | -| 脚本 | 若使用JAVA运行类型则需要编写JAVA代码。代码中必须存在public类,不用写package语句 | +| 主类名 | 启动类的完整主类名(可选) | +| 主程序包 | 选择要运行程序的主程序包 | | 资源 | 可以是外部JAR包也可以是其他资源文件,它们都会被加入到类路径或模块路径中,您可以在自己的JAVA脚本中轻松获取 | ## 任务样例 -java任务类型有两种运行模式,这里以JAVA模式为例进行演示。 +java任务类型有两种运行模式,这里进行分别进行演示。 主要配置参数如下: @@ -35,12 +36,25 @@ java任务类型有两种运行模式,这里以JAVA模式为例进行演示。 - 模块路径 - 主程序参数 - 虚拟机参数 -- 脚本文件 +- 主程序包 +- 资源文件 -![java_task](../../../../img/tasks/demo/java_task02.png) +如图所示 + +- FAT_JAR类型 + +![java_task](../../../../img/tasks/demo/java_fat.png) + +`FAT_JAR`即`uber-jar`,依赖和代码都在同一个`jar`中,只需选择这一个`jar`即可 + +- NORMAL_JAR类型 + +![java_task](../../../../img/tasks/demo/java_normal.png) + +`normal1.jar`是程序运行的入口,`normal2.jar`是所需的依赖,需要用主程序包指定程序的入口,并且在资源文件中选择所有依赖和程序入口的`jar`文件,才能正确运行 ## 注意事项 -使用JAVA运行类型时代码中必须存在public类,可以不写package语句 +1. 基于安全原因,执行JAVA任务时,请使用环境管理功能配置JDK环境,例如`JAVA_HOME`等环境变量 +2. `NORMAL_JAR` 应该提供主类名(可选),`FAT_JAR` 不需要提供主类名。 -基于安全原因,执行JAVA任务时,请使用环境管理功能配置JDK环境,例如`JAVA_HOME`等环境变量 diff --git a/docs/img/tasks/demo/java_fat.png b/docs/img/tasks/demo/java_fat.png new file mode 100644 index 000000000000..61083f91fa33 Binary files /dev/null and b/docs/img/tasks/demo/java_fat.png differ diff --git a/docs/img/tasks/demo/java_normal.png b/docs/img/tasks/demo/java_normal.png new file mode 100644 index 000000000000..3a2145a0ecac Binary files /dev/null and b/docs/img/tasks/demo/java_normal.png differ diff --git a/docs/img/tasks/demo/java_task02.png b/docs/img/tasks/demo/java_task02.png deleted file mode 100644 index 28cc4beb280e..000000000000 Binary files a/docs/img/tasks/demo/java_task02.png and /dev/null differ diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/WorkflowJavaTaskE2ETest.java b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/WorkflowJavaTaskE2ETest.java index 043db5c0d89d..8d7a662ad2a5 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/WorkflowJavaTaskE2ETest.java +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/WorkflowJavaTaskE2ETest.java @@ -19,6 +19,7 @@ import static org.assertj.core.api.Assertions.assertThat; +import org.apache.dolphinscheduler.e2e.core.Constants; import org.apache.dolphinscheduler.e2e.core.DolphinScheduler; import org.apache.dolphinscheduler.e2e.core.WebDriverWaitFactory; import org.apache.dolphinscheduler.e2e.pages.LoginPage; @@ -29,11 +30,35 @@ import org.apache.dolphinscheduler.e2e.pages.project.workflow.WorkflowForm; import org.apache.dolphinscheduler.e2e.pages.project.workflow.WorkflowInstanceTab; import org.apache.dolphinscheduler.e2e.pages.project.workflow.task.JavaTaskForm; +import org.apache.dolphinscheduler.e2e.pages.resource.FileManagePage; +import org.apache.dolphinscheduler.e2e.pages.resource.ResourcePage; import org.apache.dolphinscheduler.e2e.pages.security.EnvironmentPage; import org.apache.dolphinscheduler.e2e.pages.security.SecurityPage; import org.apache.dolphinscheduler.e2e.pages.security.TenantPage; import org.apache.dolphinscheduler.e2e.pages.security.UserPage; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.jar.Attributes; +import java.util.jar.JarEntry; +import java.util.jar.JarOutputStream; +import java.util.jar.Manifest; + +import javax.tools.JavaCompiler; +import javax.tools.JavaFileObject; +import javax.tools.StandardJavaFileManager; +import javax.tools.ToolProvider; + +import lombok.extern.slf4j.Slf4j; + import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Order; @@ -42,16 +67,20 @@ import org.openqa.selenium.By; import org.openqa.selenium.remote.RemoteWebDriver; import org.openqa.selenium.support.ui.ExpectedConditions; +import org.openqa.selenium.support.ui.WebDriverWait; import org.testcontainers.shaded.org.awaitility.Awaitility; @DolphinScheduler(composeFiles = "docker/basic/docker-compose.yaml") @DisableIfTestFails +@Slf4j public class WorkflowJavaTaskE2ETest { - private static final String project = "test-workflow-1"; + private static final String project = "test-workflow"; private static final String workflow = "test-workflow-1"; + private static final String workflow2 = "test-workflow-2"; + private static final String user = "admin"; private static final String password = "dolphinscheduler123"; @@ -70,16 +99,99 @@ public class WorkflowJavaTaskE2ETest { private static final String environmentWorkerGroup = "default"; - private static final String javaContent = "public class Test {" + - " public static void main(String[] args) {" + - " System.out.println(\"hello world\");" + - " }" + - "}"; + private static final String filePath = Constants.HOST_TMP_PATH.toString(); private static RemoteWebDriver browser; + private static void createJar(String className, String classFilePath, String entryName, String mainPackage, + String jarName) { + + String jarFilePath = Constants.HOST_TMP_PATH + "/" + jarName; + + Manifest manifest = new Manifest(); + manifest.getMainAttributes().put(Attributes.Name.MANIFEST_VERSION, "1.0"); + manifest.getMainAttributes().put(Attributes.Name.MAIN_CLASS, mainPackage); + + try ( + FileOutputStream fos = new FileOutputStream(jarFilePath); + JarOutputStream jos = new JarOutputStream(fos, manifest)) { + Path path = new File(classFilePath + className).toPath(); + JarEntry entry = new JarEntry(entryName); + jos.putNextEntry(entry); + byte[] bytes = Files.readAllBytes(path); + jos.write(bytes, 0, bytes.length); + jos.closeEntry(); + } catch (IOException e) { + throw new RuntimeException("Create jar failed:", e); + } + + } + + private static void createAndBuildJars() { + String classPath = Constants.HOST_TMP_PATH + "/"; + compileJavaFile(Arrays.asList("docker/java-task/Fat.java")); + compileJavaFile(Arrays.asList("docker/java-task/Normal1.java", "docker/java-task/Normal2.java")); + compileJavaFile(Arrays.asList("docker/java-task/Normal2.java")); + createJar("Fat.class", classPath, + "Fat.class", + "Fat", + "fat.jar"); + createJar("Normal1.class", classPath, + "Normal1.class", + "Normal1", + "normal1.jar"); + createJar("Normal2.class", classPath, + "Normal2.class", + "Normal2", + "normal2.jar"); + + } + + public static void compileJavaFile(List sourceFilePaths) { + JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); + if (compiler == null) { + log.error("Cannot find the system Java compiler.", new IllegalStateException()); + return; + } + + String outputDirPath = Constants.HOST_TMP_PATH.toString(); + + try (StandardJavaFileManager fileManager = compiler.getStandardFileManager(null, null, null)) { + List sourceFiles = new ArrayList<>(); + for (String sourceFilePath : sourceFilePaths) { + URL resourceUrl = Thread.currentThread().getContextClassLoader().getResource(sourceFilePath); + if (resourceUrl == null) { + log.error("Java file not found: " + sourceFilePath, new IllegalArgumentException()); + continue; + } + + File resourceFile = new File(resourceUrl.toURI()); + if (!resourceFile.exists()) { + log.error("Java file does not exist: " + resourceFile.getAbsolutePath(), + new IllegalArgumentException()); + continue; + } + sourceFiles.add(resourceFile); + } + + Iterable compilationUnits = fileManager.getJavaFileObjectsFromFiles(sourceFiles); + List options = Arrays.asList("--release", "8", "-d", outputDirPath); + + JavaCompiler.CompilationTask task = + compiler.getTask(null, fileManager, null, options, null, compilationUnits); + boolean success = task.call(); + + if (!success) { + throw new RuntimeException("Java compilation failed."); + } + } catch (Exception e) { + throw new RuntimeException("Compile java files failed:", e); + } + } + @BeforeAll public static void setup() { + createAndBuildJars(); UserPage userPage = new LoginPage(browser) .login(user, password) .goToNav(SecurityPage.class) @@ -97,6 +209,11 @@ public static void setup() { userPage.update(user, user, email, phone, tenant) .goToNav(ProjectPage.class) .create(project); + + ProjectPage projectPage = new ProjectPage(browser); + Awaitility.await().untilAsserted(() -> assertThat(projectPage.projectList()) + .as("The project list should include newly created projects.") + .anyMatch(it -> it.getText().contains(project))); } @AfterAll @@ -121,35 +238,47 @@ public static void cleanup() { @Test @Order(1) - void testCreateWorkflow() { - WorkflowDefinitionTab workflowDefinitionPage = - new ProjectPage(browser) - .goTo(project) - .goToTab(WorkflowDefinitionTab.class); + void testCreateFatJarWorkflow() { + FileManagePage file = new NavBarPage(browser) + .goToNav(ResourcePage.class) + .goToTab(FileManagePage.class) + .uploadFile(filePath + "/fat.jar"); - workflowDefinitionPage - .createWorkflow() - .addTask(WorkflowForm.TaskType.JAVA) - .script(javaContent) + WebDriverWait wait = WebDriverWaitFactory.createWebDriverWait(browser); + + wait.until(ExpectedConditions.visibilityOfElementLocated(By.xpath("//span[text()='fat.jar']"))); + + ProjectPage projectPage = new NavBarPage(browser) + .goToNav(ProjectPage.class); + + wait.until(ExpectedConditions.visibilityOfAllElements(projectPage.projectList())); + + WorkflowDefinitionTab workflowDefinitionPage = projectPage + .goTo(project) + .goToTab(WorkflowDefinitionTab.class); + + WorkflowForm workflow1 = workflowDefinitionPage.createWorkflow(); + workflow1.addTask(WorkflowForm.TaskType.JAVA) + .selectRunType("FAT_JAR") + .selectMainPackage("fat.jar") + .selectJavaResource("fat.jar") .name("test-1") - .addParam("today", "${system.datetime}") .selectEnv(environmentName) .submit() .submit() .name(workflow) - .addGlobalParam("global_param", "hello world") .submit(); Awaitility.await().untilAsserted(() -> assertThat(workflowDefinitionPage.workflowList()) .as("Workflow list should contain newly-created workflow") - .anyMatch( - it -> it.getText().contains(workflow))); + .anyMatch(it -> it.getText().contains(workflow))); + workflowDefinitionPage.publish(workflow); } @Test @Order(30) - void testRunWorkflow() { + void testRunFatJarWorkflow() { final ProjectDetailPage projectPage = new ProjectPage(browser) .goToNav(ProjectPage.class) @@ -163,17 +292,99 @@ void testRunWorkflow() { .run(workflow) .submit(); - Awaitility.await().untilAsserted(() -> { - browser.navigate().refresh(); + Awaitility.await() + .atMost(Duration.ofMinutes(2)) + .untilAsserted(() -> { + browser.navigate().refresh(); + + final WorkflowInstanceTab.Row row = projectPage + .goToTab(WorkflowInstanceTab.class) + .instances() + .iterator() + .next(); + + assertThat(row.isSuccess()).isTrue(); + assertThat(row.executionTime()).isEqualTo(1); + }); + + } + + @Test + @Order(60) + void testCreateNormalJarWorkflow() { + FileManagePage file = new NavBarPage(browser) + .goToNav(ResourcePage.class) + .goToTab(FileManagePage.class) + .delete("fat.jar") + .uploadFile(filePath + "/normal2.jar"); + + WebDriverWait wait = WebDriverWaitFactory.createWebDriverWait(browser); + + wait.until(ExpectedConditions.visibilityOfElementLocated(By.xpath("//span[text()='normal2.jar']"))); + + file.uploadFile(filePath + "/normal1.jar"); + + wait.until(ExpectedConditions.visibilityOfElementLocated(By.xpath("//span[text()='normal1.jar']"))); + + ProjectPage projectPage = new NavBarPage(browser) + .goToNav(ProjectPage.class); + + wait.until(ExpectedConditions.visibilityOfAllElements(projectPage.projectList())); + + WorkflowDefinitionTab workflowDefinitionPage = projectPage + .goTo(project) + .goToTab(WorkflowDefinitionTab.class); + + workflowDefinitionPage.createWorkflow() + .addTask(WorkflowForm.TaskType.JAVA) + .selectRunType("NORMAL_JAR") + .selectMainPackage("normal1.jar") + .selectJavaResource("normal1.jar") + .selectJavaResource("normal2.jar") + .name("test-2") + .selectEnv(environmentName) + .submit() + .submit() + .name(workflow2) + .submit(); + + Awaitility.await().untilAsserted(() -> assertThat(workflowDefinitionPage.workflowList()) + .as("Workflow list should contain newly-created workflow") + .anyMatch(it -> it.getText().contains(workflow2))); + + workflowDefinitionPage.publish(workflow2); + } + + @Test + @Order(90) + void testRunNormalJarWorkflow() { + final ProjectDetailPage projectPage = + new ProjectPage(browser) + .goToNav(ProjectPage.class) + .goTo(project); + + projectPage + .goToTab(WorkflowInstanceTab.class) + .deleteAll(); + projectPage + .goToTab(WorkflowDefinitionTab.class) + .run(workflow2) + .submit(); + + Awaitility.await() + .atMost(Duration.ofMinutes(2)) + .untilAsserted(() -> { + browser.navigate().refresh(); + + final WorkflowInstanceTab.Row row = projectPage + .goToTab(WorkflowInstanceTab.class) + .instances() + .iterator() + .next(); - final WorkflowInstanceTab.Row row = projectPage - .goToTab(WorkflowInstanceTab.class) - .instances() - .iterator() - .next(); + assertThat(row.isSuccess()).isTrue(); + assertThat(row.executionTime()).isEqualTo(1); + }); - assertThat(row.isSuccess()).isTrue(); - assertThat(row.executionTime()).isEqualTo(1); - }); } } diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/pages/project/workflow/task/JavaTaskForm.java b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/pages/project/workflow/task/JavaTaskForm.java index 43c5ca9a8a46..6d77530cb6bc 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/pages/project/workflow/task/JavaTaskForm.java +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/pages/project/workflow/task/JavaTaskForm.java @@ -17,27 +17,124 @@ package org.apache.dolphinscheduler.e2e.pages.project.workflow.task; -import org.apache.dolphinscheduler.e2e.pages.common.CodeEditor; +import org.apache.dolphinscheduler.e2e.core.WebDriverWaitFactory; import org.apache.dolphinscheduler.e2e.pages.project.workflow.WorkflowForm; +import java.util.List; + +import lombok.Getter; + +import org.openqa.selenium.By; +import org.openqa.selenium.JavascriptExecutor; +import org.openqa.selenium.Keys; import org.openqa.selenium.WebDriver; +import org.openqa.selenium.WebElement; +import org.openqa.selenium.support.FindBy; +import org.openqa.selenium.support.FindBys; +import org.openqa.selenium.support.PageFactory; +import org.openqa.selenium.support.ui.ExpectedConditions; +import org.openqa.selenium.support.ui.WebDriverWait; +@Getter public class JavaTaskForm extends TaskNodeForm { - private CodeEditor codeEditor; - private WebDriver driver; + @FindBys({ + @FindBy(className = "resource-select"), + @FindBy(className = "n-base-selection"), + }) + private WebElement selectResource; + + @FindBys({ + + @FindBy(className = "n-tree-select"), + @FindBy(className = "n-base-selection"), + }) + private WebElement selectMainPackage; + + @FindBys({ + @FindBy(xpath = "//div[contains(@class,'n-form-item') and .//span[text()='Run Type']]"), + @FindBy(className = "n-select"), + @FindBy(className = "n-base-selection") + + }) + private WebElement selectRunType; + public JavaTaskForm(WorkflowForm parent) { super(parent); - this.codeEditor = new CodeEditor(parent.driver()); - this.driver = parent.driver(); + + PageFactory.initElements(driver, this); } - public JavaTaskForm script(String script) { - codeEditor.content(script); + public JavaTaskForm selectJavaResource(String resourceName) { + WebDriverWait wait = WebDriverWaitFactory.createWebDriverWait(driver()); + wait.until(ExpectedConditions.elementToBeClickable(selectResource)); + + ((JavascriptExecutor) parent().driver()).executeScript("arguments[0].click();", selectResource); + final By optionsLocator = By.className("n-tree-node-content__text"); + WebDriverWaitFactory.createWebDriverWait(parent().driver()) + .until(ExpectedConditions.visibilityOfElementLocated(optionsLocator)); + wait.until(s -> { + try { + Thread.sleep(5000); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + return true; + }); + parent().driver() + .findElements(optionsLocator) + .stream() + .filter(it -> it.getText().startsWith(resourceName)) + .findFirst() + .orElseThrow(() -> new RuntimeException("No such resource: " + resourceName)) + .click(); + driver().switchTo().activeElement().sendKeys(Keys.ESCAPE); return this; } + + public JavaTaskForm selectRunType(String runType) { + WebDriverWait wait = WebDriverWaitFactory.createWebDriverWait(driver()); + + WebElement dropdown = wait.until(ExpectedConditions.elementToBeClickable(selectRunType)); + ((JavascriptExecutor) driver).executeScript("arguments[0].scrollIntoView(true); arguments[0].click();", + dropdown); + + By dropdownMenuLocator = By.xpath("//div[contains(@class, 'n-select-menu')]"); + wait.until(ExpectedConditions.visibilityOfElementLocated(dropdownMenuLocator)); + + String optionXPath = String.format( + "//div[contains(@class, 'n-select-menu')]//div[contains(@class, 'n-base-select-option') and normalize-space(text())='%s']", + runType); + WebElement option = wait.until(ExpectedConditions.elementToBeClickable(By.xpath(optionXPath))); + ((JavascriptExecutor) driver).executeScript("arguments[0].scrollIntoView(true); arguments[0].click();", option); + + return this; + } + + public JavaTaskForm selectMainPackage(String packageName) { + ((JavascriptExecutor) driver).executeScript("arguments[0].click();", selectMainPackage); + + final By optionsLocator = By.className("n-tree-node-content__text"); + + WebDriverWait wait = WebDriverWaitFactory.createWebDriverWait(driver()); + wait.until(ExpectedConditions.visibilityOfElementLocated(optionsLocator)); + + List elements = driver.findElements(optionsLocator); + + WebElement targetElement = elements.stream() + .filter(it -> it.getText().trim().equals(packageName)) + .findFirst() + .orElseThrow(() -> new RuntimeException("No such package: " + packageName)); + + targetElement.click(); + + driver.switchTo().activeElement().sendKeys(Keys.ESCAPE); + + return this; + } + } diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/pages/resource/FileManagePage.java b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/pages/resource/FileManagePage.java index 0c48fd3ec7c4..10ba4e2cbad1 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/pages/resource/FileManagePage.java +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/pages/resource/FileManagePage.java @@ -221,6 +221,14 @@ public FileManagePage uploadFile(String filePath) { driver.setFileDetector(new LocalFileDetector()); uploadFileBox().buttonUpload().sendKeys(filePath); + WebDriverWaitFactory.createWebDriverWait(driver).until(s -> { + try { + Thread.sleep(5000); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + return true; + }); uploadFileBox().buttonSubmit().click(); return this; diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/java-task/Fat.java b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/java-task/Fat.java new file mode 100644 index 000000000000..59a075d35f53 --- /dev/null +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/java-task/Fat.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + +public class Fat { + + public static void main(String[] args) { + System.out.println("Hello world-----FAT"); + } +} diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/java-task/Normal1.java b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/java-task/Normal1.java new file mode 100644 index 000000000000..e51227d78022 --- /dev/null +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/java-task/Normal1.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +public class Normal1 { + + public static void main(String[] args) { + Normal2 s1 = new Normal2(); + s1.test(); + } +} diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/java-task/Normal2.java b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/java-task/Normal2.java new file mode 100644 index 000000000000..f65a4abe8697 --- /dev/null +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/java-task/Normal2.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +public class Normal2 { + + public void test() { + System.out.println("Hello World----NORMAL"); + } +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaConstants.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaConstants.java index e9bfd581c649..2f7d77bca29b 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaConstants.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaConstants.java @@ -29,14 +29,14 @@ private JavaConstants() { public static final String JAVA_HOME_VAR = "${JAVA_HOME}"; /** - * this constant represents the use of the java command to run a task + * This constant represents the use of the java -jar command to run a task **/ - public static final String RUN_TYPE_JAVA = "JAVA"; + public static final String RUN_TYPE_FAT_JAR = "FAT_JAR"; /** - * this constant represents the use of the java -jar command to run a task + * This constant represents the use of the java -cp command to run a task **/ - public static final String RUN_TYPE_JAR = "JAR"; + public static final String RUN_TYPE_NORMAL_JAR = "NORMAL_JAR"; /** * This constant is the Classpath or module path delimiter for different operating systems @@ -48,13 +48,4 @@ private JavaConstants() { **/ public static final String CLASSPATH_CURRENT_DIR = "."; - /** - * This constant is used to construct the pre-pathname of the Java source file - **/ - public static final String JAVA_SOURCE_CODE_NAME_TEMPLATE = "%s/%s.java"; - - /** - * This constant is the regular expression to get the class name of the source file - **/ - public static final String PUBLIC_CLASS_NAME_REGEX = "(.*\\s*public\\s+class\\s+)([a-zA-Z_]+[//w_]*)([.\\s\\S]*)"; } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaParameters.java index 8ac5dc5fe7ad..4137eb733ef5 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaParameters.java @@ -29,11 +29,6 @@ @Data public class JavaParameters extends AbstractParameters { - /** - * origin java script - */ - private String rawScript; - /** * run in jar file */ @@ -59,6 +54,11 @@ public class JavaParameters extends AbstractParameters { **/ private boolean isModulePath; + /** + * full main class name + **/ + private String mainClass; + /** * resource list */ @@ -67,11 +67,11 @@ public class JavaParameters extends AbstractParameters { /** * Check that the parameters are valid * - * @returnboolean + * @return boolean */ @Override public boolean checkParameters() { - return runType != null && (StringUtils.isNotBlank(rawScript) || mainJar != null); + return StringUtils.isNotEmpty(runType) && mainJar != null; } /** diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaTask.java index a96a8bd72987..ab32c87dd5f4 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaTask.java @@ -18,8 +18,8 @@ package org.apache.dolphinscheduler.plugin.task.java; import static org.apache.dolphinscheduler.plugin.task.java.JavaConstants.JAVA_HOME_VAR; -import static org.apache.dolphinscheduler.plugin.task.java.JavaConstants.PUBLIC_CLASS_NAME_REGEX; +import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.plugin.task.api.AbstractTask; import org.apache.dolphinscheduler.plugin.task.api.ShellCommandExecutor; @@ -27,30 +27,17 @@ import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; import org.apache.dolphinscheduler.plugin.task.api.TaskException; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; -import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; import org.apache.dolphinscheduler.plugin.task.api.model.TaskResponse; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; import org.apache.dolphinscheduler.plugin.task.api.resource.ResourceContext; import org.apache.dolphinscheduler.plugin.task.api.shell.IShellInterceptorBuilder; import org.apache.dolphinscheduler.plugin.task.api.shell.ShellInterceptorBuilderFactory; -import org.apache.dolphinscheduler.plugin.task.api.utils.MapUtils; -import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; -import org.apache.dolphinscheduler.plugin.task.java.exception.JavaSourceFileExistException; -import org.apache.dolphinscheduler.plugin.task.java.exception.PublicClassNotFoundException; import org.apache.dolphinscheduler.plugin.task.java.exception.RunTypeNotFoundException; -import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; import java.io.File; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.HashMap; -import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import lombok.extern.slf4j.Slf4j; @@ -74,11 +61,6 @@ public class JavaTask extends AbstractTask { */ private TaskExecutionContext taskRequest; - /** - * class name regex pattern - */ - private static final Pattern classNamePattern = Pattern.compile(PUBLIC_CLASS_NAME_REGEX); - public JavaTask(TaskExecutionContext taskRequest) { super(taskRequest); this.taskRequest = taskRequest; @@ -109,19 +91,22 @@ public void init() { public void handle(TaskCallBack taskCallBack) throws TaskException { try { // Step 1: judge if is java or jar run type. - // Step 2 case1: the jar run type builds the command directly, adding resource to the java -jar class when + // Step 2 case1: the fat jar run type builds the command directly, adding resource to the java -jar class + // when + // building the command + // Step 2 case2: the normal jar run type builds the command directly, adding resource to the java -cp class + // when // building the command - // Step 2 case2: the java run type, first replace the custom parameters, then compile the code, and then - // build the command will add resource // Step 3: to run the command String command = null; switch (javaParameters.getRunType()) { - case JavaConstants.RUN_TYPE_JAVA: - command = buildJavaCommand(); - break; - case JavaConstants.RUN_TYPE_JAR: + + case JavaConstants.RUN_TYPE_FAT_JAR: command = buildJarCommand(); break; + case JavaConstants.RUN_TYPE_NORMAL_JAR: + command = buildNormalJarCommand(); + break; default: throw new RunTypeNotFoundException("run type is required, but it is null now."); } @@ -150,44 +135,50 @@ public void handle(TaskCallBack taskCallBack) throws TaskException { } /** - * Construct a shell command for the java Run mode + * Construct a shell command for the java -jar Run mode * * @return String - * @throws Exception **/ - protected String buildJavaCommand() throws Exception { + protected String buildJarCommand() { + ResourceContext resourceContext = taskRequest.getResourceContext(); + String mainJarAbsolutePathInLocal = resourceContext + .getResourceItem(javaParameters.getMainJar().getResourceName()) + .getResourceAbsolutePathInLocal(); StringBuilder builder = new StringBuilder(); - String sourceCode = buildJavaSourceContent(); - builder.append(buildJavaCompileCommand(sourceCode)) - .append(";") - .append(getJavaCommandPath()) - .append("java").append(" ") - .append(buildResourcePath()) - .append(" ") - .append(getPublicClassName(sourceCode)) - .append(" ") - .append(javaParameters.getMainArgs().trim()).append(" ") + builder.append(getJavaCommandPath()) + .append("java").append(Constants.SPACE) + .append(buildResourcePath()).append(Constants.SPACE) + .append("-jar").append(Constants.SPACE) + .append(mainJarAbsolutePathInLocal).append(Constants.SPACE) + .append(javaParameters.getMainArgs().trim()).append(Constants.SPACE) .append(javaParameters.getJvmArgs().trim()); return builder.toString(); } /** - * Construct a shell command for the java -jar Run mode + * Construct a shell command for the java -cp run mode * * @return String **/ - protected String buildJarCommand() { + protected String buildNormalJarCommand() { ResourceContext resourceContext = taskRequest.getResourceContext(); - String mainJarAbsolutePathInLocal = resourceContext - .getResourceItem(javaParameters.getMainJar().getResourceName()) + String mainJarAbsolutePathInLocal = resourceContext.getResourceItem( + javaParameters.getMainJar() + .getResourceName()) .getResourceAbsolutePathInLocal(); + String mainClassName = javaParameters.getMainClass(); + String mainJarName; + if (mainClassName == null || StringUtils.isEmpty(mainClassName)) { + mainJarName = MainClassExtractor.getMainClassName(mainJarAbsolutePathInLocal); + } else { + mainJarName = mainClassName; + } StringBuilder builder = new StringBuilder(); builder.append(getJavaCommandPath()) - .append("java").append(" ") - .append(buildResourcePath()).append(" ") - .append("-jar").append(" ") - .append(mainJarAbsolutePathInLocal).append(" ") - .append(javaParameters.getMainArgs().trim()).append(" ") + .append("java").append(Constants.SPACE) + .append(buildResourcePath()).append(Constants.SPACE) + .append(mainJarName).append(Constants.SPACE) + .append(javaParameters.getMainArgs().trim()).append(Constants.SPACE) .append(javaParameters.getJvmArgs().trim()); return builder.toString(); } @@ -207,36 +198,6 @@ public AbstractParameters getParameters() { return javaParameters; } - /** - * Creates a Java source file when it does not exist - * - * @param sourceCode - * @param fileName - * @return String - **/ - protected void createJavaSourceFileIfNotExists(String sourceCode, String fileName) throws IOException { - log.info("tenantCode: {}, task dir:{}", taskRequest.getTenantCode(), taskRequest.getExecutePath()); - if (!Files.exists(Paths.get(fileName))) { - log.info("the java source code:{}, will be write to the file: {}", sourceCode, fileName); - // write data to file - FileUtils.writeStringToFile(new File(fileName), - sourceCode, - StandardCharsets.UTF_8); - } else { - throw new JavaSourceFileExistException("java source file exists, please report an issue on official."); - } - } - - /** - * Construct the full path name of the Java source file from the temporary execution path of the task - * - * @return String - **/ - protected String buildJavaSourceCodeFileFullName(String publicClassName) { - return String.format(JavaConstants.JAVA_SOURCE_CODE_NAME_TEMPLATE, taskRequest.getExecutePath(), - publicClassName); - } - /** * Construct a Classpath or module path based on isModulePath * @@ -262,47 +223,6 @@ protected String buildResourcePath() { return builder.toString(); } - /** - * Constructs a shell command compiled from a Java source file - * - * @param sourceCode - * @return String - * @throws IOException - **/ - protected String buildJavaCompileCommand(String sourceCode) throws IOException { - String publicClassName = getPublicClassName(sourceCode); - String fileName = buildJavaSourceCodeFileFullName(publicClassName); - createJavaSourceFileIfNotExists(sourceCode, fileName); - - StringBuilder compilerCommand = new StringBuilder() - .append(getJavaCommandPath()) - .append("javac").append(" ") - .append(buildResourcePath()).append(" ") - .append(fileName); - return compilerCommand.toString(); - } - - /** - * Work with Java source file content, such as replacing local variables - * - * @return String - **/ - protected String buildJavaSourceContent() { - String rawJavaScript = javaParameters.getRawScript().replaceAll("\\r\\n", "\n"); - // replace placeholder - - Map paramsMap = taskRequest.getPrepareParamsMap(); - if (MapUtils.isEmpty(paramsMap)) { - paramsMap = new HashMap<>(); - } - if (MapUtils.isNotEmpty(taskRequest.getParamsMap())) { - paramsMap.putAll(taskRequest.getParamsMap()); - } - log.info("The current java source code will begin to replace the placeholder: {}", rawJavaScript); - rawJavaScript = ParameterUtils.convertParameterPlaceholders(rawJavaScript, ParameterUtils.convert(paramsMap)); - return rawJavaScript; - } - /** * Gets the operating system absolute path to the Java command * @@ -312,17 +232,4 @@ private String getJavaCommandPath() { return JAVA_HOME_VAR + File.separator + "bin" + File.separator; } - /** - * Gets the public class name from the Java source file - * - * @param sourceCode - * @return String - **/ - public String getPublicClassName(String sourceCode) { - Matcher matcher = classNamePattern.matcher(sourceCode); - if (!matcher.find()) { - throw new PublicClassNotFoundException("public class is not be found in source code : " + sourceCode); - } - return matcher.group(2).trim(); - } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/MainClassExtractor.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/MainClassExtractor.java new file mode 100644 index 000000000000..93501e6d6aa6 --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/MainClassExtractor.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.java; + +import java.io.File; +import java.util.jar.JarFile; +import java.util.jar.Manifest; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class MainClassExtractor { + + private MainClassExtractor() { + } + public static String getMainClassName(String jarFilePath) { + String mainClassName = null; + try (JarFile jarFile = new JarFile(new File(jarFilePath))) { + + Manifest manifest = jarFile.getManifest(); + mainClassName = manifest.getMainAttributes().getValue("Main-Class"); + + } catch (Exception e) { + throw new RuntimeException("get mainJarName failed:", e); + } + return mainClassName; + } +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/test/java/org/apache/dolphinscheduler/plugin/task/java/JavaTaskTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/test/java/org/apache/dolphinscheduler/plugin/task/java/JavaTaskTest.java index b8af840c69f1..dd28b53857d4 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/test/java/org/apache/dolphinscheduler/plugin/task/java/JavaTaskTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/test/java/org/apache/dolphinscheduler/plugin/task/java/JavaTaskTest.java @@ -20,60 +20,30 @@ import static com.google.common.truth.Truth.assertThat; import static org.apache.dolphinscheduler.plugin.task.api.enums.DataType.VARCHAR; import static org.apache.dolphinscheduler.plugin.task.api.enums.Direct.IN; -import static org.apache.dolphinscheduler.plugin.task.java.JavaConstants.RUN_TYPE_JAR; -import static org.apache.dolphinscheduler.plugin.task.java.JavaConstants.RUN_TYPE_JAVA; +import static org.apache.dolphinscheduler.plugin.task.java.JavaConstants.RUN_TYPE_FAT_JAR; +import static org.apache.dolphinscheduler.plugin.task.java.JavaConstants.RUN_TYPE_NORMAL_JAR; -import org.apache.dolphinscheduler.common.utils.FileUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.plugin.task.api.TaskCallBack; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; -import org.apache.dolphinscheduler.plugin.task.api.model.ApplicationInfo; import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; import org.apache.dolphinscheduler.plugin.task.api.resource.ResourceContext; -import org.apache.dolphinscheduler.plugin.task.java.exception.JavaSourceFileExistException; -import org.apache.dolphinscheduler.plugin.task.java.exception.PublicClassNotFoundException; -import org.apache.dolphinscheduler.plugin.task.java.exception.RunTypeNotFoundException; +import java.io.File; import java.io.IOException; -import java.lang.reflect.Field; import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; import java.util.ArrayList; +import java.util.jar.Attributes; +import java.util.jar.JarEntry; +import java.util.jar.JarOutputStream; +import java.util.jar.Manifest; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -public class JavaTaskTest { - - private TaskCallBack taskCallBack = new TaskCallBack() { - - @Override - public void updateRemoteApplicationInfo(int taskInstanceId, ApplicationInfo applicationInfo) { - - } - - @Override - public void updateTaskInstanceInfo(int taskInstanceId) { +import lombok.extern.slf4j.Slf4j; - } - }; +import org.junit.jupiter.api.Test; - @Test - public void testGetPubllicClassName() { - JavaTask javaTask = runJavaType(); - Assertions.assertEquals(javaTask.getPublicClassName("import java.io.IOException;\n" + - "public class JavaTaskTest {\n" + - " public static void main(String[] args) throws IOException {\n" + - " StringBuilder builder = new StringBuilder(\"Hello: \");\n" + - " for (String arg : args) {\n" + - " builder.append(arg).append(\" \");\n" + - " }\n" + - " System.out.println(builder);\n" + - " }\n" + - "}\n"), "JavaTaskTest"); - } +@Slf4j +class JavaTaskTest { /** * Construct a java -jar command @@ -81,7 +51,7 @@ public void testGetPubllicClassName() { * @return void **/ @Test - public void buildJarCommand() { + void buildJarCommand() { JavaTask javaTask = runJarType(); assertThat(javaTask.buildJarCommand()) .isEqualTo( @@ -89,126 +59,58 @@ public void buildJarCommand() { } /** - * Construct the compile command + * Construct a java -cp command * * @return void - **/ + */ @Test - public void buildJavaCompileCommand() throws IOException { - JavaTask javaTask = runJavaType(); - String sourceCode = javaTask.buildJavaSourceContent(); - String publicClassName = javaTask.getPublicClassName(sourceCode); - Assertions.assertEquals("JavaTaskTest", publicClassName); - String fileName = javaTask.buildJavaSourceCodeFileFullName(publicClassName); - try { - Path path = Paths.get(fileName); - if (Files.exists(path)) { - Files.delete(path); - } - assertThat(javaTask.buildJavaCompileCommand(sourceCode)) - .isEqualTo( - "${JAVA_HOME}/bin/javac -classpath .:/tmp/dolphinscheduler/test/executepath:/tmp/dolphinscheduler/test/executepath/opt/share/jar/resource2.jar /tmp/dolphinscheduler/test/executepath/JavaTaskTest.java"); - } finally { - Path path = Paths.get(fileName); - if (Files.exists(path)) { - Files.delete(path); - } - } - - } - - /** - * Construct java to run the command - * - * @return void - **/ - @Test - public void buildJavaCommand() throws Exception { - JavaTask javaTask = runJavaType(); - String sourceCode = javaTask.buildJavaSourceContent(); - String publicClassName = javaTask.getPublicClassName(sourceCode); - - Assertions.assertEquals("JavaTaskTest", publicClassName); - - String fileName = javaTask.buildJavaSourceCodeFileFullName(publicClassName); - Path path = Paths.get(fileName); - if (Files.exists(path)) { - Files.delete(path); - } - assertThat(javaTask.buildJavaCommand()) + void buildNormalJarCommand() { + JavaTask javaTask = runNormalJarType(); + assertThat(javaTask.buildNormalJarCommand()) .isEqualTo( - "${JAVA_HOME}/bin/javac -classpath .:/tmp/dolphinscheduler/test/executepath:/tmp/dolphinscheduler/test/executepath/opt/share/jar/resource2.jar /tmp/dolphinscheduler/test/executepath/JavaTaskTest.java;${JAVA_HOME}/bin/java -classpath .:/tmp/dolphinscheduler/test/executepath:/tmp/dolphinscheduler/test/executepath/opt/share/jar/resource2.jar JavaTaskTest -host 127.0.0.1 -port 8080 -xms:50m"); - } - - /** - * There is no exception to overwriting the Java source file - * - * @return void - * @throws IOException - **/ - @Test - public void coverJavaSourceFileExistException() throws IOException { - JavaTask javaTask = runJavaType(); - String sourceCode = javaTask.buildJavaSourceContent(); - String publicClassName = javaTask.getPublicClassName(sourceCode); - Assertions.assertEquals("JavaTaskTest", publicClassName); - String fileName = javaTask.buildJavaSourceCodeFileFullName(publicClassName); - - Assertions.assertThrows(JavaSourceFileExistException.class, () -> { - try { - Path path = Paths.get(fileName); - if (!Files.exists(path)) { - FileUtils.createDirectoryWith755(path); - } - javaTask.createJavaSourceFileIfNotExists(sourceCode, fileName); - } finally { - Path path = Paths.get(fileName); - if (Files.exists(path)) { - Files.delete(path); - } - } - }); - } - - /** - * The override class name could not find an exception - * - * @return void - **/ - @Test - public void coverPublicClassNotFoundException() { - Assertions.assertThrows(PublicClassNotFoundException.class, () -> { - JavaTask javaTask = runJavaType(); - javaTask.getPublicClassName(""); - }); + "${JAVA_HOME}/bin/java -classpath .:/tmp/dolphinscheduler/test/executepath:/tmp/dolphinscheduler/test/executepath/opt/share/jar/resource2.jar:/tmp/dolphinscheduler/test/executepath/opt/share/jar/main.jar Test -host 127.0.0.1 -port 8080 -xms:50m"); } /** - * The override run mode could not find an exception + * add the Normal Jar parameters * - * @return void - * @throws Exception - **/ - @Test - public void coverRunTypeNotFoundException() throws Exception { - JavaTask javaTask = runJavaType(); - Field javaParameters = JavaTask.class.getDeclaredField("javaParameters"); - javaParameters.setAccessible(true); - ((JavaParameters) (javaParameters.get(javaTask))).setRunType(""); - - Assertions.assertThrows(RunTypeNotFoundException.class, () -> { - javaTask.handle(taskCallBack); - javaTask.getPublicClassName(""); - }); + * @param runType + * @return JavaParameters + */ + private JavaParameters createNormalJarJavaParameters(String runType) { + JavaParameters javaParameters = new JavaParameters(); + javaParameters.setRunType(runType); + javaParameters.setModulePath(false); + javaParameters.setJvmArgs("-xms:50m"); + javaParameters.setMainArgs("-host 127.0.0.1 -port 8080"); + ResourceInfo resourceJar1 = new ResourceInfo(); + resourceJar1.setResourceName("/opt/share/jar/resource2.jar"); + ResourceInfo resourceJar2 = new ResourceInfo(); + resourceJar2.setResourceName("/opt/share/jar/main.jar"); + ArrayList resourceInfoArrayList = new ArrayList<>(); + resourceInfoArrayList.add(resourceJar1); + resourceInfoArrayList.add(resourceJar2); + javaParameters.setResourceList(resourceInfoArrayList); + ArrayList localParams = new ArrayList<>(); + Property property = new Property(); + property.setProp("name"); + property.setValue("zhangsan"); + property.setDirect(IN); + property.setType(VARCHAR); + javaParameters.setLocalParams(localParams); + ResourceInfo mainJar = new ResourceInfo(); + mainJar.setResourceName("/opt/share/jar/main.jar"); + javaParameters.setMainJar(mainJar); + return javaParameters; } /** - * Create a Java task parameter mock object + * Add the fat jar parameters * * @param runType * @return JavaParameters - **/ - public JavaParameters createJavaParametersObject(String runType) { + */ + private JavaParameters createJavaParametersObject(String runType) { JavaParameters javaParameters = new JavaParameters(); javaParameters.setRunType(runType); javaParameters.setModulePath(false); @@ -219,16 +121,6 @@ public JavaParameters createJavaParametersObject(String runType) { ArrayList resourceInfoArrayList = new ArrayList<>(); resourceInfoArrayList.add(resourceJar); javaParameters.setResourceList(resourceInfoArrayList); - javaParameters.setRawScript( - "import java.io.IOException;\n" + - "public class JavaTaskTest {\n" + - " public static void main(String[] args) throws IOException {\n" + - " StringBuilder builder = new StringBuilder(\"Hello: \");\n" + - " for (String arg : args) {\n" + - " builder.append(arg).append(\" \");\n" + - " }\n" + " System.out.println(builder);\n" + - " }\n" + - "}\n"); ArrayList localParams = new ArrayList<>(); Property property = new Property(); property.setProp("name"); @@ -243,13 +135,13 @@ public JavaParameters createJavaParametersObject(String runType) { } /** - * A Java task that constructs the Java runtime pattern + * The Java task to construct the jar run mode * * @return JavaTask **/ - public JavaTask runJavaType() { + private JavaTask runJarType() { TaskExecutionContext taskExecutionContext = new TaskExecutionContext(); - taskExecutionContext.setTaskParams(JSONUtils.toJsonString(createJavaParametersObject(RUN_TYPE_JAVA))); + taskExecutionContext.setTaskParams(JSONUtils.toJsonString(createJavaParametersObject(RUN_TYPE_FAT_JAR))); taskExecutionContext.setExecutePath("/tmp/dolphinscheduler/test/executepath"); taskExecutionContext.setTaskAppId("runJavaType"); ResourceContext.ResourceItem resourceItem1 = new ResourceContext.ResourceItem(); @@ -261,28 +153,25 @@ public JavaTask runJavaType() { resourceItem2.setResourceAbsolutePathInStorage("/opt/share/jar/main.jar"); resourceItem2.setResourceAbsolutePathInLocal("/tmp/dolphinscheduler/test/executepath/opt/share/jar/main.jar"); - ResourceContext.ResourceItem resourceItem3 = new ResourceContext.ResourceItem(); - resourceItem3.setResourceAbsolutePathInStorage("/JavaTaskTest.java"); - resourceItem3.setResourceAbsolutePathInLocal("/tmp/dolphinscheduler/test/executepath/JavaTaskTest.java"); - ResourceContext resourceContext = new ResourceContext(); resourceContext.addResourceItem(resourceItem1); resourceContext.addResourceItem(resourceItem2); - resourceContext.addResourceItem(resourceItem3); taskExecutionContext.setResourceContext(resourceContext); + JavaTask javaTask = new JavaTask(taskExecutionContext); javaTask.init(); return javaTask; } /** - * The Java task to construct the jar run mode + * The Java task to construct the normal jar run mode * * @return JavaTask - **/ - private JavaTask runJarType() { + */ + private JavaTask runNormalJarType() { + packageTestJar(); TaskExecutionContext taskExecutionContext = new TaskExecutionContext(); - taskExecutionContext.setTaskParams(JSONUtils.toJsonString(createJavaParametersObject(RUN_TYPE_JAR))); + taskExecutionContext.setTaskParams(JSONUtils.toJsonString(createNormalJarJavaParameters(RUN_TYPE_NORMAL_JAR))); taskExecutionContext.setExecutePath("/tmp/dolphinscheduler/test/executepath"); taskExecutionContext.setTaskAppId("runJavaType"); ResourceContext.ResourceItem resourceItem1 = new ResourceContext.ResourceItem(); @@ -303,4 +192,30 @@ private JavaTask runJarType() { javaTask.init(); return javaTask; } + + /** + * Package the class to Jar + */ + private void packageTestJar() { + Manifest manifest = new Manifest(); + Attributes attributes = manifest.getMainAttributes(); + attributes.put(Attributes.Name.MANIFEST_VERSION, "1.0"); + attributes.put(Attributes.Name.MAIN_CLASS, "Test"); + String jarDirPath = "/tmp/dolphinscheduler/test/executepath/opt/share/jar"; + File jarDir = new File(jarDirPath); + if (!jarDir.exists() && jarDir.mkdirs()) { + log.info("Created directory: {}", jarDirPath); + } else if (!jarDir.exists()) { + throw new RuntimeException("Failed to create directory: " + jarDirPath); + } + File jarFile = new File(jarDirPath, "main.jar"); + try (JarOutputStream jos = new JarOutputStream(Files.newOutputStream(jarFile.toPath()), manifest)) { + jos.putNextEntry(new JarEntry("META-INF/")); + jos.closeEntry(); + } catch (IOException e) { + throw new RuntimeException("An error occurred while creating the JAR file.", e); + } + log.info("main.jar created: {}", jarFile.getAbsolutePath()); + } + } diff --git a/dolphinscheduler-ui/src/locales/en_US/project.ts b/dolphinscheduler-ui/src/locales/en_US/project.ts index 7e0f0a1314c6..677f66391e95 100644 --- a/dolphinscheduler-ui/src/locales/en_US/project.ts +++ b/dolphinscheduler-ui/src/locales/en_US/project.ts @@ -470,8 +470,9 @@ export default { expression_value_tips: 'expression values(optional)', pre_tasks: 'Pre tasks', program_type: 'Program Type', - main_class: 'Main Class', - main_class_tips: 'Please enter main class', + main_class: 'Main Class Name', + main_class_tips: 'Please enter the full main class name (optional)', + main_class_invalid: 'Invalid main class name', main_package: 'Main Package', main_package_tips: 'Please enter main package', deploy_mode: 'Deploy Mode', diff --git a/dolphinscheduler-ui/src/locales/zh_CN/project.ts b/dolphinscheduler-ui/src/locales/zh_CN/project.ts index 28d0fa601765..76706e04904c 100644 --- a/dolphinscheduler-ui/src/locales/zh_CN/project.ts +++ b/dolphinscheduler-ui/src/locales/zh_CN/project.ts @@ -457,8 +457,9 @@ export default { expression_value_tips: '表达式值集(选填)', pre_tasks: '前置任务', program_type: '程序类型', - main_class: '主函数的Class', - main_class_tips: '请填写主函数的Class', + main_class: '主类名', + main_class_tips: '请填写完整主类名(可选)', + main_class_invalid: '主类名不合法', main_package: '主程序包', main_package_tips: '请选择主程序包', deploy_mode: '部署方式', diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-java-task-main-jar.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-java-task-main-jar.ts index 826d709fad98..7bd0b3819260 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-java-task-main-jar.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-java-task-main-jar.ts @@ -26,7 +26,7 @@ export function useJavaTaskMainJar(model: { [field: string]: any }): IJsonItem { const mainJarOptions = ref([] as IMainJar[]) const taskStore = useTaskNodeStore() - const mainJarSpan = computed(() => (model.runType === 'JAVA' ? 0 : 24)) + const mainJarSpan = computed(() => (model.runType === 'FAT_JAR' ? 24 : 0)) const getMainJars = async (programType: ProgramType) => { const storeMainJar = taskStore.getMainJar(programType) if (storeMainJar) { diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-java-task-normal-jar.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-java-task-normal-jar.ts new file mode 100644 index 000000000000..e8931dd3feee --- /dev/null +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-java-task-normal-jar.ts @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { computed, ref, onMounted, watch } from 'vue' +import { useI18n } from 'vue-i18n' +import { queryResourceByProgramType } from '@/service/modules/resources' +import { useTaskNodeStore } from '@/store/project/task-node' +import utils from '@/utils' +import type { IJsonItem, ProgramType, IMainJar } from '../types' + +export function useJavaTaskNormalJar(model: { + [field: string]: any +}): IJsonItem[] { + const { t } = useI18n() + const mainJarOptions = ref([] as IMainJar[]) + const taskStore = useTaskNodeStore() + + const mainJarSpan = computed(() => (model.runType === 'NORMAL_JAR' ? 24 : 0)) + const getMainJars = async (programType: ProgramType) => { + const storeMainJar = taskStore.getMainJar(programType) + if (storeMainJar) { + mainJarOptions.value = storeMainJar + return + } + const res = await queryResourceByProgramType({ + type: 'FILE', + programType + }) + utils.removeUselessChildren(res) + mainJarOptions.value = res || [] + taskStore.updateMainJar(programType, res) + } + + onMounted(() => { + getMainJars(model.programType) + }) + + watch( + () => model.programType, + (value) => { + getMainJars(value) + } + ) + + return [ + { + type: 'input', + field: 'mainClass', + name: t('project.node.main_class'), + span: mainJarSpan, + props: { + type: 'textarea', + placeholder: t('project.node.main_class_tips') + }, + validate: { + trigger: ['input', 'blur'], + validator(_: any, value: string) { + if ( + value && + !/^([A-Za-z_$][A-Za-z\d_$]*\.)*[A-Za-z_$][A-Za-z\d_$]*$/.test( + value.trim() + ) + ) { + return new Error(t('project.node.main_class_invalid')) + } + return true + } + } + }, + { + type: 'tree-select', + field: 'mainJar', + name: t('project.node.main_package'), + span: mainJarSpan, + props: { + checkable: true, + cascade: true, + showPath: true, + checkStrategy: 'child', + placeholder: t('project.node.main_package_tips'), + keyField: 'fullName', + labelField: 'name' + }, + validate: { + trigger: ['input', 'blur'], + required: true, + validator(_: any, value: string) { + if (!value) { + return new Error(t('project.node.main_package_tips')) + } + return true + } + }, + options: mainJarOptions + } + ] +} diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-java.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-java.ts index b8a47c8935d2..e45a9344f849 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-java.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-java.ts @@ -14,14 +14,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { computed } from 'vue' import { useI18n } from 'vue-i18n' import { useCustomParams, useResources, useJavaTaskMainJar } from '.' import type { IJsonItem } from '../types' +import { useJavaTaskNormalJar } from '@/views/projects/task/components/node/fields/use-java-task-normal-jar' export function useJava(model: { [field: string]: any }): IJsonItem[] { const { t } = useI18n() - const rawScriptSpan = computed(() => (model.runType === 'JAR' ? 0 : 24)) return [ { type: 'select', @@ -57,17 +56,7 @@ export function useJava(model: { [field: string]: any }): IJsonItem[] { } }, useJavaTaskMainJar(model), - { - type: 'editor', - field: 'rawScript', - span: rawScriptSpan, - name: t('project.node.script'), - validate: { - trigger: ['input', 'trigger'], - required: true, - message: t('project.node.script_tips') - } - }, + ...useJavaTaskNormalJar(model), useResources(), ...useCustomParams({ model, field: 'localParams', isSimple: false }) ] @@ -75,11 +64,11 @@ export function useJava(model: { [field: string]: any }): IJsonItem[] { export const RUN_TYPES = [ { - label: 'JAVA', - value: 'JAVA' + label: 'FAT_JAR', + value: 'FAT_JAR' }, { - label: 'JAR', - value: 'JAR' + label: 'NORMAL_JAR', + value: 'NORMAL_JAR' } ] diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts index 9e7449c43948..1767c20de929 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts @@ -43,7 +43,11 @@ export function formatParams(data: INodeData): { taskParams.mainArgs = data.mainArgs taskParams.jvmArgs = data.jvmArgs taskParams.isModulePath = data.isModulePath - if (data.runType === 'JAR' && data.mainJar) { + taskParams.mainClass = data.mainClass + if ( + (data.runType === 'FAT_JAR' || data.runType === 'NORMAL_JAR') && + data.mainJar + ) { taskParams.mainJar = { resourceName: data.mainJar } } } diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-java.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-java.ts index 875a872f0ab8..bb7f26036d7c 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-java.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-java.ts @@ -48,9 +48,10 @@ export function useJava({ timeoutNotifyStrategy: ['WARN'], timeout: 30, mainJar: undefined, - runType: 'JAVA', + runType: 'FAT_JAR', mainArgs: '', jvmArgs: '', + mainClass: '', programType: 'JAVA' } as unknown as INodeData)