Merge pull request #373 from silk-framework/fix/taskCloning

Fix/task cloning

silk-workbench/silk-workbench-workspace/app/controllers/projectApi/ProjectApi.scala

@@ -10,6 +10,7 @@ import controllers.workspaceApi.projectTask.{ItemCloneRequest, ItemCloneResponse
 import controllers.workspaceApi.search.ItemType
 import javax.inject.Inject
 import org.silkframework.config.{MetaData, Prefixes}
+import org.silkframework.runtime.resource.ResourceManager
 import org.silkframework.runtime.validation.BadUserInputException
 import org.silkframework.serialization.json.JsonSerializers
 import org.silkframework.serialization.json.JsonSerializers.MetaDataJsonFormat
@@ -19,6 +20,8 @@ import org.silkframework.workspace.io.WorkspaceIO
 import play.api.libs.json.{JsValue, Json}
 import play.api.mvc.{Accepting, Action, AnyContent, InjectedController}
 
+import scala.util.Try
+
 /**
   * REST API for project artifacts.
   */
@@ -53,8 +56,12 @@ class ProjectApi @Inject()(accessMonitor: WorkbenchAccessMonitor) extends Inject
       val clonedProjectConfig = project.config.copy(id = generatedId, metaData = request.metaData.asMetaData)
       val clonedProject = workspace.createProject(clonedProjectConfig.copy(projectResourceUriOpt = Some(clonedProjectConfig.generateDefaultUri)))
       WorkspaceIO.copyResources(project.resources, clonedProject.resources)
+      // Clone task spec, since task specs may contain state, e.g. RDF file dataset
+      implicit val resourceManager: ResourceManager = project.resources
+      implicit val prefixes: Prefixes = project.config.prefixes
       for (task <- project.allTasks) {
-        clonedProject.addAnyTask(task.id, task.data) // FIXME: CMEM-2591, re-create task specs instead of adding the existing ones
+        val clonedTaskSpec = Try(task.data.withProperties(Map.empty)).getOrElse(task.data)
+        clonedProject.addAnyTask(task.id, clonedTaskSpec, task.metaData.asNewMetaData)
       }
       val projectLink = ItemType.itemDetailsPage(ItemType.project, generatedId, generatedId).path
       Created(Json.toJson(ItemCloneResponse(generatedId, projectLink)))

silk-workbench/silk-workbench-workspace/app/controllers/workspace/TaskApi.scala

@@ -6,17 +6,17 @@ import controllers.core.util.ControllerUtilsTrait
 import controllers.core.{RequestUserContextAction, UserContextAction}
 import controllers.util.SerializationUtils
 import javax.inject.Inject
-import org.silkframework.config.{MetaData, Task, TaskSpec}
+import org.silkframework.config.{MetaData, Prefixes, Task, TaskSpec}
 import org.silkframework.dataset.DatasetSpec.GenericDatasetSpec
 import org.silkframework.dataset.ResourceBasedDataset
 import org.silkframework.runtime.activity.UserContext
 import org.silkframework.runtime.plugin.{ParameterAutoCompletion, PluginDescription, PluginObjectParameterTypeTrait}
-import org.silkframework.runtime.resource.FileResource
+import org.silkframework.runtime.resource.{FileResource, ResourceManager}
 import org.silkframework.runtime.serialization.{ReadContext, WriteContext}
 import org.silkframework.runtime.validation.BadUserInputException
 import org.silkframework.serialization.json.JsonFormat
 import org.silkframework.serialization.json.JsonSerializers
-import org.silkframework.serialization.json.JsonSerializers.{TaskFormatOptions, TaskJsonFormat, TaskSpecJsonFormat, fromJson, toJson, MetaDataJsonFormat, GenericTaskJsonFormat}
+import org.silkframework.serialization.json.JsonSerializers.{GenericTaskJsonFormat, MetaDataJsonFormat, TaskFormatOptions, TaskJsonFormat, TaskSpecJsonFormat, fromJson, toJson}
 import org.silkframework.serialization.json.JsonSerializers._
 import org.silkframework.serialization.json.{JsonSerialization, JsonSerializers}
 import org.silkframework.util.Identifier
@@ -27,6 +27,7 @@ import play.api.libs.json._
 import play.api.mvc._
 
 import scala.concurrent.ExecutionContext
+import scala.util.Try
 import scala.util.control.NonFatal
 
 class TaskApi @Inject() (accessMonitor: WorkbenchAccessMonitor) extends InjectedController with ControllerUtilsTrait {
@@ -213,7 +214,12 @@ class TaskApi @Inject() (accessMonitor: WorkbenchAccessMonitor) extends Injected
 
   def cloneTask(projectName: String, oldTask: String, newTask: String): Action[AnyContent] = UserContextAction { implicit userContext =>
     val project = WorkspaceFactory().workspace.project(projectName)
-    project.addAnyTask(newTask, project.anyTask(oldTask))
+    val fromTask = project.anyTask(oldTask)
+    // Clone task spec, since task specs may contain state, e.g. RDF file dataset
+    implicit val resourceManager: ResourceManager = project.resources
+    implicit val prefixes: Prefixes = project.config.prefixes
+    val clonedTaskSpec = Try(fromTask.data.withProperties(Map.empty)).getOrElse(fromTask.data)
+    project.addAnyTask(newTask, clonedTaskSpec)
     Ok
   }
 

silk-workbench/silk-workbench-workspace/app/controllers/workspace/WorkspaceApi.scala

@@ -32,6 +32,7 @@ import play.api.mvc._
 
 import scala.concurrent.ExecutionContext.Implicits.global
 import scala.language.existentials
+import scala.util.Try
 
 class WorkspaceApi  @Inject() (accessMonitor: WorkbenchAccessMonitor) extends InjectedController with ControllerUtilsTrait {
 
@@ -81,8 +82,12 @@ class WorkspaceApi  @Inject() (accessMonitor: WorkbenchAccessMonitor) extends In
     val clonedProjectUri = clonedProjectConfig.generateDefaultUri
     val clonedProject = workspace.createProject(clonedProjectConfig.copy(projectResourceUriOpt = Some(clonedProjectUri)))
     WorkspaceIO.copyResources(project.resources, clonedProject.resources)
+    // Clone task spec, since task specs may contain state, e.g. RDF file dataset
+    implicit val resourceManager: ResourceManager = project.resources
+    implicit val prefixes: Prefixes = project.config.prefixes
     for(task <- project.allTasks) {
-      clonedProject.addAnyTask(task.id, task.data)
+      val clonedTaskSpec = Try(task.data.withProperties(Map.empty)).getOrElse(task.data)
+      clonedProject.addAnyTask(task.id, clonedTaskSpec, task.metaData.asNewMetaData)
     }
 
     Ok

silk-workbench/silk-workbench-workspace/test/controllers/workspace/TaskApiTest.scala

@@ -5,11 +5,14 @@ import org.scalatest.MustMatchers
 import org.scalatestplus.play.PlaySpec
 import org.silkframework.config.MetaData
 import org.silkframework.dataset.DatasetSpec
+import org.silkframework.dataset.DatasetSpec.GenericDatasetSpec
 import org.silkframework.dataset.rdf.SparqlEndpointDatasetParameter
+import org.silkframework.entity.StringValueType
 import org.silkframework.plugins.dataset.rdf.datasets.{InMemoryDataset, SparqlDataset}
 import org.silkframework.plugins.dataset.rdf.tasks.SparqlSelectCustomTask
 import org.silkframework.runtime.plugin.PluginRegistry
 import org.silkframework.serialization.json.JsonSerializers.{DATA, ID, PARAMETERS, TASKTYPE, TYPE}
+import org.silkframework.util.Uri
 import org.silkframework.workspace.TestCustomTask
 import play.api.http.Status
 import play.api.libs.json._
@@ -435,7 +438,23 @@ class TaskApiTest extends PlaySpec with IntegrationTestTrait with MustMatchers {
       val datasetJson = checkResponse(datasetResponse).json
       (datasetJson \ "metadata" \ "label").as[JsString].value mustBe "changed label"
     }
+  }
 
+  "task clone endpoint" should {
+    "clone a dataset by creating a new instance" in {
+      val inMemoryDataset = InMemoryDataset()
+      val tripleSink = inMemoryDataset.tripleSink
+      tripleSink.init()
+      tripleSink.writeTriple("a", "http://prop", "c", StringValueType())
+      tripleSink.close()
+      inMemoryDataset.source.retrievePaths("").flatMap(_.propertyUri) mustBe Seq(Uri("http://prop"))
+      val datasetName = "oneTripleInMemoryDataset"
+      val newDatasetName = "newInmemoryDataset"
+      val p = retrieveOrCreateProject(project)
+      p.addAnyTask(datasetName, new DatasetSpec(inMemoryDataset))
+      checkResponse(client.url(s"$baseUrl/workspace/projects/$project/tasks/$datasetName/clone?newTask=$newDatasetName").post(""))
+      p.task[GenericDatasetSpec](newDatasetName).data.source.retrievePaths("") mustBe Seq()
+    }
   }
 
   /**

silk-workbench/silk-workbench-workspace/test/controllers/workspace/WorkspaceApiTest.scala

@@ -0,0 +1,48 @@
+package controllers.workspace
+
+import helper.IntegrationTestTrait
+import org.scalatest.{BeforeAndAfterAll, FlatSpec, MustMatchers}
+import org.scalatestplus.play.PlaySpec
+import org.silkframework.dataset.DatasetSpec
+import org.silkframework.dataset.DatasetSpec.GenericDatasetSpec
+import org.silkframework.entity.StringValueType
+import org.silkframework.plugins.dataset.rdf.datasets.InMemoryDataset
+import org.silkframework.util.Uri
+import test.Routes
+
+/**
+  * Workspace API integration tests.
+  */
+class WorkspaceApiTest extends PlaySpec with IntegrationTestTrait with MustMatchers with BeforeAndAfterAll {
+
+  private val project = "project"
+
+  override def workspaceProviderId: String = "inMemory"
+
+  protected override def routes: Option[Class[Routes]] = Some(classOf[test.Routes])
+
+  override def beforeAll(): Unit = {
+    super.beforeAll()
+    retrieveOrCreateProject(project)
+  }
+
+  "Project clone endpoint" should {
+    "re-create tasks when cloning them" in {
+      val inMemoryDataset = InMemoryDataset(clearGraphBeforeExecution = false)
+      val tripleSink = inMemoryDataset.tripleSink
+      tripleSink.init()
+      tripleSink.writeTriple("a", "http://prop", "c", StringValueType())
+      tripleSink.close()
+      inMemoryDataset.source.retrievePaths("").flatMap(_.propertyUri) mustBe Seq(Uri("http://prop"))
+      val datasetName = "oneTripleInMemoryDataset"
+      val newProject = "newProject"
+      val p = retrieveOrCreateProject(project)
+      p.addAnyTask(datasetName, new DatasetSpec(inMemoryDataset))
+      checkResponse(client.url(s"$baseUrl/workspace/projects/$project/clone?newProject=$newProject").post(""))
+      val clonedInmemoryDataset = retrieveOrCreateProject(newProject).task[GenericDatasetSpec](datasetName)
+      clonedInmemoryDataset.data.plugin.asInstanceOf[InMemoryDataset].clearGraphBeforeExecution mustBe false
+      // Check that this is a new instance and does not contain the old state
+      clonedInmemoryDataset.source.retrievePaths("") mustBe Seq.empty
+    }
+  }
+}