diff --git a/zebedee-cms/src/main/java/com/github/onsdigital/zebedee/model/approval/ApproveTask.java b/zebedee-cms/src/main/java/com/github/onsdigital/zebedee/model/approval/ApproveTask.java index 31fc47692..f02129727 100644 --- a/zebedee-cms/src/main/java/com/github/onsdigital/zebedee/model/approval/ApproveTask.java +++ b/zebedee-cms/src/main/java/com/github/onsdigital/zebedee/model/approval/ApproveTask.java @@ -51,6 +51,7 @@ import static com.github.onsdigital.logging.v2.event.SimpleEvent.info; import static com.github.onsdigital.zebedee.configuration.CMSFeatureFlags.cmsFeatureFlags; +import static com.github.onsdigital.zebedee.configuration.Configuration.getDatasetWhitelist; import static com.github.onsdigital.zebedee.json.EventType.APPROVAL_FAILED; import static com.github.onsdigital.zebedee.logging.CMSLogEvent.error; @@ -165,7 +166,11 @@ private boolean doApproval() throws Exception { eventLog.compressedZipFiles(); if (Configuration.isUploadNewEndpointEnabled() && DatasetWhitelistChecker.isURIWhitelisted(collectionReader)) { - uploadNewEndpoint(collection, collectionReader); + List reviewedList = collection.getReviewed().uris().stream().filter(i -> i.contains("datasets")).collect(Collectors.toList()); + String whitelist = getDatasetWhitelist(); + List allowed = Arrays.asList(whitelist.split(",")); + reviewedList = reviewedList.stream().filter(j -> allowed.stream().anyMatch(j::contains)).collect(Collectors.toList()); + uploadWhitelistedFiles(collectionReader, reviewedList, collection.getDescription().getId()); } approveCollection(); @@ -352,22 +357,17 @@ private void validate() { info().data("collectionId", collection.getDescription().getId()).log("approval task: validation successful"); } - protected void uploadNewEndpoint(Collection collection, CollectionReader collectionReader) + protected void uploadWhitelistedFiles(CollectionReader collectionReader, List urisToCheck, String description) throws ZebedeeException, IOException { - uploadWhitelistedFiles(collection, collectionReader); - } - - protected void uploadWhitelistedFiles(Collection collection, CollectionReader collectionReader) - throws ZebedeeException, IOException { - String correctDatasetVersion = findCorrectDatasetVersion(collectionReader.getReviewed().listUris()); - for (String uri : collectionReader.getReviewed().listUris()) { + for (String uri : urisToCheck) { if (uri.endsWith(".csv") || uri.endsWith(".xlsx") || uri.endsWith(".xls") || uri.endsWith(".csdb")) { String fileName = uri.substring(1); Resource myFile = collectionReader.getResource(fileName); if (DatasetWhitelistChecker.isWhitelisted(myFile.getName())) { - info().data("filename", fileName).data("collectionId", collection.getDescription().getId()) + info().data("filename", fileName).data("collectionId", description) .log("File is whitelisted"); - uploadFile(myFile, fileName, collection.getDescription().getId(), correctDatasetVersion); + String correctDatasetVersion = findCorrectDatasetVersion(urisToCheck, fileName); + uploadFile(myFile, fileName, description, correctDatasetVersion); } } } @@ -526,14 +526,14 @@ protected String incrementDatasetVersionByOne(String datasetVersion) { return letter + Integer.toString(number); } - protected String findCorrectDatasetVersion(List listOfUris) { + protected String findCorrectDatasetVersion(List listOfUris, String fileName) { if (listOfUris == null) { throw new IllegalArgumentException("input array can't be null"); } String datasetVersion = ""; for (String uri : listOfUris) { if (uri.endsWith(".csv") || uri.endsWith(".xlsx") || uri.endsWith(".xls") || uri.endsWith(".csdb")) { - if (uri.contains("previous")) { + if (uri.contains("previous") && uri.contains(extractFileName(fileName))) { datasetVersion = extractDatasetVersion(uri); return datasetVersion; } diff --git a/zebedee-cms/src/test/java/com/github/onsdigital/zebedee/model/approval/ApproveTaskTest.java b/zebedee-cms/src/test/java/com/github/onsdigital/zebedee/model/approval/ApproveTaskTest.java index 063f8e412..9f0bd773e 100644 --- a/zebedee-cms/src/test/java/com/github/onsdigital/zebedee/model/approval/ApproveTaskTest.java +++ b/zebedee-cms/src/test/java/com/github/onsdigital/zebedee/model/approval/ApproveTaskTest.java @@ -332,46 +332,6 @@ public void shouldApproveCmdSuccessfully() throws Exception { } } - // new unit tests - - @Test - public void testUploadNewEndpoint_HappyPath() throws ZebedeeException, IOException { - // Given - Collection collection = Mockito.mock(Collection.class); - CollectionReader collectionReader = Mockito.mock(CollectionReader.class); - System.setProperty("ENABLE_UPLOAD_NEW_ENDPOINT", "true"); - - when(collection.getDescription()).thenReturn(collectionDescription); - when(collectionReader.getReviewed()).thenReturn(contentReader); - when(collectionWriter.getReviewed()).thenReturn(contentWriter); - when(collection.getReviewed()).thenReturn(content); - - // When - task.uploadNewEndpoint(collection, collectionReader); - - // Then - verify(task, times(1)).uploadWhitelistedFiles(collection, collectionReader); - } - - @Test - public void testUploadNewEndpoint_UnHappyPath() throws ZebedeeException, IOException { - // Given - Collection collection = Mockito.mock(Collection.class); - CollectionReader collectionReader = Mockito.mock(CollectionReader.class); - System.setProperty("ENABLE_UPLOAD_NEW_ENDPOINT", "false"); - - when(collection.getDescription()).thenReturn(collectionDescription); - when(collectionReader.getReviewed()).thenReturn(contentReader); - when(collectionWriter.getReviewed()).thenReturn(contentWriter); - when(collection.getReviewed()).thenReturn(content); - - // When - task.uploadNewEndpoint(collection, collectionReader); - - // Then - verify(task, times(1)).uploadWhitelistedFiles(collection, collectionReader); - } - @Test public void testCreateUploadParams() { String resumableFilename = "test_file.csv"; @@ -574,7 +534,7 @@ public void testIncrementDatasetVersionByOne_HappyPath() { @Test public void testfindCorrectDatasetVersion_nullInput() { - IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, () -> task.findCorrectDatasetVersion(null)); + IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, () -> task.findCorrectDatasetVersion(null, "")); assertThat(ex.getMessage(), equalTo("input array can't be null")); } @@ -587,7 +547,7 @@ public void testfindCorrectDatasetVersion_correctInput() { listOfUris.add("/economy/grossdomesticproductgdp/datasets/mycollectionpagedltest1/augusttoseptember2024/previous/v1/diop.csv"); String expected = "v1"; - String actual = task.findCorrectDatasetVersion(listOfUris); + String actual = task.findCorrectDatasetVersion(listOfUris, "diop.csv"); assertEquals(expected, actual); } @@ -600,7 +560,7 @@ public void testfindCorrectDatasetVersion_anotherCorrectInput_previousVersionExi listOfUris.add("/economy/grossdomesticproductgdp/datasets/mycollectionpagedltest1/current/previous/v123/diop.csv"); String expected = "v123"; - String actual = task.findCorrectDatasetVersion(listOfUris); + String actual = task.findCorrectDatasetVersion(listOfUris, "diop.csv"); assertEquals(expected, actual); } @@ -612,7 +572,7 @@ public void testfindCorrectDatasetVersion_anotherCorrectInput_previousVersionDoe listOfUris.add("/economy/grossdomesticproductgdp/datasets/mycollectionpagedltest1/september2024/diop.xlsx"); String expected = ""; - String actual = task.findCorrectDatasetVersion(listOfUris); + String actual = task.findCorrectDatasetVersion(listOfUris, "diop.csv"); assertEquals(expected, actual); } @@ -621,8 +581,7 @@ public void testfindCorrectDatasetVersion_emptyList() { List listOfUris = new ArrayList<>(); String expected = ""; - String actual = task.findCorrectDatasetVersion(listOfUris); + String actual = task.findCorrectDatasetVersion(listOfUris, "diop.csv"); assertEquals(expected, actual); } - }