Story/CITE-173 : Reprocessing Documents

citesphere/src/main/java/edu/asu/diging/citesphere/core/service/ICitationManager.java

@@ -4,6 +4,7 @@
 import java.util.concurrent.ExecutionException;
 
 import org.springframework.data.util.CloseableIterator;
+import org.springframework.http.HttpStatus;
 import org.springframework.social.zotero.api.ZoteroUpdateItemsStatuses;
 import org.springframework.social.zotero.exception.ZoteroConnectionException;
 
@@ -88,10 +89,15 @@ CloseableIterator<ICitation> getAllGroupItems(IUser user, String groupId, String
 
     void deleteLocalGroupCitations(String groupId);
 
+    HttpStatus reprocessFile(IUser user, String zoteroGroupId, String itemId, String documentId)
+            throws GroupDoesNotExistException, CannotFindCitationException, ZoteroHttpStatusException,
+            ZoteroConnectionException, CitationIsOutdatedException, ZoteroItemCreationFailedException;
+
+
     Citations findAuthorityCitations(IAuthorityEntry entry, IUser user);
 
     ICitation addCitationToReferences(IUser user, ICitation citation, String zoteroGroupId, String referenceCitationKey, 
             String reference) throws SelfCitationException, ZoteroConnectionException, CitationIsOutdatedException, 
             ZoteroHttpStatusException, ZoteroItemCreationFailedException;
 
-}
+}

citesphere/src/main/java/edu/asu/diging/citesphere/core/service/giles/GilesUploadChecker.java

@@ -1,5 +1,6 @@
 package edu.asu.diging.citesphere.core.service.giles;
 
+import edu.asu.diging.citesphere.model.bib.IGilesUpload;
 import edu.asu.diging.citesphere.user.IUser;
 
 public interface GilesUploadChecker {
@@ -12,5 +13,7 @@ public interface GilesUploadChecker {
     void checkUploadStatus(String citationKey);
 
     void checkFileUploadStatus(String itemId, IUser principal, String fileId);
+
+    boolean canReprocess(IGilesUpload upload, IUser user);
 
 }

citesphere/src/main/java/edu/asu/diging/citesphere/core/service/giles/IGilesConnector.java

@@ -1,15 +1,17 @@
 package edu.asu.diging.citesphere.core.service.giles;
 
+import org.springframework.http.HttpMethod;
 import org.springframework.http.ResponseEntity;
 import org.springframework.web.client.HttpClientErrorException;
 
 import edu.asu.diging.citesphere.user.IUser;
 
 public interface IGilesConnector {
 
-    <T> ResponseEntity<T> sendRequest(IUser user, String endpoint, Class<T> returnType)
+    <T> ResponseEntity<T> sendRequest(IUser user, String endpoint, Class<T> returnType, HttpMethod httpMethod)
             throws HttpClientErrorException;
 
     byte[] getFile(IUser user, String fileId);
-
-}
+
+    ResponseEntity<String> reprocessDocument(IUser user, String documentId);
+}

citesphere/src/main/java/edu/asu/diging/citesphere/core/service/giles/impl/GilesConnector.java

@@ -8,6 +8,7 @@
 import org.springframework.http.HttpEntity;
 import org.springframework.http.HttpHeaders;
 import org.springframework.http.HttpMethod;
+import org.springframework.http.HttpStatus;
 import org.springframework.http.ResponseEntity;
 import org.springframework.stereotype.Component;
 import org.springframework.util.MultiValueMap;
@@ -29,6 +30,9 @@ public class GilesConnector implements IGilesConnector {
     @Value("${giles_file_endpoint}")
     private String fileEndpoint;
 
+    @Value("${giles_file_reprocessing_endpoint}")
+    private String reprocessingEndpoint;
+
     @Autowired
     private InternalTokenManager internalTokenManager;
 
@@ -44,7 +48,7 @@ public void init() {
      * @see edu.asu.diging.citesphere.core.service.giles.impl.IGilesConnector#sendRequest(edu.asu.diging.citesphere.user.IUser, java.lang.String, java.lang.Class)
      */
     @Override
-    public <T> ResponseEntity<T> sendRequest(IUser user, String endpoint,Class<T> returnType) throws HttpClientErrorException {
+    public <T> ResponseEntity<T> sendRequest(IUser user, String endpoint,Class<T> returnType, HttpMethod httpMethod) throws HttpClientErrorException {
         String token = internalTokenManager.getAccessToken(user).getValue();
 
         HttpHeaders headers = new HttpHeaders();
@@ -54,15 +58,20 @@ public <T> ResponseEntity<T> sendRequest(IUser user, String endpoint,Class<T> re
 
         return restTemplate.exchange(
                     gilesBaseurl + endpoint,
-                    HttpMethod.GET, requestEntity, returnType);
+                    httpMethod, requestEntity, returnType);
     }
 
     /* (non-Javadoc)
      * @see edu.asu.diging.citesphere.core.service.giles.impl.IGilesConnector#getFile(edu.asu.diging.citesphere.user.IUser, java.lang.String)
      */
     @Override
     public byte[] getFile(IUser user, String fileId) {
-        ResponseEntity<byte[]> content = sendRequest(user, fileEndpoint.replace("{0}",  fileId), byte[].class);
+        ResponseEntity<byte[]> content = sendRequest(user, fileEndpoint.replace("{0}",  fileId), byte[].class, HttpMethod.GET);
         return content.getBody();
     }
+
+    @Override
+    public ResponseEntity<String> reprocessDocument(IUser user, String documentId) {
+        return sendRequest(user, reprocessingEndpoint.replace("{0}", documentId), String.class, HttpMethod.POST);
+    }
 }

citesphere/src/main/java/edu/asu/diging/citesphere/core/service/giles/impl/GilesUploadCheckerImpl.java

@@ -283,4 +283,33 @@ public void checkFileUploadStatus(String citationKey, IUser user, String process
             updateCitation(citation, checkedUploads, user, currentCitation);
         }        
     }
+
+    /**
+     * Check if an upload can be reprocessed based on Giles API v2 status
+     * @param upload The upload to check
+     * @param user The user making the request
+     * @return true if upload can be reprocessed, false otherwise
+     */
+    public boolean canReprocess(IGilesUpload upload, IUser user) {
+        if (upload.getProgressId() == null) {
+            return false;
+        }
+
+        String token = internalTokenManager.getAccessToken(user).getValue();
+        HttpHeaders headers = new HttpHeaders();
+        headers.setBearerAuth(token);
+        HttpEntity<MultiValueMap<String, Object>> requestEntity = new HttpEntity<>(headers);
+
+        try {
+            ResponseEntity<String> response = restTemplate.exchange(
+                gilesBaseurl + gilesCheckEndpoint + upload.getProgressId(),
+                HttpMethod.GET, requestEntity, String.class);
+
+            return response.getStatusCode() == HttpStatus.OK;
+
+        } catch (HttpClientErrorException ex) {
+            return ex.getStatusCode() == HttpStatus.NOT_FOUND || 
+                   ex.getStatusCode() == HttpStatus.INTERNAL_SERVER_ERROR;
+        }
+    }
 }

citesphere/src/main/java/edu/asu/diging/citesphere/core/service/impl/CitationManager.java

@@ -1,5 +1,6 @@
 package edu.asu.diging.citesphere.core.service.impl;
 
+import java.io.IOException;
 import java.time.OffsetDateTime;
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -22,12 +23,16 @@
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.context.annotation.PropertySource;
 import org.springframework.data.util.CloseableIterator;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
 import org.springframework.social.zotero.api.ZoteroUpdateItemsStatuses;
 import org.springframework.social.zotero.exception.ZoteroConnectionException;
 import org.springframework.stereotype.Service;
 import org.springframework.web.client.HttpClientErrorException;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
 
 import edu.asu.diging.citesphere.core.exceptions.AccessForbiddenException;
 import edu.asu.diging.citesphere.core.exceptions.CannotFindCitationException;
@@ -42,18 +47,22 @@
 import edu.asu.diging.citesphere.core.service.ICitationManager;
 import edu.asu.diging.citesphere.core.service.ICitationStore;
 import edu.asu.diging.citesphere.core.service.IGroupManager;
+import edu.asu.diging.citesphere.core.service.giles.GilesUploadChecker;
+import edu.asu.diging.citesphere.core.service.giles.IGilesConnector;
 import edu.asu.diging.citesphere.core.zotero.IZoteroManager;
 import edu.asu.diging.citesphere.data.bib.CitationGroupRepository;
 import edu.asu.diging.citesphere.data.bib.ICitationDao;
 import edu.asu.diging.citesphere.model.authority.IAuthorityEntry;
 import edu.asu.diging.citesphere.model.bib.ICitation;
 import edu.asu.diging.citesphere.model.bib.ICitationCollection;
 import edu.asu.diging.citesphere.model.bib.ICitationGroup;
+import edu.asu.diging.citesphere.model.bib.IGilesUpload;
 import edu.asu.diging.citesphere.model.bib.IReference;
 import edu.asu.diging.citesphere.model.bib.ItemType;
 import edu.asu.diging.citesphere.model.bib.impl.BibField;
 import edu.asu.diging.citesphere.model.bib.impl.CitationGroup;
 import edu.asu.diging.citesphere.model.bib.impl.CitationResults;
+import edu.asu.diging.citesphere.model.bib.impl.GilesUpload;
 import edu.asu.diging.citesphere.model.transfer.impl.Citations;
 import edu.asu.diging.citesphere.model.bib.impl.Reference;
 import edu.asu.diging.citesphere.user.IUser;
@@ -63,6 +72,8 @@
 @Transactional
 public class CitationManager implements ICitationManager {
 
+    private final Logger logger = LoggerFactory.getLogger(getClass());
+
     @Value("${_zotero_page_size}")
     private Integer zoteroPageSize;
 
@@ -90,6 +101,12 @@
 
     @Autowired
     private IAsyncCitationProcessor asyncCitationProcessor;
+
+    @Autowired
+    private IGilesConnector gilesConnector;
+
+    @Autowired
+    private GilesUploadChecker gilesUploadChecker;
 
     private Map<String, BiFunction<ICitation, ICitation, Integer>> sortFunctions;
 
@@ -380,78 +397,78 @@
         throw new GroupDoesNotExistException("Group " + groupId + " does not exist.");
     }
 
     @Override
     public CitationResults getGroupItems(IUser user, String groupId, String collectionId, int page, String sortBy, List<String> conceptIds)
             throws GroupDoesNotExistException, ZoteroHttpStatusException {
 
         ICitationGroup group = null;
         Optional<ICitationGroup> groupOptional = groupRepository.findFirstByGroupId(new Long(groupId));
         if (!groupOptional.isPresent() || !groupOptional.get().getUsers().contains(user.getUsername())) {
             group = zoteroManager.getGroup(user, groupId, false);
             if (group != null) {
                 if (groupOptional.isPresent()){
                     group.setId(groupOptional.get().getId());
                 }
                 group.getUsers().add(user.getUsername());
                 groupRepository.save((CitationGroup) group);
             }
         } else {
             group = groupOptional.get();
         }
 
         if (group == null) {
             throw new GroupDoesNotExistException("There is no group with id " + groupId);
         }
 
         boolean isModified = zoteroManager.isGroupModified(user, groupId, group.getContentVersion());
         CitationResults results = new CitationResults();
         if (isModified) {
             long previousVersion = group.getContentVersion();
             // first update the group info
             // if we are using a previously stored group, delete it
             ICitationGroup zoteroGroup = null;
             if (group.getId() != null) {
                 zoteroGroup = zoteroManager.getGroup(user, groupId + "", true);
                 zoteroGroup.setId(group.getId());
             }
             zoteroGroup.setUpdatedOn(OffsetDateTime.now().toString());
             addUserToGroup(zoteroGroup, user);
             group = groupRepository.save((CitationGroup) zoteroGroup);
 
             // then update content
             results.setNotModified(false);
             asyncCitationProcessor.sync(user, group.getGroupId() + "", previousVersion, collectionId);
         } else {
             results.setNotModified(true);
         }
 
         List<ICitation> citations = null;
         long total = 0;
         if (collectionId != null && !collectionId.trim().isEmpty()) {
             citations = (List<ICitation>) citationDao.findCitationsInCollection(groupId, collectionId, (page - 1) * zoteroPageSize, zoteroPageSize, conceptIds);
             ICitationCollection collection = collectionManager.getCollection(user, groupId, collectionId);
             if (collection != null) {
                 total = collection.getNumberOfItems();
             } else {
                 total = citations.size();
             }
         } else {
             citations = (List<ICitation>) citationDao.findCitations(groupId, (page - 1) * zoteroPageSize,
                     zoteroPageSize, false, conceptIds);
             if (groupOptional.isPresent()) {
                 updateCitationGroup(user, groupId);
 
                 total = groupRepository.findFirstByGroupId(new Long(groupId)).get().getNumItems();
             } else {
                 total = citations.size();
             }
         }
         results.setCitations(citations != null ? citations : new ArrayList<>());
         results.setTotalResults(total);
         return results;
 
     }
 
     @Override
     public void forceGroupItemsRefresh(IUser user, String groupId, String collectionId, int page, String sortBy) {
         Optional<ICitationGroup> groupOptional = groupRepository.findFirstByGroupId(new Long(groupId));
@@ -518,6 +535,106 @@
         citationStore.deleteCitationByGroupId(groupId);
     }
 
+    @Override
+    public HttpStatus reprocessFile(IUser user, String zoteroGroupId, String itemId, String documentId)
+            throws GroupDoesNotExistException, CannotFindCitationException, ZoteroHttpStatusException,
+            ZoteroConnectionException, CitationIsOutdatedException, ZoteroItemCreationFailedException {
+
+        if (documentId == null || documentId.trim().isEmpty()) {
+            logger.warn("Cannot reprocess file: documentId is null or empty for citation {}", itemId);
+            return HttpStatus.BAD_REQUEST;
+        }
+
+        ICitation citation = getCitation(user, zoteroGroupId, itemId);
+        List<IGilesUpload> uploadsToReprocess = citation.getGilesUploads().stream()
+            .filter(upload -> upload.getDocumentId() != null && upload.getDocumentId().equals(documentId))
+            .collect(Collectors.toList());
+
+        if (uploadsToReprocess.isEmpty()) {
+            logger.warn("No uploads found for document ID {} in citation {}", documentId, itemId);
+            return HttpStatus.NOT_FOUND;
+        }
+
+        boolean canReprocess = uploadsToReprocess.stream()
+            .anyMatch(upload -> gilesUploadChecker.canReprocess(upload, user));
+
+        if (!canReprocess) {
+            logger.warn("Upload for document ID {} in citation {} cannot be reprocessed (still processing)", documentId, itemId);
+            return HttpStatus.CONFLICT;
+        }
+
+        HttpStatus reprocessingStatus = null;
+        for(IGilesUpload upload : uploadsToReprocess) {
+            reprocessingStatus = initiateReprocessing(user, documentId, citation);
+        }
+
+        if (reprocessingStatus == null) {
+            logger.error("Reprocessing returned null status for document {} in citation {}", documentId, itemId);
+            return HttpStatus.INTERNAL_SERVER_ERROR;
+        }
+
+        return reprocessingStatus;
+    }
+
+    private HttpStatus initiateReprocessing(IUser user, String documentId, ICitation citation) throws GroupDoesNotExistException, CannotFindCitationException, ZoteroHttpStatusException {
+        ResponseEntity<String> reprocessingResponse = gilesConnector.reprocessDocument(user, documentId);
+
+        if (!reprocessingResponse.getStatusCode().equals(HttpStatus.OK)) {
+            logger.error("Document reprocessing failed for document {}. Server returned status: {}", documentId, reprocessingResponse.getStatusCode());
+            return reprocessingResponse.getStatusCode();
+        }
+        IGilesUpload reprocessedUpload = new GilesUpload();
+        String responseBody = reprocessingResponse.getBody();
+        ObjectMapper objectMapper = new ObjectMapper();
+        String progressId = null;
+        try {
+            JsonNode jsonNode = objectMapper.readTree(responseBody);
+            if (jsonNode != null && jsonNode.has("id")) {
+                progressId = jsonNode.get("id").asText();
+            }
+        } catch (IOException e) {
+            logger.error("Could not deserialize response for document {}. This means reprocessing cannot continue properly.", documentId, e);
+            return HttpStatus.INTERNAL_SERVER_ERROR;
+        }
+
+        if (progressId == null || progressId.trim().isEmpty()) {
+            logger.error("Could not extract valid progress ID for document {}. Reprocessing cannot continue.", documentId);
+            return HttpStatus.INTERNAL_SERVER_ERROR;
+        }
+
+        reprocessedUpload.setUploadingUser(user.getUsername());
+        reprocessedUpload.setProgressId(progressId);
+        Set<IGilesUpload> checkedUploads = new HashSet<>();
+        checkedUploads.add(reprocessedUpload);
+        updateReprocessedUpload(checkedUploads, user, citation, documentId);
+        gilesUploadChecker.add(citation.getKey());
+        return reprocessingResponse.getStatusCode();
+    }
+
+    private void updateReprocessedUpload(Set<IGilesUpload> checkedUploads, IUser user, ICitation citation, String documentId) throws GroupDoesNotExistException, CannotFindCitationException, ZoteroHttpStatusException {
+        ICitation currentCitation =getCitation(user, citation.getGroup(), citation.getKey());
+        if (currentCitation != null) {
+            for (IGilesUpload upload : checkedUploads) {
+                Optional<IGilesUpload> oldUpload = currentCitation
+                        .getGilesUploads().stream()
+                        .filter(u -> u.getDocumentId() != null && u
+                                .getDocumentId().equals(documentId))
+                        .findFirst();
+                if (oldUpload.isPresent()) {
+                    currentCitation.getGilesUploads().remove(oldUpload.get());
+                }
+                currentCitation.getGilesUploads().add(upload);
+            }
+            try {
+                updateCitation(user, citation.getGroup(),
+                        currentCitation);
+            } catch (ZoteroConnectionException | CitationIsOutdatedException
+                    | ZoteroHttpStatusException | ZoteroItemCreationFailedException e) {
+                logger.error("Could not update citation.", e);
+            }
+        }
+    }
+
     @Override
     public Citations findAuthorityCitations(IAuthorityEntry entry, IUser user) {  
         List<ICitationGroup> groups = getGroups(user);

citesphere/src/main/java/edu/asu/diging/citesphere/web/user/ItemController.java

@@ -5,16 +5,22 @@
 import java.util.List;
 
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
 import org.springframework.security.core.Authentication;
+import org.springframework.social.zotero.exception.ZoteroConnectionException;
 import org.springframework.stereotype.Controller;
 import org.springframework.ui.Model;
 import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestMethod;
 import org.springframework.web.bind.annotation.RequestParam;
 
 import edu.asu.diging.citesphere.core.exceptions.CannotFindCitationException;
+import edu.asu.diging.citesphere.core.exceptions.CitationIsOutdatedException;
 import edu.asu.diging.citesphere.core.exceptions.GroupDoesNotExistException;
 import edu.asu.diging.citesphere.core.exceptions.ZoteroHttpStatusException;
+import edu.asu.diging.citesphere.core.exceptions.ZoteroItemCreationFailedException;
 import edu.asu.diging.citesphere.core.search.service.SearchEngine;
 import edu.asu.diging.citesphere.core.service.ICitationManager;
 import edu.asu.diging.citesphere.core.service.IGroupManager;
@@ -74,4 +80,15 @@ public String getItem(Authentication authentication, Model model, @PathVariable(
         }
         return "auth/group/item";
     }
+
+    @RequestMapping(value = "/auth/group/{zoteroGroupId}/file/reprocess", method = RequestMethod.POST)
+    public ResponseEntity<String> deleteFile(Authentication authentication,
+            @PathVariable("zoteroGroupId") String zoteroGroupId,
+            @RequestParam(value = "documentId", required = false) String documentId,
+            @RequestParam(value = "itemId", required = false) String itemId)
+            throws GroupDoesNotExistException, CannotFindCitationException, ZoteroHttpStatusException,
+            ZoteroConnectionException, CitationIsOutdatedException, ZoteroItemCreationFailedException {
+        HttpStatus status = citationManager.reprocessFile((IUser) authentication.getPrincipal(), zoteroGroupId, itemId, documentId);
+        return new ResponseEntity<>(status);
+    }
 }

citesphere/src/main/resources/config.properties

@@ -94,5 +94,6 @@ elasticsearch.connect.timeout=${elasticsearch.connect.timeout}
 giles_upload_endpoint=/api/v2/files/upload
 giles_check_endpoint=/api/v2/files/upload/check/
 giles_file_endpoint=/api/v2/resources/files/{0}/content
+giles_file_reprocessing_endpoint=/api/v2/resources/documents/{0}/reprocess
 
 javers_default_author=${javers.default.author}