Merge branch 'master' of github.com:NUKnightLab/cityhallmonitor

cityhallmonitor/management/commands/data_lookup.py

@@ -154,7 +154,11 @@ def handle(self, *args, **options):
 
                     print('\nDatabase record')
                     print('- hyperlink', attachment.hyperlink) 
+                    print('- last_modified', attachment.last_modified) 
+                    print('- link_obtained_at', attachment.link_obtained_at)                    
+                    print('- matter.id', attachment.matter.id)
                     print('- matter.last_modified', attachment.matter.last_modified)
+                    print('- matter.attachments_obtained_at', attachment.matter.attachments_obtained_at)
                 except MatterAttachment.DoesNotExist:
                     print('\nMatterAttachment does not exist in database')
             else:
@@ -166,7 +170,11 @@ def handle(self, *args, **options):
 
                     print('\nDatabase record')
                     print('- id', attachment.id) 
+                    print('- last_modified', attachment.last_modified) 
+                    print('- link_obtained_at', attachment.link_obtained_at) 
+                    print('- matter.id', attachment.matter.id)
                     print('- matter.last_modified', attachment.matter.last_modified)
+                    print('- matter.attachments_obtained_at', attachment.matter.attachments_obtained_at)
                 except MatterAttachment.DoesNotExist:
                     print('\nMatterAttachment does not exist in database')
 

cityhallmonitor/management/commands/process_alerts.py

@@ -62,7 +62,7 @@ def send_subscription_alert(self, subscription, document_list):
 
     def process_subscription(self, subscription):
         """Process subscription"""
-        query = 'account:%s project:"%s" %s' % (
+        query = 'account:%s project:"%s" access:public %s' % (
             settings.DOCUMENT_CLOUD_ACCOUNT, 
             DEFAULT_PROJECT, 
             subscription.query)

cityhallmonitor/management/commands/pull_attachments.py

@@ -1,26 +1,71 @@
+from django.conf import settings
 from django.core.management.base import BaseCommand, CommandError
 from django.db.models import F, Q
 from django.utils import timezone
 import json
 import logging
 import requests
 from cityhallmonitor.models import Matter, MatterAttachment
+from documentcloud import DocumentCloud
 
 
 logger = logging.getLogger(__name__)
 
 _legistar_matter_attachments_url = \
     'http://webapi.legistar.com/v1/chicago/Matters/%(matter_id)s/Attachments'
 
+USERNAME = settings.DOCUMENT_CLOUD_USERNAME
+PASSWORD = settings.DOCUMENT_CLOUD_PASSWORD
+DOCUMENT_CLOUD_ACCOUNT = settings.DOCUMENT_CLOUD_ACCOUNT
+DEFAULT_PROJECT = 'Chicago City Hall Monitor'
+
 
 class Command(BaseCommand):
     help = 'Get attachments for updated matters from the Chicago Legistar API.'
 
+    _client = None
+
     def add_arguments(self, parser):
         parser.add_argument('matter_id', nargs='?', help='Matter ID')
 
+    def client(self):
+        if self._client is None:
+            self._client = DocumentCloud(USERNAME, PASSWORD)
+        return self._client
+
+    def search(self, query):
+        """
+        Seach DocumentCloud
+        """
+        r = self.client().documents.search(query)
+        assert type(r) is list, \
+            'DocumentCloud search response is %s: %s' % (type(r), repr(r))
+        return r
+
+    def privatize_doc(self, hyperlink):
+        """
+        Make doc with source=hyperlink private
+        """
+        logger.debug('Privatizing %s' % hyperlink)
+
+        r = self.search('account:%s access:public source: "%s"' % (
+            DOCUMENT_CLOUD_ACCOUNT, hyperlink))
+        if not r:
+            logger.info('Skipping privatization (no public version found): %s' \
+                % hyperlink)
+            return            
+        if len(r) > 1:
+            raise Exception(
+                'Multiple instances exist in DocumentCloud for '\
+                'source: %s' % hyperlink)               
+        doc = r[0]    
+        doc.access = 'private'
+        doc.save()
+
     def fetch(self, matter):
-        """Fetch attachments for matter"""
+        """
+        Fetch attachments for matter
+        """
         url = _legistar_matter_attachments_url % {'matter_id': matter.id}        
         logger.debug('Downloading %s', url)   
 
@@ -30,9 +75,20 @@ def fetch(self, matter):
             raise Exception('Error downloading %s [%d]' \
                 % (url, r.status_code))
         for i, item in enumerate(r.json()):
+
+            try:
+                r = MatterAttachment.objects.get(id=item['MatterAttachmentId'])
+                hyperlink = r.hyperlink
+            except MatterAttachment.DoesNotExist:
+                hyperlink = ''
+
             try:
                 r = MatterAttachment.from_json(matter.id, item)
-                r.save()
+
+                if r.hyperlink != hyperlink:
+                    self.privatize_doc(hyperlink)
+
+                r.save()                
             except Exception as e:
                 logger.info(item)
                 raise e

cityhallmonitor/management/commands/pull_pdfs.py

@@ -204,7 +204,12 @@ def handle(self, *args, **options):
                 logger.info('Fetching files for matter ID %s.', matter_id)
                 for attachment in MatterAttachment.objects.filter(
                         matter_id=matter_id):
-                    self.fetch(attchment, project.id)
+                    self.fetch(attachment, project.id)
+                    attachment.link_obtained_at = timezone.now()
+                    attachment.save()
+                    logger.debug(
+                        'Updated link_obtained_at timestamp for '\
+                        'MatterAttachment: %s', attachment.id)
             else:
                 q = MatterAttachment.objects.all()
                 if options['all']:

conf/prd/cityhallmonitor.cron

@@ -12,4 +12,4 @@ DJANGO_SETTINGS_MODULE=core.settings.prd
 30 2,10,18 * * * apps /home/apps/sites/cityhallmonitor/manage.py pull_attachments
 30 3,11,19 * * * apps /home/apps/sites/cityhallmonitor/manage.py pull_pdfs
 30 4,12,20 * * * apps /home/apps/sites/cityhallmonitor/manage.py get_descriptions
-
+0 12 * * * apps /home/apps/sites/cityhallmonitor/manage.py process_alerts