move to core==1.0:

- update dependencies and API to reflect 1.0 changes
- for building test assets, use `ocrd bag` to create self-contained
  directories atomically (but still work around core#176)
- for testing, depend on ocrd_tesserocr==0.3 (with overwrite_words,
  and thereby rid of this manual step)
- when creating output files, try to use the input file ID
  (modulo input file grp) in the basename

note: testing depends on core#266!

Author: bertsky

Makefile

@@ -39,6 +39,7 @@ test: test/assets
 
 # prepare test assets
 test/assets:
+	# TODO: instead of this, use bag repos, or add something useful to OCR-D/assets
 	test/prepare_gt.bash $@
 
 .PHONY: help deps deps-test install test

ocrd_keraslm/scripts/run.py

@@ -8,7 +8,7 @@
 import json
 import click
 
-from ocrd_keraslm import lib
+from .. import lib
 
 class SortedGroup(click.Group):
     def list_commands(self, ctx):

ocrd_keraslm/wrapper/cli.py

@@ -1,7 +1,7 @@
 import click
 
 from ocrd.decorators import ocrd_cli_options, ocrd_cli_wrap_processor
-from ocrd_keraslm.wrapper import KerasRate
+from . import KerasRate
 
 @click.command()
 @ocrd_cli_options

ocrd_keraslm/wrapper/rate.py

@@ -1,16 +1,25 @@
 from __future__ import absolute_import
+import os
 from math import log, ceil
 
-from ocrd import Processor, MIMETYPE_PAGE
-from ocrd.validator.page_validator import PageValidator, ConsistencyError
-from ocrd.utils import getLogger, concat_padded, xywh_from_points, points_from_xywh
-from ocrd.model.ocrd_page import from_file, to_xml, GlyphType, CoordsType, TextEquivType
-from ocrd.model.ocrd_page_generateds import MetadataItemType, LabelsType, LabelType
+from ocrd import Processor
+from ocrd_validators.page_validator import PageValidator, ConsistencyError
+from ocrd_utils import (
+    getLogger, concat_padded,
+    xywh_from_points, points_from_xywh,
+    MIMETYPE_PAGE
+)
+from ocrd_modelfactory import page_from_file
+from ocrd_models.ocrd_page import (
+    to_xml, GlyphType,
+    MetadataItemType, LabelsType, LabelType,
+    CoordsType, TextEquivType
+)
 
 import networkx as nx
 
-from ocrd_keraslm.wrapper.config import OCRD_TOOL
-from ocrd_keraslm import lib
+from .config import OCRD_TOOL
+from .. import lib
 
 LOG = getLogger('processor.KerasRate')
 
@@ -54,13 +63,15 @@ def process(self):
 
         prev_traceback = None
         prev_pcgts = None
+        prev_file_id = None
         for (n, input_file) in enumerate(self.input_files):
-            LOG.info("INPUT FILE %i / %s", n, input_file)
-            pcgts = from_file(self.workspace.download_file(input_file))
+            page_id = input_file.pageId or input_file.ID
+            LOG.info("INPUT FILE %i / %s", n, page_id)
+            pcgts = page_from_file(self.workspace.download_file(input_file))
             LOG.info("Scoring text in page '%s' at the %s level", pcgts.get_pcGtsId(), level)
 
             # annotate processing metadata:
-            metadata = pcgts.get_Metadata() # ensured by from_file()
+            metadata = pcgts.get_Metadata() # ensured by page_from_file()
             metadata.add_MetadataItem(
                 MetadataItemType(type_="processingStep",
                                  name=OCRD_TOOL['tools']['ocrd-keraslm-rate']['steps'][0],
@@ -115,11 +126,13 @@ def process(self):
                 page_update_higher_textequiv_levels(level, pcgts)
 
                 # write back result
-                file_id = concat_padded(self.output_file_grp, n)
+                file_id = input_file.ID.replace(self.input_file_grp, self.output_file_grp)
+                if file_id == input_file.ID:
+                    file_id = concat_padded(self.output_file_grp, n)
                 self.workspace.add_file(
                     ID=file_id,
                     file_grp=self.output_file_grp,
-                    basename=file_id + '.xml', # with suffix or bare?
+                    local_filename=os.path.join(self.output_file_grp, file_id + '.xml'),
                     mimetype=MIMETYPE_PAGE,
                     content=to_xml(pcgts),
                 )
@@ -140,15 +153,18 @@ def process(self):
                     page_update_higher_textequiv_levels(level, prev_pcgts)
 
                     # write back result
-                    file_id = concat_padded(self.output_file_grp, n - 1)
+                    file_id = prev_file_id.replace(self.input_file_grp, self.output_file_grp)
+                    if file_id == prev_file_id:
+                        file_id = concat_padded(self.output_file_grp, n - 1)
                     self.workspace.add_file(
                         ID=file_id,
                         file_grp=self.output_file_grp,
-                        basename=file_id + '.xml', # with suffix or bare?
+                        local_filename=os.path.join(self.output_file_grp, file_id + '.xml'),
                         mimetype=MIMETYPE_PAGE,
                         content=to_xml(prev_pcgts),
                     )
-                
+
+                prev_file_id = input_file.ID
                 prev_pcgts = pcgts
                 prev_traceback = traceback
 
@@ -160,11 +176,13 @@ def process(self):
             page_update_higher_textequiv_levels(level, prev_pcgts)
 
             # write back result
-            file_id = concat_padded(self.output_file_grp, n)
+            file_id = input_file.ID.replace(self.input_file_grp, self.output_file_grp)
+            if file_id == input_file.ID:
+                file_id = concat_padded(self.output_file_grp, n)
             self.workspace.add_file(
                 ID=file_id,
                 file_grp=self.output_file_grp,
-                basename=file_id + '.xml', # with suffix or bare?
+                local_filename=os.path.join(self.output_file_grp, file_id + '.xml'),
                 mimetype=MIMETYPE_PAGE,
                 content=to_xml(prev_pcgts),
             )

requirements.txt

@@ -1,4 +1,4 @@
-ocrd >= 0.15.2
+ocrd >= 1.0.0b10
 click
 keras >= 2.2.4
 numpy

requirements_test.txt

@@ -1,2 +1,2 @@
 pytest
-ocrd_tesserocr
+ocrd_tesserocr >= 0.3.0

setup.py

@@ -2,6 +2,7 @@
 """
 Installs:
     - keraslm-rate
+    - ocrd-keraslm-rate
 """
 import codecs
 
@@ -20,15 +21,7 @@
     url='https://github.com/OCR-D/ocrd_keraslm',
     license='Apache License 2.0',
     packages=find_packages(exclude=('tests', 'docs')),
-    install_requires=[
-        'ocrd >= 0.15.2',
-        'keras',
-        'click',
-        'numpy',
-        'tensorflow',
-        'h5py',
-        'networkx',
-    ],
+    install_requires=open('requirements.txt').read().split('\n'),
     extras_require={
         'plotting': [
             'sklearn',

test/prepare_gt.bash

@@ -42,37 +42,28 @@ EOF
 for GT_FILE in $GT_FILES; do
     test -f "$CACHE_DIR/${GT_FILE}.zip" ||
         wget -P "$CACHE_DIR" http://www.ocr-d.de/sites/all/GTDaten/${GT_FILE}.zip
-    unzip -d "$TMP_DIR" "$CACHE_DIR/${GT_FILE}.zip"
-    pushd "$TMP_DIR/$GT_FILE/$GT_FILE"
+    unzip -jod "$TMP_DIR/$GT_FILE" "$CACHE_DIR/${GT_FILE}.zip"
+    pushd "$TMP_DIR/$GT_FILE"
     ocrd workspace init .
     ZEROS=0000
     i=0
-    for PAGE_FILE in page/*.xml; do
+    for PAGE_FILE in *.xml; do
+		test "x$PAGE_FILE" = xmets.xml && continue
         i=$((i+1))
         ID=${ZEROS:0:$((4-${#i}))}$i
         IMG_FILE=$(xsltproc "$TMP_DIR/page-extract-imagefilename.xsl" "$PAGE_FILE")
         test -f "$IMG_FILE"
-        ocrd workspace add -G OCR-D-IMG -i OCR-D-IMG_$ID -g OCR-D-IMG_$ID -m image/tiff "$IMG_FILE"
-        ocrd workspace add -G OCR-D-GT-PAGE -i OCR-D-GT-PAGE_$ID -g OCR-D-IMG_$ID -m application/vnd.prima.page+xml "$PAGE_FILE"
+        ocrd workspace add -G OCR-D-IMG -i OCR-D-IMG_$ID -g phys_$ID -m image/tiff "$IMG_FILE"
+        ocrd workspace add -G OCR-D-GT-PAGE -i OCR-D-GT-PAGE_$ID -g phys_$ID -m application/vnd.prima.page+xml "$PAGE_FILE"
+		# workaround for OCR-D/core/issues/176 (still true for ocrd v1.0.0b10 !!)
+        sed -i -e "s|imageFilename=\"[^\"]*\"|imageFilename=\"OCR-D-IMG/OCR-D-IMG_$ID\"|" "$PAGE_FILE"
     done
+	ocrd zip bag -i ${GT_FILE}.zip -D full -Z -I
     popd
 done
 
-# this would break URIs: (still true for ocrd v0.15.2 !!)
-#mv "$TMP_DIR" "$1" # atomic
-# clone+cp instead:
-trap "rm -fr '$TMP_DIR' '$1'" ERR
-mkdir -p "$1"
-for GT_FILE in $GT_FILES; do # not so atomic
-    WORKSPACE="$TMP_DIR/$GT_FILE/$GT_FILE"
-    ocrd workspace clone -l "$WORKSPACE/mets.xml" "$1/$GT_FILE"
-    # workaround for OCR-D/core/issues/176 (still true for ocrd v0.15.2 !!)
-    for PAGE_FILE in "$1/$GT_FILE/OCR-D-GT-PAGE/"*.xml; do
-        sed -ie "s|imageFilename=\"|imageFilename=\"file://$PWD/$1/$GT_FILE/OCR-D-IMG/|" "$PAGE_FILE"
-    done
-    cp "$TMP_DIR/${GT_FILE}.txt" "$1"
-done
-rm -fr "$TMP_DIR"
+mv "$TMP_DIR" "$1" # atomic
+
 
 
 

test/test_wrapper.py

@@ -3,8 +3,9 @@
 from unittest import TestCase, main
 
 from ocrd.resolver import Resolver
-from ocrd.model.ocrd_page import from_file, to_xml
-from ocrd import MIMETYPE_PAGE
+from ocrd_models.ocrd_page import to_xml
+from ocrd_modelfactory import page_from_file
+from ocrd_utils import MIMETYPE_PAGE
 from ocrd_tesserocr.recognize import TesserocrRecognize
 from ocrd_keraslm.wrapper import KerasRate
 
@@ -20,8 +21,11 @@ def setUp(self):
 
     def runTest(self):
         resolver = Resolver()
-        workspace = resolver.workspace_from_url('test/assets/kant_aufklaerung_1784/mets.xml', dst_dir=WORKSPACE_DIR, download=True)
+        workspace = resolver.workspace_from_url('test/assets/kant_aufklaerung_1784/data/mets.xml', dst_dir=WORKSPACE_DIR, download=True)
         self.assertIsNotNone(workspace)
+        #
+        # rate text alternative 1 on the word level:
+        #
         KerasRate(
             workspace,
             input_file_grp='OCR-D-GT-PAGE', # has wrong tokenisation but that's ok now
@@ -33,36 +37,23 @@ def runTest(self):
         workspace.save_mets()
         for file in workspace.mets.find_files(fileGrp='OCR-D-LM-WORD'):
             continue # todo: for some reason, from_file yields NoneType here
-            pcgts = from_file(file)
+            pcgts = page_from_file(file)
             metadata = pcgts.get_Metadata()
             self.assertIsNotNone(metadata)
             metadataitems = metadata.get_MetadataItem()
             self.assertIsNotNone(metadataitems)
             rated = any([i for i in metadataitems if i.get_value() == 'ocrd-keraslm-rate'])
             self.assertTrue(rated)
-        for file in workspace.mets.find_files(fileGrp='OCR-D-GT-PAGE'):
-            grp='OCR-D-GT-SEG-LINE'
-            ID=grp + '_' + file.ID.split(sep='_')[-1]
-            pcgts = from_file(file)
-            page = pcgts.get_Page()
-            for region in page.get_TextRegion():
-                for line in region.get_TextLine():
-                    line.set_TextEquiv([]) # remove text results (interferes with ocrd_tesserocr)
-                    line.set_Word([]) # remove word annotation (interferes with ocrd_tesserocr, has wrong tokenization)
-            self.assertIsNotNone(
-                workspace.add_file(
-                    ID=ID,
-                    file_grp=grp,
-                    basename=ID + '.xml',
-                    mimetype=MIMETYPE_PAGE,
-                    content=to_xml(pcgts)))
+        # 
+        # rate and viterbi-decode all text alternatives on the glyph level:
+        # 
         TesserocrRecognize( # we need this to get alternatives to decode
             workspace,
-            input_file_grp='OCR-D-GT-SEG-LINE',
-            #input_file_grp='OCR-D-GT-PAGE', # only possible with ocrd_tesserocr >= 0.3.0
+            input_file_grp='OCR-D-GT-PAGE', # has wrong tokenisation but that's ok now
             output_file_grp='OCR-D-OCR-TESS-GLYPH',
             parameter={'textequiv_level': 'glyph',
-                       'model': 'deu-frak'}
+                       'overwrite_words': True,
+                       'model': 'deu-frak'} # old model for alternatives
             ).process()
         workspace.save_mets()
         KerasRate(
@@ -77,7 +68,7 @@ def runTest(self):
         workspace.save_mets()
         for file in workspace.mets.find_files(fileGrp='OCR-D-LM-GLYPH'):
             continue # todo: for some reason, from_file yields NoneType here
-            pcgts = from_file(file)
+            pcgts = page_from_file(file)
             metadata = pcgts.get_Metadata()
             self.assertIsNotNone(metadata)
             metadataitems = metadata.get_MetadataItem()