Merge pull request #71 from UUDigitalHumanitieslab/feature/alpino-lookup

Feature/alpino lookup
CentreForDigitalHumanities · Aug 28, 2023 · 5cef2fe · 5cef2fe
2 parents addfe85 + 27217a7
commit 5cef2fe
Show file tree

Hide file tree

Showing 116 changed files with 10,741 additions and 1,703 deletions.
diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml
@@ -11,7 +11,7 @@ on:
 jobs:
   build:
 
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-20.04
     strategy:
       fail-fast: false
       matrix:
@@ -37,16 +37,17 @@ jobs:
           #   alpino: Alpino-x86_64-Linux-glibc-2.17-git819-sicstus
           # versions known to works
           - dependencies: pinned
-            alpino: Alpino-x86_64-Linux-glibc-2.17-git819-sicstus
-            ticcutils: fd408a4db606ded411b002c04c96369120449a0f # 0.30
-            libfolia: 2722c53122d7e47028a999a919fd0bbc6e58a664 # 2.12
-            uctodata: a8be6cf7512998f4c8963d1021f402b8b1290085 # 0.9.1
-            ucto: 58766ef94740e6c9277c8ad77eda80f4df115443 # 0.26
-            timbl: 7dca5c783c4730f48da0be27de8e54450f3e509d # 6.8
-            mbt: 3bebc5512edf4fda9e6c38ef106567546c434c30 # 3.8
-            mbtserver: f4ef14850e242ecb60db0f99070435fe79f5d6fd # 0.16
-            frogdata: 6c8cd0bee122d2703d61395ff527592268192785 # 0.22
-            frog: 7818ce301dfdfdc026e36474adf83e3e0d769ac6 # 0.26
+            # removed for speed, using pre-parsed trees only
+            # alpino: Alpino-x86_64-Linux-glibc-2.17-git819-sicstus
+            ticcutils: ecb62eb116ffc3677626c53f012f34e00854e4c4 # 0.33 2023-04-29
+            libfolia: 1a3f462f8f048df60889817325701130b6271e8a # 2.15 2023-05-01
+            uctodata: a8be6cf7512998f4c8963d1021f402b8b1290085 # 0.9.1 2022-10-04
+            ucto: d22029833b264a969351ead0ee3ab9b3f97db97d # 0.30 2023-04-29
+            timbl: 61929ce6dc7d7077cb5eeceafef86de38eb40daa # 6.9 2023-04-29
+            mbt: fd7cb7ebdd52bef2794f16f329569bedad3143e9 # 3.10 2023-04-29
+            mbtserver: a6f04f30f62965c8660ee92be99d4eb86fc4bf65 # 0.17 2023-04-30
+            frogdata: 99de9597105c2304faeb797264231ba180fcdb20 # 0.22 2023-03-07
+            frog: 991f6977cfd81b9a6538db0c6de3d26908ec16b7 # 0.29 2023-05-03
     env:
       CC: ${{ matrix.compiler.CC }}
       CXX: ${{ matrix.compiler.CXX }}
@@ -89,11 +90,11 @@ jobs:
         mkdir -p ~/.tscan-deps
         ls -la ~/.tscan-deps
 
-    - name: Install Alpino
-      env:
-        ALPINO_VERSION: ${{ matrix.alpino }}
-      run: |
-        .github/workflows/cpp-prep-alpino.sh $ALPINO_VERSION
+    # - name: Install Alpino
+    #   env:
+    #     ALPINO_VERSION: ${{ matrix.alpino }}
+    #   run: |
+    #     .github/workflows/cpp-prep-alpino.sh $ALPINO_VERSION
 
     - name: Install ticcutils
       run: |
@@ -133,31 +134,29 @@ jobs:
         sudo make install
     - name: Start services
       run: |
-        export ALPINO_HOME=~/.tscan-deps/Alpino
+        # export ALPINO_HOME=~/.tscan-deps/Alpino
         export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH
         cd webservice
-        ls -la ~/.tscan-deps/Alpino
-        ./startalpino.sh &
+        # ls -la ~/.tscan-deps/Alpino
+        # ./startalpino.sh &
         ./startfrog.sh &
     - name: Services up?
-      run: |
+      run: |        
+        # .github/workflows/cpp-check-service.sh 7003
         .github/workflows/cpp-check-service.sh 7001
-        .github/workflows/cpp-check-service.sh 7003
         sleep 1
     - name: Run unit tests
       run: |
         make check
-    - name: Clear extracted Alpino files
-      run: |
-        rm -rf ~/.tscan-deps/Alpino
+    # - name: Clear extracted Alpino files
+    #   run: |
+    #     rm -rf ~/.tscan-deps/Alpino
     - name: Debug information
       if: ${{ !success() }}
       run: |
         cat config.h
         [ -f src/test-suite.log ] && cat src/test-suite.log
         [ -f src/test.sh.log ] && cat src/test.sh.log
         tests/logdiffs.sh
-        echo "*** ALPINO LOG ***"
-        cat /tmp/alpino_server.log
         echo "*** FROG LOG ***"
         cat /tmp/frog-tscan.log
diff --git a/.github/workflows/webservice.yml b/.github/workflows/webservice.yml
@@ -27,7 +27,7 @@ jobs:
       run: |
         sudo apt install libmagic-dev antiword
         python -m pip install --upgrade pip
-        pip install flake8 textract python-magic
+        pip install flake8 lxml textract python-magic
     - name: Lint with flake8
       run: |
         cd webservice

diff --git a/include/tscan/stats.h b/include/tscan/stats.h
@@ -919,7 +919,7 @@ struct structStats: public basicStats {
 
 
 struct sentStats : public structStats {
-  sentStats( int, folia::Sentence*, const sentStats* );
+  sentStats( const std::string&, int, folia::Sentence*, const sentStats* );
   bool isSentence() const override { return true; };
   void resolveConnectives();
   void resolveSituations();
@@ -944,13 +944,13 @@ struct sentStats : public structStats {
 
 
 struct parStats: public structStats {
-  parStats( int, folia::Paragraph* );
+  parStats( const std::string&, int, folia::Paragraph* );
   void addMetrics() const override;
 };
 
 
 struct docStats : public structStats {
-  explicit docStats( folia::Document* );
+  explicit docStats( const std::string&, folia::Document* );
   bool isDocument() const override { return true; };
   void toCSV( const std::string&, csvKind ) const;
   double rarity( int level ) const override;