Merge branch 'release/0.13.0'

* release/0.13.0:
  Version 0.13.0
  Remove line markers within line continuations added by fypp for NAG compiler (fixes ecmwf/atlas#193)
  GitHub ci notifications (#42)
  Upgrade fypp installation to pep517
  Fix copyright header dates and replace message with ecbuild_info
  Make python virtual environment installation optional
  Downgrade minimum cmake to 3.17
  Disable pip upgrade warnings
  Restore fckit-eval script processing of fypp
  Add tests for fckit_yaml_reader
  VENV: install python virtualenv with ruamel and fypp
  Increase CMake version to 3.20 to use cmake_path utils
  YAML reader: create minimal yaml reader class that exports the same API as pyyaml
  RUAMEL: add contributed ruamel.yaml repo
  Fix memory leak in fckit_configuration%get_config_list (Thanks Mirco)

.github/workflows/ci.yml

@@ -74,3 +74,20 @@ jobs:
           repository: private-downstream-ci
           event_type: downstream-ci-hpc
           payload: '{"fckit": "ecmwf/fckit@${{ github.event.pull_request.head.sha || github.sha }}"}'
+
+  notify:
+    runs-on: ubuntu-latest
+    needs:
+      - downstream-ci
+      - private-downstream-ci
+      - downstream-ci-hpc
+      - private-downstream-ci-hpc
+    if: ${{ always() && !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
+    steps:
+      - name: Trigger Teams notification
+        uses: ecmwf-actions/notify-teams@v1
+        with:
+          incoming_webhook: ${{ secrets.MS_TEAMS_INCOMING_WEBHOOK }}
+          needs_context: ${{ toJSON(needs) }}
+
+

.github/workflows/notify-new-issue.yml

@@ -0,0 +1,15 @@
+name: Notify new issue
+
+on:
+  issues:
+    types:
+      - "opened"
+
+jobs:
+  notify:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Notify new issue
+        uses: ecmwf-actions/notify-teams-issue@v1
+        with:
+          incoming_webhook: ${{ secrets.MS_TEAMS_INCOMING_WEBHOOK }}

.github/workflows/notify-new-pr.yml

@@ -0,0 +1,16 @@
+name: Notify new PR
+
+# Needs the worklow to be located in the branche the PR is merged to
+on:
+  pull_request_target:
+    types:
+      - "opened"
+
+jobs:
+  notify:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Notify new PR
+        uses: ecmwf-actions/notify-teams-pr@v1
+        with:
+          incoming_webhook: ${{ secrets.MS_TEAMS_INCOMING_WEBHOOK }}

.gitignore

@@ -1,4 +1,6 @@
-build/*
+build
 install/*
 CMakeLists.txt.user
+**/*.egg-info/
+__pycache__
 

CMakeLists.txt

@@ -9,7 +9,7 @@
 ############################################################################################
 # FCKIT
 
-cmake_minimum_required( VERSION 3.12 FATAL_ERROR )
+cmake_minimum_required( VERSION 3.17 FATAL_ERROR )
 
 find_package( ecbuild 3.4 REQUIRED HINTS ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/../ecbuild )
 
@@ -22,8 +22,6 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON)
 ################################################################################################
 # options & dependencies
 
-ecbuild_find_python( VERSION 3.4 REQUIRED NO_LIBS )
-
 ### Fortran ...
 ecbuild_enable_fortran( REQUIRED MODULE_DIRECTORY ${PROJECT_BINARY_DIR}/module )
 
@@ -116,19 +114,34 @@ endif()
 set( FCKIT_LIBRARIES  fckit )
 
 ################################################################################################
-# find fypp
+# install python venv with rumael.yaml and fypp
+include( fckit_install_venv )
+set( install_permissions OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE )
+
+# Discover only system install Python 3
+set( Python3_FIND_VIRTUALENV STANDARD )
+find_package( Python3 COMPONENTS Interpreter REQUIRED )
+
+ecbuild_add_option( FEATURE FCKIT_VENV
+                    DEFAULT OFF
+                    DESCRIPTION "Install Python virtual environment with fypp and a yaml parser"
+                    CONDITION Python3_VERSION VERSION_GREATER_EQUAL 3.8 )
+
+if( HAVE_FCKIT_VENV )
+   fckit_install_venv()
+else()
+   # install fypp runner script only
+   set( FYPP ${CMAKE_CURRENT_SOURCE_DIR}/tools/fckit-eval.sh
+             ${CMAKE_CURRENT_SOURCE_DIR}/contrib/fypp-3.2-b8dd58b-20230822/bin/fypp )
 
-find_program(FYPP fypp)
-if( FYPP )
-  ecbuild_info("fckit found fypp: ${FYPP}")
+   install( FILES contrib/fypp-3.2-b8dd58b-20230822/bin/fypp DESTINATION libexec RENAME fckit-fypp.py
+            PERMISSIONS ${install_permissions} )
 endif()
 
+################################################################################################
 # sources
 
-if( NOT FYPP )
-  set( FYPP ${CMAKE_CURRENT_SOURCE_DIR}/tools/fckit-eval.sh ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tools/fckit-fypp.py )
-endif()
-set( FCTEST_GENERATOR ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tools/fctest-generate-runner.py )
+set( FCTEST_GENERATOR ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tools/fctest-generate-runner.py )
 include( fckit_preprocess_fypp )
 include( add_fctest )
 
@@ -148,16 +161,12 @@ if( ECBUILD_INSTALL_FORTRAN_MODULES )
 endif()
 if( NOT PROJECT_NAME STREQUAL CMAKE_PROJECT_NAME )
   set( FCTEST_GENERATOR ${FCTEST_GENERATOR} PARENT_SCOPE )
-  if( NOT FYPP )
-    set( FYPP ${FYPP} PARENT_SCOPE )
-  endif()
+  set( FYPP ${FYPP} PARENT_SCOPE )
 endif()
 file(READ ${CMAKE_CURRENT_SOURCE_DIR}/cmake/add_fctest.cmake ADD_FCTEST)
 file(READ ${CMAKE_CURRENT_SOURCE_DIR}/cmake/fckit_preprocess_fypp.cmake FCKIT_PREPROCESS_FYPP )
 
-set( install_permissions OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE )
 install( FILES tools/fctest-generate-runner.py DESTINATION libexec PERMISSIONS ${install_permissions} )
 install( FILES tools/fckit-eval.sh DESTINATION libexec PERMISSIONS ${install_permissions} )
-install( FILES contrib/fypp-3.2-b8dd58b-20230822/bin/fypp DESTINATION libexec RENAME fckit-fypp.py PERMISSIONS ${install_permissions} )
 
 ecbuild_install_project( NAME fckit )

VERSION

@@ -1,2 +1,2 @@
-0.12.1
+0.13.0
 

cmake/fckit-import.cmake.in

@@ -7,11 +7,24 @@
 # does it submit to any jurisdiction.
 
 if( @PROJECT_NAME@_IS_BUILD_DIR_EXPORT )
-  set( FCTEST_GENERATOR @PYTHON_EXECUTABLE@ @CMAKE_CURRENT_SOURCE_DIR@/tools/fctest-generate-runner.py )
-  set( FYPP @CMAKE_CURRENT_SOURCE_DIR@/tools/fckit-eval.sh @PYTHON_EXECUTABLE@ @CMAKE_CURRENT_SOURCE_DIR@/tools/fckit-fypp.py )
+  set( FCTEST_GENERATOR @Python3_EXECUTABLE@ @CMAKE_CURRENT_SOURCE_DIR@/tools/fctest-generate-runner.py )
+  set( FYPP @FYPP@ )
+  if( @HAVE_FCKIT_VENV@ )
+     set( FCKIT_VENV_EXE @FCKIT_VENV_EXE@ )
+  endif()
 else()
-  set( FCTEST_GENERATOR @PYTHON_EXECUTABLE@ ${fckit_BASE_DIR}/libexec/fctest-generate-runner.py )
-  set( FYPP ${fckit_BASE_DIR}/libexec/fckit-eval.sh @PYTHON_EXECUTABLE@ ${fckit_BASE_DIR}/libexec/fckit-fypp.py )
+  if( @HAVE_FCKIT_VENV@ AND NOT @ECBUILD_INSTALL_LIBRARY_HEADERS@ )
+     message( FATAL_ERROR "Installation of fckit python vritual environment was disabled" )
+  elseif( @HAVE_FCKIT_VENV@ )
+     set( _fckit_eval_script ${fckit_BASE_DIR}/libexec/fckit-eval.sh )
+     set( FCKIT_VENV_EXE ${fckit_BASE_DIR}/@rel_venv_exe_path@ )
+     set( FYPP ${_fckit_eval_script} ${FCKIT_VENV_EXE} -m fypp )
+  else()
+     set( FYPP ${fckit_BASE_DIR}/libexec/fckit-eval.sh 
+               ${fckit_BASE_DIR}/libexec/fckit-fypp.py )
+  endif()
+
+  set( FCTEST_GENERATOR @Python3_EXECUTABLE@ ${fckit_BASE_DIR}/libexec/fctest-generate-runner.py )
 endif()
 
 @FCKIT_PREPROCESS_FYPP@

cmake/fckit_install_venv.cmake

@@ -0,0 +1,69 @@
+# (C) Copyright 2024 ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation nor
+# does it submit to any jurisdiction.
+
+macro( fckit_install_venv )
+
+    # Create a virtualenv
+    set( VENV_PATH ${CMAKE_CURRENT_BINARY_DIR}/fckit_venv )
+    ecbuild_info( "Create Python virtual environment ${VENV_PATH}" )
+    execute_process( COMMAND ${Python3_EXECUTABLE} -m venv --copies "${VENV_PATH}" )
+
+    # Make the virtualenv portable by automatically deducing the VIRTUAL_ENV path from
+    # the 'activate' script's location in the filesystem
+    execute_process(
+        COMMAND
+            sed -i "s/^VIRTUAL_ENV=\".*\"$/VIRTUAL_ENV=\"$(cd \"$(dirname \"$(dirname \"\${BASH_SOURCE[0]}\" )\")\" \\&\\& pwd)\"/" "${VENV_PATH}/bin/activate"
+    )
+
+    # Change the context of the search to only find the venv
+    set( Python3_FIND_VIRTUALENV ONLY )
+    set( Python3_EXECUTABLE_CACHE ${Python3_EXECUTABLE} )
+
+    # Unset Python3_EXECUTABLE because it is also an input variable
+    #  (see documentation, Artifacts Specification section)
+    unset( Python3_EXECUTABLE )
+    # To allow cmake to discover the newly created venv if Python3_ROOT_DIR
+    # was passed as an argument at build-time
+    set( Python3_ROOT_DIR "${VENV_PATH}" )
+
+    # Find newly created python venv
+    find_package( Python3 COMPONENTS Interpreter REQUIRED )
+
+    set( _pkg_name "fckit_yaml_reader")
+    if( HAVE_TESTS )
+       set( _pkg_name "fckit_yaml_reader/[tests]")
+    endif()
+
+    ecbuild_info( "Install fckit_yaml_reader in virtual environment ${VENV_PATH}" )
+    execute_process( COMMAND ${Python3_EXECUTABLE} -m pip install --disable-pip-version-check ${CMAKE_CURRENT_SOURCE_DIR}/src/fckit/${_pkg_name} OUTPUT_QUIET )
+
+    # install ruamel
+    ecbuild_info( "Install ruamel.yaml in virtual environment ${VENV_PATH}" )
+    execute_process( COMMAND ${Python3_EXECUTABLE} -m pip install --disable-pip-version-check ${CMAKE_CURRENT_SOURCE_DIR}/contrib/ruamel.yaml-0.18.6 OUTPUT_QUIET )
+
+    # install fypp
+    ecbuild_info( "Install fypp in virtual environment ${VENV_PATH}" )
+    execute_process( COMMAND ${Python3_EXECUTABLE} -m pip install --use-pep517 --disable-pip-version-check
+                     ${CMAKE_CURRENT_SOURCE_DIR}/contrib/fypp-3.2-b8dd58b-20230822 OUTPUT_QUIET )
+
+    if( ECBUILD_INSTALL_LIBRARY_HEADERS )
+       install( DIRECTORY ${VENV_PATH} DESTINATION . PATTERN "bin/*" PERMISSIONS ${install_permissions} )
+    endif()
+
+    # add python interpreter of venv as executable target
+    set( FCKIT_VENV_EXE ${Python3_EXECUTABLE} )
+
+    # compute relative path to venv to aid with installation
+    string(REPLACE "${CMAKE_CURRENT_BINARY_DIR}/" "" rel_venv_exe_path ${FCKIT_VENV_EXE})
+
+    set( FYPP ${CMAKE_CURRENT_SOURCE_DIR}/tools/fckit-eval.sh ${FCKIT_VENV_EXE} -m fypp )
+
+    # reset Python3_EXECUTABLE to the system install
+    set( Python3_EXECUTABLE ${Python3_EXECUTABLE_CACHE} )
+
+endmacro()    

cmake/fckit_preprocess_fypp.cmake

@@ -108,6 +108,10 @@ function( fckit_preprocess_fypp_sources output )
       if( CMAKE_Fortran_COMPILER_ID MATCHES "Intel" )
         list( APPEND args --line-marker-format=gfortran5 )
       endif()
+      if( CMAKE_Fortran_COMPILER_ID MATCHES "NAG" )
+        # workaround for line markers in continuation lines ( see e.g. https://github.com/ecmwf/atlas/pull/193 )
+        list( APPEND args --line-numbering-mode=nocontlines )
+      endif()
       # list( APPEND args -N nocontlines )  # workaround for line numbers in continuation lines
     endif()