.gitlab-ci.yml

include: '.gitlab-ci/all_years_build_and_aggregates.yml'

variables:
  PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
  CI_REGISTRY: https://index.docker.io/v1/
  CI_REGISTRY_IMAGE: leximpact/openfisca-france-data
  # OUT_FOLDER: "$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA" # For branch-commit_id
  OUT_FOLDER: "$CI_COMMIT_REF_NAME" # For just branch
  ROOT_FOLDER: "/mnt/data-out/openfisca-france-data"
#  GIT_DEPTH: 1000 # To be able to get last tag (default 50)

cache:
  paths:
    - .cache/pip
    - ./figures/

stages:
  - docker
  - test
  - build_collection
  - build_input_data
  - diagnostics
  - aggregates
  - pote
  - run_on_all_years
  - build_input_data_all
  - aggregates_all
  - anaconda

before_script:
  # To be sure we are up to date even if we do not rebuild docker image
  - make install
  - cp ./.gitlab-ci/openfisca_survey_manager_raw_data.ini ~/.config/openfisca-survey-manager/raw_data.ini
  - echo "End of before_script"

build docker image:
  stage: docker
  tags:
    - openfisca
  image:
    name: gcr.io/kaniko-project/executor:debug
    entrypoint: [""]
  # Prevent call of before_script because it will fail in this context
  before_script:
    - ''
  script:
    # From https://github.com/GoogleContainerTools/kaniko#pushing-to-docker-hub
    - DOCKER_HUB_AUTH=$(echo -n $DOCKER_HUB_USER:$DOCKER_HUB_PASSWORD | base64)
    - echo "{\"auths\":{\"$CI_REGISTRY\":{\"auth\":\"$DOCKER_HUB_AUTH\"}}}" > /kaniko/.docker/config.json
    - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/docker/Dockerfile --destination $CI_REGISTRY_IMAGE:latest
  # Build Docker is needed only if code as changed.
  when: manual

test:
  image: $CI_REGISTRY_IMAGE:latest
  script:
  - make test
  stage: test
  tags:
  - openfisca

clean_folder:
  before_script:
    - ''
  script: rm -rf $ROOT_FOLDER/$OUT_FOLDER || true
  stage: build_collection
  tags:
  - openfisca
  when: manual

build_collection:
  image: $CI_REGISTRY_IMAGE:latest
  script:
  - echo "Begin with fresh config"
  - rm -rf $ROOT_FOLDER/$OUT_FOLDER || true
  - mkdir -p $ROOT_FOLDER/$OUT_FOLDER/data_collections/
  - mkdir -p $ROOT_FOLDER/$OUT_FOLDER/data_output/
  - cp ./.gitlab-ci/openfisca_survey_manager_config.ini ~/.config/openfisca-survey-manager/config.ini
  - echo "Custom output folder"
  - sed -i "s/BRANCH_NAME/$OUT_FOLDER/" ~/.config/openfisca-survey-manager/config.ini
  - 'echo "{\"name\": \"erfs_fpr\", \"surveys\": {}}" > $ROOT_FOLDER/$OUT_FOLDER/data_collections/erfs_fpr.json'
  - 'echo "{\"name\": \"openfisca_erfs_fpr\", \"surveys\": {}}" > $ROOT_FOLDER/$OUT_FOLDER/data_collections/openfisca_erfs_fpr.json'
  - cat ~/.config/openfisca-survey-manager/config.ini
  - cat ~/.config/openfisca-survey-manager/raw_data.ini
  - '#build-collection -c enquete_logement -d -m -s 2013'
  - build-collection -c erfs_fpr -d -m -v -p ~/.config/openfisca-survey-manager/
  - echo "Backup updated config"
  - cp ~/.config/openfisca-survey-manager/config.ini $ROOT_FOLDER/$OUT_FOLDER/openfisca_survey_manager_config-after-build-collection.ini
  stage: build_collection
  tags:
  - openfisca
  when: manual

copy_previous_build_collections:
  before_script:
    - ''
  script: |
    if [[ -f "$ROOT_FOLDER/$OUT_FOLDER/data_collections/erfs_fpr.json" ]]; then
        echo "Files already exists, do nothing."
    else
        rm -rf $ROOT_FOLDER/$OUT_FOLDER || true
        mkdir -p $ROOT_FOLDER/$OUT_FOLDER/data_collections/
        mkdir -p $ROOT_FOLDER/$OUT_FOLDER/data_output/
        cp $ROOT_FOLDER/master/openfisca_survey_manager_config-after-build-collection.ini $ROOT_FOLDER/$OUT_FOLDER/openfisca_survey_manager_config-after-build-collection.ini
        sed -i "s/master/$OUT_FOLDER/" $ROOT_FOLDER/$OUT_FOLDER/openfisca_survey_manager_config-after-build-collection.ini
        cp $ROOT_FOLDER/master/data_collections/erfs_fpr.json $ROOT_FOLDER/$OUT_FOLDER/data_collections/erfs_fpr.json
        cp ./.gitlab-ci/empty_openfisca_erfs_fpr.json $ROOT_FOLDER/$OUT_FOLDER/data_collections/openfisca_erfs_fpr.json
    fi
  stage: build_collection
  tags:
  - openfisca
  except:
  - master

diagnostics:
  artifacts:
    paths:
    - figures_directory
  image: $CI_REGISTRY_IMAGE:latest
  needs:
  - input_data-2019
  - input_data-2021
  script:
  - echo "diagnotics-2021"
  - cp $ROOT_FOLDER/$OUT_FOLDER/openfisca_survey_manager_config_input_data-after-build-erfs-fprs-2021.ini
    ~/.config/openfisca-survey-manager/config.ini
  - mkdir -p ~/.config/openfisca-france-data
  - cp ./.gitlab-ci/openfisca_france_data_config.ini
    ~/.config/openfisca-france-data/config.ini
  - sed -i "s/BRANCH_NAME/$OUT_FOLDER/" ~/.config/openfisca-france-data/config.ini
  - cat ~/.config/openfisca-france-data/config.ini
  - compare-erfs-fpr-input -u -s -v
  - cp -r /mnt/data-out/openfisca-france-data/$OUT_FOLDER/tests/figures_directory .
  stage: diagnostics
  tags:
  - openfisca

build_and_test_pote:
  image: $CI_REGISTRY_IMAGE:latest
  script:
  - cp ./.gitlab-ci/pote_openfisca_survey_manager_config.ini ~/.config/openfisca-survey-manager/config.ini
  - python tests/pote/create_fake_data.py -y 2022 -p /tests/pote/fake_data/raw/
  - build-pote -y 2022 -c ~/.config/openfisca-survey-manager/
  - pytest tests/pote/test_pote_survey_scenario.py
  stage: pote

check-version-and-changelog:
  stage: diagnostics
  before_script:
  - ''
  needs:
  - input_data-2019
  - input_data-2021
  script:
  - .gitlab-ci/is-version-number-acceptable.sh

run_on_all_years:
  stage: run_on_all_years
  # Prevent call of before_script because it will fail in this context
  before_script:
    - ''
  script:
    - echo "On ne fait rien"
  when: manual

check-for-functional-changes:
  stage: anaconda
  needs:
  - check-version-and-changelog
  before_script:
  - ''
  script:
  - if `.gitlab-ci/has-functional-changes.sh` ; then echo "OK to build package" ; fi
  only:
  - master

build_conda_package:
  stage: anaconda
  needs:
  - check-for-functional-changes
  before_script:
  - ''
  only:
  - master
  image: continuumio/miniconda3
  script:
  - conda install -y conda-build anaconda-client
  - conda build -c conda-forge -c openfisca --token $ANACONDA_TOKEN --user OpenFisca .conda