ARROW-17353: [Release][R] Validate binaries version (#14396)

We generate version in Crossbow not CI job.
We can use file name validation feature by using version generated by Crossbow.

We use "X.Y.Z" version for RC version such as "10.0.0" for "10.0.0-rc1".

Authored-by: Sutou Kouhei <kou@clear-code.com>
Signed-off-by: Sutou Kouhei <kou@clear-code.com>

Author: kou

dev/archery/archery/crossbow/core.py

@@ -738,14 +738,16 @@ class Target(Serializable):
     (currently only an email address where the notification should be sent).
     """
 
-    def __init__(self, head, branch, remote, version, email=None):
+    def __init__(self, head, branch, remote, version, r_version, email=None):
         self.head = head
         self.email = email
         self.branch = branch
         self.remote = remote
         self.github_repo = "/".join(_parse_github_user_repo(remote))
         self.version = version
+        self.r_version = r_version
         self.no_rc_version = re.sub(r'-rc\d+\Z', '', version)
+        self.no_rc_r_version = re.sub(r'-rc\d+\Z', '', r_version)
         # TODO(ARROW-17552): Remove "master" from default_branch after
         #                    migration to "main".
         self.default_branch = ['main', 'master']
@@ -791,8 +793,39 @@ def from_repo(cls, repo, head=None, branch=None, remote=None, version=None,
         if email is None:
             email = repo.user_email
 
+        version_dev_match = re.match(r".*\.dev(\d+)$", version)
+        if version_dev_match:
+            with open(f"{repo.path}/r/DESCRIPTION") as description_file:
+                description = description_file.read()
+                r_version_pattern = re.compile(r"^Version:\s*(.*)$",
+                                               re.MULTILINE)
+                r_version = re.findall(r_version_pattern, description)[0]
+            if r_version:
+                version_dev = int(version_dev_match[1])
+                # "1_0000_00_00 +" is for generating a greater version
+                # than YYYYMMDD. For example, 1_0000_00_01
+                # (version_dev == 1 case) is greater than 2022_10_16.
+                #
+                # Why do we need a greater version than YYYYMMDD? It's
+                # for keeping backward compatibility. We used
+                # MAJOR.MINOR.PATCH.YYYYMMDD as our nightly package
+                # version. (See also ARROW-16403). If we use "9000 +
+                # version_dev" here, a developer that used
+                # 9.0.0.20221016 can't upgrade to the later nightly
+                # package unless we release 10.0.0. Because 9.0.0.9234
+                # or something is less than 9.0.0.20221016.
+                r_version_dev = 1_0000_00_00 + version_dev
+                # version: 10.0.0.dev234
+                # r_version: 9.0.0.9000
+                # -> 9.0.0.100000234
+                r_version = re.sub(r"\.9000\Z", f".{r_version_dev}", r_version)
+            else:
+                r_version = version
+        else:
+            r_version = version
+
         return cls(head=head, email=email, branch=branch, remote=remote,
-                   version=version)
+                   version=version, r_version=r_version)
 
     def is_default_branch(self):
         # TODO(ARROW-17552): Switch the condition to "is" instead of "in"
@@ -1105,7 +1138,10 @@ def from_config(cls, config, target, tasks=None, groups=None, params=None):
             'version': target.version,
             'no_rc_version': target.no_rc_version,
             'no_rc_semver_version': target.no_rc_semver_version,
-            'no_rc_snapshot_version': target.no_rc_snapshot_version}
+            'no_rc_snapshot_version': target.no_rc_snapshot_version,
+            'r_version': target.r_version,
+            'no_rc_r_version': target.no_rc_r_version,
+        }
         for task_name, task in task_definitions.items():
             task = task.copy()
             artifacts = task.pop('artifacts', None) or []  # because of yaml
@@ -1260,6 +1296,7 @@ def validate(self):
             branch='master',
             remote='https://github.com/apache/arrow',
             version='1.0.0dev123',
+            r_version='0.13.0.100000123',
             email='dummy@example.ltd'
         )
         job = Job.from_config(config=self,

dev/archery/archery/crossbow/tests/fixtures/crossbow-job-no-failure.yaml

@@ -5,7 +5,9 @@ target: !Target
   branch: refs/pull/4435/merge
   remote: https://github.com/apache/arrow
   version: 0.13.0.dev306
+  r_version: 0.12.0.100000306
   no_rc_version: 0.13.0.dev306
+  no_rc_r_version: 0.12.0.100000306
 tasks:
   docker-cpp-cmake32: !Task
     ci: circle
@@ -64,4 +66,4 @@ branch: ursabot-1
 _queue: !Queue
   path: the_path
   github_token: xxxxxxxxx
-  _remote_url: https://github.com/apache/crossbow
+  _remote_url: https://github.com/apache/crossbow

dev/archery/archery/crossbow/tests/fixtures/crossbow-job.yaml

@@ -5,7 +5,9 @@ target: !Target
   branch: refs/pull/4435/merge
   remote: https://github.com/apache/arrow
   version: 0.13.0.dev306
+  r_version: 0.12.0.100000306
   no_rc_version: 0.13.0.dev306
+  no_rc_r_version: 0.12.0.100000306
 tasks:
   docker-cpp-cmake32: !Task
     ci: circle
@@ -64,4 +66,4 @@ branch: ursabot-1
 _queue: !Queue
   path: the_path
   github_token: xxxxxxxxx
-  _remote_url: https://github.com/apache/crossbow
+  _remote_url: https://github.com/apache/crossbow

dev/tasks/macros.jinja

@@ -269,7 +269,7 @@ on:
       rm -f apache-arrow*.rb.bak
 {% endmacro %}
 
-{%- macro github_change_r_pkg_version(is_fork, version = '\\2.\'\"$(date +%Y%m%d)\"\'' ) -%}
+{%- macro github_change_r_pkg_version(is_fork, version) -%}
   - name: Modify version
     shell: bash
     run: |
@@ -339,12 +339,16 @@ on:
     # tree not available in git-bash on windows
     run: |
       ls -R repo
-  - name: Add dev repo to .Rprofile
+  - name: Add repos to .Rprofile
     shell: Rscript {0}
     run: |
-      str <- paste0("options(arrow.dev_repo ='file://", getwd(), "/repo' )")
-      print(str)
       profile_path <- file.path(getwd(), ".Rprofile")
+      repo <- paste0("file://", getwd(), "/repo")
+      str <- paste0("options(arrow.repo = '", repo, "' )")
+      print(str)
+      write(str, file = profile_path, append = TRUE)
+      str <- paste0("options(arrow.dev_repo = '", repo, "' )")
+      print(str)
       write(str, file = profile_path, append = TRUE)
       # Set envvar for later steps by appending to $GITHUB_ENV
       write(paste0("R_PROFILE_USER=", profile_path), file = Sys.getenv("GITHUB_ENV"), append = TRUE)

dev/tasks/r/github.packages.yml

@@ -17,10 +17,6 @@
 
 {% import 'macros.jinja' as macros with context %}
 
-# This allows us to set a custom version via param:
-# crossbow submit --param custom_version=8.5.3 r-binary-packages
-# if the param is unset defaults to the usual Ymd naming scheme
-{% set package_version = custom_version|replace("Unset", "\\2.\'\"$(date +%Y%m%d)\"\'") %}
 {% set is_fork = macros.is_fork %}
 
 {{ macros.github_header() }}
@@ -35,7 +31,7 @@ jobs:
       pkg_version: {{ '${{ steps.save-version.outputs.pkg_version }}' }}
     steps:
       {{ macros.github_checkout_arrow()|indent }}
-      {{ macros.github_change_r_pkg_version(is_fork, package_version)|indent }}
+      {{ macros.github_change_r_pkg_version(is_fork, arrow.no_rc_r_version)|indent }}
       - name: Save Version
         id: save-version
         shell: bash
@@ -281,7 +277,7 @@ jobs:
           {{  macros.github_test_r_src_pkg()|indent(8) }}
           '
       - name: Upload binary artifact
-        if:  matrix.config.devtoolset
+        if: matrix.config.devtoolset
         uses: actions/upload-artifact@v3
         with:
           name: r-pkg_centos7

dev/tasks/tasks.yml

@@ -957,17 +957,17 @@ tasks:
     params:
       custom_version: Unset
     artifacts:
-      - r-lib__libarrow__bin__windows__arrow-[0-9\.]+\.zip
-      - r-lib__libarrow__bin__centos-7__arrow-[0-9\.]+\.zip
-      - r-lib__libarrow__bin__ubuntu-18.04__arrow-[0-9\.]+\.zip
-      - r-lib__libarrow__bin__ubuntu-22.04__arrow-[0-9\.]+\.zip
-      - r-pkg__bin__windows__contrib__4.1__arrow_[0-9\.]+\.zip
-      - r-pkg__bin__windows__contrib__4.2__arrow_[0-9\.]+\.zip
-      - r-pkg__bin__macosx__contrib__4.1__arrow_[0-9\.]+\.tgz
-      - r-pkg__bin__macosx__contrib__4.2__arrow_[0-9\.]+\.tgz
-      - r-pkg__bin__macosx__big-sur-arm64__contrib__4.1__arrow_[0-9\.]+\.tgz
-      - r-pkg__bin__macosx__big-sur-arm64__contrib__4.2__arrow_[0-9\.]+\.tgz
-      - r-pkg__src__contrib__arrow_[0-9\.]+\.tar\.gz
+      - r-lib__libarrow__bin__windows__arrow-{no_rc_r_version}\.zip
+      - r-lib__libarrow__bin__centos-7__arrow-{no_rc_r_version}\.zip
+      - r-lib__libarrow__bin__ubuntu-18.04__arrow-{no_rc_r_version}\.zip
+      - r-lib__libarrow__bin__ubuntu-22.04__arrow-{no_rc_r_version}\.zip
+      - r-pkg__bin__windows__contrib__4.1__arrow_{no_rc_r_version}\.zip
+      - r-pkg__bin__windows__contrib__4.2__arrow_{no_rc_r_version}\.zip
+      - r-pkg__bin__macosx__contrib__4.1__arrow_{no_rc_r_version}\.tgz
+      - r-pkg__bin__macosx__contrib__4.2__arrow_{no_rc_r_version}\.tgz
+      - r-pkg__bin__macosx__big-sur-arm64__contrib__4.1__arrow_{no_rc_r_version}\.tgz
+      - r-pkg__bin__macosx__big-sur-arm64__contrib__4.2__arrow_{no_rc_r_version}\.tgz
+      - r-pkg__src__contrib__arrow_{no_rc_r_version}\.tar\.gz
 
 
   ########################### Release verification ############################

r/tools/nixlibs.R

@@ -32,7 +32,7 @@ dev_version <- package_version(VERSION)[1, 4]
 # Small dev versions are added for R-only changes during CRAN submission.
 if (is.na(dev_version) || dev_version < 100) {
   VERSION <- package_version(VERSION)[1, 1:3]
-  arrow_repo <- sprintf("https://apache.jfrog.io/artifactory/arrow/r/%s/libarrow/", VERSION)
+  arrow_repo <- paste0(getOption("arrow.repo", sprintf("https://apache.jfrog.io/artifactory/arrow/r/%s", VERSION)), "/libarrow/")
 } else {
   arrow_repo <- paste0(getOption("arrow.dev_repo", "https://nightlies.apache.org/arrow/r"), "/libarrow/")
 }

r/tools/winlibs.R

@@ -44,7 +44,10 @@ if (!file.exists(sprintf("windows/arrow-%s/include/arrow/api.h", VERSION))) {
       "/libarrow/bin/windows/arrow-%s.zip"
     )
     # %1$s uses the first variable for both substitutions
-    artifactory <- "https://apache.jfrog.io/artifactory/arrow/r/%1$s/libarrow/bin/windows/arrow-%1$s.zip"
+    artifactory <- paste0(
+      getOption("arrow.repo", "https://apache.jfrog.io/artifactory/arrow/r/%1$s"),
+      "/libarrow/bin/windows/arrow-%1$s.zip"
+    )
     rwinlib <- "https://github.com/rwinlib/arrow/archive/v%s.zip"
 
     dev_version <- package_version(VERSION)[1, 4]