apacheGH-44062: [Dev][Archery][Integration] Reduce needless test matr…

…ix (apache#44099)

### Rationale for this change

If we enable C++, Java and Rust, we use the following patterns:

| Producer | Consumer |
|----------|----------|
| C++      | C++      |
| C++      | Java     |
| C++      | Rust     |
| Java     | C++      |
| Java     | Java     |
| Java     | Rust     |
| Rust     | C++      |
| Rust     | Java     |
| Rust     | Rust     |

In apache/arrow, the following patterns are redundant because they should be done in apache/arrow-rs:

| Producer | Consumer |
|----------|----------|
| Rust     | Rust     |

In apache/arror-rs, the following patterns are redundant because they should be done in apache/arrow:

| Producer | Consumer |
|----------|----------|
| C++      | C++      |
| C++      | Java     |
| Java     | C++      |
| Java     | Java     |

### What changes are included in this PR?

Add `--target-languages` option. We can specify target languages by this. Here are expected usages:

In apache/arrow:
* `--target-languages=cpp,csharp,go,java,js`

In apache/arrow-rs
* `--target-languages=rust`

Here is an example in apache/arrow-rs:

Used matrix:

| Producer | Consumer |
|----------|----------|
| Rust     | Rust     |
| Rust     | C++      |
| Rust     | C#       |
| Rust     | Go       |
| Rust     | Java     |
| Rust     | JS       |
| Rust     | nanoarrow|
| C++      | Rust     |
| C#       | Rust     |
| Go       | Rust     |
| Java     | Rust     |
| JS       | Rust     |
| nanoarrow| Rust     |

If no `--target-languages` is specified, all enabled languages are the target languages. (The same as the current behavior.)

### Are these changes tested?

Yes.

### Are there any user-facing changes?

No.
* GitHub Issue: apache#44062

Authored-by: Sutou Kouhei <kou@clear-code.com>
Signed-off-by: Sutou Kouhei <kou@clear-code.com>

ci/scripts/integration_arrow.sh

@@ -30,6 +30,9 @@ gold_dir=$arrow_dir/testing/data/arrow-ipc-stream/integration
 : ${ARROW_INTEGRATION_JAVA:=ON}
 : ${ARROW_INTEGRATION_JS:=ON}
 
+: ${ARCHERY_INTEGRATION_TARGET_LANGUAGES:=cpp,csharp,go,java,js}
+export ARCHERY_INTEGRATION_TARGET_LANGUAGES
+
 . ${arrow_dir}/ci/scripts/util_log.sh
 
 github_actions_group_begin "Integration: Prepare: Archery"

dev/archery/archery/cli.py

@@ -744,6 +744,9 @@ def _set_default(opt, default):
 @click.option('--with-rust', type=bool, default=False,
  help='Include Rust in integration tests',
  envvar="ARCHERY_INTEGRATION_WITH_RUST")
+@click.option('--target-languages', default='',
+ help=('Target languages in this integration tests'),
+ envvar="ARCHERY_INTEGRATION_TARGET_LANGUAGES")
 @click.option('--write_generated_json', default="",
  help='Generate test JSON to indicated path')
 @click.option('--run-ipc', is_flag=True, default=False,

dev/archery/archery/integration/runner.py

@@ -67,12 +67,13 @@ class IntegrationRunner(object):
 
  def __init__(self, json_files,
  flight_scenarios: List[Scenario],
- testers: List[Tester], tempdir=None,
- debug=False, stop_on_error=True, gold_dirs=None,
+ testers: List[Tester], other_testers: List[Tester],
+ tempdir=None, debug=False, stop_on_error=True, gold_dirs=None,
  serial=False, match=None, **unused_kwargs):
  self.json_files = json_files
  self.flight_scenarios = flight_scenarios
  self.testers = testers
+ self.other_testers = other_testers
  self.temp_dir = tempdir or tempfile.mkdtemp()
  self.debug = debug
  self.stop_on_error = stop_on_error
@@ -100,6 +101,20 @@ def run_ipc(self):
  producer, consumer, self._produce_consume,
  self.json_files)
 
+ for producer, consumer in itertools.product(
+ filter(lambda t: t.PRODUCER, self.testers),
+ filter(lambda t: t.CONSUMER, self.other_testers)):
+ self._compare_ipc_implementations(
+ producer, consumer, self._produce_consume,
+ self.json_files)
+
+ for producer, consumer in itertools.product(
+ filter(lambda t: t.PRODUCER, self.other_testers),
+ filter(lambda t: t.CONSUMER, self.testers)):
+ self._compare_ipc_implementations(
+ producer, consumer, self._produce_consume,
+ self.json_files)
+
  if self.gold_dirs:
  for gold_dir, consumer in itertools.product(
  self.gold_dirs,
@@ -124,7 +139,7 @@ def run_flight(self):
  """
  servers = filter(lambda t: t.FLIGHT_SERVER, self.testers)
  clients = filter(lambda t: (t.FLIGHT_CLIENT and t.CONSUMER),
- self.testers)
+ self.testers + self.other_testers)
  for server, client in itertools.product(servers, clients):
  self._compare_flight_implementations(server, client)
  log('\n')
@@ -138,6 +153,14 @@ def run_c_data(self):
  filter(lambda t: t.C_DATA_SCHEMA_EXPORTER, self.testers),
  filter(lambda t: t.C_DATA_SCHEMA_IMPORTER, self.testers)):
  self._compare_c_data_implementations(producer, consumer)
+ for producer, consumer in itertools.product(
+ filter(lambda t: t.C_DATA_SCHEMA_EXPORTER, self.testers),
+ filter(lambda t: t.C_DATA_SCHEMA_IMPORTER, self.other_testers)):
+ self._compare_c_data_implementations(producer, consumer)
+ for producer, consumer in itertools.product(
+ filter(lambda t: t.C_DATA_SCHEMA_EXPORTER, self.other_testers),
+ filter(lambda t: t.C_DATA_SCHEMA_IMPORTER, self.testers)):
+ self._compare_c_data_implementations(producer, consumer)
  log('\n')
 
  def _gold_tests(self, gold_dir):
@@ -560,31 +583,42 @@ def get_static_json_files():
 def run_all_tests(with_cpp=True, with_java=True, with_js=True,
  with_csharp=True, with_go=True, with_rust=False,
  with_nanoarrow=False, run_ipc=False, run_flight=False,
- run_c_data=False, tempdir=None, **kwargs):
+ run_c_data=False, tempdir=None, target_languages="",
+ **kwargs):
  tempdir = tempdir or tempfile.mkdtemp(prefix='arrow-integration-')
+ print(["before", target_languages])
+ target_languages = list(filter(len, target_languages.split(",")))
+ print(["after", target_languages])
 
  testers: List[Tester] = []
+ other_testers: List[Tester] = []
+
+ def append_tester(language, tester):
+ if len(target_languages) == 0 or language in target_languages:
+ testers.append(tester)
+ else:
+ other_testers.append(tester)
 
  if with_cpp:
- testers.append(CppTester(**kwargs))
+ append_tester("cpp", CppTester(**kwargs))
 
  if with_java:
- testers.append(JavaTester(**kwargs))
+ append_tester("java", JavaTester(**kwargs))
 
  if with_js:
- testers.append(JSTester(**kwargs))
+ append_tester("js", JSTester(**kwargs))
 
  if with_csharp:
- testers.append(CSharpTester(**kwargs))
+ append_tester("csharp", CSharpTester(**kwargs))
 
  if with_go:
- testers.append(GoTester(**kwargs))
+ append_tester("go", GoTester(**kwargs))
 
  if with_nanoarrow:
- testers.append(NanoarrowTester(**kwargs))
+ append_tester("nanoarrow", NanoarrowTester(**kwargs))
 
  if with_rust:
- testers.append(RustTester(**kwargs))
+ append_tester("rust", RustTester(**kwargs))
 
  static_json_files = get_static_json_files()
  generated_json_files = datagen.get_generated_json_files(tempdir=tempdir)
@@ -666,7 +700,8 @@ def run_all_tests(with_cpp=True, with_java=True, with_js=True,
  ),
  ]
 
- runner = IntegrationRunner(json_files, flight_scenarios, testers, **kwargs)
+ runner = IntegrationRunner(json_files, flight_scenarios, testers,
+ other_testers, **kwargs)
  if run_ipc:
  runner.run_ipc()
  if run_flight: