From ce2f25ed376152c15775138bdaee93a0a37368ff Mon Sep 17 00:00:00 2001 From: MINGYJ Date: Thu, 3 Jul 2025 11:36:40 -0400 Subject: [PATCH 01/20] iRODS: Replace Jargon with irods4j --- irods/bin/pom.xml | 78 ++++++++ ...DS (iPlant Collaborative).cyberduckprofile | 28 +++ irods/output.txt | 151 ++++++++++++++++ irods/pom.xml | 28 +-- .../ch/cyberduck/core/irods/ChunkWorker.java | 75 ++++++++ .../DefaultTransferOptionsConfigurer.java | 38 ---- ...DefaultTransferStatusCallbackListener.java | 90 ---------- .../irods/IRODSAttributesFinderFeature.java | 84 ++++++--- .../core/irods/IRODSCopyFeature.java | 82 +++++---- .../core/irods/IRODSDeleteFeature.java | 40 +++-- .../core/irods/IRODSDirectoryFeature.java | 21 ++- .../core/irods/IRODSDownloadFeature.java | 127 +++++++++---- .../irods/IRODSExceptionMappingService.java | 40 +---- .../core/irods/IRODSFindFeature.java | 18 +- .../core/irods/IRODSHomeFinderService.java | 8 +- .../core/irods/IRODSListService.java | 121 +++++++++---- .../core/irods/IRODSMoveFeature.java | 27 ++- .../cyberduck/core/irods/IRODSProtocol.java | 8 +- .../core/irods/IRODSReadFeature.java | 60 +++---- .../ch/cyberduck/core/irods/IRODSSession.java | 167 ++++++------------ .../core/irods/IRODSTouchFeature.java | 23 ++- .../core/irods/IRODSUploadFeature.java | 145 ++++++++------- .../core/irods/IRODSWriteFeature.java | 100 ++++++++--- .../IRODSExceptionMappingServiceTest.java | 12 +- .../core/irods/IRODSReadFeatureTest.java | 6 +- .../core/irods/IRODSWriteFeatureTest.java | 8 +- .../java/ch/cyberduck/core/irods/input.txt | 151 ++++++++++++++++ .../java/ch/cyberduck/core/irods/output.txt | 151 ++++++++++++++++ 28 files changed, 1270 insertions(+), 617 deletions(-) create mode 100644 irods/bin/pom.xml create mode 100644 irods/bin/src/test/resources/iRODS (iPlant Collaborative).cyberduckprofile create mode 100644 irods/output.txt create mode 100644 irods/src/main/java/ch/cyberduck/core/irods/ChunkWorker.java delete mode 100644 irods/src/main/java/ch/cyberduck/core/irods/DefaultTransferOptionsConfigurer.java delete mode 100644 irods/src/main/java/ch/cyberduck/core/irods/DefaultTransferStatusCallbackListener.java create mode 100644 irods/src/test/java/ch/cyberduck/core/irods/input.txt create mode 100644 irods/src/test/java/ch/cyberduck/core/irods/output.txt diff --git a/irods/bin/pom.xml b/irods/bin/pom.xml new file mode 100644 index 00000000000..ac92788e04c --- /dev/null +++ b/irods/bin/pom.xml @@ -0,0 +1,78 @@ + + + + 4.0.0 + + ch.cyberduck + parent + 9.2.0-SNAPSHOT + + irods + jar + + + + ch.cyberduck + core + ${project.version} + + + ch.cyberduck + test + pom + test + ${project.version} + + + + + + + + org.apache.maven.plugins + maven-failsafe-plugin + + true + + + + + + diff --git a/irods/bin/src/test/resources/iRODS (iPlant Collaborative).cyberduckprofile b/irods/bin/src/test/resources/iRODS (iPlant Collaborative).cyberduckprofile new file mode 100644 index 00000000000..7d5484f947f --- /dev/null +++ b/irods/bin/src/test/resources/iRODS (iPlant Collaborative).cyberduckprofile @@ -0,0 +1,28 @@ + + + + + Protocol + irods + Vendor + iplant + Description + iPlant Data Store + Hostname Configurable + + Port Configurable + + Default Hostname + data.iplantcollaborative.org + Region + iplant + Default Port + 1247 + Username Placeholder + iPlant username + Password Placeholder + iPlant password + Authorization + STANDARD + + diff --git a/irods/output.txt b/irods/output.txt new file mode 100644 index 00000000000..5adda2b1cb5 --- /dev/null +++ b/irods/output.txt @@ -0,0 +1,151 @@ +irods.host=172.20.0.2 +irods.port=1247 +irods.zoneName=tempZone + +# STANDARD | PAM AUTH +irods.auth.scheme=STANDARD +default.storage.resource= + +# sets jargon ssl negotiation policy for the client. Leaving to DONT_CARE defers to the server, and is recommended +# NO_NEGOTIATION, CS_NEG_REFUSE, CS_NEG_REQUIRE, CS_NEG_DONT_CARE +ssl.negotiation.policy=CS_NEG_REFUSE + +########################################################## +# jargon properties settings +utilize.packing.streams=true + +# jargon now supports checksum calculation for streaming uploads. This does not currently verify, but does store if set to true +compute.checksum=true + +###################################### +# other irods environment + +# HTTP connection for RMD +rmd.connection.timeout=500 +rmd.connection.port=8000 + +# Reverse DNS lookup on dashboard +reverse.dns.lookup=false + +###################################### +# msi props +populate.msi.enabled=false +illumina.msi.enabled=true + +# MSI API version supported by this application +msi.api.version=1.X.X + +msi.metalnx.list=libmsiget_illumina_meta.so,libmsiobjget_microservices.so,libmsiobjget_version.so,libmsiobjjpeg_extract.so,libmsiobjput_mdbam.so,libmsiobjput_mdbam.so,libmsiobjput_mdmanifest.so,libmsiobjput_mdvcf.so,libmsiobjput_populate.so + +msi.irods.list=libmsisync_to_archive.so,libmsi_update_unixfilesystem_resource_free_space.so,libmsiobjput_http.so,libmsiobjput_irods.so,libmsiobjget_irods.so,libmsiobjget_http.so,libmsiobjput_slink.so,libmsiobjget_slink.so + +msi.irods.42.list=libmsisync_to_archive.so,libmsi_update_unixfilesystem_resource_free_space.so + +msi.other.list= +###################################### +# global feature flags that serve as defaults. Note that the info handling will manipulate aspects of the data profiler, +# so by default some things are set to null to be turned on by the service depending on the view requested (e.g. acl, metadata, replicas) and should be left 'false' as a default, +# but other aspects, such as metadata templating and mime type detection, can be globally turned on or off depending on the implmenetation. +# controls access to features globally +metalnx.enable.tickets=false +# disable automatic detection and running of rules on upload +metalnx.enable.upload.rules=false +# download size limit in megabytes (6000=6GB) +metalnx.download.limit=6000 +# show dashboard (off by default due to performance issues) +metalnx.enable.dashboard=false +###################################### +# info home page feature flags +# this controls the behavior of the data profiler and what information it will gather +irodsext.dataprofiler.retrieve.tickets=false +# process starred or favorites +irodsext.dataprofiler.retrieve.starred=true +# process shared +irodsext.dataprofiler.retrieve.shared=false +# tags and comments +irodsext.dataprofiler.retrieve.tags.and.comments=false +# metadata templates (currently not implemented) +irodsext.dataprofiler.retrieve.metadata.templates=false +# save data type information for later use +irodsext.datatyper.persist.data.types=false +# perform a detailed versus a lightweight data typing, which may involve processing the file contents +irodsext.datatyper.detailed.determination=false + +############################# +# misc ui configuration niceties +############################# +# allow translation of iRODS auth types to user friendly names in login +# in the form irodstype:displaytype| +metalnx.authtype.mappings=PAM:PAM|STANDARD:Standard + +############################# +# JWT configuration (necessary when using search and notification services). Otherwise can be left as-is and ignored +############################# +jwt.issuer=metalnx +jwt.secret=thisisasecretthatisverysecretyouwillneverguessthiskey +jwt.algo=HS384 + +############################# +# Pluggable search configuration. Turn on and off pluggable search globally, and configure search endpoints. +# N.B. pluggable search also requires provisioning of the jwt.* information above +############################# +# configured endpoints, comma delimited in form https://host.com/v1 +# Note the commented out URL which matches up to the irods-contrib/file-and-metadata-indexer docker compose assembly. In order to +# utilize this assembly you need to uncomment the URL and set pluggablesearch.enabled to true +pluggablesearch.endpointRegistryList= +# enable pluggable search globally and show the search GUI components +pluggablesearch.enabled=false +# display the older file and properties search in the menu, if you are running the elasticsearch standard plugin this is probably +# a menu item to turn off +classicsearch.enabled=true +# JWT subject claim used to access search endpoint for data gathering. User searches will utilize the name of the individual +pluggablesearch.endpointAccessSubject=metalnx +# timeout for info/attribute gathering, set to 0 for no timeout +pluggablesearch.info.timeout=0 +# timeout for actual search, set to 0 for no timeout +pluggablesearch.search.timeout=0 + +############################# +# Pluggable shopping cart and export plugin configuration. +# Turn on and off pluggable shopping cart globally, and configure export endpoints. +# N.B. plugins also requires provisioning of the jwt.* information above +############################# + +# enable pluggable export globally and show the export GUI components +pluggableshoppingcart.enabled=false + +# configured endpoints, comma delimited in form https://host.com/v1 +pluggablepublishing.endpointRegistryList= +# timeout for info/attribute gathering, set to 0 for no timeout +pluggablepublishing.info.timeout=0 + +# timeout for actual publishing, set to 0 for no timeout +pluggablepublishing.publishing.timeout=0 + +# server rule engine instance that will provide the galleryview listing +gallery_view.rule_engine_plugin.instance_name=irods_rule_engine_plugin-irods_rule_language-instance + +########################################################## +# Metadata Masking Properties +# +# Excludes metadata when the attribute name starts with at least one prefix. +# Multiple prefixes can be defined by separating them with the character sequence defined +# by metalnx.metadata.mask.delimiter. +# +# For example, the configuration below will hide any metadata that contains an attribute +# starting with "irods::", "metalnx-", or "_system_". +# +# metalnx.metadata.mask.prefixes=irods::;metalnx-;_system_ +# metalnx.metadata.mask.delimiter=; +# +# Use the iRODS Metadata Guard rule engine plugin to protect your metadata namespaces from +# being modified. +metalnx.metadata.mask.prefixes= +metalnx.metadata.mask.delimiter=, + + +########################################################## +# Setting to enable/disable the "Public" sidebar link. +# The default is "false" (hidden) +########################################################## +sidebar.show.public=false diff --git a/irods/pom.xml b/irods/pom.xml index ba6609fe2c5..2d183a05682 100644 --- a/irods/pom.xml +++ b/irods/pom.xml @@ -24,6 +24,11 @@ jar + + org.irods + irods4j + 0.2.0-java8 + ch.cyberduck core @@ -36,29 +41,6 @@ test ${project.version} - - ch.iterate.jargon - jargon-core - 4.2.0.1 - - - com.claymoresystems - puretls - - - org.globus.jglobus - cog-jglobus - - - org.perf4j - perf4j - - - log4j - log4j - - - diff --git a/irods/src/main/java/ch/cyberduck/core/irods/ChunkWorker.java b/irods/src/main/java/ch/cyberduck/core/irods/ChunkWorker.java new file mode 100644 index 00000000000..af1a532c949 --- /dev/null +++ b/irods/src/main/java/ch/cyberduck/core/irods/ChunkWorker.java @@ -0,0 +1,75 @@ +package ch.cyberduck.core.irods; + +import java.io.IOException; +import java.io.RandomAccessFile; + +import org.irods.irods4j.high_level.io.IRODSDataObjectInputStream; +import org.irods.irods4j.high_level.io.IRODSDataObjectOutputStream; +import org.irods.irods4j.high_level.io.IRODSDataObjectStream.SeekDirection; +import org.irods.irods4j.low_level.api.IRODSException; + +public class ChunkWorker implements Runnable{ + private final Object stream; + private final RandomAccessFile file; + private final long offset; + private final long chunkSize; + private final byte[] buffer; + public ChunkWorker(Object stream, String localfilePath,long offset, long chunkSize, int bufferSize) throws IOException, IRODSException { + this.stream= stream; + this.file=new RandomAccessFile(localfilePath,"rw"); + this.offset=offset; + this.chunkSize=chunkSize; + this.buffer=new byte[bufferSize]; + + + file.seek(offset); + } + @Override + public void run(){ + try { + if (stream instanceof IRODSDataObjectInputStream) { + IRODSDataObjectInputStream in = (IRODSDataObjectInputStream) (stream); + in.seek((int) offset, SeekDirection.CURRENT); + long remaining = chunkSize; + while (remaining > 0) { + int readLength = (int) Math.min(buffer.length, remaining); + int read = in.read(buffer, 0, readLength); + file.write(buffer, 0, read); + remaining -= read; + } + + }else if (stream instanceof IRODSDataObjectOutputStream) { + IRODSDataObjectOutputStream out = (IRODSDataObjectOutputStream) (stream); + out.seek((int) offset, SeekDirection.CURRENT); + long remaining = chunkSize; + while (remaining > 0) { + int writeLength = (int) Math.min(buffer.length, remaining); + int read = file.read(buffer, 0, writeLength); + if (read == -1) break; + out.write(buffer, 0, read); + remaining -= read; + } + }else { + throw new IllegalArgumentException("Unsupported stream type"); + } + } catch (IOException | IRODSException e) { + e.printStackTrace(); + } + try { + close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + public void close() throws IOException { + file.close(); + if (stream instanceof IRODSDataObjectInputStream) { + IRODSDataObjectInputStream in = (IRODSDataObjectInputStream) (stream); + in.close(); + } else if (stream instanceof IRODSDataObjectOutputStream) { + IRODSDataObjectOutputStream out = (IRODSDataObjectOutputStream) (stream); + out.close(); + } + } +} diff --git a/irods/src/main/java/ch/cyberduck/core/irods/DefaultTransferOptionsConfigurer.java b/irods/src/main/java/ch/cyberduck/core/irods/DefaultTransferOptionsConfigurer.java deleted file mode 100644 index f57c2e21217..00000000000 --- a/irods/src/main/java/ch/cyberduck/core/irods/DefaultTransferOptionsConfigurer.java +++ /dev/null @@ -1,38 +0,0 @@ -package ch.cyberduck.core.irods; - -/* - * Copyright (c) 2002-2015 David Kocher. All rights reserved. - * http://cyberduck.ch/ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * Bug fixes, suggestions and comments should be sent to feedback@cyberduck.ch - */ - -import ch.cyberduck.core.preferences.Preferences; -import ch.cyberduck.core.preferences.PreferencesFactory; - -import org.irods.jargon.core.packinstr.TransferOptions; - -public class DefaultTransferOptionsConfigurer { - - private final Preferences preferences = PreferencesFactory.get(); - - public TransferOptions configure(final TransferOptions options) { - options.setPutOption(TransferOptions.PutOptions.NORMAL); - options.setForceOption(TransferOptions.ForceOption.ASK_CALLBACK_LISTENER); - options.setMaxThreads(preferences.getInteger("queue.connections.limit.default")); - // Enable progress callbacks - options.setIntraFileStatusCallbacks(true); - options.setIntraFileStatusCallbacksNumberCallsInterval(1); - return options; - } -} diff --git a/irods/src/main/java/ch/cyberduck/core/irods/DefaultTransferStatusCallbackListener.java b/irods/src/main/java/ch/cyberduck/core/irods/DefaultTransferStatusCallbackListener.java deleted file mode 100644 index accf71879f9..00000000000 --- a/irods/src/main/java/ch/cyberduck/core/irods/DefaultTransferStatusCallbackListener.java +++ /dev/null @@ -1,90 +0,0 @@ -package ch.cyberduck.core.irods; - -/* - * Copyright (c) 2002-2015 David Kocher. All rights reserved. - * http://cyberduck.ch/ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * Bug fixes, suggestions and comments should be sent to feedback@cyberduck.ch - */ - -import ch.cyberduck.core.BytecountStreamListener; -import ch.cyberduck.core.exception.ConnectionCanceledException; -import ch.cyberduck.core.io.StreamListener; -import ch.cyberduck.core.transfer.TransferStatus; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.irods.jargon.core.transfer.TransferControlBlock; -import org.irods.jargon.core.transfer.TransferStatusCallbackListener; - -public class DefaultTransferStatusCallbackListener implements TransferStatusCallbackListener { - private static final Logger log = LogManager.getLogger(DefaultTransferStatusCallbackListener.class); - - private final TransferStatus status; - private final BytecountStreamListener listener; - private final TransferControlBlock block; - - public DefaultTransferStatusCallbackListener(final TransferStatus status, final StreamListener listener, - final TransferControlBlock block) { - this.status = status; - this.listener = new BytecountStreamListener(listener); - this.block = block; - } - - @Override - public FileStatusCallbackResponse statusCallback(final org.irods.jargon.core.transfer.TransferStatus t) { - log.debug("Progress with {}", t); - final long bytes = t.getBytesTransfered() - listener.getSent(); - switch(t.getTransferType()) { - case GET: - listener.recv(bytes); - break; - case PUT: - listener.sent(bytes); - break; - } - try { - status.validate(); - if(!t.isIntraFileStatusReport()) { - if(t.getTotalFilesTransferredSoFar() == t.getTotalFilesToTransfer()) { - status.setComplete(); - } - } - } - catch(ConnectionCanceledException e) { - log.debug("Set canceled for block {}", block); - block.setCancelled(true); - return FileStatusCallbackResponse.SKIP; - } - return FileStatusCallbackResponse.CONTINUE; - } - - @Override - public void overallStatusCallback(final org.irods.jargon.core.transfer.TransferStatus t) { - // - } - - @Override - public CallbackResponse transferAsksWhetherToForceOperation(final String irodsAbsolutePath, final boolean isCollection) { - try { - status.validate(); - } - catch(ConnectionCanceledException e) { - return CallbackResponse.CANCEL; - } - if(status.isAppend()) { - return CallbackResponse.NO_THIS_FILE; - } - return CallbackResponse.YES_THIS_FILE; - } -} diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSAttributesFinderFeature.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSAttributesFinderFeature.java index 8abbc111bda..7b4ce7b4f8c 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSAttributesFinderFeature.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSAttributesFinderFeature.java @@ -1,5 +1,18 @@ package ch.cyberduck.core.irods; +import java.io.IOException; +import java.util.List; + +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.lang3.StringUtils; +import org.irods.irods4j.common.Versioning; +import org.irods.irods4j.high_level.catalog.IRODSQuery; +import org.irods.irods4j.high_level.catalog.IRODSQuery.GenQuery1QueryArgs; +import org.irods.irods4j.high_level.connection.IRODSConnection; +import org.irods.irods4j.high_level.vfs.IRODSFilesystem; +import org.irods.irods4j.low_level.api.GenQuery1Columns; +import org.irods.irods4j.low_level.api.IRODSException; + /* * Copyright (c) 2002-2016 iterate GmbH. All rights reserved. * https://cyberduck.io/ @@ -24,14 +37,8 @@ import ch.cyberduck.core.features.AttributesFinder; import ch.cyberduck.core.io.Checksum; -import org.apache.commons.codec.binary.Base64; -import org.apache.commons.codec.binary.Hex; -import org.irods.jargon.core.exception.JargonException; -import org.irods.jargon.core.pub.IRODSFileSystemAO; -import org.irods.jargon.core.pub.domain.ObjStat; -import org.irods.jargon.core.pub.io.IRODSFile; -public class IRODSAttributesFinderFeature implements AttributesFinder, AttributesAdapter { +public class IRODSAttributesFinderFeature implements AttributesFinder, AttributesAdapter> { private final IRODSSession session; @@ -42,28 +49,65 @@ public IRODSAttributesFinderFeature(final IRODSSession session) { @Override public PathAttributes find(final Path file, final ListProgressListener listener) throws BackgroundException { try { - final IRODSFileSystemAO fs = session.getClient(); - final IRODSFile f = fs.getIRODSFileFactory().instanceIRODSFile(file.getAbsolute()); - if(!f.exists()) { + final PathAttributes[] attributes = new PathAttributes[1]; + final IRODSConnection conn = session.getClient(); + if(!IRODSFilesystem.exists(this.session.getClient().getRcComm(), file.getAbsolute())) { throw new NotfoundException(file.getAbsolute()); } - final ObjStat stats = fs.getObjStat(f.getAbsolutePath()); - return this.toAttributes(stats); + String logicalPath = file.getAbsolute(); + String parentPath = FilenameUtils.getFullPathNoEndSeparator(logicalPath); + String fileName = FilenameUtils.getName(logicalPath); + if(Versioning.compareVersions(conn.getRcComm().relVersion.substring(4), "4.3.4") > 0) { + String query = String.format("select DATA_MODIFY_TIME, DATA_CREATE_TIME, DATA_SIZE, DATA_CHECKSUM, DATA_OWNER_NAME, DATA_OWNER_ZONE where COLL_NAME = '%s' and DATA_NAME = '%s'", parentPath, fileName); + List> rows = IRODSQuery.executeGenQuery2(conn.getRcComm(), query); + List row = rows.get(0); + attributes[0]=toAttributes(row); + }else { + GenQuery1QueryArgs input = new GenQuery1QueryArgs(); + + // select COLL_NAME, DATA_NAME, DATA_ACCESS_TIME + input.addColumnToSelectClause(GenQuery1Columns.COL_D_MODIFY_TIME); + input.addColumnToSelectClause(GenQuery1Columns.COL_D_CREATE_TIME); + input.addColumnToSelectClause(GenQuery1Columns.COL_DATA_SIZE); + input.addColumnToSelectClause(GenQuery1Columns.COL_D_DATA_CHECKSUM); + input.addColumnToSelectClause(GenQuery1Columns.COL_D_OWNER_NAME); + input.addColumnToSelectClause(GenQuery1Columns.COL_D_OWNER_ZONE); + + + // where COLL_NAME like '/tempZone/home/rods and DATA_NAME = 'atime.txt' + String collNameCondStr = String.format("= '%s'", parentPath); + String dataNameCondStr = String.format("= '%s'", fileName); + input.addConditionToWhereClause(GenQuery1Columns.COL_COLL_NAME, collNameCondStr); + input.addConditionToWhereClause(GenQuery1Columns.COL_DATA_NAME, dataNameCondStr); + + StringBuilder output = new StringBuilder(); + + IRODSQuery.executeGenQuery1(conn.getRcComm(), input, row -> { + attributes[0]=toAttributes(row); + return false; + }); + } + return attributes[0]; } - catch(JargonException e) { + catch(IOException | IRODSException e) { throw new IRODSExceptionMappingService().map("Failure to read attributes of {0}", e, file); } } @Override - public PathAttributes toAttributes(final ObjStat stats) { + public PathAttributes toAttributes(final List row) { + final IRODSConnection conn = session.getClient(); final PathAttributes attributes = new PathAttributes(); - attributes.setModificationDate(stats.getModifiedAt().getTime()); - attributes.setCreationDate(stats.getCreatedAt().getTime()); - attributes.setSize(stats.getObjSize()); - attributes.setChecksum(Checksum.parse(Hex.encodeHexString(Base64.decodeBase64(stats.getChecksum())))); - attributes.setOwner(stats.getOwnerName()); - attributes.setGroup(stats.getOwnerZone()); + attributes.setModificationDate(Long.parseLong(row.get(0)) * 1000); // seconds to ms + attributes.setCreationDate(Long.parseLong(row.get(1)) * 1000); + attributes.setSize(Long.parseLong(row.get(2))); + String checksum = row.get(3); + if (!StringUtils.isEmpty(checksum)) { + attributes.setChecksum(Checksum.parse(checksum)); + } + + attributes.setOwner(conn.getRcComm().relVersion); + attributes.setGroup(row.get(5)); return attributes; } } diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSCopyFeature.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSCopyFeature.java index 0687bdf24d7..98c1a187c1c 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSCopyFeature.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSCopyFeature.java @@ -1,5 +1,16 @@ package ch.cyberduck.core.irods; +import java.io.IOException; +import java.util.EnumSet; + +import org.irods.irods4j.high_level.connection.IRODSConnection; +import org.irods.irods4j.high_level.vfs.CollectionEntry; +import org.irods.irods4j.high_level.vfs.IRODSCollectionIterator; +import org.irods.irods4j.high_level.vfs.IRODSFilesystem; +import org.irods.irods4j.high_level.vfs.ObjectStatus; +import org.irods.irods4j.low_level.api.IRODSApi.RcComm; +import org.irods.irods4j.low_level.api.IRODSException; + /* * Copyright (c) 2002-2015 David Kocher. All rights reserved. * http://cyberduck.ch/ @@ -22,17 +33,6 @@ import ch.cyberduck.core.exception.BackgroundException; import ch.cyberduck.core.features.Copy; import ch.cyberduck.core.io.StreamListener; -import ch.cyberduck.core.preferences.HostPreferencesFactory; - -import org.apache.commons.lang3.StringUtils; -import org.irods.jargon.core.exception.JargonException; -import org.irods.jargon.core.pub.DataTransferOperations; -import org.irods.jargon.core.pub.IRODSFileSystemAO; -import org.irods.jargon.core.transfer.DefaultTransferControlBlock; -import org.irods.jargon.core.transfer.TransferStatus; -import org.irods.jargon.core.transfer.TransferStatusCallbackListener; - -import java.util.EnumSet; public class IRODSCopyFeature implements Copy { @@ -45,37 +45,49 @@ public IRODSCopyFeature(final IRODSSession session) { @Override public Path copy(final Path source, final Path target, final ch.cyberduck.core.transfer.TransferStatus status, final ConnectionCallback callback, final StreamListener listener) throws BackgroundException { try { - final IRODSFileSystemAO fs = session.getClient(); - final DataTransferOperations transfer = fs.getIRODSAccessObjectFactory() - .getDataTransferOperations(fs.getIRODSAccount()); - transfer.copy(fs.getIRODSFileFactory().instanceIRODSFile(source.getAbsolute()), - fs.getIRODSFileFactory().instanceIRODSFile(target.getAbsolute()), new TransferStatusCallbackListener() { - @Override - public FileStatusCallbackResponse statusCallback(final TransferStatus transferStatus) { - return FileStatusCallbackResponse.CONTINUE; - } + final IRODSConnection conn = session.getClient(); + final String from = source.getAbsolute(); + final String to = target.getAbsolute(); + if (source.isFile()) { + IRODSFilesystem.copyDataObject(conn.getRcComm(), from, to); - @Override - public void overallStatusCallback(final TransferStatus transferStatus) { - switch(transferStatus.getTransferState()) { - case OVERALL_COMPLETION: - listener.sent(status.getLength()); - } - } - - @Override - public CallbackResponse transferAsksWhetherToForceOperation(final String irodsAbsolutePath, final boolean isCollection) { - return CallbackResponse.YES_THIS_FILE; - } - }, DefaultTransferControlBlock.instance(StringUtils.EMPTY, - HostPreferencesFactory.get(session.getHost()).getInteger("connection.retry"))); + if (listener != null && status.getLength() > 0) { + listener.sent(status.getLength()); + } + } + if(source.isDirectory()) { + this.copyDirectoryRecursively(conn.getRcComm(), from, to); + } return target; } - catch(JargonException e) { + catch(IOException | IRODSException e) { throw new IRODSExceptionMappingService().map("Cannot copy {0}", e, source); } } + + public static void copyDirectoryRecursively(RcComm rcComm, String source, String target) throws IOException, IRODSException { + // First, create the root of the target directory + if (!IRODSFilesystem.exists(rcComm, target)) { + IRODSFilesystem.createCollection(rcComm, target); + } + + // Recursively iterate through the source collection + for (CollectionEntry entry : new IRODSCollectionIterator(rcComm, source)) { + String relative = entry.path().substring(source.length()); // relative path from source + String targetPath = target + relative; + ObjectStatus status = entry.status(); + + if (status.getType() == ObjectStatus.ObjectType.COLLECTION) { + // Create directory in target + IRODSFilesystem.createCollection(rcComm, targetPath); + } else if (status.getType() == ObjectStatus.ObjectType.DATA_OBJECT) { + // Copy file + IRODSFilesystem.copyDataObject(rcComm, entry.path(), targetPath); + } + } + } + @Override public EnumSet features(final Path source, final Path target) { return EnumSet.of(Flags.recursive); diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSDeleteFeature.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSDeleteFeature.java index 47261db3efa..029730dfe3b 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSDeleteFeature.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSDeleteFeature.java @@ -1,5 +1,17 @@ package ch.cyberduck.core.irods; +import java.io.IOException; +import java.util.ArrayList; +import java.util.EnumSet; +import java.util.List; +import java.util.Map; + +import org.irods.irods4j.high_level.vfs.IRODSFilesystem; +import org.irods.irods4j.high_level.vfs.IRODSFilesystem.RemoveOptions; +import org.irods.irods4j.high_level.vfs.ObjectStatus; +import org.irods.irods4j.high_level.vfs.ObjectStatus.ObjectType; +import org.irods.irods4j.low_level.api.IRODSException; + /* * Copyright (c) 2002-2015 David Kocher. All rights reserved. * http://cyberduck.ch/ @@ -24,14 +36,6 @@ import ch.cyberduck.core.features.Delete; import ch.cyberduck.core.transfer.TransferStatus; -import org.irods.jargon.core.exception.JargonException; -import org.irods.jargon.core.pub.io.IRODSFile; - -import java.util.ArrayList; -import java.util.EnumSet; -import java.util.List; -import java.util.Map; - public class IRODSDeleteFeature implements Delete { private final IRODSSession session; @@ -56,19 +60,19 @@ public void delete(final Map files, final PasswordCallback } deleted.add(file); callback.delete(file); + final String absolute = file.getAbsolute(); try { - final IRODSFile f = session.getClient().getIRODSFileFactory().instanceIRODSFile(file.getAbsolute()); - if(!f.exists()) { - throw new NotfoundException(String.format("%s doesn't exist", file.getAbsolute())); - } - if(f.isFile()) { - session.getClient().fileDeleteForce(f); - } - else if(f.isDirectory()) { - session.getClient().directoryDeleteForce(f); + if (!IRODSFilesystem.exists(this.session.getClient().getRcComm(), absolute)) { + throw new NotfoundException(String.format("%s doesn't exist", absolute)); } + ObjectStatus status = IRODSFilesystem.status(this.session.getClient().getRcComm(), absolute); + if (status.equals(ObjectType.DATA_OBJECT)) { + IRODSFilesystem.remove(this.session.getClient().getRcComm(), absolute, RemoveOptions.NO_TRASH); + } else if (status.equals(ObjectType.COLLECTION)) { + IRODSFilesystem.removeAll(this.session.getClient().getRcComm(), absolute, RemoveOptions.NO_TRASH); + } } - catch(JargonException e) { + catch(IOException | IRODSException e) { throw new IRODSExceptionMappingService().map("Cannot delete {0}", e, file); } } diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSDirectoryFeature.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSDirectoryFeature.java index 7f4e6ee90da..854b8c9349b 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSDirectoryFeature.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSDirectoryFeature.java @@ -1,5 +1,11 @@ package ch.cyberduck.core.irods; +import java.io.IOException; + +import org.irods.irods4j.high_level.connection.IRODSConnection; +import org.irods.irods4j.high_level.vfs.IRODSFilesystem; +import org.irods.irods4j.high_level.vfs.IRODSFilesystemException; + /* * Copyright (c) 2002-2015 David Kocher. All rights reserved. * http://cyberduck.ch/ @@ -23,10 +29,6 @@ import ch.cyberduck.core.features.Write; import ch.cyberduck.core.transfer.TransferStatus; -import org.irods.jargon.core.exception.JargonException; -import org.irods.jargon.core.pub.IRODSFileSystemAO; -import org.irods.jargon.core.pub.io.IRODSFile; - public class IRODSDirectoryFeature implements Directory { private final IRODSSession session; @@ -38,12 +40,15 @@ public IRODSDirectoryFeature(final IRODSSession session) { @Override public Path mkdir(final Write writer, final Path folder, final TransferStatus status) throws BackgroundException { try { - final IRODSFileSystemAO fs = session.getClient(); - final IRODSFile f = fs.getIRODSFileFactory().instanceIRODSFile(folder.getAbsolute()); - fs.mkdir(f, false); + final IRODSConnection conn = session.getClient(); + String path = folder.getAbsolute(); + boolean created = IRODSFilesystem.createCollection(conn.getRcComm(), path); + if (!created) { + throw new IOException("Failed to create collection: " + path); + } return folder; } - catch(JargonException e) { + catch(IOException | IRODSFilesystemException e) { throw new IRODSExceptionMappingService().map("Cannot create folder {0}", e, folder); } } diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSDownloadFeature.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSDownloadFeature.java index bcf7b24e554..da352fcde4a 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSDownloadFeature.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSDownloadFeature.java @@ -1,5 +1,20 @@ package ch.cyberduck.core.irods; +import java.io.RandomAccessFile; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +import org.irods.irods4j.high_level.connection.IRODSConnectionPool; +import org.irods.irods4j.high_level.connection.IRODSConnectionPool.PoolConnection; +import org.irods.irods4j.high_level.connection.QualifiedUsername; +import org.irods.irods4j.high_level.io.IRODSDataObjectInputStream; +import org.irods.irods4j.high_level.vfs.IRODSFilesystem; +import org.irods.irods4j.low_level.api.IRODSApi; +import org.irods.irods4j.low_level.api.IRODSApi.RcComm; + /* * Copyright (c) 2002-2015 David Kocher. All rights reserved. * http://cyberduck.ch/ @@ -18,7 +33,6 @@ */ import ch.cyberduck.core.ConnectionCallback; -import ch.cyberduck.core.Host; import ch.cyberduck.core.Local; import ch.cyberduck.core.Path; import ch.cyberduck.core.exception.BackgroundException; @@ -27,24 +41,14 @@ import ch.cyberduck.core.features.Read; import ch.cyberduck.core.io.BandwidthThrottle; import ch.cyberduck.core.io.StreamListener; -import ch.cyberduck.core.preferences.HostPreferencesFactory; -import ch.cyberduck.core.preferences.PreferencesFactory; import ch.cyberduck.core.transfer.TransferStatus; -import org.apache.commons.lang3.StringUtils; -import org.irods.jargon.core.exception.JargonException; -import org.irods.jargon.core.packinstr.TransferOptions; -import org.irods.jargon.core.pub.DataTransferOperations; -import org.irods.jargon.core.pub.IRODSFileSystemAO; -import org.irods.jargon.core.pub.io.IRODSFile; -import org.irods.jargon.core.transfer.DefaultTransferControlBlock; -import org.irods.jargon.core.transfer.TransferControlBlock; - -import java.io.File; - public class IRODSDownloadFeature implements Download { private final IRODSSession session; + private boolean truncate = true; + private boolean append = false; + private static final int BUFFER_SIZE = 4 * 1024 * 1024; public IRODSDownloadFeature(final IRODSSession session) { this.session = session; @@ -54,32 +58,79 @@ public IRODSDownloadFeature(final IRODSSession session) { public void download(final Read read, final Path file, final Local local, final BandwidthThrottle throttle, final StreamListener listener, final TransferStatus status, final ConnectionCallback callback) throws BackgroundException { + final int numThread = 3; try { - final IRODSFileSystemAO fs = session.getClient(); - final IRODSFile f = fs.getIRODSFileFactory().instanceIRODSFile(file.getAbsolute()); - if(f.exists()) { - final TransferControlBlock block = DefaultTransferControlBlock.instance(StringUtils.EMPTY, - HostPreferencesFactory.get(session.getHost()).getInteger("connection.retry")); - final TransferOptions options = new DefaultTransferOptionsConfigurer().configure(new TransferOptions()); - if(Host.TransferType.unknown.equals(session.getHost().getTransferType())) { - options.setUseParallelTransfer(Host.TransferType.valueOf(PreferencesFactory.get().getProperty("queue.transfer.type")).equals(Host.TransferType.concurrent)); - } - else { - options.setUseParallelTransfer(session.getHost().getTransferType().equals(Host.TransferType.concurrent)); - } - block.setTransferOptions(options); - final DataTransferOperations transfer = fs.getIRODSAccessObjectFactory() - .getDataTransferOperations(fs.getIRODSAccount()); - transfer.getOperation(f, new File(local.getAbsolute()), - new DefaultTransferStatusCallbackListener(status, listener, block), - block); - } - else { - throw new NotfoundException(file.getAbsolute()); - } + + final RcComm primaryConn = session.getClient().getRcComm(); + final String logicalPath = file.getAbsolute(); + + if (!IRODSFilesystem.exists(primaryConn, logicalPath)) { + throw new NotfoundException(logicalPath); + } + + final long fileSize = IRODSFilesystem.dataObjectSize(primaryConn, logicalPath); + + // Step 1: Get replica token & number via primary stream + try (IRODSDataObjectInputStream primary = new IRODSDataObjectInputStream(primaryConn, logicalPath)) { + final String replicaToken = primary.getReplicaToken(); + final long replicaNumber = primary.getReplicaNumber(); + + // Step 2: Setup connection pool + final IRODSConnectionPool pool = new IRODSConnectionPool(numThread); + pool.start( + session.getHost().getHostname(), + session.getHost().getPort(), + new QualifiedUsername(session.getHost().getCredentials().getUsername(), session.getRegion()), + conn -> { + try { + IRODSApi.rcAuthenticateClient(conn, "native", session.getHost().getCredentials().getPassword()); + return true; + } catch (Exception e) { + return false; + } + }); + + final ExecutorService executor = Executors.newFixedThreadPool(numThread); + + //TODO:fileSize/ + final long chunkSize = fileSize / numThread; + final long remainChunkSize = fileSize % numThread; + + + // Step 3: Create empty target file + try (RandomAccessFile out = new RandomAccessFile(local.getAbsolute(), "rw")) { + out.setLength(fileSize); + } + + // Step 4: Parallel readers + List> tasks = new ArrayList<>(); + for (int i = 0; i < numThread; i++) { + final long start = i * chunkSize; + final PoolConnection conn = pool.getConnection(); + IRODSDataObjectInputStream stream = new IRODSDataObjectInputStream(conn.getRcComm(), replicaToken, replicaNumber); + ChunkWorker worker = new ChunkWorker( + stream, + local.getAbsolute(), + start, + (numThread - 1 == i) ? chunkSize + remainChunkSize : chunkSize, + BUFFER_SIZE + ); + Future task = executor.submit(worker); + tasks.add(task); + } + + for (Future task : tasks) { + task.get(); + } + + + executor.shutdown(); + pool.close(); + } + } - catch(JargonException e) { - throw new IRODSExceptionMappingService().map("Download {0} failed", e, file); + catch(Exception e) { + throw new IRODSExceptionMappingService().map("Download {0} failed", e); } } diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSExceptionMappingService.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSExceptionMappingService.java index 244566915b0..b2030444ec9 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSExceptionMappingService.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSExceptionMappingService.java @@ -1,5 +1,8 @@ package ch.cyberduck.core.irods; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + /* * Copyright (c) 2002-2015 David Kocher. All rights reserved. * http://cyberduck.ch/ @@ -18,47 +21,18 @@ */ import ch.cyberduck.core.AbstractExceptionMappingService; -import ch.cyberduck.core.exception.AccessDeniedException; import ch.cyberduck.core.exception.BackgroundException; -import ch.cyberduck.core.exception.LoginFailureException; -import ch.cyberduck.core.exception.NotfoundException; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.irods.jargon.core.exception.AuthenticationException; -import org.irods.jargon.core.exception.CatNoAccessException; -import org.irods.jargon.core.exception.DataNotFoundException; -import org.irods.jargon.core.exception.FileNotFoundException; -import org.irods.jargon.core.exception.InvalidGroupException; -import org.irods.jargon.core.exception.InvalidUserException; -import org.irods.jargon.core.exception.JargonException; -public class IRODSExceptionMappingService extends AbstractExceptionMappingService { +public class IRODSExceptionMappingService extends AbstractExceptionMappingService { private static final Logger log = LogManager.getLogger(IRODSExceptionMappingService.class); @Override - public BackgroundException map(final JargonException e) { + public BackgroundException map(final Exception e) { + //TODO: write a more complete exception mapping services log.warn("Map failure {}", e.toString()); final StringBuilder buffer = new StringBuilder(); this.append(buffer, e.getMessage()); - if(e instanceof CatNoAccessException) { - return new AccessDeniedException(buffer.toString(), e); - } - if(e instanceof FileNotFoundException) { - return new NotfoundException(buffer.toString(), e); - } - if(e instanceof DataNotFoundException) { - return new NotfoundException(buffer.toString(), e); - } - if(e instanceof AuthenticationException) { - return new LoginFailureException(buffer.toString(), e); - } - if(e instanceof InvalidUserException) { - return new LoginFailureException(buffer.toString(), e); - } - if(e instanceof InvalidGroupException) { - return new LoginFailureException(buffer.toString(), e); - } return this.wrap(e, buffer); } + } diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSFindFeature.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSFindFeature.java index ee3154370e9..a350da3d429 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSFindFeature.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSFindFeature.java @@ -1,5 +1,11 @@ package ch.cyberduck.core.irods; +import java.io.IOException; + +import org.irods.irods4j.high_level.connection.IRODSConnection; +import org.irods.irods4j.high_level.vfs.IRODSFilesystem; +import org.irods.irods4j.low_level.api.IRODSException; + /* * Copyright (c) 2002-2015 David Kocher. All rights reserved. * http://cyberduck.ch/ @@ -22,10 +28,6 @@ import ch.cyberduck.core.exception.BackgroundException; import ch.cyberduck.core.features.Find; -import org.irods.jargon.core.exception.JargonException; -import org.irods.jargon.core.pub.IRODSFileSystemAO; -import org.irods.jargon.core.pub.io.IRODSFile; - public class IRODSFindFeature implements Find { private final IRODSSession session; @@ -40,12 +42,12 @@ public boolean find(final Path file, final ListProgressListener listener) throws return true; } try { - final IRODSFileSystemAO fs = session.getClient(); - final IRODSFile f = fs.getIRODSFileFactory().instanceIRODSFile(file.getAbsolute()); - return fs.isFileExists(f); + final IRODSConnection conn = session.getClient(); + return IRODSFilesystem.exists(conn.getRcComm(), file.getAbsolute()); } - catch(JargonException e) { + catch(IOException | IRODSException e) { throw new IRODSExceptionMappingService().map("Failure to read attributes of {0}", e, file); } + } } diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSHomeFinderService.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSHomeFinderService.java index 6720017524d..344ca0af4f0 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSHomeFinderService.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSHomeFinderService.java @@ -1,5 +1,9 @@ package ch.cyberduck.core.irods; +import java.util.EnumSet; + +import org.apache.commons.lang3.StringUtils; + /* * Copyright (c) 2002-2016 iterate GmbH. All rights reserved. * https://cyberduck.io/ @@ -20,10 +24,6 @@ import ch.cyberduck.core.exception.BackgroundException; import ch.cyberduck.core.shared.AbstractHomeFeature; -import org.apache.commons.lang3.StringUtils; - -import java.util.EnumSet; - public class IRODSHomeFinderService extends AbstractHomeFeature { private final IRODSSession session; diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSListService.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSListService.java index 9df21070b9e..439b364b501 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSListService.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSListService.java @@ -1,5 +1,21 @@ package ch.cyberduck.core.irods; +import java.io.IOException; +import java.util.EnumSet; +import java.util.List; + +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.lang3.StringUtils; +import org.irods.irods4j.common.Versioning; +import org.irods.irods4j.high_level.catalog.IRODSQuery; +import org.irods.irods4j.high_level.catalog.IRODSQuery.GenQuery1QueryArgs; +import org.irods.irods4j.high_level.connection.IRODSConnection; +import org.irods.irods4j.high_level.vfs.CollectionEntry; +import org.irods.irods4j.high_level.vfs.IRODSCollectionIterator; +import org.irods.irods4j.high_level.vfs.IRODSFilesystem; +import org.irods.irods4j.low_level.api.GenQuery1Columns; +import org.irods.irods4j.low_level.api.IRODSException; + /* * Copyright (c) 2002-2015 David Kocher. All rights reserved. * http://cyberduck.ch/ @@ -27,18 +43,6 @@ import ch.cyberduck.core.exception.NotfoundException; import ch.cyberduck.core.io.Checksum; -import org.apache.commons.codec.binary.Base64; -import org.apache.commons.codec.binary.Hex; -import org.apache.commons.io.filefilter.TrueFileFilter; -import org.apache.commons.lang3.StringUtils; -import org.irods.jargon.core.exception.JargonException; -import org.irods.jargon.core.pub.IRODSFileSystemAO; -import org.irods.jargon.core.pub.domain.ObjStat; -import org.irods.jargon.core.pub.io.IRODSFile; - -import java.io.File; -import java.util.EnumSet; - public class IRODSListService implements ListService { private final IRODSSession session; @@ -51,32 +55,81 @@ public IRODSListService(IRODSSession session) { public AttributedList list(final Path directory, final ListProgressListener listener) throws BackgroundException { try { final AttributedList children = new AttributedList(); - final IRODSFileSystemAO fs = session.getClient(); - final IRODSFile f = fs.getIRODSFileFactory().instanceIRODSFile(directory.getAbsolute()); - if(!f.exists()) { - throw new NotfoundException(directory.getAbsolute()); + final IRODSConnection conn = session.getClient(); + String path = directory.getAbsolute(); + if (!IRODSFilesystem.exists(conn.getRcComm(), path)) { + throw new NotfoundException(path); } - for(File file : fs.getListInDirWithFileFilter(f, TrueFileFilter.TRUE)) { - final String normalized = PathNormalizer.normalize(file.getAbsolutePath(), true); - if(StringUtils.equals(normalized, directory.getAbsolute())) { - continue; - } - final PathAttributes attributes = new PathAttributes(); - final ObjStat stats = fs.getObjStat(file.getAbsolutePath()); - attributes.setModificationDate(stats.getModifiedAt().getTime()); - attributes.setCreationDate(stats.getCreatedAt().getTime()); - attributes.setSize(stats.getObjSize()); - attributes.setChecksum(Checksum.parse(Hex.encodeHexString(Base64.decodeBase64(stats.getChecksum())))); - attributes.setOwner(stats.getOwnerName()); - attributes.setGroup(stats.getOwnerZone()); - children.add(new Path(directory, PathNormalizer.name(normalized), - file.isDirectory() ? EnumSet.of(Path.Type.directory) : EnumSet.of(Path.Type.file), - attributes)); - listener.chunk(directory, children); + final IRODSCollectionIterator iterator = new IRODSCollectionIterator(conn.getRcComm(), path); + + for (CollectionEntry entry : iterator) { + final String normalized = PathNormalizer.normalize(entry.path(), true); + if(StringUtils.equals(normalized, directory.getAbsolute())) { + continue; + } + final PathAttributes attributes = new PathAttributes(); + String logicalPath = entry.path(); + String parentPath = FilenameUtils.getFullPathNoEndSeparator(logicalPath); + String fileName = FilenameUtils.getName(logicalPath); + String query = ""; + //check version + if(Versioning.compareVersions(conn.getRcComm().relVersion.substring(4), "4.3.4") > 0) { + query = String.format("select DATA_MODIFY_TIME, DATA_CREATE_TIME, DATA_SIZE, DATA_CHECKSUM, DATA_OWNER_NAME, DATA_OWNER_ZONE where COLL_NAME = '%s' and DATA_NAME = '%s'", parentPath, fileName); + List> rows = IRODSQuery.executeGenQuery2(conn.getRcComm(), query); + List row = rows.get(0); + attributes.setModificationDate(Long.parseLong(row.get(0)) * 1000); // seconds to ms + attributes.setCreationDate(Long.parseLong(row.get(1)) * 1000); + attributes.setSize(Long.parseLong(row.get(2))); + String checksum = row.get(3); + if (!StringUtils.isEmpty(checksum)) { + attributes.setChecksum(Checksum.parse(checksum)); + } + + attributes.setOwner(row.get(4)); + attributes.setGroup(row.get(5)); + }else { + //if older version, use GenQuery1 + GenQuery1QueryArgs input = new GenQuery1QueryArgs(); + + // select COLL_NAME, DATA_NAME, DATA_ACCESS_TIME + input.addColumnToSelectClause(GenQuery1Columns.COL_D_MODIFY_TIME); + input.addColumnToSelectClause(GenQuery1Columns.COL_D_CREATE_TIME); + input.addColumnToSelectClause(GenQuery1Columns.COL_DATA_SIZE); + input.addColumnToSelectClause(GenQuery1Columns.COL_D_DATA_CHECKSUM); + input.addColumnToSelectClause(GenQuery1Columns.COL_D_OWNER_NAME); + input.addColumnToSelectClause(GenQuery1Columns.COL_D_OWNER_ZONE); + + + // where COLL_NAME like '/tempZone/home/rods and DATA_NAME = 'atime.txt' + String collNameCondStr = String.format("= '%s'", parentPath); + String dataNameCondStr = String.format("= '%s'", fileName); + input.addConditionToWhereClause(GenQuery1Columns.COL_COLL_NAME, collNameCondStr); + input.addConditionToWhereClause(GenQuery1Columns.COL_DATA_NAME, dataNameCondStr); + + IRODSQuery.executeGenQuery1(conn.getRcComm(), input, row -> { + attributes.setModificationDate(Long.parseLong(row.get(0)) * 1000); // seconds to ms + attributes.setCreationDate(Long.parseLong(row.get(1)) * 1000); + attributes.setSize(Long.parseLong(row.get(2))); + String checksum = row.get(3); + if (!StringUtils.isEmpty(checksum)) { + attributes.setChecksum(Checksum.parse(checksum)); + } + + attributes.setOwner(row.get(4)); + attributes.setGroup(row.get(5)); + return false; + }); + } + EnumSet type = entry.isCollection() + ? EnumSet.of(Path.Type.directory) + : EnumSet.of(Path.Type.file); + + children.add(new Path(directory, PathNormalizer.name(normalized), type, attributes)); + listener.chunk(directory, children); } return children; } - catch(JargonException e) { + catch(IRODSException | IOException e) { throw new IRODSExceptionMappingService().map("Listing directory {0} failed", e, directory); } } diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSMoveFeature.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSMoveFeature.java index 67c70daee15..b36eb99b8da 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSMoveFeature.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSMoveFeature.java @@ -1,5 +1,13 @@ package ch.cyberduck.core.irods; +import java.io.IOException; +import java.util.Collections; +import java.util.EnumSet; + +import org.irods.irods4j.high_level.connection.IRODSConnection; +import org.irods.irods4j.high_level.vfs.IRODSFilesystem; +import org.irods.irods4j.low_level.api.IRODSException; + /* * Copyright (c) 2002-2015 David Kocher. All rights reserved. * http://cyberduck.ch/ @@ -25,13 +33,6 @@ import ch.cyberduck.core.features.Move; import ch.cyberduck.core.transfer.TransferStatus; -import org.irods.jargon.core.exception.JargonException; -import org.irods.jargon.core.pub.IRODSFileSystemAO; -import org.irods.jargon.core.pub.io.IRODSFile; - -import java.util.Collections; -import java.util.EnumSet; - public class IRODSMoveFeature implements Move { private final IRODSSession session; @@ -45,19 +46,17 @@ public IRODSMoveFeature(IRODSSession session) { @Override public Path move(final Path file, final Path renamed, final TransferStatus status, final Delete.Callback callback, final ConnectionCallback connectionCallback) throws BackgroundException { try { - final IRODSFileSystemAO fs = session.getClient(); - final IRODSFile s = fs.getIRODSFileFactory().instanceIRODSFile(file.getAbsolute()); - if(!s.exists()) { + final IRODSConnection conn = session.getClient(); + if(!IRODSFilesystem.exists(conn.getRcComm(), file.getAbsolute())) { throw new NotfoundException(String.format("%s doesn't exist", file.getAbsolute())); } if(status.isExists()) { delete.delete(Collections.singletonMap(renamed, status), connectionCallback, callback); } - final IRODSFile d = fs.getIRODSFileFactory().instanceIRODSFile(renamed.getAbsolute()); - s.renameTo(d); - return renamed; + IRODSFilesystem.rename(conn.getRcComm(), file.getAbsolute(), renamed.getAbsolute()); + return renamed; } - catch(JargonException e) { + catch(IOException | IRODSException e) { throw new IRODSExceptionMappingService().map("Cannot rename {0}", e, file); } } diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSProtocol.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSProtocol.java index c8e48bdea4e..8a7ecc913af 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSProtocol.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSProtocol.java @@ -1,5 +1,9 @@ package ch.cyberduck.core.irods; +import org.apache.commons.lang3.StringUtils; + +import com.google.auto.service.AutoService; + /* * Copyright (c) 2002-2015 David Kocher. All rights reserved. * http://cyberduck.ch/ @@ -22,10 +26,6 @@ import ch.cyberduck.core.Protocol; import ch.cyberduck.core.Scheme; -import org.apache.commons.lang3.StringUtils; - -import com.google.auto.service.AutoService; - @AutoService(Protocol.class) public final class IRODSProtocol extends AbstractProtocol { diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSReadFeature.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSReadFeature.java index 636cf788056..81ef579d3d4 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSReadFeature.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSReadFeature.java @@ -1,5 +1,13 @@ package ch.cyberduck.core.irods; +import java.io.IOException; +import java.io.InputStream; + +import org.irods.irods4j.high_level.io.IRODSDataObjectInputStream; +import org.irods.irods4j.high_level.vfs.IRODSFilesystem; +import org.irods.irods4j.low_level.api.IRODSApi.RcComm; +import org.irods.irods4j.low_level.api.IRODSException; + /* * Copyright (c) 2002-2015 David Kocher. All rights reserved. * http://cyberduck.ch/ @@ -22,18 +30,7 @@ import ch.cyberduck.core.exception.BackgroundException; import ch.cyberduck.core.exception.NotfoundException; import ch.cyberduck.core.features.Read; -import ch.cyberduck.core.io.StreamCopier; import ch.cyberduck.core.transfer.TransferStatus; -import ch.cyberduck.core.worker.DefaultExceptionMappingService; - -import org.irods.jargon.core.exception.JargonException; -import org.irods.jargon.core.exception.JargonRuntimeException; -import org.irods.jargon.core.pub.IRODSFileSystemAO; -import org.irods.jargon.core.pub.io.IRODSFile; -import org.irods.jargon.core.pub.io.IRODSFileFactory; -import org.irods.jargon.core.pub.io.PackingIrodsInputStream; - -import java.io.InputStream; public class IRODSReadFeature implements Read { @@ -45,31 +42,28 @@ public IRODSReadFeature(IRODSSession session) { @Override public InputStream read(final Path file, final TransferStatus status, final ConnectionCallback callback) throws BackgroundException { - try { - try { - final IRODSFileSystemAO fs = session.getClient(); - final IRODSFileFactory factory = fs.getIRODSFileFactory(); - final IRODSFile f = factory.instanceIRODSFile(file.getAbsolute()); - if(f.exists()) { - final InputStream in = new PackingIrodsInputStream(factory.instanceIRODSFileInputStream(f)); - if(status.isAppend()) { - return StreamCopier.skip(in, status.getOffset()); - } - return in; + + try { + final RcComm rcComm = session.getClient().getRcComm(); + final String logicalPath = file.getAbsolute(); // e.g., "/zone/home/user/file.txt" + + if (!IRODSFilesystem.exists(rcComm, logicalPath)) { + throw new NotfoundException(logicalPath); } - else { - throw new NotfoundException(file.getAbsolute()); + + // Open input stream + InputStream in = new IRODSDataObjectInputStream(rcComm,logicalPath); + + // If resuming from offset, skip ahead + if(status.isAppend() && status.getOffset() > 0) { + in.skip(status.getOffset()); } + + return in; } - catch(JargonRuntimeException e) { - if(e.getCause() instanceof JargonException) { - throw (JargonException) e.getCause(); - } - throw new DefaultExceptionMappingService().map(e); + catch(IOException | IRODSException e) { + throw new IRODSExceptionMappingService().map("Download {0} failed", e, file); } - } - catch(JargonException e) { - throw new IRODSExceptionMappingService().map("Download {0} failed", e, file); - } + } } diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSSession.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSSession.java index fa8303775b2..3329fe29ccf 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSSession.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSSession.java @@ -1,5 +1,15 @@ package ch.cyberduck.core.irods; +import java.text.MessageFormat; + +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.irods.irods4j.high_level.connection.IRODSConnection; +import org.irods.irods4j.high_level.connection.QualifiedUsername; +import org.irods.irods4j.low_level.api.IRODSApi.ConnectionOptions; + + /* * Copyright (c) 2002-2015 David Kocher. All rights reserved. * http://cyberduck.ch/ @@ -25,7 +35,6 @@ import ch.cyberduck.core.ListService; import ch.cyberduck.core.LocaleFactory; import ch.cyberduck.core.LoginCallback; -import ch.cyberduck.core.URIEncoder; import ch.cyberduck.core.exception.BackgroundException; import ch.cyberduck.core.exception.LoginFailureException; import ch.cyberduck.core.features.AttributesFinder; @@ -49,23 +58,8 @@ import ch.cyberduck.core.ssl.X509TrustManager; import ch.cyberduck.core.threading.CancelCallback; -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.irods.jargon.core.connection.AuthScheme; -import org.irods.jargon.core.connection.IRODSAccount; -import org.irods.jargon.core.connection.SettableJargonProperties; -import org.irods.jargon.core.connection.auth.AuthResponse; -import org.irods.jargon.core.exception.JargonException; -import org.irods.jargon.core.pub.IRODSAccessObjectFactory; -import org.irods.jargon.core.pub.IRODSFileSystem; -import org.irods.jargon.core.pub.IRODSFileSystemAO; - -import java.net.URI; -import java.net.URISyntaxException; -import java.text.MessageFormat; -public class IRODSSession extends SSLSession { +public class IRODSSession extends SSLSession { private static final Logger log = LogManager.getLogger(IRODSSession.class); public IRODSSession(final Host h) { @@ -77,38 +71,33 @@ public IRODSSession(final Host h, final X509TrustManager trust, final X509KeyMan } @Override - protected IRODSFileSystemAO connect(final ProxyFinder proxy, final HostKeyCallback key, final LoginCallback prompt, final CancelCallback cancel) throws BackgroundException { + protected IRODSConnection connect(final ProxyFinder proxy, final HostKeyCallback key, final LoginCallback prompt, final CancelCallback cancel) throws BackgroundException { try { - final IRODSFileSystem fs = this.configure(IRODSFileSystem.instance()); - final IRODSAccessObjectFactory factory = fs.getIRODSAccessObjectFactory(); - final String region = this.getRegion(); - final String resource = this.getResource(); - final Credentials credentials = host.getCredentials(); - try { - return factory.getIRODSFileSystemAO(new URIEncodingIRODSAccount(credentials.getUsername(), credentials.getPassword(), - new IRODSHomeFinderService(IRODSSession.this).find().getAbsolute(), region, resource)); - } - catch(IllegalArgumentException e) { - throw new LoginFailureException(e.getMessage(), e); - } - } - catch(JargonException e) { - throw new IRODSExceptionMappingService().map(e); + final String host = "localhost"; + final int port = 1247; + final String zone = "tempZone"; + + ConnectionOptions options=new ConnectionOptions(); + + + IRODSConnection conn = new IRODSConnection(options); + conn.connect(host, port, new QualifiedUsername("rods", zone)); + + + return conn; + } catch (Exception e) { + String msg = String.format("exception=[%s], host=[%s], port=[%d], username=[%s], zone=[%s]", e.getMessage(), this.host.getHostname(),this.host.getPort(), this.host.getCredentials().getUsername(),getRegion()); + throw new BackgroundException("Failed to connect to iRODS - "+ msg,e); } } - protected IRODSFileSystem configure(final IRODSFileSystem client) { - final SettableJargonProperties properties = new SettableJargonProperties(client.getJargonProperties()); - properties.setEncoding(host.getEncoding()); - final int timeout = ConnectionTimeoutFactory.get(preferences).getTimeout() * 1000; - properties.setIrodsSocketTimeout(timeout); - properties.setIrodsParallelSocketTimeout(timeout); - properties.setGetBufferSize(preferences.getInteger("connection.chunksize")); - properties.setPutBufferSize(preferences.getInteger("connection.chunksize")); - log.debug("Configure client {} with properties {}", client, properties); - client.getIrodsSession().setJargonProperties(properties); - client.getIrodsSession().setX509TrustManager(trust); - return client; + protected ConnectionOptions configure(final ConnectionOptions options) { + //TODO update to use configure + final PreferencesReader preferences = HostPreferencesFactory.get(host); + options.tcpReceiveBufferSize=preferences.getInteger("connection.chunksize"); + options.tcpSendBufferSize=preferences.getInteger("connection.chunksize"); + + return options; } protected String getRegion() { @@ -127,30 +116,35 @@ protected String getResource() { @Override public void login(final LoginCallback prompt, final CancelCallback cancel) throws BackgroundException { - try { - final IRODSAccount account = client.getIRODSAccount(); + try { final Credentials credentials = host.getCredentials(); - account.setUserName(credentials.getUsername()); - account.setPassword(credentials.getPassword()); - final AuthResponse response = client.getIRODSAccessObjectFactory().authenticateIRODSAccount(account); - log.debug("Connected to {}", response.getStartupResponse()); - if(!response.isSuccessful()) { - throw new LoginFailureException(MessageFormat.format(LocaleFactory.localizedString( - "Login {0} with username and password", "Credentials"), BookmarkNameProvider.toString(host))); - } - } - catch(JargonException e) { - throw new IRODSExceptionMappingService().map(e); + final String username = credentials.getUsername(); + final String password = credentials.getPassword(); + final String authScheme = StringUtils.defaultIfBlank( + host.getProtocol().getAuthorization(), + "native" + ); + //irods4j authenticate + client.authenticate(authScheme, password); + + log.debug("Authenticated to iRODS as {}", username); + } + catch (Exception e) { + throw new LoginFailureException(MessageFormat.format(LocaleFactory.localizedString( + "Login {0} with username and password", "Credentials"), + BookmarkNameProvider.toString(host)), e); } } @Override - public void disconnect() throws BackgroundException { - try { - client.getIRODSSession().closeSession(); - } - catch(JargonException e) { - throw new IRODSExceptionMappingService().map(e); + protected void logout() throws BackgroundException { + try { + if (client != null) { + client.disconnect(); + client = null; + } + } catch (Exception e) { + throw new BackgroundException("Failed to disconnect from iRODS", e); } finally { super.disconnect(); @@ -196,50 +190,5 @@ public T _getFeature(final Class type) { return super._getFeature(type); } - private final class URIEncodingIRODSAccount extends IRODSAccount { - public URIEncodingIRODSAccount(final String user, final String password, final String home, final String region, final String resource) { - super(host.getHostname(), host.getPort(), StringUtils.isBlank(user) ? StringUtils.EMPTY : user, password, home, region, resource); - this.setUserName(user); - } - - @Override - public URI toURI(final boolean includePassword) throws JargonException { - try { - return new URI(String.format("irods://%s.%s%s@%s:%d%s", - this.getUserName(), - this.getZone(), - includePassword ? String.format(":%s", this.getPassword()) : StringUtils.EMPTY, - this.getHost(), - this.getPort(), - URIEncoder.encode(this.getHomeDirectory()))); - } - catch(URISyntaxException e) { - throw new JargonException(e.getMessage()); - } - } - @Override - public void setUserName(final String input) { - final String user; - final AuthScheme scheme; - if(StringUtils.contains(input, ':')) { - // Support non default auth scheme (PAM) - user = StringUtils.splitPreserveAllTokens(input, ':')[1]; - // Defaults to standard if not found - scheme = AuthScheme.findTypeByString(StringUtils.splitPreserveAllTokens(input, ':')[0]); - } - else { - user = input; - if(StringUtils.isNotBlank(host.getProtocol().getAuthorization())) { - scheme = AuthScheme.findTypeByString(host.getProtocol().getAuthorization()); - } - else { - // We can default to Standard if not specified - scheme = AuthScheme.STANDARD; - } - } - super.setUserName(user); - this.setAuthenticationScheme(scheme); - } - } } diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSTouchFeature.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSTouchFeature.java index ac159c7fdd5..e7b5a7de3d3 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSTouchFeature.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSTouchFeature.java @@ -1,5 +1,11 @@ package ch.cyberduck.core.irods; +import java.io.IOException; + +import org.irods.irods4j.high_level.connection.IRODSConnection; +import org.irods.irods4j.high_level.io.IRODSDataObjectOutputStream; +import org.irods.irods4j.low_level.api.IRODSException; + /* * Copyright (c) 2002-2015 David Kocher. All rights reserved. * http://cyberduck.ch/ @@ -23,9 +29,6 @@ import ch.cyberduck.core.features.Write; import ch.cyberduck.core.transfer.TransferStatus; -import org.irods.jargon.core.exception.JargonException; -import org.irods.jargon.core.packinstr.DataObjInp; -import org.irods.jargon.core.pub.IRODSFileSystemAO; public class IRODSTouchFeature implements Touch { @@ -38,13 +41,17 @@ public IRODSTouchFeature(final IRODSSession session) { @Override public Path touch(final Write writer, final Path file, final TransferStatus status) throws BackgroundException { try { - final IRODSFileSystemAO fs = session.getClient(); - final int descriptor = fs.createFile(file.getAbsolute(), - DataObjInp.OpenFlags.WRITE_TRUNCATE, DataObjInp.DEFAULT_CREATE_MODE); - fs.fileClose(descriptor, false); + + // Open and immediately close the file to create/truncate it + final IRODSConnection conn=session.getClient(); + try (IRODSDataObjectOutputStream out = new IRODSDataObjectOutputStream(conn.getRcComm(), file.getAbsolute(), + true /* truncate if exists */, false /* don't append */)) { + // File is created or truncated by opening the stream + } + return file; } - catch(JargonException e) { + catch(IOException|IRODSException e) { throw new IRODSExceptionMappingService().map("Cannot create {0}", e, file); } } diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSUploadFeature.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSUploadFeature.java index 93f4b591879..1fcd23f48ee 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSUploadFeature.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSUploadFeature.java @@ -1,5 +1,21 @@ package ch.cyberduck.core.irods; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.irods.irods4j.high_level.connection.IRODSConnectionPool; +import org.irods.irods4j.high_level.connection.IRODSConnectionPool.PoolConnection; +import org.irods.irods4j.high_level.connection.QualifiedUsername; +import org.irods.irods4j.high_level.io.IRODSDataObjectOutputStream; +import org.irods.irods4j.high_level.vfs.IRODSFilesystem; +import org.irods.irods4j.low_level.api.IRODSApi; +import org.irods.irods4j.low_level.api.IRODSApi.RcComm; + /* * Copyright (c) 2002-2015 David Kocher. All rights reserved. * http://cyberduck.ch/ @@ -18,95 +34,92 @@ */ import ch.cyberduck.core.ConnectionCallback; -import ch.cyberduck.core.Host; import ch.cyberduck.core.Local; -import ch.cyberduck.core.LocaleFactory; import ch.cyberduck.core.Path; import ch.cyberduck.core.ProgressListener; import ch.cyberduck.core.exception.BackgroundException; -import ch.cyberduck.core.exception.ChecksumException; import ch.cyberduck.core.features.Upload; import ch.cyberduck.core.features.Write; import ch.cyberduck.core.io.BandwidthThrottle; import ch.cyberduck.core.io.Checksum; -import ch.cyberduck.core.io.ChecksumComputeFactory; import ch.cyberduck.core.io.StreamListener; -import ch.cyberduck.core.preferences.HostPreferencesFactory; -import ch.cyberduck.core.preferences.PreferencesFactory; import ch.cyberduck.core.transfer.TransferStatus; -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.irods.jargon.core.checksum.ChecksumValue; -import org.irods.jargon.core.exception.JargonException; -import org.irods.jargon.core.packinstr.TransferOptions; -import org.irods.jargon.core.pub.DataObjectChecksumUtilitiesAO; -import org.irods.jargon.core.pub.DataTransferOperations; -import org.irods.jargon.core.pub.IRODSFileSystemAO; -import org.irods.jargon.core.pub.io.IRODSFile; -import org.irods.jargon.core.transfer.DefaultTransferControlBlock; -import org.irods.jargon.core.transfer.TransferControlBlock; - -import java.io.File; -import java.text.MessageFormat; - -public class IRODSUploadFeature implements Upload { +public class IRODSUploadFeature implements Upload { private static final Logger log = LogManager.getLogger(IRODSUploadFeature.class); private final IRODSSession session; + private boolean truncate = true; + private boolean append = false; + private int numThread=3; + private static final int BUFFER_SIZE = 4 * 1024 * 1024; public IRODSUploadFeature(final IRODSSession session) { this.session = session; } @Override - public Void upload(final Write write, final Path file, final Local local, final BandwidthThrottle throttle, - final ProgressListener progress, final StreamListener streamListener, final TransferStatus status, - final ConnectionCallback callback) throws BackgroundException { - try { - final IRODSFileSystemAO fs = session.getClient(); - final IRODSFile f = fs.getIRODSFileFactory().instanceIRODSFile(file.getAbsolute()); - final TransferControlBlock block = DefaultTransferControlBlock.instance(StringUtils.EMPTY, - HostPreferencesFactory.get(session.getHost()).getInteger("connection.retry")); - final TransferOptions options = new DefaultTransferOptionsConfigurer().configure(new TransferOptions()); - if(Host.TransferType.unknown.equals(session.getHost().getTransferType())) { - options.setUseParallelTransfer(Host.TransferType.valueOf(PreferencesFactory.get().getProperty("queue.transfer.type")).equals(Host.TransferType.concurrent)); - } - else { - options.setUseParallelTransfer(session.getHost().getTransferType().equals(Host.TransferType.concurrent)); - } - block.setTransferOptions(options); - final DataTransferOperations transfer = fs.getIRODSAccessObjectFactory().getDataTransferOperations(fs.getIRODSAccount()); - transfer.putOperation(new File(local.getAbsolute()), f, new DefaultTransferStatusCallbackListener( - status, streamListener, block - ), block); - if(status.isComplete()) { - final DataObjectChecksumUtilitiesAO checksum = fs - .getIRODSAccessObjectFactory() - .getDataObjectChecksumUtilitiesAO(fs.getIRODSAccount()); - final ChecksumValue value = checksum.computeChecksumOnDataObject(f); - final Checksum fingerprint = Checksum.parse(value.getChecksumStringValue()); - if(null == fingerprint) { - log.warn("Unsupported checksum algorithm {}", value.getChecksumEncoding()); - } - else { - if(file.getType().contains(Path.Type.encrypted)) { - log.warn("Skip checksum verification for {} with client side encryption enabled", file); - } - else { - final Checksum expected = ChecksumComputeFactory.get(fingerprint.algorithm).compute(local.getInputStream(), new TransferStatus(status)); - if(!expected.equals(fingerprint)) { - throw new ChecksumException(MessageFormat.format(LocaleFactory.localizedString("Upload {0} failed", "Error"), file.getName()), - MessageFormat.format("Mismatch between {0} hash {1} of uploaded data and ETag {2} returned by the server", - fingerprint.algorithm.toString(), expected, fingerprint.hash)); - } + public Checksum upload(final Path file, final Local local, final BandwidthThrottle throttle, + final ProgressListener progress, final StreamListener streamListener, final TransferStatus status, + final ConnectionCallback callback) throws BackgroundException { + try { + final RcComm primaryConn = session.getClient().getRcComm(); + final long fileSize = local.attributes().getSize(); + + // Step 1: Open one primary output stream to get token and replica info + IRODSDataObjectOutputStream primaryOut = new IRODSDataObjectOutputStream(); + primaryOut.open(primaryConn, file.getAbsolute(), truncate, append); + final String replicaToken = primaryOut.getReplicaToken(); + final long replicaNumber = primaryOut.getReplicaNumber(); + primaryOut.close(); + + // Step 2: Connection pool + final IRODSConnectionPool pool = new IRODSConnectionPool(numThread); + pool.start( + session.getHost().getHostname(), + session.getHost().getPort(), + new QualifiedUsername(session.getHost().getCredentials().getUsername(), session.getRegion()), + conn -> { + try { + IRODSApi.rcAuthenticateClient(conn, "native", session.getHost().getCredentials().getPassword()); + return true; + } catch (Exception e) { + return false; } - } + }); + + // Step 3: Thread pool & chunking + final ExecutorService executor = Executors.newFixedThreadPool(numThread); + final long chunkSize = fileSize / numThread; + final long remainChunkSize = fileSize % numThread; + + List> tasks = new ArrayList<>(); + for (int i = 0; i < numThread; i++) { + final long start = i * chunkSize; + final PoolConnection conn = pool.getConnection(); + IRODSDataObjectOutputStream stream = new IRODSDataObjectOutputStream(); + stream.open(conn.getRcComm(), file.getAbsolute(),replicaToken,replicaNumber, truncate, append); + ChunkWorker worker = new ChunkWorker( + stream, + local.getAbsolute(), + start, + (i == numThread - 1) ? chunkSize + remainChunkSize : chunkSize, + BUFFER_SIZE + ); + tasks.add(executor.submit(worker)); } - return null; + + for (Future task : tasks) { + task.get(); + } + + executor.shutdown(); + pool.close(); + + final String fingerprintValue = IRODSFilesystem.dataObjectChecksum(primaryConn, file.getAbsolute()); + return Checksum.parse(fingerprintValue); } - catch(JargonException e) { + catch(Exception e) { throw new IRODSExceptionMappingService().map(e); } } diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSWriteFeature.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSWriteFeature.java index e567a368af5..32ad40edbdd 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSWriteFeature.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSWriteFeature.java @@ -1,5 +1,19 @@ package ch.cyberduck.core.irods; +import java.io.IOException; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.io.FilenameUtils; +import org.irods.irods4j.common.Versioning; +import org.irods.irods4j.high_level.catalog.IRODSQuery; +import org.irods.irods4j.high_level.catalog.IRODSQuery.GenQuery1QueryArgs; +import org.irods.irods4j.high_level.connection.IRODSConnection; +import org.irods.irods4j.high_level.io.IRODSDataObjectOutputStream; +import org.irods.irods4j.low_level.api.GenQuery1Columns; +import org.irods.irods4j.low_level.api.IRODSException; + /* * Copyright (c) 2002-2015 David Kocher. All rights reserved. * http://cyberduck.ch/ @@ -24,16 +38,8 @@ import ch.cyberduck.core.io.StatusOutputStream; import ch.cyberduck.core.io.VoidStatusOutputStream; import ch.cyberduck.core.transfer.TransferStatus; -import ch.cyberduck.core.worker.DefaultExceptionMappingService; - -import org.irods.jargon.core.exception.JargonException; -import org.irods.jargon.core.exception.JargonRuntimeException; -import org.irods.jargon.core.packinstr.DataObjInp; -import org.irods.jargon.core.pub.IRODSFileSystemAO; -import org.irods.jargon.core.pub.io.IRODSFileOutputStream; -import org.irods.jargon.core.pub.io.PackingIrodsOutputStream; -public class IRODSWriteFeature implements Write { +public class IRODSWriteFeature implements Write> { private final IRODSSession session; @@ -42,22 +48,70 @@ public IRODSWriteFeature(IRODSSession session) { } @Override - public StatusOutputStream write(final Path file, final TransferStatus status, final ConnectionCallback callback) throws BackgroundException { - try { + public StatusOutputStream> write(final Path file, final TransferStatus status, final ConnectionCallback callback) throws BackgroundException { try { - final IRODSFileSystemAO fs = session.getClient(); - final IRODSFileOutputStream out = fs.getIRODSFileFactory().instanceIRODSFileOutputStream( - file.getAbsolute(), status.isAppend() ? DataObjInp.OpenFlags.READ_WRITE : DataObjInp.OpenFlags.WRITE_TRUNCATE); - return new VoidStatusOutputStream(new PackingIrodsOutputStream(out)); - } - catch(JargonRuntimeException e) { - if(e.getCause() instanceof JargonException) { - throw (JargonException) e.getCause(); - } - throw new DefaultExceptionMappingService().map(e); - } + // Step 1: Get the active iRODS client connection and parameters + final IRODSConnection conn = session.getClient(); + boolean append = status.isAppend(); + boolean truncate = !append; + final OutputStream out = new IRODSDataObjectOutputStream(conn.getRcComm(), file.getAbsolute(), truncate, append); + // Step 2: Return a wrapped StatusOutputStream that provides file metadata on completion + return new StatusOutputStream>(out) { + @Override + public List getStatus() throws BackgroundException { + // Step 3: Extract parent directory and filename from the logical path + String logicalPath = file.getAbsolute(); + String parentPath = FilenameUtils.getFullPathNoEndSeparator(logicalPath); + String fileName = FilenameUtils.getName(logicalPath); + final List status = new ArrayList<>();; + // Step 4: Check iRODS version to decide which query mechanism to use + if(Versioning.compareVersions(conn.getRcComm().relVersion.substring(4), "4.3.4") > 0) { + String query = String.format("select DATA_MODIFY_TIME, DATA_CREATE_TIME, DATA_SIZE, DATA_CHECKSUM, DATA_OWNER_NAME, DATA_OWNER_ZONE where COLL_NAME = '%s' and DATA_NAME = '%s'", parentPath, fileName); + List> rows; + try { + // Step 5: Execute the query and add the first result row to the status list + rows = IRODSQuery.executeGenQuery2(conn.getRcComm(), query); + status.addAll(rows.get(0)); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch (IRODSException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + }else { + //if older version, use Query1 + GenQuery1QueryArgs input = new GenQuery1QueryArgs(); + + // select COLL_NAME, DATA_NAME, DATA_ACCESS_TIME + input.addColumnToSelectClause(GenQuery1Columns.COL_D_MODIFY_TIME); + input.addColumnToSelectClause(GenQuery1Columns.COL_D_CREATE_TIME); + input.addColumnToSelectClause(GenQuery1Columns.COL_DATA_SIZE); + input.addColumnToSelectClause(GenQuery1Columns.COL_D_DATA_CHECKSUM); + input.addColumnToSelectClause(GenQuery1Columns.COL_D_OWNER_NAME); + input.addColumnToSelectClause(GenQuery1Columns.COL_D_OWNER_ZONE); + + + // where COLL_NAME like '/tempZone/home/rods and DATA_NAME = 'atime.txt' + String collNameCondStr = String.format("= '%s'", parentPath); + String dataNameCondStr = String.format("= '%s'", fileName); + input.addConditionToWhereClause(GenQuery1Columns.COL_COLL_NAME, collNameCondStr); + input.addConditionToWhereClause(GenQuery1Columns.COL_DATA_NAME, dataNameCondStr); + + try { + IRODSQuery.executeGenQuery1(conn.getRcComm(), input, row -> { + status.addAll(row); + return false; + }); + } catch (IOException | IRODSException e) { + e.printStackTrace(); + } + } + return status; + } + }; } - catch(JargonException e) { + catch(IRODSException | IOException e) { throw new IRODSExceptionMappingService().map("Uploading {0} failed", e, file); } } diff --git a/irods/src/test/java/ch/cyberduck/core/irods/IRODSExceptionMappingServiceTest.java b/irods/src/test/java/ch/cyberduck/core/irods/IRODSExceptionMappingServiceTest.java index c7b0664aa76..03e8654f70d 100644 --- a/irods/src/test/java/ch/cyberduck/core/irods/IRODSExceptionMappingServiceTest.java +++ b/irods/src/test/java/ch/cyberduck/core/irods/IRODSExceptionMappingServiceTest.java @@ -21,9 +21,9 @@ import ch.cyberduck.core.exception.LoginFailureException; import ch.cyberduck.core.exception.NotfoundException; -import org.irods.jargon.core.exception.AuthenticationException; -import org.irods.jargon.core.exception.CatNoAccessException; -import org.irods.jargon.core.exception.FileNotFoundException; +//import org.irods.jargon.core.exception.AuthenticationException; +//import org.irods.jargon.core.exception.CatNoAccessException; +//import org.irods.jargon.core.exception.FileNotFoundException; import org.junit.Test; import static org.junit.Assert.assertTrue; @@ -32,8 +32,8 @@ public class IRODSExceptionMappingServiceTest { @Test public void testMap() { - assertTrue(new IRODSExceptionMappingService().map(new CatNoAccessException("no access")) instanceof AccessDeniedException); - assertTrue(new IRODSExceptionMappingService().map(new FileNotFoundException("no file")) instanceof NotfoundException); - assertTrue(new IRODSExceptionMappingService().map(new AuthenticationException("no user")) instanceof LoginFailureException); +// assertTrue(new IRODSExceptionMappingService().map(new CatNoAccessException("no access")) instanceof AccessDeniedException); +// assertTrue(new IRODSExceptionMappingService().map(new FileNotFoundException("no file")) instanceof NotfoundException); +// assertTrue(new IRODSExceptionMappingService().map(new AuthenticationException("no user")) instanceof LoginFailureException); } } diff --git a/irods/src/test/java/ch/cyberduck/core/irods/IRODSReadFeatureTest.java b/irods/src/test/java/ch/cyberduck/core/irods/IRODSReadFeatureTest.java index e77c9083589..92c0d7b63e6 100644 --- a/irods/src/test/java/ch/cyberduck/core/irods/IRODSReadFeatureTest.java +++ b/irods/src/test/java/ch/cyberduck/core/irods/IRODSReadFeatureTest.java @@ -43,6 +43,7 @@ import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.RandomUtils; +//import org.irods.jargon.core.pub.domain.ObjStat; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -53,6 +54,7 @@ import java.util.Collections; import java.util.EnumSet; import java.util.HashSet; +import java.util.List; import java.util.UUID; import static org.junit.Assert.*; @@ -136,8 +138,8 @@ public void testReadRange() throws Exception { assertNotNull(out); IOUtils.write(content, out); out.close(); - new DefaultUploadFeature(session).upload( - new IRODSWriteFeature(session), test, local, new BandwidthThrottle(BandwidthThrottle.UNLIMITED), new DisabledProgressListener(), new DisabledStreamListener(), + new DefaultUploadFeature>(new IRODSWriteFeature(session)).upload( + test, local, new BandwidthThrottle(BandwidthThrottle.UNLIMITED), new DisabledProgressListener(), new DisabledStreamListener(), new TransferStatus().setLength(content.length), new DisabledConnectionCallback()); final TransferStatus status = new TransferStatus(); diff --git a/irods/src/test/java/ch/cyberduck/core/irods/IRODSWriteFeatureTest.java b/irods/src/test/java/ch/cyberduck/core/irods/IRODSWriteFeatureTest.java index 4212f1f739d..6dee5762300 100644 --- a/irods/src/test/java/ch/cyberduck/core/irods/IRODSWriteFeatureTest.java +++ b/irods/src/test/java/ch/cyberduck/core/irods/IRODSWriteFeatureTest.java @@ -41,6 +41,7 @@ import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.RandomUtils; +//import org.irods.jargon.core.pub.domain.ObjStat; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -50,6 +51,7 @@ import java.util.Collections; import java.util.EnumSet; import java.util.HashSet; +import java.util.List; import java.util.UUID; import java.util.concurrent.CountDownLatch; @@ -232,7 +234,7 @@ public void testWrite() throws Exception { assertEquals(0L, new IRODSUploadFeature(session).append(test, status).offset, 0L); - final StatusOutputStream out = feature.write(test, status, new DisabledConnectionCallback()); + final StatusOutputStream> out = feature.write(test, status, new DisabledConnectionCallback()); assertNotNull(out); new StreamCopier(new TransferStatus(), new TransferStatus()).transfer(new ByteArrayInputStream(content), out); @@ -259,8 +261,8 @@ public void testWrite() throws Exception { assertTrue(new IRODSUploadFeature(session).append(test, status).append); assertEquals(content.length, new IRODSUploadFeature(session).append(test, status).offset, 0L); - final StatusOutputStream out = feature.write(test, status, new DisabledConnectionCallback()); - assertNull(out); + final StatusOutputStream> out = feature.write(test, status, new DisabledConnectionCallback()); + assertNotNull(out); new StreamCopier(new TransferStatus(), new TransferStatus()).transfer(new ByteArrayInputStream(newcontent), out); assertTrue(session.getFeature(Find.class).find(test)); diff --git a/irods/src/test/java/ch/cyberduck/core/irods/input.txt b/irods/src/test/java/ch/cyberduck/core/irods/input.txt new file mode 100644 index 00000000000..5adda2b1cb5 --- /dev/null +++ b/irods/src/test/java/ch/cyberduck/core/irods/input.txt @@ -0,0 +1,151 @@ +irods.host=172.20.0.2 +irods.port=1247 +irods.zoneName=tempZone + +# STANDARD | PAM AUTH +irods.auth.scheme=STANDARD +default.storage.resource= + +# sets jargon ssl negotiation policy for the client. Leaving to DONT_CARE defers to the server, and is recommended +# NO_NEGOTIATION, CS_NEG_REFUSE, CS_NEG_REQUIRE, CS_NEG_DONT_CARE +ssl.negotiation.policy=CS_NEG_REFUSE + +########################################################## +# jargon properties settings +utilize.packing.streams=true + +# jargon now supports checksum calculation for streaming uploads. This does not currently verify, but does store if set to true +compute.checksum=true + +###################################### +# other irods environment + +# HTTP connection for RMD +rmd.connection.timeout=500 +rmd.connection.port=8000 + +# Reverse DNS lookup on dashboard +reverse.dns.lookup=false + +###################################### +# msi props +populate.msi.enabled=false +illumina.msi.enabled=true + +# MSI API version supported by this application +msi.api.version=1.X.X + +msi.metalnx.list=libmsiget_illumina_meta.so,libmsiobjget_microservices.so,libmsiobjget_version.so,libmsiobjjpeg_extract.so,libmsiobjput_mdbam.so,libmsiobjput_mdbam.so,libmsiobjput_mdmanifest.so,libmsiobjput_mdvcf.so,libmsiobjput_populate.so + +msi.irods.list=libmsisync_to_archive.so,libmsi_update_unixfilesystem_resource_free_space.so,libmsiobjput_http.so,libmsiobjput_irods.so,libmsiobjget_irods.so,libmsiobjget_http.so,libmsiobjput_slink.so,libmsiobjget_slink.so + +msi.irods.42.list=libmsisync_to_archive.so,libmsi_update_unixfilesystem_resource_free_space.so + +msi.other.list= +###################################### +# global feature flags that serve as defaults. Note that the info handling will manipulate aspects of the data profiler, +# so by default some things are set to null to be turned on by the service depending on the view requested (e.g. acl, metadata, replicas) and should be left 'false' as a default, +# but other aspects, such as metadata templating and mime type detection, can be globally turned on or off depending on the implmenetation. +# controls access to features globally +metalnx.enable.tickets=false +# disable automatic detection and running of rules on upload +metalnx.enable.upload.rules=false +# download size limit in megabytes (6000=6GB) +metalnx.download.limit=6000 +# show dashboard (off by default due to performance issues) +metalnx.enable.dashboard=false +###################################### +# info home page feature flags +# this controls the behavior of the data profiler and what information it will gather +irodsext.dataprofiler.retrieve.tickets=false +# process starred or favorites +irodsext.dataprofiler.retrieve.starred=true +# process shared +irodsext.dataprofiler.retrieve.shared=false +# tags and comments +irodsext.dataprofiler.retrieve.tags.and.comments=false +# metadata templates (currently not implemented) +irodsext.dataprofiler.retrieve.metadata.templates=false +# save data type information for later use +irodsext.datatyper.persist.data.types=false +# perform a detailed versus a lightweight data typing, which may involve processing the file contents +irodsext.datatyper.detailed.determination=false + +############################# +# misc ui configuration niceties +############################# +# allow translation of iRODS auth types to user friendly names in login +# in the form irodstype:displaytype| +metalnx.authtype.mappings=PAM:PAM|STANDARD:Standard + +############################# +# JWT configuration (necessary when using search and notification services). Otherwise can be left as-is and ignored +############################# +jwt.issuer=metalnx +jwt.secret=thisisasecretthatisverysecretyouwillneverguessthiskey +jwt.algo=HS384 + +############################# +# Pluggable search configuration. Turn on and off pluggable search globally, and configure search endpoints. +# N.B. pluggable search also requires provisioning of the jwt.* information above +############################# +# configured endpoints, comma delimited in form https://host.com/v1 +# Note the commented out URL which matches up to the irods-contrib/file-and-metadata-indexer docker compose assembly. In order to +# utilize this assembly you need to uncomment the URL and set pluggablesearch.enabled to true +pluggablesearch.endpointRegistryList= +# enable pluggable search globally and show the search GUI components +pluggablesearch.enabled=false +# display the older file and properties search in the menu, if you are running the elasticsearch standard plugin this is probably +# a menu item to turn off +classicsearch.enabled=true +# JWT subject claim used to access search endpoint for data gathering. User searches will utilize the name of the individual +pluggablesearch.endpointAccessSubject=metalnx +# timeout for info/attribute gathering, set to 0 for no timeout +pluggablesearch.info.timeout=0 +# timeout for actual search, set to 0 for no timeout +pluggablesearch.search.timeout=0 + +############################# +# Pluggable shopping cart and export plugin configuration. +# Turn on and off pluggable shopping cart globally, and configure export endpoints. +# N.B. plugins also requires provisioning of the jwt.* information above +############################# + +# enable pluggable export globally and show the export GUI components +pluggableshoppingcart.enabled=false + +# configured endpoints, comma delimited in form https://host.com/v1 +pluggablepublishing.endpointRegistryList= +# timeout for info/attribute gathering, set to 0 for no timeout +pluggablepublishing.info.timeout=0 + +# timeout for actual publishing, set to 0 for no timeout +pluggablepublishing.publishing.timeout=0 + +# server rule engine instance that will provide the galleryview listing +gallery_view.rule_engine_plugin.instance_name=irods_rule_engine_plugin-irods_rule_language-instance + +########################################################## +# Metadata Masking Properties +# +# Excludes metadata when the attribute name starts with at least one prefix. +# Multiple prefixes can be defined by separating them with the character sequence defined +# by metalnx.metadata.mask.delimiter. +# +# For example, the configuration below will hide any metadata that contains an attribute +# starting with "irods::", "metalnx-", or "_system_". +# +# metalnx.metadata.mask.prefixes=irods::;metalnx-;_system_ +# metalnx.metadata.mask.delimiter=; +# +# Use the iRODS Metadata Guard rule engine plugin to protect your metadata namespaces from +# being modified. +metalnx.metadata.mask.prefixes= +metalnx.metadata.mask.delimiter=, + + +########################################################## +# Setting to enable/disable the "Public" sidebar link. +# The default is "false" (hidden) +########################################################## +sidebar.show.public=false diff --git a/irods/src/test/java/ch/cyberduck/core/irods/output.txt b/irods/src/test/java/ch/cyberduck/core/irods/output.txt new file mode 100644 index 00000000000..adcc2d8497b --- /dev/null +++ b/irods/src/test/java/ch/cyberduck/core/irods/output.txt @@ -0,0 +1,151 @@ +te that the info handling will manipulate aspects of the data profiler, +# so by default some things are set to null to be turned on by the service depending on the view requested (e.g. acl, metadata, replicas) and should be left 'false' as a default, +# but other aspects, such as metadata templating and mime type detection, can be globally turned on or off depending on the implmenetation. +# controls access to features globally +metalnx.enable.tickets=false +# disable automatic detection and running of rules on upload +metalnx.enable.upload.rules=false +# download size limit in megabytes (6000=6GB) +metalnx.download.limit=6000 +# show dashboard (off by default due to performance issues) +metalnx.enable.dashboard=false +###################################### +# info home page feature flags +# this controls the behavior of the data profiler and what information it will gather +irodsext.dataprofiler.retrieve.tickets=false +# process starred or favorites +irodsext.dataprofiler.retrieve.starred=true +# process shared +irodsext.dataprofiler.retrieve.shared=false +# tags and comments +irodsext.dataprofiler.retrieve.tags.and.comments=false +# metadata templates (currently not implemented) +irodsext.dataprofiler.retrieve.metadata.templates=false +# save data type information for later use +irodsext.datatyper.persist.data.types=false +# perform a detailed versus a lightweight data typing, which may involve processing the file contents +irodsext.datatyper.detailed.determination=false + +############################# +# misc ui configuration niceties +############################# +# allow translation of iRODS auth types to user friendly names in login +# in the form irodstype:displaytype| +metalnx.auirods.host=172.20.0.2 +irods.port=1247 +irods.zoneName=tempZone + +# STANDARD | PAM AUTH +irods.auth.scheme=STANDARD +default.storage.resource= + +# sets jargon ssl negotiation policy for the client. Leaving to DONT_CARE defers to the server, and is recommended +# NO_NEGOTIATION, CS_NEG_REFUSE, CS_NEG_REQUIRE, CS_NEG_DONT_CARE +ssl.negotiation.policy=CS_NEG_REFUSE + +########################################################## +# jargon properties settings +utilize.packing.streams=true + +# jargon now supports checksum calculation for streaming uploads. This does not currently verify, but does store if set to true +compute.checksum=true + +###################################### +# other irods environment + +# HTTP connection for RMD +rmd.connection.timeout=500 +rmd.connection.port=8000 + +# Reverse DNS lookup on dashboard +reverse.dns.lookup=false + +###################################### +# msi props +populate.msi.enabled=false +illumina.msi.enabled=true + +# MSI API version supported by this application +msi.api.version=1.X.X + +msi.metalnx.list=libmsiget_illumina_meta.so,libmsiobjget_microservices.so,libmsiobjget_version.so,libmsiobjjpeg_extract.so,libmsiobjput_mdbam.so,libmsiobjput_mdbam.so,libmsiobjput_mdmanifest.so,libmsiobjput_mdvcf.so,libmsiobjput_populate.so + +msi.irods.list=libmsisync_to_archive.so,libmsi_update_unixfilesystem_resource_free_space.so,libmsiobjput_http.so,libmsiobjput_irods.so,libmsiobjget_irods.so,libmsiobjget_http.so,libmsiobjput_slink.so,libmsiobjget_slink.so + +msi.irods.42.list=libmsisync_to_archive.so,libmsi_update_unixfilesystem_resource_free_space.so + +msi.other.list= +###################################### +# global feature flags that serve as defaults. Nothtype.mappings=PAM:PAM|STANDARD:Standard + +############################# +# JWT configuration (necessary when using search and notification services). Otherwise can be left as-is and ignored +############################# +jwt.issuer=metalnx +jwt.secret=thisisasecretthatisverysecretyouwillneverguessthiskey +jwt.algo=HS384 + +############################# +# Pluggable search configuration. Turn on and off pluggable search globally, and configure search endpoints. +# N.B. pluggable search also requires provisioning of the jwt.* information above +############################# +# configured endpoints, comma delimited in form https://host.com/v1 +# Note the commented out URL which matches up to the irods-contrib/file-and-metadata-indexer docker compose assembly. In order to +# utilize this assembly you need to uncomment the URL and set pluggablesearch.enabled to true +pluggablesearch.endpointRegistryList= +# enable pluggable search globally and show the search GUI components +pluggablesearch.enabled=false +# display the older file and properties search in the menu, if you are running the elasticsearch standard plugin this is probably +# a menu item to turn off +classicsearch.enabled=true +# JWT subject claim used to access search endpoint for data gathering. User searches will utilize the name of the individual +pluggablesearch.endpointAccessSubject=metalnx +# timeout for info/attribute gathering, set to 0 for no timeout +pluggablesearch.info.timeout=0 +# timeout for actual search, set to 0 for no timeout +pluggablesearch.search.timeout=0 + +############################# +# Pluggable shopping cart and export plugin configuration. +# Turn on and off pluggable shopping cart globally, and configure export endpoints. +# N.B. plugins also requires provisioning of the jwt.* information above +############################# + +# enable pluggable export globally and show the export GUI components +pluggableshoppingcart.enabled=false + +# configured endpoints, comma delimited in form https://host.com/v1 +pluggablepublishing.endpointRegistryList= +# timeout for info/attribute gathering, set to 0 for no timeout +pluggablepublishing.info.timeout=0 + +# timeout for actual publishing, set to 0 for no timeout +pluggablepublishing.publishing.timeout=0 + +# server rule engine instance that will provide the galleryview listing +gallery_view.rule_engine_plugin.instance_name=irods_rule_engine_plugin-irods_rule_language-instance + +########################################################## +# Metadata Masking Properties +# +# Excludes metadata when the attribute name starts with at least one prefix. +# Multiple prefixes can be defined by separating them with the character sequence defined +# by metalnx.metadata.mask.delimiter. +# +# For example, the configuration below will hide any metadata that contains an attribute +# starting with "irods::", "metalnx-", or "_system_". +# +# metalnx.metadata.mask.prefixes=irods::;metalnx-;_system_ +# metalnx.metadata.mask.delimiter=; +# +# Use the iRODS Metadata Guard rule engine plugin to protect your metadata namespaces from +# being modified. +metalnx.metadata.mask.prefixes= +metalnx.metadata.mask.delimiter=, + + +########################################################## +# Setting to enable/disable the "Public" sidebar link. +# The default is "false" (hidden) +########################################################## +sidebar.show.public=false From 2870ae7a622d03dad38740eafaac08b1efc67dad Mon Sep 17 00:00:00 2001 From: Kory Draughn Date: Tue, 5 Aug 2025 12:29:17 -0400 Subject: [PATCH 02/20] iRODS: Fix bugs, expand options, and clean up implementation --- irods/bin/pom.xml | 78 ---- ...DS (iPlant Collaborative).cyberduckprofile | 28 -- irods/output.txt | 151 -------- irods/pom.xml | 10 +- .../ch/cyberduck/core/irods/ChunkWorker.java | 75 ---- .../irods/IRODSAttributesFinderFeature.java | 140 +++---- .../core/irods/IRODSChecksumUtils.java | 52 +++ .../core/irods/IRODSChunkWorker.java | 101 +++++ .../core/irods/IRODSConnectionUtils.java | 112 ++++++ .../core/irods/IRODSCopyFeature.java | 65 +--- .../core/irods/IRODSDeleteFeature.java | 67 ++-- .../core/irods/IRODSDirectoryFeature.java | 24 +- .../core/irods/IRODSDownloadFeature.java | 141 ------- .../irods/IRODSExceptionMappingService.java | 64 ++- .../core/irods/IRODSFindFeature.java | 22 +- .../core/irods/IRODSHomeFinderService.java | 20 +- .../core/irods/IRODSIntegerUtils.java | 32 ++ .../core/irods/IRODSListService.java | 137 +++---- .../core/irods/IRODSMoveFeature.java | 29 +- .../cyberduck/core/irods/IRODSProtocol.java | 49 ++- .../core/irods/IRODSReadFeature.java | 63 ++- .../ch/cyberduck/core/irods/IRODSSession.java | 133 ++++--- .../core/irods/IRODSStreamUtils.java | 74 ++++ .../cyberduck/core/irods/IRODSTimestamp.java | 96 +++++ .../core/irods/IRODSTouchFeature.java | 44 ++- .../core/irods/IRODSUploadFeature.java | 363 ++++++++++++++---- .../core/irods/IRODSWriteFeature.java | 102 +---- .../IRODSExceptionMappingServiceTest.java | 13 +- .../core/irods/IRODSListServiceTest.java | 1 - .../core/irods/IRODSReadFeatureTest.java | 5 +- .../core/irods/IRODSSessionTest.java | 1 - .../core/irods/IRODSWriteFeatureTest.java | 5 +- .../java/ch/cyberduck/core/irods/input.txt | 151 -------- .../java/ch/cyberduck/core/irods/output.txt | 151 -------- ...DS (iPlant Collaborative).cyberduckprofile | 11 +- 35 files changed, 1251 insertions(+), 1359 deletions(-) delete mode 100644 irods/bin/pom.xml delete mode 100644 irods/bin/src/test/resources/iRODS (iPlant Collaborative).cyberduckprofile delete mode 100644 irods/output.txt delete mode 100644 irods/src/main/java/ch/cyberduck/core/irods/ChunkWorker.java create mode 100644 irods/src/main/java/ch/cyberduck/core/irods/IRODSChecksumUtils.java create mode 100644 irods/src/main/java/ch/cyberduck/core/irods/IRODSChunkWorker.java create mode 100644 irods/src/main/java/ch/cyberduck/core/irods/IRODSConnectionUtils.java delete mode 100644 irods/src/main/java/ch/cyberduck/core/irods/IRODSDownloadFeature.java create mode 100644 irods/src/main/java/ch/cyberduck/core/irods/IRODSIntegerUtils.java create mode 100644 irods/src/main/java/ch/cyberduck/core/irods/IRODSStreamUtils.java create mode 100644 irods/src/main/java/ch/cyberduck/core/irods/IRODSTimestamp.java delete mode 100644 irods/src/test/java/ch/cyberduck/core/irods/input.txt delete mode 100644 irods/src/test/java/ch/cyberduck/core/irods/output.txt diff --git a/irods/bin/pom.xml b/irods/bin/pom.xml deleted file mode 100644 index ac92788e04c..00000000000 --- a/irods/bin/pom.xml +++ /dev/null @@ -1,78 +0,0 @@ - - - - 4.0.0 - - ch.cyberduck - parent - 9.2.0-SNAPSHOT - - irods - jar - - - - ch.cyberduck - core - ${project.version} - - - ch.cyberduck - test - pom - test - ${project.version} - - - - - - - - org.apache.maven.plugins - maven-failsafe-plugin - - true - - - - - - diff --git a/irods/bin/src/test/resources/iRODS (iPlant Collaborative).cyberduckprofile b/irods/bin/src/test/resources/iRODS (iPlant Collaborative).cyberduckprofile deleted file mode 100644 index 7d5484f947f..00000000000 --- a/irods/bin/src/test/resources/iRODS (iPlant Collaborative).cyberduckprofile +++ /dev/null @@ -1,28 +0,0 @@ - - - - - Protocol - irods - Vendor - iplant - Description - iPlant Data Store - Hostname Configurable - - Port Configurable - - Default Hostname - data.iplantcollaborative.org - Region - iplant - Default Port - 1247 - Username Placeholder - iPlant username - Password Placeholder - iPlant password - Authorization - STANDARD - - diff --git a/irods/output.txt b/irods/output.txt deleted file mode 100644 index 5adda2b1cb5..00000000000 --- a/irods/output.txt +++ /dev/null @@ -1,151 +0,0 @@ -irods.host=172.20.0.2 -irods.port=1247 -irods.zoneName=tempZone - -# STANDARD | PAM AUTH -irods.auth.scheme=STANDARD -default.storage.resource= - -# sets jargon ssl negotiation policy for the client. Leaving to DONT_CARE defers to the server, and is recommended -# NO_NEGOTIATION, CS_NEG_REFUSE, CS_NEG_REQUIRE, CS_NEG_DONT_CARE -ssl.negotiation.policy=CS_NEG_REFUSE - -########################################################## -# jargon properties settings -utilize.packing.streams=true - -# jargon now supports checksum calculation for streaming uploads. This does not currently verify, but does store if set to true -compute.checksum=true - -###################################### -# other irods environment - -# HTTP connection for RMD -rmd.connection.timeout=500 -rmd.connection.port=8000 - -# Reverse DNS lookup on dashboard -reverse.dns.lookup=false - -###################################### -# msi props -populate.msi.enabled=false -illumina.msi.enabled=true - -# MSI API version supported by this application -msi.api.version=1.X.X - -msi.metalnx.list=libmsiget_illumina_meta.so,libmsiobjget_microservices.so,libmsiobjget_version.so,libmsiobjjpeg_extract.so,libmsiobjput_mdbam.so,libmsiobjput_mdbam.so,libmsiobjput_mdmanifest.so,libmsiobjput_mdvcf.so,libmsiobjput_populate.so - -msi.irods.list=libmsisync_to_archive.so,libmsi_update_unixfilesystem_resource_free_space.so,libmsiobjput_http.so,libmsiobjput_irods.so,libmsiobjget_irods.so,libmsiobjget_http.so,libmsiobjput_slink.so,libmsiobjget_slink.so - -msi.irods.42.list=libmsisync_to_archive.so,libmsi_update_unixfilesystem_resource_free_space.so - -msi.other.list= -###################################### -# global feature flags that serve as defaults. Note that the info handling will manipulate aspects of the data profiler, -# so by default some things are set to null to be turned on by the service depending on the view requested (e.g. acl, metadata, replicas) and should be left 'false' as a default, -# but other aspects, such as metadata templating and mime type detection, can be globally turned on or off depending on the implmenetation. -# controls access to features globally -metalnx.enable.tickets=false -# disable automatic detection and running of rules on upload -metalnx.enable.upload.rules=false -# download size limit in megabytes (6000=6GB) -metalnx.download.limit=6000 -# show dashboard (off by default due to performance issues) -metalnx.enable.dashboard=false -###################################### -# info home page feature flags -# this controls the behavior of the data profiler and what information it will gather -irodsext.dataprofiler.retrieve.tickets=false -# process starred or favorites -irodsext.dataprofiler.retrieve.starred=true -# process shared -irodsext.dataprofiler.retrieve.shared=false -# tags and comments -irodsext.dataprofiler.retrieve.tags.and.comments=false -# metadata templates (currently not implemented) -irodsext.dataprofiler.retrieve.metadata.templates=false -# save data type information for later use -irodsext.datatyper.persist.data.types=false -# perform a detailed versus a lightweight data typing, which may involve processing the file contents -irodsext.datatyper.detailed.determination=false - -############################# -# misc ui configuration niceties -############################# -# allow translation of iRODS auth types to user friendly names in login -# in the form irodstype:displaytype| -metalnx.authtype.mappings=PAM:PAM|STANDARD:Standard - -############################# -# JWT configuration (necessary when using search and notification services). Otherwise can be left as-is and ignored -############################# -jwt.issuer=metalnx -jwt.secret=thisisasecretthatisverysecretyouwillneverguessthiskey -jwt.algo=HS384 - -############################# -# Pluggable search configuration. Turn on and off pluggable search globally, and configure search endpoints. -# N.B. pluggable search also requires provisioning of the jwt.* information above -############################# -# configured endpoints, comma delimited in form https://host.com/v1 -# Note the commented out URL which matches up to the irods-contrib/file-and-metadata-indexer docker compose assembly. In order to -# utilize this assembly you need to uncomment the URL and set pluggablesearch.enabled to true -pluggablesearch.endpointRegistryList= -# enable pluggable search globally and show the search GUI components -pluggablesearch.enabled=false -# display the older file and properties search in the menu, if you are running the elasticsearch standard plugin this is probably -# a menu item to turn off -classicsearch.enabled=true -# JWT subject claim used to access search endpoint for data gathering. User searches will utilize the name of the individual -pluggablesearch.endpointAccessSubject=metalnx -# timeout for info/attribute gathering, set to 0 for no timeout -pluggablesearch.info.timeout=0 -# timeout for actual search, set to 0 for no timeout -pluggablesearch.search.timeout=0 - -############################# -# Pluggable shopping cart and export plugin configuration. -# Turn on and off pluggable shopping cart globally, and configure export endpoints. -# N.B. plugins also requires provisioning of the jwt.* information above -############################# - -# enable pluggable export globally and show the export GUI components -pluggableshoppingcart.enabled=false - -# configured endpoints, comma delimited in form https://host.com/v1 -pluggablepublishing.endpointRegistryList= -# timeout for info/attribute gathering, set to 0 for no timeout -pluggablepublishing.info.timeout=0 - -# timeout for actual publishing, set to 0 for no timeout -pluggablepublishing.publishing.timeout=0 - -# server rule engine instance that will provide the galleryview listing -gallery_view.rule_engine_plugin.instance_name=irods_rule_engine_plugin-irods_rule_language-instance - -########################################################## -# Metadata Masking Properties -# -# Excludes metadata when the attribute name starts with at least one prefix. -# Multiple prefixes can be defined by separating them with the character sequence defined -# by metalnx.metadata.mask.delimiter. -# -# For example, the configuration below will hide any metadata that contains an attribute -# starting with "irods::", "metalnx-", or "_system_". -# -# metalnx.metadata.mask.prefixes=irods::;metalnx-;_system_ -# metalnx.metadata.mask.delimiter=; -# -# Use the iRODS Metadata Guard rule engine plugin to protect your metadata namespaces from -# being modified. -metalnx.metadata.mask.prefixes= -metalnx.metadata.mask.delimiter=, - - -########################################################## -# Setting to enable/disable the "Public" sidebar link. -# The default is "false" (hidden) -########################################################## -sidebar.show.public=false diff --git a/irods/pom.xml b/irods/pom.xml index 2d183a05682..7d185e7071d 100644 --- a/irods/pom.xml +++ b/irods/pom.xml @@ -24,11 +24,11 @@ jar - - org.irods - irods4j - 0.2.0-java8 - + + org.irods + irods4j + 0.5.0-java8 + ch.cyberduck core diff --git a/irods/src/main/java/ch/cyberduck/core/irods/ChunkWorker.java b/irods/src/main/java/ch/cyberduck/core/irods/ChunkWorker.java deleted file mode 100644 index af1a532c949..00000000000 --- a/irods/src/main/java/ch/cyberduck/core/irods/ChunkWorker.java +++ /dev/null @@ -1,75 +0,0 @@ -package ch.cyberduck.core.irods; - -import java.io.IOException; -import java.io.RandomAccessFile; - -import org.irods.irods4j.high_level.io.IRODSDataObjectInputStream; -import org.irods.irods4j.high_level.io.IRODSDataObjectOutputStream; -import org.irods.irods4j.high_level.io.IRODSDataObjectStream.SeekDirection; -import org.irods.irods4j.low_level.api.IRODSException; - -public class ChunkWorker implements Runnable{ - private final Object stream; - private final RandomAccessFile file; - private final long offset; - private final long chunkSize; - private final byte[] buffer; - public ChunkWorker(Object stream, String localfilePath,long offset, long chunkSize, int bufferSize) throws IOException, IRODSException { - this.stream= stream; - this.file=new RandomAccessFile(localfilePath,"rw"); - this.offset=offset; - this.chunkSize=chunkSize; - this.buffer=new byte[bufferSize]; - - - file.seek(offset); - } - @Override - public void run(){ - try { - if (stream instanceof IRODSDataObjectInputStream) { - IRODSDataObjectInputStream in = (IRODSDataObjectInputStream) (stream); - in.seek((int) offset, SeekDirection.CURRENT); - long remaining = chunkSize; - while (remaining > 0) { - int readLength = (int) Math.min(buffer.length, remaining); - int read = in.read(buffer, 0, readLength); - file.write(buffer, 0, read); - remaining -= read; - } - - }else if (stream instanceof IRODSDataObjectOutputStream) { - IRODSDataObjectOutputStream out = (IRODSDataObjectOutputStream) (stream); - out.seek((int) offset, SeekDirection.CURRENT); - long remaining = chunkSize; - while (remaining > 0) { - int writeLength = (int) Math.min(buffer.length, remaining); - int read = file.read(buffer, 0, writeLength); - if (read == -1) break; - out.write(buffer, 0, read); - remaining -= read; - } - }else { - throw new IllegalArgumentException("Unsupported stream type"); - } - } catch (IOException | IRODSException e) { - e.printStackTrace(); - } - try { - close(); - } catch (IOException e) { - e.printStackTrace(); - } - } - - public void close() throws IOException { - file.close(); - if (stream instanceof IRODSDataObjectInputStream) { - IRODSDataObjectInputStream in = (IRODSDataObjectInputStream) (stream); - in.close(); - } else if (stream instanceof IRODSDataObjectOutputStream) { - IRODSDataObjectOutputStream out = (IRODSDataObjectOutputStream) (stream); - out.close(); - } - } -} diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSAttributesFinderFeature.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSAttributesFinderFeature.java index 7b4ce7b4f8c..d4aeb6fe18a 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSAttributesFinderFeature.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSAttributesFinderFeature.java @@ -1,20 +1,7 @@ package ch.cyberduck.core.irods; -import java.io.IOException; -import java.util.List; - -import org.apache.commons.io.FilenameUtils; -import org.apache.commons.lang3.StringUtils; -import org.irods.irods4j.common.Versioning; -import org.irods.irods4j.high_level.catalog.IRODSQuery; -import org.irods.irods4j.high_level.catalog.IRODSQuery.GenQuery1QueryArgs; -import org.irods.irods4j.high_level.connection.IRODSConnection; -import org.irods.irods4j.high_level.vfs.IRODSFilesystem; -import org.irods.irods4j.low_level.api.GenQuery1Columns; -import org.irods.irods4j.low_level.api.IRODSException; - /* - * Copyright (c) 2002-2016 iterate GmbH. All rights reserved. + * Copyright (c) 2002-2025 iterate GmbH. All rights reserved. * https://cyberduck.io/ * * This program is free software; you can redistribute it and/or modify @@ -35,11 +22,23 @@ import ch.cyberduck.core.exception.NotfoundException; import ch.cyberduck.core.features.AttributesAdapter; import ch.cyberduck.core.features.AttributesFinder; -import ch.cyberduck.core.io.Checksum; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.irods.irods4j.high_level.catalog.IRODSQuery; +import org.irods.irods4j.high_level.connection.IRODSConnection; +import org.irods.irods4j.high_level.vfs.IRODSFilesystem; +import org.irods.irods4j.high_level.vfs.LogicalPath; +import org.irods.irods4j.high_level.vfs.ObjectStatus; +import org.irods.irods4j.low_level.api.IRODSException; + +import java.io.IOException; +import java.util.List; public class IRODSAttributesFinderFeature implements AttributesFinder, AttributesAdapter> { + private static final Logger log = LogManager.getLogger(IRODSAttributesFinderFeature.class); + private final IRODSSession session; public IRODSAttributesFinderFeature(final IRODSSession session) { @@ -49,45 +48,56 @@ public IRODSAttributesFinderFeature(final IRODSSession session) { @Override public PathAttributes find(final Path file, final ListProgressListener listener) throws BackgroundException { try { - final PathAttributes[] attributes = new PathAttributes[1]; + log.debug("looking up path attributes."); + + final String logicalPath = file.getAbsolute(); final IRODSConnection conn = session.getClient(); - if(!IRODSFilesystem.exists(this.session.getClient().getRcComm(), file.getAbsolute())) { - throw new NotfoundException(file.getAbsolute()); + + ObjectStatus status = IRODSFilesystem.status(conn.getRcComm(), logicalPath); + + if(IRODSFilesystem.isDataObject(status)) { + log.debug("data object exists in iRODS. fetching data using GenQuery2."); + String query = String.format( + "select DATA_CREATE_TIME, DATA_MODIFY_TIME, DATA_SIZE, DATA_CHECKSUM, DATA_REPL_STATUS where COLL_NAME = '%s' and DATA_NAME = '%s' order by DATA_REPL_STATUS desc, DATA_MODIFY_TIME desc", + LogicalPath.parentPath(logicalPath), + LogicalPath.objectName(logicalPath)); + log.debug("query = [{}]", query); + List> rows = IRODSQuery.executeGenQuery2(conn.getRcComm(), query); + + PathAttributes attrs = new PathAttributes(); + + if(!rows.isEmpty()) { + List row = rows.get(0); + if("0".equals(row.get(4)) || "1".equals(row.get(4))) { + setAttributes(attrs, row); + } + } + + return attrs; } - String logicalPath = file.getAbsolute(); - String parentPath = FilenameUtils.getFullPathNoEndSeparator(logicalPath); - String fileName = FilenameUtils.getName(logicalPath); - if(Versioning.compareVersions(conn.getRcComm().relVersion.substring(4), "4.3.4") > 0) { - String query = String.format("select DATA_MODIFY_TIME, DATA_CREATE_TIME, DATA_SIZE, DATA_CHECKSUM, DATA_OWNER_NAME, DATA_OWNER_ZONE where COLL_NAME = '%s' and DATA_NAME = '%s'", parentPath, fileName); - List> rows = IRODSQuery.executeGenQuery2(conn.getRcComm(), query); - List row = rows.get(0); - attributes[0]=toAttributes(row); - }else { - GenQuery1QueryArgs input = new GenQuery1QueryArgs(); - - // select COLL_NAME, DATA_NAME, DATA_ACCESS_TIME - input.addColumnToSelectClause(GenQuery1Columns.COL_D_MODIFY_TIME); - input.addColumnToSelectClause(GenQuery1Columns.COL_D_CREATE_TIME); - input.addColumnToSelectClause(GenQuery1Columns.COL_DATA_SIZE); - input.addColumnToSelectClause(GenQuery1Columns.COL_D_DATA_CHECKSUM); - input.addColumnToSelectClause(GenQuery1Columns.COL_D_OWNER_NAME); - input.addColumnToSelectClause(GenQuery1Columns.COL_D_OWNER_ZONE); - - - // where COLL_NAME like '/tempZone/home/rods and DATA_NAME = 'atime.txt' - String collNameCondStr = String.format("= '%s'", parentPath); - String dataNameCondStr = String.format("= '%s'", fileName); - input.addConditionToWhereClause(GenQuery1Columns.COL_COLL_NAME, collNameCondStr); - input.addConditionToWhereClause(GenQuery1Columns.COL_DATA_NAME, dataNameCondStr); - - StringBuilder output = new StringBuilder(); - - IRODSQuery.executeGenQuery1(conn.getRcComm(), input, row -> { - attributes[0]=toAttributes(row); - return false; - }); - } - return attributes[0]; + + if(IRODSFilesystem.isCollection(status)) { + log.debug("collection exists in iRODS. fetching data using GenQuery2."); + String query = String.format("select COLL_CREATE_TIME, COLL_MODIFY_TIME where COLL_NAME = '%s'", logicalPath); + log.debug("query = [{}]", query); + List> rows = IRODSQuery.executeGenQuery2(conn.getRcComm(), query); + + PathAttributes attrs = new PathAttributes(); + + if(!rows.isEmpty()) { + // Collections do not have the same properties as data objects + // so fill in the gaps to satisfy requirements of setAttributes. + List row = rows.get(0); + row.add("0"); // Data size + row.add(""); // Checksum + row.add(""); // Replica status + setAttributes(attrs, row); + } + + return attrs; + } + + throw new NotfoundException(logicalPath); } catch(IOException | IRODSException e) { throw new IRODSExceptionMappingService().map("Failure to read attributes of {0}", e, file); @@ -96,18 +106,18 @@ public PathAttributes find(final Path file, final ListProgressListener listener) @Override public PathAttributes toAttributes(final List row) { - final IRODSConnection conn = session.getClient(); - final PathAttributes attributes = new PathAttributes(); - attributes.setModificationDate(Long.parseLong(row.get(0)) * 1000); // seconds to ms - attributes.setCreationDate(Long.parseLong(row.get(1)) * 1000); - attributes.setSize(Long.parseLong(row.get(2))); - String checksum = row.get(3); - if (!StringUtils.isEmpty(checksum)) { - attributes.setChecksum(Checksum.parse(checksum)); - } - - attributes.setOwner(conn.getRcComm().relVersion); - attributes.setGroup(row.get(5)); - return attributes; + PathAttributes attrs = new PathAttributes(); + setAttributes(attrs, row); + return attrs; } + + private static void setAttributes(final PathAttributes attrs, final List row) { + log.debug("path attribute info: created at [{}], modified at [{}], data size = [{}], checksum = [{}]", + row.get(0), row.get(1), row.get(2), row.get(3)); + attrs.setCreationDate(Long.parseLong(row.get(0)) * 1000); // seconds to ms + attrs.setModificationDate(Long.parseLong(row.get(1)) * 1000); + attrs.setSize(Long.parseLong(row.get(2))); + attrs.setChecksum(IRODSChecksumUtils.toChecksum(row.get(3))); + } + } diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSChecksumUtils.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSChecksumUtils.java new file mode 100644 index 00000000000..ddd41ff66fd --- /dev/null +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSChecksumUtils.java @@ -0,0 +1,52 @@ +package ch.cyberduck.core.irods; + +/* + * Copyright (c) 2002-2025 iterate GmbH. All rights reserved. + * https://cyberduck.io/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +import ch.cyberduck.core.io.Checksum; + +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.codec.binary.Hex; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public final class IRODSChecksumUtils { + + private static final Logger log = LogManager.getLogger(IRODSChecksumUtils.class); + + public static Checksum toChecksum(String irodsChecksum) { + if(StringUtils.isBlank(irodsChecksum)) { + return Checksum.NONE; + } + + int colon = irodsChecksum.indexOf(':'); + if(-1 == colon) { + log.debug("no hash algorithm prefix found in iRODS checksum. ignoring checksum."); + return Checksum.NONE; + } + + if(colon + 1 >= irodsChecksum.length()) { + log.debug("iRODS checksum may be corrupted. ignoring checksum."); + return Checksum.NONE; + } + + log.debug("checksum from iRODS server is [{}].", irodsChecksum); + String checksum = irodsChecksum.substring(colon + 1); + checksum = Hex.encodeHexString(Base64.decodeBase64(checksum)); + log.debug("base64-decoded, hex-encoded checksum is [{}].", checksum); + return Checksum.parse(checksum); + } +} diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSChunkWorker.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSChunkWorker.java new file mode 100644 index 00000000000..0e33be839bf --- /dev/null +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSChunkWorker.java @@ -0,0 +1,101 @@ +package ch.cyberduck.core.irods; + +/* + * Copyright (c) 2002-2025 iterate GmbH. All rights reserved. + * https://cyberduck.io/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +import ch.cyberduck.core.exception.ConnectionCanceledException; +import ch.cyberduck.core.io.StreamListener; +import ch.cyberduck.core.transfer.TransferStatus; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.irods.irods4j.low_level.api.IRODSException; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.concurrent.Callable; + +public class IRODSChunkWorker implements Callable { + + private static final Logger log = LogManager.getLogger(IRODSChunkWorker.class); + + private final TransferStatus status; + private final StreamListener streamListener; + private final InputStream in; + private final OutputStream out; + private final long offset; + private final long chunkSize; + private final byte[] buffer; + + public IRODSChunkWorker(TransferStatus status, StreamListener streamListener, InputStream in, OutputStream out, long offset, long chunkSize, int bufferSize) { + log.info("constructing iRODS chunk worker."); + log.info("offset = [{}]", offset); + log.info("chunk size = [{}]", chunkSize); + log.info("buffer size = [{}]", bufferSize); + this.status = status; + this.streamListener = streamListener; + this.in = in; + this.out = out; + this.offset = offset; + this.chunkSize = chunkSize; + this.buffer = new byte[bufferSize]; + log.info("iRODS chunk worker constructed."); + } + + @Override + public Boolean call() { + try { + IRODSStreamUtils.seek(in, offset); + IRODSStreamUtils.seek(out, offset); + + long remaining = chunkSize; + while(remaining > 0) { + try { + status.validate(); + } + catch(ConnectionCanceledException e) { + log.info("transfer cancelled."); + return false; + } + + int count = (int) Math.min(buffer.length, remaining); + + int bytesRead = in.read(buffer, 0, count); + log.info("read [{}] of [{}] requested bytes from input stream.", bytesRead, count); + if(-1 == bytesRead) { + break; + } + + streamListener.recv(bytesRead); + out.write(buffer, 0, bytesRead); + log.info("wrote [{}] bytes to output stream.", bytesRead); + streamListener.sent(bytesRead); + remaining -= bytesRead; + } + + log.info("total bytes remaining = [{}]", remaining); + log.info("done. wrote [{}] of [{}] bytes to the replica.", chunkSize - remaining, chunkSize); + + return true; + } + catch(IOException | IRODSException e) { + log.error(e.getMessage()); + } + + return false; + } + +} diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSConnectionUtils.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSConnectionUtils.java new file mode 100644 index 00000000000..5eabc8ad09e --- /dev/null +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSConnectionUtils.java @@ -0,0 +1,112 @@ +package ch.cyberduck.core.irods; + +/* + * Copyright (c) 2002-2025 iterate GmbH. All rights reserved. + * https://cyberduck.io/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +import ch.cyberduck.core.preferences.HostPreferencesFactory; +import ch.cyberduck.core.preferences.PreferencesReader; + +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.irods.irods4j.authentication.AuthPlugin; +import org.irods.irods4j.authentication.NativeAuthPlugin; +import org.irods.irods4j.authentication.PamPasswordAuthPlugin; +import org.irods.irods4j.high_level.connection.IRODSConnection; +import org.irods.irods4j.high_level.connection.IRODSConnectionPool; +import org.irods.irods4j.high_level.connection.QualifiedUsername; +import org.irods.irods4j.low_level.api.IRODSApi; +import org.irods.irods4j.low_level.api.IRODSException; + +import java.io.IOException; + +final class IRODSConnectionUtils { + + private static final Logger log = LogManager.getLogger(IRODSConnectionUtils.class); + + public static IRODSApi.ConnectionOptions initConnectionOptions(IRODSSession session) { + log.debug("configuring iRODS connection."); + final PreferencesReader preferences = HostPreferencesFactory.get(session.getHost()); + final IRODSApi.ConnectionOptions options = new IRODSApi.ConnectionOptions(); + + options.clientServerNegotiation = preferences.getProperty(IRODSProtocol.CLIENT_SERVER_NEGOTIATION); + options.sslProtocol = preferences.getProperty(IRODSProtocol.TLS_PROTOCOL); + options.sslTruststore = preferences.getProperty(IRODSProtocol.TLS_TRUSTSTORE); + options.sslTruststorePassword = preferences.getProperty(IRODSProtocol.TLS_TRUSTSTORE_PASSWORD); + log.debug("client server negotiation = [{}], ssl protocol = [{}], ssl truststore = [{}]", + options.clientServerNegotiation, options.sslProtocol, options.sslTruststore); + + options.encryptionAlgorithm = preferences.getProperty(IRODSProtocol.ENCRYPTION_ALGORITHM); + options.encryptionKeySize = preferences.getInteger(IRODSProtocol.ENCRYPTION_KEY_SIZE); + options.encryptionSaltSize = preferences.getInteger(IRODSProtocol.ENCRYPTION_SALT_SIZE); + options.encryptionNumHashRounds = preferences.getInteger(IRODSProtocol.ENCRYPTION_HASH_ROUNDS); + log.debug("encryption algorithm = [{}], encryption key size = [{}], encryption salt size = [{}], encryption hash rounds = [{}]", + options.encryptionAlgorithm, options.encryptionKeySize, options.encryptionSaltSize, options.encryptionNumHashRounds); + + return options; + } + + public static AuthPlugin newAuthPlugin(IRODSSession session) { + AuthPlugin plugin = null; + + final String authScheme = StringUtils.defaultIfBlank(session.getHost().getProtocol().getAuthorization(), "native"); + if("native".equalsIgnoreCase(authScheme) || "standard".equalsIgnoreCase(authScheme)) { + plugin = new NativeAuthPlugin(); + } + else if("pam_password".equalsIgnoreCase(authScheme)) { + plugin = new PamPasswordAuthPlugin(true); + } + else { + throw new IllegalArgumentException(String.format("Authentication scheme not recognized: %s", authScheme)); + } + + return plugin; + } + + public static IRODSConnection newAuthenticatedConnection(IRODSSession session) throws Exception { + String host = session.getHost().getHostname(); + int port = session.getHost().getPort(); + String zone = session.getRegion(); + String username = session.getHost().getCredentials().getUsername(); + String password = session.getHost().getCredentials().getPassword(); + IRODSConnection conn = new IRODSConnection(initConnectionOptions(session)); + conn.connect(host, port, new QualifiedUsername(username, zone)); + conn.authenticate(newAuthPlugin(session), password); + return conn; + } + + public static void startIRODSConnectionPool(IRODSSession session, IRODSConnectionPool connPool) throws IRODSException, IOException { + String host = session.getHost().getHostname(); + int port = session.getHost().getPort(); + String zone = session.getRegion(); + String username = session.getHost().getCredentials().getUsername(); + String password = session.getHost().getCredentials().getPassword(); + + connPool.start( + host, + port, + new QualifiedUsername(username, zone), + conn -> { + try { + IRODSApi.rcAuthenticateClient(conn, newAuthPlugin(session), password); + return true; + } + catch(Exception e) { + log.error(e.getMessage()); + return false; + } + }); + } +} diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSCopyFeature.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSCopyFeature.java index 98c1a187c1c..0718e7f17ec 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSCopyFeature.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSCopyFeature.java @@ -1,19 +1,8 @@ package ch.cyberduck.core.irods; -import java.io.IOException; -import java.util.EnumSet; - -import org.irods.irods4j.high_level.connection.IRODSConnection; -import org.irods.irods4j.high_level.vfs.CollectionEntry; -import org.irods.irods4j.high_level.vfs.IRODSCollectionIterator; -import org.irods.irods4j.high_level.vfs.IRODSFilesystem; -import org.irods.irods4j.high_level.vfs.ObjectStatus; -import org.irods.irods4j.low_level.api.IRODSApi.RcComm; -import org.irods.irods4j.low_level.api.IRODSException; - /* - * Copyright (c) 2002-2015 David Kocher. All rights reserved. - * http://cyberduck.ch/ + * Copyright (c) 2002-2025 iterate GmbH. All rights reserved. + * https://cyberduck.io/ * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -24,8 +13,6 @@ * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. - * - * Bug fixes, suggestions and comments should be sent to feedback@cyberduck.ch */ import ch.cyberduck.core.ConnectionCallback; @@ -34,6 +21,15 @@ import ch.cyberduck.core.features.Copy; import ch.cyberduck.core.io.StreamListener; +import ch.cyberduck.core.transfer.TransferStatus; + +import org.irods.irods4j.high_level.connection.IRODSConnection; +import org.irods.irods4j.high_level.vfs.IRODSFilesystem; +import org.irods.irods4j.low_level.api.IRODSException; + +import java.io.IOException; +import java.util.EnumSet; + public class IRODSCopyFeature implements Copy { private final IRODSSession session; @@ -43,51 +39,24 @@ public IRODSCopyFeature(final IRODSSession session) { } @Override - public Path copy(final Path source, final Path target, final ch.cyberduck.core.transfer.TransferStatus status, final ConnectionCallback callback, final StreamListener listener) throws BackgroundException { + public Path copy(final Path source, final Path target, final TransferStatus status, + final ConnectionCallback callback, final StreamListener listener) throws BackgroundException { try { final IRODSConnection conn = session.getClient(); final String from = source.getAbsolute(); final String to = target.getAbsolute(); - if (source.isFile()) { - IRODSFilesystem.copyDataObject(conn.getRcComm(), from, to); - if (listener != null && status.getLength() > 0) { - listener.sent(status.getLength()); - } - } - if(source.isDirectory()) { - this.copyDirectoryRecursively(conn.getRcComm(), from, to); - } + // TODO If we're dealing with a collection, should existing data objects sharing + // the same name be overwritten? This should probably be a configurable option. + IRODSFilesystem.copy(conn.getRcComm(), from, to, IRODSFilesystem.CopyOptions.RECURSIVE); + return target; } catch(IOException | IRODSException e) { throw new IRODSExceptionMappingService().map("Cannot copy {0}", e, source); } } - - public static void copyDirectoryRecursively(RcComm rcComm, String source, String target) throws IOException, IRODSException { - // First, create the root of the target directory - if (!IRODSFilesystem.exists(rcComm, target)) { - IRODSFilesystem.createCollection(rcComm, target); - } - - // Recursively iterate through the source collection - for (CollectionEntry entry : new IRODSCollectionIterator(rcComm, source)) { - String relative = entry.path().substring(source.length()); // relative path from source - String targetPath = target + relative; - - ObjectStatus status = entry.status(); - if (status.getType() == ObjectStatus.ObjectType.COLLECTION) { - // Create directory in target - IRODSFilesystem.createCollection(rcComm, targetPath); - } else if (status.getType() == ObjectStatus.ObjectType.DATA_OBJECT) { - // Copy file - IRODSFilesystem.copyDataObject(rcComm, entry.path(), targetPath); - } - } - } - @Override public EnumSet features(final Path source, final Path target) { return EnumSet.of(Flags.recursive); diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSDeleteFeature.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSDeleteFeature.java index 029730dfe3b..70af16c5043 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSDeleteFeature.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSDeleteFeature.java @@ -1,20 +1,8 @@ package ch.cyberduck.core.irods; -import java.io.IOException; -import java.util.ArrayList; -import java.util.EnumSet; -import java.util.List; -import java.util.Map; - -import org.irods.irods4j.high_level.vfs.IRODSFilesystem; -import org.irods.irods4j.high_level.vfs.IRODSFilesystem.RemoveOptions; -import org.irods.irods4j.high_level.vfs.ObjectStatus; -import org.irods.irods4j.high_level.vfs.ObjectStatus.ObjectType; -import org.irods.irods4j.low_level.api.IRODSException; - /* - * Copyright (c) 2002-2015 David Kocher. All rights reserved. - * http://cyberduck.ch/ + * Copyright (c) 2002-2025 iterate GmbH. All rights reserved. + * https://cyberduck.io/ * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -25,8 +13,6 @@ * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. - * - * Bug fixes, suggestions and comments should be sent to feedback@cyberduck.ch */ import ch.cyberduck.core.PasswordCallback; @@ -34,14 +20,38 @@ import ch.cyberduck.core.exception.BackgroundException; import ch.cyberduck.core.exception.NotfoundException; import ch.cyberduck.core.features.Delete; +import ch.cyberduck.core.preferences.HostPreferencesFactory; +import ch.cyberduck.core.preferences.PreferencesReader; import ch.cyberduck.core.transfer.TransferStatus; +import org.irods.irods4j.high_level.connection.IRODSConnection; +import org.irods.irods4j.high_level.vfs.IRODSFilesystem; +import org.irods.irods4j.high_level.vfs.IRODSFilesystem.RemoveOptions; +import org.irods.irods4j.high_level.vfs.ObjectStatus; +import org.irods.irods4j.high_level.vfs.ObjectStatus.ObjectType; +import org.irods.irods4j.low_level.api.IRODSException; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.EnumSet; +import java.util.List; +import java.util.Map; + public class IRODSDeleteFeature implements Delete { private final IRODSSession session; + private final RemoveOptions removeOptions; public IRODSDeleteFeature(IRODSSession session) { this.session = session; + + PreferencesReader prefs = HostPreferencesFactory.get(session.getHost()); + if("yes".equalsIgnoreCase(prefs.getProperty(IRODSProtocol.DELETE_OBJECTS_PERMANTENTLY))) { + removeOptions = RemoveOptions.NO_TRASH; + } + else { + removeOptions = RemoveOptions.NONE; + } } @Override @@ -55,22 +65,29 @@ public void delete(final Map files, final PasswordCallback break; } } + if(skip) { continue; } + deleted.add(file); callback.delete(file); - final String absolute = file.getAbsolute(); + try { - if (!IRODSFilesystem.exists(this.session.getClient().getRcComm(), absolute)) { - throw new NotfoundException(String.format("%s doesn't exist", absolute)); + final IRODSConnection conn = session.getClient(); + final String logicalPath = file.getAbsolute(); + final ObjectStatus status = IRODSFilesystem.status(conn.getRcComm(), logicalPath); + + if(!IRODSFilesystem.exists(status)) { + throw new NotfoundException(String.format("%s doesn't exist", logicalPath)); + } + + if(status.getType() == ObjectType.DATA_OBJECT) { + IRODSFilesystem.remove(conn.getRcComm(), logicalPath, removeOptions); + } + else if(status.getType() == ObjectType.COLLECTION) { + IRODSFilesystem.removeAll(conn.getRcComm(), logicalPath, removeOptions); } - ObjectStatus status = IRODSFilesystem.status(this.session.getClient().getRcComm(), absolute); - if (status.equals(ObjectType.DATA_OBJECT)) { - IRODSFilesystem.remove(this.session.getClient().getRcComm(), absolute, RemoveOptions.NO_TRASH); - } else if (status.equals(ObjectType.COLLECTION)) { - IRODSFilesystem.removeAll(this.session.getClient().getRcComm(), absolute, RemoveOptions.NO_TRASH); - } } catch(IOException | IRODSException e) { throw new IRODSExceptionMappingService().map("Cannot delete {0}", e, file); diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSDirectoryFeature.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSDirectoryFeature.java index 854b8c9349b..3f473ea686b 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSDirectoryFeature.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSDirectoryFeature.java @@ -1,14 +1,8 @@ package ch.cyberduck.core.irods; -import java.io.IOException; - -import org.irods.irods4j.high_level.connection.IRODSConnection; -import org.irods.irods4j.high_level.vfs.IRODSFilesystem; -import org.irods.irods4j.high_level.vfs.IRODSFilesystemException; - /* - * Copyright (c) 2002-2015 David Kocher. All rights reserved. - * http://cyberduck.ch/ + * Copyright (c) 2002-2025 iterate GmbH. All rights reserved. + * https://cyberduck.io/ * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -19,8 +13,6 @@ * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. - * - * Bug fixes, suggestions and comments should be sent to feedback@cyberduck.ch */ import ch.cyberduck.core.Path; @@ -29,6 +21,12 @@ import ch.cyberduck.core.features.Write; import ch.cyberduck.core.transfer.TransferStatus; +import org.irods.irods4j.high_level.connection.IRODSConnection; +import org.irods.irods4j.high_level.vfs.IRODSFilesystem; +import org.irods.irods4j.high_level.vfs.IRODSFilesystemException; + +import java.io.IOException; + public class IRODSDirectoryFeature implements Directory { private final IRODSSession session; @@ -41,11 +39,7 @@ public IRODSDirectoryFeature(final IRODSSession session) { public Path mkdir(final Write writer, final Path folder, final TransferStatus status) throws BackgroundException { try { final IRODSConnection conn = session.getClient(); - String path = folder.getAbsolute(); - boolean created = IRODSFilesystem.createCollection(conn.getRcComm(), path); - if (!created) { - throw new IOException("Failed to create collection: " + path); - } + IRODSFilesystem.createCollection(conn.getRcComm(), folder.getAbsolute()); return folder; } catch(IOException | IRODSFilesystemException e) { diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSDownloadFeature.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSDownloadFeature.java deleted file mode 100644 index da352fcde4a..00000000000 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSDownloadFeature.java +++ /dev/null @@ -1,141 +0,0 @@ -package ch.cyberduck.core.irods; - -import java.io.RandomAccessFile; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; - -import org.irods.irods4j.high_level.connection.IRODSConnectionPool; -import org.irods.irods4j.high_level.connection.IRODSConnectionPool.PoolConnection; -import org.irods.irods4j.high_level.connection.QualifiedUsername; -import org.irods.irods4j.high_level.io.IRODSDataObjectInputStream; -import org.irods.irods4j.high_level.vfs.IRODSFilesystem; -import org.irods.irods4j.low_level.api.IRODSApi; -import org.irods.irods4j.low_level.api.IRODSApi.RcComm; - -/* - * Copyright (c) 2002-2015 David Kocher. All rights reserved. - * http://cyberduck.ch/ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * Bug fixes, suggestions and comments should be sent to feedback@cyberduck.ch - */ - -import ch.cyberduck.core.ConnectionCallback; -import ch.cyberduck.core.Local; -import ch.cyberduck.core.Path; -import ch.cyberduck.core.exception.BackgroundException; -import ch.cyberduck.core.exception.NotfoundException; -import ch.cyberduck.core.features.Download; -import ch.cyberduck.core.features.Read; -import ch.cyberduck.core.io.BandwidthThrottle; -import ch.cyberduck.core.io.StreamListener; -import ch.cyberduck.core.transfer.TransferStatus; - -public class IRODSDownloadFeature implements Download { - - private final IRODSSession session; - private boolean truncate = true; - private boolean append = false; - private static final int BUFFER_SIZE = 4 * 1024 * 1024; - - public IRODSDownloadFeature(final IRODSSession session) { - this.session = session; - } - - @Override - public void download(final Read read, final Path file, final Local local, final BandwidthThrottle throttle, - final StreamListener listener, final TransferStatus status, - final ConnectionCallback callback) throws BackgroundException { - final int numThread = 3; - try { - - final RcComm primaryConn = session.getClient().getRcComm(); - final String logicalPath = file.getAbsolute(); - - if (!IRODSFilesystem.exists(primaryConn, logicalPath)) { - throw new NotfoundException(logicalPath); - } - - final long fileSize = IRODSFilesystem.dataObjectSize(primaryConn, logicalPath); - - // Step 1: Get replica token & number via primary stream - try (IRODSDataObjectInputStream primary = new IRODSDataObjectInputStream(primaryConn, logicalPath)) { - final String replicaToken = primary.getReplicaToken(); - final long replicaNumber = primary.getReplicaNumber(); - - // Step 2: Setup connection pool - final IRODSConnectionPool pool = new IRODSConnectionPool(numThread); - pool.start( - session.getHost().getHostname(), - session.getHost().getPort(), - new QualifiedUsername(session.getHost().getCredentials().getUsername(), session.getRegion()), - conn -> { - try { - IRODSApi.rcAuthenticateClient(conn, "native", session.getHost().getCredentials().getPassword()); - return true; - } catch (Exception e) { - return false; - } - }); - - final ExecutorService executor = Executors.newFixedThreadPool(numThread); - - //TODO:fileSize/ - final long chunkSize = fileSize / numThread; - final long remainChunkSize = fileSize % numThread; - - - // Step 3: Create empty target file - try (RandomAccessFile out = new RandomAccessFile(local.getAbsolute(), "rw")) { - out.setLength(fileSize); - } - - // Step 4: Parallel readers - List> tasks = new ArrayList<>(); - for (int i = 0; i < numThread; i++) { - final long start = i * chunkSize; - final PoolConnection conn = pool.getConnection(); - IRODSDataObjectInputStream stream = new IRODSDataObjectInputStream(conn.getRcComm(), replicaToken, replicaNumber); - ChunkWorker worker = new ChunkWorker( - stream, - local.getAbsolute(), - start, - (numThread - 1 == i) ? chunkSize + remainChunkSize : chunkSize, - BUFFER_SIZE - ); - Future task = executor.submit(worker); - tasks.add(task); - } - - for (Future task : tasks) { - task.get(); - } - - - executor.shutdown(); - pool.close(); - } - - } - catch(Exception e) { - throw new IRODSExceptionMappingService().map("Download {0} failed", e); - } - } - - @Override - public boolean offset(final Path file) { - return false; - } -} diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSExceptionMappingService.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSExceptionMappingService.java index b2030444ec9..32f22de554a 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSExceptionMappingService.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSExceptionMappingService.java @@ -1,11 +1,8 @@ package ch.cyberduck.core.irods; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - /* - * Copyright (c) 2002-2015 David Kocher. All rights reserved. - * http://cyberduck.ch/ + * Copyright (c) 2002-2025 iterate GmbH. All rights reserved. + * https://cyberduck.io/ * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -16,23 +13,72 @@ * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. - * - * Bug fixes, suggestions and comments should be sent to feedback@cyberduck.ch */ import ch.cyberduck.core.AbstractExceptionMappingService; +import ch.cyberduck.core.exception.AccessDeniedException; import ch.cyberduck.core.exception.BackgroundException; +import ch.cyberduck.core.exception.ConnectionRefusedException; +import ch.cyberduck.core.exception.LockedException; +import ch.cyberduck.core.exception.LoginFailureException; + +import ch.cyberduck.core.exception.NotfoundException; +import ch.cyberduck.core.exception.QuotaException; +import ch.cyberduck.core.exception.SSLNegotiateException; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.irods.irods4j.low_level.api.IRODSErrorCodes; +import org.irods.irods4j.low_level.api.IRODSException; + public class IRODSExceptionMappingService extends AbstractExceptionMappingService { + private static final Logger log = LogManager.getLogger(IRODSExceptionMappingService.class); @Override public BackgroundException map(final Exception e) { - //TODO: write a more complete exception mapping services log.warn("Map failure {}", e.toString()); final StringBuilder buffer = new StringBuilder(); this.append(buffer, e.getMessage()); + + if(e instanceof IRODSException) { + switch(((IRODSException) e).getErrorCode()) { + case IRODSErrorCodes.AUTHENTICATION_ERROR: + case IRODSErrorCodes.CAT_INVALID_AUTHENTICATION: + case IRODSErrorCodes.CAT_PASSWORD_EXPIRED: + case IRODSErrorCodes.CAT_INVALID_USER: + return new LoginFailureException(buffer.toString(), e); + + case IRODSErrorCodes.CAT_NO_ACCESS_PERMISSION: + case IRODSErrorCodes.CAT_INSUFFICIENT_PRIVILEGE_LEVEL: + case IRODSErrorCodes.SYS_NOT_ALLOWED: + return new AccessDeniedException(buffer.toString(), e); + + case IRODSErrorCodes.INTERMEDIATE_REPLICA_ACCESS: + case IRODSErrorCodes.SYS_REPLICA_INACCESSIBLE: + return new LockedException(buffer.toString(), e); + + case IRODSErrorCodes.SSL_CERT_ERROR: + case IRODSErrorCodes.SSL_HANDSHAKE_ERROR: + case IRODSErrorCodes.SSL_INIT_ERROR: + case IRODSErrorCodes.SSL_SHUTDOWN_ERROR: + case IRODSErrorCodes.SSL_NOT_BUILT_INTO_CLIENT: + case IRODSErrorCodes.SSL_NOT_BUILT_INTO_SERVER: + return new SSLNegotiateException(buffer.toString(), e); + + case IRODSErrorCodes.SYS_RESC_QUOTA_EXCEEDED: + return new QuotaException(buffer.toString(), e); + + case IRODSErrorCodes.CAT_NO_ROWS_FOUND: + case IRODSErrorCodes.CAT_NOT_A_DATAOBJ_AND_NOT_A_COLLECTION: + return new NotfoundException(buffer.toString(), e); + + case IRODSErrorCodes.CONNECTION_REFUSED: + return new ConnectionRefusedException(buffer.toString(), e); + } + } + return this.wrap(e, buffer); } - } diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSFindFeature.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSFindFeature.java index a350da3d429..52364eb011b 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSFindFeature.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSFindFeature.java @@ -1,14 +1,8 @@ package ch.cyberduck.core.irods; -import java.io.IOException; - -import org.irods.irods4j.high_level.connection.IRODSConnection; -import org.irods.irods4j.high_level.vfs.IRODSFilesystem; -import org.irods.irods4j.low_level.api.IRODSException; - /* - * Copyright (c) 2002-2015 David Kocher. All rights reserved. - * http://cyberduck.ch/ + * Copyright (c) 2002-2025 iterate GmbH. All rights reserved. + * https://cyberduck.io/ * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -19,8 +13,6 @@ * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. - * - * Bug fixes, suggestions and comments should be sent to feedback@cyberduck.ch */ import ch.cyberduck.core.ListProgressListener; @@ -28,6 +20,12 @@ import ch.cyberduck.core.exception.BackgroundException; import ch.cyberduck.core.features.Find; +import org.irods.irods4j.high_level.connection.IRODSConnection; +import org.irods.irods4j.high_level.vfs.IRODSFilesystem; +import org.irods.irods4j.low_level.api.IRODSException; + +import java.io.IOException; + public class IRODSFindFeature implements Find { private final IRODSSession session; @@ -41,13 +39,13 @@ public boolean find(final Path file, final ListProgressListener listener) throws if(file.isRoot()) { return true; } + try { final IRODSConnection conn = session.getClient(); return IRODSFilesystem.exists(conn.getRcComm(), file.getAbsolute()); } catch(IOException | IRODSException e) { - throw new IRODSExceptionMappingService().map("Failure to read attributes of {0}", e, file); + throw new IRODSExceptionMappingService().map("Failure to find {0}", e, file); } - } } diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSHomeFinderService.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSHomeFinderService.java index 344ca0af4f0..8833694c733 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSHomeFinderService.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSHomeFinderService.java @@ -1,11 +1,7 @@ package ch.cyberduck.core.irods; -import java.util.EnumSet; - -import org.apache.commons.lang3.StringUtils; - /* - * Copyright (c) 2002-2016 iterate GmbH. All rights reserved. + * Copyright (c) 2002-2025 iterate GmbH. All rights reserved. * https://cyberduck.io/ * * This program is free software; you can redistribute it and/or modify @@ -24,6 +20,10 @@ import ch.cyberduck.core.exception.BackgroundException; import ch.cyberduck.core.shared.AbstractHomeFeature; +import org.apache.commons.lang3.StringUtils; + +import java.util.EnumSet; + public class IRODSHomeFinderService extends AbstractHomeFeature { private final IRODSSession session; @@ -36,16 +36,18 @@ public IRODSHomeFinderService(final IRODSSession session) { public Path find() throws BackgroundException { final String user; final Credentials credentials = session.getHost().getCredentials(); + if(StringUtils.contains(credentials.getUsername(), ':')) { user = StringUtils.splitPreserveAllTokens(credentials.getUsername(), ':')[1]; } else { user = credentials.getUsername(); } + return new Path(new StringBuilder() - .append(Path.DELIMITER).append(session.getRegion()) - .append(Path.DELIMITER).append("home") - .append(Path.DELIMITER).append(user) - .toString(), EnumSet.of(Path.Type.directory, Path.Type.volume)); + .append(Path.DELIMITER).append(session.getRegion()) + .append(Path.DELIMITER).append("home") + .append(Path.DELIMITER).append(user) + .toString(), EnumSet.of(Path.Type.directory, Path.Type.volume)); } } diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSIntegerUtils.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSIntegerUtils.java new file mode 100644 index 00000000000..99b2c40ef0f --- /dev/null +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSIntegerUtils.java @@ -0,0 +1,32 @@ +package ch.cyberduck.core.irods; + +/* + * Copyright (c) 2002-2025 iterate GmbH. All rights reserved. + * https://cyberduck.io/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +public final class IRODSIntegerUtils { + + static > T clamp(T value, T low, T high) { + if(value.compareTo(low) < 0) { + return low; + } + + if(value.compareTo(high) > 0) { + return high; + } + + return value; + } + +} diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSListService.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSListService.java index 439b364b501..64117f4ee77 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSListService.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSListService.java @@ -1,24 +1,8 @@ package ch.cyberduck.core.irods; -import java.io.IOException; -import java.util.EnumSet; -import java.util.List; - -import org.apache.commons.io.FilenameUtils; -import org.apache.commons.lang3.StringUtils; -import org.irods.irods4j.common.Versioning; -import org.irods.irods4j.high_level.catalog.IRODSQuery; -import org.irods.irods4j.high_level.catalog.IRODSQuery.GenQuery1QueryArgs; -import org.irods.irods4j.high_level.connection.IRODSConnection; -import org.irods.irods4j.high_level.vfs.CollectionEntry; -import org.irods.irods4j.high_level.vfs.IRODSCollectionIterator; -import org.irods.irods4j.high_level.vfs.IRODSFilesystem; -import org.irods.irods4j.low_level.api.GenQuery1Columns; -import org.irods.irods4j.low_level.api.IRODSException; - /* - * Copyright (c) 2002-2015 David Kocher. All rights reserved. - * http://cyberduck.ch/ + * Copyright (c) 2002-2025 iterate GmbH. All rights reserved. + * https://cyberduck.io/ * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -29,8 +13,6 @@ * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. - * - * Bug fixes, suggestions and comments should be sent to feedback@cyberduck.ch */ import ch.cyberduck.core.AttributedList; @@ -41,7 +23,16 @@ import ch.cyberduck.core.PathNormalizer; import ch.cyberduck.core.exception.BackgroundException; import ch.cyberduck.core.exception.NotfoundException; -import ch.cyberduck.core.io.Checksum; + +import org.apache.commons.lang3.StringUtils; +import org.irods.irods4j.high_level.connection.IRODSConnection; +import org.irods.irods4j.high_level.vfs.CollectionEntry; +import org.irods.irods4j.high_level.vfs.IRODSCollectionIterator; +import org.irods.irods4j.high_level.vfs.IRODSFilesystem; +import org.irods.irods4j.low_level.api.IRODSException; + +import java.io.IOException; +import java.util.EnumSet; public class IRODSListService implements ListService { @@ -54,83 +45,45 @@ public IRODSListService(IRODSSession session) { @Override public AttributedList list(final Path directory, final ListProgressListener listener) throws BackgroundException { try { - final AttributedList children = new AttributedList(); final IRODSConnection conn = session.getClient(); - String path = directory.getAbsolute(); - if (!IRODSFilesystem.exists(conn.getRcComm(), path)) { - throw new NotfoundException(path); + + String logicalPath = directory.getAbsolute(); + if(!IRODSFilesystem.exists(conn.getRcComm(), logicalPath)) { + throw new NotfoundException(logicalPath); } - final IRODSCollectionIterator iterator = new IRODSCollectionIterator(conn.getRcComm(), path); - - for (CollectionEntry entry : iterator) { - final String normalized = PathNormalizer.normalize(entry.path(), true); - if(StringUtils.equals(normalized, directory.getAbsolute())) { - continue; - } - final PathAttributes attributes = new PathAttributes(); - String logicalPath = entry.path(); - String parentPath = FilenameUtils.getFullPathNoEndSeparator(logicalPath); - String fileName = FilenameUtils.getName(logicalPath); - String query = ""; - //check version - if(Versioning.compareVersions(conn.getRcComm().relVersion.substring(4), "4.3.4") > 0) { - query = String.format("select DATA_MODIFY_TIME, DATA_CREATE_TIME, DATA_SIZE, DATA_CHECKSUM, DATA_OWNER_NAME, DATA_OWNER_ZONE where COLL_NAME = '%s' and DATA_NAME = '%s'", parentPath, fileName); - List> rows = IRODSQuery.executeGenQuery2(conn.getRcComm(), query); - List row = rows.get(0); - attributes.setModificationDate(Long.parseLong(row.get(0)) * 1000); // seconds to ms - attributes.setCreationDate(Long.parseLong(row.get(1)) * 1000); - attributes.setSize(Long.parseLong(row.get(2))); - String checksum = row.get(3); - if (!StringUtils.isEmpty(checksum)) { - attributes.setChecksum(Checksum.parse(checksum)); - } - - attributes.setOwner(row.get(4)); - attributes.setGroup(row.get(5)); - }else { - //if older version, use GenQuery1 - GenQuery1QueryArgs input = new GenQuery1QueryArgs(); - - // select COLL_NAME, DATA_NAME, DATA_ACCESS_TIME - input.addColumnToSelectClause(GenQuery1Columns.COL_D_MODIFY_TIME); - input.addColumnToSelectClause(GenQuery1Columns.COL_D_CREATE_TIME); - input.addColumnToSelectClause(GenQuery1Columns.COL_DATA_SIZE); - input.addColumnToSelectClause(GenQuery1Columns.COL_D_DATA_CHECKSUM); - input.addColumnToSelectClause(GenQuery1Columns.COL_D_OWNER_NAME); - input.addColumnToSelectClause(GenQuery1Columns.COL_D_OWNER_ZONE); - - - // where COLL_NAME like '/tempZone/home/rods and DATA_NAME = 'atime.txt' - String collNameCondStr = String.format("= '%s'", parentPath); - String dataNameCondStr = String.format("= '%s'", fileName); - input.addConditionToWhereClause(GenQuery1Columns.COL_COLL_NAME, collNameCondStr); - input.addConditionToWhereClause(GenQuery1Columns.COL_DATA_NAME, dataNameCondStr); - - IRODSQuery.executeGenQuery1(conn.getRcComm(), input, row -> { - attributes.setModificationDate(Long.parseLong(row.get(0)) * 1000); // seconds to ms - attributes.setCreationDate(Long.parseLong(row.get(1)) * 1000); - attributes.setSize(Long.parseLong(row.get(2))); - String checksum = row.get(3); - if (!StringUtils.isEmpty(checksum)) { - attributes.setChecksum(Checksum.parse(checksum)); - } - - attributes.setOwner(row.get(4)); - attributes.setGroup(row.get(5)); - return false; - }); - } - EnumSet type = entry.isCollection() - ? EnumSet.of(Path.Type.directory) - : EnumSet.of(Path.Type.file); - - children.add(new Path(directory, PathNormalizer.name(normalized), type, attributes)); - listener.chunk(directory, children); + + final AttributedList children = new AttributedList(); + + for(CollectionEntry entry : new IRODSCollectionIterator(conn.getRcComm(), logicalPath)) { + final String normalized = PathNormalizer.normalize(entry.path(), true); + if(StringUtils.equals(normalized, directory.getAbsolute())) { + continue; + } + + PathAttributes attrs = new PathAttributes(); + attrs.setCreationDate(entry.createdAt() * 1000L); + attrs.setModificationDate(entry.modifiedAt() * 1000L); + + EnumSet type = EnumSet.of(Path.Type.file); + + if(entry.isCollection()) { + attrs.setDirectoryId(entry.id()); + type = EnumSet.of(Path.Type.directory); + } + else if(entry.isDataObject()) { + attrs.setFileId(entry.id()); + attrs.setSize(entry.dataSize()); + attrs.setChecksum(IRODSChecksumUtils.toChecksum(entry.checksum())); + } + + children.add(new Path(directory, PathNormalizer.name(normalized), type, attrs)); + listener.chunk(directory, children); } + return children; } catch(IRODSException | IOException e) { - throw new IRODSExceptionMappingService().map("Listing directory {0} failed", e, directory); + throw new IRODSExceptionMappingService().map("Listing {0} failed", e, directory); } } } diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSMoveFeature.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSMoveFeature.java index b36eb99b8da..537a852efe1 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSMoveFeature.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSMoveFeature.java @@ -1,16 +1,8 @@ package ch.cyberduck.core.irods; -import java.io.IOException; -import java.util.Collections; -import java.util.EnumSet; - -import org.irods.irods4j.high_level.connection.IRODSConnection; -import org.irods.irods4j.high_level.vfs.IRODSFilesystem; -import org.irods.irods4j.low_level.api.IRODSException; - /* - * Copyright (c) 2002-2015 David Kocher. All rights reserved. - * http://cyberduck.ch/ + * Copyright (c) 2002-2025 iterate GmbH. All rights reserved. + * https://cyberduck.io/ * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -21,8 +13,6 @@ * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. - * - * Bug fixes, suggestions and comments should be sent to feedback@cyberduck.ch */ import ch.cyberduck.core.ConnectionCallback; @@ -33,6 +23,14 @@ import ch.cyberduck.core.features.Move; import ch.cyberduck.core.transfer.TransferStatus; +import org.irods.irods4j.high_level.connection.IRODSConnection; +import org.irods.irods4j.high_level.vfs.IRODSFilesystem; +import org.irods.irods4j.low_level.api.IRODSException; + +import java.io.IOException; +import java.util.Collections; +import java.util.EnumSet; + public class IRODSMoveFeature implements Move { private final IRODSSession session; @@ -44,17 +42,18 @@ public IRODSMoveFeature(IRODSSession session) { } @Override - public Path move(final Path file, final Path renamed, final TransferStatus status, final Delete.Callback callback, final ConnectionCallback connectionCallback) throws BackgroundException { + public Path move(final Path file, final Path renamed, final TransferStatus status, + final Delete.Callback callback, final ConnectionCallback connectionCallback) throws BackgroundException { try { final IRODSConnection conn = session.getClient(); if(!IRODSFilesystem.exists(conn.getRcComm(), file.getAbsolute())) { - throw new NotfoundException(String.format("%s doesn't exist", file.getAbsolute())); + throw new NotfoundException(String.format("[%s] doesn't exist", file.getAbsolute())); } if(status.isExists()) { delete.delete(Collections.singletonMap(renamed, status), connectionCallback, callback); } IRODSFilesystem.rename(conn.getRcComm(), file.getAbsolute(), renamed.getAbsolute()); - return renamed; + return renamed; } catch(IOException | IRODSException e) { throw new IRODSExceptionMappingService().map("Cannot rename {0}", e, file); diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSProtocol.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSProtocol.java index 8a7ecc913af..8e76eafd3f5 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSProtocol.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSProtocol.java @@ -1,12 +1,8 @@ package ch.cyberduck.core.irods; -import org.apache.commons.lang3.StringUtils; - -import com.google.auto.service.AutoService; - /* - * Copyright (c) 2002-2015 David Kocher. All rights reserved. - * http://cyberduck.ch/ + * Copyright (c) 2002-2025 iterate GmbH. All rights reserved. + * https://cyberduck.io/ * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -17,8 +13,6 @@ * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. - * - * Bug fixes, suggestions and comments should be sent to feedback@cyberduck.ch */ import ch.cyberduck.core.AbstractProtocol; @@ -26,9 +20,30 @@ import ch.cyberduck.core.Protocol; import ch.cyberduck.core.Scheme; +import org.apache.commons.lang3.StringUtils; + +import com.google.auto.service.AutoService; + +import java.util.HashMap; +import java.util.Map; + @AutoService(Protocol.class) public final class IRODSProtocol extends AbstractProtocol { + public static final String DESTINATION_RESOURCE = "Destination Resource"; + public static final String DELETE_OBJECTS_PERMANTENTLY = "Delete Objects Permanently"; + public static final String CLIENT_SERVER_NEGOTIATION = "Client Server Negotiation"; + public static final String TLS_PROTOCOL = "TLS Protocol"; + public static final String TLS_TRUSTSTORE = "TLS Truststore"; + public static final String TLS_TRUSTSTORE_PASSWORD = "TLS Truststore Password"; + public static final String ENCRYPTION_ALGORITHM = "Encryption Algorithm"; + public static final String ENCRYPTION_KEY_SIZE = "Encryption Key Size"; + public static final String ENCRYPTION_SALT_SIZE = "Encryption Salt Size"; + public static final String ENCRYPTION_HASH_ROUNDS = "Encryption Hash Rounds"; + public static final String PARALLEL_TRANSFER_THRESHOLD = "Parallel Transfer Threshold"; + public static final String PARALLEL_TRANSFER_CONNECTIONS = "Parallel Transfer Connections"; + public static final String PARALLEL_TRANSFER_BUFFER_SIZE = "Parallel Transfer Buffer Size"; + @Override public String getIdentifier() { return this.getScheme().name(); @@ -68,4 +83,22 @@ public String getPrefix() { public VersioningMode getVersioningMode() { return VersioningMode.none; } + + @Override + public Map getProperties() { + final Map props = new HashMap<>(); + props.put(DELETE_OBJECTS_PERMANTENTLY, "no"); + props.put(CLIENT_SERVER_NEGOTIATION, "CS_NEG_REFUSE"); + props.put(TLS_PROTOCOL, "TLSv1.2"); + props.put(TLS_TRUSTSTORE, "NOT SET"); + props.put(TLS_TRUSTSTORE_PASSWORD, "NOT SET"); + props.put(ENCRYPTION_ALGORITHM, "AES-256-CBC"); + props.put(ENCRYPTION_KEY_SIZE, "32"); + props.put(ENCRYPTION_SALT_SIZE, "8"); + props.put(ENCRYPTION_HASH_ROUNDS, "16"); + props.put(PARALLEL_TRANSFER_THRESHOLD, "33554432"); // 32MB + props.put(PARALLEL_TRANSFER_CONNECTIONS, "3"); + props.put(PARALLEL_TRANSFER_BUFFER_SIZE, "4194304"); // 4MB + return props; + } } diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSReadFeature.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSReadFeature.java index 81ef579d3d4..19c27e1da68 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSReadFeature.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSReadFeature.java @@ -1,16 +1,8 @@ package ch.cyberduck.core.irods; -import java.io.IOException; -import java.io.InputStream; - -import org.irods.irods4j.high_level.io.IRODSDataObjectInputStream; -import org.irods.irods4j.high_level.vfs.IRODSFilesystem; -import org.irods.irods4j.low_level.api.IRODSApi.RcComm; -import org.irods.irods4j.low_level.api.IRODSException; - /* - * Copyright (c) 2002-2015 David Kocher. All rights reserved. - * http://cyberduck.ch/ + * Copyright (c) 2002-2025 iterate GmbH. All rights reserved. + * https://cyberduck.io/ * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -21,8 +13,6 @@ * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. - * - * Bug fixes, suggestions and comments should be sent to feedback@cyberduck.ch */ import ch.cyberduck.core.ConnectionCallback; @@ -32,6 +22,14 @@ import ch.cyberduck.core.features.Read; import ch.cyberduck.core.transfer.TransferStatus; +import org.irods.irods4j.high_level.io.IRODSDataObjectInputStream; +import org.irods.irods4j.high_level.vfs.IRODSFilesystem; +import org.irods.irods4j.low_level.api.IRODSApi.RcComm; +import org.irods.irods4j.low_level.api.IRODSException; + +import java.io.IOException; +import java.io.InputStream; + public class IRODSReadFeature implements Read { private final IRODSSession session; @@ -42,28 +40,29 @@ public IRODSReadFeature(IRODSSession session) { @Override public InputStream read(final Path file, final TransferStatus status, final ConnectionCallback callback) throws BackgroundException { - - try { - final RcComm rcComm = session.getClient().getRcComm(); - final String logicalPath = file.getAbsolute(); // e.g., "/zone/home/user/file.txt" - - if (!IRODSFilesystem.exists(rcComm, logicalPath)) { - throw new NotfoundException(logicalPath); - } - - // Open input stream - InputStream in = new IRODSDataObjectInputStream(rcComm,logicalPath); - - // If resuming from offset, skip ahead - if(status.isAppend() && status.getOffset() > 0) { - in.skip(status.getOffset()); - } + try { + final RcComm rcComm = session.getClient().getRcComm(); + final String logicalPath = file.getAbsolute(); // e.g. /tempZone/home/rods/data_object.txt - return in; + if(!IRODSFilesystem.exists(rcComm, logicalPath)) { + throw new NotfoundException(logicalPath); } - catch(IOException | IRODSException e) { - throw new IRODSExceptionMappingService().map("Download {0} failed", e, file); + + IRODSDataObjectInputStream in = new IRODSDataObjectInputStream(rcComm, logicalPath); + + if(status.isAppend() && status.getOffset() > 0) { + IRODSStreamUtils.seek(in, status.getOffset()); } - + + return in; + } + catch(IOException | IRODSException e) { + throw new IRODSExceptionMappingService().map("Download of {0} failed", e, file); + } + } + + @Override + public boolean offset(final Path file) { + return true; } } diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSSession.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSSession.java index 3329fe29ccf..8bc4802eafc 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSSession.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSSession.java @@ -1,18 +1,8 @@ package ch.cyberduck.core.irods; -import java.text.MessageFormat; - -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.irods.irods4j.high_level.connection.IRODSConnection; -import org.irods.irods4j.high_level.connection.QualifiedUsername; -import org.irods.irods4j.low_level.api.IRODSApi.ConnectionOptions; - - /* - * Copyright (c) 2002-2015 David Kocher. All rights reserved. - * http://cyberduck.ch/ + * Copyright (c) 2002-2025 iterate GmbH. All rights reserved. + * https://cyberduck.io/ * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -23,12 +13,9 @@ * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. - * - * Bug fixes, suggestions and comments should be sent to feedback@cyberduck.ch */ import ch.cyberduck.core.BookmarkNameProvider; -import ch.cyberduck.core.ConnectionTimeoutFactory; import ch.cyberduck.core.Credentials; import ch.cyberduck.core.Host; import ch.cyberduck.core.HostKeyCallback; @@ -45,7 +32,9 @@ import ch.cyberduck.core.features.Home; import ch.cyberduck.core.features.Move; import ch.cyberduck.core.features.Read; +import ch.cyberduck.core.features.Timestamp; import ch.cyberduck.core.features.Touch; +import ch.cyberduck.core.features.Upload; import ch.cyberduck.core.features.Write; import ch.cyberduck.core.proxy.ProxyFinder; import ch.cyberduck.core.shared.DefaultPathHomeFeature; @@ -58,8 +47,21 @@ import ch.cyberduck.core.ssl.X509TrustManager; import ch.cyberduck.core.threading.CancelCallback; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.irods.irods4j.high_level.connection.IRODSConnection; +import org.irods.irods4j.high_level.connection.QualifiedUsername; +import org.irods.irods4j.low_level.api.IRODSApi; + +import java.text.MessageFormat; public class IRODSSession extends SSLSession { + + static { + IRODSApi.setApplicationName("Cyberduck"); + } + private static final Logger log = LogManager.getLogger(IRODSSession.class); public IRODSSession(final Host h) { @@ -70,34 +72,39 @@ public IRODSSession(final Host h, final X509TrustManager trust, final X509KeyMan super(h, trust, key); } + @Override + public boolean isConnected() { + return super.isConnected() && null != client && client.isConnected(); + } + @Override protected IRODSConnection connect(final ProxyFinder proxy, final HostKeyCallback key, final LoginCallback prompt, final CancelCallback cancel) throws BackgroundException { try { - final String host = "localhost"; - final int port = 1247; - final String zone = "tempZone"; - - ConnectionOptions options=new ConnectionOptions(); - - - IRODSConnection conn = new IRODSConnection(options); - conn.connect(host, port, new QualifiedUsername("rods", zone)); - - + log.debug("connecting to iRODS server."); + + final String host = this.host.getHostname(); + final int port = this.host.getPort(); + final String username = this.host.getCredentials().getUsername(); + final String zone = getRegion(); + + log.debug("iRODS server: host=[{}], port=[{}], username=[{}], zone=[{}]", host, port, username, zone); + + IRODSConnection conn = new IRODSConnection(IRODSConnectionUtils.initConnectionOptions(this)); + conn.connect(host, port, new QualifiedUsername(username, zone)); + log.debug("connected to iRODS server successfully."); + return conn; - } catch (Exception e) { - String msg = String.format("exception=[%s], host=[%s], port=[%d], username=[%s], zone=[%s]", e.getMessage(), this.host.getHostname(),this.host.getPort(), this.host.getCredentials().getUsername(),getRegion()); - throw new BackgroundException("Failed to connect to iRODS - "+ msg,e); } - } + catch(Exception e) { + final String host = this.host.getHostname(); + final int port = this.host.getPort(); + final String username = this.host.getCredentials().getUsername(); + final String zone = getRegion(); - protected ConnectionOptions configure(final ConnectionOptions options) { - //TODO update to use configure - final PreferencesReader preferences = HostPreferencesFactory.get(host); - options.tcpReceiveBufferSize=preferences.getInteger("connection.chunksize"); - options.tcpSendBufferSize=preferences.getInteger("connection.chunksize"); - - return options; + String msg = String.format("Could not connect to iRODS server at [%s:%d] as [%s#%s]: %s", + host, port, username, zone, e.getMessage()); + throw new IRODSExceptionMappingService().map(msg, e); + } } protected String getRegion() { @@ -116,39 +123,42 @@ protected String getResource() { @Override public void login(final LoginCallback prompt, final CancelCallback cancel) throws BackgroundException { - try { + try { + log.debug("authenticating with iRODS server."); + final Credentials credentials = host.getCredentials(); - final String username = credentials.getUsername(); final String password = credentials.getPassword(); - final String authScheme = StringUtils.defaultIfBlank( - host.getProtocol().getAuthorization(), - "native" - ); - //irods4j authenticate - client.authenticate(authScheme, password); - log.debug("Authenticated to iRODS as {}", username); + client.authenticate(IRODSConnectionUtils.newAuthPlugin(this), password); + log.debug("authenticated with iRODS server successfully."); } - catch (Exception e) { + catch(Exception e) { throw new LoginFailureException(MessageFormat.format(LocaleFactory.localizedString( - "Login {0} with username and password", "Credentials"), - BookmarkNameProvider.toString(host)), e); + "Login {0} with username and password", "Credentials"), + BookmarkNameProvider.toString(host)), e); } } @Override - protected void logout() throws BackgroundException { - try { - if (client != null) { + protected void logout() { + // iRODS does not provide a logout operation. + // It only supports connecting, authenticating, and disconnecting. + } + + @Override + protected void disconnect() { + try { + if(null != client) { client.disconnect(); client = null; + log.debug("disconnected from iRODS server."); } - } catch (Exception e) { - throw new BackgroundException("Failed to disconnect from iRODS", e); } - finally { - super.disconnect(); + catch(Exception e) { + log.error(e.getMessage()); } + + super.disconnect(); } @Override @@ -172,8 +182,8 @@ public T _getFeature(final Class type) { if(type == Move.class) { return (T) new IRODSMoveFeature(this); } - if(type == Write.class) { - return (T) new IRODSWriteFeature(this); + if(type == Upload.class) { + return (T) new IRODSUploadFeature(this); } if(type == Touch.class) { return (T) new IRODSTouchFeature(this); @@ -187,8 +197,13 @@ public T _getFeature(final Class type) { if(type == AttributesFinder.class) { return (T) new IRODSAttributesFinderFeature(this); } + if(type == Write.class) { + return (T) new IRODSWriteFeature(this); + } + if(type == Timestamp.class) { + return (T) new IRODSTimestamp(this); + } return super._getFeature(type); } - } diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSStreamUtils.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSStreamUtils.java new file mode 100644 index 00000000000..b95fcbb05fd --- /dev/null +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSStreamUtils.java @@ -0,0 +1,74 @@ +package ch.cyberduck.core.irods; + +/* + * Copyright (c) 2002-2025 iterate GmbH. All rights reserved. + * https://cyberduck.io/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.irods.irods4j.high_level.io.IRODSDataObjectInputStream; +import org.irods.irods4j.high_level.io.IRODSDataObjectOutputStream; +import org.irods.irods4j.high_level.io.IRODSDataObjectStream; +import org.irods.irods4j.low_level.api.IRODSException; + +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.channels.FileChannel; + +class IRODSStreamUtils { + + private static final Logger log = LogManager.getLogger(IRODSStreamUtils.class); + + static void seek(InputStream in, long offset) throws IRODSException, IOException { + if(in instanceof IRODSDataObjectInputStream) { + IRODSDataObjectInputStream stream = (IRODSDataObjectInputStream) in; + long totalOffset = offset; + log.debug("input stream: total offset = [{}]", totalOffset); + while(totalOffset > 0) { + long intermediateOffset = Math.min(totalOffset, Integer.MAX_VALUE); + totalOffset -= intermediateOffset; + log.debug("input stream: offsetting by [{}]. remaining offset = [{}]", intermediateOffset, totalOffset); + stream.seek((int) intermediateOffset, IRODSDataObjectStream.SeekDirection.CURRENT); + } + } + else if(in instanceof FileInputStream) { + log.debug("input stream: seeking to position [{}]", offset); + FileChannel fc = ((FileInputStream) in).getChannel().position(offset); + log.debug("input stream: position = [{}]", fc.position()); + } + } + + static void seek(OutputStream out, long offset) throws IRODSException, IOException { + if(out instanceof IRODSDataObjectOutputStream) { + IRODSDataObjectOutputStream stream = (IRODSDataObjectOutputStream) out; + long totalOffset = offset; + log.debug("output stream: total offset = [{}]", totalOffset); + while(totalOffset > 0) { + long intermediateOffset = Math.min(totalOffset, Integer.MAX_VALUE); + totalOffset -= intermediateOffset; + log.debug("output stream: offsetting by [{}]. remaining offset = [{}]", intermediateOffset, totalOffset); + stream.seek((int) intermediateOffset, IRODSDataObjectStream.SeekDirection.CURRENT); + } + } + else if(out instanceof FileOutputStream) { + log.debug("output stream: seeking to position [{}]", offset); + FileChannel fc = ((FileOutputStream) out).getChannel().position(offset); + log.debug("output stream: position = [{}]", fc.position()); + } + } + +} diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSTimestamp.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSTimestamp.java new file mode 100644 index 00000000000..4e7fa167ffe --- /dev/null +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSTimestamp.java @@ -0,0 +1,96 @@ +package ch.cyberduck.core.irods; + +/* + * Copyright (c) 2002-2025 iterate GmbH. All rights reserved. + * https://cyberduck.io/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +import ch.cyberduck.core.Path; +import ch.cyberduck.core.exception.BackgroundException; +import ch.cyberduck.core.features.Timestamp; +import ch.cyberduck.core.transfer.TransferStatus; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.irods.irods4j.high_level.catalog.IRODSQuery; +import org.irods.irods4j.high_level.connection.IRODSConnection; +import org.irods.irods4j.high_level.vfs.IRODSFilesystem; +import org.irods.irods4j.high_level.vfs.IRODSReplicas; +import org.irods.irods4j.high_level.vfs.LogicalPath; +import org.irods.irods4j.high_level.vfs.ObjectStatus; +import org.irods.irods4j.low_level.api.IRODSException; + +import java.io.IOException; +import java.util.List; +import java.util.concurrent.TimeUnit; + +public class IRODSTimestamp implements Timestamp { + + private static final Logger log = LogManager.getLogger(IRODSTimestamp.class); + + private IRODSSession session; + + public IRODSTimestamp(IRODSSession session) { + this.session = session; + } + + @Override + public void setTimestamp(final Path file, final TransferStatus status) throws BackgroundException { + if(status.getModified() == null) { + return; + } + + final String logicalPath = file.getAbsolute(); + final long seconds = TimeUnit.MILLISECONDS.toSeconds(status.getModified()); + log.debug("setting timestamp for [{}] to [{}] seconds (since epoch).", logicalPath, seconds); + + try { + ObjectStatus objectStatus = IRODSFilesystem.status(session.getClient().getRcComm(), logicalPath); + boolean updated = true; + + if(IRODSFilesystem.isDataObject(objectStatus)) { + long replicaNumber = getReplicaNumberOfLatestGoodReplica(logicalPath); + IRODSReplicas.lastWriteTime(session.getClient().getRcComm(), logicalPath, replicaNumber, seconds); + } + else if(IRODSFilesystem.isCollection(objectStatus)) { + IRODSFilesystem.lastWriteTime(session.getClient().getRcComm(), logicalPath, seconds); + } + else { + updated = false; + log.debug("path does not point to a data object or collection. cannot update timestamp."); + } + + if(updated) { + log.debug("timestamp set to [{}] seconds (since epoch) on [{}] successfully.", seconds, logicalPath); + } + } + catch(IOException | IRODSException e) { + throw new IRODSExceptionMappingService().map(e); + } + } + + private long getReplicaNumberOfLatestGoodReplica(String logicalPath) throws IRODSException, IOException { + final IRODSConnection conn = session.getClient(); + + log.debug("getting replica number of latest (good) replica."); + String query = String.format( + "select DATA_REPL_NUM, DATA_REPL_STATUS, DATA_MODIFY_TIME where COLL_NAME = '%s' and DATA_NAME = '%s' order by DATA_REPL_STATUS desc, DATA_MODIFY_TIME desc", + LogicalPath.parentPath(logicalPath), + LogicalPath.objectName(logicalPath)); + log.debug("query = [{}]", query); + List> rows = IRODSQuery.executeGenQuery2(conn.getRcComm(), query); + + return Long.parseLong(rows.get(0).get(0)); + } + +} diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSTouchFeature.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSTouchFeature.java index e7b5a7de3d3..df8782d9d55 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSTouchFeature.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSTouchFeature.java @@ -1,14 +1,8 @@ package ch.cyberduck.core.irods; -import java.io.IOException; - -import org.irods.irods4j.high_level.connection.IRODSConnection; -import org.irods.irods4j.high_level.io.IRODSDataObjectOutputStream; -import org.irods.irods4j.low_level.api.IRODSException; - /* - * Copyright (c) 2002-2015 David Kocher. All rights reserved. - * http://cyberduck.ch/ + * Copyright (c) 2002-2025 iterate GmbH. All rights reserved. + * https://cyberduck.io/ * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -19,8 +13,6 @@ * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. - * - * Bug fixes, suggestions and comments should be sent to feedback@cyberduck.ch */ import ch.cyberduck.core.Path; @@ -29,9 +21,20 @@ import ch.cyberduck.core.features.Write; import ch.cyberduck.core.transfer.TransferStatus; +import com.fasterxml.jackson.databind.ObjectMapper; + +import org.irods.irods4j.high_level.connection.IRODSConnection; +import org.irods.irods4j.low_level.api.IRODSApi; +import org.irods.irods4j.low_level.api.IRODSException; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; public class IRODSTouchFeature implements Touch { + private static final ObjectMapper mapper = new ObjectMapper(); + private final IRODSSession session; public IRODSTouchFeature(final IRODSSession session) { @@ -41,18 +44,27 @@ public IRODSTouchFeature(final IRODSSession session) { @Override public Path touch(final Write writer, final Path file, final TransferStatus status) throws BackgroundException { try { + final IRODSConnection conn = session.getClient(); + + Map input = new HashMap<>(); + input.put("logical_path", file.getAbsolute()); - // Open and immediately close the file to create/truncate it - final IRODSConnection conn=session.getClient(); - try (IRODSDataObjectOutputStream out = new IRODSDataObjectOutputStream(conn.getRcComm(), file.getAbsolute(), - true /* truncate if exists */, false /* don't append */)) { - // File is created or truncated by opening the stream + String jsonInput = mapper.writeValueAsString(input); + + int ec = IRODSApi.rcTouch(conn.getRcComm(), jsonInput); + if(ec < 0) { + throw new IRODSException(ec, "rcTouch error"); } return file; } - catch(IOException|IRODSException e) { + catch(IOException | IRODSException e) { throw new IRODSExceptionMappingService().map("Cannot create {0}", e, file); } } + + @Override + public boolean isSupported(final Path workdir, final String filename) { + return true; + } } diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSUploadFeature.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSUploadFeature.java index 1fcd23f48ee..f0bb635db0d 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSUploadFeature.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSUploadFeature.java @@ -1,24 +1,8 @@ package ch.cyberduck.core.irods; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.irods.irods4j.high_level.connection.IRODSConnectionPool; -import org.irods.irods4j.high_level.connection.IRODSConnectionPool.PoolConnection; -import org.irods.irods4j.high_level.connection.QualifiedUsername; -import org.irods.irods4j.high_level.io.IRODSDataObjectOutputStream; -import org.irods.irods4j.high_level.vfs.IRODSFilesystem; -import org.irods.irods4j.low_level.api.IRODSApi; -import org.irods.irods4j.low_level.api.IRODSApi.RcComm; - /* - * Copyright (c) 2002-2015 David Kocher. All rights reserved. - * http://cyberduck.ch/ + * Copyright (c) 2002-2025 iterate GmbH. All rights reserved. + * https://cyberduck.io/ * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -29,8 +13,6 @@ * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. - * - * Bug fixes, suggestions and comments should be sent to feedback@cyberduck.ch */ import ch.cyberduck.core.ConnectionCallback; @@ -38,86 +20,235 @@ import ch.cyberduck.core.Path; import ch.cyberduck.core.ProgressListener; import ch.cyberduck.core.exception.BackgroundException; +import ch.cyberduck.core.exception.ConnectionCanceledException; import ch.cyberduck.core.features.Upload; import ch.cyberduck.core.features.Write; import ch.cyberduck.core.io.BandwidthThrottle; -import ch.cyberduck.core.io.Checksum; import ch.cyberduck.core.io.StreamListener; +import ch.cyberduck.core.preferences.HostPreferencesFactory; +import ch.cyberduck.core.preferences.PreferencesReader; import ch.cyberduck.core.transfer.TransferStatus; -public class IRODSUploadFeature implements Upload { +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.irods.irods4j.high_level.connection.IRODSConnection; +import org.irods.irods4j.high_level.connection.IRODSConnectionPool; +import org.irods.irods4j.high_level.connection.IRODSConnectionPool.PoolConnection; +import org.irods.irods4j.high_level.io.IRODSDataObjectOutputStream; +import org.irods.irods4j.high_level.io.IRODSDataObjectStream; + +import java.io.FileInputStream; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +public class IRODSUploadFeature implements Upload { + private static final Logger log = LogManager.getLogger(IRODSUploadFeature.class); private final IRODSSession session; - private boolean truncate = true; - private boolean append = false; - private int numThread=3; - private static final int BUFFER_SIZE = 4 * 1024 * 1024; public IRODSUploadFeature(final IRODSSession session) { this.session = session; } @Override - public Checksum upload(final Path file, final Local local, final BandwidthThrottle throttle, - final ProgressListener progress, final StreamListener streamListener, final TransferStatus status, - final ConnectionCallback callback) throws BackgroundException { - try { - final RcComm primaryConn = session.getClient().getRcComm(); + public Void upload(final Write write, final Path file, final Local local, final BandwidthThrottle throttle, + final ProgressListener progress, final StreamListener streamListener, final TransferStatus status, + final ConnectionCallback callback) throws BackgroundException { + try { + final PreferencesReader preferences = HostPreferencesFactory.get(session.getHost()); + final long fileSize = local.attributes().getSize(); + final String logicalPath = file.getAbsolute(); + final String dstRootResource = getResource(preferences).orElse(StringUtils.EMPTY); + + log.debug("status.getLength() = [{}]", status.getLength()); + log.debug("fileSize = [{}]", fileSize); + log.debug("local file = [{}]", local.getAbsolute()); + log.debug("logicalPath = [{}]", logicalPath); + log.debug("dst root resource = [{}]", dstRootResource); + + // Signals whether the completion flag should be set following the transfer. + // An upload is considered to be successful if and only if no errors occurred. + // That includes waiting for background threads to terminate and closing output + // streams. This is important for certain features (e.g. mtime preservation). + boolean setCompletionFlag = false; + + final long threshold = IRODSIntegerUtils.clamp( + preferences.getInteger(IRODSProtocol.PARALLEL_TRANSFER_THRESHOLD), 1, Integer.MAX_VALUE); + final int bufferSize = IRODSIntegerUtils.clamp( + preferences.getInteger(IRODSProtocol.PARALLEL_TRANSFER_BUFFER_SIZE), 1, (int) (128 * TransferStatus.MEGA)); + + // Transfer the bytes over multiple connections if the size of the local file + // exceeds a certain number of bytes - e.g. 32MB. + if(fileSize < threshold) { + log.debug("local file does not exceed parallel transfer threshold [{}]. performing single-threaded transfer.", threshold); + + byte[] buffer = new byte[bufferSize]; + boolean truncate = true; + boolean append = false; + + try(FileInputStream in = new FileInputStream(local.getAbsolute()); + IRODSConnection conn = IRODSConnectionUtils.newAuthenticatedConnection(session)) { + + IRODSDataObjectOutputStream out; + if(StringUtils.isNotBlank(dstRootResource)) { + out = new IRODSDataObjectOutputStream(conn.getRcComm(), logicalPath, dstRootResource, truncate, append); + } + else { + out = new IRODSDataObjectOutputStream(conn.getRcComm(), logicalPath, truncate, append); + } - // Step 1: Open one primary output stream to get token and replica info - IRODSDataObjectOutputStream primaryOut = new IRODSDataObjectOutputStream(); - primaryOut.open(primaryConn, file.getAbsolute(), truncate, append); - final String replicaToken = primaryOut.getReplicaToken(); - final long replicaNumber = primaryOut.getReplicaNumber(); - primaryOut.close(); - - // Step 2: Connection pool - final IRODSConnectionPool pool = new IRODSConnectionPool(numThread); - pool.start( - session.getHost().getHostname(), - session.getHost().getPort(), - new QualifiedUsername(session.getHost().getCredentials().getUsername(), session.getRegion()), - conn -> { try { - IRODSApi.rcAuthenticateClient(conn, "native", session.getHost().getCredentials().getPassword()); - return true; - } catch (Exception e) { - return false; + while(true) { + try { + status.validate(); // Throws if transfer is cancelled. + } + catch(ConnectionCanceledException e) { + log.info("transfer cancelled."); + return null; + } + + int bytesRead = in.read(buffer); + if(bytesRead == -1) { + setCompletionFlag = true; + return null; + } + streamListener.recv(bytesRead); + out.write(buffer, 0, bytesRead); + streamListener.sent(bytesRead); + } + } + finally { + out.close(); + + if(setCompletionFlag) { + status.setComplete(); + } } - }); - - // Step 3: Thread pool & chunking - final ExecutorService executor = Executors.newFixedThreadPool(numThread); - final long chunkSize = fileSize / numThread; - final long remainChunkSize = fileSize % numThread; - - List> tasks = new ArrayList<>(); - for (int i = 0; i < numThread; i++) { - final long start = i * chunkSize; - final PoolConnection conn = pool.getConnection(); - IRODSDataObjectOutputStream stream = new IRODSDataObjectOutputStream(); - stream.open(conn.getRcComm(), file.getAbsolute(),replicaToken,replicaNumber, truncate, append); - ChunkWorker worker = new ChunkWorker( - stream, - local.getAbsolute(), - start, - (i == numThread - 1) ? chunkSize + remainChunkSize : chunkSize, - BUFFER_SIZE - ); - tasks.add(executor.submit(worker)); + } } - for (Future task : tasks) { - task.get(); + // + // The data object is larger than the threshold so use parallel transfer. + // + + log.debug("local file exceeds parallel transfer threshold [{}]. performing multi-threaded transfer.", threshold); + + final int threadCount = IRODSIntegerUtils.clamp( + preferences.getInteger(IRODSProtocol.PARALLEL_TRANSFER_CONNECTIONS), 2, 10); + log.debug("thread count = [{}]; starting thread pool.", threadCount); + final ExecutorService executor = Executors.newFixedThreadPool(threadCount); + + final long chunkSize = fileSize / threadCount; + final long remainingBytes = fileSize % threadCount; + log.debug("chunk size = [{}]", chunkSize); + log.debug("remaining bytes = [{}]", remainingBytes); + + final List localFileStreams = new ArrayList<>(); + final List irodsStreams = new ArrayList<>(); + + log.debug("launching connection pool with [{}] connections.", threadCount); + try(IRODSConnectionPool pool = new IRODSConnectionPool(IRODSConnectionUtils.initConnectionOptions(session), threadCount)) { + try { + status.validate(); // Throws if transfer is cancelled. + } + catch(ConnectionCanceledException e) { + log.info("transfer cancelled."); + return null; + } + + IRODSConnectionUtils.startIRODSConnectionPool(session, pool); + log.debug("connection pool started."); + + try { + String replicaToken = null; + long replicaNumber = -1; + + for(int i = 0; i < threadCount; ++i) { + // We cannot use Files.newInputStream() does not report the concrete + // type of the stream. The concrete type is needed for seek operations. + localFileStreams.add(new FileInputStream(local.getAbsolute())); + + // The pooled connection will never be returned to the pool. This is + // okay because after the transfer, no connection is reused. + PoolConnection conn = pool.getConnection(); + + if(0 == i) { + log.debug("opened primary iRODS stream."); + // The first iRODS output stream is the primary stream. The opened + // replica is always truncated upon success. + if(StringUtils.isNotBlank(dstRootResource)) { + irodsStreams.add(new IRODSDataObjectOutputStream( + conn.getRcComm(), logicalPath, dstRootResource, true, false)); + } + else { + irodsStreams.add(new IRODSDataObjectOutputStream( + conn.getRcComm(), logicalPath, true, false)); + } + replicaToken = irodsStreams.get(0).getReplicaToken(); + replicaNumber = irodsStreams.get(0).getReplicaNumber(); + log.debug("replica token = [{}]", replicaToken); + log.debug("replica number = [{}]", replicaNumber); + } + else { + log.debug("opened secondary iRODS stream."); + irodsStreams.add(new IRODSDataObjectOutputStream( + conn.getRcComm(), replicaToken, logicalPath, replicaNumber, false, false)); + } + } + + try { + status.validate(); // Throws if transfer is cancelled. + } + catch(ConnectionCanceledException e) { + log.info("transfer cancelled."); + return null; + } + + // Holds handles to tasks running on the thread pool. This allows us to wait for + // all tasks to complete before shutting down everything. + List> tasks = new ArrayList<>(); + + // Launch remaining IO tasks. + log.debug("launch parallel IO tasks."); + for(int i = 0; i < threadCount; ++i) { + tasks.add(executor.submit(new IRODSChunkWorker( + status, + streamListener, + localFileStreams.get(i), + irodsStreams.get(i), + i * chunkSize, + (threadCount - 1 == i) ? chunkSize + remainingBytes : chunkSize, + bufferSize + ))); + } + + setCompletionFlag = waitForTasksToComplete(tasks); + } + finally { + final boolean closedOutputStreams = closeOutputStreams(irodsStreams); + if(setCompletionFlag && closedOutputStreams) { + status.setComplete(); + } + + closeInputStreams(localFileStreams); + } } + log.debug("shutting down thread pool executor."); executor.shutdown(); - pool.close(); + executor.awaitTermination(5, TimeUnit.SECONDS); + log.debug("done."); - final String fingerprintValue = IRODSFilesystem.dataObjectChecksum(primaryConn, file.getAbsolute()); - return Checksum.parse(fingerprintValue); + return null; } catch(Exception e) { throw new IRODSExceptionMappingService().map(e); @@ -125,7 +256,87 @@ public Checksum upload(final Path file, final Local local, final BandwidthThrott } @Override - public Write.Append append(final Path file, final TransferStatus status) throws BackgroundException { + public Write.Append append(Path file, TransferStatus status) throws BackgroundException { return new Write.Append(status.isExists()).withStatus(status); } + + private static boolean closeOutputStreams(List streams) { + log.debug("closing output streams."); + + final IRODSDataObjectStream.OnCloseSuccess closeInstructions = new IRODSDataObjectStream.OnCloseSuccess(); + closeInstructions.updateSize = false; + closeInstructions.updateStatus = false; + closeInstructions.computeChecksum = false; + closeInstructions.sendNotifications = false; + + boolean success = true; + + for(int i = 1; i < streams.size(); ++i) { + try { + streams.get(i).close(closeInstructions); + } + catch(Exception e) { + log.error(e.getMessage()); + success = false; + } + } + + try { + streams.get(0).close(); + } + catch(Exception e) { + log.error(e.getMessage()); + success = false; + } + + return success; + } + + private static void closeInputStreams(List streams) { + log.debug("closing input streams."); + + streams.forEach(out -> { + try { + out.close(); + } + catch(Exception e) { + log.error(e.getMessage()); + } + }); + } + + private static boolean waitForTasksToComplete(List> tasks) { + log.debug("waiting for parallel IO tasks to finish."); + boolean success = true; + + for(Future task : tasks) { + try { + if(!task.get()) { + success = false; + } + } + catch(Exception e) { + success = false; + log.error(e.getMessage()); + } + } + + log.debug("parallel IO tasks have finished."); + return success; + } + + private Optional getResource(PreferencesReader prefs) { + final String resc = prefs.getProperty(IRODSProtocol.DESTINATION_RESOURCE); + if(StringUtils.isNotBlank(resc)) { + return Optional.of(resc); + } + + final String region = session.getHost().getRegion(); + int index = region.indexOf(':'); + if(index != -1 && ++index < region.length()) { + return Optional.of(region.substring(index)); + } + + return Optional.empty(); + } } diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSWriteFeature.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSWriteFeature.java index 32ad40edbdd..77d6efe4667 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSWriteFeature.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSWriteFeature.java @@ -1,22 +1,8 @@ package ch.cyberduck.core.irods; -import java.io.IOException; -import java.io.OutputStream; -import java.util.ArrayList; -import java.util.List; - -import org.apache.commons.io.FilenameUtils; -import org.irods.irods4j.common.Versioning; -import org.irods.irods4j.high_level.catalog.IRODSQuery; -import org.irods.irods4j.high_level.catalog.IRODSQuery.GenQuery1QueryArgs; -import org.irods.irods4j.high_level.connection.IRODSConnection; -import org.irods.irods4j.high_level.io.IRODSDataObjectOutputStream; -import org.irods.irods4j.low_level.api.GenQuery1Columns; -import org.irods.irods4j.low_level.api.IRODSException; - /* - * Copyright (c) 2002-2015 David Kocher. All rights reserved. - * http://cyberduck.ch/ + * Copyright (c) 2002-2025 iterate GmbH. All rights reserved. + * https://cyberduck.io/ * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -27,8 +13,6 @@ * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. - * - * Bug fixes, suggestions and comments should be sent to feedback@cyberduck.ch */ import ch.cyberduck.core.ConnectionCallback; @@ -39,7 +23,18 @@ import ch.cyberduck.core.io.VoidStatusOutputStream; import ch.cyberduck.core.transfer.TransferStatus; -public class IRODSWriteFeature implements Write> { +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.irods.irods4j.high_level.connection.IRODSConnection; +import org.irods.irods4j.high_level.io.IRODSDataObjectOutputStream; +import org.irods.irods4j.low_level.api.IRODSException; + +import java.io.IOException; +import java.io.OutputStream; + +public class IRODSWriteFeature implements Write { + + private static final Logger log = LogManager.getLogger(IRODSWriteFeature.class); private final IRODSSession session; @@ -48,68 +43,13 @@ public IRODSWriteFeature(IRODSSession session) { } @Override - public StatusOutputStream> write(final Path file, final TransferStatus status, final ConnectionCallback callback) throws BackgroundException { - try { - // Step 1: Get the active iRODS client connection and parameters - final IRODSConnection conn = session.getClient(); - boolean append = status.isAppend(); - boolean truncate = !append; - final OutputStream out = new IRODSDataObjectOutputStream(conn.getRcComm(), file.getAbsolute(), truncate, append); - // Step 2: Return a wrapped StatusOutputStream that provides file metadata on completion - return new StatusOutputStream>(out) { - @Override - public List getStatus() throws BackgroundException { - // Step 3: Extract parent directory and filename from the logical path - String logicalPath = file.getAbsolute(); - String parentPath = FilenameUtils.getFullPathNoEndSeparator(logicalPath); - String fileName = FilenameUtils.getName(logicalPath); - final List status = new ArrayList<>();; - // Step 4: Check iRODS version to decide which query mechanism to use - if(Versioning.compareVersions(conn.getRcComm().relVersion.substring(4), "4.3.4") > 0) { - String query = String.format("select DATA_MODIFY_TIME, DATA_CREATE_TIME, DATA_SIZE, DATA_CHECKSUM, DATA_OWNER_NAME, DATA_OWNER_ZONE where COLL_NAME = '%s' and DATA_NAME = '%s'", parentPath, fileName); - List> rows; - try { - // Step 5: Execute the query and add the first result row to the status list - rows = IRODSQuery.executeGenQuery2(conn.getRcComm(), query); - status.addAll(rows.get(0)); - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } catch (IRODSException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - }else { - //if older version, use Query1 - GenQuery1QueryArgs input = new GenQuery1QueryArgs(); - - // select COLL_NAME, DATA_NAME, DATA_ACCESS_TIME - input.addColumnToSelectClause(GenQuery1Columns.COL_D_MODIFY_TIME); - input.addColumnToSelectClause(GenQuery1Columns.COL_D_CREATE_TIME); - input.addColumnToSelectClause(GenQuery1Columns.COL_DATA_SIZE); - input.addColumnToSelectClause(GenQuery1Columns.COL_D_DATA_CHECKSUM); - input.addColumnToSelectClause(GenQuery1Columns.COL_D_OWNER_NAME); - input.addColumnToSelectClause(GenQuery1Columns.COL_D_OWNER_ZONE); - - - // where COLL_NAME like '/tempZone/home/rods and DATA_NAME = 'atime.txt' - String collNameCondStr = String.format("= '%s'", parentPath); - String dataNameCondStr = String.format("= '%s'", fileName); - input.addConditionToWhereClause(GenQuery1Columns.COL_COLL_NAME, collNameCondStr); - input.addConditionToWhereClause(GenQuery1Columns.COL_DATA_NAME, dataNameCondStr); - - try { - IRODSQuery.executeGenQuery1(conn.getRcComm(), input, row -> { - status.addAll(row); - return false; - }); - } catch (IOException | IRODSException e) { - e.printStackTrace(); - } - } - return status; - } - }; + public StatusOutputStream write(final Path file, final TransferStatus status, final ConnectionCallback callback) throws BackgroundException { + try { + final IRODSConnection conn = session.getClient(); + boolean append = status.isAppend(); + boolean truncate = !append; + final OutputStream out = new IRODSDataObjectOutputStream(conn.getRcComm(), file.getAbsolute(), truncate, append); + return new VoidStatusOutputStream(out); } catch(IRODSException | IOException e) { throw new IRODSExceptionMappingService().map("Uploading {0} failed", e, file); diff --git a/irods/src/test/java/ch/cyberduck/core/irods/IRODSExceptionMappingServiceTest.java b/irods/src/test/java/ch/cyberduck/core/irods/IRODSExceptionMappingServiceTest.java index 03e8654f70d..61951917a12 100644 --- a/irods/src/test/java/ch/cyberduck/core/irods/IRODSExceptionMappingServiceTest.java +++ b/irods/src/test/java/ch/cyberduck/core/irods/IRODSExceptionMappingServiceTest.java @@ -18,12 +18,12 @@ */ import ch.cyberduck.core.exception.AccessDeniedException; + import ch.cyberduck.core.exception.LoginFailureException; import ch.cyberduck.core.exception.NotfoundException; -//import org.irods.jargon.core.exception.AuthenticationException; -//import org.irods.jargon.core.exception.CatNoAccessException; -//import org.irods.jargon.core.exception.FileNotFoundException; +import org.irods.irods4j.low_level.api.IRODSErrorCodes; +import org.irods.irods4j.low_level.api.IRODSException; import org.junit.Test; import static org.junit.Assert.assertTrue; @@ -32,8 +32,9 @@ public class IRODSExceptionMappingServiceTest { @Test public void testMap() { -// assertTrue(new IRODSExceptionMappingService().map(new CatNoAccessException("no access")) instanceof AccessDeniedException); -// assertTrue(new IRODSExceptionMappingService().map(new FileNotFoundException("no file")) instanceof NotfoundException); -// assertTrue(new IRODSExceptionMappingService().map(new AuthenticationException("no user")) instanceof LoginFailureException); + final IRODSExceptionMappingService mappingService = new IRODSExceptionMappingService(); + assertTrue(mappingService.map(new IRODSException(IRODSErrorCodes.CAT_NO_ACCESS_PERMISSION, "no access")) instanceof AccessDeniedException); + assertTrue(mappingService.map(new IRODSException(IRODSErrorCodes.CAT_NO_ROWS_FOUND, "no file")) instanceof NotfoundException); + assertTrue(mappingService.map(new IRODSException(IRODSErrorCodes.AUTHENTICATION_ERROR, "no user")) instanceof LoginFailureException); } } diff --git a/irods/src/test/java/ch/cyberduck/core/irods/IRODSListServiceTest.java b/irods/src/test/java/ch/cyberduck/core/irods/IRODSListServiceTest.java index 6b9b4cd806c..0a08507f1db 100644 --- a/irods/src/test/java/ch/cyberduck/core/irods/IRODSListServiceTest.java +++ b/irods/src/test/java/ch/cyberduck/core/irods/IRODSListServiceTest.java @@ -29,7 +29,6 @@ import ch.cyberduck.core.ProtocolFactory; import ch.cyberduck.core.exception.NotfoundException; import ch.cyberduck.core.proxy.DisabledProxyFinder; -import ch.cyberduck.core.proxy.Proxy; import ch.cyberduck.core.serializer.impl.dd.ProfilePlistReader; import ch.cyberduck.test.IntegrationTest; import ch.cyberduck.test.VaultTest; diff --git a/irods/src/test/java/ch/cyberduck/core/irods/IRODSReadFeatureTest.java b/irods/src/test/java/ch/cyberduck/core/irods/IRODSReadFeatureTest.java index 92c0d7b63e6..33ebbc4c261 100644 --- a/irods/src/test/java/ch/cyberduck/core/irods/IRODSReadFeatureTest.java +++ b/irods/src/test/java/ch/cyberduck/core/irods/IRODSReadFeatureTest.java @@ -43,7 +43,6 @@ import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.RandomUtils; -//import org.irods.jargon.core.pub.domain.ObjStat; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -138,8 +137,8 @@ public void testReadRange() throws Exception { assertNotNull(out); IOUtils.write(content, out); out.close(); - new DefaultUploadFeature>(new IRODSWriteFeature(session)).upload( - test, local, new BandwidthThrottle(BandwidthThrottle.UNLIMITED), new DisabledProgressListener(), new DisabledStreamListener(), + new DefaultUploadFeature(session).upload( + new IRODSWriteFeature(session), test, local, new BandwidthThrottle(BandwidthThrottle.UNLIMITED), new DisabledProgressListener(), new DisabledStreamListener(), new TransferStatus().setLength(content.length), new DisabledConnectionCallback()); final TransferStatus status = new TransferStatus(); diff --git a/irods/src/test/java/ch/cyberduck/core/irods/IRODSSessionTest.java b/irods/src/test/java/ch/cyberduck/core/irods/IRODSSessionTest.java index 160b00ee40f..c6ecb71a635 100644 --- a/irods/src/test/java/ch/cyberduck/core/irods/IRODSSessionTest.java +++ b/irods/src/test/java/ch/cyberduck/core/irods/IRODSSessionTest.java @@ -29,7 +29,6 @@ import ch.cyberduck.core.ProtocolFactory; import ch.cyberduck.core.exception.LoginFailureException; import ch.cyberduck.core.proxy.DisabledProxyFinder; -import ch.cyberduck.core.proxy.Proxy; import ch.cyberduck.core.serializer.impl.dd.ProfilePlistReader; import ch.cyberduck.test.IntegrationTest; import ch.cyberduck.test.VaultTest; diff --git a/irods/src/test/java/ch/cyberduck/core/irods/IRODSWriteFeatureTest.java b/irods/src/test/java/ch/cyberduck/core/irods/IRODSWriteFeatureTest.java index 6dee5762300..7dd00a92d1c 100644 --- a/irods/src/test/java/ch/cyberduck/core/irods/IRODSWriteFeatureTest.java +++ b/irods/src/test/java/ch/cyberduck/core/irods/IRODSWriteFeatureTest.java @@ -41,7 +41,6 @@ import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.RandomUtils; -//import org.irods.jargon.core.pub.domain.ObjStat; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -234,7 +233,7 @@ public void testWrite() throws Exception { assertEquals(0L, new IRODSUploadFeature(session).append(test, status).offset, 0L); - final StatusOutputStream> out = feature.write(test, status, new DisabledConnectionCallback()); + final StatusOutputStream out = feature.write(test, status, new DisabledConnectionCallback()); assertNotNull(out); new StreamCopier(new TransferStatus(), new TransferStatus()).transfer(new ByteArrayInputStream(content), out); @@ -261,7 +260,7 @@ public void testWrite() throws Exception { assertTrue(new IRODSUploadFeature(session).append(test, status).append); assertEquals(content.length, new IRODSUploadFeature(session).append(test, status).offset, 0L); - final StatusOutputStream> out = feature.write(test, status, new DisabledConnectionCallback()); + final StatusOutputStream out = feature.write(test, status, new DisabledConnectionCallback()); assertNotNull(out); new StreamCopier(new TransferStatus(), new TransferStatus()).transfer(new ByteArrayInputStream(newcontent), out); diff --git a/irods/src/test/java/ch/cyberduck/core/irods/input.txt b/irods/src/test/java/ch/cyberduck/core/irods/input.txt deleted file mode 100644 index 5adda2b1cb5..00000000000 --- a/irods/src/test/java/ch/cyberduck/core/irods/input.txt +++ /dev/null @@ -1,151 +0,0 @@ -irods.host=172.20.0.2 -irods.port=1247 -irods.zoneName=tempZone - -# STANDARD | PAM AUTH -irods.auth.scheme=STANDARD -default.storage.resource= - -# sets jargon ssl negotiation policy for the client. Leaving to DONT_CARE defers to the server, and is recommended -# NO_NEGOTIATION, CS_NEG_REFUSE, CS_NEG_REQUIRE, CS_NEG_DONT_CARE -ssl.negotiation.policy=CS_NEG_REFUSE - -########################################################## -# jargon properties settings -utilize.packing.streams=true - -# jargon now supports checksum calculation for streaming uploads. This does not currently verify, but does store if set to true -compute.checksum=true - -###################################### -# other irods environment - -# HTTP connection for RMD -rmd.connection.timeout=500 -rmd.connection.port=8000 - -# Reverse DNS lookup on dashboard -reverse.dns.lookup=false - -###################################### -# msi props -populate.msi.enabled=false -illumina.msi.enabled=true - -# MSI API version supported by this application -msi.api.version=1.X.X - -msi.metalnx.list=libmsiget_illumina_meta.so,libmsiobjget_microservices.so,libmsiobjget_version.so,libmsiobjjpeg_extract.so,libmsiobjput_mdbam.so,libmsiobjput_mdbam.so,libmsiobjput_mdmanifest.so,libmsiobjput_mdvcf.so,libmsiobjput_populate.so - -msi.irods.list=libmsisync_to_archive.so,libmsi_update_unixfilesystem_resource_free_space.so,libmsiobjput_http.so,libmsiobjput_irods.so,libmsiobjget_irods.so,libmsiobjget_http.so,libmsiobjput_slink.so,libmsiobjget_slink.so - -msi.irods.42.list=libmsisync_to_archive.so,libmsi_update_unixfilesystem_resource_free_space.so - -msi.other.list= -###################################### -# global feature flags that serve as defaults. Note that the info handling will manipulate aspects of the data profiler, -# so by default some things are set to null to be turned on by the service depending on the view requested (e.g. acl, metadata, replicas) and should be left 'false' as a default, -# but other aspects, such as metadata templating and mime type detection, can be globally turned on or off depending on the implmenetation. -# controls access to features globally -metalnx.enable.tickets=false -# disable automatic detection and running of rules on upload -metalnx.enable.upload.rules=false -# download size limit in megabytes (6000=6GB) -metalnx.download.limit=6000 -# show dashboard (off by default due to performance issues) -metalnx.enable.dashboard=false -###################################### -# info home page feature flags -# this controls the behavior of the data profiler and what information it will gather -irodsext.dataprofiler.retrieve.tickets=false -# process starred or favorites -irodsext.dataprofiler.retrieve.starred=true -# process shared -irodsext.dataprofiler.retrieve.shared=false -# tags and comments -irodsext.dataprofiler.retrieve.tags.and.comments=false -# metadata templates (currently not implemented) -irodsext.dataprofiler.retrieve.metadata.templates=false -# save data type information for later use -irodsext.datatyper.persist.data.types=false -# perform a detailed versus a lightweight data typing, which may involve processing the file contents -irodsext.datatyper.detailed.determination=false - -############################# -# misc ui configuration niceties -############################# -# allow translation of iRODS auth types to user friendly names in login -# in the form irodstype:displaytype| -metalnx.authtype.mappings=PAM:PAM|STANDARD:Standard - -############################# -# JWT configuration (necessary when using search and notification services). Otherwise can be left as-is and ignored -############################# -jwt.issuer=metalnx -jwt.secret=thisisasecretthatisverysecretyouwillneverguessthiskey -jwt.algo=HS384 - -############################# -# Pluggable search configuration. Turn on and off pluggable search globally, and configure search endpoints. -# N.B. pluggable search also requires provisioning of the jwt.* information above -############################# -# configured endpoints, comma delimited in form https://host.com/v1 -# Note the commented out URL which matches up to the irods-contrib/file-and-metadata-indexer docker compose assembly. In order to -# utilize this assembly you need to uncomment the URL and set pluggablesearch.enabled to true -pluggablesearch.endpointRegistryList= -# enable pluggable search globally and show the search GUI components -pluggablesearch.enabled=false -# display the older file and properties search in the menu, if you are running the elasticsearch standard plugin this is probably -# a menu item to turn off -classicsearch.enabled=true -# JWT subject claim used to access search endpoint for data gathering. User searches will utilize the name of the individual -pluggablesearch.endpointAccessSubject=metalnx -# timeout for info/attribute gathering, set to 0 for no timeout -pluggablesearch.info.timeout=0 -# timeout for actual search, set to 0 for no timeout -pluggablesearch.search.timeout=0 - -############################# -# Pluggable shopping cart and export plugin configuration. -# Turn on and off pluggable shopping cart globally, and configure export endpoints. -# N.B. plugins also requires provisioning of the jwt.* information above -############################# - -# enable pluggable export globally and show the export GUI components -pluggableshoppingcart.enabled=false - -# configured endpoints, comma delimited in form https://host.com/v1 -pluggablepublishing.endpointRegistryList= -# timeout for info/attribute gathering, set to 0 for no timeout -pluggablepublishing.info.timeout=0 - -# timeout for actual publishing, set to 0 for no timeout -pluggablepublishing.publishing.timeout=0 - -# server rule engine instance that will provide the galleryview listing -gallery_view.rule_engine_plugin.instance_name=irods_rule_engine_plugin-irods_rule_language-instance - -########################################################## -# Metadata Masking Properties -# -# Excludes metadata when the attribute name starts with at least one prefix. -# Multiple prefixes can be defined by separating them with the character sequence defined -# by metalnx.metadata.mask.delimiter. -# -# For example, the configuration below will hide any metadata that contains an attribute -# starting with "irods::", "metalnx-", or "_system_". -# -# metalnx.metadata.mask.prefixes=irods::;metalnx-;_system_ -# metalnx.metadata.mask.delimiter=; -# -# Use the iRODS Metadata Guard rule engine plugin to protect your metadata namespaces from -# being modified. -metalnx.metadata.mask.prefixes= -metalnx.metadata.mask.delimiter=, - - -########################################################## -# Setting to enable/disable the "Public" sidebar link. -# The default is "false" (hidden) -########################################################## -sidebar.show.public=false diff --git a/irods/src/test/java/ch/cyberduck/core/irods/output.txt b/irods/src/test/java/ch/cyberduck/core/irods/output.txt deleted file mode 100644 index adcc2d8497b..00000000000 --- a/irods/src/test/java/ch/cyberduck/core/irods/output.txt +++ /dev/null @@ -1,151 +0,0 @@ -te that the info handling will manipulate aspects of the data profiler, -# so by default some things are set to null to be turned on by the service depending on the view requested (e.g. acl, metadata, replicas) and should be left 'false' as a default, -# but other aspects, such as metadata templating and mime type detection, can be globally turned on or off depending on the implmenetation. -# controls access to features globally -metalnx.enable.tickets=false -# disable automatic detection and running of rules on upload -metalnx.enable.upload.rules=false -# download size limit in megabytes (6000=6GB) -metalnx.download.limit=6000 -# show dashboard (off by default due to performance issues) -metalnx.enable.dashboard=false -###################################### -# info home page feature flags -# this controls the behavior of the data profiler and what information it will gather -irodsext.dataprofiler.retrieve.tickets=false -# process starred or favorites -irodsext.dataprofiler.retrieve.starred=true -# process shared -irodsext.dataprofiler.retrieve.shared=false -# tags and comments -irodsext.dataprofiler.retrieve.tags.and.comments=false -# metadata templates (currently not implemented) -irodsext.dataprofiler.retrieve.metadata.templates=false -# save data type information for later use -irodsext.datatyper.persist.data.types=false -# perform a detailed versus a lightweight data typing, which may involve processing the file contents -irodsext.datatyper.detailed.determination=false - -############################# -# misc ui configuration niceties -############################# -# allow translation of iRODS auth types to user friendly names in login -# in the form irodstype:displaytype| -metalnx.auirods.host=172.20.0.2 -irods.port=1247 -irods.zoneName=tempZone - -# STANDARD | PAM AUTH -irods.auth.scheme=STANDARD -default.storage.resource= - -# sets jargon ssl negotiation policy for the client. Leaving to DONT_CARE defers to the server, and is recommended -# NO_NEGOTIATION, CS_NEG_REFUSE, CS_NEG_REQUIRE, CS_NEG_DONT_CARE -ssl.negotiation.policy=CS_NEG_REFUSE - -########################################################## -# jargon properties settings -utilize.packing.streams=true - -# jargon now supports checksum calculation for streaming uploads. This does not currently verify, but does store if set to true -compute.checksum=true - -###################################### -# other irods environment - -# HTTP connection for RMD -rmd.connection.timeout=500 -rmd.connection.port=8000 - -# Reverse DNS lookup on dashboard -reverse.dns.lookup=false - -###################################### -# msi props -populate.msi.enabled=false -illumina.msi.enabled=true - -# MSI API version supported by this application -msi.api.version=1.X.X - -msi.metalnx.list=libmsiget_illumina_meta.so,libmsiobjget_microservices.so,libmsiobjget_version.so,libmsiobjjpeg_extract.so,libmsiobjput_mdbam.so,libmsiobjput_mdbam.so,libmsiobjput_mdmanifest.so,libmsiobjput_mdvcf.so,libmsiobjput_populate.so - -msi.irods.list=libmsisync_to_archive.so,libmsi_update_unixfilesystem_resource_free_space.so,libmsiobjput_http.so,libmsiobjput_irods.so,libmsiobjget_irods.so,libmsiobjget_http.so,libmsiobjput_slink.so,libmsiobjget_slink.so - -msi.irods.42.list=libmsisync_to_archive.so,libmsi_update_unixfilesystem_resource_free_space.so - -msi.other.list= -###################################### -# global feature flags that serve as defaults. Nothtype.mappings=PAM:PAM|STANDARD:Standard - -############################# -# JWT configuration (necessary when using search and notification services). Otherwise can be left as-is and ignored -############################# -jwt.issuer=metalnx -jwt.secret=thisisasecretthatisverysecretyouwillneverguessthiskey -jwt.algo=HS384 - -############################# -# Pluggable search configuration. Turn on and off pluggable search globally, and configure search endpoints. -# N.B. pluggable search also requires provisioning of the jwt.* information above -############################# -# configured endpoints, comma delimited in form https://host.com/v1 -# Note the commented out URL which matches up to the irods-contrib/file-and-metadata-indexer docker compose assembly. In order to -# utilize this assembly you need to uncomment the URL and set pluggablesearch.enabled to true -pluggablesearch.endpointRegistryList= -# enable pluggable search globally and show the search GUI components -pluggablesearch.enabled=false -# display the older file and properties search in the menu, if you are running the elasticsearch standard plugin this is probably -# a menu item to turn off -classicsearch.enabled=true -# JWT subject claim used to access search endpoint for data gathering. User searches will utilize the name of the individual -pluggablesearch.endpointAccessSubject=metalnx -# timeout for info/attribute gathering, set to 0 for no timeout -pluggablesearch.info.timeout=0 -# timeout for actual search, set to 0 for no timeout -pluggablesearch.search.timeout=0 - -############################# -# Pluggable shopping cart and export plugin configuration. -# Turn on and off pluggable shopping cart globally, and configure export endpoints. -# N.B. plugins also requires provisioning of the jwt.* information above -############################# - -# enable pluggable export globally and show the export GUI components -pluggableshoppingcart.enabled=false - -# configured endpoints, comma delimited in form https://host.com/v1 -pluggablepublishing.endpointRegistryList= -# timeout for info/attribute gathering, set to 0 for no timeout -pluggablepublishing.info.timeout=0 - -# timeout for actual publishing, set to 0 for no timeout -pluggablepublishing.publishing.timeout=0 - -# server rule engine instance that will provide the galleryview listing -gallery_view.rule_engine_plugin.instance_name=irods_rule_engine_plugin-irods_rule_language-instance - -########################################################## -# Metadata Masking Properties -# -# Excludes metadata when the attribute name starts with at least one prefix. -# Multiple prefixes can be defined by separating them with the character sequence defined -# by metalnx.metadata.mask.delimiter. -# -# For example, the configuration below will hide any metadata that contains an attribute -# starting with "irods::", "metalnx-", or "_system_". -# -# metalnx.metadata.mask.prefixes=irods::;metalnx-;_system_ -# metalnx.metadata.mask.delimiter=; -# -# Use the iRODS Metadata Guard rule engine plugin to protect your metadata namespaces from -# being modified. -metalnx.metadata.mask.prefixes= -metalnx.metadata.mask.delimiter=, - - -########################################################## -# Setting to enable/disable the "Public" sidebar link. -# The default is "false" (hidden) -########################################################## -sidebar.show.public=false diff --git a/irods/src/test/resources/iRODS (iPlant Collaborative).cyberduckprofile b/irods/src/test/resources/iRODS (iPlant Collaborative).cyberduckprofile index 7d5484f947f..f51cceb78a1 100644 --- a/irods/src/test/resources/iRODS (iPlant Collaborative).cyberduckprofile +++ b/irods/src/test/resources/iRODS (iPlant Collaborative).cyberduckprofile @@ -8,21 +8,26 @@ iplant Description iPlant Data Store + Hostname Configurable Port Configurable + Default Hostname data.iplantcollaborative.org - Region - iplant Default Port 1247 + Username Placeholder iPlant username Password Placeholder iPlant password + + Region + iplant + Authorization - STANDARD + native From 99b1ac6b0effa871f6b47b30c52518d69bc63678 Mon Sep 17 00:00:00 2001 From: Kory Draughn Date: Wed, 15 Oct 2025 15:15:10 -0400 Subject: [PATCH 03/20] iRODS: Add Docker Compose project for testing --- .../test/resources/docker/docker-compose.yml | 27 +++ .../resources/docker/irods_catalog/Dockerfile | 3 + .../docker/irods_catalog/init-user-db.sh | 12 ++ .../docker/irods_catalog_provider/Dockerfile | 61 +++++++ .../irods_catalog_provider/entrypoint.sh | 24 +++ .../unattended_install.json | 169 ++++++++++++++++++ 6 files changed, 296 insertions(+) create mode 100644 irods/src/test/resources/docker/docker-compose.yml create mode 100644 irods/src/test/resources/docker/irods_catalog/Dockerfile create mode 100644 irods/src/test/resources/docker/irods_catalog/init-user-db.sh create mode 100644 irods/src/test/resources/docker/irods_catalog_provider/Dockerfile create mode 100644 irods/src/test/resources/docker/irods_catalog_provider/entrypoint.sh create mode 100644 irods/src/test/resources/docker/irods_catalog_provider/unattended_install.json diff --git a/irods/src/test/resources/docker/docker-compose.yml b/irods/src/test/resources/docker/docker-compose.yml new file mode 100644 index 00000000000..9397c839929 --- /dev/null +++ b/irods/src/test/resources/docker/docker-compose.yml @@ -0,0 +1,27 @@ +name: cyberduck-irods-testing + +services: + irods-catalog: + build: + context: irods_catalog + environment: + - POSTGRES_PASSWORD=testpassword + restart: always + + irods-catalog-provider: + build: + context: irods_catalog_provider + init: true + ports: + - "1247:1247" + shm_size: 100mb + healthcheck: + test: ["CMD", 'echo -e "\x00\x00\x00\x33HEARTBEAT" | (exec 3<>/dev/tcp/127.0.0.1/1247; cat >&3; cat <&3; exec 3<&-)'] + interval: 10s + timeout: 10s + retries: 3 + start_period: 20s + start_interval: 10s + restart: always + depends_on: + - irods-catalog \ No newline at end of file diff --git a/irods/src/test/resources/docker/irods_catalog/Dockerfile b/irods/src/test/resources/docker/irods_catalog/Dockerfile new file mode 100644 index 00000000000..cfdacba890a --- /dev/null +++ b/irods/src/test/resources/docker/irods_catalog/Dockerfile @@ -0,0 +1,3 @@ +FROM postgres:17 + +COPY init-user-db.sh /docker-entrypoint-initdb.d/init-user-db.sh \ No newline at end of file diff --git a/irods/src/test/resources/docker/irods_catalog/init-user-db.sh b/irods/src/test/resources/docker/irods_catalog/init-user-db.sh new file mode 100644 index 00000000000..692340c5f9a --- /dev/null +++ b/irods/src/test/resources/docker/irods_catalog/init-user-db.sh @@ -0,0 +1,12 @@ +#! /bin/bash + +# Adapted from "Initialization script" in documentation for official Postgres dockerhub: +# https://hub.docker.com/_/postgres/ +set -e + +psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL + CREATE USER irods WITH PASSWORD 'testpassword'; + CREATE DATABASE "ICAT"; + GRANT ALL PRIVILEGES ON DATABASE "ICAT" to irods; + ALTER DATABASE "ICAT" OWNER TO irods; +EOSQL \ No newline at end of file diff --git a/irods/src/test/resources/docker/irods_catalog_provider/Dockerfile b/irods/src/test/resources/docker/irods_catalog_provider/Dockerfile new file mode 100644 index 00000000000..8836f0fea61 --- /dev/null +++ b/irods/src/test/resources/docker/irods_catalog_provider/Dockerfile @@ -0,0 +1,61 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && \ + apt-get install -y \ + apt-transport-https \ + gnupg \ + wget \ + netcat-traditional \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* + +RUN mkdir -p /etc/apt/keyrings && \ + wget -qO - https://packages.irods.org/irods-signing-key.asc | \ + gpg \ + --no-options \ + --no-default-keyring \ + --no-auto-check-trustdb \ + --homedir /dev/null \ + --no-keyring \ + --import-options import-export \ + --output /etc/apt/keyrings/renci-irods-archive-keyring.pgp \ + --import \ + && \ + echo "deb [signed-by=/etc/apt/keyrings/renci-irods-archive-keyring.pgp arch=amd64] https://packages.irods.org/apt/ noble main" | \ + tee /etc/apt/sources.list.d/renci-irods.list + +RUN apt-get update && \ + apt-get install -y \ + libcurl4-gnutls-dev \ + python3 \ + python3-distro \ + python3-jsonschema \ + python3-pip \ + python3-psutil \ + python3-requests \ + rsyslog \ + unixodbc \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* + +ARG irods_version=5.0.2 +ARG irods_package_version_suffix=-0~noble +ARG irods_package_version=${irods_version}${irods_package_version_suffix} + +RUN apt-get update && \ + apt-get install -y \ + irods-database-plugin-postgres=${irods_package_version} \ + irods-runtime=${irods_package_version} \ + irods-server=${irods_package_version} \ + irods-icommands=${irods_package_version} \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* + +COPY unattended_install.json / +COPY --chmod=755 entrypoint.sh / +ENTRYPOINT ["./entrypoint.sh"] \ No newline at end of file diff --git a/irods/src/test/resources/docker/irods_catalog_provider/entrypoint.sh b/irods/src/test/resources/docker/irods_catalog_provider/entrypoint.sh new file mode 100644 index 00000000000..306cb564c15 --- /dev/null +++ b/irods/src/test/resources/docker/irods_catalog_provider/entrypoint.sh @@ -0,0 +1,24 @@ +#! /bin/bash -e + +echo "Waiting for iRODS catalog database to be ready" +catalog_db_hostname=irods-catalog +until pg_isready -h ${catalog_db_hostname} -d ICAT -U irods -q; do + sleep 1 +done +echo "iRODS catalog database is ready" + +unattended_install_file=/unattended_install.json +if [ -f "${unattended_install_file}" ]; then + echo "Running iRODS setup" + + # Add generated hostname as a recognizable alias. + sed -i "s/CONTAINER_HOSTNAME_ALIAS/${HOSTNAME}/g" ${unattended_install_file} + python3 /var/lib/irods/scripts/setup_irods.py --json_configuration_file ${unattended_install_file} + + # Move the input file used to configure the server out of the way so + # the container is restartable. + mv ${unattended_install_file} ${unattended_install_file}.processed +fi + +echo "Starting server" +su - irods -c 'irodsServer --stdout' diff --git a/irods/src/test/resources/docker/irods_catalog_provider/unattended_install.json b/irods/src/test/resources/docker/irods_catalog_provider/unattended_install.json new file mode 100644 index 00000000000..2369cea5d34 --- /dev/null +++ b/irods/src/test/resources/docker/irods_catalog_provider/unattended_install.json @@ -0,0 +1,169 @@ +{ + "admin_password": "rods", + "default_resource_directory": "/var/lib/irods/Vault", + "default_resource_name": "demoResc", + "host_system_information": { + "service_account_user_name": "irods", + "service_account_group_name": "irods" + }, + "service_account_environment": { + "irods_client_server_policy": "CS_NEG_REFUSE", + "irods_connection_pool_refresh_time_in_seconds": 300, + "irods_cwd": "/tempZone/home/rods", + "irods_default_hash_scheme": "SHA256", + "irods_default_number_of_transfer_threads": 4, + "irods_default_resource": "demoResc", + "irods_encryption_algorithm": "AES-256-CBC", + "irods_encryption_key_size": 32, + "irods_encryption_num_hash_rounds": 16, + "irods_encryption_salt_size": 8, + "irods_home": "/tempZone/home/rods", + "irods_host": "irods-catalog-provider", + "irods_match_hash_policy": "compatible", + "irods_maximum_size_for_single_buffer_in_megabytes": 32, + "irods_port": 1247, + "irods_transfer_buffer_size_for_parallel_transfer_in_megabytes": 4, + "irods_user_name": "rods", + "irods_zone_name": "tempZone", + "schema_name": "service_account_environment", + "schema_version": "v5" + }, + "server_config": { + "advanced_settings": { + "checksum_read_buffer_size_in_bytes": 1048576, + "default_number_of_transfer_threads": 4, + "default_temporary_password_lifetime_in_seconds": 120, + "delay_rule_executors": [], + "delay_server_sleep_time_in_seconds": 30, + "dns_cache": { + "eviction_age_in_seconds": 3600, + "cache_clearer_sleep_time_in_seconds": 600, + "shared_memory_size_in_bytes": 5000000 + }, + "hostname_cache": { + "eviction_age_in_seconds": 3600, + "cache_clearer_sleep_time_in_seconds": 600, + "shared_memory_size_in_bytes": 2500000 + }, + "maximum_size_for_single_buffer_in_megabytes": 32, + "maximum_size_of_delay_queue_in_bytes": 0, + "maximum_temporary_password_lifetime_in_seconds": 1000, + "migrate_delay_server_sleep_time_in_seconds": 5, + "number_of_concurrent_delay_rule_executors": 4, + "stacktrace_file_processor_sleep_time_in_seconds": 10, + "transfer_buffer_size_for_parallel_transfer_in_megabytes": 4, + "transfer_chunk_size_for_parallel_transfer_in_megabytes": 40 + }, + "catalog_provider_hosts": [ + "irods-catalog-provider" + ], + "catalog_service_role": "provider", + "client_server_policy": "CS_NEG_REFUSE", + "connection_pool_refresh_time_in_seconds": 300, + "controlled_user_connection_list": { + "control_type": "denylist", + "users": [] + }, + "default_dir_mode": "0750", + "default_file_mode": "0600", + "default_hash_scheme": "SHA256", + "default_resource_name": "demoResc", + "encryption": { + "algorithm": "AES-256-CBC", + "key_size": 32, + "num_hash_rounds": 16, + "salt_size": 8 + }, + "environment_variables": {}, + "federation": [], + "graceful_shutdown_timeout_in_seconds": 30, + "host": "irods-catalog-provider", + "host_access_control": { + "access_entries": [] + }, + "host_resolution": { + "host_entries": [ + { + "address_type": "local", + "addresses": [ + "irods-catalog-provider", + "CONTAINER_HOSTNAME_ALIAS" + ] + } + ] + }, + "log_level": { + "agent": "info", + "agent_factory": "info", + "api": "info", + "authentication": "info", + "database": "info", + "delay_server": "info", + "genquery1": "info", + "genquery2": "info", + "legacy": "info", + "microservice": "info", + "network": "info", + "resource": "info", + "rule_engine": "info", + "server": "info", + "sql": "info" + }, + "match_hash_policy": "compatible", + "negotiation_key": "32_byte_server_negotiation_key__", + "plugin_configuration": { + "authentication": {}, + "database": { + "technology": "postgres", + "host": "irods-catalog", + "name": "ICAT", + "odbc_driver": "PostgreSQL ANSI", + "password": "testpassword", + "port": 5432, + "username": "irods" + }, + "network": {}, + "resource": {}, + "rule_engines": [ + { + "instance_name": "irods_rule_engine_plugin-irods_rule_language-instance", + "plugin_name": "irods_rule_engine_plugin-irods_rule_language", + "plugin_specific_configuration": { + "re_data_variable_mapping_set": [ + "core" + ], + "re_function_name_mapping_set": [ + "core" + ], + "re_rulebase_set": [ + "core" + ], + "regexes_for_supported_peps": [ + "ac[^ ]*", + "msi[^ ]*", + "[^ ]*pep_[^ ]*_(pre|post|except|finally)" + ] + }, + "shared_memory_instance": "irods_rule_language_rule_engine" + }, + { + "instance_name": "irods_rule_engine_plugin-cpp_default_policy-instance", + "plugin_name": "irods_rule_engine_plugin-cpp_default_policy", + "plugin_specific_configuration": {} + } + ] + }, + "rule_engine_namespaces": [ + "" + ], + "schema_name": "server_config", + "schema_version": "v5", + "server_port_range_end": 20199, + "server_port_range_start": 20000, + "zone_auth_scheme": "native", + "zone_key": "TEMPORARY_ZONE_KEY", + "zone_name": "tempZone", + "zone_port": 1247, + "zone_user": "rods" + } +} \ No newline at end of file From e7bac883081f3563645ccc0af97935c4a41968a9 Mon Sep 17 00:00:00 2001 From: Kory Draughn Date: Thu, 30 Oct 2025 18:04:37 -0400 Subject: [PATCH 04/20] iRODS: Fix tests --- irods/pom.xml | 6 ++ .../IRODSAttributesFinderFeatureTest.java | 13 ++-- .../core/irods/IRODSCopyFeatureTest.java | 9 ++- .../core/irods/IRODSDeleteFeatureTest.java | 11 ++-- .../core/irods/IRODSDirectoryFeatureTest.java | 9 ++- .../core/irods/IRODSDockerComposeManager.java | 50 ++++++++++++++ .../core/irods/IRODSFindFeatureTest.java | 9 ++- .../core/irods/IRODSListServiceTest.java | 15 ++--- .../core/irods/IRODSMoveFeatureTest.java | 13 ++-- .../core/irods/IRODSReadFeatureTest.java | 14 ++-- .../core/irods/IRODSSessionTest.java | 17 +++-- .../core/irods/IRODSTouchFeatureTest.java | 9 ++- .../core/irods/IRODSUploadFeatureTest.java | 66 ++++++++++++++++--- .../core/irods/IRODSWriteFeatureTest.java | 27 +++++--- ...yberduckprofile => iRODS.cyberduckprofile} | 18 +++-- 15 files changed, 195 insertions(+), 91 deletions(-) create mode 100644 irods/src/test/java/ch/cyberduck/core/irods/IRODSDockerComposeManager.java rename irods/src/test/resources/{iRODS (iPlant Collaborative).cyberduckprofile => iRODS.cyberduckprofile} (63%) diff --git a/irods/pom.xml b/irods/pom.xml index 7d185e7071d..9b50155d52e 100644 --- a/irods/pom.xml +++ b/irods/pom.xml @@ -41,6 +41,12 @@ test ${project.version} + + testcontainers + org.testcontainers + 1.21.3 + test + diff --git a/irods/src/test/java/ch/cyberduck/core/irods/IRODSAttributesFinderFeatureTest.java b/irods/src/test/java/ch/cyberduck/core/irods/IRODSAttributesFinderFeatureTest.java index c5e479a359e..58817f36df9 100644 --- a/irods/src/test/java/ch/cyberduck/core/irods/IRODSAttributesFinderFeatureTest.java +++ b/irods/src/test/java/ch/cyberduck/core/irods/IRODSAttributesFinderFeatureTest.java @@ -29,8 +29,7 @@ import ch.cyberduck.core.proxy.DisabledProxyFinder; import ch.cyberduck.core.serializer.impl.dd.ProfilePlistReader; import ch.cyberduck.core.transfer.TransferStatus; -import ch.cyberduck.test.IntegrationTest; -import ch.cyberduck.test.VaultTest; +import ch.cyberduck.test.TestcontainerTest; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -43,14 +42,14 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -@Category(IntegrationTest.class) -public class IRODSAttributesFinderFeatureTest extends VaultTest { +@Category(TestcontainerTest.class) +public class IRODSAttributesFinderFeatureTest extends IRODSDockerComposeManager { @Test(expected = NotfoundException.class) public void testFindNotFound() throws Exception { final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new IRODSProtocol()))); final Profile profile = new ProfilePlistReader(factory).read( - this.getClass().getResourceAsStream("/iRODS (iPlant Collaborative).cyberduckprofile")); + this.getClass().getResourceAsStream("/iRODS.cyberduckprofile")); final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials( PROPERTIES.get("irods.key"), PROPERTIES.get("irods.secret") )); @@ -64,7 +63,7 @@ public void testFindNotFound() throws Exception { public void testFind() throws Exception { final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new IRODSProtocol()))); final Profile profile = new ProfilePlistReader(factory).read( - this.getClass().getResourceAsStream("/iRODS (iPlant Collaborative).cyberduckprofile")); + this.getClass().getResourceAsStream("/iRODS.cyberduckprofile")); final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials( PROPERTIES.get("irods.key"), PROPERTIES.get("irods.secret") )); @@ -81,8 +80,6 @@ public void testFind() throws Exception { assertEquals(folderTimestamp, f.find(folder).getModificationDate()); final PathAttributes attributes = f.find(test); assertEquals(0L, attributes.getSize()); - assertEquals("iterate", attributes.getOwner()); - assertEquals("iplant", attributes.getGroup()); new IRODSDeleteFeature(session).delete(Collections.singletonList(test), new DisabledLoginCallback(), new Delete.DisabledCallback()); assertFalse(new IRODSFindFeature(session).find(test)); session.close(); diff --git a/irods/src/test/java/ch/cyberduck/core/irods/IRODSCopyFeatureTest.java b/irods/src/test/java/ch/cyberduck/core/irods/IRODSCopyFeatureTest.java index 6db993b2ab8..5f92bc8358a 100644 --- a/irods/src/test/java/ch/cyberduck/core/irods/IRODSCopyFeatureTest.java +++ b/irods/src/test/java/ch/cyberduck/core/irods/IRODSCopyFeatureTest.java @@ -31,8 +31,7 @@ import ch.cyberduck.core.proxy.DisabledProxyFinder; import ch.cyberduck.core.serializer.impl.dd.ProfilePlistReader; import ch.cyberduck.core.transfer.TransferStatus; -import ch.cyberduck.test.IntegrationTest; -import ch.cyberduck.test.VaultTest; +import ch.cyberduck.test.TestcontainerTest; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -45,14 +44,14 @@ import static org.junit.Assert.assertTrue; -@Category(IntegrationTest.class) -public class IRODSCopyFeatureTest extends VaultTest { +@Category(TestcontainerTest.class) +public class IRODSCopyFeatureTest extends IRODSDockerComposeManager { @Test public void testCopy() throws Exception { final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new IRODSProtocol()))); final Profile profile = new ProfilePlistReader(factory).read( - this.getClass().getResourceAsStream("/iRODS (iPlant Collaborative).cyberduckprofile")); + this.getClass().getResourceAsStream("/iRODS.cyberduckprofile")); final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials( PROPERTIES.get("irods.key"), PROPERTIES.get("irods.secret") )); diff --git a/irods/src/test/java/ch/cyberduck/core/irods/IRODSDeleteFeatureTest.java b/irods/src/test/java/ch/cyberduck/core/irods/IRODSDeleteFeatureTest.java index 2936c7c3df5..8ce521c2c94 100644 --- a/irods/src/test/java/ch/cyberduck/core/irods/IRODSDeleteFeatureTest.java +++ b/irods/src/test/java/ch/cyberduck/core/irods/IRODSDeleteFeatureTest.java @@ -30,8 +30,7 @@ import ch.cyberduck.core.proxy.DisabledProxyFinder; import ch.cyberduck.core.serializer.impl.dd.ProfilePlistReader; import ch.cyberduck.core.transfer.TransferStatus; -import ch.cyberduck.test.IntegrationTest; -import ch.cyberduck.test.VaultTest; +import ch.cyberduck.test.TestcontainerTest; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -44,14 +43,14 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -@Category(IntegrationTest.class) -public class IRODSDeleteFeatureTest extends VaultTest { +@Category(TestcontainerTest.class) +public class IRODSDeleteFeatureTest extends IRODSDockerComposeManager { @Test public void testDeleteDirectory() throws Exception { final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new IRODSProtocol()))); final Profile profile = new ProfilePlistReader(factory).read( - this.getClass().getResourceAsStream("/iRODS (iPlant Collaborative).cyberduckprofile")); + this.getClass().getResourceAsStream("/iRODS.cyberduckprofile")); final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials( PROPERTIES.get("irods.key"), PROPERTIES.get("irods.secret") )); @@ -76,7 +75,7 @@ public void testDeleteDirectory() throws Exception { public void testDeleteNotFound() throws Exception { final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new IRODSProtocol()))); final Profile profile = new ProfilePlistReader(factory).read( - this.getClass().getResourceAsStream("/iRODS (iPlant Collaborative).cyberduckprofile")); + this.getClass().getResourceAsStream("/iRODS.cyberduckprofile")); final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials( PROPERTIES.get("irods.key"), PROPERTIES.get("irods.secret") )); diff --git a/irods/src/test/java/ch/cyberduck/core/irods/IRODSDirectoryFeatureTest.java b/irods/src/test/java/ch/cyberduck/core/irods/IRODSDirectoryFeatureTest.java index 9b4e7168862..35a0e7c051b 100644 --- a/irods/src/test/java/ch/cyberduck/core/irods/IRODSDirectoryFeatureTest.java +++ b/irods/src/test/java/ch/cyberduck/core/irods/IRODSDirectoryFeatureTest.java @@ -31,8 +31,7 @@ import ch.cyberduck.core.proxy.DisabledProxyFinder; import ch.cyberduck.core.serializer.impl.dd.ProfilePlistReader; import ch.cyberduck.core.transfer.TransferStatus; -import ch.cyberduck.test.IntegrationTest; -import ch.cyberduck.test.VaultTest; +import ch.cyberduck.test.TestcontainerTest; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -45,14 +44,14 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -@Category(IntegrationTest.class) -public class IRODSDirectoryFeatureTest extends VaultTest { +@Category(TestcontainerTest.class) +public class IRODSDirectoryFeatureTest extends IRODSDockerComposeManager { @Test public void testMakeDirectory() throws Exception { final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new IRODSProtocol()))); final Profile profile = new ProfilePlistReader(factory).read( - this.getClass().getResourceAsStream("/iRODS (iPlant Collaborative).cyberduckprofile")); + this.getClass().getResourceAsStream("/iRODS.cyberduckprofile")); final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials( PROPERTIES.get("irods.key"), PROPERTIES.get("irods.secret") )); diff --git a/irods/src/test/java/ch/cyberduck/core/irods/IRODSDockerComposeManager.java b/irods/src/test/java/ch/cyberduck/core/irods/IRODSDockerComposeManager.java new file mode 100644 index 00000000000..cf20df441d4 --- /dev/null +++ b/irods/src/test/java/ch/cyberduck/core/irods/IRODSDockerComposeManager.java @@ -0,0 +1,50 @@ +package ch.cyberduck.core.irods; + +/* + * Copyright (c) 2002-2025 iterate GmbH. All rights reserved. + * https://cyberduck.io/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +import ch.cyberduck.test.TestcontainerTest; + +import org.junit.experimental.categories.Category; +import org.testcontainers.containers.ComposeContainer; +import org.testcontainers.containers.wait.strategy.Wait; + +import java.io.File; +import java.util.HashMap; +import java.util.Map; + +@Category(TestcontainerTest.class) +public abstract class IRODSDockerComposeManager { + + private static final ComposeContainer container; + + // Launch the docker compose project once and run all tests against that environment. + // Ryuk will clean it up at the end of the run. + static { + container = new ComposeContainer( + new File(IRODSDockerComposeManager.class.getResource("/docker/docker-compose.yml").getFile())) + .withPull(false) + .withLocalCompose(true) + .withExposedService("irods-catalog-provider-1", 1247, Wait.forLogMessage(".*\"log_message\":\"Initializing delay server.\".*", 1)); + + container.start(); + } + + protected static final Map PROPERTIES = new HashMap() {{ + put("irods.key", "rods"); + put("irods.secret", "rods"); + }}; + +} diff --git a/irods/src/test/java/ch/cyberduck/core/irods/IRODSFindFeatureTest.java b/irods/src/test/java/ch/cyberduck/core/irods/IRODSFindFeatureTest.java index a1e9fb93142..bbdf9d67836 100644 --- a/irods/src/test/java/ch/cyberduck/core/irods/IRODSFindFeatureTest.java +++ b/irods/src/test/java/ch/cyberduck/core/irods/IRODSFindFeatureTest.java @@ -29,8 +29,7 @@ import ch.cyberduck.core.proxy.DisabledProxyFinder; import ch.cyberduck.core.serializer.impl.dd.ProfilePlistReader; import ch.cyberduck.core.transfer.TransferStatus; -import ch.cyberduck.test.IntegrationTest; -import ch.cyberduck.test.VaultTest; +import ch.cyberduck.test.TestcontainerTest; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -43,14 +42,14 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -@Category(IntegrationTest.class) -public class IRODSFindFeatureTest extends VaultTest { +@Category(TestcontainerTest.class) +public class IRODSFindFeatureTest extends IRODSDockerComposeManager { @Test public void testFind() throws Exception { final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new IRODSProtocol()))); final Profile profile = new ProfilePlistReader(factory).read( - this.getClass().getResourceAsStream("/iRODS (iPlant Collaborative).cyberduckprofile")); + this.getClass().getResourceAsStream("/iRODS.cyberduckprofile")); final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials( PROPERTIES.get("irods.key"), PROPERTIES.get("irods.secret") )); diff --git a/irods/src/test/java/ch/cyberduck/core/irods/IRODSListServiceTest.java b/irods/src/test/java/ch/cyberduck/core/irods/IRODSListServiceTest.java index 0a08507f1db..b0c0eff309c 100644 --- a/irods/src/test/java/ch/cyberduck/core/irods/IRODSListServiceTest.java +++ b/irods/src/test/java/ch/cyberduck/core/irods/IRODSListServiceTest.java @@ -30,10 +30,8 @@ import ch.cyberduck.core.exception.NotfoundException; import ch.cyberduck.core.proxy.DisabledProxyFinder; import ch.cyberduck.core.serializer.impl.dd.ProfilePlistReader; -import ch.cyberduck.test.IntegrationTest; -import ch.cyberduck.test.VaultTest; +import ch.cyberduck.test.TestcontainerTest; -import org.junit.Ignore; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -44,15 +42,14 @@ import static org.junit.Assert.*; -@Category(IntegrationTest.class) -public class IRODSListServiceTest extends VaultTest { +@Category(TestcontainerTest.class) +public class IRODSListServiceTest extends IRODSDockerComposeManager { @Test - @Ignore public void testList() throws Exception { final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new IRODSProtocol()))); final Profile profile = new ProfilePlistReader(factory).read( - this.getClass().getResourceAsStream("/iRODS (iPlant Collaborative).cyberduckprofile")); + this.getClass().getResourceAsStream("/iRODS.cyberduckprofile")); final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials( PROPERTIES.get("irods.key"), PROPERTIES.get("irods.secret") )); @@ -64,7 +61,7 @@ public void testList() throws Exception { assertNotNull(session.getClient()); session.login(new DisabledLoginCallback(), new DisabledCancelCallback()); final AttributedList list = new IRODSListService(session).list(new IRODSHomeFinderService(session).find(), new DisabledListProgressListener()); - assertFalse(list.isEmpty()); + assertTrue(list.isEmpty()); for(Path p : list) { assertEquals(new IRODSHomeFinderService(session).find(), p.getParent()); assertNotEquals(-1L, p.attributes().getModificationDate()); @@ -77,7 +74,7 @@ public void testList() throws Exception { public void testListNotfound() throws Exception { final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new IRODSProtocol()))); final Profile profile = new ProfilePlistReader(factory).read( - this.getClass().getResourceAsStream("/iRODS (iPlant Collaborative).cyberduckprofile")); + this.getClass().getResourceAsStream("/iRODS.cyberduckprofile")); final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials( PROPERTIES.get("irods.key"), PROPERTIES.get("irods.secret") )); diff --git a/irods/src/test/java/ch/cyberduck/core/irods/IRODSMoveFeatureTest.java b/irods/src/test/java/ch/cyberduck/core/irods/IRODSMoveFeatureTest.java index ca9a7a90d45..7a6dd1016cb 100644 --- a/irods/src/test/java/ch/cyberduck/core/irods/IRODSMoveFeatureTest.java +++ b/irods/src/test/java/ch/cyberduck/core/irods/IRODSMoveFeatureTest.java @@ -33,8 +33,7 @@ import ch.cyberduck.core.proxy.DisabledProxyFinder; import ch.cyberduck.core.serializer.impl.dd.ProfilePlistReader; import ch.cyberduck.core.transfer.TransferStatus; -import ch.cyberduck.test.IntegrationTest; -import ch.cyberduck.test.VaultTest; +import ch.cyberduck.test.TestcontainerTest; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -46,14 +45,14 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -@Category(IntegrationTest.class) -public class IRODSMoveFeatureTest extends VaultTest { +@Category(TestcontainerTest.class) +public class IRODSMoveFeatureTest extends IRODSDockerComposeManager { @Test public void testMoveDirectory() throws Exception { final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new IRODSProtocol()))); final Profile profile = new ProfilePlistReader(factory).read( - this.getClass().getResourceAsStream("/iRODS (iPlant Collaborative).cyberduckprofile")); + this.getClass().getResourceAsStream("/iRODS.cyberduckprofile")); final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials( PROPERTIES.get("irods.key"), PROPERTIES.get("irods.secret") )); @@ -83,7 +82,7 @@ public void testMoveDirectory() throws Exception { public void testMoveFile() throws Exception { final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new IRODSProtocol()))); final Profile profile = new ProfilePlistReader(factory).read( - this.getClass().getResourceAsStream("/iRODS (iPlant Collaborative).cyberduckprofile")); + this.getClass().getResourceAsStream("/iRODS.cyberduckprofile")); final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials( PROPERTIES.get("irods.key"), PROPERTIES.get("irods.secret") )); @@ -108,7 +107,7 @@ public void testMoveFile() throws Exception { public void testMoveNotFound() throws Exception { final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new IRODSProtocol()))); final Profile profile = new ProfilePlistReader(factory).read( - this.getClass().getResourceAsStream("/iRODS (iPlant Collaborative).cyberduckprofile")); + this.getClass().getResourceAsStream("/iRODS.cyberduckprofile")); final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials( PROPERTIES.get("irods.key"), PROPERTIES.get("irods.secret") )); diff --git a/irods/src/test/java/ch/cyberduck/core/irods/IRODSReadFeatureTest.java b/irods/src/test/java/ch/cyberduck/core/irods/IRODSReadFeatureTest.java index 33ebbc4c261..91f11b7267f 100644 --- a/irods/src/test/java/ch/cyberduck/core/irods/IRODSReadFeatureTest.java +++ b/irods/src/test/java/ch/cyberduck/core/irods/IRODSReadFeatureTest.java @@ -38,8 +38,7 @@ import ch.cyberduck.core.serializer.impl.dd.ProfilePlistReader; import ch.cyberduck.core.shared.DefaultUploadFeature; import ch.cyberduck.core.transfer.TransferStatus; -import ch.cyberduck.test.IntegrationTest; -import ch.cyberduck.test.VaultTest; +import ch.cyberduck.test.TestcontainerTest; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.RandomUtils; @@ -53,19 +52,18 @@ import java.util.Collections; import java.util.EnumSet; import java.util.HashSet; -import java.util.List; import java.util.UUID; import static org.junit.Assert.*; -@Category(IntegrationTest.class) -public class IRODSReadFeatureTest extends VaultTest { +@Category(TestcontainerTest.class) +public class IRODSReadFeatureTest extends IRODSDockerComposeManager { @Test public void testRead() throws Exception { final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new IRODSProtocol()))); final Profile profile = new ProfilePlistReader(factory).read( - this.getClass().getResourceAsStream("/iRODS (iPlant Collaborative).cyberduckprofile")); + this.getClass().getResourceAsStream("/iRODS.cyberduckprofile")); final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials( PROPERTIES.get("irods.key"), PROPERTIES.get("irods.secret") )); @@ -101,7 +99,7 @@ public void testRead() throws Exception { public void testReadNotFound() throws Exception { final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new IRODSProtocol()))); final Profile profile = new ProfilePlistReader(factory).read( - this.getClass().getResourceAsStream("/iRODS (iPlant Collaborative).cyberduckprofile")); + this.getClass().getResourceAsStream("/iRODS.cyberduckprofile")); final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials( PROPERTIES.get("irods.key"), PROPERTIES.get("irods.secret") )); @@ -121,7 +119,7 @@ public void testReadNotFound() throws Exception { public void testReadRange() throws Exception { final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new IRODSProtocol()))); final Profile profile = new ProfilePlistReader(factory).read( - this.getClass().getResourceAsStream("/iRODS (iPlant Collaborative).cyberduckprofile")); + this.getClass().getResourceAsStream("/iRODS.cyberduckprofile")); final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials( PROPERTIES.get("irods.key"), PROPERTIES.get("irods.secret") )); diff --git a/irods/src/test/java/ch/cyberduck/core/irods/IRODSSessionTest.java b/irods/src/test/java/ch/cyberduck/core/irods/IRODSSessionTest.java index c6ecb71a635..7db1304fa4a 100644 --- a/irods/src/test/java/ch/cyberduck/core/irods/IRODSSessionTest.java +++ b/irods/src/test/java/ch/cyberduck/core/irods/IRODSSessionTest.java @@ -30,8 +30,7 @@ import ch.cyberduck.core.exception.LoginFailureException; import ch.cyberduck.core.proxy.DisabledProxyFinder; import ch.cyberduck.core.serializer.impl.dd.ProfilePlistReader; -import ch.cyberduck.test.IntegrationTest; -import ch.cyberduck.test.VaultTest; +import ch.cyberduck.test.TestcontainerTest; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -41,14 +40,14 @@ import static org.junit.Assert.*; -@Category(IntegrationTest.class) -public class IRODSSessionTest extends VaultTest { +@Category(TestcontainerTest.class) +public class IRODSSessionTest extends IRODSDockerComposeManager { @Test public void testConnect() throws Exception { final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new IRODSProtocol()))); final Profile profile = new ProfilePlistReader(factory).read( - this.getClass().getResourceAsStream("/iRODS (iPlant Collaborative).cyberduckprofile")); + this.getClass().getResourceAsStream("/iRODS.cyberduckprofile")); final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials( PROPERTIES.get("irods.key"), PROPERTIES.get("irods.secret") )); @@ -67,7 +66,7 @@ public void testConnect() throws Exception { public void testLoginDefault() throws Exception { final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new IRODSProtocol()))); final Profile profile = new ProfilePlistReader(factory).read( - this.getClass().getResourceAsStream("/iRODS (iPlant Collaborative).cyberduckprofile")); + this.getClass().getResourceAsStream("/iRODS.cyberduckprofile")); final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials( PROPERTIES.get("irods.key"), PROPERTIES.get("irods.secret") )); @@ -81,7 +80,7 @@ public void testLoginDefault() throws Exception { session.login(new DisabledLoginCallback(), new DisabledCancelCallback()); final AttributedList list = new IRODSListService(session).list(new IRODSHomeFinderService(session).find(), new DisabledListProgressListener()); - assertFalse(list.isEmpty()); + assertTrue(list.isEmpty()); assertTrue(session.isConnected()); session.close(); @@ -92,7 +91,7 @@ public void testLoginDefault() throws Exception { public void testLoginWhitespaceHomeDirectory() throws Exception { final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new IRODSProtocol()))); final Profile profile = new ProfilePlistReader(factory).read( - this.getClass().getResourceAsStream("/iRODS (iPlant Collaborative).cyberduckprofile")); + this.getClass().getResourceAsStream("/iRODS.cyberduckprofile")); final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials( PROPERTIES.get("irods.key"), PROPERTIES.get("irods.secret") )); @@ -115,7 +114,7 @@ public void testLoginWhitespaceHomeDirectory() throws Exception { public void testLoginFailure() throws Exception { final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new IRODSProtocol()))); final Profile profile = new ProfilePlistReader(factory).read( - this.getClass().getResourceAsStream("/iRODS (iPlant Collaborative).cyberduckprofile")); + this.getClass().getResourceAsStream("/iRODS.cyberduckprofile")); final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials("a", "a")); final IRODSSession session = new IRODSSession(host); diff --git a/irods/src/test/java/ch/cyberduck/core/irods/IRODSTouchFeatureTest.java b/irods/src/test/java/ch/cyberduck/core/irods/IRODSTouchFeatureTest.java index a7a9fb03fc1..8b68b5f76ad 100644 --- a/irods/src/test/java/ch/cyberduck/core/irods/IRODSTouchFeatureTest.java +++ b/irods/src/test/java/ch/cyberduck/core/irods/IRODSTouchFeatureTest.java @@ -29,8 +29,7 @@ import ch.cyberduck.core.proxy.DisabledProxyFinder; import ch.cyberduck.core.serializer.impl.dd.ProfilePlistReader; import ch.cyberduck.core.transfer.TransferStatus; -import ch.cyberduck.test.IntegrationTest; -import ch.cyberduck.test.VaultTest; +import ch.cyberduck.test.TestcontainerTest; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -43,14 +42,14 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -@Category(IntegrationTest.class) -public class IRODSTouchFeatureTest extends VaultTest { +@Category(TestcontainerTest.class) +public class IRODSTouchFeatureTest extends IRODSDockerComposeManager { @Test public void testTouch() throws Exception { final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new IRODSProtocol()))); final Profile profile = new ProfilePlistReader(factory).read( - this.getClass().getResourceAsStream("/iRODS (iPlant Collaborative).cyberduckprofile")); + this.getClass().getResourceAsStream("/iRODS.cyberduckprofile")); final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials( PROPERTIES.get("irods.key"), PROPERTIES.get("irods.secret") )); diff --git a/irods/src/test/java/ch/cyberduck/core/irods/IRODSUploadFeatureTest.java b/irods/src/test/java/ch/cyberduck/core/irods/IRODSUploadFeatureTest.java index 947a3ac28e5..b80402751e1 100644 --- a/irods/src/test/java/ch/cyberduck/core/irods/IRODSUploadFeatureTest.java +++ b/irods/src/test/java/ch/cyberduck/core/irods/IRODSUploadFeatureTest.java @@ -36,8 +36,7 @@ import ch.cyberduck.core.proxy.DisabledProxyFinder; import ch.cyberduck.core.serializer.impl.dd.ProfilePlistReader; import ch.cyberduck.core.transfer.TransferStatus; -import ch.cyberduck.test.IntegrationTest; -import ch.cyberduck.test.VaultTest; +import ch.cyberduck.test.TestcontainerTest; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.RandomUtils; @@ -54,15 +53,15 @@ import static org.junit.Assert.*; -@Category(IntegrationTest.class) -public class IRODSUploadFeatureTest extends VaultTest { +@Category(TestcontainerTest.class) +public class IRODSUploadFeatureTest extends IRODSDockerComposeManager { @Test @Ignore public void testAppend() throws Exception { final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new IRODSProtocol()))); final Profile profile = new ProfilePlistReader(factory).read( - this.getClass().getResourceAsStream("/iRODS (iPlant Collaborative).cyberduckprofile")); + this.getClass().getResourceAsStream("/iRODS.cyberduckprofile")); final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials( PROPERTIES.get("irods.key"), PROPERTIES.get("irods.secret") )); @@ -108,7 +107,7 @@ public void testAppend() throws Exception { public void testWrite() throws Exception { final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new IRODSProtocol()))); final Profile profile = new ProfilePlistReader(factory).read( - this.getClass().getResourceAsStream("/iRODS (iPlant Collaborative).cyberduckprofile")); + this.getClass().getResourceAsStream("/iRODS.cyberduckprofile")); final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials( PROPERTIES.get("irods.key"), PROPERTIES.get("irods.secret") )); @@ -139,10 +138,10 @@ public void testWrite() throws Exception { } @Test - public void testInterruptStatus() throws Exception { + public void testInterruptStatusForSmallFiles() throws Exception { final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new IRODSProtocol()))); final Profile profile = new ProfilePlistReader(factory).read( - this.getClass().getResourceAsStream("/iRODS (iPlant Collaborative).cyberduckprofile")); + this.getClass().getResourceAsStream("/iRODS.cyberduckprofile")); final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials( PROPERTIES.get("irods.key"), PROPERTIES.get("irods.secret") )); @@ -159,7 +158,10 @@ public void testInterruptStatus() throws Exception { final Path test = new Path(new IRODSHomeFinderService(session).find(), UUID.randomUUID().toString(), EnumSet.of(Path.Type.file)); final TransferStatus status = new TransferStatus().setLength(content.length); new IRODSUploadFeature(session).upload( - new IRODSWriteFeature(session), test, local, new BandwidthThrottle(BandwidthThrottle.UNLIMITED), new DisabledProgressListener(), new DisabledStreamListener() { + new IRODSWriteFeature(session), test, local, + new BandwidthThrottle(BandwidthThrottle.UNLIMITED), + new DisabledProgressListener(), + new DisabledStreamListener() { @Override public void sent(final long bytes) { super.sent(bytes); @@ -176,6 +178,52 @@ public void sent(final long bytes) { // } assertFalse(status.isComplete()); + new IRODSDeleteFeature(session).delete(Collections.singletonList(test), new DisabledLoginCallback(), new Delete.DisabledCallback()); + session.close(); + } + + @Test + public void testInterruptStatusForLargeFiles() throws Exception { + final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new IRODSProtocol()))); + final Profile profile = new ProfilePlistReader(factory).read( + this.getClass().getResourceAsStream("/iRODS.cyberduckprofile")); + final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials( + PROPERTIES.get("irods.key"), PROPERTIES.get("irods.secret") + )); + + final IRODSSession session = new IRODSSession(host); + session.open(new DisabledProxyFinder(), new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); + session.login(new DisabledLoginCallback(), new DisabledCancelCallback()); + final Local local = new Local(System.getProperty("java.io.tmpdir"), UUID.randomUUID().toString()); + final int length = 33 * 1024 * 1024; // Triggers parallel transfer + final byte[] content = RandomUtils.nextBytes(length); + final OutputStream out = local.getOutputStream(false); + IOUtils.write(content, out); + out.close(); + final Path test = new Path(new IRODSHomeFinderService(session).find(), UUID.randomUUID().toString(), EnumSet.of(Path.Type.file)); + final TransferStatus status = new TransferStatus().setLength(content.length); + new IRODSUploadFeature(session).upload( + new IRODSWriteFeature(session), test, local, + new BandwidthThrottle(BandwidthThrottle.UNLIMITED), + new DisabledProgressListener(), + new DisabledStreamListener() { + @Override + public void sent(final long bytes) { + super.sent(bytes); + status.setCanceled(); + } + }, + status, + new DisabledConnectionCallback()); + try { + status.validate(); + fail(); + } + catch(ConnectionCanceledException e) { + // + } + assertFalse(status.isComplete()); + new IRODSDeleteFeature(session).delete(Collections.singletonList(test), new DisabledLoginCallback(), new Delete.DisabledCallback()); session.close(); } } diff --git a/irods/src/test/java/ch/cyberduck/core/irods/IRODSWriteFeatureTest.java b/irods/src/test/java/ch/cyberduck/core/irods/IRODSWriteFeatureTest.java index 7dd00a92d1c..1a71866c0e6 100644 --- a/irods/src/test/java/ch/cyberduck/core/irods/IRODSWriteFeatureTest.java +++ b/irods/src/test/java/ch/cyberduck/core/irods/IRODSWriteFeatureTest.java @@ -36,8 +36,7 @@ import ch.cyberduck.core.proxy.DisabledProxyFinder; import ch.cyberduck.core.serializer.impl.dd.ProfilePlistReader; import ch.cyberduck.core.transfer.TransferStatus; -import ch.cyberduck.test.IntegrationTest; -import ch.cyberduck.test.VaultTest; +import ch.cyberduck.test.TestcontainerTest; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.RandomUtils; @@ -50,20 +49,19 @@ import java.util.Collections; import java.util.EnumSet; import java.util.HashSet; -import java.util.List; import java.util.UUID; import java.util.concurrent.CountDownLatch; import static org.junit.Assert.*; -@Category(IntegrationTest.class) -public class IRODSWriteFeatureTest extends VaultTest { +@Category(TestcontainerTest.class) +public class IRODSWriteFeatureTest extends IRODSDockerComposeManager { @Test public void testWriteConcurrent() throws Exception { final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new IRODSProtocol()))); final Profile profile = new ProfilePlistReader(factory).read( - this.getClass().getResourceAsStream("/iRODS (iPlant Collaborative).cyberduckprofile")); + this.getClass().getResourceAsStream("/iRODS.cyberduckprofile")); final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials( PROPERTIES.get("irods.key"), PROPERTIES.get("irods.secret") )); @@ -101,7 +99,13 @@ public void testWriteConcurrent() throws Exception { in2.close(); assertArrayEquals(content, buffer2); } + + session1.getFeature(Delete.class).delete(Collections.singletonList(test1), new DisabledLoginCallback(), new Delete.DisabledCallback()); + assertFalse(session1.getFeature(Find.class).find(test1)); session1.close(); + + session2.getFeature(Delete.class).delete(Collections.singletonList(test2), new DisabledLoginCallback(), new Delete.DisabledCallback()); + assertFalse(session2.getFeature(Find.class).find(test2)); session2.close(); } @@ -109,7 +113,7 @@ public void testWriteConcurrent() throws Exception { public void testWriteThreaded() throws Exception { final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new IRODSProtocol()))); final Profile profile = new ProfilePlistReader(factory).read( - this.getClass().getResourceAsStream("/iRODS (iPlant Collaborative).cyberduckprofile")); + this.getClass().getResourceAsStream("/iRODS.cyberduckprofile")); final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials( PROPERTIES.get("irods.key"), PROPERTIES.get("irods.secret") )); @@ -204,7 +208,12 @@ public void run() { cr1.await(); cr2.await(); + session1.getFeature(Delete.class).delete(Collections.singletonList(test1), new DisabledLoginCallback(), new Delete.DisabledCallback()); + assertFalse(session1.getFeature(Find.class).find(test1)); session1.close(); + + session2.getFeature(Delete.class).delete(Collections.singletonList(test2), new DisabledLoginCallback(), new Delete.DisabledCallback()); + assertFalse(session2.getFeature(Find.class).find(test2)); session2.close(); } @@ -212,7 +221,7 @@ public void run() { public void testWrite() throws Exception { final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new IRODSProtocol()))); final Profile profile = new ProfilePlistReader(factory).read( - this.getClass().getResourceAsStream("/iRODS (iPlant Collaborative).cyberduckprofile")); + this.getClass().getResourceAsStream("/iRODS.cyberduckprofile")); final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials( PROPERTIES.get("irods.key"), PROPERTIES.get("irods.secret") )); @@ -285,7 +294,7 @@ public void testWrite() throws Exception { public void testWriteAppend() throws Exception { final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new IRODSProtocol()))); final Profile profile = new ProfilePlistReader(factory).read( - this.getClass().getResourceAsStream("/iRODS (iPlant Collaborative).cyberduckprofile")); + this.getClass().getResourceAsStream("/iRODS.cyberduckprofile")); final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials( PROPERTIES.get("irods.key"), PROPERTIES.get("irods.secret") )); diff --git a/irods/src/test/resources/iRODS (iPlant Collaborative).cyberduckprofile b/irods/src/test/resources/iRODS.cyberduckprofile similarity index 63% rename from irods/src/test/resources/iRODS (iPlant Collaborative).cyberduckprofile rename to irods/src/test/resources/iRODS.cyberduckprofile index f51cceb78a1..ab84678a88f 100644 --- a/irods/src/test/resources/iRODS (iPlant Collaborative).cyberduckprofile +++ b/irods/src/test/resources/iRODS.cyberduckprofile @@ -5,9 +5,9 @@ Protocol irods Vendor - iplant + iRODS Consortium Description - iPlant Data Store + iRODS (Integrated Rule-Oriented Data System) Hostname Configurable @@ -15,19 +15,25 @@ Default Hostname - data.iplantcollaborative.org + localhost Default Port 1247 Username Placeholder - iPlant username + iRODS username Password Placeholder - iPlant password + iRODS password Region - iplant + tempZone Authorization native + + Properties + + Delete Objects Permanently + yes + From 03c9c17ec54df1a60bf2c63e25c82b33536d4221 Mon Sep 17 00:00:00 2001 From: Kory Draughn Date: Sun, 9 Nov 2025 20:57:18 -0500 Subject: [PATCH 05/20] squash w/ 2870ae7. rename timestamp class --- .../src/main/java/ch/cyberduck/core/irods/IRODSSession.java | 2 +- .../{IRODSTimestamp.java => IRODSTimestampFeature.java} | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) rename irods/src/main/java/ch/cyberduck/core/irods/{IRODSTimestamp.java => IRODSTimestampFeature.java} (96%) diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSSession.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSSession.java index 8bc4802eafc..e5679e32232 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSSession.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSSession.java @@ -201,7 +201,7 @@ public T _getFeature(final Class type) { return (T) new IRODSWriteFeature(this); } if(type == Timestamp.class) { - return (T) new IRODSTimestamp(this); + return (T) new IRODSTimestampFeature(this); } return super._getFeature(type); } diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSTimestamp.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSTimestampFeature.java similarity index 96% rename from irods/src/main/java/ch/cyberduck/core/irods/IRODSTimestamp.java rename to irods/src/main/java/ch/cyberduck/core/irods/IRODSTimestampFeature.java index 4e7fa167ffe..93d081f9868 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSTimestamp.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSTimestampFeature.java @@ -34,13 +34,13 @@ import java.util.List; import java.util.concurrent.TimeUnit; -public class IRODSTimestamp implements Timestamp { +public class IRODSTimestampFeature implements Timestamp { - private static final Logger log = LogManager.getLogger(IRODSTimestamp.class); + private static final Logger log = LogManager.getLogger(IRODSTimestampFeature.class); private IRODSSession session; - public IRODSTimestamp(IRODSSession session) { + public IRODSTimestampFeature(IRODSSession session) { this.session = session; } From c3f25060f527247234888ca05317b7d032228338 Mon Sep 17 00:00:00 2001 From: Kory Draughn Date: Sun, 9 Nov 2025 22:09:04 -0500 Subject: [PATCH 06/20] squash w/ 2870ae7. give string literals names --- .../cyberduck/core/irods/IRODSAttributesFinderFeature.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSAttributesFinderFeature.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSAttributesFinderFeature.java index d4aeb6fe18a..7734623836b 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSAttributesFinderFeature.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSAttributesFinderFeature.java @@ -39,6 +39,9 @@ public class IRODSAttributesFinderFeature implements AttributesFinder, Attribute private static final Logger log = LogManager.getLogger(IRODSAttributesFinderFeature.class); + private static final String REPLICA_STATUS_GOOD = "1"; + private static final String REPLICA_STATUS_STALE = "0"; + private final IRODSSession session; public IRODSAttributesFinderFeature(final IRODSSession session) { @@ -68,7 +71,7 @@ public PathAttributes find(final Path file, final ListProgressListener listener) if(!rows.isEmpty()) { List row = rows.get(0); - if("0".equals(row.get(4)) || "1".equals(row.get(4))) { + if(REPLICA_STATUS_STALE.equals(row.get(4)) || REPLICA_STATUS_GOOD.equals(row.get(4))) { setAttributes(attrs, row); } } From fda6abb45da30914cc00416841379fa74e9303cb Mon Sep 17 00:00:00 2001 From: Kory Draughn Date: Sun, 9 Nov 2025 22:09:37 -0500 Subject: [PATCH 07/20] squash w/ 2870ae7. use convenience function for boolean --- .../main/java/ch/cyberduck/core/irods/IRODSDeleteFeature.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSDeleteFeature.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSDeleteFeature.java index 70af16c5043..3cc797ba0fd 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSDeleteFeature.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSDeleteFeature.java @@ -46,7 +46,7 @@ public IRODSDeleteFeature(IRODSSession session) { this.session = session; PreferencesReader prefs = HostPreferencesFactory.get(session.getHost()); - if("yes".equalsIgnoreCase(prefs.getProperty(IRODSProtocol.DELETE_OBJECTS_PERMANTENTLY))) { + if(prefs.getBoolean(IRODSProtocol.DELETE_OBJECTS_PERMANTENTLY)) { removeOptions = RemoveOptions.NO_TRASH; } else { From 1d455b9848cd666bf3bae5ae506b968a401d4dbc Mon Sep 17 00:00:00 2001 From: Kory Draughn Date: Sun, 9 Nov 2025 22:10:28 -0500 Subject: [PATCH 08/20] squash w/ 2870ae7. get app name dynamically --- irods/src/main/java/ch/cyberduck/core/irods/IRODSSession.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSSession.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSSession.java index e5679e32232..c45a784fc7e 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSSession.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSSession.java @@ -22,6 +22,7 @@ import ch.cyberduck.core.ListService; import ch.cyberduck.core.LocaleFactory; import ch.cyberduck.core.LoginCallback; +import ch.cyberduck.core.PreferencesUseragentProvider; import ch.cyberduck.core.exception.BackgroundException; import ch.cyberduck.core.exception.LoginFailureException; import ch.cyberduck.core.features.AttributesFinder; @@ -59,7 +60,7 @@ public class IRODSSession extends SSLSession { static { - IRODSApi.setApplicationName("Cyberduck"); + IRODSApi.setApplicationName(new PreferencesUseragentProvider().get()); } private static final Logger log = LogManager.getLogger(IRODSSession.class); From 4b2809089e1b1e64dc3e854de7c76058a66104cc Mon Sep 17 00:00:00 2001 From: Kory Draughn Date: Sun, 9 Nov 2025 22:11:16 -0500 Subject: [PATCH 09/20] squash w/ 2870ae7. remove override for logout() method --- .../src/main/java/ch/cyberduck/core/irods/IRODSSession.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSSession.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSSession.java index c45a784fc7e..80177c87409 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSSession.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSSession.java @@ -140,12 +140,6 @@ public void login(final LoginCallback prompt, final CancelCallback cancel) throw } } - @Override - protected void logout() { - // iRODS does not provide a logout operation. - // It only supports connecting, authenticating, and disconnecting. - } - @Override protected void disconnect() { try { From 8bed7e5abe563d9c376478de47079071cee8b8d7 Mon Sep 17 00:00:00 2001 From: Kory Draughn Date: Sun, 9 Nov 2025 22:12:45 -0500 Subject: [PATCH 10/20] squash w/ 2870ae7. wrap call to super.disconnect() in try-catch block --- .../main/java/ch/cyberduck/core/irods/IRODSSession.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSSession.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSSession.java index 80177c87409..26b8bda712c 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSSession.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSSession.java @@ -153,7 +153,12 @@ protected void disconnect() { log.error(e.getMessage()); } - super.disconnect(); + try { + super.disconnect(); + } + catch(BackgroundException e) { + log.error(e.getMessage()); + } } @Override From 2a5ae405f26ce4005dd31ee4655190737c21d023 Mon Sep 17 00:00:00 2001 From: Kory Draughn Date: Mon, 10 Nov 2025 14:46:10 -0500 Subject: [PATCH 11/20] squash w/ 2870ae7. use enum --- .../core/irods/IRODSConnectionUtils.java | 44 +++++++++++++++---- 1 file changed, 35 insertions(+), 9 deletions(-) diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSConnectionUtils.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSConnectionUtils.java index 5eabc8ad09e..918589ef003 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSConnectionUtils.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSConnectionUtils.java @@ -36,6 +36,27 @@ final class IRODSConnectionUtils { private static final Logger log = LogManager.getLogger(IRODSConnectionUtils.class); + private enum AuthScheme { + NATIVE, + PAM_PASSWORD; + + public static AuthScheme fromString(String s) { + if(null == s) { + throw new IllegalArgumentException("Cannot convert null to AuthScheme"); + } + + if("native".equalsIgnoreCase(s) || "standard".equalsIgnoreCase(s)) { + return NATIVE; + } + + if("pam_password".equalsIgnoreCase(s)) { + return PAM_PASSWORD; + } + + throw new IllegalArgumentException(String.format("Cannot convert [%s] to AuthScheme", s)); + } + } + public static IRODSApi.ConnectionOptions initConnectionOptions(IRODSSession session) { log.debug("configuring iRODS connection."); final PreferencesReader preferences = HostPreferencesFactory.get(session.getHost()); @@ -61,15 +82,20 @@ public static IRODSApi.ConnectionOptions initConnectionOptions(IRODSSession sess public static AuthPlugin newAuthPlugin(IRODSSession session) { AuthPlugin plugin = null; - final String authScheme = StringUtils.defaultIfBlank(session.getHost().getProtocol().getAuthorization(), "native"); - if("native".equalsIgnoreCase(authScheme) || "standard".equalsIgnoreCase(authScheme)) { - plugin = new NativeAuthPlugin(); - } - else if("pam_password".equalsIgnoreCase(authScheme)) { - plugin = new PamPasswordAuthPlugin(true); - } - else { - throw new IllegalArgumentException(String.format("Authentication scheme not recognized: %s", authScheme)); + final String authSchemeStr = StringUtils.defaultIfBlank(session.getHost().getProtocol().getAuthorization(), "native"); + final AuthScheme authScheme = AuthScheme.fromString(authSchemeStr); + switch(authScheme) { + case NATIVE: + plugin = new NativeAuthPlugin(); + break; + + case PAM_PASSWORD: + plugin = new PamPasswordAuthPlugin(true); + break; + + default: + // Should never get here. + throw new IllegalArgumentException("Cannot resolve authentication scheme to plugin implementation"); } return plugin; From 27f4dc9c3f5ce3792eaf03f385313fe3b828caf5 Mon Sep 17 00:00:00 2001 From: Kory Draughn Date: Mon, 10 Nov 2025 14:47:41 -0500 Subject: [PATCH 12/20] squash w/ 2870ae7. react to status.isExists() --- .../java/ch/cyberduck/core/irods/IRODSCopyFeature.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSCopyFeature.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSCopyFeature.java index 0718e7f17ec..e672e477546 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSCopyFeature.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSCopyFeature.java @@ -46,9 +46,12 @@ public Path copy(final Path source, final Path target, final TransferStatus stat final String from = source.getAbsolute(); final String to = target.getAbsolute(); - // TODO If we're dealing with a collection, should existing data objects sharing - // the same name be overwritten? This should probably be a configurable option. - IRODSFilesystem.copy(conn.getRcComm(), from, to, IRODSFilesystem.CopyOptions.RECURSIVE); + int options = IRODSFilesystem.CopyOptions.RECURSIVE; + if(status.isExists()) { + options |= IRODSFilesystem.CopyOptions.OVERWRITE_EXISTING; + } + + IRODSFilesystem.copy(conn.getRcComm(), from, to, options); return target; } From b3eb7f85500222be0d9a90475204d21decd35037 Mon Sep 17 00:00:00 2001 From: Kory Draughn Date: Mon, 10 Nov 2025 14:47:59 -0500 Subject: [PATCH 13/20] squash w/ 2870ae7. add test for checksum utils --- .../core/irods/IRODSChecksumUtilsTest.java | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 irods/src/test/java/ch/cyberduck/core/irods/IRODSChecksumUtilsTest.java diff --git a/irods/src/test/java/ch/cyberduck/core/irods/IRODSChecksumUtilsTest.java b/irods/src/test/java/ch/cyberduck/core/irods/IRODSChecksumUtilsTest.java new file mode 100644 index 00000000000..154adf205a5 --- /dev/null +++ b/irods/src/test/java/ch/cyberduck/core/irods/IRODSChecksumUtilsTest.java @@ -0,0 +1,49 @@ +package ch.cyberduck.core.irods; + +/* + * Copyright (c) 2002-2015 David Kocher. All rights reserved. + * http://cyberduck.ch/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Bug fixes, suggestions and comments should be sent to feedback@cyberduck.ch + */ + +import ch.cyberduck.core.io.Checksum; + +import ch.cyberduck.core.io.HashAlgorithm; + +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.codec.binary.Hex; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class IRODSChecksumUtilsTest { + + @Test + public void testInvalidInputs() { + assertEquals(Checksum.NONE, IRODSChecksumUtils.toChecksum(null)); + assertEquals(Checksum.NONE, IRODSChecksumUtils.toChecksum("")); + assertEquals(Checksum.NONE, IRODSChecksumUtils.toChecksum("no_colon")); + assertEquals(Checksum.NONE, IRODSChecksumUtils.toChecksum("sha2:")); + } + + @Test + public void testValidChecksum() { + final String irodsChecksum = "sha2:Qu9ZCAIUVS3AR4gaILMZevr5PK7XAO9vqlhSF0u+ha0="; + final String checksumNoPrefix = irodsChecksum.substring(irodsChecksum.indexOf(':') + 1); + final String hexEncodedChecksum = Hex.encodeHexString(Base64.decodeBase64(checksumNoPrefix)); + final Checksum expectedChecksum = Checksum.parse(hexEncodedChecksum); + assertEquals(expectedChecksum, IRODSChecksumUtils.toChecksum(irodsChecksum)); + assertEquals(HashAlgorithm.sha256, expectedChecksum.algorithm); + } +} From 78664bdcd4b31c70265a46dc42b136d3dcac3d4f Mon Sep 17 00:00:00 2001 From: Kory Draughn Date: Mon, 10 Nov 2025 14:51:13 -0500 Subject: [PATCH 14/20] squash w/ 2870ae7. lock exception mapper to IRODSException --- .../irods/IRODSExceptionMappingService.java | 72 +++++++++---------- 1 file changed, 35 insertions(+), 37 deletions(-) diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSExceptionMappingService.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSExceptionMappingService.java index 32f22de554a..da5dfa0840a 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSExceptionMappingService.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSExceptionMappingService.java @@ -32,51 +32,49 @@ import org.irods.irods4j.low_level.api.IRODSErrorCodes; import org.irods.irods4j.low_level.api.IRODSException; -public class IRODSExceptionMappingService extends AbstractExceptionMappingService { +public class IRODSExceptionMappingService extends AbstractExceptionMappingService { private static final Logger log = LogManager.getLogger(IRODSExceptionMappingService.class); @Override - public BackgroundException map(final Exception e) { + public BackgroundException map(final IRODSException e) { log.warn("Map failure {}", e.toString()); final StringBuilder buffer = new StringBuilder(); this.append(buffer, e.getMessage()); - if(e instanceof IRODSException) { - switch(((IRODSException) e).getErrorCode()) { - case IRODSErrorCodes.AUTHENTICATION_ERROR: - case IRODSErrorCodes.CAT_INVALID_AUTHENTICATION: - case IRODSErrorCodes.CAT_PASSWORD_EXPIRED: - case IRODSErrorCodes.CAT_INVALID_USER: - return new LoginFailureException(buffer.toString(), e); - - case IRODSErrorCodes.CAT_NO_ACCESS_PERMISSION: - case IRODSErrorCodes.CAT_INSUFFICIENT_PRIVILEGE_LEVEL: - case IRODSErrorCodes.SYS_NOT_ALLOWED: - return new AccessDeniedException(buffer.toString(), e); - - case IRODSErrorCodes.INTERMEDIATE_REPLICA_ACCESS: - case IRODSErrorCodes.SYS_REPLICA_INACCESSIBLE: - return new LockedException(buffer.toString(), e); - - case IRODSErrorCodes.SSL_CERT_ERROR: - case IRODSErrorCodes.SSL_HANDSHAKE_ERROR: - case IRODSErrorCodes.SSL_INIT_ERROR: - case IRODSErrorCodes.SSL_SHUTDOWN_ERROR: - case IRODSErrorCodes.SSL_NOT_BUILT_INTO_CLIENT: - case IRODSErrorCodes.SSL_NOT_BUILT_INTO_SERVER: - return new SSLNegotiateException(buffer.toString(), e); - - case IRODSErrorCodes.SYS_RESC_QUOTA_EXCEEDED: - return new QuotaException(buffer.toString(), e); - - case IRODSErrorCodes.CAT_NO_ROWS_FOUND: - case IRODSErrorCodes.CAT_NOT_A_DATAOBJ_AND_NOT_A_COLLECTION: - return new NotfoundException(buffer.toString(), e); - - case IRODSErrorCodes.CONNECTION_REFUSED: - return new ConnectionRefusedException(buffer.toString(), e); - } + switch(e.getErrorCode()) { + case IRODSErrorCodes.AUTHENTICATION_ERROR: + case IRODSErrorCodes.CAT_INVALID_AUTHENTICATION: + case IRODSErrorCodes.CAT_PASSWORD_EXPIRED: + case IRODSErrorCodes.CAT_INVALID_USER: + return new LoginFailureException(buffer.toString(), e); + + case IRODSErrorCodes.CAT_NO_ACCESS_PERMISSION: + case IRODSErrorCodes.CAT_INSUFFICIENT_PRIVILEGE_LEVEL: + case IRODSErrorCodes.SYS_NOT_ALLOWED: + return new AccessDeniedException(buffer.toString(), e); + + case IRODSErrorCodes.INTERMEDIATE_REPLICA_ACCESS: + case IRODSErrorCodes.SYS_REPLICA_INACCESSIBLE: + return new LockedException(buffer.toString(), e); + + case IRODSErrorCodes.SSL_CERT_ERROR: + case IRODSErrorCodes.SSL_HANDSHAKE_ERROR: + case IRODSErrorCodes.SSL_INIT_ERROR: + case IRODSErrorCodes.SSL_SHUTDOWN_ERROR: + case IRODSErrorCodes.SSL_NOT_BUILT_INTO_CLIENT: + case IRODSErrorCodes.SSL_NOT_BUILT_INTO_SERVER: + return new SSLNegotiateException(buffer.toString(), e); + + case IRODSErrorCodes.SYS_RESC_QUOTA_EXCEEDED: + return new QuotaException(buffer.toString(), e); + + case IRODSErrorCodes.CAT_NO_ROWS_FOUND: + case IRODSErrorCodes.CAT_NOT_A_DATAOBJ_AND_NOT_A_COLLECTION: + return new NotfoundException(buffer.toString(), e); + + case IRODSErrorCodes.CONNECTION_REFUSED: + return new ConnectionRefusedException(buffer.toString(), e); } return this.wrap(e, buffer); From 7b6176ab5ad62ab79f95a51f7e7afdbffe3c6194 Mon Sep 17 00:00:00 2001 From: Kory Draughn Date: Mon, 10 Nov 2025 14:52:31 -0500 Subject: [PATCH 15/20] squash w/ 2870ae7. use cyberduck threadpool for multipart upload. split exception handling logic --- .../core/irods/IRODSUploadFeature.java | 27 ++++++++++++------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSUploadFeature.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSUploadFeature.java index f0bb635db0d..10da289e5d5 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSUploadFeature.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSUploadFeature.java @@ -16,6 +16,7 @@ */ import ch.cyberduck.core.ConnectionCallback; +import ch.cyberduck.core.DefaultIOExceptionMappingService; import ch.cyberduck.core.Local; import ch.cyberduck.core.Path; import ch.cyberduck.core.ProgressListener; @@ -27,8 +28,12 @@ import ch.cyberduck.core.io.StreamListener; import ch.cyberduck.core.preferences.HostPreferencesFactory; import ch.cyberduck.core.preferences.PreferencesReader; +import ch.cyberduck.core.threading.ThreadPool; +import ch.cyberduck.core.threading.ThreadPoolFactory; import ch.cyberduck.core.transfer.TransferStatus; +import ch.cyberduck.core.worker.DefaultExceptionMappingService; + import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -37,16 +42,15 @@ import org.irods.irods4j.high_level.connection.IRODSConnectionPool.PoolConnection; import org.irods.irods4j.high_level.io.IRODSDataObjectOutputStream; import org.irods.irods4j.high_level.io.IRODSDataObjectStream; +import org.irods.irods4j.low_level.api.IRODSException; import java.io.FileInputStream; +import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.List; import java.util.Optional; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; public class IRODSUploadFeature implements Upload { @@ -145,7 +149,7 @@ public Void upload(final Write write, final Path file, final Local local, final int threadCount = IRODSIntegerUtils.clamp( preferences.getInteger(IRODSProtocol.PARALLEL_TRANSFER_CONNECTIONS), 2, 10); log.debug("thread count = [{}]; starting thread pool.", threadCount); - final ExecutorService executor = Executors.newFixedThreadPool(threadCount); + final ThreadPool threadPool = ThreadPoolFactory.get("multipart-iRODS", threadCount); final long chunkSize = fileSize / threadCount; final long remainingBytes = fileSize % threadCount; @@ -220,7 +224,7 @@ public Void upload(final Write write, final Path file, final Local local, // Launch remaining IO tasks. log.debug("launch parallel IO tasks."); for(int i = 0; i < threadCount; ++i) { - tasks.add(executor.submit(new IRODSChunkWorker( + tasks.add(threadPool.execute(new IRODSChunkWorker( status, streamListener, localFileStreams.get(i), @@ -243,16 +247,21 @@ public Void upload(final Write write, final Path file, final Local local, } } - log.debug("shutting down thread pool executor."); - executor.shutdown(); - executor.awaitTermination(5, TimeUnit.SECONDS); + log.debug("shutting down thread pool."); + threadPool.shutdown(false); log.debug("done."); return null; } - catch(Exception e) { + catch(IRODSException e) { throw new IRODSExceptionMappingService().map(e); } + catch(IOException e) { + throw new DefaultIOExceptionMappingService().map(e); + } + catch(Exception e) { + throw new DefaultExceptionMappingService().map(e); + } } @Override From 12beedaa93792f0f942c0ed3c915d7e9830a5263 Mon Sep 17 00:00:00 2001 From: Kory Draughn Date: Mon, 10 Nov 2025 14:54:37 -0500 Subject: [PATCH 16/20] squash w/ 2870ae7. handle redundant variables. split exception handling logic --- .../ch/cyberduck/core/irods/IRODSSession.java | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSSession.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSSession.java index 26b8bda712c..f550a00b486 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSSession.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSSession.java @@ -48,12 +48,15 @@ import ch.cyberduck.core.ssl.X509TrustManager; import ch.cyberduck.core.threading.CancelCallback; +import ch.cyberduck.core.worker.DefaultExceptionMappingService; + import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.irods.irods4j.high_level.connection.IRODSConnection; import org.irods.irods4j.high_level.connection.QualifiedUsername; import org.irods.irods4j.low_level.api.IRODSApi; +import org.irods.irods4j.low_level.api.IRODSException; import java.text.MessageFormat; @@ -80,14 +83,13 @@ public boolean isConnected() { @Override protected IRODSConnection connect(final ProxyFinder proxy, final HostKeyCallback key, final LoginCallback prompt, final CancelCallback cancel) throws BackgroundException { + final String host = this.host.getHostname(); + final int port = this.host.getPort(); + final String username = this.host.getCredentials().getUsername(); + final String zone = getRegion(); + try { log.debug("connecting to iRODS server."); - - final String host = this.host.getHostname(); - final int port = this.host.getPort(); - final String username = this.host.getCredentials().getUsername(); - final String zone = getRegion(); - log.debug("iRODS server: host=[{}], port=[{}], username=[{}], zone=[{}]", host, port, username, zone); IRODSConnection conn = new IRODSConnection(IRODSConnectionUtils.initConnectionOptions(this)); @@ -96,16 +98,16 @@ protected IRODSConnection connect(final ProxyFinder proxy, final HostKeyCallback return conn; } - catch(Exception e) { - final String host = this.host.getHostname(); - final int port = this.host.getPort(); - final String username = this.host.getCredentials().getUsername(); - final String zone = getRegion(); - + catch(IRODSException e) { String msg = String.format("Could not connect to iRODS server at [%s:%d] as [%s#%s]: %s", host, port, username, zone, e.getMessage()); throw new IRODSExceptionMappingService().map(msg, e); } + catch(Exception e) { + String msg = String.format("Problem connecting to iRODS server at [%s:%d] as [%s#%s]: %s", + host, port, username, zone, e.getMessage()); + throw new DefaultExceptionMappingService().map(msg, e); + } } protected String getRegion() { From d6d3f9b11ba479e7767165c8e30549477992c699 Mon Sep 17 00:00:00 2001 From: Kory Draughn Date: Mon, 10 Nov 2025 14:58:40 -0500 Subject: [PATCH 17/20] squash w/ 2870ae7. split exception handling logic in feature impls --- .../cyberduck/core/irods/IRODSAttributesFinderFeature.java | 6 +++++- .../main/java/ch/cyberduck/core/irods/IRODSCopyFeature.java | 6 +++++- .../java/ch/cyberduck/core/irods/IRODSDeleteFeature.java | 6 +++++- .../java/ch/cyberduck/core/irods/IRODSDirectoryFeature.java | 6 +++++- .../main/java/ch/cyberduck/core/irods/IRODSFindFeature.java | 6 +++++- .../main/java/ch/cyberduck/core/irods/IRODSListService.java | 6 +++++- .../main/java/ch/cyberduck/core/irods/IRODSMoveFeature.java | 6 +++++- .../main/java/ch/cyberduck/core/irods/IRODSReadFeature.java | 6 +++++- .../java/ch/cyberduck/core/irods/IRODSTimestampFeature.java | 6 +++++- .../java/ch/cyberduck/core/irods/IRODSTouchFeature.java | 6 +++++- .../java/ch/cyberduck/core/irods/IRODSWriteFeature.java | 6 +++++- 11 files changed, 55 insertions(+), 11 deletions(-) diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSAttributesFinderFeature.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSAttributesFinderFeature.java index 7734623836b..67dc93e433d 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSAttributesFinderFeature.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSAttributesFinderFeature.java @@ -15,6 +15,7 @@ * GNU General Public License for more details. */ +import ch.cyberduck.core.DefaultIOExceptionMappingService; import ch.cyberduck.core.ListProgressListener; import ch.cyberduck.core.Path; import ch.cyberduck.core.PathAttributes; @@ -102,9 +103,12 @@ public PathAttributes find(final Path file, final ListProgressListener listener) throw new NotfoundException(logicalPath); } - catch(IOException | IRODSException e) { + catch(IRODSException e) { throw new IRODSExceptionMappingService().map("Failure to read attributes of {0}", e, file); } + catch(IOException e) { + throw new DefaultIOExceptionMappingService().map("Failure to read attributes of {0}", e, file); + } } @Override diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSCopyFeature.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSCopyFeature.java index e672e477546..30674f5e9ae 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSCopyFeature.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSCopyFeature.java @@ -16,6 +16,7 @@ */ import ch.cyberduck.core.ConnectionCallback; +import ch.cyberduck.core.DefaultIOExceptionMappingService; import ch.cyberduck.core.Path; import ch.cyberduck.core.exception.BackgroundException; import ch.cyberduck.core.features.Copy; @@ -55,9 +56,12 @@ public Path copy(final Path source, final Path target, final TransferStatus stat return target; } - catch(IOException | IRODSException e) { + catch(IRODSException e) { throw new IRODSExceptionMappingService().map("Cannot copy {0}", e, source); } + catch(IOException e) { + throw new DefaultIOExceptionMappingService().map("Cannot copy {0}", e, source); + } } @Override diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSDeleteFeature.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSDeleteFeature.java index 3cc797ba0fd..3bf9d376358 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSDeleteFeature.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSDeleteFeature.java @@ -15,6 +15,7 @@ * GNU General Public License for more details. */ +import ch.cyberduck.core.DefaultIOExceptionMappingService; import ch.cyberduck.core.PasswordCallback; import ch.cyberduck.core.Path; import ch.cyberduck.core.exception.BackgroundException; @@ -89,9 +90,12 @@ else if(status.getType() == ObjectType.COLLECTION) { IRODSFilesystem.removeAll(conn.getRcComm(), logicalPath, removeOptions); } } - catch(IOException | IRODSException e) { + catch(IRODSException e) { throw new IRODSExceptionMappingService().map("Cannot delete {0}", e, file); } + catch(IOException e) { + throw new DefaultIOExceptionMappingService().map("Cannot delete {0}", e, file); + } } } diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSDirectoryFeature.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSDirectoryFeature.java index 3f473ea686b..746990a36b9 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSDirectoryFeature.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSDirectoryFeature.java @@ -15,6 +15,7 @@ * GNU General Public License for more details. */ +import ch.cyberduck.core.DefaultIOExceptionMappingService; import ch.cyberduck.core.Path; import ch.cyberduck.core.exception.BackgroundException; import ch.cyberduck.core.features.Directory; @@ -42,8 +43,11 @@ public Path mkdir(final Write writer, final Path folder, final TransferSta IRODSFilesystem.createCollection(conn.getRcComm(), folder.getAbsolute()); return folder; } - catch(IOException | IRODSFilesystemException e) { + catch(IRODSFilesystemException e) { throw new IRODSExceptionMappingService().map("Cannot create folder {0}", e, folder); } + catch(IOException e) { + throw new DefaultIOExceptionMappingService().map("Cannot create folder {0}", e, folder); + } } } diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSFindFeature.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSFindFeature.java index 52364eb011b..78ca3b86ea8 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSFindFeature.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSFindFeature.java @@ -15,6 +15,7 @@ * GNU General Public License for more details. */ +import ch.cyberduck.core.DefaultIOExceptionMappingService; import ch.cyberduck.core.ListProgressListener; import ch.cyberduck.core.Path; import ch.cyberduck.core.exception.BackgroundException; @@ -44,8 +45,11 @@ public boolean find(final Path file, final ListProgressListener listener) throws final IRODSConnection conn = session.getClient(); return IRODSFilesystem.exists(conn.getRcComm(), file.getAbsolute()); } - catch(IOException | IRODSException e) { + catch(IRODSException e) { throw new IRODSExceptionMappingService().map("Failure to find {0}", e, file); } + catch(IOException e) { + throw new DefaultIOExceptionMappingService().map("Failure to find {0}", e, file); + } } } diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSListService.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSListService.java index 64117f4ee77..228a8f94bef 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSListService.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSListService.java @@ -16,6 +16,7 @@ */ import ch.cyberduck.core.AttributedList; +import ch.cyberduck.core.DefaultIOExceptionMappingService; import ch.cyberduck.core.ListProgressListener; import ch.cyberduck.core.ListService; import ch.cyberduck.core.Path; @@ -82,8 +83,11 @@ else if(entry.isDataObject()) { return children; } - catch(IRODSException | IOException e) { + catch(IRODSException e) { throw new IRODSExceptionMappingService().map("Listing {0} failed", e, directory); } + catch(IOException e) { + throw new DefaultIOExceptionMappingService().map("Listing {0} failed", e, directory); + } } } diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSMoveFeature.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSMoveFeature.java index 537a852efe1..5bdac39520a 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSMoveFeature.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSMoveFeature.java @@ -16,6 +16,7 @@ */ import ch.cyberduck.core.ConnectionCallback; +import ch.cyberduck.core.DefaultIOExceptionMappingService; import ch.cyberduck.core.Path; import ch.cyberduck.core.exception.BackgroundException; import ch.cyberduck.core.exception.NotfoundException; @@ -55,9 +56,12 @@ public Path move(final Path file, final Path renamed, final TransferStatus statu IRODSFilesystem.rename(conn.getRcComm(), file.getAbsolute(), renamed.getAbsolute()); return renamed; } - catch(IOException | IRODSException e) { + catch(IRODSException e) { throw new IRODSExceptionMappingService().map("Cannot rename {0}", e, file); } + catch(IOException e) { + throw new DefaultIOExceptionMappingService().map("Cannot rename {0}", e, file); + } } @Override diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSReadFeature.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSReadFeature.java index 19c27e1da68..229d1008dea 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSReadFeature.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSReadFeature.java @@ -16,6 +16,7 @@ */ import ch.cyberduck.core.ConnectionCallback; +import ch.cyberduck.core.DefaultIOExceptionMappingService; import ch.cyberduck.core.Path; import ch.cyberduck.core.exception.BackgroundException; import ch.cyberduck.core.exception.NotfoundException; @@ -56,9 +57,12 @@ public InputStream read(final Path file, final TransferStatus status, final Conn return in; } - catch(IOException | IRODSException e) { + catch(IRODSException e) { throw new IRODSExceptionMappingService().map("Download of {0} failed", e, file); } + catch(IOException e) { + throw new DefaultIOExceptionMappingService().map("Download of {0} failed", e, file); + } } @Override diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSTimestampFeature.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSTimestampFeature.java index 93d081f9868..02c90195587 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSTimestampFeature.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSTimestampFeature.java @@ -15,6 +15,7 @@ * GNU General Public License for more details. */ +import ch.cyberduck.core.DefaultIOExceptionMappingService; import ch.cyberduck.core.Path; import ch.cyberduck.core.exception.BackgroundException; import ch.cyberduck.core.features.Timestamp; @@ -74,9 +75,12 @@ else if(IRODSFilesystem.isCollection(objectStatus)) { log.debug("timestamp set to [{}] seconds (since epoch) on [{}] successfully.", seconds, logicalPath); } } - catch(IOException | IRODSException e) { + catch(IRODSException e) { throw new IRODSExceptionMappingService().map(e); } + catch(IOException e) { + throw new DefaultIOExceptionMappingService().map(e); + } } private long getReplicaNumberOfLatestGoodReplica(String logicalPath) throws IRODSException, IOException { diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSTouchFeature.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSTouchFeature.java index df8782d9d55..608fc277e79 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSTouchFeature.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSTouchFeature.java @@ -15,6 +15,7 @@ * GNU General Public License for more details. */ +import ch.cyberduck.core.DefaultIOExceptionMappingService; import ch.cyberduck.core.Path; import ch.cyberduck.core.exception.BackgroundException; import ch.cyberduck.core.features.Touch; @@ -58,9 +59,12 @@ public Path touch(final Write writer, final Path file, final TransferStatus stat return file; } - catch(IOException | IRODSException e) { + catch(IRODSException e) { throw new IRODSExceptionMappingService().map("Cannot create {0}", e, file); } + catch(IOException e) { + throw new DefaultIOExceptionMappingService().map("Cannot create {0}", e, file); + } } @Override diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSWriteFeature.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSWriteFeature.java index 77d6efe4667..906c56abd01 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSWriteFeature.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSWriteFeature.java @@ -16,6 +16,7 @@ */ import ch.cyberduck.core.ConnectionCallback; +import ch.cyberduck.core.DefaultIOExceptionMappingService; import ch.cyberduck.core.Path; import ch.cyberduck.core.exception.BackgroundException; import ch.cyberduck.core.features.Write; @@ -51,8 +52,11 @@ public StatusOutputStream write(final Path file, final TransferStatus stat final OutputStream out = new IRODSDataObjectOutputStream(conn.getRcComm(), file.getAbsolute(), truncate, append); return new VoidStatusOutputStream(out); } - catch(IRODSException | IOException e) { + catch(IRODSException e) { throw new IRODSExceptionMappingService().map("Uploading {0} failed", e, file); } + catch(IOException e) { + throw new DefaultIOExceptionMappingService().map("Uploading {0} failed", e, file); + } } } From 907f492c8edf5a165113aca426bc116eea351721 Mon Sep 17 00:00:00 2001 From: Kory Draughn Date: Thu, 27 Nov 2025 13:13:32 -0500 Subject: [PATCH 18/20] squash w/ 99b1ac6. test infra for pam --- .../resources/docker/docker-compose.pam.yml | 27 +++ .../docker/irods_catalog_provider/Dockerfile | 1 + .../irods_catalog_provider_pam/Dockerfile | 62 ++++++ .../irods_catalog_provider_pam/entrypoint.sh | 44 +++++ .../irods_catalog_provider_pam/pam_password | 7 + .../unattended_install.json | 180 ++++++++++++++++++ .../test/resources/iRODS_pam.cyberduckprofile | 39 ++++ 7 files changed, 360 insertions(+) create mode 100644 irods/src/test/resources/docker/docker-compose.pam.yml create mode 100644 irods/src/test/resources/docker/irods_catalog_provider_pam/Dockerfile create mode 100644 irods/src/test/resources/docker/irods_catalog_provider_pam/entrypoint.sh create mode 100644 irods/src/test/resources/docker/irods_catalog_provider_pam/pam_password create mode 100644 irods/src/test/resources/docker/irods_catalog_provider_pam/unattended_install.json create mode 100644 irods/src/test/resources/iRODS_pam.cyberduckprofile diff --git a/irods/src/test/resources/docker/docker-compose.pam.yml b/irods/src/test/resources/docker/docker-compose.pam.yml new file mode 100644 index 00000000000..62d414af2cb --- /dev/null +++ b/irods/src/test/resources/docker/docker-compose.pam.yml @@ -0,0 +1,27 @@ +name: cyberduck-irods-testing + +services: + irods-catalog: + build: + context: irods_catalog + environment: + - POSTGRES_PASSWORD=testpassword + restart: always + + irods-catalog-provider: + build: + context: irods_catalog_provider_pam + init: true + ports: + - "1347:1247" + shm_size: 100mb + healthcheck: + test: ["CMD-SHELL", "su - irods -c 'ils || exit 1'"] + interval: 10s + timeout: 10s + retries: 3 + start_period: 20s + start_interval: 10s + restart: always + depends_on: + - irods-catalog \ No newline at end of file diff --git a/irods/src/test/resources/docker/irods_catalog_provider/Dockerfile b/irods/src/test/resources/docker/irods_catalog_provider/Dockerfile index 8836f0fea61..451ef1397ec 100644 --- a/irods/src/test/resources/docker/irods_catalog_provider/Dockerfile +++ b/irods/src/test/resources/docker/irods_catalog_provider/Dockerfile @@ -1,5 +1,6 @@ FROM ubuntu:24.04 +SHELL [ "/bin/bash", "-c" ] ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update && \ diff --git a/irods/src/test/resources/docker/irods_catalog_provider_pam/Dockerfile b/irods/src/test/resources/docker/irods_catalog_provider_pam/Dockerfile new file mode 100644 index 00000000000..63be835a607 --- /dev/null +++ b/irods/src/test/resources/docker/irods_catalog_provider_pam/Dockerfile @@ -0,0 +1,62 @@ +FROM ubuntu:24.04 + +SHELL [ "/bin/bash", "-c" ] +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && \ + apt-get install -y \ + apt-transport-https \ + gnupg \ + wget \ + netcat-traditional \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* + +RUN mkdir -p /etc/apt/keyrings && \ + wget -qO - https://packages.irods.org/irods-signing-key.asc | \ + gpg \ + --no-options \ + --no-default-keyring \ + --no-auto-check-trustdb \ + --homedir /dev/null \ + --no-keyring \ + --import-options import-export \ + --output /etc/apt/keyrings/renci-irods-archive-keyring.pgp \ + --import \ + && \ + echo "deb [signed-by=/etc/apt/keyrings/renci-irods-archive-keyring.pgp arch=amd64] https://packages.irods.org/apt/ noble main" | \ + tee /etc/apt/sources.list.d/renci-irods.list + +RUN apt-get update && \ + apt-get install -y \ + libcurl4-gnutls-dev \ + python3 \ + python3-distro \ + python3-jsonschema \ + python3-pip \ + python3-psutil \ + python3-requests \ + rsyslog \ + unixodbc \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* + +ARG irods_version=5.0.2 +ARG irods_package_version_suffix=-0~noble +ARG irods_package_version=${irods_version}${irods_package_version_suffix} + +RUN apt-get update && \ + apt-get install -y \ + irods-database-plugin-postgres=${irods_package_version} \ + irods-runtime=${irods_package_version} \ + irods-server=${irods_package_version} \ + irods-icommands=${irods_package_version} \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* + +COPY unattended_install.json pam_password / +COPY --chmod=755 entrypoint.sh / +ENTRYPOINT ["./entrypoint.sh"] \ No newline at end of file diff --git a/irods/src/test/resources/docker/irods_catalog_provider_pam/entrypoint.sh b/irods/src/test/resources/docker/irods_catalog_provider_pam/entrypoint.sh new file mode 100644 index 00000000000..ee7f474c303 --- /dev/null +++ b/irods/src/test/resources/docker/irods_catalog_provider_pam/entrypoint.sh @@ -0,0 +1,44 @@ +#! /bin/bash -e + +echo "Waiting for iRODS catalog database to be ready" +catalog_db_hostname=irods-catalog +until pg_isready -h ${catalog_db_hostname} -d ICAT -U irods -q; do + sleep 1 +done +echo "iRODS catalog database is ready" + +unattended_install_file=/unattended_install.json +if [ -f "${unattended_install_file}" ]; then + echo "Running iRODS setup" + + # Configure the server with secure communication (TLS) disabled. This + # avoids issues with setup (e.g. the post install test). + + # Add generated hostname as a recognizable alias. + sed -i "s/CONTAINER_HOSTNAME_ALIAS/${HOSTNAME}/g" ${unattended_install_file} + python3 /var/lib/irods/scripts/setup_irods.py --json_configuration_file ${unattended_install_file} + + # Move the input file used to configure the server out of the way so + # the container is restartable. + mv ${unattended_install_file} ${unattended_install_file}.processed + + # Generate a self-signed certificate for the server. + openssl genrsa -out /tmp/irods_server.key 2048 + openssl req -batch -new -x509 -key /tmp/irods_server.key -out /tmp/irods_server.crt -days 1 + openssl dhparam -2 -out /tmp/irods_dhparams.pem 2048 + chown irods:irods /tmp/irods_server.key + + # Update the server's configuration to require secure communication. + sed -i 's/="CS_NEG_REFUSE"/="CS_NEG_REQUIRE"/g' /etc/irods/core.re + sed -i 's/CS_NEG_REFUSE/CS_NEG_REQUIRE/g' /etc/irods/server_config.json /var/lib/irods/.irods/irods_environment.json + + # Configure PAM. + ln -s /pam_password /etc/pam.d/irods + + # Set up the test user, for PAM authentication. + useradd -m john + echo "john:=i;r@o\\d&s" | chpasswd +fi + +echo "Starting server" +su - irods -c 'irodsServer --stdout' diff --git a/irods/src/test/resources/docker/irods_catalog_provider_pam/pam_password b/irods/src/test/resources/docker/irods_catalog_provider_pam/pam_password new file mode 100644 index 00000000000..637c8cf0daf --- /dev/null +++ b/irods/src/test/resources/docker/irods_catalog_provider_pam/pam_password @@ -0,0 +1,7 @@ +# This file is for testing PAM authentication with iRODS +# using the pam_password authentication scheme. + +auth required pam_env.so +auth sufficient pam_unix.so +auth requisite pam_succeed_if.so uid >= 500 quiet +auth required pam_deny.so diff --git a/irods/src/test/resources/docker/irods_catalog_provider_pam/unattended_install.json b/irods/src/test/resources/docker/irods_catalog_provider_pam/unattended_install.json new file mode 100644 index 00000000000..8820a53787e --- /dev/null +++ b/irods/src/test/resources/docker/irods_catalog_provider_pam/unattended_install.json @@ -0,0 +1,180 @@ +{ + "admin_password": "rods", + "default_resource_directory": "/var/lib/irods/Vault", + "default_resource_name": "demoResc", + "host_system_information": { + "service_account_user_name": "irods", + "service_account_group_name": "irods" + }, + "service_account_environment": { + "irods_client_server_policy": "CS_NEG_REFUSE", + "irods_connection_pool_refresh_time_in_seconds": 300, + "irods_cwd": "/tempZone/home/rods", + "irods_default_hash_scheme": "SHA256", + "irods_default_number_of_transfer_threads": 4, + "irods_default_resource": "demoResc", + "irods_encryption_algorithm": "AES-256-CBC", + "irods_encryption_key_size": 32, + "irods_encryption_num_hash_rounds": 16, + "irods_encryption_salt_size": 8, + "irods_home": "/tempZone/home/rods", + "irods_host": "irods-catalog-provider", + "irods_match_hash_policy": "compatible", + "irods_maximum_size_for_single_buffer_in_megabytes": 32, + "irods_port": 1247, + "irods_ssl_ca_certificate_file": "/tmp/irods_server.crt", + "irods_ssl_verify_server": "none", + "irods_transfer_buffer_size_for_parallel_transfer_in_megabytes": 4, + "irods_user_name": "rods", + "irods_zone_name": "tempZone", + "schema_name": "service_account_environment", + "schema_version": "v5" + }, + "server_config": { + "advanced_settings": { + "checksum_read_buffer_size_in_bytes": 1048576, + "default_number_of_transfer_threads": 4, + "default_temporary_password_lifetime_in_seconds": 120, + "delay_rule_executors": [], + "delay_server_sleep_time_in_seconds": 30, + "dns_cache": { + "eviction_age_in_seconds": 3600, + "cache_clearer_sleep_time_in_seconds": 600, + "shared_memory_size_in_bytes": 5000000 + }, + "hostname_cache": { + "eviction_age_in_seconds": 3600, + "cache_clearer_sleep_time_in_seconds": 600, + "shared_memory_size_in_bytes": 2500000 + }, + "maximum_size_for_single_buffer_in_megabytes": 32, + "maximum_size_of_delay_queue_in_bytes": 0, + "maximum_temporary_password_lifetime_in_seconds": 1000, + "migrate_delay_server_sleep_time_in_seconds": 5, + "number_of_concurrent_delay_rule_executors": 4, + "stacktrace_file_processor_sleep_time_in_seconds": 10, + "transfer_buffer_size_for_parallel_transfer_in_megabytes": 4, + "transfer_chunk_size_for_parallel_transfer_in_megabytes": 40 + }, + "catalog_provider_hosts": [ + "irods-catalog-provider" + ], + "catalog_service_role": "provider", + "client_server_policy": "CS_NEG_REFUSE", + "connection_pool_refresh_time_in_seconds": 300, + "controlled_user_connection_list": { + "control_type": "denylist", + "users": [] + }, + "default_dir_mode": "0750", + "default_file_mode": "0600", + "default_hash_scheme": "SHA256", + "default_resource_name": "demoResc", + "encryption": { + "algorithm": "AES-256-CBC", + "key_size": 32, + "num_hash_rounds": 16, + "salt_size": 8 + }, + "environment_variables": {}, + "federation": [], + "graceful_shutdown_timeout_in_seconds": 30, + "host": "irods-catalog-provider", + "host_access_control": { + "access_entries": [] + }, + "host_resolution": { + "host_entries": [ + { + "address_type": "local", + "addresses": [ + "irods-catalog-provider", + "CONTAINER_HOSTNAME_ALIAS" + ] + } + ] + }, + "log_level": { + "agent": "info", + "agent_factory": "info", + "api": "info", + "authentication": "info", + "database": "info", + "delay_server": "info", + "genquery1": "info", + "genquery2": "info", + "legacy": "info", + "microservice": "info", + "network": "info", + "resource": "info", + "rule_engine": "info", + "server": "info", + "sql": "info" + }, + "match_hash_policy": "compatible", + "negotiation_key": "32_byte_server_negotiation_key__", + "plugin_configuration": { + "authentication": {}, + "database": { + "technology": "postgres", + "host": "irods-catalog", + "name": "ICAT", + "odbc_driver": "PostgreSQL ANSI", + "password": "testpassword", + "port": 5432, + "username": "irods" + }, + "network": {}, + "resource": {}, + "rule_engines": [ + { + "instance_name": "irods_rule_engine_plugin-irods_rule_language-instance", + "plugin_name": "irods_rule_engine_plugin-irods_rule_language", + "plugin_specific_configuration": { + "re_data_variable_mapping_set": [ + "core" + ], + "re_function_name_mapping_set": [ + "core" + ], + "re_rulebase_set": [ + "core" + ], + "regexes_for_supported_peps": [ + "ac[^ ]*", + "msi[^ ]*", + "[^ ]*pep_[^ ]*_(pre|post|except|finally)" + ] + }, + "shared_memory_instance": "irods_rule_language_rule_engine" + }, + { + "instance_name": "irods_rule_engine_plugin-cpp_default_policy-instance", + "plugin_name": "irods_rule_engine_plugin-cpp_default_policy", + "plugin_specific_configuration": {} + } + ] + }, + "rule_engine_namespaces": [ + "" + ], + "schema_name": "server_config", + "schema_version": "v5", + "server_port_range_end": 20199, + "server_port_range_start": 20000, + "tls_client": { + "ca_certificate_file": "/tmp/irods_server.crt", + "verify_server": "none" + }, + "tls_server": { + "certificate_chain_file": "/tmp/irods_server.crt", + "certificate_key_file": "/tmp/irods_server.key", + "dh_params_file": "/tmp/irods_dhparams.pem" + }, + "zone_auth_scheme": "native", + "zone_key": "TEMPORARY_ZONE_KEY", + "zone_name": "tempZone", + "zone_port": 1247, + "zone_user": "rods" + } +} \ No newline at end of file diff --git a/irods/src/test/resources/iRODS_pam.cyberduckprofile b/irods/src/test/resources/iRODS_pam.cyberduckprofile new file mode 100644 index 00000000000..8a62c5463fc --- /dev/null +++ b/irods/src/test/resources/iRODS_pam.cyberduckprofile @@ -0,0 +1,39 @@ + + + + + Protocol + irods + Vendor + iRODS Consortium + Description + iRODS (Integrated Rule-Oriented Data System) + + Hostname Configurable + + Port Configurable + + + Default Hostname + localhost + Default Port + 1347 + + Username Placeholder + iRODS username + Password Placeholder + iRODS password + + Region + tempZone + + Authorization + pam_password + + Properties + + Client Server Negotiation + CS_NEG_REQUIRE + + + From c8c84133c862b7b05ec618e688f68586e8422778 Mon Sep 17 00:00:00 2001 From: Kory Draughn Date: Thu, 27 Nov 2025 13:15:23 -0500 Subject: [PATCH 19/20] squash w/ 2870ae7. remove TLS/JKS options --- .../ch/cyberduck/core/irods/IRODSConnectionUtils.java | 10 +++++----- .../java/ch/cyberduck/core/irods/IRODSProtocol.java | 6 ------ 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSConnectionUtils.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSConnectionUtils.java index 918589ef003..893cd74a66a 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSConnectionUtils.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSConnectionUtils.java @@ -18,6 +18,8 @@ import ch.cyberduck.core.preferences.HostPreferencesFactory; import ch.cyberduck.core.preferences.PreferencesReader; +import ch.cyberduck.core.ssl.X509TrustManager; + import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -30,6 +32,7 @@ import org.irods.irods4j.low_level.api.IRODSApi; import org.irods.irods4j.low_level.api.IRODSException; +import javax.net.ssl.TrustManager; import java.io.IOException; final class IRODSConnectionUtils { @@ -63,11 +66,8 @@ public static IRODSApi.ConnectionOptions initConnectionOptions(IRODSSession sess final IRODSApi.ConnectionOptions options = new IRODSApi.ConnectionOptions(); options.clientServerNegotiation = preferences.getProperty(IRODSProtocol.CLIENT_SERVER_NEGOTIATION); - options.sslProtocol = preferences.getProperty(IRODSProtocol.TLS_PROTOCOL); - options.sslTruststore = preferences.getProperty(IRODSProtocol.TLS_TRUSTSTORE); - options.sslTruststorePassword = preferences.getProperty(IRODSProtocol.TLS_TRUSTSTORE_PASSWORD); - log.debug("client server negotiation = [{}], ssl protocol = [{}], ssl truststore = [{}]", - options.clientServerNegotiation, options.sslProtocol, options.sslTruststore); + options.trustManagers = new TrustManager[]{session.getFeature(X509TrustManager.class)}; + log.debug("client server negotiation = [{}], ssl protocol = [{}]", options.clientServerNegotiation, options.sslProtocol); options.encryptionAlgorithm = preferences.getProperty(IRODSProtocol.ENCRYPTION_ALGORITHM); options.encryptionKeySize = preferences.getInteger(IRODSProtocol.ENCRYPTION_KEY_SIZE); diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSProtocol.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSProtocol.java index 8e76eafd3f5..40381f50a86 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSProtocol.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSProtocol.java @@ -33,9 +33,6 @@ public final class IRODSProtocol extends AbstractProtocol { public static final String DESTINATION_RESOURCE = "Destination Resource"; public static final String DELETE_OBJECTS_PERMANTENTLY = "Delete Objects Permanently"; public static final String CLIENT_SERVER_NEGOTIATION = "Client Server Negotiation"; - public static final String TLS_PROTOCOL = "TLS Protocol"; - public static final String TLS_TRUSTSTORE = "TLS Truststore"; - public static final String TLS_TRUSTSTORE_PASSWORD = "TLS Truststore Password"; public static final String ENCRYPTION_ALGORITHM = "Encryption Algorithm"; public static final String ENCRYPTION_KEY_SIZE = "Encryption Key Size"; public static final String ENCRYPTION_SALT_SIZE = "Encryption Salt Size"; @@ -89,9 +86,6 @@ public Map getProperties() { final Map props = new HashMap<>(); props.put(DELETE_OBJECTS_PERMANTENTLY, "no"); props.put(CLIENT_SERVER_NEGOTIATION, "CS_NEG_REFUSE"); - props.put(TLS_PROTOCOL, "TLSv1.2"); - props.put(TLS_TRUSTSTORE, "NOT SET"); - props.put(TLS_TRUSTSTORE_PASSWORD, "NOT SET"); props.put(ENCRYPTION_ALGORITHM, "AES-256-CBC"); props.put(ENCRYPTION_KEY_SIZE, "32"); props.put(ENCRYPTION_SALT_SIZE, "8"); From f62c22114236485cb243c670ee7d3265edf4bdfa Mon Sep 17 00:00:00 2001 From: Kory Draughn Date: Thu, 27 Nov 2025 13:21:32 -0500 Subject: [PATCH 20/20] iRODS: Add tests for PAM authentication --- .../irods/IRODSPamAuthenticationTest.java | 190 ++++++++++++++++++ 1 file changed, 190 insertions(+) create mode 100644 irods/src/test/java/ch/cyberduck/core/irods/IRODSPamAuthenticationTest.java diff --git a/irods/src/test/java/ch/cyberduck/core/irods/IRODSPamAuthenticationTest.java b/irods/src/test/java/ch/cyberduck/core/irods/IRODSPamAuthenticationTest.java new file mode 100644 index 00000000000..6f406a64595 --- /dev/null +++ b/irods/src/test/java/ch/cyberduck/core/irods/IRODSPamAuthenticationTest.java @@ -0,0 +1,190 @@ +package ch.cyberduck.core.irods; + +/* + * Copyright (c) 2002-2025 iterate GmbH. All rights reserved. + * https://cyberduck.io/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +import ch.cyberduck.core.Credentials; +import ch.cyberduck.core.DisabledCancelCallback; +import ch.cyberduck.core.DisabledHostKeyCallback; +import ch.cyberduck.core.DisabledLoginCallback; +import ch.cyberduck.core.Host; +import ch.cyberduck.core.Profile; +import ch.cyberduck.core.ProtocolFactory; +import ch.cyberduck.core.exception.BackgroundException; +import ch.cyberduck.core.proxy.DisabledProxyFinder; +import ch.cyberduck.core.serializer.impl.dd.ProfilePlistReader; +import ch.cyberduck.core.ssl.X509TrustManager; +import ch.cyberduck.test.TestcontainerTest; + +import org.irods.irods4j.authentication.NativeAuthPlugin; +import org.irods.irods4j.high_level.administration.IRODSUsers; +import org.irods.irods4j.high_level.administration.IRODSZones; +import org.irods.irods4j.high_level.connection.IRODSConnection; +import org.irods.irods4j.high_level.connection.QualifiedUsername; +import org.irods.irods4j.low_level.api.IRODSApi; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.testcontainers.containers.ComposeContainer; +import org.testcontainers.containers.wait.strategy.Wait; + +import javax.net.ssl.TrustManager; +import java.io.File; +import java.io.IOException; +import java.security.cert.CertificateException; +import java.security.cert.X509Certificate; +import java.util.Collections; +import java.util.HashSet; +import java.util.Optional; + +import static org.junit.Assert.*; + +@Category(TestcontainerTest.class) +public class IRODSPamAuthenticationTest { + + private static final X509TrustManager cyberduckTrustManager = new X509TrustManager() { + @Override + public void checkClientTrusted(final X509Certificate[] chain, final String authType) throws CertificateException { + } + + @Override + public void checkServerTrusted(final X509Certificate[] chain, final String authType) throws CertificateException { + } + + @Override + public X509Certificate[] getAcceptedIssuers() { + return new X509Certificate[0]; + } + + @Override + public X509TrustManager init() throws IOException { + return null; + } + + @Override + public void verify(final String hostname, final X509Certificate[] certs, final String cipher) throws CertificateException { + } + }; + + private static final TrustManager[] irodsTrustManagers = new TrustManager[]{new javax.net.ssl.X509TrustManager() { + @Override + public void checkClientTrusted(final X509Certificate[] chain, final String authType) throws CertificateException { + } + + @Override + public void checkServerTrusted(final X509Certificate[] chain, final String authType) throws CertificateException { + } + + @Override + public X509Certificate[] getAcceptedIssuers() { + return new X509Certificate[0]; + } + }}; + + private static final ComposeContainer container = new ComposeContainer( + new File(IRODSDockerComposeManager.class.getResource("/docker/docker-compose.pam.yml").getFile())) + .withPull(false) + .withLocalCompose(true) + .withExposedService("irods-catalog-provider-1", 1347, Wait.forLogMessage(".*\"log_message\":\"Initializing delay server.\".*", 1)); + + @BeforeClass + public static void start() { + container.start(); + } + + @AfterClass + public static void shutdown() { + container.stop(); + } + + @Test + public void testPamPasswordsContainingSpecialCharactersAreHandledCorrectly() throws Exception { + final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new IRODSProtocol()))); + final Profile profile = new ProfilePlistReader(factory).read( + this.getClass().getResourceAsStream("/iRODS_pam.cyberduckprofile")); + + final String username = "john"; + final String password = "=i;r@o\\d&s"; + final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials(username, password)); + final IRODSUsers.User testUser = new IRODSUsers.User(username, Optional.of(host.getRegion())); + + IRODSApi.ConnectionOptions options = new IRODSApi.ConnectionOptions(); + options.clientServerNegotiation = "CS_NEG_REQUIRE"; + options.trustManagers = irodsTrustManagers; + + try(IRODSConnection conn = new IRODSConnection(options)) { + // Create a test user named john. We do not set a password for this user because + // they are using PAM authentication. + conn.connect(host.getHostname(), host.getPort(), new QualifiedUsername("rods", host.getRegion())); + conn.authenticate(new NativeAuthPlugin(), "rods"); + IRODSUsers.addUser(conn.getRcComm(), testUser, IRODSUsers.UserType.RODSUSER, IRODSZones.ZoneType.LOCAL); + + try { + final IRODSSession session = new IRODSSession(host, cyberduckTrustManager, null); + + assertNotNull(session.open(new DisabledProxyFinder(), new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback())); + assertTrue(session.isConnected()); + assertNotNull(session.getClient()); + session.login(new DisabledLoginCallback(), new DisabledCancelCallback()); + + session.close(); + assertFalse(session.isConnected()); + } + finally { + IRODSUsers.removeUser(conn.getRcComm(), testUser); + } + } + } + + @Test + public void testIncorrectPamPasswordFails() throws Exception { + final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new IRODSProtocol()))); + final Profile profile = new ProfilePlistReader(factory).read( + this.getClass().getResourceAsStream("/iRODS_pam.cyberduckprofile")); + + final String username = "john"; + final String password = "incorrect"; + final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials(username, password)); + final IRODSUsers.User testUser = new IRODSUsers.User(username, Optional.of(host.getRegion())); + + IRODSApi.ConnectionOptions options = new IRODSApi.ConnectionOptions(); + options.clientServerNegotiation = "CS_NEG_REQUIRE"; + options.trustManagers = irodsTrustManagers; + + try(IRODSConnection conn = new IRODSConnection(options)) { + // Create a test user named john. We do not set a password for this user because + // they are using PAM authentication. + conn.connect(host.getHostname(), host.getPort(), new QualifiedUsername("rods", host.getRegion())); + conn.authenticate(new NativeAuthPlugin(), "rods"); + IRODSUsers.addUser(conn.getRcComm(), testUser, IRODSUsers.UserType.RODSUSER, IRODSZones.ZoneType.LOCAL); + + try { + final IRODSSession session = new IRODSSession(host, cyberduckTrustManager, null); + + assertNotNull(session.open(new DisabledProxyFinder(), new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback())); + assertTrue(session.isConnected()); + assertNotNull(session.getClient()); + assertThrows(BackgroundException.class, () -> session.login(new DisabledLoginCallback(), new DisabledCancelCallback())); + + session.close(); + assertFalse(session.isConnected()); + } + finally { + IRODSUsers.removeUser(conn.getRcComm(), testUser); + } + } + } +}