iRODS: v0.9.8

deploy-os_servers.yml

@@ -326,6 +326,16 @@
  remote_ip_prefix: 0.0.0.0/0
  wait: true
  timeout: "{{ openstack_api_timeout }}"
+ - name: "Add rule to {{ slurm_cluster_name }}_irods security group: allow PostgreSQL inbound on port 5432."
+ openstack.cloud.security_group_rule:
+ security_group: "{{ slurm_cluster_name }}_irods"
+ direction: ingress
+ protocol: tcp
+ port_range_min: 5432
+ port_range_max: 5432
+ remote_ip_prefix: 0.0.0.0/0
+ wait: true
+ timeout: "{{ openstack_api_timeout }}"
  - name: "Add rule to {{ slurm_cluster_name }}_irods security group: allow SSH inbound on port 20000:20199."
  openstack.cloud.security_group_rule:
  security_group: "{{ slurm_cluster_name }}_irods"

files/nibbler_cluster/nemi_irods/README.md

@@ -1,22 +1,22 @@
 ## To-do until next meeting:
 - [ ] change the password
-- [ ] make davrods working
- - [ ] add to playbook yum install irods-resource-plugin-s3
-- [ ] test the current irods test environment
- - [ ] add users
- - [ ] copy the files
-- [ ] test alternative clients to connect to irods and davrods
- - [ ] cyberduck,
+- [x] make davrods working
+ - [x] add to playbook yum install irods-resource-plugin-s3
+- [x] test the current irods test environment
+ - [x] add users
+ - [x] copy the files
+- [x] test alternative clients to connect to irods and davrods
+ - [x] cyberduck,
  - [x] linux dav:// client
- - [ ] Windows 10 default client 4GB limitation?
-- [ ] update playbook
+ - [x] Windows 10 default client 4GB limitation?
+- [x] update playbook
 - [x] update documentation
-- [ ] permanently set firewall
+- [x] permanently set firewall
  - [ ] limit 1247 port incoming connection (on the surf IP and specific client list only?)
- - [ ] open ports 80 and 443 to docker davrods implementation
- - [ ] check if 80 can be disabled, and if then the davrods clients can still use it webdav
+ - [x] open ports 80 and 443 to docker davrods implementation
+ - [x] check if 80 can be disabled, and if then the davrods clients can still use it webdav
  - [ ] limit port 22 to jumphost
-- [ ] remove demoResc resource and add the rootResc
+- [x] remove demoResc resource and add the rootResc
 
 Extra
 - [ ] think about the implemenation of the authentication - sRAM
@@ -25,16 +25,46 @@ Extra
  - [ ] change the irods password
  - [ ] test the current irods test environment
 
-Links to check:
-- [] https://github.com/irods/irods_capability_storage_tiering
-- [] https://cyberduck.io
-- [] https://github.com/MaastrichtUniversity/sram-sync
-- [] https://hub.docker.com/r/jboss/keycloak/
+### October meeting (A - implemented as the ansible role)
 
-Clients to add:
+- Physical access
+ - [X] AA provide IP address and wall socket network connection number (PDS punt nummer) for 2 data processing machines.
+ - [X] GCC (GvdV/SC/PN) will arrange iRODS account for Ahmed. 
+
+- Clients
+ - [X] (A) Will start with Davrods on Windows and mounted network drives (webdav)
+ - [X] Other options for (Windows) clients are Cyberduck (also webdav) Raidrive (mounted drive)
+
+- Resource selection
+ - [X] (A) We should prevent data from ending up on the wrong storage resource -> Tell davrods to use swift S3 resource as default as opposed to some local file system.
+ - [X] (A) In the future we may want to enforce this default storage resource with a policy on the iRODS server side to prevent a misconfigured client from filing data on the wrong storage resource.
+
+- How to trigger workflows?
+ - [X] Upload goes to disk (swift S3 bucket) first after a manual upload via davrods or another client.
+ - [ ] Next the data will need to go to the tape resources. This can be triggered manually, but it would be nice to do this automatically to make it easier for data managers like Ahmed. 
+
+- Elegant way is to use the iRODS tiering plugin. We should make an iRODS rule that:
+ - [ ] looks at last access time of a file
+ - [ ] file size (only transfer files over certain size to tape as tapes don't handle small files well.)
+ - [ ] Maastricht uses file size limit of 265 GB. We should check with with SURF if that is the advised/optimal file size limit for deciding wether to migrate a file to tape or keep it on disk.
+
+- How to add meta-data when ingesting data into iRODs?
+ - [ ] We should try to use the meta-data model that was already put in a Molgenis and use that as "web interface" for upload of meta-data.
+ - [ ] To prevent having large files in iRODS with missing meta-data:
+ - [ ] A. Upload meta-data first
+ - [ ] B. Triggers creation of folder with correct permissions on swift S3 disk resources, which allows upload of data files via davrods.
+ - [ ] C. When both meta-data and data have arrived -> ingest.
+
+### Links to check:
+- [x] https://github.com/irods/irods_capability_storage_tiering
+- [x] https://cyberduck.io
+- [x] https://github.com/MaastrichtUniversity/sram-sync
+- [x] https://hub.docker.com/r/jboss/keycloak/
+
+### Clients to add:
 - We'll start with two data processing workstations from microscopy dept.:
- - [] 129.125.130.209 (3215.-174.T65)
- - [] 192.168.20.37 (3215.-174.D84)
+ - [ ] 129.125.130.209 (3215.-174.T65)
+ - [ ] 192.168.20.37 (3215.-174.D84)
 
 ## For the iRODS scale-out service SURFsara needs the following from us:
 

group_vars/irods.yml

@@ -1,11 +1,26 @@
 ---
-irods_icat_fqdn: "umcg-icat01.hpc.rug.nl" # fqdn of iCAT server
 firewall_allowed_tcp_ports: # list of open ports on iCAT server
  - "22" # SSH.
+ - "443" # davrods SSL
  - "1247" # irods
+ - "5432" # PostgreSQL
  - "20000:20199" # irods
 irods_ssl_certificate_chain_file: "localhost_and_chain_umcg-icat01.crt"
 irods_ssl_certificate_key_file: "localhost-umcg01.key"
 irods_ssl_dh_params_file: "dhparams.pem"
 irods_zone: 'nlumcg' # default main iRODS zone name
+irods_local_resource: 'rootResc' # local iRODS resource
+irods_vault_path: '/var/lib/irods/Vault' # default path to store files for local resource
+irods_default_resource: 'surfObjStore' # default resource iRODS uploads to
+irods_service_account: 'irods' # linux account under which iRODS runs
+irods_admin_name: 'rods' # iRODS (and zone) account
+irods_admin_home_path: '/nlumcg/home/rods' # iRODS admin's home path
+irods_icat_fqdn: 'umcg-icat01.hpc.rug.nl' # iRODS iCAT external FQDN address
+server_type: 'icat' # iRODS Server Type 
+irods_db_user: '{{ irods_service_account }}' # db Username, usually same as irods_service_account
+irods_db_server: '127.0.0.1' # iRODS Database Server
+irods_db_name: 'ICAT' # iRODS Database Name
+davrods_install: true # to install davrods docker
+davrods_docker_folder: "davrods_docker" # davrods docker folder name, relative to the user home directory
+davrods_default_resource: "surfObjStore" # default resource to upload files via davrods
 ...

group_vars/nibbler_cluster/secrets.yml

@@ -1,78 +1,84 @@
 $ANSIBLE_VAULT;1.2;AES256;nibbler
-64393961393334303930653239373366653764646532326536343937626238666632373836336365
-6566663733643139643236666466323939346465323930350a663835303236333137633334333439
-35646430656465376331313236393864326631646232623736623462306232653466613335313161
-3062623534653030310a333635666461393162366564356436303635633264323634366364633861
-65336162373636343964396533386233363734333633663238623639666261336235643330383233
-66333463653136353338393361326331323264313239363934343031333265343462333665393532
-39373130373632323562383331643363613138383164376462626661663736336263326361633364
-62383234633730366539303639633839373062616336356231393962666338653834643330653362
-37356432323261356262386631313734376362616363646664626339633032323466363664383238
-36666361646563333036306133396535396430333538306232363530656536303864373631333933
-33353230383365316239393935363938363566396361316566386331336630396531646130346537
-30396464376664333632656139306263376565383433303535366330383565363561356363663931
-61643133386432316566663964393633343437613430636365326562336136653164323233386235
-62326661363236326566666438303864363439356335353734663431633064313436333162653936
-30383734303832303834646565386466633531353664613931306236373933343531346135303335
-66376332346666656161333161663238346162623761323164623963386431653435383936623635
-31616435633733323162313236666337623164613866613437346633633930643231343132323864
-33643065646561643835306365333937333038323632353931356263343064383437653338343461
-31313263353130326337393739656638346266363163323765623762326637303134616133346535
-36383733613930663538666336616137643030336363393364333366613362303966376466326432
-64623565363063383232643262633766633232373930353866666638663862623534663930393739
-62303438376161313138636538663465393437666337643636323638303964303836393439396361
-31363838343334636235346434326234363562366537666230363766383066363539646663366438
-30643662656564366531393130396435383834623261386162326430633638386230633730633735
-30636666656661326563663665396638393938356332326365356466653237313362663130633066
-39396137393033343465653761323663313265346236353032353931613963326662616433316531
-34366639326664616633636130653735383535626261356363343232613038623065623934633534
-39353130663330656131663638663030643335353035303338373561363030356261366666373635
-31313137313133353938343330316663613265343236323332303731343939386535366538356332
-61373730356133373964303463373138383231613862383737313636303735666339396361373233
-63383665653965356436613539316234383634353130633032343061313539373936316533326165
-61346266386230323239343366393866396530613735323262306136393336656630353231336337
-61356237336532623736633338353765666266353036343931343062646630396532363039303139
-30636630356531616237633062663237346232633864376663613662373536313733356265633133
-34646531313366653935303230343030633134346462623531386239646334313931643963346231
-31646162656531633134393538333736613632333535353362323939633262316339656530373930
-65393463343033643432633864616634353265343539613966646336323032636262326530636531
-39346136613938313436346139383935613362343235303131393236643137336136646433376437
-34353432636638393137653765636238383134336135343637306462343439376236363633636366
-39303536626333636635623539353164626630363733333736303062396332306538613538383538
-34356336393462633230626439353639373030383964653838623639366633333962326432323131
-38306562303135663665323664383938626232616461326265323632636263663032663638373434
-38623537326364316562313639646165616466633336356531646261633265353838623330346335
-33353236356463316361383861306232626636316430323666396261646635633061386561386539
-33316662633766626632613633386263313735363264396466306561363733333737663034623437
-62633037613535663930626539663661363639616238373834386335623034383831316461306264
-64393265626235356538386635316236636464643862343361663832636536363530363337613662
-31323538653461383230393132383166663432383563343938633066393037333936313764613031
-32353866663866653335663761316162646139653230346633643435623139323263653832393239
-64363431376336383532623535646564306363613839346135613063626230616439396330316639
-63366663323739656166613839646231666334613434356466626262363337663631363034303365
-33646335643064336261366530653431646332376163656336623137646639336233333165363738
-37663266656238336366333833313031373539623834663966353662663334666366393238343532
-65333162636561303939343038336339646566306331323866323865373266336265613835643431
-62386364636233316463366231303435353431656561313266346233356362666664636263363630
-63366566313963396562633736616462343965383363613933383831373465386164633932666238
-30343638323731313539626438626462373236393065666336343861653838386462643935376633
-63663266313764333363633265393334356662376238336365623538316639373663343063353865
-32343836616230333537663261663330386563643138366262303765373833363731653765393030
-35663733616532616338616265323962346261653866633739343762666366313832626333313564
-34653461356634646264383738636463643934346534303934343463396436643130356238613661
-35633861323135373661393332666462626235383436323030623934373363656363613461333161
-30303630393263663134336432336538376436333966643662376432346137356235613333343935
-64373135653465313330386333353537626632313664653065353461633838666365653138623137
-64373566346335363432653965333139333663643138376166326566626466633261626366396264
-32333264376437613637306337333136323339366362336262633464366365656230623330386232
-62636136373230376533353838343561356366323865343766386337303637663030623263316336
-32353831323364353536313964323734373935383831666563623031313439376137323866393332
-30366661396430306462356266346464323837386638303637376266363338323064623432393433
-35653636316161653131313136366531613533326534383264303132326664353963376630656133
-34663363653434373565306663346530313264333964646439393937356531666433646233393862
-37353230666236396634653533306566623432663563626439326238373764313032656239623734
-61356539653234353133363634396463653238326435613937626666306664393633623062333734
-31613966363536626133313535373236636137656466373263313564373634646165343137653736
-30333232336537653263316531623636343037633663656135333762366266363936373339656432
-31336237646138313536656332386165306163396431623262393435636136393330336462343133
-34626238316164636135383432666638303264393162333330616531306437333232
+34306463636163303533643933303465313033303933323366316639316639303961323039376636
+3336303464313532323464386561363234393362363831620a393030353237633832316635313938
+38336134346231623130376262393131626231643934303031326130646565613264313637346565
+3830323264313366340a316131623034643731383663646637306161646230653164633035373934
+66396438646463633135343862386532326234623131343466343830643834656436323334386638
+31346131623766663337623331653062356264356237326331303039643736613732386330323766
+38326433626239623265633938313538623734393634373638646233316631643539333564666466
+61353966386565336661353435303735363336393539326337356632643536313532656665346432
+38316536653763326631663434613930363530393062663661393162383335353261316636323962
+63643666343832336637316134396532383062363937666565373937336633396365393839363363
+33666363636539383337376438653032636335623831383064616636316136626665353837613266
+32323366666130623538393365353261383363613736306435333966313638333631346439626638
+63316666353731656537623139663466663638333961356166613866646261323436636462616330
+37663062393230333238343637303834303638336164326236666631393139336630333334633630
+36396536343366396639343837633032356531343236666637363535626362353631633966663531
+63616431663434333031383935653363306261626337623261396363663733323133323237663431
+39396230396563393063396532373839363736366435643462633331353161326634313761653362
+64333665616130616366376364306664616463303435306334636330383964343631373834373862
+62393766393661343138303862303264383365646130633330343066633937333261356138636136
+63316232626134303563656666393030303935386532383138383236303533383837376666633931
+30343239373465366163663065353061386231386539386231396464356463623061343162366361
+33653436666532353764636435663862323233616265386136323862316239373730393830636637
+34363237613634663865386163323564396463363864653937636462376163396466306439396435
+62643161643364323661323862316331666263326432363230376232353838373630346530393531
+65666162326233336337616666333631303335636632333831633563323938393336633437353931
+66383231616562373862393131326132656162613638313737636436366365333638373734646538
+33353933373733656130643665323162633639323734356432396136623464323838396331663466
+37636636363162386136396662623837323730373164383562336436316236303465343035316663
+37616361366363336264346561346234303562343264366133383938366532313939613766303066
+30323834613035656563646238653632373830616238383461613262316332653932356661616433
+38623937353762653166313532666430343766333165646239333834646161633862383435653263
+33316231656462626235333463386164643035383931386366666138376666623663306133356561
+32353962386136613031623238613534643730303931636135303933653236613865623437356165
+37373537393166313362323337393466303530656438333465373531343834333033306366666535
+33646634656330636466326237386665303736383762393466663630646438653662303139626661
+64646564306165333465656239656431373232613133646262633735633233373233393532396430
+33333264356239363937326136656533333235373031643462303166333133323735396261353963
+61623964336362346335623839643964653862323866343762666431373066356638303130626439
+38366461333933333132623761663761343537623234393232333033626439613265353063656463
+65383132613337306662373064613163393039633066336164306666353438343161636433343737
+61666133643437643735616662313838346530663166306462353535396264663161653165613666
+65653935316438333538316461336561376631633366653163633330356233643431656135323561
+37333364346166346234306362313662313837613737656365323262346530343837363038383037
+65356164613335336465646538663262356633323539643864363838303564386532396366623631
+34656564633066646635366465373562396165323537663935626263643438373533366435623239
+36353732363938316433366264396463333963346138646563383332386536373466383033633363
+62646561323661373936616236313131623935393133356266356137316136636233626231656135
+63366666323131386236383637376363626235356462633162636134663164646664343732366134
+34326132356132323063396631633166643131623038376339636535303631313862336635633266
+31366662303835386363336163323865646234313133653638306436373865663233653433373864
+64626664313630326639633566373461363564393564383832633334646438626136303731326237
+35613563663264303632376163363730666233633861666662383736666232613238323163663863
+39353237333937656435346433333234306562376530366534613130333930306338333636663330
+34343332393363373261393266346637333862336562643239393862303662663338363334373063
+35326530343466326338666233356361393436633463303539306266323335313036613463356337
+62323535303336643333313833366539663162373738333335653637326436323938316165326635
+64363731663039393766313737333334616435653630333562396364376436336530346336316433
+36316634356437613432303933643061366164656434353566333963636135626534636565626362
+33336433613236363835303832613831623636353065646630303564363165376337353534303762
+39646538666338333339346163656633386362343164393330383236376337363963646531366332
+33376332643937333537386266313038333163306162323064303762623539633738313038356534
+36653039353139346464343534366363363737643435313133346663633038356532626639346330
+33393937316330333161333038376138306631373033373236636138326561366566313566303436
+30613966343430643830613465363532336333653961316536623863376536353434343330343463
+32353134393332306530633232383833613466316635636561353664326633323533386138663832
+32303765633838613331323533333930623734313630646661633463366635383838393138626236
+61666236303731303937393066633938313337623731386362613932613363393831396231353963
+30356338333064666431316433343138613331653734653439623831373562306237326661316462
+30646635343937333366383837393934333031346231326332306536356439373432656334303339
+65396235306239323330373464376463396133666337633466626139386637633764326332323332
+62663263323862303665343638363934323739633664303131356231363464643230363135656264
+35343932366538313665313065356231376566363533393537316361336532303665356266623139
+36623432626437346631653865353663306239363833646430616634656563313165363565376634
+34393733393064343535326265313366623066363339663136613039646333343466346633353463
+61386437626530393166353634373365363364313165393862376532313830663934303237393463
+61383935323061393064343930303435366332613234386261323532653766343831383939363562
+30376165373365346263623932376461333432316365383564313932336463376234326138366563
+61363733333766386662663434383231613339366264353933376361633366613837303162653432
+31313664616563316535633630363763306263626464613431313632353264346431306139653766
+32343539336162343933623465353434663632386630396333643334653332393337333564666239
+39316230393737313765313936396364333839663235633061663663626431653334333332656430
+65333034646266323930333734303032313130306334633931323337396432323138373830303165
+333434363839336533333536356463326430

roles/docker/tasks/main.yml

@@ -4,8 +4,9 @@
  name:
  - docker
  - python2-pip
+ - docker-compose
  state: latest
- update_cache: yes
+ update_cache: true
  become: true
 
 - name: Install docker-py.