From aecbb5a7dd87c927f44be64c75a313d6c0327072 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Tue, 27 Oct 2020 15:14:59 +0300 Subject: [PATCH 01/20] tmp --- .../create-task-page/create-task-content.tsx | 8 ++++++++ cvat-ui/src/components/file-manager/file-manager.tsx | 10 ++++++---- cvat-ui/src/containers/file-manager/file-manager.tsx | 4 +++- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/cvat-ui/src/components/create-task-page/create-task-content.tsx b/cvat-ui/src/components/create-task-page/create-task-content.tsx index f30e79810104..4710fc3cc15f 100644 --- a/cvat-ui/src/components/create-task-page/create-task-content.tsx +++ b/cvat-ui/src/components/create-task-page/create-task-content.tsx @@ -114,6 +114,13 @@ class CreateTaskContent extends React.PureComponent { + // todo + // add field to state CreateTaskData.AdvancedConfiguration + // change state here + // draw checkbox depending on the state + }; + private handleSubmitClick = (): void => { if (!this.validateLabels()) { notification.error({ @@ -192,6 +199,7 @@ class CreateTaskContent extends React.PureComponent* Select files: { this.fileManagerContainer = container; }} diff --git a/cvat-ui/src/components/file-manager/file-manager.tsx b/cvat-ui/src/components/file-manager/file-manager.tsx index d62d724b412a..c4b9e2a875ad 100644 --- a/cvat-ui/src/components/file-manager/file-manager.tsx +++ b/cvat-ui/src/components/file-manager/file-manager.tsx @@ -31,6 +31,7 @@ interface Props { withRemote: boolean; treeData: TreeNodeNormal[]; onLoadData: (key: string, success: () => void, failure: () => void) => void; + onChangeActiveKey(key: string): void; } export default class FileManager extends React.PureComponent { @@ -215,7 +216,7 @@ export default class FileManager extends React.PureComponent { } public render(): JSX.Element { - const { withRemote } = this.props; + const { withRemote, onChangeActiveKey } = this.props; const { active } = this.state; return ( @@ -224,11 +225,12 @@ export default class FileManager extends React.PureComponent { type='card' activeKey={active} tabBarGutter={5} - onChange={(activeKey: string): void => + onChange={(activeKey: string): void => { + onChangeActiveKey(activeKey); this.setState({ active: activeKey as any, - }) - } + }); + }} > {this.renderLocalSelector()} {this.renderShareSelector()} diff --git a/cvat-ui/src/containers/file-manager/file-manager.tsx b/cvat-ui/src/containers/file-manager/file-manager.tsx index 8a74964c1205..3db64770e166 100644 --- a/cvat-ui/src/containers/file-manager/file-manager.tsx +++ b/cvat-ui/src/containers/file-manager/file-manager.tsx @@ -14,6 +14,7 @@ import { ShareItem, CombinedState } from 'reducers/interfaces'; interface OwnProps { ref: any; withRemote: boolean; + onChangeActiveKey(key: string): void; } interface StateToProps { @@ -68,12 +69,13 @@ export class FileManagerContainer extends React.PureComponent { } public render(): JSX.Element { - const { treeData, getTreeData, withRemote } = this.props; + const { treeData, getTreeData, withRemote, onChangeActiveKey } = this.props; return ( { this.managerComponentRef = component; From 5762fddf1a3b2593dec1736f8f5e9e76b469f3a3 Mon Sep 17 00:00:00 2001 From: Maya Date: Fri, 30 Oct 2020 16:34:27 +0300 Subject: [PATCH 02/20] Added checkbox 'copy_data' --- cvat-core/src/session.js | 20 +++++++++++++ cvat-ui/src/actions/tasks-actions.ts | 3 ++ .../advanced-configuration-form.tsx | 29 +++++++++++++++++-- .../create-task-page/create-task-content.tsx | 13 ++++++--- 4 files changed, 59 insertions(+), 6 deletions(-) diff --git a/cvat-core/src/session.js b/cvat-core/src/session.js index 8c47fa43a898..35a3125afbc8 100644 --- a/cvat-core/src/session.js +++ b/cvat-core/src/session.js @@ -875,6 +875,7 @@ data_original_chunk_type: undefined, use_zip_chunks: undefined, use_cache: undefined, + copy_data: undefined, }; for (const property in data) { @@ -1117,6 +1118,22 @@ data.use_cache = useCache; }, }, + /** + * @name copyData + * @type {boolean} + * @memberof module:API.cvat.classes.Task + * @instance + * @throws {module:API.cvat.exceptions.ArgumentError} + */ + copyData: { + get: () => data.copy_data, + set: (copyData) => { + if (typeof copyData !== 'boolean') { + throw new ArgumentError('Value must be a boolean'); + } + data.copy_data = copyData; + }, + }, /** * After task has been created value can be appended only. * @name labels @@ -1695,6 +1712,9 @@ if (typeof this.dataChunkSize !== 'undefined') { taskDataSpec.chunk_size = this.dataChunkSize; } + if (typeof this.copyData !== 'undefined') { + taskDataSpec.copy_data = this.copyData; + } const task = await serverProxy.tasks.createTask(taskSpec, taskDataSpec, onUpdate); return new Task(task); diff --git a/cvat-ui/src/actions/tasks-actions.ts b/cvat-ui/src/actions/tasks-actions.ts index 1f6eebdb95e4..58b492f202e3 100644 --- a/cvat-ui/src/actions/tasks-actions.ts +++ b/cvat-ui/src/actions/tasks-actions.ts @@ -405,6 +405,9 @@ export function createTaskAsync(data: any): ThunkAction, {}, {}, A if (data.advanced.dataChunkSize) { description.data_chunk_size = data.advanced.dataChunkSize; } + if (data.advanced.copyData) { + description.copy_data = data.advanced.copyData; + } const taskInstance = new cvat.classes.Task(description); taskInstance.clientFiles = data.files.local; diff --git a/cvat-ui/src/components/create-task-page/advanced-configuration-form.tsx b/cvat-ui/src/components/create-task-page/advanced-configuration-form.tsx index 88175b7e6aca..c92c7042965b 100644 --- a/cvat-ui/src/components/create-task-page/advanced-configuration-form.tsx +++ b/cvat-ui/src/components/create-task-page/advanced-configuration-form.tsx @@ -26,11 +26,14 @@ export interface AdvancedConfiguration { useZipChunks: boolean; dataChunkSize?: number; useCache: boolean; + activeTab: string; + copyData?: boolean; } type Props = FormComponentProps & { onSubmit(values: AdvancedConfiguration): void; installedGit: boolean; + activeTab: string; }; function isPositiveInteger(_: any, value: any, callback: any): void { @@ -114,6 +117,26 @@ class AdvancedConfigurationForm extends React.PureComponent { form.resetFields(); } + renderCopyDataChechbox(): JSX.Element { + const { form } = this.props; + return ( + + + + {form.getFieldDecorator('copyData', { + initialValue: false, + valuePropName: 'checked', + })( + + Copy data into CVAT + , + )} + + + + ); + } + private renderImageQuality(): JSX.Element { const { form } = this.props; @@ -386,10 +409,12 @@ class AdvancedConfigurationForm extends React.PureComponent { } public render(): JSX.Element { - const { installedGit } = this.props; - + const { installedGit, activeTab } = this.props; return (
+ + {activeTab === 'share' ? this.renderCopyDataChechbox() : null} + {this.renderUzeZipChunks()} diff --git a/cvat-ui/src/components/create-task-page/create-task-content.tsx b/cvat-ui/src/components/create-task-page/create-task-content.tsx index 4710fc3cc15f..c8964256f07b 100644 --- a/cvat-ui/src/components/create-task-page/create-task-content.tsx +++ b/cvat-ui/src/components/create-task-page/create-task-content.tsx @@ -42,6 +42,7 @@ const defaultState = { lfs: false, useZipChunks: true, useCache: true, + activeTab: 'local', }, labels: [], files: { @@ -115,10 +116,13 @@ class CreateTaskContent extends React.PureComponent { - // todo - // add field to state CreateTaskData.AdvancedConfiguration - // change state here - // draw checkbox depending on the state + const values = this.state.advanced; + this.setState({ + advanced: { + ...values, + activeTab: key + } + }); }; private handleSubmitClick = (): void => { @@ -217,6 +221,7 @@ class CreateTaskContent extends React.PureComponentAdvanced configuration}> { this.advancedConfigurationComponent = component; }} From dac61289a358c5c160d2fb35dd55e7e7dbab9de8 Mon Sep 17 00:00:00 2001 From: Maya Date: Fri, 30 Oct 2020 16:37:20 +0300 Subject: [PATCH 03/20] Updated versions --- cvat-core/package-lock.json | 2 +- cvat-core/package.json | 2 +- cvat-ui/package-lock.json | 2 +- cvat-ui/package.json | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cvat-core/package-lock.json b/cvat-core/package-lock.json index 0b31ce418f77..6fc9b5a1375f 100644 --- a/cvat-core/package-lock.json +++ b/cvat-core/package-lock.json @@ -1,6 +1,6 @@ { "name": "cvat-core", - "version": "3.8.1", + "version": "3.9.0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/cvat-core/package.json b/cvat-core/package.json index ece057acd52e..e7e86118f821 100644 --- a/cvat-core/package.json +++ b/cvat-core/package.json @@ -1,6 +1,6 @@ { "name": "cvat-core", - "version": "3.8.1", + "version": "3.9.0", "description": "Part of Computer Vision Tool which presents an interface for client-side integration", "main": "babel.config.js", "scripts": { diff --git a/cvat-ui/package-lock.json b/cvat-ui/package-lock.json index 4ed13697d6b8..b854e4c3a580 100644 --- a/cvat-ui/package-lock.json +++ b/cvat-ui/package-lock.json @@ -1,6 +1,6 @@ { "name": "cvat-ui", - "version": "1.9.14", + "version": "1.10.0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/cvat-ui/package.json b/cvat-ui/package.json index 432c439d0f74..7304204ed580 100644 --- a/cvat-ui/package.json +++ b/cvat-ui/package.json @@ -1,6 +1,6 @@ { "name": "cvat-ui", - "version": "1.9.14", + "version": "1.10.0", "description": "CVAT single-page application", "main": "src/index.tsx", "scripts": { From b97a6cda105a4eb2cc8dcf73fbe7804c4d498136 Mon Sep 17 00:00:00 2001 From: Maya Date: Fri, 30 Oct 2020 20:04:36 +0300 Subject: [PATCH 04/20] Added location of uploaded data --- ...033_data_uploaded_data_storage_location.py | 23 +++++++++++++++++++ cvat/apps/engine/models.py | 14 +++++++++++ 2 files changed, 37 insertions(+) create mode 100644 cvat/apps/engine/migrations/0033_data_uploaded_data_storage_location.py diff --git a/cvat/apps/engine/migrations/0033_data_uploaded_data_storage_location.py b/cvat/apps/engine/migrations/0033_data_uploaded_data_storage_location.py new file mode 100644 index 000000000000..84caba3e7fdb --- /dev/null +++ b/cvat/apps/engine/migrations/0033_data_uploaded_data_storage_location.py @@ -0,0 +1,23 @@ +# Generated by Django 3.1.1 on 2020-10-22 09:29 + +import cvat.apps.engine.models +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('engine', '0032_remove_task_z_order'), + ] + + operations = [ + migrations.AddField( + model_name='data', + name='uploaded_data_storage_location', + field=models.CharField( + choices=[('local', 'LOCAL'), ('share', 'SHARE')], + default=cvat.apps.engine.models.UploadedDataStorageLocationChoice['LOCAL'], + max_length=15 + ), + ), + ] diff --git a/cvat/apps/engine/models.py b/cvat/apps/engine/models.py index d99181bcc97f..92ab5dfbe5b5 100644 --- a/cvat/apps/engine/models.py +++ b/cvat/apps/engine/models.py @@ -54,6 +54,18 @@ def choices(cls): def __str__(self): return self.value +class UploadedDataStorageLocationChoice(str, Enum): + #AWS_S3 = 'aws_s3_bucket' + LOCAL = 'local' + SHARE = 'share' + + @classmethod + def choices(cls): + return tuple((x.value, x.name) for x in cls) + + def __str__(self): + return self.value + class Data(models.Model): chunk_size = models.PositiveIntegerField(null=True) size = models.PositiveIntegerField(default=0) @@ -66,6 +78,8 @@ class Data(models.Model): original_chunk_type = models.CharField(max_length=32, choices=DataChoice.choices(), default=DataChoice.IMAGESET) storage_method = models.CharField(max_length=15, choices=StorageMethodChoice.choices(), default=StorageMethodChoice.FILE_SYSTEM) + uploaded_data_storage_location = models.CharField(max_length=15, choices=UploadedDataStorageLocationChoice.choices(), + default=UploadedDataStorageLocationChoice.LOCAL) class Meta: default_permissions = () From a1b32b406d612faddf01c5e308083977ef550dd6 Mon Sep 17 00:00:00 2001 From: Maya Date: Fri, 30 Oct 2020 20:11:13 +0300 Subject: [PATCH 05/20] temp --- cvat/apps/engine/cache.py | 40 ++++++++++++++++++++------------- cvat/apps/engine/serializers.py | 4 +++- cvat/apps/engine/task.py | 16 ++++++++----- cvat/apps/engine/views.py | 8 +++++-- 4 files changed, 43 insertions(+), 25 deletions(-) diff --git a/cvat/apps/engine/cache.py b/cvat/apps/engine/cache.py index bfe79a4e18ee..5d7c01ee6c3a 100644 --- a/cvat/apps/engine/cache.py +++ b/cvat/apps/engine/cache.py @@ -10,9 +10,9 @@ from cvat.apps.engine.media_extractors import (Mpeg4ChunkWriter, Mpeg4CompressedChunkWriter, ZipChunkWriter, ZipCompressedChunkWriter) -from cvat.apps.engine.models import DataChoice +from cvat.apps.engine.models import DataChoice, UploadedDataStorageLocationChoice from cvat.apps.engine.prepare import PrepareInfo - +from .log import slogger class CacheInteraction: def __init__(self): @@ -31,28 +31,36 @@ def get_buff_mime(self, chunk_number, quality, db_data): def prepare_chunk_buff(self, db_data, quality, chunk_number): from cvat.apps.engine.frame_provider import FrameProvider # TODO: remove circular dependency - extractor_classes = { + writer_classes = { FrameProvider.Quality.COMPRESSED : Mpeg4CompressedChunkWriter if db_data.compressed_chunk_type == DataChoice.VIDEO else ZipCompressedChunkWriter, FrameProvider.Quality.ORIGINAL : Mpeg4ChunkWriter if db_data.original_chunk_type == DataChoice.VIDEO else ZipChunkWriter, } - image_quality = 100 if extractor_classes[quality] in [Mpeg4ChunkWriter, ZipChunkWriter] else db_data.image_quality - mime_type = 'video/mp4' if extractor_classes[quality] in [Mpeg4ChunkWriter, Mpeg4CompressedChunkWriter] else 'application/zip' + image_quality = 100 if writer_classes[quality] in [Mpeg4ChunkWriter, ZipChunkWriter] else db_data.image_quality + mime_type = 'video/mp4' if writer_classes[quality] in [Mpeg4ChunkWriter, Mpeg4CompressedChunkWriter] else 'application/zip' - extractor = extractor_classes[quality](image_quality) + extractor = writer_classes[quality](image_quality) images = [] buff = BytesIO() - if os.path.exists(db_data.get_meta_path()): - source_path = os.path.join(db_data.get_upload_dirname(), db_data.video.path) - meta = PrepareInfo(source_path=source_path, meta_path=db_data.get_meta_path()) - for frame in meta.decode_needed_frames(chunk_number, db_data): - images.append(frame) - extractor.save_as_chunk([(image, source_path, None) for image in images], buff) - else: - with open(db_data.get_dummy_chunk_path(chunk_number), 'r') as dummy_file: - images = [os.path.join(db_data.get_upload_dirname(), line.strip()) for line in dummy_file] - extractor.save_as_chunk([(image, image, None) for image in images], buff) + upload_dir = { + UploadedDataStorageLocationChoice.LOCAL: db_data.get_upload_dirname(), + UploadedDataStorageLocationChoice.SHARE: settings.SHARE_ROOT + }[db_data.uploaded_data_storage_location] + try: + if os.path.exists(db_data.get_meta_path()): + source_path = os.path.join(upload_dir, db_data.video.path) + meta = PrepareInfo(source_path=source_path, meta_path=db_data.get_meta_path()) + for frame in meta.decode_needed_frames(chunk_number, db_data): + images.append(frame) + extractor.save_as_chunk([(image, source_path, None) for image in images], buff) + else: + with open(db_data.get_dummy_chunk_path(chunk_number), 'r') as dummy_file: + images = [os.path.join(upload_dir, line.strip()) for line in dummy_file] + extractor.save_as_chunk([(image, image, None) for image in images], buff) + except FileNotFoundError as ex: + slogger.glob.exception(f"{ex.strerror} {ex.filename}") + buff.seek(0) return buff, mime_type diff --git a/cvat/apps/engine/serializers.py b/cvat/apps/engine/serializers.py index 71843878313f..99b38676a36e 100644 --- a/cvat/apps/engine/serializers.py +++ b/cvat/apps/engine/serializers.py @@ -171,12 +171,13 @@ class DataSerializer(serializers.ModelSerializer): server_files = ServerFileSerializer(many=True, default=[]) remote_files = RemoteFileSerializer(many=True, default=[]) use_cache = serializers.BooleanField(default=False) + copy_data = serializers.BooleanField(default=False) class Meta: model = models.Data fields = ('chunk_size', 'size', 'image_quality', 'start_frame', 'stop_frame', 'frame_filter', 'compressed_chunk_type', 'original_chunk_type', 'client_files', 'server_files', 'remote_files', 'use_zip_chunks', - 'use_cache') + 'use_cache', 'copy_data') # pylint: disable=no-self-use def validate_frame_filter(self, value): @@ -205,6 +206,7 @@ def create(self, validated_data): remote_files = validated_data.pop('remote_files') validated_data.pop('use_zip_chunks') validated_data.pop('use_cache') + validated_data.pop('copy_data') db_data = models.Data.objects.create(**validated_data) data_path = db_data.get_data_dirname() diff --git a/cvat/apps/engine/task.py b/cvat/apps/engine/task.py index fad3654fc385..8d5a0cb0363d 100644 --- a/cvat/apps/engine/task.py +++ b/cvat/apps/engine/task.py @@ -15,7 +15,7 @@ from urllib import request as urlrequest from cvat.apps.engine.media_extractors import get_mime, MEDIA_TYPES, Mpeg4ChunkWriter, ZipChunkWriter, Mpeg4CompressedChunkWriter, ZipCompressedChunkWriter -from cvat.apps.engine.models import DataChoice, StorageMethodChoice +from cvat.apps.engine.models import DataChoice, StorageMethodChoice, UploadedDataStorageLocationChoice as LocationChoice from cvat.apps.engine.utils import av_scan_paths from cvat.apps.engine.prepare import prepare_meta @@ -232,7 +232,10 @@ def _create_thread(tid, data): "File with meta information can be uploaded if 'Use cache' option is also selected" if data['server_files']: - _copy_data_from_share(data['server_files'], upload_dir) + if db_data.uploaded_data_storage_location == LocationChoice.LOCAL: + _copy_data_from_share(data['server_files'], upload_dir) + else: + upload_dir = settings.SHARE_ROOT av_scan_paths(upload_dir) @@ -303,10 +306,11 @@ def update_progress(progress): try: from cvat.apps.engine.prepare import UploadedMeta if os.path.split(meta_info_file[0])[0]: - os.replace( - os.path.join(upload_dir, meta_info_file[0]), - db_data.get_meta_path() - ) + if db_data.uploaded_data_storage_location is not LocationChoice.SHARE: + os.replace( + os.path.join(upload_dir, meta_info_file[0]), + db_data.get_meta_path() + ) meta_info = UploadedMeta(source_path=os.path.join(upload_dir, media_files[0]), meta_path=db_data.get_meta_path()) meta_info.check_seek_key_frames() diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index 50798c732c5a..1edcea6197ac 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -36,7 +36,9 @@ from cvat.apps.authentication import auth from cvat.apps.dataset_manager.serializers import DatasetFormatsSerializer from cvat.apps.engine.frame_provider import FrameProvider -from cvat.apps.engine.models import Job, StatusChoice, Task, StorageMethodChoice +from cvat.apps.engine.models import ( + Job, StatusChoice, Task, StorageMethodChoice, UploadedDataStorageLocationChoice +) from cvat.apps.engine.serializers import ( AboutSerializer, AnnotationFileSerializer, BasicUserSerializer, DataMetaSerializer, DataSerializer, ExceptionSerializer, @@ -398,7 +400,9 @@ def data(self, request, pk): if data['use_cache']: db_task.data.storage_method = StorageMethodChoice.CACHE db_task.data.save(update_fields=['storage_method']) - + if data['server_files'] and data.get('copy_data') == False: + db_task.data.uploaded_data_storage_location = UploadedDataStorageLocationChoice.SHARE + db_task.data.save(update_fields=['uploaded_data_storage_location']) # if the value of stop_frame is 0, then inside the function we cannot know # the value specified by the user or it's default value from the database if 'stop_frame' not in serializer.validated_data: From c378989ce1d87d54ce696f148f890873d1df897d Mon Sep 17 00:00:00 2001 From: Maya Date: Fri, 30 Oct 2020 20:12:15 +0300 Subject: [PATCH 06/20] Added guide how mount cloud storages as fuse --- cvat/apps/documentation/installation.md | 3 + .../documentation/mounting_cloud_storages.md | 306 ++++++++++++++++++ 2 files changed, 309 insertions(+) create mode 100644 cvat/apps/documentation/mounting_cloud_storages.md diff --git a/cvat/apps/documentation/installation.md b/cvat/apps/documentation/installation.md index ca536ea5580e..8abd316daa1e 100644 --- a/cvat/apps/documentation/installation.md +++ b/cvat/apps/documentation/installation.md @@ -373,6 +373,9 @@ You can change the share device path to your actual share. For user convenience we have defined the environment variable \$CVAT_SHARE_URL. This variable contains a text (url for example) which is shown in the client-share browser. +You can [mount](/cvat/apps/documentation/mounting_cloud_storages.md) +your cloud storage as a FUSE and use it later as a share. + ### Email verification You can enable email verification for newly registered users. diff --git a/cvat/apps/documentation/mounting_cloud_storages.md b/cvat/apps/documentation/mounting_cloud_storages.md new file mode 100644 index 000000000000..713ff2a785ce --- /dev/null +++ b/cvat/apps/documentation/mounting_cloud_storages.md @@ -0,0 +1,306 @@ +- [Mounting cloud storage](#mounting-cloud-storage) + - [AWS S3 bucket](#aws-s3-bucket-as-filesystem) + - [Ubuntu 20.04](#ubuntu-20.04) + - [Mount](#mount) + - [Cheking](#cheking) + - [Automounting](#automounting) + - [Using /etc/fstab]('#using-fstab') + - [Using systemd](#using-systemd) + - [Unmount](#unmount-filesystem) + - [Azure container](#microsoft-azure-container-as-filesystem) + - [Ubuntu 20.04](#ubuntu-20.04) + - [Mount](#mount) + - [Cheking](#cheking) + - [Automounting](#automounting) + - [Using /etc/fstab]('#using-fstab') + - [Using systemd](#using-systemd) + - [Unmount](#unmount-filesystem) + - [Google Drive](#google-drive-as-filesystem) + - [Ubuntu 20.04](#ubuntu-20.04) + - [Mount](#mount) + - [Cheking](#cheking) + - [Automounting](#automounting) + - [Using /etc/fstab]('#using-fstab') + - [Using systemd](#using-systemd) + - [Unmount](#unmount-filesystem) + - +# Mounting cloud storage +## AWS S3 bucket as filesystem +### Ubuntu 20.04 +#### Mount + +1. Install s3fs +```bash +sudo apt install s3fs +``` + +2. Enter your credentials in a file `${HOME}/.passwd-s3fs` and set owner-only permissions: +```bash +echo ACCESS_KEY_ID:SECRET_ACCESS_KEY > ${HOME}/.passwd-s3fs +chmod 600 ${HOME}/.passwd-s3fs +``` + +3. Uncomment `user_allow_other` in the `/etc/fuse.conf` file: `sudo nano /etc/fuse.conf` + +4. Run s3fs, replace `bucket_name`, `mount_point`: +```bash +s3fs -o allow_other +``` + +For more details see [here](https://github.com/s3fs-fuse/s3fs-fuse). + +#### Cheking +A file `/etc/mtab` contains records of currently mounted filesystems. +```bash +cat /etc/mtab | grep 's3fs' +``` + +#### Automounting +##### Using fstab +TODO +``` +cat /etc/mtab | grep s3fs >> /etc/fstab +doesn't work +``` +##### Using systemd + +1. Create unit file `sudo nano /etc/systemd/system/s3fs.service`. +Replace `user_name`, `bucket_name`, `mount_point`, `/path/to/.passwd-s3fs` + +``` +[Unit] +Description=FUSE filesystem over AWS S3 bucket +After=network.target + +[Service] +Environment=MOUNT_POINT= +User= +Group= +ExecStart=s3fs ${MOUNT_POINT} -o passwd_file=/path/to/.passwd-s3fs -o allow_other +ExecStop=fusermount -u ${MOUNT_POINT} +Restart=always +Type=forking + +[Install] +WantedBy=multi-user.target +``` + +2. Update the system configurations, enable unit autorun when the system boots. +```bash +sudo systemctl daemon-reload +sudo systemctl enable s3fs.service +``` + + +#### Unmount filesystem +```bash +fusermount -u +``` + +## Microsoft Azure container as filesystem +### Ubuntu 20.04 +#### Mount +1. Set up the Microsoft package repository.(More [here](https://docs.microsoft.com/en-us/windows-server/administration/Linux-Package-Repository-for-Microsoft-Software#configuring-the-repositories)) + +```bash +wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb +sudo dpkg -i packages-microsoft-prod.deb +sudo apt-get update +``` +2. Install `blobfuse` and `fuse` +```bash +sudo apt-get install blobfuse fuse +``` +For more details see [here](https://github.com/Azure/azure-storage-fuse/wiki/1.-Installation) + +3. Create enviroments(replace `account_name`, `account_key`, `mount_point`) +```bash +export AZURE_STORAGE_ACCOUNT= +export AZURE_STORAGE_ACCESS_KEY= +MOUNT_POINT= +``` + +3. Create a folder for cache: +```bash +sudo mkdir -p /mnt/blobfusetmp +``` + +4. Make sure the file must be owned by the user who mounts the container: +```bash +sudo chown /mnt/blobfusetmp +``` + +5. Create the mount point, if it doesn't exists: +```bash +mkdir -p ${MOUNT_POINT} +``` + +6. Uncomment `user_allow_other` in the `/etc/fuse.conf` file: `sudo nano /etc/fuse.conf` + +7. Mount container(replace `your_container`): +```bash +blobfuse ${MOUNT_POINT} --container-name= --tmp-path=/mnt/blobfusetmp -o allow_other +``` + +#### Automounting + +##### Using fstab +1. Create configuration file `connection.cfg` with same content, change accountName, select one from accountKey or sasToken and replace with your value +``` +accountName +# Please provide either an account key or a SAS token, and delete the other line. +accountKey +#change authType to specify only 1 +sasToken +authType +containerName +``` + +2. create `mount.sh` with content below: +``` +#!/bin/bash +BLOBFS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +cd $BLOBFS_DIR/build +./blobfuse $1 --tmp-path=/mnt/blobfusetmp --use-attr-cache=true -o attr_timeout=240 +-o entry_timeout=240 -o negative_timeout=120 -o allow_other --config-file=../connection.cfg +``` + +4. Edit `/etc/fstab` with the blobfuse script. Add the following line: +`//mount.sh fuse _netdev,allow_other` + +##### Using systemd +1. Create unit file `sudo nano /etc/systemd/system/blobfuse.service`. +Replace `user_name`, `mount_point`, `container_name`,`/path/to/connection.cfg` + +``` +[Unit] +Description=FUSE filesystem over Azure container +After=network.target + +[Service] +Environment=MOUNT_POINT= +User= +Group= +ExecStart=blobfuse ${MOUNT_POINT} --container-name= --tmp-path=/mnt/blobfusetmp --config-file=/path/to/connection.cfg -o allow_other +ExecStop=fusermount -u ${MOUNT_POINT} +Restart=always +Type=forking + +[Install] +WantedBy=multi-user.target +``` + +2. Update the system configurations, enable unit autorun when the system boots: +```bash +sudo systemctl daemon-reload +sudo systemctl enable blobfuse.service +``` + +Or for more detail [see here](https://github.com/Azure/azure-storage-fuse/tree/master/systemd) + +#### Unmount the filesystem +```bash +fusermount -u +``` + +If you have any mounting problems, check out the [answers](https://github.com/Azure/azure-storage-fuse/wiki/3.-Troubleshoot-FAQ) +to common problems + +## Google Drive as filesystem +### Ubuntu 20.04 +#### Mount +To mount a google drive as a filesystem in user space(FUSE) +you can use [google-drive-ocamlfuse](https://github.com/astrada/google-drive-ocamlfuse) +To do this follow the instructions below: +1. Install google-drive-ocamlfuse +``` +sudo add-apt-repository ppa:alessandro-strada/ppa +sudo apt-get update +sudo apt-get install google-drive-ocamlfuse +``` + +2. Run `google-drive-ocamlfuse` without parameters +``` +google-drive-ocamlfuse +``` +This command will create the default application directory (~/.gdfuse/default), containing the configuration file config +(see the [wiki](https://github.com/astrada/google-drive-ocamlfuse/wiki) page for more details about configuration). +And it will start a web browser to obtain authorization to access your Google Drive. +This will let you modify default configuration before mounting the filesystem. + +Then you can choose a local directory to mount your Google Drive (e.g.: ~/GoogleDrive). + +3. Create the mount point, if it doesn't exist(replace mount_point): +```bash +mountpoint="" +mkdir -p $mountpoint +``` + +4. Uncomment `user_allow_other` in the `/etc/fuse.conf` file: `sudo nano /etc/fuse.conf` + +5. Mount the filesystem +```bash +google-drive-ocamlfuse -o allow_other $mountpoint +``` + +#### Cheking +A file `/etc/mtab` contains records of currently mounted filesystems. +```bash +cat /etc/mtab | grep 'google-drive-ocamlfuse' +``` + +#### Automounting +##### Using fstab +1. Create a shell script named gdfuse in /usr/bin (as root) with this content: +```bash +#!/bin/bash + +su $USERNAME -l -c "google-drive-ocamlfuse -label $1 $*" +exit 0 +``` +2. Give it the exec permission: +```bash +sudo chmod +x /usr/bin/gdfuse +``` + +3. Edit `/etc/fstab` adding a line like this, replace `label`(dafualt `label=default`): +``` +gdfuse#