From 980bce8b3f4b9c56f1dc935a7ac21491a6b37220 Mon Sep 17 00:00:00 2001 From: angelsl Date: Thu, 5 Aug 2021 02:48:29 +0800 Subject: [PATCH 1/2] Allow custom course prefix --- lib/cadet/assets/assets.ex | 25 ++--- lib/cadet/courses/course.ex | 3 + lib/cadet/courses/courses.ex | 6 ++ .../admin_assets_controller.ex | 13 ++- .../admin_courses_controller.ex | 2 +- lib/cadet_web/controllers/user_controller.ex | 4 +- lib/cadet_web/views/courses_view.ex | 3 +- lib/cadet_web/views/user_view.ex | 5 +- .../20210804182725_add_assets_prefix.exs | 9 ++ test/cadet/assets/assets_test.exs | 14 +-- .../admin_assets_controller_test.exs | 58 ++++++++++++ .../controllers/user_controller_test.exs | 8 +- .../aws/controller_delete_asset#3.json | 65 +++++++++++++ .../aws/controller_list_assets#2.json | 33 +++++++ .../aws/controller_upload_asset#3.json | 93 +++++++++++++++++++ 15 files changed, 313 insertions(+), 28 deletions(-) create mode 100644 priv/repo/migrations/20210804182725_add_assets_prefix.exs create mode 100644 test/fixtures/vcr_cassettes/aws/controller_delete_asset#3.json create mode 100644 test/fixtures/vcr_cassettes/aws/controller_list_assets#2.json create mode 100644 test/fixtures/vcr_cassettes/aws/controller_upload_asset#3.json diff --git a/lib/cadet/assets/assets.ex b/lib/cadet/assets/assets.ex index dd1076812..b9244829c 100644 --- a/lib/cadet/assets/assets.ex +++ b/lib/cadet/assets/assets.ex @@ -9,18 +9,18 @@ defmodule Cadet.Assets.Assets do if(Mix.env() == :test, do: ["testFolder"], else: []) @accepted_file_types ~w(.jpg .jpeg .gif .png .wav .mp3 .txt) - def upload_to_s3(upload_params, course_id, folder_name, file_name) do + def upload_to_s3(upload_params, prefix, folder_name, file_name) do file_type = Path.extname(file_name) with :ok <- validate_file_name(file_name), :ok <- validate_folder_name(folder_name), :ok <- validate_file_type(file_type) do - if object_exists?(course_id, folder_name, file_name) do + if object_exists?(prefix, folder_name, file_name) do {:error, {:bad_request, "File already exists"}} else file = upload_params.path - s3_path = "#{prefix()}#{course_id}/#{folder_name}/#{file_name}" + s3_path = "#{prefix}#{folder_name}/#{file_name}" file |> Upload.stream_file() @@ -32,11 +32,11 @@ defmodule Cadet.Assets.Assets do end end - def list_assets(course_id, folder_name) do + def list_assets(prefix, folder_name) do case validate_folder_name(folder_name) do :ok -> bucket() - |> S3.list_objects(prefix: "#{prefix()}#{course_id}/" <> folder_name <> "/") + |> S3.list_objects(prefix: prefix <> folder_name <> "/") |> ExAws.stream!() |> Enum.map(fn file -> file.key end) @@ -45,12 +45,12 @@ defmodule Cadet.Assets.Assets do end end - def delete_object(course_id, folder_name, file_name) do + def delete_object(prefix, folder_name, file_name) do with :ok <- validate_file_name(file_name), :ok <- validate_folder_name(folder_name) do - if object_exists?(course_id, folder_name, file_name) do + if object_exists?(prefix, folder_name, file_name) do bucket() - |> S3.delete_object("#{prefix()}#{course_id}/#{folder_name}/#{file_name}") + |> S3.delete_object("#{prefix}#{folder_name}/#{file_name}") |> ExAws.request!() :ok @@ -60,11 +60,11 @@ defmodule Cadet.Assets.Assets do end end - @spec object_exists?(integer(), binary, binary) :: boolean() - def object_exists?(course_id, folder_name, file_name) do + @spec object_exists?(integer() | binary(), binary, binary) :: boolean() + def object_exists?(prefix, folder_name, file_name) do response = bucket() - |> S3.head_object("#{prefix()}#{course_id}/#{folder_name}/#{file_name}") + |> S3.head_object("#{prefix}#{folder_name}/#{file_name}") |> ExAws.request() case response do @@ -99,5 +99,6 @@ defmodule Cadet.Assets.Assets do defp bucket, do: :cadet |> Application.fetch_env!(:uploader) |> Keyword.get(:assets_bucket) - defp prefix, do: :cadet |> Application.fetch_env!(:uploader) |> Keyword.get(:assets_prefix, "") + def assets_prefix, + do: :cadet |> Application.fetch_env!(:uploader) |> Keyword.get(:assets_prefix, "") end diff --git a/lib/cadet/courses/course.ex b/lib/cadet/courses/course.ex index e87a62d78..c224c875d 100644 --- a/lib/cadet/courses/course.ex +++ b/lib/cadet/courses/course.ex @@ -17,6 +17,9 @@ defmodule Cadet.Courses.Course do field(:source_variant, :string) field(:module_help_text, :string) + # for now, only settable from database + field(:assets_prefix, :string, default: nil) + has_many(:assessment_config, AssessmentConfig) timestamps() diff --git a/lib/cadet/courses/courses.ex b/lib/cadet/courses/courses.ex index 431cb4212..1a5f236dd 100644 --- a/lib/cadet/courses/courses.ex +++ b/lib/cadet/courses/courses.ex @@ -20,6 +20,7 @@ defmodule Cadet.Courses do alias Cadet.Assessments alias Cadet.Assessments.Assessment + alias Cadet.Assets.Assets @doc """ Creates a new course configuration, course registration, and sets @@ -408,4 +409,9 @@ defmodule Cadet.Courses do |> Repo.all() |> Repo.preload(:uploader) end + + @spec assets_prefix(%Course{}) :: binary() + def assets_prefix(course) do + course.assets_prefix || "#{Assets.assets_prefix()}#{course.id}/" + end end diff --git a/lib/cadet_web/admin_controllers/admin_assets_controller.ex b/lib/cadet_web/admin_controllers/admin_assets_controller.ex index d2eec9ec5..8fd2f7003 100644 --- a/lib/cadet_web/admin_controllers/admin_assets_controller.ex +++ b/lib/cadet_web/admin_controllers/admin_assets_controller.ex @@ -2,12 +2,14 @@ defmodule CadetWeb.AdminAssetsController do use CadetWeb, :controller use PhoenixSwagger + alias Cadet.Assets.Assets + alias Cadet.Courses def index(conn, _params = %{"foldername" => foldername}) do course_reg = conn.assigns.course_reg - case Assets.list_assets(course_reg.course_id, foldername) do + case Assets.list_assets(Courses.assets_prefix(course_reg.course), foldername) do {:error, {status, message}} -> conn |> put_status(status) |> text(message) assets -> render(conn, "index.json", assets: assets) end @@ -18,7 +20,7 @@ defmodule CadetWeb.AdminAssetsController do course_reg = conn.assigns.course_reg filename = Enum.join(filename, "/") - case Assets.delete_object(course_reg.course_id, foldername, filename) do + case Assets.delete_object(Courses.assets_prefix(course_reg.course), foldername, filename) do {:error, {status, message}} -> conn |> put_status(status) |> text(message) _ -> conn |> put_status(204) |> text('') end @@ -32,7 +34,12 @@ defmodule CadetWeb.AdminAssetsController do course_reg = conn.assigns.course_reg filename = Enum.join(filename, "/") - case Assets.upload_to_s3(upload_params, course_reg.course_id, foldername, filename) do + case Assets.upload_to_s3( + upload_params, + Courses.assets_prefix(course_reg.course), + foldername, + filename + ) do {:error, {status, message}} -> conn |> put_status(status) |> text(message) resp -> render(conn, "show.json", resp: resp) end diff --git a/lib/cadet_web/admin_controllers/admin_courses_controller.ex b/lib/cadet_web/admin_controllers/admin_courses_controller.ex index 2f0a2f7e7..e1b268b1b 100644 --- a/lib/cadet_web/admin_controllers/admin_courses_controller.ex +++ b/lib/cadet_web/admin_controllers/admin_courses_controller.ex @@ -84,7 +84,7 @@ defmodule CadetWeb.AdminCoursesController do end swagger_path :update_course_config do - put("/v2/courses/{course_id}/admin/onfig") + put("/v2/courses/{course_id}/admin/config") summary("Updates the course configuration for the specified course") diff --git a/lib/cadet_web/controllers/user_controller.ex b/lib/cadet_web/controllers/user_controller.ex index 728e946ab..465bb4bf7 100644 --- a/lib/cadet_web/controllers/user_controller.ex +++ b/lib/cadet_web/controllers/user_controller.ex @@ -291,6 +291,7 @@ defmodule CadetWeb.UserController do source_variant(Schema.ref(:SourceVariant), "Source Variant name", required: true) module_help_text(:string, "Module help text", required: true) assessment_types(:list, "Assessment Types", required: true) + assets_prefix(:string, "Assets prefix, used by the game") end example(%{ @@ -303,7 +304,8 @@ defmodule CadetWeb.UserController do source_chapter: 1, source_variant: "default", module_help_text: "Help text", - assessment_types: ["Missions", "Quests", "Paths", "Contests", "Others"] + assessment_types: ["Missions", "Quests", "Paths", "Contests", "Others"], + assets_prefix: "courses-prod/1/" }) end, SourceVariant: diff --git a/lib/cadet_web/views/courses_view.ex b/lib/cadet_web/views/courses_view.ex index 285e78584..6ab298c73 100644 --- a/lib/cadet_web/views/courses_view.ex +++ b/lib/cadet_web/views/courses_view.ex @@ -14,7 +14,8 @@ defmodule CadetWeb.CoursesView do sourceChapter: :source_chapter, sourceVariant: :source_variant, moduleHelpText: :module_help_text, - assessmentTypes: :assessment_configs + assessmentTypes: :assessment_configs, + assetsPrefix: :assets_prefix }) } end diff --git a/lib/cadet_web/views/user_view.ex b/lib/cadet_web/views/user_view.ex index 06cff4af0..f28599024 100644 --- a/lib/cadet_web/views/user_view.ex +++ b/lib/cadet_web/views/user_view.ex @@ -1,6 +1,8 @@ defmodule CadetWeb.UserView do use CadetWeb, :view + alias Cadet.Courses + def render("index.json", %{ user: user, courses: courses, @@ -104,7 +106,8 @@ defmodule CadetWeb.UserView do enableSourcecast: :enable_sourcecast, sourceChapter: :source_chapter, sourceVariant: :source_variant, - moduleHelpText: :module_help_text + moduleHelpText: :module_help_text, + assetsPrefix: &Courses.assets_prefix/1 }) end end diff --git a/priv/repo/migrations/20210804182725_add_assets_prefix.exs b/priv/repo/migrations/20210804182725_add_assets_prefix.exs new file mode 100644 index 000000000..fed8f5c81 --- /dev/null +++ b/priv/repo/migrations/20210804182725_add_assets_prefix.exs @@ -0,0 +1,9 @@ +defmodule Cadet.Repo.Migrations.AddAssetsPrefix do + use Ecto.Migration + + def change do + alter table(:courses) do + add(:assets_prefix, :string, null: true) + end + end +end diff --git a/test/cadet/assets/assets_test.exs b/test/cadet/assets/assets_test.exs index 3d16451d2..fee9ea21c 100644 --- a/test/cadet/assets/assets_test.exs +++ b/test/cadet/assets/assets_test.exs @@ -7,7 +7,7 @@ defmodule Cadet.Assets.AssetsTest do describe "Manage assets" do test "accessible folder" do use_cassette "aws/model_list_assets#1" do - assert Assets.list_assets(1, "testFolder") === [ + assert Assets.list_assets(prefix(1), "testFolder") === [ "1/testFolder/", "1/testFolder/test.png", "1/testFolder/test2.png" @@ -17,20 +17,20 @@ defmodule Cadet.Assets.AssetsTest do test "access another course with 0 folder" do use_cassette "aws/model_list_assets#2" do - assert Assets.list_assets(2, "testFolder") === [] + assert Assets.list_assets(prefix(2), "testFolder") === [] end end test "delete nonexistent file" do use_cassette "aws/model_delete_asset#1" do - assert Assets.delete_object(1, "testFolder", "test4.png") === + assert Assets.delete_object(prefix(1), "testFolder", "test4.png") === {:error, {:not_found, "File not found"}} end end test "delete ok file" do use_cassette "aws/model_delete_asset#2" do - assert Assets.delete_object(1, "testFolder", "test.png") === :ok + assert Assets.delete_object(prefix(1), "testFolder", "test.png") === :ok end end @@ -38,7 +38,7 @@ defmodule Cadet.Assets.AssetsTest do use_cassette "aws/model_upload_asset#1" do assert Assets.upload_to_s3( build_upload("test/fixtures/upload.png"), - 1, + prefix(1), "testFolder", "test2.png" ) === @@ -50,7 +50,7 @@ defmodule Cadet.Assets.AssetsTest do use_cassette "aws/model_upload_asset#2" do assert Assets.upload_to_s3( build_upload("test/fixtures/upload.png"), - 1, + prefix(1), "testFolder", "test1.png" ) === @@ -64,4 +64,6 @@ defmodule Cadet.Assets.AssetsTest do end defp bucket, do: :cadet |> Application.fetch_env!(:uploader) |> Keyword.get(:assets_bucket) + + defp prefix(course_id), do: "#{Assets.assets_prefix()}#{course_id}/" end diff --git a/test/cadet_web/admin_controllers/admin_assets_controller_test.exs b/test/cadet_web/admin_controllers/admin_assets_controller_test.exs index d62b771b7..d2d422361 100644 --- a/test/cadet_web/admin_controllers/admin_assets_controller_test.exs +++ b/test/cadet_web/admin_controllers/admin_assets_controller_test.exs @@ -2,8 +2,12 @@ defmodule CadetWeb.AdminAssetsControllerTest do use CadetWeb.ConnCase use ExVCR.Mock, adapter: ExVCR.Adapter.Hackney + alias Cadet.Courses.Course + alias Cadet.Repo alias CadetWeb.AdminAssetsController + import Ecto.Query, only: [where: 2] + setup_all do HTTPoison.start() end @@ -196,6 +200,52 @@ defmodule CadetWeb.AdminAssetsControllerTest do end end + describe "course with custom assets_prefix" do + @tag authenticate: :staff, course_id: 117 + test "index file", %{conn: conn} do + course_id = conn.assigns.course_id + + set_course_prefix(course_id) + + use_cassette "aws/controller_list_assets#2" do + conn = get(conn, build_url(course_id, "testFolder"), %{}) + + assert json_response(conn, 200) === + ["testFolder/", "testFolder/test.png", "testFolder/test2.png"] + end + end + + @tag authenticate: :staff, course_id: 117 + test "delete file", %{conn: conn} do + course_id = conn.assigns.course_id + + set_course_prefix(course_id) + + use_cassette "aws/controller_delete_asset#3" do + conn = delete(conn, build_url(course_id, "testFolder/nestedFolder/test2.png")) + + assert response(conn, 204) + end + end + + @tag authenticate: :staff, course_id: 117 + test "upload file", %{conn: conn} do + course_id = conn.assigns.course_id + + set_course_prefix(course_id) + + use_cassette "aws/controller_upload_asset#3" do + conn = + post(conn, build_url(course_id, "testFolder/nestedFolder/test.png"), %{ + "upload" => build_upload("test/fixtures/upload.png") + }) + + assert json_response(conn, 200) === + "https://#{bucket()}.s3.amazonaws.com/#{custom_prefix()}testFolder/nestedFolder/test.png" + end + end + end + defp build_url(course_id), do: "/v2/courses/#{course_id}/admin/assets/" defp build_url(course_id, url), do: "#{build_url(course_id)}/#{url}" @@ -203,5 +253,13 @@ defmodule CadetWeb.AdminAssetsControllerTest do %Plug.Upload{path: path, filename: Path.basename(path), content_type: content_type} end + defp set_course_prefix(course_id) do + Course + |> where(id: ^course_id) + |> Repo.update_all(set: [assets_prefix: custom_prefix()]) + end + defp bucket, do: :cadet |> Application.fetch_env!(:uploader) |> Keyword.get(:assets_bucket) + + defp custom_prefix, do: "this-is-a-prefix/" end diff --git a/test/cadet_web/controllers/user_controller_test.exs b/test/cadet_web/controllers/user_controller_test.exs index 99ae00200..cb6770ba7 100644 --- a/test/cadet_web/controllers/user_controller_test.exs +++ b/test/cadet_web/controllers/user_controller_test.exs @@ -3,10 +3,10 @@ defmodule CadetWeb.UserControllerTest do import Cadet.Factory - alias Cadet.Repo alias CadetWeb.UserController # alias Cadet.Assessments.{Assessment, Submission} alias Cadet.Accounts.{User, CourseRegistration} + alias Cadet.{Repo, Courses} test "swagger" do assert is_map(UserController.swagger_definitions()) @@ -100,7 +100,8 @@ defmodule CadetWeb.UserControllerTest do "courseName" => "Programming Methodology", "sourceChapter" => 1, "sourceVariant" => "default", - "viewable" => true + "viewable" => true, + "assetsPrefix" => Courses.assets_prefix(course) }, "assessmentConfigurations" => [ %{ @@ -226,7 +227,8 @@ defmodule CadetWeb.UserControllerTest do "courseName" => "Programming Methodology", "sourceChapter" => 1, "sourceVariant" => "default", - "viewable" => true + "viewable" => true, + "assetsPrefix" => Courses.assets_prefix(course) }, "assessmentConfigurations" => [] } diff --git a/test/fixtures/vcr_cassettes/aws/controller_delete_asset#3.json b/test/fixtures/vcr_cassettes/aws/controller_delete_asset#3.json new file mode 100644 index 000000000..17c8584c9 --- /dev/null +++ b/test/fixtures/vcr_cassettes/aws/controller_delete_asset#3.json @@ -0,0 +1,65 @@ +[ + { + "request": { + "body": "", + "headers": { + "Authorization": "***", + "host": "s3.ap-southeast-1.amazonaws.com", + "x-amz-date": "20200704T190423Z", + "content-length": "0", + "x-amz-content-sha256": "***" + }, + "method": "head", + "options": { + "with_body": "true", + "recv_timeout": 660000 + }, + "request_body": "", + "url": "https://s3.ap-southeast-1.amazonaws.com/test-sa-assets/this-is-a-prefix/testFolder/nestedFolder/test2.png" + }, + "response": { + "binary": false, + "body": null, + "headers": { + "Date": "Sat, 04 Jul 2020 19:04:24 GMT", + "Last-Modified": "Sat, 04 Jul 2020 18:58:52 GMT", + "ETag": "\"3104001edec38fadeb925b9dbddab198\"", + "Accept-Ranges": "bytes", + "Content-Type": "image/png", + "Content-Length": "8035", + "Server": "AmazonS3" + }, + "status_code": 200, + "type": "ok" + } + }, + { + "request": { + "body": "", + "headers": { + "Authorization": "***", + "host": "s3.ap-southeast-1.amazonaws.com", + "x-amz-date": "20200704T190423Z", + "content-length": "0", + "x-amz-content-sha256": "***" + }, + "method": "delete", + "options": { + "with_body": "true", + "recv_timeout": 660000 + }, + "request_body": "", + "url": "https://s3.ap-southeast-1.amazonaws.com/test-sa-assets/this-is-a-prefix/testFolder/nestedFolder/test2.png" + }, + "response": { + "binary": false, + "body": "", + "headers": { + "Date": "Sat, 04 Jul 2020 19:04:24 GMT", + "Server": "AmazonS3" + }, + "status_code": 204, + "type": "ok" + } + } +] diff --git a/test/fixtures/vcr_cassettes/aws/controller_list_assets#2.json b/test/fixtures/vcr_cassettes/aws/controller_list_assets#2.json new file mode 100644 index 000000000..ed453772e --- /dev/null +++ b/test/fixtures/vcr_cassettes/aws/controller_list_assets#2.json @@ -0,0 +1,33 @@ +[ + { + "request": { + "body": "", + "headers": { + "Authorization": "***", + "host": "s3.ap-southeast-1.amazonaws.com", + "x-amz-date": "20200704T190455Z", + "x-amz-content-sha256": "***" + }, + "method": "get", + "options": { + "with_body": "true", + "recv_timeout": 660000 + }, + "request_body": "", + "url": "https://s3.ap-southeast-1.amazonaws.com/test-sa-assets/" + }, + "response": { + "binary": false, + "body": "\ntest-sa-assetsthis-is-a-prefix/testFolder/1000falsetestFolder/2020-07-04T18:53:11.000Z"d41d8cd98f00b204e9800998ecf8427e"098bd2bd2de0c976fb511f741fea454cb1026b9e1f9ac9160fd9f51d07e765f19unixsp+cs1101sSTANDARDtestFolder/test.png2020-07-04T19:04:50.000Z"3104001edec38fadeb925b9dbddab198"803598bd2bd2de0c976fb511f741fea454cb1026b9e1f9ac9160fd9f51d07e765f19unixsp+cs1101sSTANDARDtestFolder/test2.png2020-07-04T19:04:50.000Z"3104001edec38fadeb925b9dbddab198"803598bd2bd2de0c976fb511f741fea454cb1026b9e1f9ac9160fd9f51d07e765f19unixsp+cs1101sSTANDARD", + "headers": { + "Date": "Sat, 04 Jul 2020 19:04:57 GMT", + "x-amz-bucket-region": "ap-southeast-1", + "Content-Type": "application/xml", + "Transfer-Encoding": "chunked", + "Server": "AmazonS3" + }, + "status_code": 200, + "type": "ok" + } + } +] diff --git a/test/fixtures/vcr_cassettes/aws/controller_upload_asset#3.json b/test/fixtures/vcr_cassettes/aws/controller_upload_asset#3.json new file mode 100644 index 000000000..84a8fc07f --- /dev/null +++ b/test/fixtures/vcr_cassettes/aws/controller_upload_asset#3.json @@ -0,0 +1,93 @@ +[ + { + "request": { + "body": "", + "headers": { + "Authorization": "***", + "host": "s3.ap-southeast-1.amazonaws.com", + "x-amz-date": "20210802T110004Z", + "content-length": "0", + "x-amz-content-sha256": "***" + }, + "method": "post", + "options": { + "with_body": "true", + "recv_timeout": 660000 + }, + "request_body": "", + "url": "https://s3.ap-southeast-1.amazonaws.com/test-sa-assets/this-is-a-prefix/testFolder/nestedFolder/test.png" + }, + "response": { + "binary": false, + "body": "\ntest-sa-assetsthis-is-a-prefix/testFolder/nestedFolder/test.pngjQOloaBCXjlYK5iRAQKuUVpX4bDPKZ0RSW8T3ndp4N.ZnW_e3eoE.AermyCbfytWRJ.i.tQ4lKSvruRteBTviJ5.492AlTIm6D08VEaszDayFRN_gGrdPclOZgxcT1b.UZ9V.Nku10KjcT7fc3BWbQ--", + "headers": { + "Date": "Mon, 02 Aug 2021 11:00:04 GMT", + "Transfer-Encoding": "chunked", + "Server": "AmazonS3" + }, + "status_code": 200, + "type": "ok" + } + }, + { + "request": { + "body": "", + "headers": { + "Authorization": "***", + "host": "s3.ap-southeast-1.amazonaws.com", + "x-amz-date": "20210802T110004Z", + "content-length": "0", + "x-amz-content-sha256": "***" + }, + "method": "head", + "options": { + "with_body": "true", + "recv_timeout": 660000 + }, + "request_body": "", + "url": "https://s3.ap-southeast-1.amazonaws.com/test-sa-assets/this-is-a-prefix/testFolder/nestedFolder/test.png" + }, + "response": { + "binary": false, + "body": null, + "headers": { + "Content-Type": "application/xml", + "Date": "Mon, 02 Aug 2021 11:00:02 GMT", + "Server": "AmazonS3" + }, + "status_code": 404, + "type": "ok" + } + }, + { + "request": { + "body": "�PNG\r\n\u001a\n\u0000\u0000\u0000\rIHDR\u0000\u0000\u0000\u0001\u0000\u0000\u0000\u0001\u0001\u0003\u0000\u0000\u0000%�V�\u0000\u0000\u0000\u0003PLTE�M\u0000\\58\u0000\u0000\u0000\u0001tRNS��4V�\u0000\u0000\u0000\nIDATx�cb\u0000\u0000\u0000\u0006\u0000\u000367|�\u0000\u0000\u0000\u0000IEND�B`�", + "headers": { + "Authorization": "***", + "host": "s3.ap-southeast-1.amazonaws.com", + "x-amz-date": "20210802T110004Z", + "content-length": "95", + "x-amz-content-sha256": "***" + }, + "method": "put", + "options": { + "with_body": "true", + "recv_timeout": 660000 + }, + "request_body": "", + "url": "https://s3.ap-southeast-1.amazonaws.com/test-sa-assets/this-is-a-prefix/testFolder/nestedFolder/test.png" + }, + "response": { + "binary": false, + "body": "", + "headers": { + "Date": "Mon, 02 Aug 2021 11:00:04 GMT", + "ETag": "\"60cf42b4d05caf10cf8bb15c0817a7b4\"", + "Server": "AmazonS3", + "Content-Length": "0" + }, + "status_code": 200, + "type": "ok" + } + } +] From e6819757bdedb876f4df01fb8482f62e4443e7ab Mon Sep 17 00:00:00 2001 From: angelsl Date: Thu, 5 Aug 2021 03:33:38 +0800 Subject: [PATCH 2/2] Strip prefix from list result --- lib/cadet/assets/assets.ex | 6 +++++- test/cadet/assets/assets_test.exs | 6 +++--- .../vcr_cassettes/aws/controller_list_assets#1.json | 2 +- .../vcr_cassettes/aws/controller_list_assets#2.json | 2 +- test/fixtures/vcr_cassettes/aws/model_list_assets#1.json | 2 +- 5 files changed, 11 insertions(+), 7 deletions(-) diff --git a/lib/cadet/assets/assets.ex b/lib/cadet/assets/assets.ex index b9244829c..379726b35 100644 --- a/lib/cadet/assets/assets.ex +++ b/lib/cadet/assets/assets.ex @@ -33,12 +33,16 @@ defmodule Cadet.Assets.Assets do end def list_assets(prefix, folder_name) do + prefix_len = byte_size(prefix) + case validate_folder_name(folder_name) do :ok -> bucket() |> S3.list_objects(prefix: prefix <> folder_name <> "/") |> ExAws.stream!() - |> Enum.map(fn file -> file.key end) + |> Enum.map(fn file -> + binary_part(file.key, prefix_len, byte_size(file.key) - prefix_len) + end) {:error, _} = error -> error diff --git a/test/cadet/assets/assets_test.exs b/test/cadet/assets/assets_test.exs index fee9ea21c..413861dee 100644 --- a/test/cadet/assets/assets_test.exs +++ b/test/cadet/assets/assets_test.exs @@ -8,9 +8,9 @@ defmodule Cadet.Assets.AssetsTest do test "accessible folder" do use_cassette "aws/model_list_assets#1" do assert Assets.list_assets(prefix(1), "testFolder") === [ - "1/testFolder/", - "1/testFolder/test.png", - "1/testFolder/test2.png" + "testFolder/", + "testFolder/test.png", + "testFolder/test2.png" ] end end diff --git a/test/fixtures/vcr_cassettes/aws/controller_list_assets#1.json b/test/fixtures/vcr_cassettes/aws/controller_list_assets#1.json index cc525f19b..0f2ecc1c6 100644 --- a/test/fixtures/vcr_cassettes/aws/controller_list_assets#1.json +++ b/test/fixtures/vcr_cassettes/aws/controller_list_assets#1.json @@ -18,7 +18,7 @@ }, "response": { "binary": false, - "body": "\ntest-sa-assetscourses-test/117/testFolder/1000falsetestFolder/2020-07-04T18:53:11.000Z"d41d8cd98f00b204e9800998ecf8427e"098bd2bd2de0c976fb511f741fea454cb1026b9e1f9ac9160fd9f51d07e765f19unixsp+cs1101sSTANDARDtestFolder/test.png2020-07-04T19:04:50.000Z"3104001edec38fadeb925b9dbddab198"803598bd2bd2de0c976fb511f741fea454cb1026b9e1f9ac9160fd9f51d07e765f19unixsp+cs1101sSTANDARDtestFolder/test2.png2020-07-04T19:04:50.000Z"3104001edec38fadeb925b9dbddab198"803598bd2bd2de0c976fb511f741fea454cb1026b9e1f9ac9160fd9f51d07e765f19unixsp+cs1101sSTANDARD", + "body": "\ntest-sa-assetscourses-test/117/testFolder/1000falsecourses-test/117/testFolder/2020-07-04T18:53:11.000Z"d41d8cd98f00b204e9800998ecf8427e"098bd2bd2de0c976fb511f741fea454cb1026b9e1f9ac9160fd9f51d07e765f19unixsp+cs1101sSTANDARDcourses-test/117/testFolder/test.png2020-07-04T19:04:50.000Z"3104001edec38fadeb925b9dbddab198"803598bd2bd2de0c976fb511f741fea454cb1026b9e1f9ac9160fd9f51d07e765f19unixsp+cs1101sSTANDARDcourses-test/117/testFolder/test2.png2020-07-04T19:04:50.000Z"3104001edec38fadeb925b9dbddab198"803598bd2bd2de0c976fb511f741fea454cb1026b9e1f9ac9160fd9f51d07e765f19unixsp+cs1101sSTANDARD", "headers": { "Date": "Sat, 04 Jul 2020 19:04:57 GMT", "x-amz-bucket-region": "ap-southeast-1", diff --git a/test/fixtures/vcr_cassettes/aws/controller_list_assets#2.json b/test/fixtures/vcr_cassettes/aws/controller_list_assets#2.json index ed453772e..ecf77d37b 100644 --- a/test/fixtures/vcr_cassettes/aws/controller_list_assets#2.json +++ b/test/fixtures/vcr_cassettes/aws/controller_list_assets#2.json @@ -18,7 +18,7 @@ }, "response": { "binary": false, - "body": "\ntest-sa-assetsthis-is-a-prefix/testFolder/1000falsetestFolder/2020-07-04T18:53:11.000Z"d41d8cd98f00b204e9800998ecf8427e"098bd2bd2de0c976fb511f741fea454cb1026b9e1f9ac9160fd9f51d07e765f19unixsp+cs1101sSTANDARDtestFolder/test.png2020-07-04T19:04:50.000Z"3104001edec38fadeb925b9dbddab198"803598bd2bd2de0c976fb511f741fea454cb1026b9e1f9ac9160fd9f51d07e765f19unixsp+cs1101sSTANDARDtestFolder/test2.png2020-07-04T19:04:50.000Z"3104001edec38fadeb925b9dbddab198"803598bd2bd2de0c976fb511f741fea454cb1026b9e1f9ac9160fd9f51d07e765f19unixsp+cs1101sSTANDARD", + "body": "\ntest-sa-assetsthis-is-a-prefix/testFolder/1000falsethis-is-a-prefix/testFolder/2020-07-04T18:53:11.000Z"d41d8cd98f00b204e9800998ecf8427e"098bd2bd2de0c976fb511f741fea454cb1026b9e1f9ac9160fd9f51d07e765f19unixsp+cs1101sSTANDARDthis-is-a-prefix/testFolder/test.png2020-07-04T19:04:50.000Z"3104001edec38fadeb925b9dbddab198"803598bd2bd2de0c976fb511f741fea454cb1026b9e1f9ac9160fd9f51d07e765f19unixsp+cs1101sSTANDARDthis-is-a-prefix/testFolder/test2.png2020-07-04T19:04:50.000Z"3104001edec38fadeb925b9dbddab198"803598bd2bd2de0c976fb511f741fea454cb1026b9e1f9ac9160fd9f51d07e765f19unixsp+cs1101sSTANDARD", "headers": { "Date": "Sat, 04 Jul 2020 19:04:57 GMT", "x-amz-bucket-region": "ap-southeast-1", diff --git a/test/fixtures/vcr_cassettes/aws/model_list_assets#1.json b/test/fixtures/vcr_cassettes/aws/model_list_assets#1.json index 788ff2393..819085eff 100644 --- a/test/fixtures/vcr_cassettes/aws/model_list_assets#1.json +++ b/test/fixtures/vcr_cassettes/aws/model_list_assets#1.json @@ -18,7 +18,7 @@ }, "response": { "binary": false, - "body": "\ntest-sa-assetscourses-test/1/testFolder/1000false1/testFolder/2021-08-02T10:23:39.000Z"d41d8cd98f00b204e9800998ecf8427e"098bd2bd2de0c976fb511f741fea454cb1026b9e1f9ac9160fd9f51d07e765f19unixsp+cs1101sSTANDARD1/testFolder/test.png2021-08-02T10:56:36.000Z"33c5d4f094a7727780a3c63babae1083"256898bd2bd2de0c976fb511f741fea454cb1026b9e1f9ac9160fd9f51d07e765f19unixsp+cs1101sSTANDARD1/testFolder/test2.png2021-08-02T10:25:15.000Z"af2ab457d8b118efa176bc12cff4895f"299698bd2bd2de0c976fb511f741fea454cb1026b9e1f9ac9160fd9f51d07e765f19unixsp+cs1101sSTANDARD", + "body": "\ntest-sa-assetscourses-test/1/testFolder/1000falsecourses-test/1/testFolder/2021-08-02T10:23:39.000Z"d41d8cd98f00b204e9800998ecf8427e"098bd2bd2de0c976fb511f741fea454cb1026b9e1f9ac9160fd9f51d07e765f19unixsp+cs1101sSTANDARDcourses-test/1/testFolder/test.png2021-08-02T10:56:36.000Z"33c5d4f094a7727780a3c63babae1083"256898bd2bd2de0c976fb511f741fea454cb1026b9e1f9ac9160fd9f51d07e765f19unixsp+cs1101sSTANDARDcourses-test/1/testFolder/test2.png2021-08-02T10:25:15.000Z"af2ab457d8b118efa176bc12cff4895f"299698bd2bd2de0c976fb511f741fea454cb1026b9e1f9ac9160fd9f51d07e765f19unixsp+cs1101sSTANDARD", "headers": { "Date": "Mon, 02 Aug 2021 10:59:51 GMT", "x-amz-bucket-region": "ap-southeast-1",