Merge pull request #5914 from BengangY/feature/non-cdn-update

Merge feature/non-cdn-update to master

ocaml/idl/datamodel.ml

@@ -8432,6 +8432,7 @@ let http_actions =
       , []
       )
     )
+  ; ("put_bundle", (Put, Constants.put_bundle_uri, true, [], _R_POOL_OP, []))
   ]
 
 (* these public http actions will NOT be checked by RBAC *)

ocaml/idl/datamodel_common.ml

@@ -10,7 +10,7 @@ open Datamodel_roles
               to leave a gap for potential hotfixes needing to increment the schema version.*)
 let schema_major_vsn = 5
 
-let schema_minor_vsn = 780
+let schema_minor_vsn = 781
 
 (* Historical schema versions just in case this is useful later *)
 let rio_schema_major_vsn = 5

ocaml/idl/datamodel_errors.ml

@@ -1895,6 +1895,19 @@ let _ =
     ~doc:"The GPG public key file name in the repository is invalid." () ;
   error Api_errors.repository_already_exists ["ref"]
     ~doc:"The repository already exists." () ;
+  error Api_errors.bundle_repository_already_exists ["ref"]
+    ~doc:"The bundle repository already exists." () ;
+  error Api_errors.bundle_unpack_failed ["error"]
+    ~doc:"Failed to unpack bundle file." () ;
+  error Api_errors.bundle_repo_not_enabled []
+    ~doc:"Cannot sync bundle as the bundle repository is not enabled." () ;
+  error Api_errors.can_not_sync_updates []
+    ~doc:"Cannot sync updates as the bundle repository is enabled." () ;
+  error Api_errors.bundle_repo_should_be_single_enabled []
+    ~doc:
+      "If the bundle repository is enabled, it should be the only one enabled \
+       repository of the pool."
+    () ;
   error Api_errors.repository_is_in_use [] ~doc:"The repository is in use." () ;
   error Api_errors.repository_cleanup_failed []
     ~doc:"Failed to clean up local repository on coordinator." () ;
@@ -1907,8 +1920,15 @@ let _ =
       "The operation could not be performed because syncing updates is in \
        progress."
     () ;
+  error Api_errors.sync_bundle_in_progress []
+    ~doc:
+      "The operation could not be performed because syncing bundle is in \
+       progress."
+    () ;
   error Api_errors.reposync_failed []
     ~doc:"Syncing with remote YUM repository failed." () ;
+  error Api_errors.bundle_sync_failed []
+    ~doc:"Syncing with bundle repository failed." () ;
   error Api_errors.invalid_repomd_xml [] ~doc:"The repomd.xml is invalid." () ;
   error Api_errors.invalid_updateinfo_xml []
     ~doc:"The updateinfo.xml is invalid." () ;

ocaml/idl/datamodel_lifecycle.ml

@@ -25,6 +25,8 @@ let prototyped_of_field = function
       Some "23.14.0"
   | "Observer", "uuid" ->
       Some "23.14.0"
+  | "Repository", "origin" ->
+      Some "24.21.0-next"
   | "Repository", "gpgkey_path" ->
       Some "22.12.0"
   | "Certificate", "fingerprint_sha1" ->
@@ -123,6 +125,8 @@ let prototyped_of_message = function
       Some "22.20.0"
   | "Repository", "set_gpgkey_path" ->
       Some "22.12.0"
+  | "Repository", "introduce_bundle" ->
+      Some "24.21.0-next"
   | "PCI", "get_dom0_access_status" ->
       Some "24.14.0"
   | "PCI", "enable_dom0_access" ->

ocaml/idl/datamodel_pool.ml

@@ -21,6 +21,9 @@ let operations =
       ; ( "sync_updates"
         , "Indicates this pool is in the process of syncing updates"
         )
+      ; ( "sync_bundle"
+        , "Indicates this pool is in the process of syncing bundle"
+        )
       ; ( "get_updates"
         , "Indicates this pool is in the process of getting updates"
         )

ocaml/idl/datamodel_repository.ml

@@ -18,10 +18,19 @@ open Datamodel_roles
 
 let lifecycle = [(Lifecycle.Published, "1.301.0", "")]
 
+let origin =
+  Enum
+    ( "origin"
+    , [
+        ("remote", "The origin of the repository is a remote one")
+      ; ("bundle", "The origin of the repository is a local bundle file")
+      ]
+    )
+
 let introduce =
   call ~name:"introduce" ~in_oss_since:None
     ~lifecycle:[(Published, "1.301.0", "")]
-    ~doc:"Add the configuration for a new repository"
+    ~doc:"Add the configuration for a new remote repository"
     ~versioned_params:
       [
         {
@@ -73,6 +82,18 @@ let introduce =
     ~allowed_roles:(_R_POOL_OP ++ _R_CLIENT_CERT)
     ()
 
+let introduce_bundle =
+  call ~name:"introduce_bundle" ~in_oss_since:None ~lifecycle:[]
+    ~doc:"Add the configuration for a new bundle repository"
+    ~params:
+      [
+        (String, "name_label", "The name of the repository")
+      ; (String, "name_description", "The description of the repository")
+      ]
+    ~result:(Ref _repository, "The ref of the created repository record.")
+    ~allowed_roles:(_R_POOL_OP ++ _R_CLIENT_CERT)
+    ()
+
 let forget =
   call ~name:"forget" ~in_oss_since:None
     ~lifecycle:[(Published, "1.301.0", "")]
@@ -148,7 +169,15 @@ let t =
     ~lifecycle:[(Published, "1.301.0", "")]
     ~persist:PersistEverything ~in_oss_since:None
     ~messages_default_allowed_roles:(_R_POOL_OP ++ _R_CLIENT_CERT)
-    ~messages:[introduce; forget; apply; set_gpgkey_path; apply_livepatch]
+    ~messages:
+      [
+        introduce
+      ; introduce_bundle
+      ; forget
+      ; apply
+      ; set_gpgkey_path
+      ; apply_livepatch
+      ]
     ~contents:
       [
         uid _repository ~lifecycle:[(Published, "1.301.0", "")]
@@ -187,5 +216,10 @@ let t =
       ; field ~qualifier:StaticRO ~lifecycle:[] ~ty:String
           ~default_value:(Some (VString "")) "gpgkey_path"
           "The file name of the GPG public key of this repository"
+      ; field ~qualifier:StaticRO ~lifecycle:[] ~ty:origin "origin"
+          ~default_value:(Some (VEnum "remote"))
+          "The origin of the repository. 'remote' if the origin of the \
+           repository is a remote one, 'bundle' if the origin of the \
+           repository is a local bundle file."
       ]
     ()

ocaml/idl/schematest.ml

@@ -3,7 +3,7 @@ let hash x = Digest.string x |> Digest.to_hex
 (* BEWARE: if this changes, check that schema has been bumped accordingly in
    ocaml/idl/datamodel_common.ml, usually schema_minor_vsn *)
 
-let last_known_schema_hash = "7885f7b085e4a5e32977a4b222030412"
+let last_known_schema_hash = "4417b0087b481c3038e73f170b7d4d01"
 
 let current_schema_hash : string =
   let open Datamodel_types in

ocaml/tests/dune

@@ -9,7 +9,7 @@
       test_vm_placement test_vm_helpers test_repository test_repository_helpers
       test_ref test_xapi_helpers test_vm_group
       test_livepatch test_rpm test_updateinfo test_storage_smapiv1_wrapper test_storage_quicktest test_observer
-      test_pool_periodic_update_sync test_pkg_mgr))
+      test_pool_periodic_update_sync test_pkg_mgr test_tar_ext test_pool_repository))
   (libraries
     alcotest
     angstrom
@@ -80,16 +80,17 @@
   (names test_vm_helpers test_vm_placement test_network_sriov test_vdi_cbt
     test_clustering test_pusb test_daemon_manager test_repository test_repository_helpers
     test_livepatch test_rpm test_updateinfo test_pool_periodic_update_sync test_pkg_mgr
-    test_xapi_helpers)
+    test_xapi_helpers test_tar_ext test_pool_repository)
   (package xapi)
   (modes exe)
   (modules test_vm_helpers test_vm_placement test_network_sriov test_vdi_cbt
     test_event test_clustering test_cluster_host test_cluster test_pusb
     test_daemon_manager test_repository test_repository_helpers test_livepatch test_rpm
     test_updateinfo test_pool_periodic_update_sync test_pkg_mgr
-    test_xapi_helpers)
+    test_xapi_helpers test_tar_ext test_pool_repository)
   (libraries
     alcotest
+    bos
     fmt
     ptime
     result

ocaml/tests/test_data/gen_tar_ext_test_file.sh

@@ -0,0 +1,62 @@
+#!/bin/bash
+#
+# Copyright (c) Cloud Software Group, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published
+# by the Free Software Foundation; version 2.1 only. with the special
+# exception on linking described in file LICENSE.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+
+test_file_dir=$1
+mkdir -p "${test_file_dir}"
+cd "${test_file_dir}" || exit
+
+echo "========= Generating regular tar file ========="
+echo "This is file-1" > file1.txt
+echo "This is file-2" > file2.txt
+tar -cvf test_tar_ext_regular.tar file1.txt file2.txt
+
+echo "========= Generating tar file with illegal path ========="
+mkdir test_illegal_dir
+touch test_illegal_dir/file
+tar --absolute-names -cvf test_tar_ext_illegal_path.tar test_illegal_dir/../file1.txt
+
+echo "========= Generating tar file trying to escape the current dir ========="
+mkdir current_dir
+mkdir another_dir
+touch current_dir/file
+touch another_dir/escaped_file
+tar --absolute-names -cvf current_dir/test_tar_ext_trying_to_escape.tar current_dir/../another_dir/escaped_file
+
+echo "========= Generating tar file with absolute path starting from '/' ========="
+tar --absolute-names -cvf test_tar_ext_absolute_path.tar /usr/bin/ls
+
+echo "========= Generating tar file with unsupported file type ========="
+ln -s file1.txt link
+tar -cvf test_tar_ext_unsupported_file_type.tar link
+
+echo "========= Generating tar file unpacked exceeds max size limit ========="
+dd if=/dev/zero of=file1 bs=1M count=1
+dd if=/dev/zero of=file2 bs=1M count=1
+dd if=/dev/zero of=file3 bs=1M count=1
+tar -cvf test_tar_ext_unpacked_exceeds_max_size.tar file1 file2 file3
+
+echo "========= Generating size mismatch tar file ========="
+split -b 100000 test_tar_ext_unpacked_exceeds_max_size.tar test_tar_ext_file_size_mismatch.
+mv test_tar_ext_file_size_mismatch.aa test_tar_ext_file_size_mismatch.tar
+
+echo "========= Generating incomplete tar file ========="
+mv file1.txt test_tar_ext_file_incomplete.tar
+
+echo "========= Generating corrupted tar file ========="
+cp test_tar_ext_regular.tar test_tar_ext_corrupted_file.tar
+sed -i 's/file1.txt/file3.txt/g' test_tar_ext_corrupted_file.tar
+
+echo "========= Generating unpacking failure file ========="
+cp test_tar_ext_regular.tar test_tar_ext_unpacking_failure.tar
+sed -i 's/file1.txt/file.txt/g' test_tar_ext_unpacking_failure.tar

ocaml/tests/test_pool_repository.ml

@@ -0,0 +1,116 @@
+(*
+ * Copyright (c) Cloud Software Group, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation; version 2.1 only. with the special
+ * exception on linking described in file LICENSE.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *)
+
+module T = Test_common
+
+let test_set_remote_and_bundle_repos () =
+  let __context = T.make_test_database () in
+  let name_label = "remote" in
+  let name_description = "remote" in
+  let binary_url = "https://repo.example.com" in
+  let source_url = "https://repo-src.example.com" in
+  let gpgkey_path = "" in
+  let ref_remote =
+    Repository.introduce ~__context ~name_label ~name_description ~binary_url
+      ~source_url ~update:true ~gpgkey_path
+  in
+  let ref_bundle =
+    Repository.introduce_bundle ~__context ~name_label:"bundle"
+      ~name_description:"bundle"
+  in
+  let self = Helpers.get_pool ~__context in
+  Alcotest.check_raises "test_set_remote_and_bundle_repos"
+    Api_errors.(Server_error (bundle_repo_should_be_single_enabled, []))
+    (fun () ->
+      Xapi_pool.set_repositories ~__context ~self
+        ~value:[ref_remote; ref_bundle]
+    )
+
+let test_add_bundle_repo () =
+  let __context = T.make_test_database () in
+  let name_label = "remote" in
+  let name_description = "remote" in
+  let binary_url = "https://repo.example.com" in
+  let source_url = "https://repo-src.example.com" in
+  let gpgkey_path = "" in
+  let ref_remote =
+    Repository.introduce ~__context ~name_label ~name_description ~binary_url
+      ~source_url ~update:true ~gpgkey_path
+  in
+  let ref_bundle =
+    Repository.introduce_bundle ~__context ~name_label:"bundle"
+      ~name_description:"bundle"
+  in
+  let self = Helpers.get_pool ~__context in
+  Alcotest.check_raises "test_add_bundle_repo"
+    Api_errors.(Server_error (bundle_repo_should_be_single_enabled, []))
+    (fun () ->
+      Xapi_pool.set_repositories ~__context ~self ~value:[ref_remote] ;
+      Xapi_pool.add_repository ~__context ~self ~value:ref_bundle
+    )
+
+let test_add_remote_repo () =
+  let __context = T.make_test_database () in
+  let name_label = "remote" in
+  let name_description = "remote" in
+  let binary_url = "https://repo.example.com" in
+  let source_url = "https://repo-src.example.com" in
+  let gpgkey_path = "" in
+  let ref_remote =
+    Repository.introduce ~__context ~name_label ~name_description ~binary_url
+      ~source_url ~update:true ~gpgkey_path
+  in
+  let ref_bundle =
+    Repository.introduce_bundle ~__context ~name_label:"bundle"
+      ~name_description:"bundle"
+  in
+  let self = Helpers.get_pool ~__context in
+  Alcotest.check_raises "test_add_remote_repo"
+    Api_errors.(Server_error (bundle_repo_should_be_single_enabled, []))
+    (fun () ->
+      Xapi_pool.set_repositories ~__context ~self ~value:[ref_bundle] ;
+      Xapi_pool.add_repository ~__context ~self ~value:ref_remote
+    )
+
+let test_can_not_enable_bundle_repo_auto_sync () =
+  let __context = T.make_test_database () in
+  let ref_bundle =
+    Repository.introduce_bundle ~__context ~name_label:"bundle"
+      ~name_description:"bundle"
+  in
+  let self = Helpers.get_pool ~__context in
+  Alcotest.check_raises "test_can_not_enable_bundle_repo_auto_sync"
+    Api_errors.(Server_error (can_not_sync_updates, []))
+    (fun () ->
+      Xapi_pool.set_repositories ~__context ~self ~value:[ref_bundle] ;
+      Xapi_pool.set_update_sync_enabled ~__context ~self ~value:true
+    )
+
+let test =
+  [
+    ( "test_set_remote_and_bundle_repos"
+    , `Quick
+    , test_set_remote_and_bundle_repos
+    )
+  ; ("test_add_bundle_repo", `Quick, test_add_bundle_repo)
+  ; ("test_add_remote_repo", `Quick, test_add_remote_repo)
+  ; ( "test_can_not_enable_bundle_repo_auto_sync"
+    , `Quick
+    , test_can_not_enable_bundle_repo_auto_sync
+    )
+  ]
+
+let () =
+  Suite_init.harness_init () ;
+  Alcotest.run "Test Pool Repository suite" [("Test_pool_repository", test)]