diff --git a/Gemfile b/Gemfile index 0041f61f..0fd9ef4f 100644 --- a/Gemfile +++ b/Gemfile @@ -2,7 +2,8 @@ source 'https://rubygems.org' gemspec gem 'rest-client', :github => 'opscode/rest-client' -gem 'chef-pedant', :github => 'opscode/chef-pedant', :tag => '1.0.41' + +gem 'chef-pedant', :github => 'opscode/chef-pedant', :tag => '1.0.42' gem 'chef', :github => 'opscode/chef', :ref => '92eefc79bb28d217b15099655244228b9e1efec7' diff --git a/chef-zero.gemspec b/chef-zero.gemspec index c6bc909e..3db39a16 100644 --- a/chef-zero.gemspec +++ b/chef-zero.gemspec @@ -19,7 +19,9 @@ Gem::Specification.new do |s| s.add_dependency 'rack' s.add_development_dependency 'rake' - s.add_development_dependency 'rspec' + + # pedant incompatible with RSpec 3.2 as of pedant version 1.0.42 + s.add_development_dependency 'rspec', '~> 3.1.0' s.bindir = 'bin' s.executables = ['chef-zero'] diff --git a/lib/chef_zero/data_store/raw_file_store.rb b/lib/chef_zero/data_store/raw_file_store.rb index dce6374f..6da7dd99 100644 --- a/lib/chef_zero/data_store/raw_file_store.rb +++ b/lib/chef_zero/data_store/raw_file_store.rb @@ -44,18 +44,20 @@ def clear if destructible Dir.entries(root).each do |entry| next if entry == '.' || entry == '..' - FileUtils.rm_rf(Path.join(root, entry)) + FileUtils.rm_rf(File.join(root, entry)) end end end def create_dir(path, name, *options) + pp create_dir: {path: path, name: name} real_path = path_to(path, name) + pp create_dir_real_path: real_path if options.include?(:recursive) FileUtils.mkdir_p(real_path) else begin - Dir.mkdir(File.join(path, name)) + Dir.mkdir(real_path) rescue Errno::ENOENT raise DataNotFoundError.new(path) rescue Errno::EEXIST diff --git a/lib/chef_zero/endpoints/cookbook_artifact_endpoint.rb b/lib/chef_zero/endpoints/cookbook_artifact_endpoint.rb new file mode 100644 index 00000000..ced0d9f5 --- /dev/null +++ b/lib/chef_zero/endpoints/cookbook_artifact_endpoint.rb @@ -0,0 +1,69 @@ +require 'chef_zero/endpoints/cookbooks_base' + +module ChefZero + module Endpoints + # /cookbook_artifacts/NAME + class CookbookArtifactEndpoint < CookbooksBase + def get(request) + filter = request.rest_path[3] + case filter + when '_latest' + result = {} + filter_cookbooks(all_cookbooks_list(request), {}, 1) do |name, versions| + if versions.size > 0 + result[name] = build_uri(request.base_uri, request.rest_path[0..1] + ['cookbook_artifacts', name, versions[0]]) + end + end + json_response(200, result) + when '_recipes' + result = [] + filter_cookbooks(all_cookbooks_list(request), {}, 1) do |name, versions| + if versions.size > 0 + cookbook = FFI_Yajl::Parser.parse(get_data(request, request.rest_path[0..1] + ['cookbook_artifacts', name, versions[0]]), :create_additions => false) + result += recipe_names(name, cookbook) + end + end + json_response(200, result.sort) + else + cookbook_list = { filter => list_data(request, request.rest_path) } + json_response(200, format_cookbooks_list(request, cookbook_list)) + end + end + + def latest_version(versions) + sorted = versions.sort_by { |version| Gem::Version.new(version.dup) } + sorted[-1] + end + + ## CookbooksBase Overrides + # Methods here override behavior in CookbooksBase that is otherwise + # hard-coded to 'cookbooks' + + def format_cookbooks_list(request, cookbooks_list, constraints = {}, num_versions = nil) + results = {} + filter_cookbooks(cookbooks_list, constraints, num_versions) do |name, versions| + versions_list = versions.map do |version| + { + 'url' => build_uri(request.base_uri, request.rest_path[0..1] + ['cookbook_artifacts', name, version]), + 'version' => version + } + end + results[name] = { + 'url' => build_uri(request.base_uri, request.rest_path[0..1] + ['cookbook_artifacts', name]), + 'versions' => versions_list + } + end + results + end + + def all_cookbooks_list(request) + result = {} + # Race conditions exist here (if someone deletes while listing). I don't care. + data_store.list(request.rest_path[0..1] + ['cookbook_artifacts']).each do |name| + result[name] = data_store.list(request.rest_path[0..1] + ['cookbook_artifacts', name]) + end + result + end + end + end +end diff --git a/lib/chef_zero/endpoints/cookbook_artifact_version_endpoint.rb b/lib/chef_zero/endpoints/cookbook_artifact_version_endpoint.rb new file mode 100644 index 00000000..d6f1d28b --- /dev/null +++ b/lib/chef_zero/endpoints/cookbook_artifact_version_endpoint.rb @@ -0,0 +1,119 @@ +require 'ffi_yajl' +require 'chef_zero/endpoints/rest_object_endpoint' +require 'chef_zero/rest_error_response' +require 'chef_zero/chef_data/data_normalizer' +require 'chef_zero/data_store/data_not_found_error' + +module ChefZero + module Endpoints + # /organizations/ORG/cookbook_artifacts/NAME/VERSION + class CookbookArtifactVersionEndpoint < RestObjectEndpoint + def get(request) + if request.rest_path[4] == "_latest" || request.rest_path[4] == "latest" + request.rest_path[4] = latest_version(list_data(request, request.rest_path[0..3])) + end + super(request) + end + + def put(request) + name = request.rest_path[3] + version = request.rest_path[4] + existing_cookbook = get_data(request, request.rest_path, :nil) + + # Honor frozen + if existing_cookbook + existing_cookbook_json = FFI_Yajl::Parser.parse(existing_cookbook, :create_additions => false) + if existing_cookbook_json['frozen?'] + if request.query_params['force'] != "true" + raise RestErrorResponse.new(409, "The cookbook #{name} at version #{version} is frozen. Use the 'force' option to override.") + end + # For some reason, you are forever unable to modify "frozen?" on a frozen cookbook. + request_body = FFI_Yajl::Parser.parse(request.body, :create_additions => false) + if !request_body['frozen?'] + request_body['frozen?'] = true + request.body = FFI_Yajl::Encoder.encode(request_body, :pretty => true) + end + end + end + + # Set the cookbook + set_data(request, request.rest_path, request.body, :create_dir, :create) + + # If the cookbook was updated, check for deleted files and clean them up + if existing_cookbook + missing_checksums = get_checksums(existing_cookbook) - get_checksums(request.body) + if missing_checksums.size > 0 + hoover_unused_checksums(missing_checksums, request) + end + end + + already_json_response(existing_cookbook ? 200 : 201, populate_defaults(request, request.body)) + end + + def delete(request) + if request.rest_path[4] == "_latest" || request.rest_path[4] == "latest" + request.rest_path[4] = latest_version(list_data(request, request.rest_path[0..3])) + end + + deleted_cookbook = get_data(request) + + response = super(request) + cookbook_name = request.rest_path[3] + cookbook_path = request.rest_path[0..1] + ['cookbook_artifacts', cookbook_name] + if exists_data_dir?(request, cookbook_path) && list_data(request, cookbook_path).size == 0 + delete_data_dir(request, cookbook_path) + end + + # Hoover deleted files, if they exist + hoover_unused_checksums(get_checksums(deleted_cookbook), request) + response + end + + def get_checksums(cookbook) + result = [] + FFI_Yajl::Parser.parse(cookbook, :create_additions => false).each_pair do |key, value| + if value.is_a?(Array) + value.each do |file| + if file.is_a?(Hash) && file.has_key?('checksum') + result << file['checksum'] + end + end + end + end + result.uniq + end + + private + + def hoover_unused_checksums(deleted_checksums, request) + data_store.list(request.rest_path[0..1] + ['cookbook_artifacts']).each do |cookbook_name| + data_store.list(request.rest_path[0..1] + ['cookbook_artifacts', cookbook_name]).each do |version| + cookbook = data_store.get(request.rest_path[0..1] + ['cookbook_artifacts', cookbook_name, version], request) + deleted_checksums = deleted_checksums - get_checksums(cookbook) + end + end + deleted_checksums.each do |checksum| + # There can be a race here if multiple cookbooks are uploading. + # This deals with an exception on delete, but things can still get deleted + # that shouldn't be. + begin + delete_data(request, request.rest_path[0..1] + ['file_store', 'checksums', checksum], :data_store_exceptions) + rescue ChefZero::DataStore::DataNotFoundError + end + end + end + + def populate_defaults(request, response_json) + # Inject URIs into each cookbook file + cookbook = FFI_Yajl::Parser.parse(response_json, :create_additions => false) + cookbook = ChefData::DataNormalizer.normalize_cookbook(self, request.rest_path[0..1], cookbook, request.rest_path[3], request.rest_path[4], request.base_uri, request.method) + FFI_Yajl::Encoder.encode(cookbook, :pretty => true) + end + + def latest_version(versions) + sorted = versions.sort_by { |version| Gem::Version.new(version.dup) } + sorted[-1] + end + end + end +end diff --git a/lib/chef_zero/endpoints/cookbook_artifacts_endpoint.rb b/lib/chef_zero/endpoints/cookbook_artifacts_endpoint.rb new file mode 100644 index 00000000..d8761daf --- /dev/null +++ b/lib/chef_zero/endpoints/cookbook_artifacts_endpoint.rb @@ -0,0 +1,50 @@ +require 'chef_zero/endpoints/cookbooks_base' + +module ChefZero + module Endpoints + # /cookbook_artifacts + class CookbookArtifactsEndpoint < CookbooksBase + def get(request) + if request.query_params['num_versions'] == 'all' + num_versions = nil + elsif request.query_params['num_versions'] + num_versions = request.query_params['num_versions'].to_i + else + num_versions = 1 + end + json_response(200, format_cookbooks_list(request, all_cookbooks_list(request), {}, num_versions)) + end + + ## CookbooksBase Overrides + # Methods here override behavior in CookbooksBase that is otherwise + # hard-coded to 'cookbooks' + + def format_cookbooks_list(request, cookbooks_list, constraints = {}, num_versions = nil) + results = {} + filter_cookbooks(cookbooks_list, constraints, num_versions) do |name, versions| + versions_list = versions.map do |version| + { + 'url' => build_uri(request.base_uri, request.rest_path[0..1] + ['cookbook_artifacts', name, version]), + 'version' => version + } + end + results[name] = { + 'url' => build_uri(request.base_uri, request.rest_path[0..1] + ['cookbook_artifacts', name]), + 'versions' => versions_list + } + end + results + end + + def all_cookbooks_list(request) + result = {} + # Race conditions exist here (if someone deletes while listing). I don't care. + data_store.list(request.rest_path[0..1] + ['cookbook_artifacts']).each do |name| + result[name] = data_store.list(request.rest_path[0..1] + ['cookbook_artifacts', name]) + end + result + end + + end + end +end diff --git a/lib/chef_zero/rest_base.rb b/lib/chef_zero/rest_base.rb index 3fa017a3..ec43f5e0 100644 --- a/lib/chef_zero/rest_base.rb +++ b/lib/chef_zero/rest_base.rb @@ -116,10 +116,12 @@ def set_data(request, rest_path, data, *options) rest_path ||= request.rest_path begin data_store.set(rest_path, data, *options, :requestor => request.requestor) - rescue DataStore::DataNotFoundError + rescue DataStore::DataNotFoundError => e if options.include?(:data_store_exceptions) raise else + puts e + puts e.backtrace raise RestErrorResponse.new(404, "Object not found: #{build_uri(request.base_uri, request.rest_path)}") end end diff --git a/lib/chef_zero/rest_router.rb b/lib/chef_zero/rest_router.rb index f2770d35..5b198d89 100644 --- a/lib/chef_zero/rest_router.rb +++ b/lib/chef_zero/rest_router.rb @@ -38,7 +38,7 @@ def call(request) private def find_endpoint(clean_path) - _, endpoint = routes.find { |route, endpoint| route.match(clean_path) } + _, endpoint = routes.find { |route, _endpoint| route.match(clean_path) } endpoint || not_found end end diff --git a/lib/chef_zero/server.rb b/lib/chef_zero/server.rb index de2a3f50..ba13541e 100644 --- a/lib/chef_zero/server.rb +++ b/lib/chef_zero/server.rb @@ -41,6 +41,11 @@ require 'chef_zero/endpoints/acl_endpoint' require 'chef_zero/endpoints/actors_endpoint' require 'chef_zero/endpoints/actor_endpoint' + +require 'chef_zero/endpoints/cookbook_artifact_endpoint' +require 'chef_zero/endpoints/cookbook_artifact_version_endpoint' +require 'chef_zero/endpoints/cookbook_artifacts_endpoint' + require 'chef_zero/endpoints/cookbooks_endpoint' require 'chef_zero/endpoints/cookbook_endpoint' require 'chef_zero/endpoints/cookbook_version_endpoint' @@ -500,6 +505,11 @@ def open_source_endpoints # Both [ "/organizations/*/clients", ActorsEndpoint.new(self) ], [ "/organizations/*/clients/*", ActorEndpoint.new(self) ], + + [ "/organizations/*/cookbook_artifacts", CookbookArtifactsEndpoint.new(self) ], + [ "/organizations/*/cookbook_artifacts/*", CookbookArtifactEndpoint.new(self) ], + [ "/organizations/*/cookbook_artifacts/*/*", CookbookArtifactVersionEndpoint.new(self) ], + [ "/organizations/*/cookbooks", CookbooksEndpoint.new(self) ], [ "/organizations/*/cookbooks/*", CookbookEndpoint.new(self) ], [ "/organizations/*/cookbooks/*/*", CookbookVersionEndpoint.new(self) ], diff --git a/spec/run_pedant.rb b/spec/run_pedant.rb index 2caf1fff..74bcba43 100644 --- a/spec/run_pedant.rb +++ b/spec/run_pedant.rb @@ -55,11 +55,24 @@ def start_local_server(chef_repo_path) tmpdir = Dir.mktmpdir data_store = ChefZero::DataStore::RawFileStore.new(tmpdir, true) data_store = ChefZero::DataStore::DefaultFacade.new(data_store, true, false) - server = ChefZero::Server.new(:port => 8889, :single_org => 'chef', :data_store => data_store) + + # TODO: Without this, tests don't run at all for me. But when I add this, + # there are a handful of extra clients present, which makes some tests + # fail. + data_store.create_dir([ 'organizations' ], 'pedant') + + server_opts = { :port => 8889, :single_org => 'pedant', :data_store => data_store } + server_opts[:log_level] = :debug if ENV['DEBUG_ZERO'] + + server = ChefZero::Server.new(server_opts) server.start_background else - server = ChefZero::Server.new(:port => 8889, :single_org => false, :osc_compat => true) + server_opts = { :port => 8889, :single_org => false, :osc_compat => true } + server_opts[:log_level] = :debug if ENV['DEBUG_ZERO'] + + server = ChefZero::Server.new(server_opts) + server.data_store.create_dir([ 'organizations' ], 'pedant') server.start_background end diff --git a/spec/support/oc_pedant.rb b/spec/support/oc_pedant.rb index 60e36292..626517fa 100644 --- a/spec/support/oc_pedant.rb +++ b/spec/support/oc_pedant.rb @@ -132,3 +132,7 @@ ruby_users_endpoint? false ruby_acls_endpoint? false ruby_org_assoc? false + +# The Policies endpoint is feature-flagged during development. Zero supports +# the policies endpoint, so turn it on: +policies? true