Cookbook Artifact API

Gemfile

@@ -2,7 +2,8 @@ source 'https://rubygems.org'
 gemspec
 
 gem 'rest-client', :github => 'opscode/rest-client'
-gem 'chef-pedant', :github => 'opscode/chef-pedant', :tag => '1.0.41'
+
+gem 'chef-pedant', :github => 'opscode/chef-pedant', :tag => '1.0.42'
 
 gem 'chef', :github => 'opscode/chef', :ref => '92eefc79bb28d217b15099655244228b9e1efec7'
 

chef-zero.gemspec

@@ -19,7 +19,9 @@ Gem::Specification.new do |s|
   s.add_dependency 'rack'
 
   s.add_development_dependency 'rake'
-  s.add_development_dependency 'rspec'
+
+  # pedant incompatible with RSpec 3.2 as of pedant version 1.0.42
+  s.add_development_dependency 'rspec', '~> 3.1.0'
 
   s.bindir       = 'bin'
   s.executables  = ['chef-zero']

lib/chef_zero/data_store/raw_file_store.rb

@@ -44,18 +44,20 @@ def clear
         if destructible
           Dir.entries(root).each do |entry|
             next if entry == '.' || entry == '..'
-            FileUtils.rm_rf(Path.join(root, entry))
+            FileUtils.rm_rf(File.join(root, entry))
           end
         end
       end
 
       def create_dir(path, name, *options)
+        pp create_dir: {path: path, name: name}
         real_path = path_to(path, name)
+        pp create_dir_real_path: real_path
         if options.include?(:recursive)
           FileUtils.mkdir_p(real_path)
         else
           begin
-            Dir.mkdir(File.join(path, name))
+            Dir.mkdir(real_path)
           rescue Errno::ENOENT
             raise DataNotFoundError.new(path)
           rescue Errno::EEXIST

lib/chef_zero/endpoints/cookbook_artifact_endpoint.rb

@@ -0,0 +1,69 @@
+require 'chef_zero/endpoints/cookbooks_base'
+
+module ChefZero
+  module Endpoints
+    # /cookbook_artifacts/NAME
+    class CookbookArtifactEndpoint < CookbooksBase
+      def get(request)
+        filter = request.rest_path[3]
+        case filter
+        when '_latest'
+          result = {}
+          filter_cookbooks(all_cookbooks_list(request), {}, 1) do |name, versions|
+            if versions.size > 0
+              result[name] = build_uri(request.base_uri, request.rest_path[0..1] + ['cookbook_artifacts', name, versions[0]])
+            end
+          end
+          json_response(200, result)
+        when '_recipes'
+          result = []
+          filter_cookbooks(all_cookbooks_list(request), {}, 1) do |name, versions|
+            if versions.size > 0
+              cookbook = FFI_Yajl::Parser.parse(get_data(request, request.rest_path[0..1] + ['cookbook_artifacts', name, versions[0]]), :create_additions => false)
+              result += recipe_names(name, cookbook)
+            end
+          end
+          json_response(200, result.sort)
+        else
+          cookbook_list = { filter => list_data(request, request.rest_path) }
+          json_response(200, format_cookbooks_list(request, cookbook_list))
+        end
+      end
+
+      def latest_version(versions)
+        sorted = versions.sort_by { |version| Gem::Version.new(version.dup) }
+        sorted[-1]
+      end
+
+      ## CookbooksBase Overrides
+      # Methods here override behavior in CookbooksBase that is otherwise
+      # hard-coded to 'cookbooks'
+
+      def format_cookbooks_list(request, cookbooks_list, constraints = {}, num_versions = nil)
+        results = {}
+        filter_cookbooks(cookbooks_list, constraints, num_versions) do |name, versions|
+          versions_list = versions.map do |version|
+            {
+              'url' => build_uri(request.base_uri, request.rest_path[0..1] + ['cookbook_artifacts', name, version]),
+              'version' => version
+            }
+          end
+          results[name] = {
+            'url' => build_uri(request.base_uri, request.rest_path[0..1] + ['cookbook_artifacts', name]),
+            'versions' => versions_list
+          }
+        end
+        results
+      end
+
+      def all_cookbooks_list(request)
+        result = {}
+        # Race conditions exist here (if someone deletes while listing).  I don't care.
+        data_store.list(request.rest_path[0..1] + ['cookbook_artifacts']).each do |name|
+          result[name] = data_store.list(request.rest_path[0..1] + ['cookbook_artifacts', name])
+        end
+        result
+      end
+    end
+  end
+end

lib/chef_zero/endpoints/cookbook_artifact_version_endpoint.rb

@@ -0,0 +1,119 @@
+require 'ffi_yajl'
+require 'chef_zero/endpoints/rest_object_endpoint'
+require 'chef_zero/rest_error_response'
+require 'chef_zero/chef_data/data_normalizer'
+require 'chef_zero/data_store/data_not_found_error'
+
+module ChefZero
+  module Endpoints
+    # /organizations/ORG/cookbook_artifacts/NAME/VERSION
+    class CookbookArtifactVersionEndpoint < RestObjectEndpoint
+      def get(request)
+        if request.rest_path[4] == "_latest" || request.rest_path[4] == "latest"
+          request.rest_path[4] = latest_version(list_data(request, request.rest_path[0..3]))
+        end
+        super(request)
+      end
+
+      def put(request)
+        name = request.rest_path[3]
+        version = request.rest_path[4]
+        existing_cookbook = get_data(request, request.rest_path, :nil)
+
+        # Honor frozen
+        if existing_cookbook
+          existing_cookbook_json = FFI_Yajl::Parser.parse(existing_cookbook, :create_additions => false)
+          if existing_cookbook_json['frozen?']
+            if request.query_params['force'] != "true"
+              raise RestErrorResponse.new(409, "The cookbook #{name} at version #{version} is frozen. Use the 'force' option to override.")
+            end
+            # For some reason, you are forever unable to modify "frozen?" on a frozen cookbook.
+            request_body = FFI_Yajl::Parser.parse(request.body, :create_additions => false)
+            if !request_body['frozen?']
+              request_body['frozen?'] = true
+              request.body = FFI_Yajl::Encoder.encode(request_body, :pretty => true)
+            end
+          end
+        end
+
+        # Set the cookbook
+        set_data(request, request.rest_path, request.body, :create_dir, :create)
+
+        # If the cookbook was updated, check for deleted files and clean them up
+        if existing_cookbook
+          missing_checksums = get_checksums(existing_cookbook) - get_checksums(request.body)
+          if missing_checksums.size > 0
+            hoover_unused_checksums(missing_checksums, request)
+          end
+        end
+
+        already_json_response(existing_cookbook ? 200 : 201, populate_defaults(request, request.body))
+      end
+
+      def delete(request)
+        if request.rest_path[4] == "_latest" || request.rest_path[4] == "latest"
+          request.rest_path[4] = latest_version(list_data(request, request.rest_path[0..3]))
+        end
+
+        deleted_cookbook = get_data(request)
+
+        response = super(request)
+        cookbook_name = request.rest_path[3]
+        cookbook_path = request.rest_path[0..1] + ['cookbook_artifacts', cookbook_name]
+        if exists_data_dir?(request, cookbook_path) && list_data(request, cookbook_path).size == 0
+          delete_data_dir(request, cookbook_path)
+        end
+
+        # Hoover deleted files, if they exist
+        hoover_unused_checksums(get_checksums(deleted_cookbook), request)
+        response
+      end
+
+      def get_checksums(cookbook)
+        result = []
+        FFI_Yajl::Parser.parse(cookbook, :create_additions => false).each_pair do |key, value|
+          if value.is_a?(Array)
+            value.each do |file|
+              if file.is_a?(Hash) && file.has_key?('checksum')
+                result << file['checksum']
+              end
+            end
+          end
+        end
+        result.uniq
+      end
+
+      private
+
+      def hoover_unused_checksums(deleted_checksums, request)
+        data_store.list(request.rest_path[0..1] + ['cookbook_artifacts']).each do |cookbook_name|
+          data_store.list(request.rest_path[0..1] + ['cookbook_artifacts', cookbook_name]).each do |version|
+            cookbook = data_store.get(request.rest_path[0..1] + ['cookbook_artifacts', cookbook_name, version], request)
+            deleted_checksums = deleted_checksums - get_checksums(cookbook)
+          end
+        end
+        deleted_checksums.each do |checksum|
+          # There can be a race here if multiple cookbooks are uploading.
+          # This deals with an exception on delete, but things can still get deleted
+          # that shouldn't be.
+          begin
+            delete_data(request, request.rest_path[0..1] + ['file_store', 'checksums', checksum], :data_store_exceptions)
+          rescue ChefZero::DataStore::DataNotFoundError
+          end
+        end
+      end
+
+      def populate_defaults(request, response_json)
+        # Inject URIs into each cookbook file
+        cookbook = FFI_Yajl::Parser.parse(response_json, :create_additions => false)
+        cookbook = ChefData::DataNormalizer.normalize_cookbook(self, request.rest_path[0..1], cookbook, request.rest_path[3], request.rest_path[4], request.base_uri, request.method)
+        FFI_Yajl::Encoder.encode(cookbook, :pretty => true)
+      end
+
+      def latest_version(versions)
+        sorted = versions.sort_by { |version| Gem::Version.new(version.dup) }
+        sorted[-1]
+      end
+    end
+  end
+end

lib/chef_zero/endpoints/cookbook_artifacts_endpoint.rb

@@ -0,0 +1,50 @@
+require 'chef_zero/endpoints/cookbooks_base'
+
+module ChefZero
+  module Endpoints
+    # /cookbook_artifacts
+    class CookbookArtifactsEndpoint < CookbooksBase
+      def get(request)
+        if request.query_params['num_versions'] == 'all'
+          num_versions = nil
+        elsif request.query_params['num_versions']
+          num_versions = request.query_params['num_versions'].to_i
+        else
+          num_versions = 1
+        end
+        json_response(200, format_cookbooks_list(request, all_cookbooks_list(request), {}, num_versions))
+      end
+
+      ## CookbooksBase Overrides
+      # Methods here override behavior in CookbooksBase that is otherwise
+      # hard-coded to 'cookbooks'
+
+      def format_cookbooks_list(request, cookbooks_list, constraints = {}, num_versions = nil)
+        results = {}
+        filter_cookbooks(cookbooks_list, constraints, num_versions) do |name, versions|
+          versions_list = versions.map do |version|
+            {
+              'url' => build_uri(request.base_uri, request.rest_path[0..1] + ['cookbook_artifacts', name, version]),
+              'version' => version
+            }
+          end
+          results[name] = {
+            'url' => build_uri(request.base_uri, request.rest_path[0..1] + ['cookbook_artifacts', name]),
+            'versions' => versions_list
+          }
+        end
+        results
+      end
+
+      def all_cookbooks_list(request)
+        result = {}
+        # Race conditions exist here (if someone deletes while listing).  I don't care.
+        data_store.list(request.rest_path[0..1] + ['cookbook_artifacts']).each do |name|
+          result[name] = data_store.list(request.rest_path[0..1] + ['cookbook_artifacts', name])
+        end
+        result
+      end
+
+    end
+  end
+end

lib/chef_zero/rest_base.rb

@@ -116,10 +116,12 @@ def set_data(request, rest_path, data, *options)
       rest_path ||= request.rest_path
       begin
         data_store.set(rest_path, data, *options, :requestor => request.requestor)
-      rescue DataStore::DataNotFoundError
+      rescue DataStore::DataNotFoundError => e
         if options.include?(:data_store_exceptions)
           raise
         else
+          puts e
+          puts e.backtrace
           raise RestErrorResponse.new(404, "Object not found: #{build_uri(request.base_uri, request.rest_path)}")
         end
       end

lib/chef_zero/rest_router.rb

@@ -38,7 +38,7 @@ def call(request)
     private
 
       def find_endpoint(clean_path)
-        _, endpoint = routes.find { |route, endpoint| route.match(clean_path) }
+        _, endpoint = routes.find { |route, _endpoint| route.match(clean_path) }
         endpoint || not_found
       end
   end

lib/chef_zero/server.rb

@@ -41,6 +41,11 @@
 require 'chef_zero/endpoints/acl_endpoint'
 require 'chef_zero/endpoints/actors_endpoint'
 require 'chef_zero/endpoints/actor_endpoint'
+
+require 'chef_zero/endpoints/cookbook_artifact_endpoint'
+require 'chef_zero/endpoints/cookbook_artifact_version_endpoint'
+require 'chef_zero/endpoints/cookbook_artifacts_endpoint'
+
 require 'chef_zero/endpoints/cookbooks_endpoint'
 require 'chef_zero/endpoints/cookbook_endpoint'
 require 'chef_zero/endpoints/cookbook_version_endpoint'
@@ -500,6 +505,11 @@ def open_source_endpoints
         # Both
         [ "/organizations/*/clients", ActorsEndpoint.new(self) ],
         [ "/organizations/*/clients/*", ActorEndpoint.new(self) ],
+
+        [ "/organizations/*/cookbook_artifacts", CookbookArtifactsEndpoint.new(self) ],
+        [ "/organizations/*/cookbook_artifacts/*", CookbookArtifactEndpoint.new(self) ],
+        [ "/organizations/*/cookbook_artifacts/*/*", CookbookArtifactVersionEndpoint.new(self) ],
+
         [ "/organizations/*/cookbooks", CookbooksEndpoint.new(self) ],
         [ "/organizations/*/cookbooks/*", CookbookEndpoint.new(self) ],
         [ "/organizations/*/cookbooks/*/*", CookbookVersionEndpoint.new(self) ],

spec/run_pedant.rb

@@ -55,11 +55,24 @@ def start_local_server(chef_repo_path)
     tmpdir = Dir.mktmpdir
     data_store = ChefZero::DataStore::RawFileStore.new(tmpdir, true)
     data_store = ChefZero::DataStore::DefaultFacade.new(data_store, true, false)
-    server = ChefZero::Server.new(:port => 8889, :single_org => 'chef', :data_store => data_store)
+
+    # TODO: Without this, tests don't run at all for me. But when I add this,
+    # there are a handful of extra clients present, which makes some tests
+    # fail.
+    data_store.create_dir([ 'organizations' ], 'pedant')
+
+    server_opts =  { :port => 8889, :single_org => 'pedant', :data_store => data_store }
+    server_opts[:log_level] = :debug if ENV['DEBUG_ZERO']
+
+    server = ChefZero::Server.new(server_opts)
     server.start_background
 
   else
-    server = ChefZero::Server.new(:port => 8889, :single_org => false, :osc_compat => true)
+    server_opts =  { :port => 8889, :single_org => false, :osc_compat => true }
+    server_opts[:log_level] = :debug if ENV['DEBUG_ZERO']
+
+    server = ChefZero::Server.new(server_opts)
+
     server.data_store.create_dir([ 'organizations' ], 'pedant')
     server.start_background
   end

spec/support/oc_pedant.rb

@@ -132,3 +132,7 @@
 ruby_users_endpoint? false
 ruby_acls_endpoint? false
 ruby_org_assoc? false
+
+# The Policies endpoint is feature-flagged during development. Zero supports
+# the policies endpoint, so turn it on:
+policies? true