From 84dd088677e28b1aa39a8af59e2daff9af021de4 Mon Sep 17 00:00:00 2001
From: Ivan Kapelyukhin <ikapelyukhin@suse.com>
Date: Wed, 3 Jan 2018 15:16:17 +0100
Subject: [PATCH 01/14] Modify the schema and the existing specs

---
 app/models/product.rb                         | 16 ++++--
 app/models/products_extensions_association.rb |  1 +
 app/serializers/v3/product_serializer.rb      |  6 ++-
 ..._and_recommended_to_products_extensions.rb | 22 ++++++++
 db/schema.rb                                  |  5 +-
 lib/rmt/scc.rb                                |  1 +
 spec/factories/products.rb                    | 12 +++--
 spec/models/product_spec.rb                   | 54 ++++++++++++++++++-
 .../v4/systems/products_controller_spec.rb    |  5 +-
 9 files changed, 105 insertions(+), 17 deletions(-)
 create mode 100644 db/migrate/20180102134941_add_root_and_recommended_to_products_extensions.rb

diff --git a/app/models/product.rb b/app/models/product.rb
index 2842f6365..fb60e7971 100644
--- a/app/models/product.rb
+++ b/app/models/product.rb
@@ -16,13 +16,21 @@ class Product < ApplicationRecord
            class_name: 'ProductsExtensionsAssociation',
            foreign_key: :product_id
 
-  has_many :extensions,
-           through: :extension_products_associations,
-           source: :extension
+  has_many :extensions, -> { distinct },
+    through: :extension_products_associations,
+    source: :extension do
+    def for_root_product(root_product)
+      where('products_extensions.root_product_id = %s', root_product.id)
+    end
+  end
 
   has_many :mirrored_extensions, -> { mirrored },
     through: :extension_products_associations,
-    source: :extension
+    source: :extension do
+    def for_root_product(root_product)
+      where('products_extensions.root_product_id = %s', root_product.id)
+    end
+  end
 
   has_and_belongs_to_many :predecessors, class_name: 'Product', join_table: :product_predecessors,
     association_foreign_key: :predecessor_id
diff --git a/app/models/products_extensions_association.rb b/app/models/products_extensions_association.rb
index 60bd09285..6b1994b2e 100644
--- a/app/models/products_extensions_association.rb
+++ b/app/models/products_extensions_association.rb
@@ -2,6 +2,7 @@ class ProductsExtensionsAssociation < ApplicationRecord
 
   self.table_name = 'products_extensions'
   belongs_to :product, class_name: 'Product', foreign_key: :product_id
+  belongs_to :root_product, class_name: 'Product', foreign_key: :root_product_id
   belongs_to :extension, class_name: 'Product', foreign_key: :extension_id
 
 end
diff --git a/app/serializers/v3/product_serializer.rb b/app/serializers/v3/product_serializer.rb
index bef0b0bf1..58927d2e3 100644
--- a/app/serializers/v3/product_serializer.rb
+++ b/app/serializers/v3/product_serializer.rb
@@ -14,7 +14,7 @@ def repositories
              :friendly_name, :product_class, :cpe, :free, :description, :eula_url, :repositories, :product_type, :extensions
 
   def extensions
-    object.mirrored_extensions.map do |extension|
+    object.mirrored_extensions.for_root_product(root_product).map do |extension|
       ::V3::ProductSerializer.new(extension, base_url: base_url).attributes
     end
   end
@@ -37,4 +37,8 @@ def free
     true
   end
 
+  def root_product
+    @instance_options[:root_product] ||= object
+  end
+
 end
diff --git a/db/migrate/20180102134941_add_root_and_recommended_to_products_extensions.rb b/db/migrate/20180102134941_add_root_and_recommended_to_products_extensions.rb
new file mode 100644
index 000000000..ca815c1ca
--- /dev/null
+++ b/db/migrate/20180102134941_add_root_and_recommended_to_products_extensions.rb
@@ -0,0 +1,22 @@
+class AddRootAndRecommendedToProductsExtensions < ActiveRecord::Migration[5.1]
+
+  def change
+    add_column :products_extensions, :recommended, :boolean
+    add_column :products_extensions, :root_product_id, :integer
+    add_index :products_extensions, %i[product_id extension_id root_product_id],
+              unique: true, name: 'index_products_extensions_on_product_extension_root'
+
+    reversible do |dir|
+      dir.up do
+        ProductsExtensionsAssociation.find_each.each do |pa|
+          base = pa.product
+          pa.root_product = base.bases.present? ? base.bases.first : base
+          pa.recommended = false
+          pa.save!
+        end
+        change_column_null(:products_extensions, :root_product_id, false)
+      end
+    end
+  end
+
+end
diff --git a/db/schema.rb b/db/schema.rb
index 890412e97..d07dc8b99 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20171214093115) do
+ActiveRecord::Schema.define(version: 20180102134941) do
 
   create_table "activations", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8" do |t|
     t.integer "service_id", null: false
@@ -56,7 +56,10 @@
   create_table "products_extensions", force: :cascade, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8" do |t|
     t.bigint "product_id", null: false
     t.bigint "extension_id", null: false
+    t.boolean "recommended"
+    t.integer "root_product_id", null: false
     t.index ["extension_id"], name: "index_products_extensions_on_extension_id"
+    t.index ["product_id", "extension_id", "root_product_id"], name: "index_products_extensions_on_product_extension_root", unique: true
     t.index ["product_id"], name: "index_products_extensions_on_product_id"
   end
 
diff --git a/lib/rmt/scc.rb b/lib/rmt/scc.rb
index b00401a5d..d1be6c46b 100644
--- a/lib/rmt/scc.rb
+++ b/lib/rmt/scc.rb
@@ -126,6 +126,7 @@ def create_product(item)
       association = ProductsExtensionsAssociation.new
       association.product_id = product.id
       association.extension_id = extension.id
+      association.root_product_id = product.id # FIXME: correct root_product_id !!!
       association.save!
     end
 
diff --git a/spec/factories/products.rb b/spec/factories/products.rb
index 70b8e806a..d1448fa52 100644
--- a/spec/factories/products.rb
+++ b/spec/factories/products.rb
@@ -17,11 +17,15 @@
     transient do
       base_products []
       predecessor nil
+      root_product nil
     end
 
     after :create do |product, evaluator|
       evaluator.base_products.each do |base_product|
-        product.product_extensions_associations << ProductsExtensionsAssociation.create(product: base_product)
+        product.product_extensions_associations << ProductsExtensionsAssociation.create(
+          product: base_product,
+          root_product: evaluator.root_product || base_product
+        )
       end
 
       product.predecessors << evaluator.predecessor if evaluator.predecessor
@@ -38,8 +42,7 @@
     trait :with_extensions do
       after :create do |product, _evaluator|
         5.times do
-          extension = create :product, :extension
-          product.extensions << extension
+          create(:product, :extension, base_products: [product])
         end
       end
     end
@@ -47,8 +50,7 @@
     trait :with_mirrored_extensions do
       after :create do |product, _evaluator|
         5.times do
-          extension = create :product, :extension, :with_mirrored_repositories
-          product.extensions << extension
+          create(:product, :extension, :with_mirrored_repositories, base_products: [product])
         end
       end
     end
diff --git a/spec/models/product_spec.rb b/spec/models/product_spec.rb
index 5c628e7cd..3c39ff187 100644
--- a/spec/models/product_spec.rb
+++ b/spec/models/product_spec.rb
@@ -10,14 +10,14 @@
     subject { product.has_extension? }
 
     let(:product) { create :product }
-    let(:extension) { create :product }
+
 
     context 'when has no extensions' do
       it { is_expected.to eq false }
     end
 
     context 'when has extension' do
-      before { product.extensions << extension }
+      before { create(:product, :extension, base_products: [product]) }
       it { is_expected.to eq true }
     end
   end
@@ -65,4 +65,54 @@
       it { is_expected.to eq('42') }
     end
   end
+
+  describe '#extensions' do
+    let(:base_a) { FactoryGirl.create(:product) }
+    let(:base_b) { FactoryGirl.create(:product) }
+
+    let(:level_one_extension) { FactoryGirl.create(:product, :extension, base_products: [base_a, base_b]) }
+
+    let!(:level_two_extension_a) { FactoryGirl.create(:product, :extension, base_products: [level_one_extension], root_product: base_a) }
+    let!(:level_two_extension_b) { FactoryGirl.create(:product, :extension, base_products: [level_one_extension], root_product: base_b) }
+
+    it 'returns correct extensions for base A' do
+      expect(level_one_extension.extensions.for_root_product(base_a)).to eq([level_two_extension_a])
+    end
+
+    it 'returns correct extensions for base B' do
+      expect(level_one_extension.extensions.for_root_product(base_b)).to eq([level_two_extension_b])
+    end
+  end
+
+  describe '#mirrored_extensions' do
+    let(:base_a) { FactoryGirl.create(:product) }
+    let(:base_b) { FactoryGirl.create(:product) }
+
+    let(:level_one_extension) { FactoryGirl.create(:product, :extension, base_products: [base_a, base_b]) }
+
+    let(:level_two_extension_a_not_mirrored) { FactoryGirl.create(:product, :extension, base_products: [level_one_extension], root_product: base_a) }
+    let(:level_two_extension_b_not_mirrored) { FactoryGirl.create(:product, :extension, base_products: [level_one_extension], root_product: base_b) }
+
+    let(:level_two_extension_a_mirrored) do
+      FactoryGirl.create(:product, :extension, :with_mirrored_repositories, base_products: [level_one_extension], root_product: base_a)
+    end
+    let(:level_two_extension_b_mirrored) do
+      FactoryGirl.create(:product, :extension, :with_mirrored_repositories, base_products: [level_one_extension], root_product: base_b)
+    end
+
+    before do
+      level_two_extension_a_not_mirrored
+      level_two_extension_b_not_mirrored
+      level_two_extension_a_mirrored
+      level_two_extension_b_mirrored
+    end
+
+    it 'returns correct extensions for base A' do
+      expect(level_one_extension.mirrored_extensions.for_root_product(base_a)).to eq([level_two_extension_a_mirrored])
+    end
+
+    it 'returns correct extensions for base B' do
+      expect(level_one_extension.mirrored_extensions.for_root_product(base_b)).to eq([level_two_extension_b_mirrored])
+    end
+  end
 end
diff --git a/spec/requests/api/connect/v4/systems/products_controller_spec.rb b/spec/requests/api/connect/v4/systems/products_controller_spec.rb
index 3a0b1e7a0..c2e3bd332 100644
--- a/spec/requests/api/connect/v4/systems/products_controller_spec.rb
+++ b/spec/requests/api/connect/v4/systems/products_controller_spec.rb
@@ -53,10 +53,7 @@
       context 'has products depending on it and is activated' do
         let(:product) do
           product = FactoryGirl.create(:product, :extension, :with_mirrored_repositories, :activated, system: system)
-          ext_product = FactoryGirl.create(:product, :extension, :with_mirrored_repositories, :activated, system: system)
-          ext_product.bases << product
-          ext_product.save!
-
+          FactoryGirl.create(:product, :extension, :with_mirrored_repositories, :activated, system: system, base_products: [product])
           product
         end
 

From f022bcd0ea813b30d46fffed7e54111f7c04b172 Mon Sep 17 00:00:00 2001
From: Ivan Kapelyukhin <ikapelyukhin@suse.com>
Date: Thu, 4 Jan 2018 16:48:58 +0100
Subject: [PATCH 02/14] Modify serializer and SCC sync to walk products
 recursively

---
 app/models/product.rb                         |  4 ++
 app/models/products_extensions_association.rb |  3 ++
 app/serializers/v3/product_serializer.rb      | 11 ++++--
 lib/rmt/scc.rb                                | 39 ++++++++-----------
 4 files changed, 31 insertions(+), 26 deletions(-)

diff --git a/app/models/product.rb b/app/models/product.rb
index fb60e7971..74af61732 100644
--- a/app/models/product.rb
+++ b/app/models/product.rb
@@ -77,4 +77,8 @@ def change_repositories_mirroring!(conditions, mirroring_enabled)
     repositories.where(conditions).update_all(mirroring_enabled: mirroring_enabled)
   end
 
+  def recommended_for?(root_product)
+    product_extensions_associations.where(recommended: true).where(root_product: root_product).present?
+  end
+
 end
diff --git a/app/models/products_extensions_association.rb b/app/models/products_extensions_association.rb
index 6b1994b2e..5ea4887f8 100644
--- a/app/models/products_extensions_association.rb
+++ b/app/models/products_extensions_association.rb
@@ -5,4 +5,7 @@ class ProductsExtensionsAssociation < ApplicationRecord
   belongs_to :root_product, class_name: 'Product', foreign_key: :root_product_id
   belongs_to :extension, class_name: 'Product', foreign_key: :extension_id
 
+  validates :product_id, :extension_id, :root_product_id, presence: true
+  validates :product_id, uniqueness: { scope: %i[extension_id root_product_id] }
+
 end
diff --git a/app/serializers/v3/product_serializer.rb b/app/serializers/v3/product_serializer.rb
index 58927d2e3..4b87b14e0 100644
--- a/app/serializers/v3/product_serializer.rb
+++ b/app/serializers/v3/product_serializer.rb
@@ -11,11 +11,12 @@ def repositories
   end
 
   attributes :id, :name, :identifier, :former_identifier, :version, :release_type, :arch,
-             :friendly_name, :product_class, :cpe, :free, :description, :eula_url, :repositories, :product_type, :extensions
+             :friendly_name, :product_class, :cpe, :free, :description, :eula_url, :repositories, :product_type,
+             :extensions, :recommended
 
   def extensions
-    object.mirrored_extensions.for_root_product(root_product).map do |extension|
-      ::V3::ProductSerializer.new(extension, base_url: base_url).attributes
+    object.extensions.for_root_product(root_product).map do |extension|
+      ::V3::ProductSerializer.new(extension, base_url: base_url, root_product: root_product).attributes
     end
   end
 
@@ -41,4 +42,8 @@ def root_product
     @instance_options[:root_product] ||= object
   end
 
+  def recommended
+    object.recommended_for? root_product
+  end
+
 end
diff --git a/lib/rmt/scc.rb b/lib/rmt/scc.rb
index d1be6c46b..7bfa1faf4 100644
--- a/lib/rmt/scc.rb
+++ b/lib/rmt/scc.rb
@@ -24,8 +24,7 @@ def sync
     data = scc_api_client.list_products
     data.each do |item|
       @logger.debug("Adding product #{item[:identifier]}/#{item[:version]}#{(item[:arch]) ? '/' + item[:arch] : ''}")
-      product = create_product(item)
-      create_service(item, product)
+      create_product(item) if (item[:product_type] == 'base')
     end
 
     @logger.info('Updating repositories')
@@ -80,8 +79,7 @@ def import(path)
     data = JSON.parse(File.read(File.join(path, 'organizations_products_scoped.json')), symbolize_names: true)
     data.each do |item|
       @logger.debug("Adding product #{item[:identifier]}/#{item[:version]}#{(item[:arch]) ? '/' + item[:arch] : ''}")
-      product = create_product(item)
-      create_service(item, product)
+      create_product(item)
     end
 
     @logger.info('Updating repositories')
@@ -101,18 +99,7 @@ def import(path)
 
   protected
 
-  def create_product(item)
-    extensions = []
-
-    item[:extensions].each do |ext_item|
-      extension = Product.find_or_create_by(id: ext_item[:id])
-      extension.attributes = ext_item.select { |k, _| extension.attributes.keys.member?(k.to_s) }
-      extension.save!
-
-      create_service(ext_item, extension)
-      extensions << extension
-    end
-
+  def create_product(item, root_product_id = nil, base_product = nil)
     product = Product.find_or_create_by(id: item[:id])
     product.attributes = item.select { |k, _| product.attributes.keys.member?(k.to_s) }
     product.save!
@@ -122,15 +109,21 @@ def create_product(item)
       ProductPredecessorAssociation.create(product_id: product.id, predecessor_id: predecessor_id)
     end
 
-    extensions.each do |extension|
-      association = ProductsExtensionsAssociation.new
-      association.product_id = product.id
-      association.extension_id = extension.id
-      association.root_product_id = product.id # FIXME: correct root_product_id !!!
-      association.save!
+    create_service(item, product)
+
+    if root_product_id
+      ProductsExtensionsAssociation.create(
+        product_id: base_product,
+        extension_id: product.id,
+        root_product_id: root_product_id
+      )
+    else
+      root_product_id = product.id
     end
 
-    product
+    item[:extensions].each do |ext_item|
+      create_product(ext_item, root_product_id, product.id)
+    end
   end
 
   def create_service(item, product)

From c5eed3af068210bd928606319b67515ca7e4bdfa Mon Sep 17 00:00:00 2001
From: Ivan Kapelyukhin <ikapelyukhin@suse.com>
Date: Thu, 4 Jan 2018 17:25:30 +0100
Subject: [PATCH 03/14] Save correct `recommended` value, add `available`
 attribute

---
 app/serializers/v3/product_serializer.rb | 7 ++++++-
 lib/rmt/scc.rb                           | 8 +++++---
 2 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/app/serializers/v3/product_serializer.rb b/app/serializers/v3/product_serializer.rb
index 4b87b14e0..1c089a7a1 100644
--- a/app/serializers/v3/product_serializer.rb
+++ b/app/serializers/v3/product_serializer.rb
@@ -12,7 +12,7 @@ def repositories
 
   attributes :id, :name, :identifier, :former_identifier, :version, :release_type, :arch,
              :friendly_name, :product_class, :cpe, :free, :description, :eula_url, :repositories, :product_type,
-             :extensions, :recommended
+             :extensions, :recommended, :available
 
   def extensions
     object.extensions.for_root_product(root_product).map do |extension|
@@ -46,4 +46,9 @@ def recommended
     object.recommended_for? root_product
   end
 
+  # This attribute is added by SMT as well, indicating whether the product is mirrored or not
+  def available
+    object.mirror?
+  end
+
 end
diff --git a/lib/rmt/scc.rb b/lib/rmt/scc.rb
index 7bfa1faf4..9b7629d1d 100644
--- a/lib/rmt/scc.rb
+++ b/lib/rmt/scc.rb
@@ -99,7 +99,7 @@ def import(path)
 
   protected
 
-  def create_product(item, root_product_id = nil, base_product = nil)
+  def create_product(item, root_product_id = nil, base_product = nil, recommended = false)
     product = Product.find_or_create_by(id: item[:id])
     product.attributes = item.select { |k, _| product.attributes.keys.member?(k.to_s) }
     product.save!
@@ -115,14 +115,16 @@ def create_product(item, root_product_id = nil, base_product = nil)
       ProductsExtensionsAssociation.create(
         product_id: base_product,
         extension_id: product.id,
-        root_product_id: root_product_id
+        root_product_id: root_product_id,
+        recommended: recommended
       )
     else
       root_product_id = product.id
+      ProductsExtensionsAssociation.where(root_product_id: root_product_id).destroy_all
     end
 
     item[:extensions].each do |ext_item|
-      create_product(ext_item, root_product_id, product.id)
+      create_product(ext_item, root_product_id, product.id, ext_item[:recommended])
     end
   end
 

From df59f3acbf833bdcb25487b55cd75f26c3e0c23d Mon Sep 17 00:00:00 2001
From: Ivan Kapelyukhin <ikapelyukhin@suse.com>
Date: Wed, 17 Jan 2018 13:26:07 +0100
Subject: [PATCH 04/14] Add specs for SCC sync with SLE15 product tree

---
 spec/fixtures/files/products/sle15_tree.json | 185 +++++++++++++++++++
 spec/lib/rmt/scc_spec.rb                     |  36 ++++
 2 files changed, 221 insertions(+)
 create mode 100644 spec/fixtures/files/products/sle15_tree.json

diff --git a/spec/fixtures/files/products/sle15_tree.json b/spec/fixtures/files/products/sle15_tree.json
new file mode 100644
index 000000000..c7fc388ee
--- /dev/null
+++ b/spec/fixtures/files/products/sle15_tree.json
@@ -0,0 +1,185 @@
+[
+  {
+    "id": 10000,
+    "name": "SLES",
+    "identifier": "SLES",
+    "former_identifier": "SLES",
+    "version": "15",
+    "release_type": null,
+    "arch": "x86_64",
+    "friendly_name": "SUSE Linux Enterprise Server",
+    "product_class": "7261",
+    "cpe": null,
+    "free": false,
+    "description": null,
+    "release_stage": "released",
+    "eula_url": "https://example.com/sles-15-x86_64/eula.txt",
+    "product_type": "base",
+    "predecessor_ids": [],
+    "shortname": null,
+    "recommended": false,
+    "repositories": [],
+    "extensions": [
+      {
+        "id": 10001,
+        "name": "Basesystem Module",
+        "identifier": "sle-module-basesystem",
+        "former_identifier": "sle-module-basesystem",
+        "version": "15",
+        "release_type": null,
+        "arch": null,
+        "friendly_name": "Basesystem Module",
+        "product_class": "MODULE",
+        "cpe": null,
+        "free": true,
+        "description": null,
+        "release_stage": "released",
+        "eula_url": "https://example.com/sle-module-basesystem-15-x86_64/eula.txt",
+        "product_type": "module",
+        "predecessor_ids": [],
+        "shortname": null,
+        "recommended": true,
+        "repositories": [],
+        "extensions": [
+          {
+            "id": 10002,
+            "name": "Desktop Module",
+            "identifier": "sle-module-desktop",
+            "former_identifier": "sle-module-desktop",
+            "version": "15",
+            "release_type": null,
+            "arch": null,
+            "friendly_name": "Desktop Module",
+            "product_class": "MODULE",
+            "cpe": null,
+            "free": true,
+            "description": null,
+            "release_stage": "released",
+            "eula_url": "https://example.com/sle-module-desktop-15-x86_64/eula.txt",
+            "product_type": "module",
+            "predecessor_ids": [],
+            "shortname": null,
+            "recommended": false,
+            "repositories": [],
+            "extensions": [
+              {
+                "id": 10003,
+                "name": "SLE WE Module",
+                "identifier": "sle-module-we",
+                "former_identifier": "sle-module-we",
+                "version": "15",
+                "release_type": null,
+                "arch": null,
+                "friendly_name": "Workstation Extensions Module",
+                "product_class": "MODULE",
+                "cpe": null,
+                "free": true,
+                "description": null,
+                "release_stage": "released",
+                "eula_url": "https://example.com/sle-module-we-15-x86_64/eula.txt",
+                "product_type": "module",
+                "predecessor_ids": [],
+                "shortname": null,
+                "recommended": false,
+                "repositories": [],
+                "extensions": []
+              }
+            ]
+          }
+        ]
+      }
+    ]
+  },
+
+  {
+    "id": 10004,
+    "name": "SLED",
+    "identifier": "SLED",
+    "former_identifier": "SLED",
+    "version": "15",
+    "release_type": null,
+    "arch": "x86_64",
+    "friendly_name": "SUSE Linux Enterprise Desktop",
+    "product_class": "7261",
+    "cpe": null,
+    "free": false,
+    "description": null,
+    "release_stage": "released",
+    "eula_url": "https://example.com/sles-15-x86_64/eula.txt",
+    "product_type": "base",
+    "predecessor_ids": [],
+    "shortname": null,
+    "recommended": false,
+    "repositories": [],
+    "extensions": [
+      {
+        "id": 10001,
+        "name": "Basesystem Module",
+        "identifier": "sle-module-basesystem",
+        "former_identifier": "sle-module-basesystem",
+        "version": "15",
+        "release_type": null,
+        "arch": null,
+        "friendly_name": "Basesystem Module",
+        "product_class": "MODULE",
+        "cpe": null,
+        "free": true,
+        "description": null,
+        "release_stage": "released",
+        "eula_url": "https://example.com/sle-module-basesystem-15-x86_64/eula.txt",
+        "product_type": "module",
+        "predecessor_ids": [],
+        "shortname": null,
+        "recommended": true,
+        "repositories": [],
+        "extensions": [
+          {
+            "id": 10002,
+            "name": "Desktop Module",
+            "identifier": "sle-module-desktop",
+            "former_identifier": "sle-module-desktop",
+            "version": "15",
+            "release_type": null,
+            "arch": null,
+            "friendly_name": "Desktop Module",
+            "product_class": "MODULE",
+            "cpe": null,
+            "free": true,
+            "description": null,
+            "release_stage": "released",
+            "eula_url": "https://example.com/sle-module-desktop-15-x86_64/eula.txt",
+            "product_type": "module",
+            "predecessor_ids": [],
+            "shortname": null,
+            "recommended": true,
+            "repositories": [],
+            "extensions": [
+              {
+                "id": 10005,
+                "name": "Desktop Productivity Module",
+                "identifier": "sle-module-desktop-productivity",
+                "former_identifier": "sle-module-desktop-productivity",
+                "version": "15",
+                "release_type": null,
+                "arch": null,
+                "friendly_name": "Desktop Productivity Module",
+                "product_class": "MODULE",
+                "cpe": null,
+                "free": true,
+                "description": null,
+                "release_stage": "released",
+                "eula_url": "https://example.com/sle-module-desktop-productivity-15-x86_64/eula.txt",
+                "product_type": "module",
+                "predecessor_ids": [],
+                "shortname": null,
+                "recommended": true,
+                "repositories": [],
+                "extensions": []
+              }
+            ]
+          }
+        ]
+      }
+    ]
+  }
+]
diff --git a/spec/lib/rmt/scc_spec.rb b/spec/lib/rmt/scc_spec.rb
index 59b0dd91a..0a6d34aa1 100644
--- a/spec/lib/rmt/scc_spec.rb
+++ b/spec/lib/rmt/scc_spec.rb
@@ -86,6 +86,42 @@
 
       include_examples 'saves in database'
     end
+
+    context 'with SLES15 product tree' do
+      let!(:products) { JSON.parse(file_fixture('products/sles15_tree.json').read, symbolize_names: true) }
+      let!(:subscriptions) { [] }
+      let(:all_repositories) { [] }
+
+      let(:sles) { Product.find_by(identifier: 'SLES') }
+      let(:sled) { Product.find_by(identifier: 'SLED') }
+
+      before do
+        allow(Settings).to receive(:scc).and_return OpenStruct.new(username: 'foo', password: 'bar')
+        described_class.new.sync
+      end
+
+      include_examples 'saves in database'
+
+      it 'SLES has the correct extension tree' do
+        basesystem = sles.extensions.first
+        desktop = basesystem.extensions.for_root_product(sles).first
+        sle_we  = desktop.extensions.for_root_product(sles).first
+
+        expect([basesystem, desktop, sle_we].map(&:identifier)).to eq(
+          ['sle-module-basesystem', 'sle-module-desktop', 'sle-module-we']
+        )
+      end
+
+      it 'SLED has the correct extension tree' do
+        basesystem = sled.extensions.first
+        desktop = basesystem.extensions.for_root_product(sled).first
+        productivity = desktop.extensions.for_root_product(sled).first
+
+        expect([basesystem, desktop, productivity].map(&:identifier)).to eq(
+          ['sle-module-basesystem', 'sle-module-desktop', 'sle-module-desktop-productivity']
+        )
+      end
+    end
   end
 
   describe '#remove_suse_repos_without_tokens' do

From 61663f7fc3df9cc4a6a9416bece3ee5d92437281 Mon Sep 17 00:00:00 2001
From: Ivan Kapelyukhin <ikapelyukhin@suse.com>
Date: Wed, 17 Jan 2018 13:30:38 +0100
Subject: [PATCH 05/14] Fix fixture filename and rubocop offenses

---
 spec/lib/rmt/scc_spec.rb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/spec/lib/rmt/scc_spec.rb b/spec/lib/rmt/scc_spec.rb
index 0a6d34aa1..d7f0fa614 100644
--- a/spec/lib/rmt/scc_spec.rb
+++ b/spec/lib/rmt/scc_spec.rb
@@ -88,8 +88,8 @@
     end
 
     context 'with SLES15 product tree' do
-      let!(:products) { JSON.parse(file_fixture('products/sles15_tree.json').read, symbolize_names: true) }
-      let!(:subscriptions) { [] }
+      let(:products) { JSON.parse(file_fixture('products/sle15_tree.json').read, symbolize_names: true) }
+      let(:subscriptions) { [] }
       let(:all_repositories) { [] }
 
       let(:sles) { Product.find_by(identifier: 'SLES') }

From d1ffc947588b832157ac3045575f9829f87016ca Mon Sep 17 00:00:00 2001
From: Ivan Kapelyukhin <ikapelyukhin@suse.com>
Date: Wed, 17 Jan 2018 16:36:24 +0100
Subject: [PATCH 06/14] Add specs for ProductSerializer

---
 spec/factories/products.rb                    |  4 +-
 .../serializers/v3/product_serializer_spec.rb | 55 +++++++++++++++++++
 2 files changed, 58 insertions(+), 1 deletion(-)
 create mode 100644 spec/serializers/v3/product_serializer_spec.rb

diff --git a/spec/factories/products.rb b/spec/factories/products.rb
index d1448fa52..e2b625331 100644
--- a/spec/factories/products.rb
+++ b/spec/factories/products.rb
@@ -18,13 +18,15 @@
       base_products []
       predecessor nil
       root_product nil
+      recommended false
     end
 
     after :create do |product, evaluator|
       evaluator.base_products.each do |base_product|
         product.product_extensions_associations << ProductsExtensionsAssociation.create(
           product: base_product,
-          root_product: evaluator.root_product || base_product
+          root_product: evaluator.root_product || base_product,
+          recommended: evaluator.recommended
         )
       end
 
diff --git a/spec/serializers/v3/product_serializer_spec.rb b/spec/serializers/v3/product_serializer_spec.rb
new file mode 100644
index 000000000..82a1a6a55
--- /dev/null
+++ b/spec/serializers/v3/product_serializer_spec.rb
@@ -0,0 +1,55 @@
+require 'rails_helper'
+
+describe V3::ProductSerializer do
+  let(:sled15) { create(:product, name: 'SLED') }
+  let(:sles15) { create(:product, name: 'SLES') }
+  let!(:basesystem) do
+    create(:product, :extension, name: 'BASESYSTEM', base_products: [sles15, sled15], recommended: true)
+  end
+  let!(:server_applications) do
+    create(:product, :extension, name: 'SERVER APPLICATIONS', base_products: [basesystem], root_product: sles15)
+  end
+  let(:base_url) { 'http://example.com' }
+
+  describe 'SLES extension tree' do
+    subject(:serializer) { described_class.new(sles15, root_product: sles15, base_url: base_url) }
+
+    let(:top_extension) { serializer.as_json[:extensions].first }
+    let(:nested_extension) { top_extension[:extensions].first }
+
+    it 'has base system extension' do
+      expect(top_extension[:name]).to eq(basesystem.name)
+    end
+
+    it 'has base system and it is recommended' do
+      expect(top_extension[:recommended]).to eq(true)
+    end
+
+    it 'has server applications extension' do
+      expect(nested_extension[:name]).to eq(server_applications.name)
+    end
+
+    it 'has server applications extension and it is not recommended' do
+      expect(nested_extension[:recommended]).to eq(false)
+    end
+  end
+
+  describe 'SLED extension tree' do
+    subject(:serializer) { described_class.new(sled15, root_product: sled15, base_url: base_url) }
+
+    let(:top_extension) { serializer.as_json[:extensions].first }
+    let(:nested_extension) { top_extension[:extensions].first }
+
+    it 'has base system extension' do
+      expect(top_extension[:name]).to eq(basesystem.name)
+    end
+
+    it 'has base system and it is recommended' do
+      expect(top_extension[:recommended]).to eq(true)
+    end
+
+    it 'has no nested extensions' do
+      expect(nested_extension).to be_nil
+    end
+  end
+end

From 80e6d50c76d2b25a44838cac2d3fc1114b1bdb48 Mon Sep 17 00:00:00 2001
From: Ivan Kapelyukhin <ikapelyukhin@suse.com>
Date: Mon, 22 Jan 2018 13:23:28 +0100
Subject: [PATCH 07/14] Skip repositories for extensions with unavailable base
 products

---
 lib/rmt/scc.rb | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/lib/rmt/scc.rb b/lib/rmt/scc.rb
index 9b7629d1d..26cd6d463 100644
--- a/lib/rmt/scc.rb
+++ b/lib/rmt/scc.rb
@@ -23,7 +23,6 @@ def sync
     @logger.info('Updating products')
     data = scc_api_client.list_products
     data.each do |item|
-      @logger.debug("Adding product #{item[:identifier]}/#{item[:version]}#{(item[:arch]) ? '/' + item[:arch] : ''}")
       create_product(item) if (item[:product_type] == 'base')
     end
 
@@ -100,6 +99,8 @@ def import(path)
   protected
 
   def create_product(item, root_product_id = nil, base_product = nil, recommended = false)
+    @logger.debug("Adding product #{item[:identifier]}/#{item[:version]}#{(item[:arch]) ? '/' + item[:arch] : ''}")
+
     product = Product.find_or_create_by(id: item[:id])
     product.attributes = item.select { |k, _| product.attributes.keys.member?(k.to_s) }
     product.save!
@@ -149,7 +150,14 @@ def update_auth_token(item)
     uri = URI(item[:url])
     auth_token = uri.query
 
-    Repository.find(item[:id]).update! auth_token: auth_token
+    # Sometimes the extension is available, but a base product is not, e.g.:
+    # sle-hae/11.3/s390x available without base product for s390x
+    # In this case no repository data was added in create_product -- can't update those repos.
+    begin
+      Repository.find(item[:id]).update! auth_token: auth_token
+    rescue ActiveRecord::RecordNotFound
+      @logger.debug("Repository #{item[:id]} is not available")
+    end
   end
 
   def create_subscription(item)

From b6543e7edcd5750c97968b1f0eb1a9e1643091ea Mon Sep 17 00:00:00 2001
From: Ivan Kapelyukhin <ikapelyukhin@suse.com>
Date: Mon, 22 Jan 2018 13:58:43 +0100
Subject: [PATCH 08/14] Add specs for extensions without base products

---
 spec/lib/rmt/scc_spec.rb | 37 +++++++++++++++++++++++++++++++++++++
 1 file changed, 37 insertions(+)

diff --git a/spec/lib/rmt/scc_spec.rb b/spec/lib/rmt/scc_spec.rb
index d7f0fa614..a8abf58ab 100644
--- a/spec/lib/rmt/scc_spec.rb
+++ b/spec/lib/rmt/scc_spec.rb
@@ -122,6 +122,43 @@
         )
       end
     end
+
+    context "with extensions that don't have base products available" do
+      let(:extra_repo) do
+        {
+          id: 999999,
+          url: 'http://example.com/extension-without-base',
+          name: 'Repo of an extension without base'
+        }
+      end
+      let(:extra_product) do
+        {
+          id: 999999,
+          identifier: 'ext-without-base',
+          version: '99',
+          arch: 'x86_64',
+          name: 'Extension without base',
+          friendly_name: 'Extension without base',
+          repositories: [ extra_repo ]
+        }
+      end
+      let(:repositories_with_extra_repos) { all_repositories + [extra_repo] }
+      let(:products_with_extra_extension) { products + [extra_product] }
+
+      before do
+        allow(api_double).to receive(:list_products).and_return products_with_extra_extension
+        allow(api_double).to receive(:list_repositories).and_return repositories_with_extra_repos
+        described_class.new.sync
+      end
+
+      it "doesn't save extensions without base products" do
+        expect { Product.find(extra_product[:id]) }.to raise_error(ActiveRecord::RecordNotFound)
+      end
+
+      it "doesn't save repos of extensions without base products" do
+        expect { Repository.find(extra_repo[:id]) }.to raise_error(ActiveRecord::RecordNotFound)
+      end
+    end
   end
 
   describe '#remove_suse_repos_without_tokens' do

From 10bed417826b0d065bc66fdb1dfbeca5c8413534 Mon Sep 17 00:00:00 2001
From: Ivan Kapelyukhin <ikapelyukhin@suse.com>
Date: Mon, 22 Jan 2018 15:52:17 +0100
Subject: [PATCH 09/14] Fix the branch after merge

---
 app/models/product.rb | 1 +
 lib/rmt/scc.rb        | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/app/models/product.rb b/app/models/product.rb
index e5aff41dc..b73db49b8 100644
--- a/app/models/product.rb
+++ b/app/models/product.rb
@@ -79,6 +79,7 @@ def change_repositories_mirroring!(conditions, mirroring_enabled)
 
   def recommended_for?(root_product)
     product_extensions_associations.where(recommended: true).where(root_product: root_product).present?
+  end
 
   def service
     Service.find_or_create_by(product_id: id)
diff --git a/lib/rmt/scc.rb b/lib/rmt/scc.rb
index edaed2a3a..4035453ee 100644
--- a/lib/rmt/scc.rb
+++ b/lib/rmt/scc.rb
@@ -145,7 +145,7 @@ def update_auth_token(item)
     # sle-hae/11.3/s390x available without base product for s390x
     # In this case no repository data was added in create_product -- can't update those repos.
     begin
-      Repository.by_id(item[:id]).update! auth_token: auth_token
+      Repository.find(item[:id]).update! auth_token: auth_token
     rescue ActiveRecord::RecordNotFound
       @logger.debug("Repository #{item[:id]} is not available")
     end

From 74a495fff57cb1d61cbbdec1cdca59764dcad8aa Mon Sep 17 00:00:00 2001
From: Ivan Kapelyukhin <ikapelyukhin@suse.com>
Date: Mon, 22 Jan 2018 16:28:02 +0100
Subject: [PATCH 10/14] Fix spec setup for Travis

---
 spec/lib/rmt/scc_spec.rb | 1 +
 1 file changed, 1 insertion(+)

diff --git a/spec/lib/rmt/scc_spec.rb b/spec/lib/rmt/scc_spec.rb
index d8892de15..1084a9ea6 100644
--- a/spec/lib/rmt/scc_spec.rb
+++ b/spec/lib/rmt/scc_spec.rb
@@ -147,6 +147,7 @@
       let(:products_with_extra_extension) { products + [extra_product] }
 
       before do
+        allow(Settings).to receive(:scc).and_return OpenStruct.new(username: 'foo', password: 'bar')
         allow(api_double).to receive(:list_products).and_return products_with_extra_extension
         allow(api_double).to receive(:list_repositories).and_return repositories_with_extra_repos
         described_class.new.sync

From 1b9e50e2e7615f66045540b0fec3b6cfd9d95a6d Mon Sep 17 00:00:00 2001
From: Ivan Kapelyukhin <ikapelyukhin@suse.com>
Date: Tue, 23 Jan 2018 10:25:26 +0100
Subject: [PATCH 11/14] Fix token saving and add missing specs for that

---
 lib/rmt/scc.rb           |  2 +-
 spec/lib/rmt/scc_spec.rb | 17 +++++++++++++++--
 2 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/lib/rmt/scc.rb b/lib/rmt/scc.rb
index 4035453ee..825cb15a3 100644
--- a/lib/rmt/scc.rb
+++ b/lib/rmt/scc.rb
@@ -145,7 +145,7 @@ def update_auth_token(item)
     # sle-hae/11.3/s390x available without base product for s390x
     # In this case no repository data was added in create_product -- can't update those repos.
     begin
-      Repository.find(item[:id]).update! auth_token: auth_token
+      Repository.find_by!(scc_id: item[:id]).update! auth_token: auth_token
     rescue ActiveRecord::RecordNotFound
       @logger.debug("Repository #{item[:id]} is not available")
     end
diff --git a/spec/lib/rmt/scc_spec.rb b/spec/lib/rmt/scc_spec.rb
index 1084a9ea6..804290ba7 100644
--- a/spec/lib/rmt/scc_spec.rb
+++ b/spec/lib/rmt/scc_spec.rb
@@ -7,9 +7,15 @@
   let!(:subscriptions) { JSON.parse(file_fixture('subscriptions/dummy_subscriptions.json').read, symbolize_names: true) }
   let(:extension) { product[:extensions][0] }
   let(:all_repositories) do
-    products.flat_map do |product|
+    repos = products.flat_map do |product|
       [product, product[:extensions][0]].flat_map { |item| item[:repositories] }
     end
+
+    # Adding tokens to repository URLs, as organization/repositories endpoint does
+    repos.deep_dup.map do |item|
+      item[:url] += "?token_#{item[:id]}"
+      item
+    end
   end
   let(:api_double) { instance_double 'SUSE::Connect::Api' }
 
@@ -31,10 +37,17 @@
       all_repositories.map.each do |repository|
         db_repository = Repository.find_by(scc_id: repository[:id])
 
-        (db_repository.attributes.keys - %w[id scc_id external_url mirroring_enabled local_path]).each do |key|
+        (db_repository.attributes.keys - %w[id scc_id external_url mirroring_enabled local_path auth_token]).each do |key|
           expect(db_repository[key].to_s).to eq(repository[key.to_sym].to_s)
         end
         expect(db_repository[:scc_id]).to eq(repository[:id])
+
+        uri = URI(repository[:url])
+        auth_token = uri.query
+        uri.query = nil
+
+        expect(db_repository[:external_url]).to eq(uri.to_s)
+        expect(db_repository[:auth_token]).to eq(auth_token)
       end
     end
 

From 049974316d827018b7b18f12c1de7a850114e253 Mon Sep 17 00:00:00 2001
From: Ivan Kapelyukhin <ikapelyukhin@suse.com>
Date: Tue, 23 Jan 2018 14:44:34 +0100
Subject: [PATCH 12/14] Add specs for #recommended_for?

---
 spec/models/product_spec.rb | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/spec/models/product_spec.rb b/spec/models/product_spec.rb
index 3c39ff187..c79ee7987 100644
--- a/spec/models/product_spec.rb
+++ b/spec/models/product_spec.rb
@@ -115,4 +115,24 @@
       expect(level_one_extension.mirrored_extensions.for_root_product(base_b)).to eq([level_two_extension_b_mirrored])
     end
   end
+
+  describe '#recommended_for?' do
+    let!(:base_a) { FactoryGirl.create(:product) }
+    let!(:base_b) { FactoryGirl.create(:product) }
+
+    it 'recommended for base product A' do
+      extension = FactoryGirl.create(:product, :extension, base_products: [base_a], recommended: true)
+      expect(extension.recommended_for?(base_a)).to be(true)
+    end
+
+    it 'not recommended for base product A' do
+      extension = FactoryGirl.create(:product, :extension, base_products: [base_a], recommended: false)
+      expect(extension.recommended_for?(base_a)).to be(false)
+    end
+
+    it 'not recommended for unrelated product B' do
+      extension = FactoryGirl.create(:product, :extension, base_products: [base_a], recommended: true)
+      expect(extension.recommended_for?(base_b)).to be(false)
+    end
+  end
 end

From dcfa1aa723f295d02fc79b43f28c5c19164f1aec Mon Sep 17 00:00:00 2001
From: Hernan Schmidt <hschmidt@suse.com>
Date: Tue, 23 Jan 2018 16:55:45 +0100
Subject: [PATCH 13/14] Refactor #recommended_for? specs

---
 spec/models/product_spec.rb | 30 +++++++++++++++++++-----------
 1 file changed, 19 insertions(+), 11 deletions(-)

diff --git a/spec/models/product_spec.rb b/spec/models/product_spec.rb
index c79ee7987..ea870a1b4 100644
--- a/spec/models/product_spec.rb
+++ b/spec/models/product_spec.rb
@@ -117,22 +117,30 @@
   end
 
   describe '#recommended_for?' do
-    let!(:base_a) { FactoryGirl.create(:product) }
-    let!(:base_b) { FactoryGirl.create(:product) }
+    let(:base) { create :product }
+    let(:extension) { create(:product, :extension, base_products: [base], recommended: recommended) }
 
-    it 'recommended for base product A' do
-      extension = FactoryGirl.create(:product, :extension, base_products: [base_a], recommended: true)
-      expect(extension.recommended_for?(base_a)).to be(true)
+    subject { extension.recommended_for?(queried_base) }
+
+    context 'when the extension is recommended for its base' do
+      let(:recommended) { true }
+      let(:queried_base) { base }
+
+      it { is_expected.to be true }
     end
 
-    it 'not recommended for base product A' do
-      extension = FactoryGirl.create(:product, :extension, base_products: [base_a], recommended: false)
-      expect(extension.recommended_for?(base_a)).to be(false)
+    context 'when the extension is not recommended for its base' do
+      let(:recommended) { false }
+      let(:queried_base) { base }
+
+      it { is_expected.to be false }
     end
 
-    it 'not recommended for unrelated product B' do
-      extension = FactoryGirl.create(:product, :extension, base_products: [base_a], recommended: true)
-      expect(extension.recommended_for?(base_b)).to be(false)
+    context "when the queried base is not the extension's base" do
+      let(:recommended) { true }
+      let(:queried_base) { create(:product) }
+
+      it { is_expected.to be false }
     end
   end
 end

From 629adfd37a4bfc4df2bfb6b740553b6312f09c1e Mon Sep 17 00:00:00 2001
From: Ivan Kapelyukhin <ikapelyukhin@suse.com>
Date: Tue, 23 Jan 2018 17:02:20 +0100
Subject: [PATCH 14/14] Fix Rubocop offenses

---
 spec/models/product_spec.rb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/spec/models/product_spec.rb b/spec/models/product_spec.rb
index ea870a1b4..2ef10ebdc 100644
--- a/spec/models/product_spec.rb
+++ b/spec/models/product_spec.rb
@@ -117,11 +117,11 @@
   end
 
   describe '#recommended_for?' do
+    subject { extension.recommended_for?(queried_base) }
+
     let(:base) { create :product }
     let(:extension) { create(:product, :extension, base_products: [base], recommended: recommended) }
 
-    subject { extension.recommended_for?(queried_base) }
-
     context 'when the extension is recommended for its base' do
       let(:recommended) { true }
       let(:queried_base) { base }