Add pre/postinstall scripts to prevent 'npm install' failure

.github/workflows/rubocop.yml

@@ -10,11 +10,12 @@ jobs:
         uses: actions/checkout@v2
       - uses: ruby/setup-ruby@v1
         with:
-          ruby-version: 2.5
+          ruby-version: 2.5.8
+      - run: 'gem install activesupport -v 6.1.4.4'
       - name: Rubocop linter
         uses: reviewdog/action-rubocop@v1
         with:
-          rubocop_version: 1.12.1
+          rubocop_version: 1.23.0
           github_token: ${{ secrets.github_token }}
           reporter: github-pr-check
           level: error

.gitmodules

@@ -1,12 +1,12 @@
 [submodule "lib/assets/javascripts/cdb"]
 	path = lib/assets/javascripts/cdb
-        url = git://github.com/CartoDB/carto.js.git
+        url = git@github.com:CartoDB/carto.js.git
 [submodule "app/assets/stylesheets/old_common"]
 	path = app/assets/stylesheets/old_common
-	url = git://github.com/CartoDB/cartodb.css.git
+	url = git@github.com:CartoDB/cartodb.css.git
 [submodule "lib/sql"]
 	path = lib/sql
-	url = git://github.com/CartoDB/cartodb-postgresql.git
+	url = git@github.com:CartoDB/cartodb-postgresql.git
 [submodule "private"]
 	path = private
 	url = git@github.com:CartoDB/cartodb-private.git

.readthedocs.yaml

@@ -0,0 +1,4 @@
+version: 2
+
+submodules:
+  exclude: all

.rubocop.yml

@@ -6,7 +6,7 @@ require:
   - rubocop-rspec
 
 AllCops:
-  TargetRubyVersion: 2.4
+  TargetRubyVersion: 2.5
   Exclude:
   - "vendor/**/*"
   - "db/schema.rb"

NEWS.md

@@ -2,9 +2,10 @@ Development
 -----------
 
 ### NOTICES
-- None yet
+- None yet 
 
 ### Features
+* Display GeoParquet link in Catalog [16453](https://github.com/CartoDB/cartodb/pull/16453)
 * Display notifications about the new CARTO platform release [16352](https://github.com/CartoDB/cartodb/pull/16352)
 * Upgrade to deck.gl 8.5.6 [16338](https://github.com/CartoDB/cartodb/pull/16338)
 * Update DO Catalog dependencies and some changes to use bundle on CARTO Workspace [#16325](https://github.com/CartoDB/cartodb/pull/16325)
@@ -17,24 +18,41 @@ Development
 * Upgrade to CARTO Viewer v1.0.8 [16347](https://github.com/CartoDB/cartodb/pull/16347)
 * Show user's database location in profile [16349](https://github.com/CartoDB/cartodb/pull/16349)
 * Setting to enable/disable import notifications [16354](https://github.com/CartoDB/cartodb/pull/16354)
+* Setting to enable/disable random username generation on SAML authentication process [16372](https://github.com/CartoDB/cartodb/pull/16372)
+* Add type guessing capabilities to the ArcGIS connector [#16385](https://github.com/CartoDB/cartodb/pull/16385)
+* Add notification about data migrations to CARTO 3 [#16405](https://github.com/CartoDB/cartodb/pull/16405)
+* Update banner to notify about data migrations to CARTO 3 [#16420](https://github.com/CartoDB/cartodb/pull/16420)
 
 ### Bug fixes / enhancements
+- Removing the full path from urls with filter parameters in the Spatial Data Catalog [#16426](https://github.com/CartoDB/cartodb/pull/16426)
+- Fix rubocop integration [#16382](https://github.com/CartoDB/cartodb/pull/16382)
 - Add marginTop to Page when notification is displayed [#16355](https://github.com/CartoDB/cartodb/pull/16355)
 - Add "element" param to DO-Catalog entry function [#16343](https://github.com/CartoDB/cartodb/pull/16343)
 - Add new DO Catalog route for internal usage [#16342](https://github.com/CartoDB/cartodb/pull/16342)
+- Propagate 'invitation_token' when there is an error signing-up with Google [#16391](https://github.com/CartoDB/cartodb/pull/16391)
+- Reverse analysis selection order on new widget form [#16412](https://github.com/CartoDB/cartodb/pull/16412)
+- Improve info for :update_user command  [#16363](https://github.com/CartoDB/cartodb/pull/16363)
 - Disable email validation in DO Premium Subscriptions [#16309](https://github.com/CartoDB/cartodb/pull/16309)
+- Invalidate sessions on 'session_salt' issue [#16376](https://github.com/CartoDB/cartodb/pull/16376)
 - Hide sharing tab from viewer in on-premises [#16299](https://github.com/CartoDB/cartodb/pull/16299)
+- Update browser version checker to allow Firefox/100.0 [#16415](https://github.com/CartoDB/cartodb/pull/16415)
+- Update analysis schemas after giving required permissions on user promotion [#16390](https://github.com/CartoDB/cartodb/pull/16390)
+- Add timeout for SQL API exports [#16377](https://github.com/CartoDB/cartodb/pull/16377)
+- Avoid deleting a user if it has shared entities [#16424](https://github.com/CartoDB/cartodb/pull/16424)
 - Remove all references to Spatial Data Catalog and Kepler GL maps in on-premises [#16293](https://github.com/CartoDB/cartodb/pull/16293)
+- Increase hard-limit of MAX_TABLES_PER_IMPORT [#16374](https://github.com/CartoDB/cartodb/pull/16374)
 - Guard code for vizjson users [#16267](https://github.com/CartoDB/cartodb/pull/16267)
 - Guard code for Users and Visualizations [#16265](https://github.com/CartoDB/cartodb/pull/16265)
 - Use the organization user's data while editing a user from organization settings [#16280](https://github.com/CartoDB/cartodb/pull/16280)
 - Fix schema name in layers created by free users [#16307](https://github.com/CartoDB/cartodb/pull/16307)
 - Limit start parameter of Dropbox connector [#16264](https://github.com/CartoDB/cartodb/pull/16264)
 - Fix messages about layer limit being reached [#16360](https://github.com/CartoDB/cartodb/pull/16360)
+- Fix 404.html page [#16369](https://github.com/CartoDB/cartodb/pull/16369)
 - Fix deck.gl dependency conflicts [#16339](https://github.com/CartoDB/cartodb/pull/16339)
 - Migrate Redis DO subscription information in inter-cloud migrations [#16315](https://github.com/CartoDB/cartodb/pull/16315)
 - OauthApps restricted by default [#16304](https://github.com/CartoDB/cartodb/pull/16304)
 - Support staging hostname in the catalog [#16258](https://github.com/CartoDB/cartodb/pull/16258)
+- Add custom redirection on `developers.carto.com/login` [#16383](https://github.com/CartoDB/cartodb/pull/16383)
 - Fix user migration export/import logs [#16298](https://github.com/CartoDB/cartodb/pull/16298)
 - Fix race condition when DO subscriptions are created [#16311](https://github.com/CartoDB/cartodb/pull/16311)
 - Allow the usage of WMTS URLs with parameters to create custom basemaps [#16271](https://github.com/CartoDB/cartodb/pull/16271)
@@ -46,6 +64,7 @@ Development
 - Fix verification process for active users [#16337](https://github.com/CartoDB/cartodb/pull/16337)
 - Avoid updating analysis nodes more than once when moving layers in Builder [#16279](https://github.com/CartoDB/cartodb/pull/16279)
 - Fix subscription/sample filter for datasets [#16254](https://github.com/CartoDB/cartodb/pull/16254)
+- Fix form to search dataset when generating a new API key [#16378](https://github.com/CartoDB/cartodb/pull/16378)
 - Use fully qualified table name while creating a new map from a shared dataset [#16241](https://github.com/CartoDB/cartodb/pull/16241)
 - Render tileset viewer features in front of basemap [#16333](https://github.com/CartoDB/cartodb/pull/16333)
 - Rake task to migrate legacy synchronizations [#16353](https://github.com/CartoDB/cartodb/pull/16353)
@@ -70,6 +89,19 @@ Development
 - Fix Auth URL generation while establishing a connection with Google Drive [#16357](https://github.com/CartoDB/cartodb/pull/16357)
 - Fix adding license metadata to a dataset [#16356](https://github.com/CartoDB/cartodb/pull/16356)
 - Fix notifications when organization seats limit is reached [#16359](https://github.com/CartoDB/cartodb/pull/16359)
+- Notify Support when a user is reaching the named maps limit [#16368](https://github.com/CartoDB/cartodb/pull/16368)
+- Remove old named maps when a user is reaching the named maps limit [#16368](https://github.com/CartoDB/cartodb/pull/16368)
+- Fix privacy dropdown when user is editing a map [#16367](https://github.com/CartoDB/cartodb/pull/16367)
+- Add a new rake to update a user username [#16370](https://github.com/CartoDB/cartodb/pull/16370)
+- Add a check before destroying user tables in order to avoid deleting dependent maps [#16381](https://github.com/CartoDB/cartodb/pull/16381)
+- Fix duplicated attributions in datasets [#16384](https://github.com/CartoDB/cartodb/pull/16384)
+- Moving assets cdn domain from global.ssl.fastly.net to libs.cartocdn.com [#16399](https://github.com/CartoDB/cartodb/pull/16399)
+- Fix error while rolling back a user migration from one cloud to another [#16421](https://github.com/CartoDB/cartodb/pull/16421)
+- Add retry if a timeout is thrown when swapping the tables related with a sync process [#16430](https://github.com/CartoDB/cartodb/pull/16430)
+- Add AUTODETECT_SIZE_LIMIT to ogr2ogr process when guessing CSV file column types [#16431](https://github.com/CartoDB/cartodb/pull/16431)
+- Log pg locks if there is any problem during a sync table import process [#16432](https://github.com/CartoDB/cartodb/pull/16432)
+- Check pg locks during sync table swap and terminate locking queries [#16433](https://github.com/CartoDB/cartodb/pull/16433)
+- Add deprecation notice in docs [#16446](https://github.com/CartoDB/cartodb/pull/16446)
 
 4.45.0 (2021-04-14)
 -------------------

README.md

@@ -1,3 +1,10 @@
+# [DEPRECATED]
+
+Hey! This content applies only to previous CARTO products.
+
+Please check if it's relevant to your use case. On October 2021 we released the current version of our platform.
+You can learn more and read the latest documentation at docs.carto.com
+
 # What is CARTO?
 
 [![Code Climate](https://codeclimate.com/github/CartoDB/cartodb20.png)](https://codeclimate.com/github/CartoDB/cartodb20)

app/commands/central_user_commands.rb

@@ -14,7 +14,12 @@ def initialize(notifications_topic:, logger:)
   def update_user(message)
     payload = message.payload
     Carto::Common::CurrentRequest.with_request_id(message.request_id) do
-      logger.info(message: 'Processing :update_user', class_name: self.class.name)
+      logger.info(
+        message: 'Processing :update_user',
+        remote_user_id: payload['remote_user_id'],
+        class_name: self.class.name
+      )
+
       user_id = payload.delete('remote_user_id')
 
       return unless user_id.present? && payload.any?

app/controllers/admin/organizations_controller.rb

@@ -159,6 +159,7 @@ def auth_update
     @organization.auth_github_enabled = attributes[:auth_github_enabled]
     @organization.strong_passwords_enabled = attributes[:strong_passwords_enabled]
     @organization.password_expiration_in_d = attributes[:password_expiration_in_d]
+    @organization.random_saml_username = attributes[:random_saml_username]
     @organization.update_in_central
     @organization.save(raise_on_failure: true)
 

app/controllers/application_controller.rb

@@ -81,6 +81,7 @@ def current_viewer
     @current_viewer
   rescue Carto::ExpiredSessionError => e
     request.reset_session
+    current_user.try(:invalidate_all_sessions!)
     not_authorized(e)
   end
 
@@ -415,7 +416,7 @@ def browser_is_html5_compliant?
     return true if user_agent.nil?
 
     banned_regex = [
-      /msie [0-9]\./, /safari\/[0-4][0-2][0-2]/, /opera\/[0-8].[0-7]/, /firefox\/[0-2].[0-5]/
+      /msie [0-9]\./, /safari\/[0-4][0-2][0-2]/, /opera\/[0-8].[0-7]/, /firefox\/[0-2]\.[0-5]/
     ]
 
     if banned_regex.map { |re| user_agent.match(re) }.compact.first

app/controllers/carto/api/organization_presenter.rb

@@ -35,8 +35,8 @@ def to_poro
           mapzen_routing_block_price: @organization.mapzen_routing_block_price,
           geocoder_provider:          @organization.geocoder_provider,
           isolines_provider:          @organization.isolines_provider,
-          routing_provider: @organization.routing_provider,
-          map_views_quota: @organization.map_views_quota,
+          routing_provider:           @organization.routing_provider,
+          map_views_quota:            @organization.map_views_quota,
           twitter_datasource_quota:   @organization.twitter_datasource_quota,
           map_view_block_price:       @organization.map_view_block_price,
           geocoding_block_price:      @organization.geocoding_block_price,
@@ -49,7 +49,8 @@ def to_poro
           admin_email:                @organization.admin_email,
           avatar_url:                 @organization.avatar_url,
           user_count:                 @organization.users.count,
-          password_expiration_in_d:   @organization.password_expiration_in_d
+          password_expiration_in_d:   @organization.password_expiration_in_d,
+          random_saml_username:       @organization.random_saml_username
         }
       end
 

app/controllers/carto/api/organization_users_controller.rb

@@ -121,8 +121,8 @@ def destroy
         force_destroy = params[:force].present?
 
         if !force_destroy && @user.has_shared_entities?
-          error_message = "Can't delete @user. 'Has shared entities"
-          render_jsonp(error_message, 410 ) and return
+          error_message = "Can't delete user. Has shared entities"
+          render_jsonp(error_message, 401) and return
         end
 
         @user.set_force_destroy if force_destroy

app/controllers/carto/api/users_controller.rb

@@ -111,7 +111,13 @@ def delete_me
         deletion_password_confirmation = params[:deletion_password_confirmation]
 
         if user.needs_password_confirmation? && !user.validate_old_password(deletion_password_confirmation)
-          render_jsonp({ message: "Error deleting user: #{PASSWORD_DOES_NOT_MATCH_MESSAGE}" }, 400) and return
+          render_jsonp({ message: "Error deleting user: #{PASSWORD_DOES_NOT_MATCH_MESSAGE}" }, 400)
+          return
+        end
+
+        if user.has_shared_entities?
+          render_jsonp({ message: "User can't be deleted because there are shared entities. Please, unshare or delete them and try again." }, 401)
+          return
         end
 
         user.destroy_account

app/controllers/sessions_controller.rb

@@ -39,6 +39,7 @@ class SessionsController < ApplicationController
   before_filter :load_organization
   before_filter :initialize_oauth_config
   before_filter :api_authorization_required, only: :show
+  before_action :custom_redirect, only: :new
   after_action  :set_last_mfa_activity, only: [:multifactor_authentication, :multifactor_authentication_verify_code]
 
   PLEASE_LOGIN = 'Please, log in to continue using CARTO.'.freeze
@@ -186,10 +187,17 @@ def saml_user_not_in_carto
       return
     end
 
+    organization_id = warden.env['warden.options'][:organization_id]
+    organization = Carto::Organization.find(organization_id)
     saml_email = warden.env['warden.options'][:saml_email]
-    username = CartoDB::UserAccountCreator.email_to_username(saml_email)
+
+    if organization.random_saml_username
+      username = CartoDB::UserAccountCreator.random_saml_username
+    else
+      username = CartoDB::UserAccountCreator.email_to_username(saml_email)
+    end
+
     unique_username = Carto::UsernameProposer.find_unique(username)
-    organization_id = warden.env['warden.options'][:organization_id]
 
     create_user(
       username: unique_username,
@@ -309,6 +317,12 @@ def saml_user?
 
   private
 
+  # HACK: CARTO internals custom redirection
+  #   https://app.shortcut.com/cartoteam/story/198852/update-developers-carto-com-login-redirection-to-carto-com-signin
+  def custom_redirect
+    redirect_to('https://carto.com/signin/') if CartoDB.extract_subdomain(request) == 'developers'
+  end
+
   def mfa_request?
     params[:code].presence || params[:skip].presence
   end

app/mailers/reporter_mailer.rb

@@ -8,4 +8,11 @@ def trending_maps_report(mail_to, trending_visualizations)
 
     mail to: mail_to, subject: @subject
   end
+
+  def named_maps_near_the_limit(message)
+    mail_to = Cartodb.get_config(:mailer, 'support_email')
+    mail_from = Cartodb.get_config(:mailer, 'internal_notifications_email')
+
+    mail from: mail_from, to: mail_to, subject: message if mail_to && mail_from
+  end
 end

app/models/carto/helpers/user_commons.rb

@@ -42,7 +42,7 @@ module Carto::UserCommons
                           salesforce_datasource_enabled builder_enabled geocoder_provider isolines_provider
                           routing_provider engine_enabled mapzen_routing_quota
                           mapzen_routing_block_price soft_mapzen_routing_limit no_map_logo org_admin
-                          user_render_timeout database_render_timeout frontend_version asset_host
+                          user_render_timeout database_render_timeout export_timeout frontend_version asset_host
                           state rate_limit_id public_map_quota regular_api_key_quota
                           maintenance_mode private_map_quota public_dataset_quota]
 

app/models/carto/user_migration_import.rb

@@ -109,6 +109,7 @@ def do_import_metadata(package, service)
         raise e
       rescue StandardError => e
         log.append('=== Error importing metadata. Rollback! ===')
+        log.append(e.backtrace.join("\n"))
         service.rollback_import_from_directory(package.meta_dir)
         raise e
       end

app/models/carto/user_table.rb

@@ -13,6 +13,9 @@ def min_value
 
 module Carto
   class UserTable < ActiveRecord::Base
+
+    attr_accessor :skip_destroy_dependent_visualizations
+
     PRIVACY_PRIVATE = 0
     PRIVACY_PUBLIC = 1
     PRIVACY_LINK = 2
@@ -68,7 +71,7 @@ class UserTable < ActiveRecord::Base
     before_destroy :ensure_not_viewer
     before_destroy :cache_dependent_visualizations, unless: :destroyed?
     before_destroy :backup_visualizations, unless: :destroyed?
-    after_destroy :destroy_dependent_visualizations
+    after_destroy :destroy_dependent_visualizations, unless: :skip_destroy_dependent_visualizations
     after_destroy :service_after_destroy
 
     def geometry_types

app/models/concerns/cartodb_central_synchronizable.rb

@@ -84,7 +84,7 @@ def allowed_attributes_from_central(action)
            salesforce_datasource_enabled geocoder_provider
            isolines_provider routing_provider engine_enabled builder_enabled
            mapzen_routing_quota mapzen_routing_block_price no_map_logo auth_github_enabled
-           password_expiration_in_d inherit_owner_ffs)
+           password_expiration_in_d inherit_owner_ffs random_saml_username)
       when :update
         %i(seats viewer_seats quota_in_bytes display_name description website
            discus_shortname twitter_username geocoding_quota map_views_quota
@@ -96,7 +96,7 @@ def allowed_attributes_from_central(action)
            salesforce_datasource_enabled geocoder_provider
            isolines_provider routing_provider engine_enabled builder_enabled
            mapzen_routing_quota mapzen_routing_block_price no_map_logo auth_github_enabled
-           password_expiration_in_d inherit_owner_ffs)
+           password_expiration_in_d inherit_owner_ffs random_saml_username)
       end
     elsif user?
       %i(account_type admin org_admin crypted_password database_host
@@ -115,7 +115,7 @@ def allowed_attributes_from_central(action)
          salesforce_datasource_enabled viewer geocoder_provider
          isolines_provider routing_provider engine_enabled builder_enabled
          mapzen_routing_quota mapzen_routing_block_price soft_mapzen_routing_limit no_map_logo
-         user_render_timeout database_render_timeout state industry company phone job_role
+         user_render_timeout database_render_timeout export_timeout state industry company phone job_role
          password_reset_token password_reset_sent_at maintenance_mode company_employees use_case private_map_quota
          session_salt public_dataset_quota dashboard_viewed_at email_verification_token email_verification_sent_at)
     end
@@ -129,7 +129,7 @@ def allowed_attributes_to_central(action)
       when :update
         allowed_attributes = %i(seats viewer_seats display_name description website discus_shortname twitter_username
                                 auth_username_password_enabled auth_google_enabled password_expiration_in_d
-                                inherit_owner_ffs)
+                                inherit_owner_ffs random_saml_username)
         attributes.symbolize_keys.slice(*allowed_attributes).merge(name: name)
       end
     elsif user?