Fix set user roles when role is None

CHANGELOG.md

@@ -2,6 +2,50 @@
 
 <!-- <START NEW CHANGELOG ENTRY> -->
 
+## 0.10.1
+
+([Full Changelog](https://github.com/mamba-org/quetz/compare/v0.10.0...ef6836a7c887dc97a89d8b5cce4472a03740b692))
+
+### Bugs fixed
+
+- Fix oauth2 revoke functionality [#665](https://github.com/mamba-org/quetz/pull/665) ([@wolfv](https://github.com/wolfv))
+- Fix docs rest model [#661](https://github.com/mamba-org/quetz/pull/661) ([@mbestipa](https://github.com/mbestipa))
+
+### Contributors to this release
+
+([GitHub contributors page for this release](https://github.com/mamba-org/quetz/graphs/contributors?from=2023-09-11&to=2023-09-28&type=c))
+
+[@codecov-commenter](https://github.com/search?q=repo%3Amamba-org%2Fquetz+involves%3Acodecov-commenter+updated%3A2023-09-11..2023-09-28&type=Issues) | [@mbestipa](https://github.com/search?q=repo%3Amamba-org%2Fquetz+involves%3Ambestipa+updated%3A2023-09-11..2023-09-28&type=Issues) | [@wolfv](https://github.com/search?q=repo%3Amamba-org%2Fquetz+involves%3Awolfv+updated%3A2023-09-11..2023-09-28&type=Issues)
+
+<!-- <END NEW CHANGELOG ENTRY> -->
+
+## 0.10.0
+
+([Full Changelog](https://github.com/mamba-org/quetz/compare/v0.9.2...0854d442d1b20b7eb90e023eef27527b722fbcc6))
+
+### Enhancements made
+
+- Fix postgres pool size [#657](https://github.com/mamba-org/quetz/pull/657) ([@beenje](https://github.com/beenje))
+- Migrate to Pydantic v2 [#656](https://github.com/mamba-org/quetz/pull/656) ([@beenje](https://github.com/beenje))
+
+### Bugs fixed
+
+- Fix double upload bug and improve test [#663](https://github.com/mamba-org/quetz/pull/663) ([@AndreasAlbertQC](https://github.com/AndreasAlbertQC))
+- Add migration script for scoped API keys [#655](https://github.com/mamba-org/quetz/pull/655) ([@beenje](https://github.com/beenje))
+- Fix crash when uploading a package through scoped API key [#647](https://github.com/mamba-org/quetz/pull/647) ([@gabm](https://github.com/gabm))
+- Consider packages.conda for index update and channel mirroring [#638](https://github.com/mamba-org/quetz/pull/638) ([@YYYasin19](https://github.com/YYYasin19))
+
+### Maintenance and upkeep improvements
+
+- Pin pydantic\<2 [#653](https://github.com/mamba-org/quetz/pull/653) ([@AndreasAlbertQC](https://github.com/AndreasAlbertQC))
+- Make upload routes consistent with each other [#635](https://github.com/mamba-org/quetz/pull/635) ([@AndreasAlbertQC](https://github.com/AndreasAlbertQC))
+
+### Contributors to this release
+
+([GitHub contributors page for this release](https://github.com/mamba-org/quetz/graphs/contributors?from=2023-06-21&to=2023-09-11&type=c))
+
+[@AndreasAlbertQC](https://github.com/search?q=repo%3Amamba-org%2Fquetz+involves%3AAndreasAlbertQC+updated%3A2023-06-21..2023-09-11&type=Issues) | [@beenje](https://github.com/search?q=repo%3Amamba-org%2Fquetz+involves%3Abeenje+updated%3A2023-06-21..2023-09-11&type=Issues) | [@codecov-commenter](https://github.com/search?q=repo%3Amamba-org%2Fquetz+involves%3Acodecov-commenter+updated%3A2023-06-21..2023-09-11&type=Issues) | [@gabm](https://github.com/search?q=repo%3Amamba-org%2Fquetz+involves%3Agabm+updated%3A2023-06-21..2023-09-11&type=Issues) | [@janjagusch](https://github.com/search?q=repo%3Amamba-org%2Fquetz+involves%3Ajanjagusch+updated%3A2023-06-21..2023-09-11&type=Issues) | [@wolfv](https://github.com/search?q=repo%3Amamba-org%2Fquetz+involves%3Awolfv+updated%3A2023-06-21..2023-09-11&type=Issues) | [@YYYasin19](https://github.com/search?q=repo%3Amamba-org%2Fquetz+involves%3AYYYasin19+updated%3A2023-06-21..2023-09-11&type=Issues)
+
 ## 0.9.2
 
 ([Full Changelog](https://github.com/mamba-org/quetz/compare/v0.9.1...efd519fd840304fb73fe6fd31ee4ae7f010ab1a2))
@@ -22,8 +66,6 @@
 
 [@AndreasAlbertQC](https://github.com/search?q=repo%3Amamba-org%2Fquetz+involves%3AAndreasAlbertQC+updated%3A2023-06-20..2023-06-21&type=Issues) | [@codecov-commenter](https://github.com/search?q=repo%3Amamba-org%2Fquetz+involves%3Acodecov-commenter+updated%3A2023-06-20..2023-06-21&type=Issues) | [@rominf](https://github.com/search?q=repo%3Amamba-org%2Fquetz+involves%3Arominf+updated%3A2023-06-20..2023-06-21&type=Issues)
 
-<!-- <END NEW CHANGELOG ENTRY> -->
-
 ## 0.9.1
 
 ([Full Changelog](https://github.com/mamba-org/quetz/compare/v0.9.0...13f588cc16560c78927e4b2406a77958a62d5ca6))

environment.yml

@@ -50,6 +50,7 @@ dependencies:
   - conda-content-trust
   - pyinstrument
   - pytest-asyncio
-  - pydantic <2
+  - pytest-timeout
+  - pydantic >=2
   - pip:
       - git+https://github.com/jupyter-server/jupyter_releaser.git@v2

pyproject.toml

@@ -20,7 +20,7 @@ check-imports = ["quetz"]
 ignore = ["W004"]
 
 [tool.tbump.version]
-current = "0.9.2"
+current = "0.10.1"
 regex = '''
   (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)
   ((?P<channel>a|b|rc|.dev)(?P<release>\d+))?

quetz/_version.py

@@ -1,2 +1,2 @@
-version_info = (0, 9, 2, "", "")
+version_info = (0, 10, 1, "", "")
 __version__ = '.'.join(filter(lambda s: len(s) > 0, map(str, version_info)))

quetz/authentication/oauth2.py

@@ -29,9 +29,11 @@ async def login(self, request: Request):
             request, str(redirect_uri)
         )
 
-    async def revoke(self, request):
-        client_id = self.client.client_id
-        return RedirectResponse(self.revoke_url.format(client_id=client_id))
+    async def revoke(self):
+        client_id = self.authenticator.client_id
+        return RedirectResponse(
+            self.authenticator.revoke_url.format(client_id=client_id)
+        )
 
 
 class OAuthAuthenticator(BaseAuthenticator):

quetz/db_models.py

@@ -55,7 +55,7 @@ class User(Base):
         'Profile', uselist=False, back_populates='user', cascade="all,delete-orphan"
     )
 
-    role = Column(String)
+    role = Column(String, nullable=True)
 
     @classmethod
     def find(cls, db, name):

quetz/jobs/api.py

@@ -17,7 +17,7 @@
 from quetz.rest_models import PaginatedResponse
 
 from .models import JobStatus, TaskStatus
-from .rest_models import Job, JobBase, JobUpdateModel, Task
+from .rest_models import Job, JobCreate, JobUpdateModel, Task
 
 api_router = APIRouter()
 
@@ -44,7 +44,7 @@ def get_jobs(
 
 @api_router.post("/api/jobs", tags=["Jobs"], status_code=201, response_model=Job)
 def create_job(
-    job: JobBase,
+    job: JobCreate,
     dao: Dao = Depends(get_dao),
     auth: authorization.Rules = Depends(get_rules),
 ):

quetz/jobs/rest_models.py

@@ -5,7 +5,7 @@
 from typing import Optional
 
 from importlib_metadata import entry_points as get_entry_points
-from pydantic import BaseModel, Field, validator
+from pydantic import BaseModel, ConfigDict, Field, field_validator
 
 from . import handlers
 from .models import JobStatus, TaskStatus
@@ -83,7 +83,6 @@ def parse_job_name(v):
 class JobBase(BaseModel):
     """New job spec"""
 
-    items_spec: str = Field(..., title='Item selector spec')
     manifest: str = Field(None, title='Name of the function')
 
     start_at: Optional[datetime] = Field(
@@ -97,7 +96,8 @@ class JobBase(BaseModel):
         ),
     )
 
-    @validator("manifest", pre=True)
+    @field_validator("manifest", mode="before")
+    @classmethod
     def validate_job_name(cls, function_name):
         if isinstance(function_name, bytes):
             return parse_job_name(function_name)
@@ -107,6 +107,12 @@ def validate_job_name(cls, function_name):
         return function_name.encode('ascii')
 
 
+class JobCreate(JobBase):
+    """Create job spec"""
+
+    items_spec: str = Field(..., title='Item selector spec')
+
+
 class JobUpdateModel(BaseModel):
     """Modify job spec items (status and items_spec)"""
 
@@ -123,10 +129,8 @@ class Job(JobBase):
 
     status: JobStatus = Field(None, title='Status of the job (running, paused, ...)')
 
-    items_spec: str = Field(None, title='Item selector spec')
-
-    class Config:
-        orm_mode = True
+    items_spec: Optional[str] = Field(None, title='Item selector spec')
+    model_config = ConfigDict(from_attributes=True)
 
 
 class Task(BaseModel):
@@ -136,12 +140,12 @@ class Task(BaseModel):
     created: datetime = Field(None, title='Created at')
     status: TaskStatus = Field(None, title='Status of the task (running, paused, ...)')
 
-    @validator("package_version", pre=True)
+    @field_validator("package_version", mode="before")
+    @classmethod
     def convert_package_version(cls, v):
         if v:
             return {'filename': v.filename, 'id': uuid.UUID(bytes=v.id).hex}
         else:
             return {}
 
-    class Config:
-        orm_mode = True
+    model_config = ConfigDict(from_attributes=True)

quetz/main.py

@@ -17,7 +17,6 @@
 from typing import Awaitable, Callable, List, Optional, Tuple, Type
 
 import pydantic
-import pydantic.error_wrappers
 import requests
 from fastapi import (
     APIRouter,
@@ -477,7 +476,7 @@ def delete_user(
 
 @api_router.get(
     "/users/{username}/role",
-    response_model=rest_models.UserRole,
+    response_model=rest_models.UserOptionalRole,
     tags=["users"],
 )
 def get_user_role(
@@ -732,7 +731,7 @@ def post_channel(
             detail="Cannot use both `includelist` and `excludelist` together.",
         )
 
-    user_attrs = new_channel.dict(exclude_unset=True)
+    user_attrs = new_channel.model_dump(exclude_unset=True)
 
     if "size_limit" in user_attrs:
         auth.assert_set_channel_size_limit()
@@ -789,7 +788,7 @@ def patch_channel(
 ):
     auth.assert_update_channel_info(channel.name)
 
-    user_attrs = channel_data.dict(exclude_unset=True)
+    user_attrs = channel_data.model_dump(exclude_unset=True)
 
     if "size_limit" in user_attrs:
         auth.assert_set_channel_size_limit()
@@ -1064,7 +1063,7 @@ def get_package_versions(
     version_list = []
 
     for version, profile, api_key_profile in version_profile_list:
-        version_data = rest_models.PackageVersion.from_orm(version)
+        version_data = rest_models.PackageVersion.model_validate(version)
         version_list.append(version_data)
 
     return version_list
@@ -1089,7 +1088,7 @@ def get_paginated_package_versions(
     version_list = []
 
     for version, profile, api_key_profile in version_profile_list['result']:
-        version_data = rest_models.PackageVersion.from_orm(version)
+        version_data = rest_models.PackageVersion.model_validate(version)
         version_list.append(version_data)
 
     return {
@@ -1396,9 +1395,6 @@ async def post_upload(
 
     dest = os.path.join(condainfo.info["subdir"], filename)
 
-    body.seek(0)
-    await pkgstore.add_package_async(body, channel_name, dest)
-
     package_name = str(condainfo.info.get("name"))
     package_data = rest_models.Package(
         name=package_name,
@@ -1435,6 +1431,9 @@ async def post_upload(
         logger.debug(f"duplicate package '{package_name}' in channel '{channel_name}'")
         raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail="Duplicate")
 
+    body.seek(0)
+    await pkgstore.add_package_async(body, channel_name, dest)
+
     pm.hook.post_add_package_version(version=version, condainfo=condainfo)
 
     wrapped_bg_task = background_task_wrapper(indexing.update_indexes, logger)
@@ -1479,7 +1478,7 @@ def _assert_filename_package_name_consistent(file_name: str, package_name: str):
     wait=wait_exponential(multiplier=1, min=4, max=10),
     after=after_log(logger, logging.WARNING),
 )
-def _extract_and_upload_package(file, channel_name, channel_proxylist):
+def _extract_and_upload_package(file, channel_name, channel_proxylist, force: bool):
     try:
         conda_info = CondaInfo(file.file, file.filename)
     except Exception as e:
@@ -1497,6 +1496,10 @@ def _extract_and_upload_package(file, channel_name, channel_proxylist):
         logger.info(f"Skip upload of proxied file {file.filename}")
         return conda_info
 
+    # Make sure that we do not upload over existing files
+    if pkgstore.file_exists(channel_name, dest) and not force:
+        raise FileExistsError(f"{file.filename}")
+
     try:
         file.file.seek(0)
         logger.debug(
@@ -1574,8 +1577,14 @@ def handle_package_files(
                     files,
                     (channel.name,) * len(files),
                     (channel_proxylist,) * len(files),
+                    (force,) * len(files),
                 )
             ]
+        except FileExistsError as e:
+            raise HTTPException(
+                status_code=status.HTTP_409_CONFLICT,
+                detail=f"Duplicate {str(e)}",
+            )
         except exceptions.PackageError as e:
             raise HTTPException(
                 status_code=status.HTTP_400_BAD_REQUEST, detail=e.detail
@@ -1650,7 +1659,7 @@ def _delete_file(condainfo, filename):
                     summary=str(condainfo.about.get("summary", "n/a")),
                     description=str(condainfo.about.get("description", "n/a")),
                 )
-            except pydantic.error_wrappers.ValidationError as err:
+            except pydantic.ValidationError as err:
                 _delete_file(condainfo, file.filename)
                 raise errors.ValidationError(
                     "Validation Error for package: "

quetz/metrics/rest_models.py

@@ -1,17 +1,15 @@
 from datetime import datetime
 from typing import Dict, List
 
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, ConfigDict, Field
 
 from quetz.metrics.db_models import IntervalType
 
 
 class PackageVersionMetricItem(BaseModel):
     timestamp: datetime
     count: int
-
-    class Config:
-        orm_mode = True
+    model_config = ConfigDict(from_attributes=True)
 
 
 class PackageVersionMetricSeries(BaseModel):

quetz/migrations/versions/3ba25f23fb7d_update_scoped_api_key_uploader_id.py

@@ -0,0 +1,61 @@
+"""Update scoped API key uploader id
+
+Revision ID: 3ba25f23fb7d
+Revises: d212023a8e0b
+Create Date: 2023-08-02 08:03:09.961559
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = '3ba25f23fb7d'
+down_revision = 'd212023a8e0b'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    package_versions = sa.sql.table("package_versions", sa.sql.column("uploader_id"))
+    conn = op.get_bind()
+    # Get all user_id/owner_id from channel scoped API keys
+    # (user is anonymous - username is null)
+    res = conn.execute(
+        sa.text(
+            """SELECT api_keys.user_id, api_keys.owner_id FROM api_keys
+            INNER JOIN users ON users.id = api_keys.user_id
+            WHERE users.username is NULL;
+            """
+        )
+    )
+    results = res.fetchall()
+    # Replace the uploader with the key owner (real user instead of the anonymous one)
+    for result in results:
+        op.execute(
+            package_versions.update()
+            .where(package_versions.c.uploader_id == result[0])
+            .values(uploader_id=result[1])
+        )
+
+
+def downgrade():
+    package_versions = sa.sql.table("package_versions", sa.sql.column("uploader_id"))
+    conn = op.get_bind()
+    # Get all user_id/owner_id from channel scoped API keys
+    # (user is anonymous - username is null)
+    res = conn.execute(
+        sa.text(
+            """SELECT api_keys.user_id, api_keys.owner_id FROM api_keys
+            INNER JOIN users ON users.id = api_keys.user_id
+            WHERE users.username is NULL;
+            """
+        )
+    )
+    results = res.fetchall()
+    # Replace the uploader with the key anonymous user
+    for result in results:
+        op.execute(
+            package_versions.update()
+            .where(package_versions.c.uploader_id == result[1])
+            .values(uploader_id=result[0])
+        )