diff --git a/docs/parameters.yaml b/docs/parameters.yaml index b8313a93c..117e70d2f 100644 --- a/docs/parameters.yaml +++ b/docs/parameters.yaml @@ -625,6 +625,57 @@ type: object default: none components: ["nsregistry"] --- +name: Registry.CustomRegistrationFields +description: >- + An array of objects specifying **additional** fields when registering namespaces. + + The schema of the object is as follows: + ``` + - name: department + type: enum + required: true + options: + - name: Math + id: math + - name: Computer Science + id: cs + optionsUrl: https://example.com/options + validationUrl: https://example.com/validate + description: The department of the organization that holds this namespace + ``` + + Note the following requirements: + - "name" must be snake case with underline connecting words, i.e. site_name. + - "type" must be one of "string", "int", "bool", "datetime" (Unix time in seconds), or "enum". + - "options" must be a non-empty yaml array for field with type "enum". "optionsUrl" will be ignored if "options" is set. + example: + + ```yaml + options: + - name: "Option A" + id: "optionA" + ``` + + - "optionsUrl" is alternative to options with a URL return an JSON array of options in the following format: + + ```json + [ + { + "name":"Option A", + "id":"optionA" + } + ] + ``` + - "validationUrl" is an optional, user-provided validation endpoint + that listens to a POST request with JSON body of {"field": ""} + and responds with a 200 code and JSON of {"valid": bool, "message": ""}. If it's non-empty, + for all create and update of namespace registration, this URL will be called + to validate the custom field. If "valid" == "false", the "message" will be returned by Pelican + - "description" will show up in the web UI as helper text to help user understand the field +type: object +default: none +components: ["nsregistry"] +--- name: Registry.InstitutionsUrl description: >- A url to get a list of available institutions for users to register their namespaces to. diff --git a/go.mod b/go.mod index f5885281a..2f8a07a9e 100644 --- a/go.mod +++ b/go.mod @@ -160,7 +160,7 @@ require ( golang.org/x/mod v0.12.0 // indirect golang.org/x/sync v0.3.0 golang.org/x/sys v0.14.0 // indirect - golang.org/x/text v0.14.0 // indirect + golang.org/x/text v0.14.0 golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.11.0 // indirect google.golang.org/appengine v1.6.7 // indirect diff --git a/launchers/registry_serve.go b/launchers/registry_serve.go index 51ebab439..bca88864c 100644 --- a/launchers/registry_serve.go +++ b/launchers/registry_serve.go @@ -42,6 +42,11 @@ func RegistryServe(ctx context.Context, engine *gin.Engine, egrp *errgroup.Group return errors.Wrap(err, "Unable to initialize the namespace registry database") } + err = registry.InitCustomRegistrationFields() + if err != nil { + return err + } + if config.GetPreferredPrefix() == "OSDF" { metrics.SetComponentHealthStatus(metrics.DirectorRegistry_Topology, metrics.StatusWarning, "Start requesting from topology, status unknown") log.Info("Populating registry with namespaces from OSG topology service...") diff --git a/param/parameters.go b/param/parameters.go index 5d113eef0..d63005a88 100644 --- a/param/parameters.go +++ b/param/parameters.go @@ -208,5 +208,6 @@ var ( GeoIPOverrides = ObjectParam{"GeoIPOverrides"} Issuer_AuthorizationTemplates = ObjectParam{"Issuer.AuthorizationTemplates"} Issuer_OIDCAuthenticationRequirements = ObjectParam{"Issuer.OIDCAuthenticationRequirements"} + Registry_CustomRegistrationFields = ObjectParam{"Registry.CustomRegistrationFields"} Registry_Institutions = ObjectParam{"Registry.Institutions"} ) diff --git a/param/parameters_struct.go b/param/parameters_struct.go index 21eba358e..aafd9f70c 100644 --- a/param/parameters_struct.go +++ b/param/parameters_struct.go @@ -116,6 +116,7 @@ type config struct { } Registry struct { AdminUsers []string + CustomRegistrationFields interface{} DbLocation string Institutions interface{} InstitutionsUrl string @@ -288,6 +289,7 @@ type configWithType struct { } Registry struct { AdminUsers struct { Type string; Value []string } + CustomRegistrationFields struct { Type string; Value interface{} } DbLocation struct { Type string; Value string } Institutions struct { Type string; Value interface{} } InstitutionsUrl struct { Type string; Value string } diff --git a/registry/registry_db.go b/registry/registry_db.go index 4578c1094..03f9099d8 100644 --- a/registry/registry_db.go +++ b/registry/registry_db.go @@ -43,7 +43,7 @@ import ( type RegistrationStatus string -// The AdminMetadata is used in [Namespace] as a marshalled JSON string +// The AdminMetadata is used in [Namespace] as a marshaled JSON string // to be stored in registry DB. // // The *UserID are meant to correspond to the "sub" claim of the user token that @@ -72,11 +72,12 @@ type AdminMetadata struct { } type Namespace struct { - ID int `json:"id" post:"exclude"` - Prefix string `json:"prefix" validate:"required"` - Pubkey string `json:"pubkey" validate:"required"` - Identity string `json:"identity" post:"exclude"` - AdminMetadata AdminMetadata `json:"admin_metadata"` + ID int `json:"id" post:"exclude"` + Prefix string `json:"prefix" validate:"required"` + Pubkey string `json:"pubkey" validate:"required"` + Identity string `json:"identity" post:"exclude"` + AdminMetadata AdminMetadata `json:"admin_metadata"` + CustomFields map[string]interface{} `json:"custom_fields"` } type NamespaceWOPubkey struct { @@ -119,6 +120,10 @@ func (rs RegistrationStatus) String() string { return string(rs) } +func (rs RegistrationStatus) LowerString() string { + return strings.ToLower(string(rs)) +} + func (a AdminMetadata) Equal(b AdminMetadata) bool { return a.UserID == b.UserID && a.Description == b.Description && @@ -145,13 +150,55 @@ func createNamespaceTable() { prefix TEXT NOT NULL UNIQUE, pubkey TEXT NOT NULL, identity TEXT, - admin_metadata TEXT CHECK (length("admin_metadata") <= 4000) + admin_metadata TEXT CHECK (length("admin_metadata") <= 4000), + custom_fields TEXT CHECK (length("custom_fields") <= 4000) );` _, err := db.Exec(query) if err != nil { log.Fatalf("Failed to create namespace table: %v", err) } + + // Run a manual migration to add "custom_fields" field + // Check if the column exists + log.Info("Run manual migration for 'custom_fields' in namespace table") + columnExists := false + rows, err := db.Query(`PRAGMA table_info(namespace);`) + if err != nil { + log.Fatal(err) + } + defer rows.Close() + + for rows.Next() { + var ( + cid int + name string + ctype string + notnull int + dfltValue interface{} + pk int + ) + err = rows.Scan(&cid, &name, &ctype, ¬null, &dfltValue, &pk) + if err != nil { + log.Fatal(err) + } + if name == "custom_fields" { + columnExists = true + break + } + } + + // If the column does not exist, add it + if !columnExists { + _, err = db.Exec(`ALTER TABLE namespace ADD COLUMN custom_fields TEXT DEFAULT ''`) + if err != nil { + log.Fatal(err) + } + log.Info("Column 'custom_fields' added.") + } else { + log.Info("Column 'custom_fields' already exists.") + } + } func createTopologyTable() { @@ -305,7 +352,7 @@ func namespaceBelongsToUserId(id int, userId string) (bool, error) { if err := rows.Scan(&adminMetadataStr); err != nil { return false, err } - // For backward compatibility, if adminMetadata is an empty string, don't unmarshall json + // For backward compatibility, if adminMetadata is an empty string, don't unmarshal json if adminMetadataStr != "" { if err := json.Unmarshal([]byte(adminMetadataStr), &ns.AdminMetadata); err != nil { return false, err @@ -354,7 +401,7 @@ func getNamespaceJwksByPrefix(prefix string) (jwk.Set, *AdminMetadata, error) { adminMetadata := AdminMetadata{} - // For backward compatibility, if adminMetadata is an empty string, don't unmarshall json + // For backward compatibility, if adminMetadata is an empty string, don't unmarshal json if adminMetadataStr != "" { if err := json.Unmarshal([]byte(adminMetadataStr), &adminMetadata); err != nil { return nil, nil, errors.Wrap(err, "error parsing admin metadata") @@ -380,7 +427,7 @@ func getNamespaceStatusById(id int) (RegistrationStatus, error) { if err != nil { return "", err } - // For backward compatibility, if adminMetadata is an empty string, don't unmarshall json + // For backward compatibility, if adminMetadata is an empty string, don't unmarshal json if adminMetadataStr != "" { if err := json.Unmarshal([]byte(adminMetadataStr), &adminMetadata); err != nil { return "", err @@ -402,17 +449,33 @@ func getNamespaceById(id int) (*Namespace, error) { } ns := &Namespace{} adminMetadataStr := "" - query := `SELECT id, prefix, pubkey, identity, admin_metadata FROM namespace WHERE id = ?` - err := db.QueryRow(query, id).Scan(&ns.ID, &ns.Prefix, &ns.Pubkey, &ns.Identity, &adminMetadataStr) + customRegFieldsStr := "" + query := `SELECT id, prefix, pubkey, identity, admin_metadata, custom_fields FROM namespace WHERE id = ?` + err := db.QueryRow(query, id).Scan(&ns.ID, &ns.Prefix, &ns.Pubkey, &ns.Identity, &adminMetadataStr, &customRegFieldsStr) if err != nil { return nil, err } - // For backward compatibility, if adminMetadata is an empty string, don't unmarshall json + // For backward compatibility, if adminMetadata is an empty string, don't unmarshal json if adminMetadataStr != "" { if err := json.Unmarshal([]byte(adminMetadataStr), &ns.AdminMetadata); err != nil { return nil, err } } + if customRegFieldsStr != "" { + if err := json.Unmarshal([]byte(customRegFieldsStr), &ns.CustomFields); err != nil { + return nil, err + } + } + // By default, JSON unmarshal will convert any generic number to float + // and we only allow integer in custom fields, so we convert them back + for key, val := range ns.CustomFields { + switch v := val.(type) { + case float64: + ns.CustomFields[key] = int(v) + case float32: + ns.CustomFields[key] = int(v) + } + } return ns, nil } @@ -422,17 +485,33 @@ func getNamespaceByPrefix(prefix string) (*Namespace, error) { } ns := &Namespace{} adminMetadataStr := "" - query := `SELECT id, prefix, pubkey, identity, admin_metadata FROM namespace WHERE prefix = ?` - err := db.QueryRow(query, prefix).Scan(&ns.ID, &ns.Prefix, &ns.Pubkey, &ns.Identity, &adminMetadataStr) + customRegFieldsStr := "" + query := `SELECT id, prefix, pubkey, identity, admin_metadata, custom_fields FROM namespace WHERE prefix = ?` + err := db.QueryRow(query, prefix).Scan(&ns.ID, &ns.Prefix, &ns.Pubkey, &ns.Identity, &adminMetadataStr, &customRegFieldsStr) if err != nil { return nil, err } - // For backward compatibility, if adminMetadata is an empty string, don't unmarshall json + // For backward compatibility, if adminMetadata is an empty string, don't unmarshal json if adminMetadataStr != "" { if err := json.Unmarshal([]byte(adminMetadataStr), &ns.AdminMetadata); err != nil { return nil, err } } + if customRegFieldsStr != "" { + if err := json.Unmarshal([]byte(customRegFieldsStr), &ns.CustomFields); err != nil { + return nil, err + } + } + // By default, JSON unmarshal will convert any generic number to float + // and we only allow integer in custom fields, so we convert them back + for key, val := range ns.CustomFields { + switch v := val.(type) { + case float64: + ns.CustomFields[key] = int(v) + case float32: + ns.CustomFields[key] = int(v) + } + } return ns, nil } @@ -452,7 +531,9 @@ func getNamespacesByFilter(filterNs Namespace, serverType ServerType) ([]*Namesp } else if serverType != "" { return nil, errors.New(fmt.Sprint("Can't get namespace: unsupported server type: ", serverType)) } - + if filterNs.CustomFields != nil { + return nil, errors.New("Unsupported operation: Can't filter against Custrom Registration field.") + } if filterNs.ID != 0 { return nil, errors.New("Unsupported operation: Can't filter against ID field.") } @@ -484,7 +565,7 @@ func getNamespacesByFilter(filterNs Namespace, serverType ServerType) ([]*Namesp if err := rows.Scan(&ns.ID, &ns.Prefix, &ns.Pubkey, &ns.Identity, &adminMetadataStr); err != nil { return nil, err } - // For backward compatibility, if adminMetadata is an empty string, don't unmarshall json + // For backward compatibility, if adminMetadata is an empty string, don't unmarshal json if adminMetadataStr == "" { // If we apply any filter against the AdminMetadata field but the // entry didn't populate this field, skip it @@ -540,7 +621,7 @@ functions) used by the client. */ func addNamespace(ns *Namespace) error { - query := `INSERT INTO namespace (prefix, pubkey, identity, admin_metadata) VALUES (?, ?, ?, ?)` + query := `INSERT INTO namespace (prefix, pubkey, identity, admin_metadata, custom_fields) VALUES (?, ?, ?, ?, ?)` tx, err := db.Begin() if err != nil { return err @@ -558,10 +639,14 @@ func addNamespace(ns *Namespace) error { strAdminMetadata, err := json.Marshal(ns.AdminMetadata) if err != nil { - return errors.Wrap(err, "Fail to marshall AdminMetadata") + return errors.Wrap(err, "Fail to marshal AdminMetadata") + } + strCustomRegFields, err := json.Marshal(ns.CustomFields) + if err != nil { + return errors.Wrap(err, "Fail to marshal custom registration fields") } - _, err = tx.Exec(query, ns.Prefix, ns.Pubkey, ns.Identity, strAdminMetadata) + _, err = tx.Exec(query, ns.Prefix, ns.Pubkey, ns.Identity, strAdminMetadata, strCustomRegFields) if err != nil { if errRoll := tx.Rollback(); errRoll != nil { log.Errorln("Failed to rollback transaction:", errRoll) @@ -595,17 +680,21 @@ func updateNamespace(ns *Namespace) error { ns.AdminMetadata.UpdatedAt = time.Now() strAdminMetadata, err := json.Marshal(ns.AdminMetadata) if err != nil { - return errors.Wrap(err, "Fail to marshall AdminMetadata") + return errors.Wrap(err, "Fail to marshal AdminMetadata") + } + strCustomRegFields, err := json.Marshal(ns.CustomFields) + if err != nil { + return errors.Wrap(err, "Fail to marshal custom registration fields") } // We intentionally exclude updating "identity" as this should only be updated // when user registered through Pelican client with identity - query := `UPDATE namespace SET prefix = ?, pubkey = ?, admin_metadata = ? WHERE id = ?` + query := `UPDATE namespace SET prefix = ?, pubkey = ?, admin_metadata = ?, custom_fields = ? WHERE id = ?` tx, err := db.Begin() if err != nil { return err } - _, err = tx.Exec(query, ns.Prefix, ns.Pubkey, strAdminMetadata, ns.ID) + _, err = tx.Exec(query, ns.Prefix, ns.Pubkey, strAdminMetadata, strCustomRegFields, ns.ID) if err != nil { if errRoll := tx.Rollback(); errRoll != nil { log.Errorln("Failed to rollback transaction:", errRoll) @@ -633,7 +722,7 @@ func updateNamespaceStatusById(id int, status RegistrationStatus, approverId str adminMetadataByte, err := json.Marshal(ns.AdminMetadata) if err != nil { - return errors.Wrap(err, "Error marshalling admin metadata") + return errors.Wrap(err, "Error marshaling admin metadata") } query := `UPDATE namespace SET admin_metadata = ? WHERE id = ?` @@ -668,7 +757,7 @@ func deleteNamespace(prefix string) error { } func getAllNamespaces() ([]*Namespace, error) { - query := `SELECT id, prefix, pubkey, identity, admin_metadata FROM namespace ORDER BY id ASC` + query := `SELECT id, prefix, pubkey, identity, admin_metadata, custom_fields FROM namespace ORDER BY id ASC` rows, err := db.Query(query) if err != nil { return nil, err @@ -679,15 +768,31 @@ func getAllNamespaces() ([]*Namespace, error) { for rows.Next() { ns := &Namespace{} adminMetadataStr := "" - if err := rows.Scan(&ns.ID, &ns.Prefix, &ns.Pubkey, &ns.Identity, &adminMetadataStr); err != nil { + customRegFieldsStr := "" + if err := rows.Scan(&ns.ID, &ns.Prefix, &ns.Pubkey, &ns.Identity, &adminMetadataStr, &customRegFieldsStr); err != nil { return nil, err } - // For backward compatibility, if adminMetadata is an empty string, don't unmarshall json + // For backward compatibility, if adminMetadata is an empty string, don't unmarshal json if adminMetadataStr != "" { if err := json.Unmarshal([]byte(adminMetadataStr), &ns.AdminMetadata); err != nil { return nil, err } } + if customRegFieldsStr != "" { + if err := json.Unmarshal([]byte(customRegFieldsStr), &ns.CustomFields); err != nil { + return nil, err + } + } + // By default, JSON unmarshal will convert any generic number to float + // and we only allow integer in custom fields, so we convert them back + for key, val := range ns.CustomFields { + switch v := val.(type) { + case float64: + ns.CustomFields[key] = int(v) + case float32: + ns.CustomFields[key] = int(v) + } + } namespaces = append(namespaces, ns) } diff --git a/registry/registry_db_test.go b/registry/registry_db_test.go index 61c25a7b0..b4227b98d 100644 --- a/registry/registry_db_test.go +++ b/registry/registry_db_test.go @@ -60,7 +60,7 @@ func teardownMockNamespaceDB(t *testing.T) { } func insertMockDBData(nss []Namespace) error { - query := `INSERT INTO namespace (prefix, pubkey, identity, admin_metadata) VALUES (?, ?, ?, ?)` + query := `INSERT INTO namespace (prefix, pubkey, identity, admin_metadata, custom_fields) VALUES (?, ?, ?, ?, ?)` tx, err := db.Begin() if err != nil { return err @@ -73,8 +73,15 @@ func insertMockDBData(nss []Namespace) error { } return err } + customFieldsStr, err := json.Marshal(ns.CustomFields) + if err != nil { + if errRoll := tx.Rollback(); errRoll != nil { + return errors.Wrap(errRoll, "Failed to rollback transaction") + } + return err + } - _, err = tx.Exec(query, ns.Prefix, ns.Pubkey, ns.Identity, adminMetaStr) + _, err = tx.Exec(query, ns.Prefix, ns.Pubkey, ns.Identity, adminMetaStr, customFieldsStr) if err != nil { if errRoll := tx.Rollback(); errRoll != nil { return errors.Wrap(errRoll, "Failed to rollback transaction") @@ -172,6 +179,12 @@ var ( mixed = append(mixed, mockNssWithCachesNotApproved...) return }() + + mockCustomFields = map[string]interface{}{ + "key1": "value1", + "key2": 2, + "key3": true, + } ) func TestNamespaceExistsByPrefix(t *testing.T) { @@ -194,7 +207,7 @@ func TestNamespaceExistsByPrefix(t *testing.T) { }) } -func TestGetNamespacesById(t *testing.T) { +func TestGetNamespaceById(t *testing.T) { setupMockRegistryDB(t) defer teardownMockNamespaceDB(t) @@ -214,6 +227,7 @@ func TestGetNamespacesById(t *testing.T) { t.Run("return-namespace-with-correct-id", func(t *testing.T) { defer resetNamespaceDB(t) mockNs := mockNamespace("/test", "", "", AdminMetadata{UserID: "foo"}) + mockNs.CustomFields = mockCustomFields err := insertMockDBData([]Namespace{mockNs}) require.NoError(t, err) nss, err := getAllNamespaces() @@ -318,6 +332,7 @@ func TestAddNamespace(t *testing.T) { t.Run("insert-data-integrity", func(t *testing.T) { defer resetNamespaceDB(t) mockNs := mockNamespace("/test", "pubkey", "identity", AdminMetadata{UserID: "someone", Description: "Some description", SiteName: "OSG", SecurityContactUserID: "security-001"}) + mockNs.CustomFields = mockCustomFields err := addNamespace(&mockNs) require.NoError(t, err) got, err := getAllNamespaces() @@ -329,6 +344,7 @@ func TestAddNamespace(t *testing.T) { assert.Equal(t, mockNs.AdminMetadata.Description, got[0].AdminMetadata.Description) assert.Equal(t, mockNs.AdminMetadata.SiteName, got[0].AdminMetadata.SiteName) assert.Equal(t, mockNs.AdminMetadata.SecurityContactUserID, got[0].AdminMetadata.SecurityContactUserID) + assert.Equal(t, mockCustomFields, got[0].CustomFields) }) } @@ -460,6 +476,12 @@ func TestGetNamespacesByFilter(t *testing.T) { _, err := getNamespacesByFilter(filterNsID, "") require.Error(t, err, "Should return error for filtering against unsupported field ID") + filterNsCF := Namespace{ + CustomFields: mockCustomFields, + } + _, err = getNamespacesByFilter(filterNsCF, "") + require.Error(t, err, "Should return error for filtering against unsupported custom fields") + filterNsIdentity := Namespace{ Identity: "someIdentity", } diff --git a/registry/registry_ui.go b/registry/registry_ui.go index fa7f4bd6d..7bab4694e 100644 --- a/registry/registry_ui.go +++ b/registry/registry_ui.go @@ -35,6 +35,7 @@ import ( "github.com/jellydator/ttlcache/v3" "github.com/pelicanplatform/pelican/config" "github.com/pelicanplatform/pelican/param" + "github.com/pelicanplatform/pelican/utils" "github.com/pelicanplatform/pelican/web_ui" "github.com/pkg/errors" log "github.com/sirupsen/logrus" @@ -52,28 +53,45 @@ type ( } registrationFieldType string - registrationField struct { - Name string `json:"name"` - Type registrationFieldType `json:"type"` - Required bool `json:"required"` - Options []interface{} `json:"options"` + + registrationFieldOption struct { + Name string `mapstructure:"name" json:"name"` + ID string `mapstructure:"id" json:"id"` + } + registrationField struct { + Name string `json:"name"` + DisplayedName string `json:"displayed_name"` + Type registrationFieldType `json:"type"` + Required bool `json:"required"` + Options []registrationFieldOption `json:"options"` + Description string `json:"description"` } Institution struct { Name string `mapstructure:"name" json:"name" yaml:"name"` ID string `mapstructure:"id" json:"id" yaml:"id"` } + + customRegFieldsConfig struct { + Name string `mapstructure:"name"` + Type string `mapstructure:"type"` + Required bool `mapstructure:"required"` + Options []registrationFieldOption `mapstructure:"options"` + Description string `mapstructure:"description"` + } ) const ( String registrationFieldType = "string" Int registrationFieldType = "int" + Boolean registrationFieldType = "bool" Enum registrationFieldType = "enum" DateTime registrationFieldType = "datetime" ) var ( registrationFields []registrationField + customRegFieldsConfigs []customRegFieldsConfig institutionsCache *ttlcache.Cache[string, []Institution] institutionsCacheMutex = sync.RWMutex{} ) @@ -110,14 +128,15 @@ func populateRegistrationFields(prefix string, data interface{}) []registrationF if splitJson != "-" { tempName = splitJson } else { - // `json:"-"` means this field should be removed from any marshalling + // `json:"-"` means this field should be removed from any marshaling continue } } regField := registrationField{ - Name: name + tempName, - Required: strings.Contains(field.Tag.Get("validate"), "required"), + Name: name + tempName, + DisplayedName: utils.SnakeCaseToHumanReadable(tempName), + Required: strings.Contains(field.Tag.Get("validate"), "required"), } switch field.Type.Kind() { @@ -146,10 +165,10 @@ func populateRegistrationFields(prefix string, data interface{}) []registrationF if field.Type == reflect.TypeOf(RegistrationStatus("")) { regField.Type = Enum - options := make([]interface{}, 3) - options[0] = Pending - options[1] = Approved - options[2] = Denied + options := make([]registrationFieldOption, 3) + options[0] = registrationFieldOption{Name: Pending.String(), ID: Pending.LowerString()} + options[1] = registrationFieldOption{Name: Approved.String(), ID: Approved.LowerString()} + options[2] = registrationFieldOption{Name: Denied.String(), ID: Denied.LowerString()} regField.Options = options fields = append(fields, regField) } else { @@ -160,7 +179,23 @@ func populateRegistrationFields(prefix string, data interface{}) []registrationF return fields } -// Helper function to exclude pubkey field from marshalling into json +func populateCustomRegFields(configFields []customRegFieldsConfig) []registrationField { + regFields := make([]registrationField, 0) + for _, field := range configFields { + customRegField := registrationField{ + Name: "custom_fields." + field.Name, + DisplayedName: utils.SnakeCaseToHumanReadable(field.Name), + Type: registrationFieldType(field.Type), + Options: field.Options, + Required: field.Required, + Description: field.Description, + } + regFields = append(regFields, customRegField) + } + return regFields +} + +// Helper function to exclude pubkey field from marshaling into json func excludePubKey(nss []*Namespace) (nssNew []NamespaceWOPubkey) { nssNew = make([]NamespaceWOPubkey, 0) for _, ns := range nss { @@ -448,6 +483,15 @@ func createUpdateNamespace(ctx *gin.Context, isUpdate bool) { return } + if validCF, err := validateCustomFields(ns.CustomFields, true); !validCF { + if err != nil { + ctx.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("Error validating custom fields: %v", err)}) + return + } + ctx.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("Invalid custom field: %s", err.Error())}) + return + } + if !isUpdate { // Create ns.AdminMetadata.UserID = user // Overwrite status to Pending to filter malicious request @@ -656,6 +700,39 @@ func listInstitutions(ctx *gin.Context) { } } +// Initialize custom registration fields provided via Registry.CustomRegistrationFields +func InitCustomRegistrationFields() error { + configFields := []customRegFieldsConfig{} + if err := param.Registry_CustomRegistrationFields.Unmarshal(&configFields); err != nil { + return errors.Wrap(err, "Error reading from config value for Registry.CustomRegistrationFields") + } + customRegFieldsConfigs = configFields + + fieldNames := make(map[string]bool, 0) + + for _, conf := range configFields { + // Duplicated name check + if fieldNames[conf.Name] { + return errors.New(fmt.Sprintf("Bad custom registration fields, duplicated field name: %q", conf.Name)) + } else { + fieldNames[conf.Name] = true + } + if conf.Type != "string" && conf.Type != "bool" && conf.Type != "int" && conf.Type != "enum" && conf.Type != "datetime" { + return errors.New(fmt.Sprintf("Bad custom registration field, unsupported field type: %q with %q", conf.Name, conf.Type)) + } + if conf.Type == "enum" { + if conf.Options == nil { + return errors.New(fmt.Sprintf("Bad custom registration field, 'enum' type field does not have options: %q", conf.Name)) + } + } + } + + additionalRegFields := populateCustomRegFields(configFields) + registrationFields = append(registrationFields, additionalRegFields...) + + return nil +} + // Define Gin APIs for registry Web UI. All endpoints are user-facing func RegisterRegistryWebAPI(router *gin.RouterGroup) error { registryWebAPI := router.Group("/api/v1.0/registry_ui") diff --git a/registry/registry_ui_test.go b/registry/registry_ui_test.go index 755c2c6bf..fad562700 100644 --- a/registry/registry_ui_test.go +++ b/registry/registry_ui_test.go @@ -78,7 +78,7 @@ func GenerateMockJWKS() (string, error) { jsonData, err := json.MarshalIndent(jwks, "", " ") if err != nil { - return "", errors.Wrap(err, "Unable to marshall the json into string") + return "", errors.Wrap(err, "Unable to marshal the json into string") } // Append a new line to the JSON data jsonData = append(jsonData, '\n') diff --git a/registry/registry_validation.go b/registry/registry_validation.go index 77f86849c..17f78bc10 100644 --- a/registry/registry_validation.go +++ b/registry/registry_validation.go @@ -20,6 +20,7 @@ package registry import ( "encoding/json" + "fmt" "strings" "github.com/lestrrat-go/jwx/v2/jwk" @@ -195,3 +196,80 @@ func validateInstitution(instID string) (bool, error) { } return false, nil } + +// Validates if customFields are valid based on config. Set exactMatch to false to be +// backward compatible with legacy custom fields that were once defined but removed +func validateCustomFields(customFields map[string]interface{}, exactMatch bool) (bool, error) { + if len(customRegFieldsConfigs) == 0 { + if len(customFields) > 0 { + return false, errors.New("Bad configuration, Registry.CustomRegistrationFields is not set while validate against custom fields") + } else { + return true, nil + } + } else { + if customFields == nil { + return false, errors.New("Can't validate against nil customFields") + } + } + for _, conf := range customRegFieldsConfigs { + val, ok := customFields[conf.Name] + if !ok && conf.Required { + return false, errors.New(fmt.Sprintf("%q is required", conf.Name)) + } + if ok { + switch conf.Type { + case "string": + if _, ok := val.(string); !ok { + return false, errors.New(fmt.Sprintf("%q is expected to be a string, but got %v", conf.Name, val)) + } + case "int": + if _, ok := val.(int); !ok { + return false, errors.New(fmt.Sprintf("%q is expected to be an int, but got %v", conf.Name, val)) + } + case "bool": + if _, ok := val.(bool); !ok { + return false, errors.New(fmt.Sprintf("%q is expected to be a boolean, but got %v", conf.Name, val)) + } + case "datetime": + switch val.(type) { + case int: + break + case int32: + break + case int64: + break + default: + return false, fmt.Errorf("%q is expected to be a Unix timestamp, but got %v", conf.Name, val) + } + case "enum": + inOpt := false + for _, item := range conf.Options { + if item.ID == val { + inOpt = true + } + } + if !inOpt { + return false, fmt.Errorf("%q is an enumeration type, but the value is not in the options. Got %v", conf.Name, val) + } + default: + return false, errors.New(fmt.Sprintf("The type of %q is not supported", conf.Name)) + } + } + } + // Optioanlly check if the customFields are defined in config + if exactMatch { + for key := range customFields { + found := false + for _, conf := range customRegFieldsConfigs { + if conf.Name == key { + found = true + break + } + } + if !found { + return false, errors.New(fmt.Sprintf("%q is not a valid custom field", key)) + } + } + } + return true, nil +} diff --git a/registry/registry_validation_test.go b/registry/registry_validation_test.go index 7a658ef15..daf687f26 100644 --- a/registry/registry_validation_test.go +++ b/registry/registry_validation_test.go @@ -9,6 +9,138 @@ import ( "github.com/stretchr/testify/require" ) +func TestValidateCustomFields(t *testing.T) { + + oldConfig := customRegFieldsConfigs + + setMockConfig := func(config []customRegFieldsConfig) { + customRegFieldsConfigs = config + } + + t.Cleanup(func() { + customRegFieldsConfigs = oldConfig + }) + + t.Run("empty-configuration-and-custom-fields", func(t *testing.T) { + setMockConfig([]customRegFieldsConfig{}) + customFields := make(map[string]interface{}) + + valid, err := validateCustomFields(customFields, false) + require.NoError(t, err, "Should not have an error with empty config and custom fields") + assert.True(t, valid, "Validation should pass with empty config and custom fields") + }) + t.Run("configuration-not-set-non-empty-custom-fields", func(t *testing.T) { + setMockConfig(nil) + customFields := map[string]interface{}{"field1": "value1"} + + _, err := validateCustomFields(customFields, false) + require.Error(t, err, "Expected an error when config is not set, but custom fields are non-empty") + }) + t.Run("required-field-missing-in-custom-fields", func(t *testing.T) { + setMockConfig([]customRegFieldsConfig{ + {Name: "requiredField", Type: "string", Required: true}, + }) + customFields := map[string]interface{}{"otherField": "value"} + + valid, err := validateCustomFields(customFields, false) + require.Error(t, err, "Expected an error when a required field is missing") + assert.False(t, valid, "Validation should fail when a required field is missing") + }) + t.Run("type-mismatch-in-custom-fields", func(t *testing.T) { + setMockConfig([]customRegFieldsConfig{ + {Name: "stringField", Type: "string"}, + }) + customFields := map[string]interface{}{"stringField": 123} // Incorrect type + + valid, err := validateCustomFields(customFields, false) + require.Error(t, err, "Expected an error due to type mismatch") + assert.False(t, valid, "Validation should fail due to type mismatch") + }) + t.Run("invalid-datetime-field-value", func(t *testing.T) { + setMockConfig([]customRegFieldsConfig{ + {Name: "datetimeField", Type: "datetime"}, + }) + customFields := map[string]interface{}{"datetimeField": "not-a-timestamp"} + + valid, err := validateCustomFields(customFields, false) + require.Error(t, err, "Expected an error due to invalid datetime value") + assert.False(t, valid, "Validation should fail due to invalid datetime value") + }) + t.Run("enum-field-with-invalid-option", func(t *testing.T) { + setMockConfig([]customRegFieldsConfig{ + {Name: "enumField", Type: "enum", Options: []registrationFieldOption{{ID: "option1"}, {ID: "option2"}}}, + }) + customFields := map[string]interface{}{"enumField": "invalidOption"} + + valid, err := validateCustomFields(customFields, false) + require.Error(t, err, "Expected an error due to invalid enum value") + assert.False(t, valid, "Validation should fail due to invalid enum value") + }) + t.Run("additional-fields-in-custom-fields-with-exactmatch-false", func(t *testing.T) { + setMockConfig([]customRegFieldsConfig{ + {Name: "field1", Type: "string"}, + }) + customFields := map[string]interface{}{"field1": "value1", "extraField": "extraValue"} + + valid, err := validateCustomFields(customFields, false) + require.NoError(t, err, "No error expected with extra fields when exactMatch is false") + assert.True(t, valid, "Validation should pass with extra fields when exactMatch is false") + + }) + t.Run("additional-fields-in-custom-fields-with-exactmatch-true", func(t *testing.T) { + setMockConfig([]customRegFieldsConfig{ + {Name: "field1", Type: "string"}, + }) + customFields := map[string]interface{}{"field1": "value1", "extraField": "extraValue"} + + valid, err := validateCustomFields(customFields, true) + require.Error(t, err, "Expected an error due to extra fields when exactMatch is true") + assert.False(t, valid, "Validation should fail with extra fields when exactMatch is true") + }) + t.Run("all-valid-fields-with-exactmatch-true", func(t *testing.T) { + setMockConfig([]customRegFieldsConfig{ + {Name: "field1", Type: "string"}, + {Name: "field2", Type: "int"}, + }) + customFields := map[string]interface{}{"field1": "value1", "field2": 10} + + valid, err := validateCustomFields(customFields, true) + require.NoError(t, err, "No error expected with all valid fields and exactMatch true") + assert.True(t, valid, "Validation should pass with all valid fields and exactMatch true") + }) + t.Run("field-present-in-custom-fields-but-not-required-in-config", func(t *testing.T) { + setMockConfig([]customRegFieldsConfig{ + {Name: "requiredField", Type: "string", Required: true}, + {Name: "optionalField", Type: "int", Required: false}, + }) + customFields := map[string]interface{}{"requiredField": "value", "optionalField": 5} + + valid, err := validateCustomFields(customFields, false) + require.NoError(t, err, "No error expected with optional field present") + assert.True(t, valid, "Validation should pass with optional field present") + }) + t.Run("invalid-field-type-in-configuration", func(t *testing.T) { + setMockConfig([]customRegFieldsConfig{ + {Name: "unsupportedField", Type: "unsupportedType"}, + }) + customFields := map[string]interface{}{"unsupportedField": "value"} + + valid, err := validateCustomFields(customFields, false) + require.Error(t, err, "Expected an error due to unsupported field type in config") + assert.False(t, valid, "Validation should fail due to unsupported field type in config") + }) + t.Run("null-or-invalid-custom-fields-map", func(t *testing.T) { + setMockConfig([]customRegFieldsConfig{ + {Name: "field1", Type: "string"}, + }) + var customFields map[string]interface{} // Invalid (nil) map + + valid, err := validateCustomFields(customFields, false) + require.Error(t, err, "Expected an error with invalid (nil) custom fields map") + assert.False(t, valid, "Validation should fail with invalid (nil) custom fields map") + }) +} + func TestValidateKeyChaining(t *testing.T) { viper.Reset() setupMockRegistryDB(t) diff --git a/server_ui/register_namespace.go b/server_ui/register_namespace.go index 99239c7dd..43dc3ef28 100644 --- a/server_ui/register_namespace.go +++ b/server_ui/register_namespace.go @@ -98,7 +98,7 @@ func keyIsRegistered(privkey jwk.Key, registryUrlStr string, prefix string) (key keyCheckReq := checkNamespaceExistsReq{Prefix: prefix, PubKey: string(pubkeyStr)} jsonData, err := json.Marshal(keyCheckReq) if err != nil { - return noKeyPresent, errors.Wrap(err, "Error marshalling request to json string") + return noKeyPresent, errors.Wrap(err, "Error marshaling request to json string") } req, err := http.NewRequest("POST", pelicanReqURL.String(), bytes.NewBuffer(jsonData)) diff --git a/utils/utils.go b/utils/utils.go new file mode 100644 index 000000000..17c5b96c7 --- /dev/null +++ b/utils/utils.go @@ -0,0 +1,35 @@ +package utils + +import ( + "strings" + "unicode" + + "golang.org/x/text/cases" + "golang.org/x/text/language" +) + +// snakeCaseToCamelCase converts a snake case string to camel case. +func SnakeCaseToCamelCase(input string) string { + isToUpper := false + isFirst := true + return strings.Map(func(r rune) rune { + if r == '_' { + isToUpper = true + return -1 + } + if isToUpper || isFirst { + isToUpper = false + return unicode.ToUpper(r) + } + return r + }, input) +} + +// snakeCaseToSnakeCase converts a snake_case string to Snake Case (CamelCase with spaces). +func SnakeCaseToHumanReadable(input string) string { + words := strings.Split(input, "_") + for i, word := range words { + words[i] = cases.Title(language.English).String(word) + } + return strings.Join(words, " ") +} diff --git a/web_ui/frontend/app/api/docs/pelican-swagger.yaml b/web_ui/frontend/app/api/docs/pelican-swagger.yaml index df7e2c6bd..aba5bcd9c 100644 --- a/web_ui/frontend/app/api/docs/pelican-swagger.yaml +++ b/web_ui/frontend/app/api/docs/pelican-swagger.yaml @@ -131,6 +131,8 @@ definitions: description: "Timestamp of the last update" AdminMetadataForRegistration: type: object + required: + - "institution" properties: description: type: string @@ -169,6 +171,9 @@ definitions: description: The user identity we get from CILogon if the namespace is registered via CLI with `--with-identity` flag admin_metadata: $ref: "#/definitions/AdminMetadata" + custom_fields: + type: object + description: The custom fields user registered, configurable by setting Registry.CustomRegistrationFields. Institution: type: object properties: @@ -198,11 +203,17 @@ definitions: type: string description: The public JWK from the origin that wants to register the namespace. - It should be a marshalled (stringfied) JSON that contains either one JWK or a JWKS + It should be a marshaled (stringfied) JSON that contains either one JWK or a JWKS admin_metadata: $ref: "#/definitions/AdminMetadata" + custom_fields: + type: object + description: The custom fields user registered, configurable by setting Registry.CustomRegistrationFields. NamespaceForRegistration: type: object + required: + - "prefix" + - "pubkey" properties: prefix: type: string @@ -212,9 +223,12 @@ definitions: type: string description: The public JWK from the origin that wants to register the namespace. - It should be a marshalled (stringfied) JSON that contains either one JWK or a JWKS + It should be a marshaled (stringfied) JSON that contains either one JWK or a JWKS admin_metadata: $ref: "#/definitions/AdminMetadataForRegistration" + custom_fields: + type: object + description: The custom fields to register, configurable by setting Registry.CustomRegistrationFields. RegistrationFieldType: type: string enum: @@ -222,13 +236,35 @@ definitions: - int - enum - datetime + example: enum RegistrationField: type: object + required: + - "name" + - "displayed_name" + - "type" + - "required" properties: name: type: string - description: The name of the field available to register - example: "prefix" + description: | + The name of the field as the key of the object to submit the request. Note that if the + name is dot '.' separated, it means the hierarchy of the object. + + For example, `custom_fields.department` means that your request needs to be + + ```json + { + "custom_fields": { + "department": "value" + } + } + ``` + example: "custom_fields.department" + displayed_name: + type: string + description: The human-readable name of the field + example: "Department" type: description: The data type of the field $ref: "#/definitions/RegistrationFieldType" @@ -239,7 +275,16 @@ definitions: description: The available options if the field is "enum" type type: array items: - type: string + type: object + properties: + name: + type: string + description: The name of the option that will appear in UI + example: "Option A" + id: + type: string + description: The unique identifier of the option that will be stored in db + example: "option1" minItems: 0 tags: