Random test failures because of work item creation error (#1676)

Using the `Last-Modified` response header when possible
Also, making sure that the list of spaces is always ordered by
`updated_at DESC` to get the most recents spaces first.

Move the `work item number sequence` in its own subpackage.

Refactored the test to use the TestFixture to initialise the
work items from the data set.

Using an `UPSERT` statement to avoid creating then updating
 the work item number sequence, which also avoids putting a `LOCK`
 on the row (using the `... FOR UPDATE` option in the `SELECT`
 statement)

Remove the `optionalKeywords` which contained the work item number
but failed when the 3rd work item was retrieved because its number
('3') would match the description. This worked in the past when the
number did not exist and the search was based on the ID (UUID).

Added some weight on the work item number (`*A`) when the URL matches 
a known one and the number can be extracted.

Also, add a test to search by number using the `number:%d` query.

Adding a test to create work items concurrently

Refactor identities creation in test: using UUIDs of created identities instead of usernames
also, move a comment to `init()` where it belongs
also, comment on use of `:*A` in fulltext search

Also: move some log statements to `DEBUG` instead of `INFO`
to reduce the noise during test executions

Fixes #1676

Signed-off-by: Xavier Coulon <xcoulon@redhat.com>

account/identity.go

@@ -161,9 +161,9 @@ func (m *GormIdentityRepository) Create(ctx context.Context, model *Identity) er
 		}, "unable to create the identity")
 		return errs.WithStack(err)
 	}
-	log.Info(ctx, map[string]interface{}{
+	log.Debug(ctx, map[string]interface{}{
 		"identity_id": model.ID,
-	}, "Identity created!")
+	}, "Identity created")
 	return nil
 }
 

controller/workitem_blackbox_test.go

@@ -797,12 +797,12 @@ func createOneRandomIteration(ctx context.Context, db *gorm.DB) *iteration.Itera
 	return &itr
 }
 
-func createOneRandomArea(ctx context.Context, db *gorm.DB, testName string) *area.Area {
+func createOneRandomArea(ctx context.Context, db *gorm.DB) *area.Area {
 	areaRepo := area.NewAreaRepository(db)
 	spaceRepo := space.NewRepository(db)
 
 	newSpace := space.Space{
-		Name: fmt.Sprintf("Space area %v %v", testName, uuid.NewV4()),
+		Name: fmt.Sprintf("Space area %v", uuid.NewV4()),
 	}
 	space, err := spaceRepo.Create(ctx, &newSpace)
 	if err != nil {
@@ -1506,7 +1506,7 @@ func (s *WorkItem2Suite) TestWI2ListByStateFilterOKModifiedUsingIfNoneMatchIfMod
 }
 
 func (s *WorkItem2Suite) setupAreaWorkItem(createWorkItem bool) (uuid.UUID, string, *app.WorkItemSingle) {
-	tempArea := createOneRandomArea(s.svc.Context, s.DB, "TestWI2ListByAreaFilter")
+	tempArea := createOneRandomArea(s.svc.Context, s.DB)
 	require.NotNil(s.T(), tempArea)
 	areaID := tempArea.ID.String()
 	c := minimumRequiredCreatePayload()

glide.yaml

@@ -106,6 +106,9 @@ import:
   - golint
 - package: github.com/fzipp/gocyclo
 - package: github.com/Sirupsen/logrus
+  version: c078b1e43f58d563c74cebe63c85789e76ddb627
+  repo: https://github.com/sirupsen/logrus
+  vcs: git
 - package: github.com/sourcegraph/syntaxhighlight
 - package: github.com/jstemmer/go-junit-report
 - package: github.com/sourcegraph/annotate

gormsupport/cleaner/db_clean.go

@@ -65,7 +65,7 @@ func DeleteCreatedEntities(db *gorm.DB) func() {
 		}
 		for i := len(entires) - 1; i >= 0; i-- {
 			entry := entires[i]
-			log.Info(nil, map[string]interface{}{
+			log.Debug(nil, map[string]interface{}{
 				"table":     entry.table,
 				"key":       entry.key,
 				"hook_name": hookName,

search/search_repository.go

@@ -174,7 +174,9 @@ func getSearchQueryFromURLPattern(patternName, stringToMatch string) string {
 		searchQueryString = fmt.Sprintf("%s:*", searchQueryString)
 		if result["id"] != "" {
 			// Look for pattern's ID field, if exists update searchQueryString
-			searchQueryString = fmt.Sprintf("(%v:* | %v)", result["id"], searchQueryString)
+			// `*A` is used to add sme weight to the work item number in the search results.
+			// See https://www.postgresql.org/docs/9.6/static/textsearch-controls.html
+			searchQueryString = fmt.Sprintf("(%v:*A | %v)", result["id"], searchQueryString)
 			// searchQueryString = "(" + result["id"] + ":*" + " | " + searchQueryString + ")"
 		}
 		return searchQueryString
@@ -239,9 +241,11 @@ func parseSearchString(ctx context.Context, rawSearchString string) (searchKeywo
 			}
 			res.workItemTypes = append(res.workItemTypes, typeID)
 		} else if govalidator.IsURL(part) {
+			log.Debug(ctx, map[string]interface{}{"url": part}, "found a URL in the query string")
 			part := strings.ToLower(part)
 			part = trimProtocolFromURLString(part)
 			searchQueryFromURL := getSearchQueryFromURLString(part)
+			log.Debug(ctx, map[string]interface{}{"url": part, "search_query": searchQueryFromURL}, "found a URL in the query string")
 			res.words = append(res.words, searchQueryFromURL)
 		} else {
 			part := strings.ToLower(part)
@@ -446,7 +450,6 @@ func (q Query) generateExpression() (criteria.Expression, error) {
 
 // parseFilterString accepts a raw string and generates a criteria expression
 func parseFilterString(ctx context.Context, rawSearchString string) (criteria.Expression, error) {
-
 	fm := map[string]interface{}{}
 	// Parsing/Unmarshalling JSON encoding/json
 	err := json.Unmarshal([]byte(rawSearchString), &fm)
@@ -481,7 +484,6 @@ func generateSQLSearchInfo(keywords searchKeyword) (sqlParameter string) {
 // extracted this function from List() in order to close the rows object with "defer" for more readability
 // workaround for https://github.com/lib/pq/issues/81
 func (r *GormSearchRepository) search(ctx context.Context, sqlSearchQueryParameter string, workItemTypes []uuid.UUID, start *int, limit *int, spaceID *string) ([]workitem.WorkItemStorage, uint64, error) {
-	log.Info(ctx, nil, "Searching work items...")
 	db := r.db.Model(workitem.WorkItemStorage{}).Where("tsv @@ query")
 	if start != nil {
 		if *start < 0 {
@@ -573,6 +575,7 @@ func (r *GormSearchRepository) SearchFullText(ctx context.Context, rawSearchStri
 
 	sqlSearchQueryParameter := generateSQLSearchInfo(parsedSearchDict)
 	var rows []workitem.WorkItemStorage
+	log.Debug(ctx, map[string]interface{}{"search query": sqlSearchQueryParameter}, "searching for work items")
 	rows, count, err := r.search(ctx, sqlSearchQueryParameter, parsedSearchDict.workItemTypes, start, limit, spaceID)
 	if err != nil {
 		return nil, 0, errs.WithStack(err)
@@ -741,10 +744,3 @@ func (r *GormSearchRepository) Filter(ctx context.Context, rawFilterString strin
 	}
 	return res, count, nil
 }
-
-func init() {
-	// While registering URLs do not include protocol because it will be removed before scanning starts
-	// Please do not include trailing slashes because it will be removed before scanning starts
-	RegisterAsKnownURL("test-work-item-list-details", `(?P<domain>demo.almighty.io)(?P<path>/work-item/list/detail/)(?P<id>\d*)`)
-	RegisterAsKnownURL("test-work-item-board-details", `(?P<domain>demo.almighty.io)(?P<path>/work-item/board/detail/)(?P<id>\d*)`)
-}