feat(article): smooth illutsration lazy loading

autogen/db/postgres/db_sql_migration.go

@@ -119,6 +119,8 @@ drop type notification_strategy_type;
 `,
 	"db_migration_14": `alter table outgoing_webhooks add column secrets varchar null;
 update outgoing_webhooks set secrets = config;
+`,
+	"db_migration_15": `alter table articles add column thumbhash varchar null;
 `,
 	"db_migration_2": `create table devices (
   id serial not null,
@@ -178,6 +180,7 @@ var DatabaseSQLMigrationChecksums = map[string]string{
 	"db_migration_12": "b24497bb03f04fb4705ae752f8a5bf69dad26f168bc8ec196af93aee29deef49",
 	"db_migration_13": "4a52465eeb50a236d7f7a94cc51cd78238de0f885a6d29da4a548b5c389ebe81",
 	"db_migration_14": "f2c6e03988386e662f943d0f37255cf6db19b69e2c4f63c312f3778b401bb96a",
+	"db_migration_15": "edf9f683832d4b5c8c0d681f479750794ca19aea115a89b69700d4f415104fc3",
 	"db_migration_2":  "0be0d1ef1e9481d61db425a7d54378f3667c091949525b9c285b18660b6e8a1d",
 	"db_migration_3":  "5cd0d3628d990556c0b85739fd376c42244da7e98b66852b6411d27eda20c3fc",
 	"db_migration_4":  "d5fb83c15b523f15291310ff27d36c099c4ba68de2fd901c5ef5b70a18fedf65",

cmd/serve/server.go

@@ -16,7 +16,6 @@ import (
 	"github.com/ncarlier/readflow/internal/metric"
 	"github.com/ncarlier/readflow/internal/server"
 	"github.com/ncarlier/readflow/internal/service"
-	"github.com/ncarlier/readflow/pkg/cache"
 	"github.com/ncarlier/readflow/pkg/logger"
 )
 
@@ -29,14 +28,8 @@ func startServer(conf *config.Config) error {
 		return fmt.Errorf("unable to configure the database: %w", err)
 	}
 
-	// configure download cache
-	downloadCache, err := cache.NewDefault("readflow-downloads")
-	if err != nil {
-		return fmt.Errorf("unable to configure the downloader cache storage: %w", err)
-	}
-
 	// configure the service registry
-	err = service.Configure(*conf, database, downloadCache)
+	err = service.Configure(*conf, database)
 	if err != nil {
 		database.Close()
 		return fmt.Errorf("unable to configure the service registry: %w", err)
@@ -93,10 +86,6 @@ func startServer(conf *config.Config) error {
 
 		service.Shutdown()
 
-		if err := downloadCache.Close(); err != nil {
-			logger.Error().Err(err).Msg("unable to gracefully shutdown the cache storage")
-		}
-
 		if err := database.Close(); err != nil {
 			logger.Fatal().Err(err).Msg("could not gracefully shutdown database connection")
 		}

go.mod

@@ -34,7 +34,7 @@ require (
 	github.com/valyala/fasttemplate v1.2.2
 	go.etcd.io/bbolt v1.3.7
 	golang.org/x/net v0.14.0
-	golang.org/x/sync v0.1.0
+	golang.org/x/sync v0.7.0
 )
 
 require (
@@ -49,6 +49,7 @@ require (
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/emersion/go-sasl v0.0.0-20200509203442-7bfe0ed36a21 // indirect
 	github.com/emersion/go-smtp v0.18.0
+	github.com/galdor/go-thumbhash v1.0.0
 	github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f // indirect
 	github.com/golang-jwt/jwt v3.2.2+incompatible
 	github.com/golang/protobuf v1.5.3 // indirect
@@ -76,8 +77,9 @@ require (
 	github.com/tdewolff/parse/v2 v2.6.6 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	golang.org/x/crypto v0.12.0
+	golang.org/x/image v0.18.0
 	golang.org/x/sys v0.11.0 // indirect
-	golang.org/x/text v0.12.0 // indirect
+	golang.org/x/text v0.16.0 // indirect
 	google.golang.org/protobuf v1.30.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect

go.sum

@@ -44,6 +44,8 @@ github.com/emersion/go-sasl v0.0.0-20200509203442-7bfe0ed36a21/go.mod h1:iL2twTe
 github.com/emersion/go-smtp v0.18.0 h1:lrVQqB0JdxYjC8CsBt55pSwB756bRRN6vK0DSr0pXfM=
 github.com/emersion/go-smtp v0.18.0/go.mod h1:qm27SGYgoIPRot6ubfQ/GpiPy/g3PaZAVRxiO/sDUgQ=
 github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
+github.com/galdor/go-thumbhash v1.0.0 h1:Q7xSnaDvSC91SuNmQI94JuUVHva29FDdA4/PkV0EHjU=
+github.com/galdor/go-thumbhash v1.0.0/go.mod h1:gEK2wZqIxS2W4mXNf48lPl6HWjX0vWsH1LpK/cU74Ho=
 github.com/getsentry/raven-go v0.2.0 h1:no+xWJRb5ZI7eE8TWgIq1jLulQiIoLG0IfYxv5JYMGs=
 github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ=
 github.com/go-shiori/dom v0.0.0-20210627111528-4e4722cd0d65 h1:zx4B0AiwqKDQq+AgqxWeHwbbLJQeidq20hgfP+aMNWI=
@@ -190,6 +192,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk=
 golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw=
+golang.org/x/image v0.18.0 h1:jGzIakQa/ZXI1I0Fxvaa9W7yP25TqT6cHIHn+6CqvSQ=
+golang.org/x/image v0.18.0/go.mod h1:4yyo5vMFQjVjUcVk4jEQcU9MGy/rulF5WvUILseCM2E=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -207,8 +211,9 @@ golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
+golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -238,8 +243,9 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
-golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc=
 golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
+golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=

internal/api/download.go

@@ -15,12 +15,20 @@ func download() http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		id := strings.TrimPrefix(r.URL.Path, "/articles/")
 		if id == "" {
-			utils.WriteJSONProblem(w, downloadProblem, "missing article ID", http.StatusBadRequest)
+			utils.WriteJSONProblem(w, utils.JSONProblem{
+				Title:  downloadProblem,
+				Detail: "missing article ID",
+				Status: http.StatusBadRequest,
+			})
 			return
 		}
 		idArticle := utils.ConvGraphQLID(id)
 		if idArticle == nil {
-			utils.WriteJSONProblem(w, downloadProblem, "invalid article ID", http.StatusBadRequest)
+			utils.WriteJSONProblem(w, utils.JSONProblem{
+				Title:  downloadProblem,
+				Detail: "invalid article ID",
+				Status: http.StatusBadRequest,
+			})
 			return
 		}
 		// Extract and validate token parameter
@@ -33,7 +41,10 @@ func download() http.Handler {
 		// Archive the article
 		asset, err := service.Lookup().DownloadArticle(r.Context(), *idArticle, format)
 		if err != nil {
-			utils.WriteJSONProblem(w, downloadProblem, err.Error(), http.StatusInternalServerError)
+			utils.WriteJSONProblem(w, utils.JSONProblem{
+				Title:  downloadProblem,
+				Detail: err.Error(),
+			})
 			return
 		}
 

internal/api/image-proxy.go

@@ -24,7 +24,7 @@ func imgProxyHandler(conf *config.Config) http.Handler {
 	if err != nil {
 		logger.Fatal().Err(err).Msg("unable to setup Image Proxy cache")
 	}
-	down := downloader.NewInternalDownloader(defaults.HTTPClient, defaults.UserAgent, c, 0)
+	down := downloader.NewInternalDownloader(defaults.HTTPClient, defaults.UserAgent, c, 0, defaults.Timeout)
 
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		start := time.Now()

internal/api/info.go

@@ -7,6 +7,7 @@ import (
 	"github.com/ncarlier/readflow/internal/config"
 	"github.com/ncarlier/readflow/internal/service"
 	"github.com/ncarlier/readflow/internal/version"
+	"github.com/ncarlier/readflow/pkg/utils"
 )
 
 // Info API informations model structure.
@@ -29,7 +30,9 @@ func info(conf *config.Config) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		data, err := json.Marshal(v)
 		if err != nil {
-			http.Error(w, err.Error(), http.StatusInternalServerError)
+			utils.WriteJSONProblem(w, utils.JSONProblem{
+				Detail: err.Error(),
+			})
 			return
 		}
 		w.Header().Set("Content-Type", "application/json")

internal/auth/methods/basic.go

@@ -43,7 +43,10 @@ func newBasicAuthMiddlleware(cfg *config.AuthNConfig) (middleware.Middleware, er
 				return
 			}
 			w.Header().Set("WWW-Authenticate", `Basic realm="readflow", charset="UTF-8"`)
-			utils.WriteJSONProblem(w, "", "invalid credentials", http.StatusUnauthorized)
+			utils.WriteJSONProblem(w, utils.JSONProblem{
+				Detail: "invalid credentials",
+				Status: http.StatusUnauthorized,
+			})
 		})
 	}, nil
 }

internal/auth/methods/oidc.go

@@ -38,13 +38,21 @@ func newOIDCAuthMiddleware(cfg *config.AuthNConfig) (middleware.Middleware, erro
 			// retrieve username from access_token
 			username, err := getUsernameFromBearer(r, oidcClient, keyFunc)
 			if err != nil {
-				utils.WriteJSONProblem(w, "", err.Error(), http.StatusUnauthorized)
+				utils.WriteJSONProblem(w, utils.JSONProblem{
+					Detail: err.Error(),
+					Status: http.StatusUnauthorized,
+					Context: map[string]interface{}{
+						"redirect": "/login",
+					},
+				})
 			}
 
 			// retrieve or register user
 			user, err := service.Lookup().GetOrRegisterUser(ctx, username)
 			if err != nil {
-				utils.WriteJSONProblem(w, "", err.Error(), http.StatusInternalServerError)
+				utils.WriteJSONProblem(w, utils.JSONProblem{
+					Detail: err.Error(),
+				})
 				return
 			}
 

internal/auth/methods/proxy.go

@@ -47,7 +47,10 @@ func newProxyAuthMiddleware(cfg *config.AuthNConfig) (middleware.Middleware, err
 				return
 			}
 			w.Header().Set("Proxy-Authenticate", `Basic realm="readflow"`)
-			utils.WriteJSONProblem(w, "", "invalid authentication headers", http.StatusUnauthorized)
+			utils.WriteJSONProblem(w, utils.JSONProblem{
+				Detail: "invalid authentication header",
+				Status: http.StatusUnauthorized,
+			})
 		})
 	}, nil
 }

internal/config/config.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/BurntSushi/toml"
 	"github.com/imdario/mergo"
+	"github.com/ncarlier/readflow/pkg/defaults"
 	ratelimiter "github.com/ncarlier/readflow/pkg/rate-limiter"
 	"github.com/ncarlier/readflow/pkg/types"
 )
@@ -50,6 +51,14 @@ func NewConfig() *Config {
 				Value: []byte("pepper"),
 			},
 		},
+		Downloader: DownloaderConfig{
+			UserAgent:             defaults.UserAgent,
+			Cache:                 "boltdb:///tmp/readflow-downloads.cache?maxSize=256,maxEntries=5000,maxEntrySize=1",
+			MaxConcurentDownloads: 10,
+			Timeout: types.Duration{
+				Duration: defaults.Timeout,
+			},
+		},
 		Avatar: AvatarConfig{
 			ServiceProvider: "https://robohash.org/{seed}?set=set4&size=48x48",
 		},

internal/config/defaults.toml

@@ -94,6 +94,20 @@ secret_key = "${READFLOW_HASH_SECRET_KEY}"
 # Default: "706570706572" (aka "pepper")
 secret_salt = "${READFLOW_HASH_SECRET_SALT}"
 
+[downloader]
+## User-Agent used by the internal downloader
+# Default: "Mozilla/5.0 (compatible; Readflow/1.0; +https://github.com/ncarlier/readflow)"
+user_agent = "${READFLOW_DOWNLOADER_USER_AGENT}"
+## Cache paramters
+# Default: "boltdb:///tmp/readflow-downloads.cache?maxSize=256,maxEntries=5000,maxEntrySize=5"
+cache = "${READFLOW_DOWNLOADER_CACHE}"
+## Max concurent downloads
+# Default: 10
+#max_concurent_downloads = 10
+## Timeout
+# Default: 5s
+timeout = "${READFLOW_DOWNLOADER_TIMEOUT}"
+
 [scraping]
 ## External Web Scraper URL, using internal if empty
 # Example: "https://example.org/scrap"

internal/config/types.go

@@ -15,6 +15,7 @@ type Config struct {
 	AuthN        AuthNConfig        `toml:"authn"`
 	UI           UIConfig           `toml:"ui"`
 	Hash         HashConfig         `toml:"hash"`
+	Downloader   DownloaderConfig   `toml:"downloader"`
 	Scraping     ScrapingConfig     `toml:"scraping"`
 	Avatar       AvatarConfig       `toml:"avatar"`
 	Image        ImageConfig        `toml:"image"`
@@ -93,6 +94,15 @@ type HashConfig struct {
 	SecretSalt types.HexString `toml:"secret_salt"`
 }
 
+// DownloaderConfig for downloader configuration
+type DownloaderConfig struct {
+	UserAgent             string         `toml:"user_agent"`
+	Cache                 string         `toml:"cache"`
+	MaxConCache           string         `toml:"cache"`
+	MaxConcurentDownloads uint           `toml:"max_concurent_downloads"`
+	Timeout               types.Duration `toml:"timeout"`
+}
+
 // ScrapingConfig for scraping configuration section
 type ScrapingConfig struct {
 	ServiceProvider string `toml:"service_provider"`

internal/db/article.go

@@ -18,4 +18,5 @@ type ArticleRepository interface {
 	DeleteArticle(id uint) error
 	DeleteReadArticlesOlderThan(delay time.Duration) (int64, error)
 	DeleteAllReadArticlesByUser(uid uint) (int64, error)
+	SetArticleThumbHash(id uint, hash string) (*model.Article, error)
 }

internal/db/postgres/article.go

@@ -24,6 +24,7 @@ var articleColumns = []string{
 	"html",
 	"url",
 	"image",
+	"thumbhash",
 	"hash",
 	"status",
 	"stars",
@@ -44,6 +45,7 @@ func mapRowToArticle(row *sql.Row) (*model.Article, error) {
 		&article.HTML,
 		&article.URL,
 		&article.Image,
+		&article.ThumbHash,
 		&article.Hash,
 		&article.Status,
 		&article.Stars,
@@ -69,6 +71,7 @@ func mapRowsToArticle(rows *sql.Rows, article *model.Article) error {
 		&article.HTML,
 		&article.URL,
 		&article.Image,
+		&article.ThumbHash,
 		&article.Hash,
 		&article.Status,
 		&article.Stars,
@@ -253,3 +256,21 @@ func (pg *DB) DeleteAllReadArticlesByUser(uid uint) (int64, error) {
 	}
 	return result.RowsAffected()
 }
+
+// SetArticleThumbHash set article thumb hash value
+func (pg *DB) SetArticleThumbHash(id uint, hash string) (*model.Article, error) {
+	update := map[string]interface{}{
+		"updated_at": "NOW()",
+		"thumbhash":  hash,
+	}
+	query, args, _ := pg.psql.Update(
+		"articles",
+	).SetMap(update).Where(
+		sq.Eq{"id": id},
+	).Suffix(
+		"RETURNING " + strings.Join(articleColumns, ","),
+	).ToSql()
+
+	row := pg.db.QueryRow(query, args...)
+	return mapRowToArticle(row)
+}

internal/db/postgres/migration.go

@@ -8,7 +8,7 @@ import (
 	"github.com/ncarlier/readflow/pkg/logger"
 )
 
-const schemaVersion = 14
+const schemaVersion = 15
 
 // Migrate executes database migrations.
 func Migrate(db *sql.DB) {

internal/db/postgres/sql/db_migration_15.sql

@@ -0,0 +1 @@
+alter table articles add column thumbhash varchar null;

internal/model/article.go

@@ -88,6 +88,7 @@ type Article struct {
 	HTML        *string    `json:"html,omitempty"`
 	URL         *string    `json:"url,omitempty"`
 	Image       *string    `json:"image,omitempty"`
+	ThumbHash   *string    `json:"thumbhash,omitempty"`
 	Hash        string     `json:"hash,omitempty"`
 	Status      string     `json:"status,omitempty"`
 	Stars       uint       `json:"stars,omitempty"`

internal/schema/article/types.go

@@ -101,6 +101,9 @@ var articleType = graphql.NewObject(
 			"image": &graphql.Field{
 				Type: graphql.String,
 			},
+			"thumbhash": &graphql.Field{
+				Type: graphql.String,
+			},
 			"thumbnails": &graphql.Field{
 				Type:    graphql.NewList(thumbnailType),
 				Resolve: thumbnailsResolver,