-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
implemented statistics services and refactor helper services
- Loading branch information
1 parent
3ad0e83
commit f1b7106
Showing
18 changed files
with
556 additions
and
9 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
73 changes: 73 additions & 0 deletions
73
...fetching/src/main/java/fr/guiguilechat/jcelechat/libs/spring/update/batch/BatchFetch.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,73 @@ | ||
| package fr.guiguilechat.jcelechat.libs.spring.update.batch; | ||
|
|
||
| import java.time.Instant; | ||
| import java.util.List; | ||
|
|
||
| import fr.guiguilechat.jcelechat.libs.spring.update.batch.BatchFetch.BatchItem; | ||
| import jakarta.persistence.Column; | ||
| import jakarta.persistence.EnumType; | ||
| import jakarta.persistence.Enumerated; | ||
| import jakarta.persistence.GeneratedValue; | ||
| import jakarta.persistence.GenerationType; | ||
| import jakarta.persistence.Id; | ||
| import jakarta.persistence.ManyToOne; | ||
| import jakarta.persistence.MappedSuperclass; | ||
| import jakarta.persistence.OneToMany; | ||
| import lombok.AllArgsConstructor; | ||
| import lombok.Getter; | ||
| import lombok.NoArgsConstructor; | ||
| import lombok.Setter; | ||
|
|
||
| /** | ||
| * a batch fetch | ||
| */ | ||
| @MappedSuperclass | ||
| @AllArgsConstructor | ||
| @NoArgsConstructor | ||
| @Getter | ||
| @Setter | ||
| public class BatchFetch<Item extends BatchItem<?, ?>> { | ||
|
|
||
| /** an, item that is created from a batch fetch */ | ||
| @MappedSuperclass | ||
| @AllArgsConstructor | ||
| @NoArgsConstructor | ||
| @Getter | ||
| @Setter | ||
| public static abstract class BatchItem<Fetch extends BatchFetch<?>, Fetched> { | ||
|
|
||
| @ManyToOne | ||
| private Fetch fetch; | ||
|
|
||
| public abstract BatchItem<Fetch, Fetched> update(Fetched fetched); | ||
|
|
||
| } | ||
|
|
||
| @OneToMany(mappedBy = "fetch") | ||
| private List<Item> items; | ||
|
|
||
| @Id | ||
| @GeneratedValue(strategy = GenerationType.SEQUENCE) | ||
| private Long id; | ||
|
|
||
| @Column(columnDefinition = "TEXT") | ||
| private String errorMessage = null; | ||
|
|
||
| private String etag; | ||
|
|
||
| private Instant expires; | ||
|
|
||
| private Instant lastModified; | ||
|
|
||
| private int nbItems = 0; | ||
|
|
||
| private int responseCode = 0; | ||
|
|
||
| public static enum STATUS { | ||
| SUCCESS, ERROR, CACHED, CREATED | ||
| } | ||
|
|
||
| @Enumerated(EnumType.STRING) | ||
| private STATUS status = STATUS.CREATED; | ||
|
|
||
| } |
13 changes: 13 additions & 0 deletions
13
...rc/main/java/fr/guiguilechat/jcelechat/libs/spring/update/batch/BatchFetchRepository.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,13 @@ | ||
| package fr.guiguilechat.jcelechat.libs.spring.update.batch; | ||
|
|
||
| import org.springframework.data.jpa.repository.JpaRepository; | ||
| import org.springframework.data.repository.NoRepositoryBean; | ||
|
|
||
| import fr.guiguilechat.jcelechat.libs.spring.update.batch.BatchFetch.STATUS; | ||
|
|
||
| @NoRepositoryBean | ||
| public interface BatchFetchRepository<Fetch extends BatchFetch<?>> extends JpaRepository<Fetch, Long> { | ||
|
|
||
| public Fetch findTop1ByStatusOrderByLastModifiedDesc(STATUS status); | ||
|
|
||
| } |
172 changes: 172 additions & 0 deletions
172
...g/src/main/java/fr/guiguilechat/jcelechat/libs/spring/update/batch/BatchFetchService.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,172 @@ | ||
| package fr.guiguilechat.jcelechat.libs.spring.update.batch; | ||
|
|
||
| import java.time.Instant; | ||
| import java.util.HashMap; | ||
| import java.util.List; | ||
| import java.util.Map; | ||
|
|
||
| import org.springframework.beans.factory.annotation.Autowired; | ||
| import org.springframework.data.jpa.repository.JpaRepository; | ||
|
|
||
| import fr.guiguilechat.jcelechat.jcesi.ConnectedImpl; | ||
| import fr.guiguilechat.jcelechat.jcesi.interfaces.Requested; | ||
| import fr.guiguilechat.jcelechat.libs.spring.update.batch.BatchFetch.BatchItem; | ||
| import fr.guiguilechat.jcelechat.libs.spring.update.batch.BatchFetch.STATUS; | ||
| import fr.guiguilechat.jcelechat.libs.spring.update.manager.IEntityUpdater; | ||
| import fr.guiguilechat.jcelechat.libs.spring.update.resolve.status.ESIStatusService; | ||
| import lombok.AccessLevel; | ||
| import lombok.Getter; | ||
| import lombok.Setter; | ||
| import lombok.experimental.Accessors; | ||
| import lombok.extern.slf4j.Slf4j; | ||
|
|
||
| @Slf4j | ||
| public abstract class BatchFetchService< | ||
| Fetch extends BatchFetch<Item>, | ||
| FetchRepo extends BatchFetchRepository<Fetch>, | ||
| Item extends BatchItem<Fetch, ?>, | ||
| ItemRepo extends JpaRepository<Item, ?>, Structure | ||
| > | ||
| implements IEntityUpdater { | ||
|
|
||
|
|
||
| @Autowired // can't use constructor injection for generic service | ||
| @Accessors(fluent = true) | ||
| @Getter(value = AccessLevel.PROTECTED) | ||
| private ESIStatusService esiStatusService; | ||
|
|
||
| @Autowired // can't use constructor injection for generic service | ||
| @Accessors(fluent = true) | ||
| @Getter(value = AccessLevel.PROTECTED) | ||
| private FetchRepo fetchRepository; | ||
|
|
||
| @Autowired // can't use constructor injection for generic service | ||
| @Accessors(fluent = true) | ||
| @Getter(value = AccessLevel.PROTECTED) | ||
| private ItemRepo itemRepository; | ||
|
|
||
|
|
||
| @Getter | ||
| private final UpdateConfig update = new UpdateConfig(); | ||
|
|
||
| /** | ||
| * following fetch date, if any. If not set, default method to use the | ||
| * {@link UpdateConfig}. It is reset during the {@link #preUpdate()} | ||
| */ | ||
| @Setter(value = AccessLevel.PROTECTED) | ||
| private Instant nextUpdate = null; | ||
|
|
||
| @Override | ||
| public Instant nextUpdate(boolean remain, Instant now) { | ||
| if (nextUpdate != null) { | ||
| return nextUpdate; | ||
| } | ||
| return IEntityUpdater.super.nextUpdate(remain, now); | ||
| } | ||
|
|
||
| protected abstract Requested<Structure> fetchData(Map<String, String> properties); | ||
|
|
||
| protected abstract Fetch createFetch(); | ||
|
|
||
| public Fetch lastSuccess() { | ||
| return fetchRepository().findTop1ByStatusOrderByLastModifiedDesc(STATUS.SUCCESS); | ||
| } | ||
|
|
||
|
|
||
| @Override | ||
| public boolean fetch() { | ||
| Fetch lastSuccess = lastSuccess(); | ||
| if (lastSuccess != null && Instant.now().isBefore(lastSuccess.getExpires())) { | ||
| return false; | ||
| } | ||
| int errorsRemain = esiStatusService().availErrors(); | ||
| if (errorsRemain < getUpdate().getErrorsMin()) { | ||
| setNextUpdate(esiStatusService().getErrorReset()); | ||
| return true; | ||
| } | ||
|
|
||
| log.debug("{} fetching data with last etag {}", fetcherName(), lastSuccess == null ? null : lastSuccess.getEtag()); | ||
| Map<String, String> properties = new HashMap<>(); | ||
| if (lastSuccess != null) { | ||
| properties.put(ConnectedImpl.IFNONEMATCH, lastSuccess.getEtag()); | ||
| } | ||
| Fetch fetchResult = createFetch(); | ||
| fetchResult = fetchRepository().save(fetchResult); | ||
| Requested<Structure> response = fetchData(properties); | ||
| if (response == null) { | ||
| updateNullResponse(fetchResult); | ||
| } else { | ||
| int responseCode = response.getResponseCode(); | ||
| fetchResult.setResponseCode(responseCode); | ||
| switch (responseCode) { | ||
| case 200: | ||
| updateOk(fetchResult, response); | ||
| break; | ||
| case 204: // no content => null | ||
| updateNullBody(fetchResult, response); | ||
| break; | ||
| case 304: | ||
| updateNoChange(fetchResult, response); | ||
| break; | ||
| default: | ||
| switch (responseCode / 100) { | ||
| case 4: | ||
| updateRequestError(fetchResult, response); | ||
| break; | ||
| case 5: | ||
| updateServerError(fetchResult, response); | ||
| break; | ||
| default: | ||
| log.error("{} while fetching, received response code {} and error {}", | ||
| fetcherName(), responseCode, response.getError()); | ||
| throw new UnsupportedOperationException("case " + responseCode + " not handled for url" + response.getURL()); | ||
| } | ||
| } | ||
| } | ||
| fetchRepository().saveAndFlush(fetchResult); | ||
| return false; | ||
| } | ||
|
|
||
| protected void updateNullResponse(Fetch fetchResult) { | ||
| fetchResult.setErrorMessage("null response"); | ||
| fetchResult.setResponseCode(0); | ||
| fetchResult.setStatus(STATUS.ERROR); | ||
| } | ||
|
|
||
| protected void updateOk(Fetch fetchResult, Requested<Structure> response) { | ||
| updateMetaOk(fetchResult, response); | ||
| List<Item> converted = convert(fetchResult, response.getOK()); | ||
| converted.forEach(it -> it.setFetch(fetchResult)); | ||
| itemRepository().saveAllAndFlush(converted); | ||
| fetchResult.setNbItems(converted.size()); | ||
| log.debug(" {} saving {} new items", fetcherName(), converted.size()); | ||
| } | ||
|
|
||
| protected void updateMetaOk(Fetch fetchResult, Requested<Structure> response) { | ||
| fetchResult.setEtag(response.getETag()); | ||
| fetchResult.setExpires(response.getExpiresInstant()); | ||
| fetchResult.setLastModified(response.getLastModifiedInstant()); | ||
| fetchResult.setStatus(STATUS.SUCCESS); | ||
| } | ||
|
|
||
| protected abstract List<Item> convert(Fetch fetchResult, Structure response); | ||
|
|
||
| private void updateNullBody(Fetch fetchResult, Requested<Structure> response) { | ||
| updateMetaOk(fetchResult, response); | ||
| } | ||
|
|
||
| private void updateNoChange(Fetch fetchResult, Requested<Structure> response) { | ||
| fetchResult.setStatus(STATUS.CACHED); | ||
| } | ||
|
|
||
| private void updateRequestError(Fetch fetchResult, Requested<Structure> response) { | ||
| fetchResult.setErrorMessage(response.getError()); | ||
| fetchResult.setStatus(STATUS.ERROR); | ||
| } | ||
|
|
||
| private void updateServerError(Fetch fetchResult, Requested<Structure> response) { | ||
| fetchResult.setErrorMessage(response.getError()); | ||
| fetchResult.setStatus(STATUS.ERROR); | ||
| } | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.