-
Notifications
You must be signed in to change notification settings - Fork 994
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
JobUpdateTask cleanups #650
Changes from 1 commit
ac1143d
59f0352
5871d13
25b9dcb
b5e0ce8
15cc078
ded7ca5
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -32,7 +32,6 @@ | |
import feast.core.job.JobUpdateTask; | ||
import feast.core.model.FeatureSet; | ||
import feast.core.model.Job; | ||
import feast.core.model.JobStatus; | ||
import feast.core.model.Source; | ||
import feast.core.model.Store; | ||
import java.util.ArrayList; | ||
|
@@ -45,6 +44,7 @@ | |
import java.util.concurrent.ExecutorService; | ||
import java.util.concurrent.Executors; | ||
import java.util.stream.Collectors; | ||
import javax.validation.constraints.Positive; | ||
import lombok.extern.slf4j.Slf4j; | ||
import org.springframework.beans.factory.annotation.Autowired; | ||
import org.springframework.scheduling.annotation.Scheduled; | ||
|
@@ -55,11 +55,11 @@ | |
@Service | ||
public class JobCoordinatorService { | ||
|
||
private JobRepository jobRepository; | ||
private FeatureSetRepository featureSetRepository; | ||
private SpecService specService; | ||
private JobManager jobManager; | ||
private JobProperties jobProperties; | ||
private final JobRepository jobRepository; | ||
private final FeatureSetRepository featureSetRepository; | ||
private final SpecService specService; | ||
private final JobManager jobManager; | ||
private final JobProperties jobProperties; | ||
|
||
@Autowired | ||
public JobCoordinatorService( | ||
|
@@ -90,54 +90,56 @@ public JobCoordinatorService( | |
@Scheduled(fixedDelayString = "${feast.jobs.polling_interval_milliseconds}") | ||
public void Poll() throws InvalidProtocolBufferException { | ||
log.info("Polling for new jobs..."); | ||
@Positive long updateTimeout = jobProperties.getJobUpdateTimeoutSeconds(); | ||
List<JobUpdateTask> jobUpdateTasks = new ArrayList<>(); | ||
ListStoresResponse listStoresResponse = specService.listStores(Filter.newBuilder().build()); | ||
for (StoreProto.Store store : listStoresResponse.getStoreList()) { | ||
Set<FeatureSetProto.FeatureSet> featureSets = new HashSet<>(); | ||
for (Subscription subscription : store.getSubscriptionsList()) { | ||
featureSets.addAll( | ||
new ArrayList<>( | ||
specService | ||
.listFeatureSets( | ||
ListFeatureSetsRequest.Filter.newBuilder() | ||
.setFeatureSetName(subscription.getName()) | ||
.setFeatureSetVersion(subscription.getVersion()) | ||
.setProject(subscription.getProject()) | ||
.build()) | ||
.getFeatureSetsList())); | ||
} | ||
if (!featureSets.isEmpty()) { | ||
featureSets.stream() | ||
.collect(Collectors.groupingBy(fs -> fs.getSpec().getSource())) | ||
.entrySet() | ||
.stream() | ||
.forEach( | ||
kv -> { | ||
Optional<Job> originalJob = | ||
getJob(Source.fromProto(kv.getKey()), Store.fromProto(store)); | ||
jobUpdateTasks.add( | ||
new JobUpdateTask( | ||
kv.getValue(), | ||
kv.getKey(), | ||
store, | ||
originalJob, | ||
jobManager, | ||
jobProperties.getJobUpdateTimeoutSeconds())); | ||
}); | ||
|
||
for (StoreProto.Store storeSpec : listStoresResponse.getStoreList()) { | ||
Set<FeatureSet> featureSets = new HashSet<>(); | ||
Store store = Store.fromProto(storeSpec); | ||
|
||
for (Subscription subscription : store.getSubscriptions()) { | ||
var featureSetSpecs = | ||
specService | ||
.listFeatureSets( | ||
ListFeatureSetsRequest.Filter.newBuilder() | ||
.setFeatureSetName(subscription.getName()) | ||
.setFeatureSetVersion(subscription.getVersion()) | ||
.setProject(subscription.getProject()) | ||
.build()) | ||
.getFeatureSetsList(); | ||
featureSets.addAll(featureSetsFromProto(featureSetSpecs)); | ||
} | ||
|
||
featureSets.stream() | ||
.collect(Collectors.groupingBy(FeatureSet::getSource)) | ||
.forEach( | ||
(source, setsForSource) -> { | ||
Optional<Job> originalJob = getJob(source, store); | ||
jobUpdateTasks.add( | ||
new JobUpdateTask( | ||
setsForSource, source, store, originalJob, jobManager, updateTimeout)); | ||
}); | ||
} | ||
if (jobUpdateTasks.size() == 0) { | ||
if (jobUpdateTasks.isEmpty()) { | ||
log.info("No jobs found."); | ||
return; | ||
} | ||
|
||
log.info("Creating/Updating {} jobs...", jobUpdateTasks.size()); | ||
ExecutorService executorService = Executors.newFixedThreadPool(jobUpdateTasks.size()); | ||
startOrUpdateJobs(jobUpdateTasks); | ||
|
||
log.info("Updating feature set status"); | ||
updateFeatureSetStatuses(jobUpdateTasks); | ||
} | ||
|
||
void startOrUpdateJobs(List<JobUpdateTask> tasks) { | ||
ExecutorService executorService = Executors.newFixedThreadPool(tasks.size()); | ||
ExecutorCompletionService<Job> ecs = new ExecutorCompletionService<>(executorService); | ||
jobUpdateTasks.forEach(ecs::submit); | ||
tasks.forEach(ecs::submit); | ||
|
||
int completedTasks = 0; | ||
while (completedTasks < jobUpdateTasks.size()) { | ||
while (completedTasks < tasks.size()) { | ||
try { | ||
Job job = ecs.take().get(); | ||
if (job != null) { | ||
|
@@ -148,27 +150,23 @@ public void Poll() throws InvalidProtocolBufferException { | |
} | ||
completedTasks++; | ||
} | ||
|
||
log.info("Updating feature set status"); | ||
updateFeatureSetStatuses(jobUpdateTasks); | ||
executorService.shutdown(); | ||
} | ||
|
||
// TODO: make this more efficient | ||
private void updateFeatureSetStatuses(List<JobUpdateTask> jobUpdateTasks) { | ||
Set<FeatureSet> ready = new HashSet<>(); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Not really related to this PR per say, but would it be possible for us to remove this There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I think you're asking: rather than cache feature set statuses in bulk, could status be returned on-demand by checking job state for one given feature set, when requested? I guess the team will have more context on the design. As it stands, I'd have to get my head more into functional consideration of what it's doing beyond the mechanical refactoring I've done to this class, but it seems like there could be more to look at later:
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Tracking your comments here: #664
That's a very low hanging fruit. |
||
Set<FeatureSet> pending = new HashSet<>(); | ||
for (JobUpdateTask jobUpdateTask : jobUpdateTasks) { | ||
Optional<Job> job = | ||
getJob( | ||
Source.fromProto(jobUpdateTask.getSourceSpec()), | ||
Store.fromProto(jobUpdateTask.getStore())); | ||
if (job.isPresent()) { | ||
if (job.get().getStatus() == JobStatus.RUNNING) { | ||
ready.addAll(job.get().getFeatureSets()); | ||
} else { | ||
pending.addAll(job.get().getFeatureSets()); | ||
} | ||
} | ||
for (JobUpdateTask task : jobUpdateTasks) { | ||
getJob(task.getSource(), task.getStore()) | ||
.ifPresent( | ||
job -> { | ||
if (job.isRunning()) { | ||
ready.addAll(job.getFeatureSets()); | ||
} else { | ||
pending.addAll(job.getFeatureSets()); | ||
} | ||
}); | ||
} | ||
ready.removeAll(pending); | ||
ready.forEach( | ||
|
@@ -196,4 +194,9 @@ public Optional<Job> getJob(Source source, Store store) { | |
// return the latest | ||
return Optional.of(jobs.get(0)); | ||
} | ||
|
||
// TODO: Put in a util somewhere? | ||
private static List<FeatureSet> featureSetsFromProto(List<FeatureSetProto.FeatureSet> protos) { | ||
return protos.stream().map(FeatureSet::fromProto).collect(Collectors.toList()); | ||
} | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The
runnerName
as we have it above refers to therunnerType
right? Should we deduplicate that from the configuration (we could do that at the configuration level as well)?https://github.com/gojek/feast/blame/ded7ca59a2bd5b40715af73fdaa4e4f19ad0b915/core/src/main/java/feast/core/config/FeastProperties.java#L95
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The idea with the configuration was to provide a more forward compatible configuration schema, but we could just as easily use type there instead of name.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Yes, from your example config,
jobManager.getRunnerType().toString()
will be"DirectRunner"
(whereasjobManager.getRunnerType().name()
would be"DIRECT"
).In
JobUpdateTask
this value is being used only for log messages.