How to handle job recovering in Quartz.Net - quartz-scheduler

Question

I use Quartz 3.0.7 in my application that use .NET Core 2.2 platform.
I use ms sql server for Quartz action tracking. Quartz tracks and stores its actions in database and it's fine.
Configuration of my StdSchedulerFactory:
["quartz.scheduler.instanceName"] = "StdScheduler",
["quartz.scheduler.instanceId"] = $"{Environment.MachineName}-{Guid.NewGuid()}",
["quartz.jobStore.type"] = "Quartz.Impl.AdoJobStore.JobStoreTX, Quartz",
["quartz.jobStore.useProperties"] = "true",
["quartz.jobStore.dataSource"] = "default",
["quartz.jobStore.tablePrefix"] = "QRTZ_",
// if running MS SQL Server we need this
["quartz.jobStore.lockHandler.type"] = "Quartz.Impl.AdoJobStore.UpdateLockRowSemaphore, Quartz",
["quartz.dataSource.default.connectionString"] = #"Server=DESKTOP-D64SJFJ\MSSQLSERVER14;Database=quartz;Trusted_Connection=True;",
["quartz.dataSource.default.provider"] = "SqlServer",
[$"{StdSchedulerFactory.PropertyObjectSerializer}.type"] = "json",
[StdSchedulerFactory.PropertySchedulerInterruptJobsOnShutdownWithWait] = "true",
I want to recover each interrupted Job. How should I organize logic of my IHostedService for supporting Job Recovering?
When I Shutdown my application during my job is running then When I start my application again interrupted job doesn't run.
My IHostedService code:
public class QuartzHostedService : IHostedService
{
private readonly ISchedulerFactory _schedulerFactory;
private readonly IJobFactory _jobFactory;
private readonly IEnumerable<JobSchedule> _jobSchedules;
public QuartzHostedService(
ISchedulerFactory schedulerFactory,
IJobFactory jobFactory,
IEnumerable<JobSchedule> jobSchedules)
{
_schedulerFactory = schedulerFactory;
_jobSchedules = jobSchedules;
_jobFactory = jobFactory;
}
public IScheduler Scheduler { get; set; }
public async Task StartAsync(CancellationToken cancellationToken)
{
Scheduler = await _schedulerFactory.GetScheduler(cancellationToken);
Scheduler.JobFactory = _jobFactory;
await Scheduler.Start(cancellationToken);
foreach (var jobSchedule in _jobSchedules)
{
var job = CreateJob(jobSchedule);
var trigger = CreateTrigger(jobSchedule);
if (!await Scheduler.CheckExists(job.Key, cancellationToken))
{
// if the job doesn't already exist, we can create it, along with its trigger. this prevents us
// from creating multiple instances of the same job when running in a clustered environment
await Scheduler.ScheduleJob(job, trigger);
}
else
{
// if the job has exactly one trigger, we can just reschedule it, which allows us to update the schedule for
// that trigger.
var triggers = await Scheduler.GetTriggersOfJob(job.Key);
if (triggers.Count == 1)
{
await Scheduler.RescheduleJob(triggers.First().Key, trigger);
}
else
{
// if for some reason the job has multiple triggers, it's easiest to just delete and re-create the job,
// since we want to enforce a one-to-one relationship between jobs and triggers
await Scheduler.DeleteJob(job.Key);
await Scheduler.ScheduleJob(job, trigger);
}
}
}
}
public async Task StopAsync(CancellationToken cancellationToken)
{
await Scheduler?.Shutdown(cancellationToken);
}
private static IJobDetail CreateJob(JobSchedule schedule)
{
var jobType = schedule.JobType;
return JobBuilder
.Create(jobType)
.WithIdentity(jobType.FullName)
.WithDescription(jobType.Name)
.RequestRecovery(true)
.StoreDurably()
.Build();
}
private static ITrigger CreateTrigger(JobSchedule schedule)
{
return TriggerBuilder
.Create()
.WithIdentity($"{schedule.JobType.FullName}.trigger")
.WithCronSchedule(schedule.CronExpression)
.WithDescription(schedule.CronExpression)
.Build();
}
}
My startup.cs:
private void ConfigureQuartz(IServiceCollection services)
{
services.AddHostedService<QuartzHostedService>();
services.AddSingleton<IJobFactory, SingletonJobFactory>();
services.AddSingleton<ISchedulerFactory>(new StdSchedulerFactory(StdSchedulerFactoryConfiguration()));
services.AddSingleton<AuthKeyExpiresJob>();
//services.AddSingleton<AuthKeyWillExpireJob>();
services.AddSingleton(new JobSchedule(
typeof(AuthKeyExpiresJob),
"0 14 11 ? * *"));
}