Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit e871dfb

Browse files
ByronByron
committed
separate processing and reporting stage, which works due to avoiding…
…to store anything non-sync/send across await points. We can do that thanks to the new 'window' version of traversal.
1 parent 349d3de commit e871dfb

4 files changed

Lines changed: 62 additions & 55 deletions

File tree

‎criner/src/engine/run.rs‎

Lines changed: 49 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,40 @@ pub async fn non_blocking(
7474
.map(|_| ()),
7575
)?;
7676

77+
let stage = process_settings;
78+
pool.spawn(
79+
repeat_every_s(
80+
stage.every.as_secs() as u32,
81+
{
82+
let p = progress.clone();
83+
move || p.add_child("Reporting Timer")
84+
},
85+
deadline,
86+
stage.at_most,
87+
{
88+
let progress = progress.clone();
89+
let db = db.clone();
90+
let assets_dir = assets_dir.clone();
91+
let pool = pool.clone();
92+
let tokio = tokio.clone();
93+
move || {
94+
stage::processing::process(
95+
db.clone(),
96+
progress.add_child("Process Crate Versions"),
97+
io_bound_processors,
98+
cpu_bound_processors,
99+
progress.add_child("Downloads"),
100+
tokio.clone(),
101+
pool.clone(),
102+
assets_dir.clone(),
103+
startup_time,
104+
)
105+
}
106+
},
107+
)
108+
.map(|_| ()),
109+
)?;
110+
77111
let stage = report_settings;
78112
repeat_every_s(
79113
stage.run.every.as_secs() as u32,
@@ -82,54 +116,23 @@ pub async fn non_blocking(
82116
move || p.add_child("Reporting Timer")
83117
},
84118
deadline,
85-
stage
86-
.run
87-
.at_most
88-
.and_then(|at_most| process_settings.at_most.map(|am| at_most.max(am)))
89-
.or_else(|| process_settings.at_most),
119+
stage.run.at_most,
90120
{
91-
let mut process_run_count = 0;
92-
let mut report_run_count = 0;
93-
let process_max_runs = process_settings.at_most.unwrap_or(std::usize::MAX);
94-
let report_max_runs = stage.run.at_most.unwrap_or(std::usize::MAX);
121+
let progress = progress.clone();
122+
let db = db.clone();
123+
let assets_dir = assets_dir.clone();
124+
let pool = pool.clone();
125+
let glob = stage.glob.clone();
95126
move || {
96-
let progress = progress.clone();
97-
let db = db.clone();
98-
let assets_dir = assets_dir.clone();
99-
let pool = pool.clone();
100-
let glob = stage.glob.clone();
101-
let tokio = tokio.clone();
102-
async move {
103-
if process_run_count < process_max_runs {
104-
process_run_count += 1;
105-
stage::processing::process(
106-
db.clone(),
107-
progress.add_child("Process Crate Versions"),
108-
io_bound_processors,
109-
cpu_bound_processors,
110-
progress.add_child("Downloads"),
111-
tokio.clone(),
112-
pool.clone(),
113-
assets_dir.clone(),
114-
startup_time,
115-
)
116-
.await?;
117-
}
118-
if report_run_count < report_max_runs {
119-
report_run_count += 1;
120-
stage::report::generate(
121-
db.clone(),
122-
progress.add_child("Reports"),
123-
assets_dir.clone(),
124-
glob.clone(),
125-
deadline,
126-
cpu_o_bound_processors,
127-
pool.clone(),
128-
)
129-
.await?;
130-
}
131-
Ok(())
132-
}
127+
stage::report::generate(
128+
db.clone(),
129+
progress.add_child("Reports"),
130+
assets_dir.clone(),
131+
glob.clone(),
132+
deadline,
133+
cpu_o_bound_processors,
134+
pool.clone(),
135+
)
133136
}
134137
},
135138
)

‎criner/src/engine/stage/processing.rs‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -72,18 +72,17 @@ pub async fn process(
7272

7373
let auto_checkpoint_every = 10000;
7474
let checkpoint_connection = db.open_connection_with_busy_wait()?;
75-
let connection = versions.into_connection();
76-
let mut guard = connection.lock();
7775
let mut fetched_versions = 0;
7876
let mut versions = Vec::with_capacity(auto_checkpoint_every);
7977
let mut last_elapsed_for_checkpointing = None;
8078

8179
loop {
8280
let abort_loop = {
8381
progress.blocked("fetching chunk of version to schedule", None);
82+
let mut connection = db.open_connection_no_async_with_busy_wait()?;
8483
let mut statement = new_value_query_recent_first(
8584
CrateVersionTable::table_name(),
86-
&mut *guard,
85+
&mut connection,
8786
fetched_versions,
8887
auto_checkpoint_every,
8988
)?;

‎src/args.rs‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -68,13 +68,17 @@ pub enum SubCommands {
6868
#[structopt(long, short = "F")]
6969
fetch_at_most: Option<usize>,
7070

71+
/// The time between each processing run, specified in humantime, like 10s, 5min, or 2h, or '3h 2min 2s'
72+
#[structopt(long, short = "p", default_value = "60s")]
73+
process_every: humantime::Duration,
74+
7175
/// If set, the amount of times the process stage will run. If set to 0, they will never run.
7276
#[structopt(long, short = "P")]
7377
process_at_most: Option<usize>,
7478

7579
/// The time between each reporting and processing run, specified in humantime, like 10s, 5min, or 2h, or '3h 2min 2s'
7680
#[structopt(long, short = "r", default_value = "60s")]
77-
report_and_process_every: humantime::Duration,
81+
report_every: humantime::Duration,
7882

7983
/// If set, the amount of times the reporting stage will run. If set to 0, they will never run.
8084
#[structopt(long, short = "R")]
@@ -123,8 +127,9 @@ impl Default for SubCommands {
123127
time_limit: None,
124128
fetch_every: std::time::Duration::from_secs(60).into(),
125129
fetch_at_most: None,
130+
process_every: std::time::Duration::from_secs(60).into(),
126131
process_at_most: None,
127-
report_and_process_every: std::time::Duration::from_secs(60).into(),
132+
report_every: std::time::Duration::from_secs(60).into(),
128133
report_at_most: None,
129134
db_path: PathBuf::from("criner.db"),
130135
glob: None,

‎src/lib.rs‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@ use std::ops::Add;
33
mod args;
44
pub mod error;
55
pub use args::*;
6-
use std::time::Duration;
76

87
pub fn run_blocking(args: Parsed) -> criner::error::Result<()> {
98
use SubCommands::*;
@@ -28,7 +27,8 @@ pub fn run_blocking(args: Parsed) -> criner::error::Result<()> {
2827
fetch_every,
2928
fetch_at_most,
3029
process_at_most,
31-
report_and_process_every,
30+
process_every,
31+
report_every,
3232
report_at_most,
3333
glob,
3434
} => criner::run::blocking(
@@ -44,12 +44,12 @@ pub fn run_blocking(args: Parsed) -> criner::error::Result<()> {
4444
at_most: fetch_at_most,
4545
},
4646
criner::run::StageRunSettings {
47-
every: Duration::default(),
47+
every: process_every.into(),
4848
at_most: process_at_most,
4949
},
5050
criner::run::GlobStageRunSettings {
5151
run: criner::run::StageRunSettings {
52-
every: report_and_process_every.into(),
52+
every: report_every.into(),
5353
at_most: report_at_most,
5454
},
5555
glob,

0 commit comments

Comments
 (0)