Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit b78ff61

Browse files
ajbt200128jmgrosen
ajbt200128
authored and
jmgrosen
committed
chore: remove taint rule groups (semgrep/semgrep-proprietary#5835)
closes ENGINE-2621 We've decided to remove the taint rule groups experiment for now Test plan: CI passes synced from Pro 267f3be7142e9e67faac539eebf60065a0f5cc96
1 parent 3c9e2b9 commit b78ff61

11 files changed

Lines changed: 31 additions & 524 deletions

‎src/engine/Match_taint_spec.ml‎

Lines changed: 1 addition & 116 deletions
Original file line numberDiff line numberDiff line change
@@ -665,7 +665,7 @@ let taint_config_of_rule ~per_file_formula_cache ~(file : Taint_rule_inst.file)
665665
( Taint_rule_inst.
666666
{
667667
file;
668-
rule_or_group = `Rule (fst rule.id);
668+
rule = fst rule.id;
669669
options;
670670
track_control = rule_needs_to_track_control_taint rule;
671671
preds;
@@ -674,118 +674,3 @@ let taint_config_of_rule ~per_file_formula_cache ~(file : Taint_rule_inst.file)
674674
raw_spec_matches,
675675
expls )
676676
[@@trace_trace]
677-
678-
(*****************************************************************************)
679-
(* EXPERIMENT: Group taint rules *)
680-
(*****************************************************************************)
681-
682-
(** This unions raw spec matches. Pre: all the raw spec matches come from the
683-
same rule group. See Taint_rule_group.mli for more details.
684-
*)
685-
let fold_raw_spec_matches raw_spec_matches_list =
686-
(* TODO(Tean): This probably should be refactored to a better spot. Can't put
687-
* it in Taint_rule_group due to dependency cycling issues. Can some of the
688-
* types here be factored out? *)
689-
let raw_propagators, raw_sanitizers =
690-
match raw_spec_matches_list with
691-
| [] ->
692-
Log.err (fun m -> m "fold_raw_spec_matches: unexpected empty list");
693-
([], [])
694-
| raw_spec_matches :: _ ->
695-
(raw_spec_matches.raw_propagators, raw_spec_matches.raw_sanitizers)
696-
in
697-
let raw_sources =
698-
raw_spec_matches_list
699-
|> List.concat_map (fun raw_spec_matches -> raw_spec_matches.raw_sources)
700-
in
701-
let raw_sinks =
702-
raw_spec_matches_list
703-
|> List.concat_map (fun raw_spec_matches -> raw_spec_matches.raw_sinks)
704-
in
705-
{ raw_sources; raw_propagators; raw_sanitizers; raw_sinks }
706-
707-
let taint_config_of_group ~per_file_formula_cache ~(file : Taint_rule_inst.file)
708-
xconf ast_and_errors group =
709-
(* See Taint_rule_group.mli for more details. *)
710-
let rules = Taint_rule_group.rules group in
711-
let group_spec_matches =
712-
(* HACK: For every rule except for the first one, we ignore their
713-
propagators and sanitizers when looking for matches, since it should
714-
be the same as the first rule. This saves on a huge memory spike
715-
because we don't need to allocate space for essentially the same
716-
matches between rules in the same group.
717-
718-
NOTE(Tean): I've tried doing a similar thing with sources and sinks
719-
in the past, because findings will look for the pm associated with
720-
the match when assigning a rule_id. This can lead to missing findings.
721-
722-
Also, sharing propagators like this can cause differences when dumping
723-
taint sigs, that's because they have slightly different pm match
724-
info, but in practice this doesn't seem to make a difference in findings. *)
725-
match rules with
726-
| [] ->
727-
(* nosemgrep: no-logs-in-library *)
728-
Logs.err (fun m ->
729-
m "BUG: taint_config_of_group: taint rule group is empty");
730-
[]
731-
| first :: rest ->
732-
let first_preds =
733-
(* Propagators and sanitizers predicates are computed for the first rule only. *)
734-
preds_of_rule ~per_file_formula_cache ~file xconf ast_and_errors first
735-
in
736-
let rest_preds =
737-
rest
738-
|> List_.map (fun (rule : Rule.taint_rule) ->
739-
let (`Taint spec) = rule.mode in
740-
(* This rule's propagators and sanitizers are already in 'first_preds'. *)
741-
let spec = { spec with propagators = []; sanitizers = None } in
742-
let rule' = { rule with mode = `Taint spec } in
743-
preds_of_rule ~per_file_formula_cache ~file xconf
744-
ast_and_errors rule')
745-
in
746-
first_preds :: rest_preds
747-
in
748-
(* TODO(iago): It may be better to just have a separate "preds_of_group" that
749-
folds over the rules in the group and iterately constructs the 'preds',
750-
'raw_spec_matches', and the 'expls'. *)
751-
let raw_spec_matches =
752-
group_spec_matches
753-
|> List_.map (fun (_, _, raw_spec_matches, _) -> raw_spec_matches)
754-
|> fold_raw_spec_matches
755-
in
756-
let expls =
757-
group_spec_matches |> List.concat_map (fun (_, _, _, expls) -> expls)
758-
in
759-
let preds =
760-
group_spec_matches
761-
|> List_.map (fun (preds, _, _, _) -> preds)
762-
|> Taint_rule_group.fold_preds
763-
in
764-
let stats =
765-
group_spec_matches
766-
|> List_.map (fun (_, stats, _, _) -> stats)
767-
|> Taint_coverage_stats.merge_file_rule_stats
768-
in
769-
let rule_or_group = `Group group in
770-
let xconf =
771-
(* Using the first rule's options, but this is fine because groups must have
772-
the same options. See 'Taint_rule_group.group_rules'. *)
773-
let first_rule = Taint_rule_group.first_rule group in
774-
Match_env.adjust_xconfig_with_rule_options xconf first_rule.options
775-
in
776-
let options = xconf.config in
777-
let track_control = rules |> List.exists rule_needs_to_track_control_taint in
778-
( Taint_rule_inst.{ file; rule_or_group; options; track_control; preds; stats },
779-
raw_spec_matches,
780-
expls )
781-
[@@trace_trace]
782-
783-
let taint_config_of_rule_or_group ~per_file_formula_cache ~file xconf
784-
ast_and_errors rule_or_group =
785-
match rule_or_group with
786-
| `Rule rule ->
787-
taint_config_of_rule ~per_file_formula_cache ~file xconf ast_and_errors
788-
rule
789-
| `Group group ->
790-
taint_config_of_group ~per_file_formula_cache ~file xconf ast_and_errors
791-
group

‎src/engine/Match_taint_spec.mli‎

Lines changed: 1 addition & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -61,16 +61,7 @@ val taint_config_of_rule :
6161
Rule.taint_rule ->
6262
Taint_rule_inst.t * raw_spec_matches * Matching_explanation.t list
6363

64-
(* EXPERIMENT: Group taint rules *)
65-
val taint_config_of_rule_or_group :
66-
per_file_formula_cache:Formula_cache.t ->
67-
file:Taint_rule_inst.file ->
68-
Match_env.xconfig ->
69-
AST_generic.program * Tok.location list ->
70-
[ `Rule of Rule.taint_rule | `Group of Taint_rule_group.t ] ->
71-
Taint_rule_inst.t * raw_spec_matches * Matching_explanation.t list
72-
73-
(* Exposed for Pro and for taint config cache reconstruction *)
64+
(* Exposed for Pro *)
7465

7566
val range_of_any : AST_generic.any -> Range.t option
7667
val overlap_with : match_range:Range.t -> Range.t -> float

‎src/tainting/OSS_dataflow_tainting.ml‎

Lines changed: 2 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -542,24 +542,7 @@ let effects_of_tainted_sinks env taints sinks : Effect.poly list =
542542
but it starts out here as just a PM variant.
543543
*)
544544
let taints_with_traces =
545-
taints
546-
|> Taints.elements
547-
(* EXPERIMENT: Group taint rules
548-
549-
Filter taints to only include those whose sources match the
550-
sink. This prevents the generation of effects like:
551-
552-
rule-1/source-A ~~~> rule-2/sink-B
553-
554-
or
555-
556-
rule-2/source-B -> rule-1/sink-A
557-
558-
in the first place.
559-
560-
See 'Taint_rule_group.mli'.
561-
*)
562-
|> List.filter (Taint.is_valid_taint_for_rule sink.pm.rule_id.id)
545+
taints |> Taints.elements
563546
|> List_.map (fun t ->
564547
{ Effect.taint = t; sink_trace = T.PM (sink.Effect.pm, ()) })
565548
in
@@ -1855,17 +1838,7 @@ and fixpoint_aux func ~lambdas ~enter_lval_env ~in_lambda fun_cfg =
18551838
(* THINK: Why I cannot just update mapping here ? if I do, the mapping gets overwritten later on! *)
18561839
(* DataflowX.display_mapping flow init_mapping show_tainted; *)
18571840
let end_mapping, timeout =
1858-
let fp_timeout =
1859-
match func.taint_inst.rule_or_group with
1860-
| `Rule _ -> Limits_semgrep.taint_FIXPOINT_TIMEOUT
1861-
| `Group group ->
1862-
(* EXPERIMENT: Group taint rules
1863-
1864-
Running fixpoint with a group of rules would cost more time to
1865-
converge, so we scale up the timeout. *)
1866-
float_of_int (Taint_rule_group.length group)
1867-
*. Limits_semgrep.taint_FIXPOINT_TIMEOUT
1868-
in
1841+
let fp_timeout = Limits_semgrep.taint_FIXPOINT_TIMEOUT in
18691842
DataflowX.fixpoint ~timeout:fp_timeout ~eq_env:Lval_env.equal
18701843
~init:init_mapping ~trans:(transfer env ~fun_cfg) ~forward:true ~flow
18711844
in

‎src/tainting/Shape_and_sig.ml‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -301,13 +301,6 @@ and Effect : sig
301301
(** The metavariable environment that results of merging the environment from
302302
matching the source and the one from matching the sink. *)
303303
}
304-
(* EXPERIMENT: Group taint rules
305-
306-
INVARIANT(taints_to_sink):
307-
Taints in 'taints_with_trace' are "valid" for the rule the sink corresponds
308-
to, that is, 'taints_with_trace.taint.rule_id = sink.pm.rule_id'.
309-
See 'Taint_rule_group.mli'.
310-
*)
311304

312305
type taints_to_return = {
313306
data_taints : Taint.taints;

‎src/tainting/Taint.ml‎

Lines changed: 17 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -241,12 +241,6 @@ type var =
241241
| Control_var
242242

243243
type source = {
244-
rule_id : Rule_ID.t;
245-
(* EXPERIMENT: Group taint rules
246-
247-
We keep track of rule_id for when we run taint on groups of taint rules.
248-
See Taint_rule_group.mli for details. But we use this to distinguish
249-
taints that belong to different rules. *)
250244
call_trace : R.taint_source call_trace;
251245
label : string;
252246
(* This is needed because we may change the label of a taint,
@@ -261,13 +255,6 @@ type source = {
261255
and orig = Src of source | Var of var
262256
and taint = { orig : orig; rev_tokens : rev_tainted_tokens }
263257

264-
(** [is_valid_taint_for_rule rule_id taint] returns [true] if [taint] is valid for the
265-
rule with id [rule_id], otherwise returns [false]. *)
266-
let is_valid_taint_for_rule (rule_id : Rule_ID.t) (taint : taint) : bool =
267-
match taint.orig with
268-
| Var _ -> true
269-
| Src source -> Rule_ID.equal source.rule_id rule_id
270-
271258
(* See NOTE "on compare functions" *)
272259
let compare_precondition (_ts1, f1) (_ts2, f2) =
273260
(* We don't consider the "incoming" taints here, assuming both
@@ -293,55 +280,36 @@ let compare_var v1 v2 =
293280

294281
(* See NOTE "on compare functions" *)
295282
let compare_source
296-
{
297-
rule_id = rule_id1;
298-
call_trace = call_trace1;
299-
label = label1;
300-
precondition = precondition1;
301-
}
302-
{
303-
rule_id = rule_id2;
304-
call_trace = call_trace2;
305-
label = label2;
306-
precondition = precondition2;
307-
} =
283+
{ call_trace = call_trace1; label = label1; precondition = precondition1 }
284+
{ call_trace = call_trace2; label = label2; precondition = precondition2 } =
308285
(* Comparing metavariable environments this way is not robust, e.g.:
309286
* [("$A",e1);("$B",e2)] is not considered equal to [("$B",e2);("$A",e1)].
310287
* For our purposes, this is OK.
311288
*)
312289
let pm1, ts1 = pm_of_trace call_trace1
313290
and pm2, ts2 = pm_of_trace call_trace2 in
314-
(* We compare rule ids, because when we run dataflow with a group of rules,
315-
* we could have taints that belong to different rules.
316-
*
317-
* Tean: We could change this comparison in the future. Right now, each
318-
* source has a rule id attached to it, but it could be the case that if they
319-
* actually correspond to the same source, we could merge them in some way? *)
320-
match Rule_ID.compare rule_id1 rule_id2 with
291+
match compare_matches pm1 pm2 with
321292
| 0 -> (
322-
match compare_matches pm1 pm2 with
293+
let l1 = label1 ^ ts1.R.label in
294+
let l2 = label2 ^ ts2.R.label in
295+
match String.compare l1 l2 with
323296
| 0 -> (
324-
let l1 = label1 ^ ts1.R.label in
325-
let l2 = label2 ^ ts2.R.label in
326-
match String.compare l1 l2 with
327-
| 0 -> (
328-
(* It's important that we include preconditions as a distinguishing factor
297+
(* It's important that we include preconditions as a distinguishing factor
329298
between two taints.
330299
331300
Otherwise, suppose that we had a taint with label A with precondition `false`
332301
and one with precondition `true`. Obviously, only one actually exists. But
333302
if we pick the wrong one, we might fallaciously say a taint label finding does
334303
not actually occur.
335304
*)
336-
match (precondition1, precondition2) with
337-
| None, _
338-
| _, None ->
339-
(* 'None' here is the same as 'true', although the `requires` of both taints
340-
* may not be the same, in this specific case we consider them "the same",
341-
* see 'pick_best_taint'. *)
342-
0
343-
| Some pre1, Some pre2 -> compare_precondition pre1 pre2)
344-
| other -> other)
305+
match (precondition1, precondition2) with
306+
| None, _
307+
| _, None ->
308+
(* 'None' here is the same as 'true', although the `requires` of both taints
309+
* may not be the same, in this specific case we consider them "the same",
310+
* see 'pick_best_taint'. *)
311+
0
312+
| Some pre1, Some pre2 -> compare_precondition pre1 pre2)
345313
| other -> other)
346314
| other -> other
347315

@@ -377,13 +345,11 @@ let show_var var =
377345
| Taint_in_shape_var lval -> "'<" ^ show_lval lval ^ ">"
378346
| Control_var -> "'<control>"
379347

380-
let rec show_source { call_trace; rule_id = _; label; precondition } =
348+
let rec show_source { call_trace; label; precondition } =
381349
(* We want to show the actual label, not the originating label.
382350
This may change, for instance, if we have ever propagated this taint to
383351
a different label.
384352
*)
385-
(* TODO: Show rule_id? Not showing by default because it clutters debug
386-
* output. It could also cause conflicts with snapshot tests. *)
387353
let pm, ts = pm_of_trace call_trace in
388354
let matched_str =
389355
let tok1, tok2 = pm.range_loc in
@@ -753,7 +719,6 @@ let src_of_pm ~incoming ((pm : PM.t), (source : Rule.taint_source)) =
753719
Some
754720
(Src
755721
{
756-
rule_id = pm.rule_id.id;
757722
call_trace = PM (pm, source);
758723
label = source.label;
759724
precondition = None;
@@ -767,12 +732,7 @@ let src_of_pm ~incoming ((pm : PM.t), (source : Rule.taint_source)) =
767732
in
768733
Some
769734
(Src
770-
{
771-
rule_id = pm.rule_id.id;
772-
call_trace = PM (pm, source);
773-
label = source.label;
774-
precondition;
775-
})
735+
{ call_trace = PM (pm, source); label = source.label; precondition })
776736

777737
let taint_of_pm ~incoming pm =
778738
match src_of_pm ~incoming pm with

‎src/tainting/Taint.mli‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -136,7 +136,6 @@ type var =
136136
| Control_var (** Polymorphic taint variable, but for the "control-flow". *)
137137

138138
type source = {
139-
rule_id : Rule_ID.t; (** EXPERIMENT: Group taint rules *)
140139
call_trace : Rule.taint_source call_trace;
141140
label : string;
142141
(** The label of this particular taint.
@@ -186,9 +185,6 @@ and taint = { orig : orig; rev_tokens : rev_tainted_tokens }
186185
(** At a given program location, taint is given by its origin (i.e. 'orig') and
187186
* the path it took from that origin to the current location (i.e. 'tokens'). *)
188187

189-
val is_valid_taint_for_rule : Rule_ID.t -> taint -> bool
190-
(** EXPERIMENT: Group taint rules *)
191-
192188
val trace_of_pm : Core_match.t * 'a -> 'a call_trace
193189
val pm_of_trace : 'a call_trace -> Core_match.t * 'a
194190

0 commit comments

Comments
 (0)